Coverage for python/lsst/faro/base/MatchedCatalogBase.py: 21%
163 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-03 12:32 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-03 12:32 +0000
1# This file is part of faro.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import warnings
24from lsst.utils.introspection import find_outside_stacklevel
25import lsst.afw.table as afwTable
26import lsst.pipe.base as pipeBase
27import lsst.pex.config as pexConfig
28import lsst.geom as geom
29from lsst.utils.logging import PeriodicLogger
30import numpy as np
32from lsst.faro.utils.matcher import matchCatalogs
34__all__ = (
35 "MatchedBaseConnections",
36 "MatchedBaseConfig",
37 "MatchedBaseTask",
38 "MatchedTractBaseTask",
39)
42class MatchedBaseConnections(
43 pipeBase.PipelineTaskConnections,
44 dimensions=(),
45 defaultTemplates={
46 "coaddName": "deep",
47 "photoCalibName": "calexp.photoCalib",
48 "wcsName": "calexp.wcs",
49 "externalPhotoCalibName": "fgcm",
50 "externalWcsName": "gbdesAstrometricFit",
51 },
52 # TODO: remove on DM-39854.
53 deprecatedTemplates={
54 "photoCalibName": "Deprecated in favor of visitSummary; will be removed after v26.",
55 "wcsName": "Deprecated in favor of visitSummary; will be removed after v26.",
56 "externalPhotoCalibName": "Deprecated in favor of visitSummary; will be removed after v26.",
57 "externalWcsName": "Deprecated in favor of visitSummary; will be removed after v26.",
58 },
59):
60 sourceCatalogs = pipeBase.connectionTypes.Input(
61 doc="Source catalogs to match up.",
62 dimensions=("instrument", "visit", "detector", "band"),
63 storageClass="SourceCatalog",
64 name="src",
65 multiple=True,
66 )
67 visitSummary = pipeBase.connectionTypes.Input(
68 doc="Exposure catalog with WCS and PhotoCalib this detector+visit combination.",
69 dimensions=("instrument", "visit"),
70 storageClass="ExposureCatalog",
71 name="finalVisitSummary",
72 multiple=True,
73 )
74 photoCalibs = pipeBase.connectionTypes.Input(
75 doc="Photometric calibration object.",
76 dimensions=("instrument", "visit", "detector", "band"),
77 storageClass="PhotoCalib",
78 name="{photoCalibName}",
79 multiple=True,
80 # TODO: remove on DM-39854.
81 deprecated="Deprecated in favor of visitSummary and already ignored; will be removed after v26."
82 )
83 astromCalibs = pipeBase.connectionTypes.Input(
84 doc="WCS for the catalog.",
85 dimensions=("instrument", "visit", "detector", "band"),
86 storageClass="Wcs",
87 name="{wcsName}",
88 multiple=True,
89 # TODO: remove on DM-39854.
90 deprecated="Deprecated in favor of visitSummary and already ignored; will be removed after v26."
91 )
92 externalSkyWcsTractCatalog = pipeBase.connectionTypes.Input(
93 doc=(
94 "Per-tract, per-visit wcs calibrations. These catalogs use the detector "
95 "id for the catalog id, sorted on id for fast lookup."
96 ),
97 name="{externalWcsName}SkyWcsCatalog",
98 storageClass="ExposureCatalog",
99 dimensions=("instrument", "visit", "tract", "band"),
100 multiple=True,
101 # TODO: remove on DM-39854.
102 deprecated="Deprecated in favor of visitSummary; will be removed after v26."
103 )
104 externalSkyWcsGlobalCatalog = pipeBase.connectionTypes.Input(
105 doc=(
106 "Per-visit wcs calibrations computed globally (with no tract information). "
107 "These catalogs use the detector id for the catalog id, sorted on id for "
108 "fast lookup."
109 ),
110 name="{externalWcsName}SkyWcsCatalog",
111 storageClass="ExposureCatalog",
112 dimensions=("instrument", "visit", "band"),
113 multiple=True,
114 # TODO: remove on DM-39854.
115 deprecated="Deprecated in favor of visitSummary; will be removed after v26."
116 )
117 externalPhotoCalibTractCatalog = pipeBase.connectionTypes.Input(
118 doc=(
119 "Per-tract, per-visit photometric calibrations. These catalogs use the "
120 "detector id for the catalog id, sorted on id for fast lookup."
121 ),
122 name="{externalPhotoCalibName}PhotoCalibCatalog",
123 storageClass="ExposureCatalog",
124 dimensions=("instrument", "visit", "tract", "band"),
125 multiple=True,
126 # TODO: remove on DM-39854.
127 deprecated="Deprecated in favor of visitSummary; will be removed after v26."
128 )
129 externalPhotoCalibGlobalCatalog = pipeBase.connectionTypes.Input(
130 doc=(
131 "Per-visit photometric calibrations computed globally (with no tract "
132 "information). These catalogs use the detector id for the catalog id, "
133 "sorted on id for fast lookup."
134 ),
135 name="{externalPhotoCalibName}PhotoCalibCatalog",
136 storageClass="ExposureCatalog",
137 dimensions=("instrument", "visit", "band"),
138 multiple=True,
139 # TODO: remove on DM-39854.
140 deprecated="Deprecated in favor of visitSummary; will be removed after v26."
141 )
142 skyMap = pipeBase.connectionTypes.Input(
143 doc="Input definition of geometry/bbox and projection/wcs for warped exposures",
144 name="skyMap",
145 storageClass="SkyMap",
146 dimensions=("skymap",),
147 )
149 def __init__(self, *, config=None):
150 super().__init__(config=config)
151 if config.doApplyExternalSkyWcs:
152 if config.useGlobalExternalSkyWcs:
153 self.inputs.remove("externalSkyWcsTractCatalog")
154 else:
155 self.inputs.remove("externalSkyWcsGlobalCatalog")
156 else:
157 self.inputs.remove("externalSkyWcsTractCatalog")
158 self.inputs.remove("externalSkyWcsGlobalCatalog")
159 if config.doApplyExternalPhotoCalib:
160 if config.useGlobalExternalPhotoCalib:
161 self.inputs.remove("externalPhotoCalibTractCatalog")
162 else:
163 self.inputs.remove("externalPhotoCalibGlobalCatalog")
164 else:
165 self.inputs.remove("externalPhotoCalibTractCatalog")
166 self.inputs.remove("externalPhotoCalibGlobalCatalog")
167 del self.photoCalibs
168 del self.astromCalibs
171class MatchedBaseConfig(
172 pipeBase.PipelineTaskConfig, pipelineConnections=MatchedBaseConnections
173):
174 match_radius = pexConfig.Field(
175 doc="Match radius in arcseconds.", dtype=float, default=1
176 )
177 snrMin = pexConfig.Field(
178 doc="Minimum SNR for a source to be included.",
179 dtype=float, default=200
180 )
181 snrMax = pexConfig.Field(
182 doc="Maximum SNR for a source to be included.",
183 dtype=float, default=np.Inf
184 )
185 brightMagCut = pexConfig.Field(
186 doc="Bright limit of catalog entries to include.", dtype=float, default=10.0
187 )
188 faintMagCut = pexConfig.Field(
189 doc="Faint limit of catalog entries to include.", dtype=float, default=30.0
190 )
191 selectExtended = pexConfig.Field(
192 doc="Whether to select extended sources", dtype=bool, default=False
193 )
194 doApplyExternalSkyWcs = pexConfig.Field(
195 doc="Whether or not to use the external wcs.", dtype=bool, default=False,
196 # TODO: remove on DM-39854.
197 deprecated="Deprecated in favor of the visitSummary connection; will be removed after v26."
198 )
199 useGlobalExternalSkyWcs = pexConfig.Field(
200 doc="Whether or not to use the global external wcs.", dtype=bool, default=False,
201 # TODO: remove on DM-39854.
202 deprecated="Deprecated in favor of the visitSummary connection; will be removed after v26."
203 )
204 doApplyExternalPhotoCalib = pexConfig.Field(
205 doc="Whether or not to use the external photoCalib.", dtype=bool, default=False,
206 # TODO: remove on DM-39854.
207 deprecated="Deprecated in favor of the visitSummary connection; will be removed after v26."
208 )
209 useGlobalExternalPhotoCalib = pexConfig.Field(
210 doc="Whether or not to use the global external photoCalib.",
211 dtype=bool,
212 default=False,
213 # TODO: remove on DM-39854.
214 deprecated="Deprecated in favor of the visitSummary connection; will be removed after v26."
215 )
218class MatchedBaseTask(pipeBase.PipelineTask):
220 ConfigClass = MatchedBaseConfig
221 _DefaultName = "matchedBaseTask"
223 def __init__(self, config: MatchedBaseConfig, *args, **kwargs):
224 super().__init__(*args, config=config, **kwargs)
225 self.radius = self.config.match_radius
226 self.level = "patch"
228 def run(
229 self,
230 sourceCatalogs,
231 photoCalibs,
232 astromCalibs,
233 dataIds,
234 wcs,
235 box,
236 doApplyExternalSkyWcs=None,
237 doApplyExternalPhotoCalib=None,
238 ):
239 # TODO: remove these arguments on DM-39854.
240 if doApplyExternalPhotoCalib is not None:
241 warnings.warn(
242 "The doApplyExternalPhotoCalib argument is deprecated and will be removed after v26.",
243 category=FutureWarning, stacklevel=find_outside_stacklevel("lsst.faro"),
244 )
245 else:
246 doApplyExternalPhotoCalib = False
247 if doApplyExternalSkyWcs is not None:
248 warnings.warn(
249 "The doApplyExternalSkyWcs argument is deprecated and will be removed after v26.",
250 category=FutureWarning, stacklevel=find_outside_stacklevel("lsst.faro"),
251 )
252 doApplyExternalSkyWcs = False
253 self.log.info("Running catalog matching")
254 periodicLog = PeriodicLogger(self.log)
255 radius = geom.Angle(self.radius, geom.arcseconds)
256 if len(sourceCatalogs) < 2:
257 self.log.warning("%s valid input catalogs: ", len(sourceCatalogs))
258 out_matched = afwTable.SimpleCatalog()
259 else:
260 srcvis, matched = matchCatalogs(
261 sourceCatalogs, photoCalibs, astromCalibs, dataIds, radius,
262 self.config, logger=self.log
263 )
264 self.log.verbose("Finished matching catalogs.")
266 # Trim the output to the patch bounding box
267 out_matched = type(matched)(matched.schema)
268 self.log.info("%s sources in matched catalog.", len(matched))
269 for record_index, record in enumerate(matched):
270 if box.contains(wcs.skyToPixel(record.getCoord())):
271 out_matched.append(record)
272 periodicLog.log("Checked %d records for trimming out of %d.", record_index + 1, len(matched))
274 self.log.info(
275 "%s sources when trimmed to %s boundaries.", len(out_matched), self.level
276 )
277 return pipeBase.Struct(outputCatalog=out_matched)
279 def get_box_wcs(self, skymap, oid):
280 tract_info = skymap.generateTract(oid["tract"])
281 wcs = tract_info.getWcs()
282 patch_info = tract_info.getPatchInfo(oid["patch"])
283 patch_box = patch_info.getInnerBBox()
284 self.log.info("Running tract: %s and patch: %s", oid["tract"], oid["patch"])
285 return patch_box, wcs
287 def runQuantum(self, butlerQC, inputRefs, outputRefs):
288 inputs = butlerQC.get(inputRefs)
289 self.log.verbose("Inputs obtained from the butler.")
290 oid = dict(outputRefs.outputCatalog.dataId.required)
291 skymap = inputs["skyMap"]
292 del inputs["skyMap"]
293 box, wcs = self.get_box_wcs(skymap, oid)
294 # Cast to float to handle fractional pixels
295 box = geom.Box2D(box)
296 inputs["dataIds"] = [el.dataId for el in inputRefs.sourceCatalogs]
297 inputs["wcs"] = wcs
298 inputs["box"] = box
299 inputs["doApplyExternalSkyWcs"] = self.config.doApplyExternalSkyWcs
300 inputs["doApplyExternalPhotoCalib"] = self.config.doApplyExternalPhotoCalib
301 visitSummary = inputs.pop("visitSummary")
303 # TODO: significant simplification should be possible here on DM-39854.
304 if self.config.doApplyExternalPhotoCalib:
305 if self.config.useGlobalExternalPhotoCalib:
306 externalPhotoCalibCatalog = inputs.pop(
307 "externalPhotoCalibGlobalCatalog"
308 )
309 else:
310 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog")
311 else:
312 externalPhotoCalibCatalog = visitSummary
314 flatPhotoCalibList = np.hstack(externalPhotoCalibCatalog)
315 visitPhotoCalibList = np.array(
316 [calib["visit"] for calib in flatPhotoCalibList]
317 )
318 detectorPhotoCalibList = np.array(
319 [calib["id"] for calib in flatPhotoCalibList]
320 )
322 if self.config.doApplyExternalSkyWcs:
323 if self.config.useGlobalExternalSkyWcs:
324 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog")
325 else:
326 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog")
327 else:
328 externalSkyWcsCatalog = visitSummary
330 flatSkyWcsList = np.hstack(externalSkyWcsCatalog)
331 visitSkyWcsList = np.array([calib["visit"] for calib in flatSkyWcsList])
332 detectorSkyWcsList = np.array([calib["id"] for calib in flatSkyWcsList])
334 remove_indices = []
335 inputs.setdefault("photoCalibs", [None] * len(inputs["dataIds"]))
336 inputs.setdefault("astromCalibs", [None] * len(inputs["dataIds"]))
338 for i in range(len(inputs["dataIds"])):
339 dataId = inputs["dataIds"][i]
340 detector = dataId["detector"]
341 visit = dataId["visit"]
342 calib_find = (visitPhotoCalibList == visit) & (
343 detectorPhotoCalibList == detector
344 )
345 if np.sum(calib_find) < 1:
346 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog "
347 "for visit %s and will not be used.",
348 detector, visit)
349 inputs["photoCalibs"][i] = None
350 remove_indices.append(i)
351 else:
352 row = flatPhotoCalibList[calib_find]
353 externalPhotoCalib = row[0].getPhotoCalib()
354 inputs["photoCalibs"][i] = externalPhotoCalib
356 for i in range(len(inputs["dataIds"])):
357 dataId = inputs["dataIds"][i]
358 detector = dataId["detector"]
359 visit = dataId["visit"]
360 calib_find = (visitSkyWcsList == visit) & (
361 detectorSkyWcsList == detector
362 )
363 if np.sum(calib_find) < 1:
364 self.log.warning("Detector id %s not found in externalSkyWcsCatalog "
365 "for visit %s and will not be used.",
366 detector, visit)
367 inputs["astromCalibs"][i] = None
368 remove_indices.append(i)
369 else:
370 row = flatSkyWcsList[calib_find]
371 externalSkyWcs = row[0].getWcs()
372 inputs["astromCalibs"][i] = externalSkyWcs
374 # Remove datasets that didn't have matching external calibs
375 remove_indices = np.unique(np.array(remove_indices))
376 if len(remove_indices) > 0:
377 for ind in sorted(remove_indices, reverse=True):
378 del inputs['sourceCatalogs'][ind]
379 del inputs['dataIds'][ind]
380 del inputs['photoCalibs'][ind]
381 del inputs['astromCalibs'][ind]
383 outputs = self.run(**inputs)
384 butlerQC.put(outputs, outputRefs)
387class MatchedTractBaseTask(MatchedBaseTask):
389 ConfigClass = MatchedBaseConfig
390 _DefaultName = "matchedTractBaseTask"
392 def __init__(self, config: MatchedBaseConfig, *args, **kwargs):
393 super().__init__(*args, config=config, **kwargs)
394 self.radius = self.config.match_radius
395 self.level = "tract"
397 def get_box_wcs(self, skymap, oid):
398 tract_info = skymap.generateTract(oid["tract"])
399 wcs = tract_info.getWcs()
400 tract_box = tract_info.getBBox()
401 self.log.info("Running tract: %s", oid["tract"])
402 return tract_box, wcs