Coverage for python/lsst/faro/base/MatchedCatalogBase.py: 21%
149 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-22 04:09 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-22 04:09 -0800
1# This file is part of faro.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import lsst.afw.table as afwTable
23import lsst.pipe.base as pipeBase
24import lsst.pex.config as pexConfig
25import lsst.geom as geom
26from lsst.utils.logging import PeriodicLogger
27import numpy as np
29from lsst.faro.utils.matcher import matchCatalogs
31__all__ = (
32 "MatchedBaseConnections",
33 "MatchedBaseConfig",
34 "MatchedBaseTask",
35 "MatchedTractBaseTask",
36)
39class MatchedBaseConnections(
40 pipeBase.PipelineTaskConnections,
41 dimensions=(),
42 defaultTemplates={
43 "coaddName": "deep",
44 "photoCalibName": "calexp.photoCalib",
45 "wcsName": "calexp.wcs",
46 "externalPhotoCalibName": "fgcm",
47 "externalWcsName": "jointcal",
48 },
49):
50 sourceCatalogs = pipeBase.connectionTypes.Input(
51 doc="Source catalogs to match up.",
52 dimensions=("instrument", "visit", "detector", "band"),
53 storageClass="SourceCatalog",
54 name="src",
55 multiple=True,
56 )
57 photoCalibs = pipeBase.connectionTypes.Input(
58 doc="Photometric calibration object.",
59 dimensions=("instrument", "visit", "detector", "band"),
60 storageClass="PhotoCalib",
61 name="{photoCalibName}",
62 multiple=True,
63 )
64 astromCalibs = pipeBase.connectionTypes.Input(
65 doc="WCS for the catalog.",
66 dimensions=("instrument", "visit", "detector", "band"),
67 storageClass="Wcs",
68 name="{wcsName}",
69 multiple=True,
70 )
71 externalSkyWcsTractCatalog = pipeBase.connectionTypes.Input(
72 doc=(
73 "Per-tract, per-visit wcs calibrations. These catalogs use the detector "
74 "id for the catalog id, sorted on id for fast lookup."
75 ),
76 name="{externalWcsName}SkyWcsCatalog",
77 storageClass="ExposureCatalog",
78 dimensions=("instrument", "visit", "tract", "band"),
79 multiple=True,
80 )
81 externalSkyWcsGlobalCatalog = pipeBase.connectionTypes.Input(
82 doc=(
83 "Per-visit wcs calibrations computed globally (with no tract information). "
84 "These catalogs use the detector id for the catalog id, sorted on id for "
85 "fast lookup."
86 ),
87 name="{externalWcsName}SkyWcsCatalog",
88 storageClass="ExposureCatalog",
89 dimensions=("instrument", "visit", "band"),
90 multiple=True,
91 )
92 externalPhotoCalibTractCatalog = pipeBase.connectionTypes.Input(
93 doc=(
94 "Per-tract, per-visit photometric calibrations. These catalogs use the "
95 "detector id for the catalog id, sorted on id for fast lookup."
96 ),
97 name="{externalPhotoCalibName}PhotoCalibCatalog",
98 storageClass="ExposureCatalog",
99 dimensions=("instrument", "visit", "tract", "band"),
100 multiple=True,
101 )
102 externalPhotoCalibGlobalCatalog = pipeBase.connectionTypes.Input(
103 doc=(
104 "Per-visit photometric calibrations computed globally (with no tract "
105 "information). These catalogs use the detector id for the catalog id, "
106 "sorted on id for fast lookup."
107 ),
108 name="{externalPhotoCalibName}PhotoCalibCatalog",
109 storageClass="ExposureCatalog",
110 dimensions=("instrument", "visit", "band"),
111 multiple=True,
112 )
113 skyMap = pipeBase.connectionTypes.Input(
114 doc="Input definition of geometry/bbox and projection/wcs for warped exposures",
115 name="skyMap",
116 storageClass="SkyMap",
117 dimensions=("skymap",),
118 )
120 def __init__(self, *, config=None):
121 super().__init__(config=config)
122 if config.doApplyExternalSkyWcs:
123 if config.useGlobalExternalSkyWcs:
124 self.inputs.remove("externalSkyWcsTractCatalog")
125 else:
126 self.inputs.remove("externalSkyWcsGlobalCatalog")
127 else:
128 self.inputs.remove("externalSkyWcsTractCatalog")
129 self.inputs.remove("externalSkyWcsGlobalCatalog")
130 if config.doApplyExternalPhotoCalib:
131 if config.useGlobalExternalPhotoCalib:
132 self.inputs.remove("externalPhotoCalibTractCatalog")
133 else:
134 self.inputs.remove("externalPhotoCalibGlobalCatalog")
135 else:
136 self.inputs.remove("externalPhotoCalibTractCatalog")
137 self.inputs.remove("externalPhotoCalibGlobalCatalog")
140class MatchedBaseConfig(
141 pipeBase.PipelineTaskConfig, pipelineConnections=MatchedBaseConnections
142):
143 match_radius = pexConfig.Field(
144 doc="Match radius in arcseconds.", dtype=float, default=1
145 )
146 snrMin = pexConfig.Field(
147 doc="Minimum SNR for a source to be included.",
148 dtype=float, default=200
149 )
150 snrMax = pexConfig.Field(
151 doc="Maximum SNR for a source to be included.",
152 dtype=float, default=np.Inf
153 )
154 brightMagCut = pexConfig.Field(
155 doc="Bright limit of catalog entries to include.", dtype=float, default=10.0
156 )
157 faintMagCut = pexConfig.Field(
158 doc="Faint limit of catalog entries to include.", dtype=float, default=30.0
159 )
160 selectExtended = pexConfig.Field(
161 doc="Whether to select extended sources", dtype=bool, default=False
162 )
163 doApplyExternalSkyWcs = pexConfig.Field(
164 doc="Whether or not to use the external wcs.", dtype=bool, default=False
165 )
166 useGlobalExternalSkyWcs = pexConfig.Field(
167 doc="Whether or not to use the global external wcs.", dtype=bool, default=False
168 )
169 doApplyExternalPhotoCalib = pexConfig.Field(
170 doc="Whether or not to use the external photoCalib.", dtype=bool, default=False
171 )
172 useGlobalExternalPhotoCalib = pexConfig.Field(
173 doc="Whether or not to use the global external photoCalib.",
174 dtype=bool,
175 default=False,
176 )
179class MatchedBaseTask(pipeBase.PipelineTask):
181 ConfigClass = MatchedBaseConfig
182 _DefaultName = "matchedBaseTask"
184 def __init__(self, config: MatchedBaseConfig, *args, **kwargs):
185 super().__init__(*args, config=config, **kwargs)
186 self.radius = self.config.match_radius
187 self.level = "patch"
189 def run(
190 self,
191 sourceCatalogs,
192 photoCalibs,
193 astromCalibs,
194 dataIds,
195 wcs,
196 box,
197 doApplyExternalSkyWcs=False,
198 doApplyExternalPhotoCalib=False,
199 ):
200 self.log.info("Running catalog matching")
201 periodicLog = PeriodicLogger(self.log)
202 radius = geom.Angle(self.radius, geom.arcseconds)
203 if len(sourceCatalogs) < 2:
204 self.log.warning("%s valid input catalogs: ", len(sourceCatalogs))
205 out_matched = afwTable.SimpleCatalog()
206 else:
207 srcvis, matched = matchCatalogs(
208 sourceCatalogs, photoCalibs, astromCalibs, dataIds, radius,
209 self.config, logger=self.log
210 )
211 self.log.verbose("Finished matching catalogs.")
213 # Trim the output to the patch bounding box
214 out_matched = type(matched)(matched.schema)
215 self.log.info("%s sources in matched catalog.", len(matched))
216 for record_index, record in enumerate(matched):
217 if box.contains(wcs.skyToPixel(record.getCoord())):
218 out_matched.append(record)
219 periodicLog.log("Checked %d records for trimming out of %d.", record_index + 1, len(matched))
221 self.log.info(
222 "%s sources when trimmed to %s boundaries.", len(out_matched), self.level
223 )
224 return pipeBase.Struct(outputCatalog=out_matched)
226 def get_box_wcs(self, skymap, oid):
227 tract_info = skymap.generateTract(oid["tract"])
228 wcs = tract_info.getWcs()
229 patch_info = tract_info.getPatchInfo(oid["patch"])
230 patch_box = patch_info.getInnerBBox()
231 self.log.info("Running tract: %s and patch: %s", oid["tract"], oid["patch"])
232 return patch_box, wcs
234 def runQuantum(self, butlerQC, inputRefs, outputRefs):
235 inputs = butlerQC.get(inputRefs)
236 self.log.verbose("Inputs obtained from the butler.")
237 oid = outputRefs.outputCatalog.dataId.byName()
238 skymap = inputs["skyMap"]
239 del inputs["skyMap"]
240 box, wcs = self.get_box_wcs(skymap, oid)
241 # Cast to float to handle fractional pixels
242 box = geom.Box2D(box)
243 inputs["dataIds"] = [
244 butlerQC.registry.expandDataId(el.dataId) for el in inputRefs.sourceCatalogs
245 ]
246 inputs["wcs"] = wcs
247 inputs["box"] = box
248 inputs["doApplyExternalSkyWcs"] = self.config.doApplyExternalSkyWcs
249 inputs["doApplyExternalPhotoCalib"] = self.config.doApplyExternalPhotoCalib
251 if self.config.doApplyExternalPhotoCalib:
252 if self.config.useGlobalExternalPhotoCalib:
253 externalPhotoCalibCatalog = inputs.pop(
254 "externalPhotoCalibGlobalCatalog"
255 )
256 else:
257 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog")
259 flatPhotoCalibList = np.hstack(externalPhotoCalibCatalog)
260 visitPhotoCalibList = np.array(
261 [calib["visit"] for calib in flatPhotoCalibList]
262 )
263 detectorPhotoCalibList = np.array(
264 [calib["id"] for calib in flatPhotoCalibList]
265 )
267 if self.config.doApplyExternalSkyWcs:
268 if self.config.useGlobalExternalSkyWcs:
269 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog")
270 else:
271 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog")
273 flatSkyWcsList = np.hstack(externalSkyWcsCatalog)
274 visitSkyWcsList = np.array([calib["visit"] for calib in flatSkyWcsList])
275 detectorSkyWcsList = np.array([calib["id"] for calib in flatSkyWcsList])
277 remove_indices = []
279 if self.config.doApplyExternalPhotoCalib:
280 for i in range(len(inputs["dataIds"])):
281 dataId = inputs["dataIds"][i]
282 detector = dataId["detector"]
283 visit = dataId["visit"]
284 calib_find = (visitPhotoCalibList == visit) & (
285 detectorPhotoCalibList == detector
286 )
287 if np.sum(calib_find) < 1:
288 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog "
289 "for visit %s and will not be used.",
290 detector, visit)
291 inputs["photoCalibs"][i] = None
292 remove_indices.append(i)
293 else:
294 row = flatPhotoCalibList[calib_find]
295 externalPhotoCalib = row[0].getPhotoCalib()
296 inputs["photoCalibs"][i] = externalPhotoCalib
298 if self.config.doApplyExternalSkyWcs:
299 for i in range(len(inputs["dataIds"])):
300 dataId = inputs["dataIds"][i]
301 detector = dataId["detector"]
302 visit = dataId["visit"]
303 calib_find = (visitSkyWcsList == visit) & (
304 detectorSkyWcsList == detector
305 )
306 if np.sum(calib_find) < 1:
307 self.log.warning("Detector id %s not found in externalSkyWcsCatalog "
308 "for visit %s and will not be used.",
309 detector, visit)
310 inputs["astromCalibs"][i] = None
311 remove_indices.append(i)
312 else:
313 row = flatSkyWcsList[calib_find]
314 externalSkyWcs = row[0].getWcs()
315 inputs["astromCalibs"][i] = externalSkyWcs
317 # Remove datasets that didn't have matching external calibs
318 remove_indices = np.unique(np.array(remove_indices))
319 if len(remove_indices) > 0:
320 for ind in sorted(remove_indices, reverse=True):
321 del inputs['sourceCatalogs'][ind]
322 del inputs['dataIds'][ind]
323 del inputs['photoCalibs'][ind]
324 del inputs['astromCalibs'][ind]
326 outputs = self.run(**inputs)
327 butlerQC.put(outputs, outputRefs)
330class MatchedTractBaseTask(MatchedBaseTask):
332 ConfigClass = MatchedBaseConfig
333 _DefaultName = "matchedTractBaseTask"
335 def __init__(self, config: MatchedBaseConfig, *args, **kwargs):
336 super().__init__(*args, config=config, **kwargs)
337 self.radius = self.config.match_radius
338 self.level = "tract"
340 def get_box_wcs(self, skymap, oid):
341 tract_info = skymap.generateTract(oid["tract"])
342 wcs = tract_info.getWcs()
343 tract_box = tract_info.getBBox()
344 self.log.info("Running tract: %s", oid["tract"])
345 return tract_box, wcs