Coverage for python/lsst/faro/base/MatchedCatalogBase.py: 24%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of faro.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import lsst.afw.table as afwTable
23import lsst.pipe.base as pipeBase
24import lsst.pex.config as pexConfig
25import lsst.geom as geom
26import numpy as np
28from lsst.faro.utils.matcher import matchCatalogs
30__all__ = (
31 "MatchedBaseConnections",
32 "MatchedBaseConfig",
33 "MatchedBaseTask",
34 "MatchedTractBaseTask",
35)
38class MatchedBaseConnections(
39 pipeBase.PipelineTaskConnections,
40 dimensions=(),
41 defaultTemplates={
42 "coaddName": "deep",
43 "photoCalibName": "calexp.photoCalib",
44 "wcsName": "calexp.wcs",
45 "externalPhotoCalibName": "fgcm",
46 "externalWcsName": "jointcal",
47 },
48):
49 sourceCatalogs = pipeBase.connectionTypes.Input(
50 doc="Source catalogs to match up.",
51 dimensions=("instrument", "visit", "detector", "band"),
52 storageClass="SourceCatalog",
53 name="src",
54 multiple=True,
55 )
56 photoCalibs = pipeBase.connectionTypes.Input(
57 doc="Photometric calibration object.",
58 dimensions=("instrument", "visit", "detector", "band"),
59 storageClass="PhotoCalib",
60 name="{photoCalibName}",
61 multiple=True,
62 )
63 astromCalibs = pipeBase.connectionTypes.Input(
64 doc="WCS for the catalog.",
65 dimensions=("instrument", "visit", "detector", "band"),
66 storageClass="Wcs",
67 name="{wcsName}",
68 multiple=True,
69 )
70 externalSkyWcsTractCatalog = pipeBase.connectionTypes.Input(
71 doc=(
72 "Per-tract, per-visit wcs calibrations. These catalogs use the detector "
73 "id for the catalog id, sorted on id for fast lookup."
74 ),
75 name="{externalWcsName}SkyWcsCatalog",
76 storageClass="ExposureCatalog",
77 dimensions=("instrument", "visit", "tract", "band"),
78 multiple=True,
79 )
80 externalSkyWcsGlobalCatalog = pipeBase.connectionTypes.Input(
81 doc=(
82 "Per-visit wcs calibrations computed globally (with no tract information). "
83 "These catalogs use the detector id for the catalog id, sorted on id for "
84 "fast lookup."
85 ),
86 name="{externalWcsName}SkyWcsCatalog",
87 storageClass="ExposureCatalog",
88 dimensions=("instrument", "visit", "band"),
89 multiple=True,
90 )
91 externalPhotoCalibTractCatalog = pipeBase.connectionTypes.Input(
92 doc=(
93 "Per-tract, per-visit photometric calibrations. These catalogs use the "
94 "detector id for the catalog id, sorted on id for fast lookup."
95 ),
96 name="{externalPhotoCalibName}PhotoCalibCatalog",
97 storageClass="ExposureCatalog",
98 dimensions=("instrument", "visit", "tract", "band"),
99 multiple=True,
100 )
101 externalPhotoCalibGlobalCatalog = pipeBase.connectionTypes.Input(
102 doc=(
103 "Per-visit photometric calibrations computed globally (with no tract "
104 "information). These catalogs use the detector id for the catalog id, "
105 "sorted on id for fast lookup."
106 ),
107 name="{externalPhotoCalibName}PhotoCalibCatalog",
108 storageClass="ExposureCatalog",
109 dimensions=("instrument", "visit", "band"),
110 multiple=True,
111 )
112 skyMap = pipeBase.connectionTypes.Input(
113 doc="Input definition of geometry/bbox and projection/wcs for warped exposures",
114 name="skyMap",
115 storageClass="SkyMap",
116 dimensions=("skymap",),
117 )
119 def __init__(self, *, config=None):
120 super().__init__(config=config)
121 if config.doApplyExternalSkyWcs:
122 if config.useGlobalExternalSkyWcs:
123 self.inputs.remove("externalSkyWcsTractCatalog")
124 else:
125 self.inputs.remove("externalSkyWcsGlobalCatalog")
126 else:
127 self.inputs.remove("externalSkyWcsTractCatalog")
128 self.inputs.remove("externalSkyWcsGlobalCatalog")
129 if config.doApplyExternalPhotoCalib:
130 if config.useGlobalExternalPhotoCalib:
131 self.inputs.remove("externalPhotoCalibTractCatalog")
132 else:
133 self.inputs.remove("externalPhotoCalibGlobalCatalog")
134 else:
135 self.inputs.remove("externalPhotoCalibTractCatalog")
136 self.inputs.remove("externalPhotoCalibGlobalCatalog")
139class MatchedBaseConfig(
140 pipeBase.PipelineTaskConfig, pipelineConnections=MatchedBaseConnections
141):
142 match_radius = pexConfig.Field(
143 doc="Match radius in arcseconds.", dtype=float, default=1
144 )
145 snrMin = pexConfig.Field(
146 doc="Minimum SNR for a source to be included.",
147 dtype=float, default=200
148 )
149 snrMax = pexConfig.Field(
150 doc="Maximum SNR for a source to be included.",
151 dtype=float, default=np.Inf
152 )
153 brightMagCut = pexConfig.Field(
154 doc="Bright limit of catalog entries to include.", dtype=float, default=10.0
155 )
156 faintMagCut = pexConfig.Field(
157 doc="Faint limit of catalog entries to include.", dtype=float, default=30.0
158 )
159 selectExtended = pexConfig.Field(
160 doc="Whether to select extended sources", dtype=bool, default=False
161 )
162 doApplyExternalSkyWcs = pexConfig.Field(
163 doc="Whether or not to use the external wcs.", dtype=bool, default=False
164 )
165 useGlobalExternalSkyWcs = pexConfig.Field(
166 doc="Whether or not to use the global external wcs.", dtype=bool, default=False
167 )
168 doApplyExternalPhotoCalib = pexConfig.Field(
169 doc="Whether or not to use the external photoCalib.", dtype=bool, default=False
170 )
171 useGlobalExternalPhotoCalib = pexConfig.Field(
172 doc="Whether or not to use the global external photoCalib.",
173 dtype=bool,
174 default=False,
175 )
178class MatchedBaseTask(pipeBase.PipelineTask):
180 ConfigClass = MatchedBaseConfig
181 _DefaultName = "matchedBaseTask"
183 def __init__(self, config: MatchedBaseConfig, *args, **kwargs):
184 super().__init__(*args, config=config, **kwargs)
185 self.radius = self.config.match_radius
186 self.level = "patch"
188 def run(
189 self,
190 sourceCatalogs,
191 photoCalibs,
192 astromCalibs,
193 dataIds,
194 wcs,
195 box,
196 doApplyExternalSkyWcs=False,
197 doApplyExternalPhotoCalib=False,
198 ):
199 self.log.info("Running catalog matching")
200 radius = geom.Angle(self.radius, geom.arcseconds)
201 if len(sourceCatalogs) < 2:
202 self.log.warning("%s valid input catalogs: ", len(sourceCatalogs))
203 out_matched = afwTable.SimpleCatalog()
204 else:
205 srcvis, matched = matchCatalogs(
206 sourceCatalogs, photoCalibs, astromCalibs, dataIds, radius,
207 self.config, logger=self.log
208 )
209 # Trim the output to the patch bounding box
210 out_matched = type(matched)(matched.schema)
211 self.log.info("%s sources in matched catalog.", len(matched))
212 for record in matched:
213 if box.contains(wcs.skyToPixel(record.getCoord())):
214 out_matched.append(record)
215 self.log.info(
216 "%s sources when trimmed to %s boundaries.", len(out_matched), self.level
217 )
218 return pipeBase.Struct(outputCatalog=out_matched)
220 def get_box_wcs(self, skymap, oid):
221 tract_info = skymap.generateTract(oid["tract"])
222 wcs = tract_info.getWcs()
223 patch_info = tract_info.getPatchInfo(oid["patch"])
224 patch_box = patch_info.getInnerBBox()
225 self.log.info("Running tract: %s and patch: %s", oid["tract"], oid["patch"])
226 return patch_box, wcs
228 def runQuantum(self, butlerQC, inputRefs, outputRefs):
229 inputs = butlerQC.get(inputRefs)
230 oid = outputRefs.outputCatalog.dataId.byName()
231 skymap = inputs["skyMap"]
232 del inputs["skyMap"]
233 box, wcs = self.get_box_wcs(skymap, oid)
234 # Cast to float to handle fractional pixels
235 box = geom.Box2D(box)
236 inputs["dataIds"] = [
237 butlerQC.registry.expandDataId(el.dataId) for el in inputRefs.sourceCatalogs
238 ]
239 inputs["wcs"] = wcs
240 inputs["box"] = box
241 inputs["doApplyExternalSkyWcs"] = self.config.doApplyExternalSkyWcs
242 inputs["doApplyExternalPhotoCalib"] = self.config.doApplyExternalPhotoCalib
244 if self.config.doApplyExternalPhotoCalib:
245 if self.config.useGlobalExternalPhotoCalib:
246 externalPhotoCalibCatalog = inputs.pop(
247 "externalPhotoCalibGlobalCatalog"
248 )
249 else:
250 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog")
252 flatPhotoCalibList = np.hstack(externalPhotoCalibCatalog)
253 visitPhotoCalibList = np.array(
254 [calib["visit"] for calib in flatPhotoCalibList]
255 )
256 detectorPhotoCalibList = np.array(
257 [calib["id"] for calib in flatPhotoCalibList]
258 )
260 if self.config.doApplyExternalSkyWcs:
261 if self.config.useGlobalExternalSkyWcs:
262 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog")
263 else:
264 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog")
266 flatSkyWcsList = np.hstack(externalSkyWcsCatalog)
267 visitSkyWcsList = np.array([calib["visit"] for calib in flatSkyWcsList])
268 detectorSkyWcsList = np.array([calib["id"] for calib in flatSkyWcsList])
270 remove_indices = []
272 if self.config.doApplyExternalPhotoCalib:
273 for i in range(len(inputs["dataIds"])):
274 dataId = inputs["dataIds"][i]
275 detector = dataId["detector"]
276 visit = dataId["visit"]
277 calib_find = (visitPhotoCalibList == visit) & (
278 detectorPhotoCalibList == detector
279 )
280 if np.sum(calib_find) < 1:
281 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog "
282 "for visit %s and will not be used.",
283 detector, visit)
284 inputs["photoCalibs"][i] = None
285 remove_indices.append(i)
286 else:
287 row = flatPhotoCalibList[calib_find]
288 externalPhotoCalib = row[0].getPhotoCalib()
289 inputs["photoCalibs"][i] = externalPhotoCalib
291 if self.config.doApplyExternalSkyWcs:
292 for i in range(len(inputs["dataIds"])):
293 dataId = inputs["dataIds"][i]
294 detector = dataId["detector"]
295 visit = dataId["visit"]
296 calib_find = (visitSkyWcsList == visit) & (
297 detectorSkyWcsList == detector
298 )
299 if np.sum(calib_find) < 1:
300 self.log.warning("Detector id %s not found in externalSkyWcsCatalog "
301 "for visit %s and will not be used.",
302 detector, visit)
303 inputs["astromCalibs"][i] = None
304 remove_indices.append(i)
305 else:
306 row = flatSkyWcsList[calib_find]
307 externalSkyWcs = row[0].getWcs()
308 inputs["astromCalibs"][i] = externalSkyWcs
310 # Remove datasets that didn't have matching external calibs
311 remove_indices = np.unique(np.array(remove_indices))
312 if len(remove_indices) > 0:
313 for ind in sorted(remove_indices, reverse=True):
314 del inputs['sourceCatalogs'][ind]
315 del inputs['dataIds'][ind]
316 del inputs['photoCalibs'][ind]
317 del inputs['astromCalibs'][ind]
319 outputs = self.run(**inputs)
320 butlerQC.put(outputs, outputRefs)
323class MatchedTractBaseTask(MatchedBaseTask):
325 ConfigClass = MatchedBaseConfig
326 _DefaultName = "matchedTractBaseTask"
328 def __init__(self, config: MatchedBaseConfig, *args, **kwargs):
329 super().__init__(*args, config=config, **kwargs)
330 self.radius = self.config.match_radius
331 self.level = "tract"
333 def get_box_wcs(self, skymap, oid):
334 tract_info = skymap.generateTract(oid["tract"])
335 wcs = tract_info.getWcs()
336 tract_box = tract_info.getBBox()
337 self.log.info("Running tract: %s", oid["tract"])
338 return tract_box, wcs