Coverage for python/lsst/faro/base/MatchedCatalogBase.py: 19%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of faro.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import lsst.afw.table as afwTable
23import lsst.pipe.base as pipeBase
24import lsst.pex.config as pexConfig
25import lsst.geom as geom
26import numpy as np
28from lsst.faro.utils.matcher import matchCatalogs
30__all__ = (
31 "MatchedBaseConnections",
32 "MatchedBaseConfig",
33 "MatchedBaseTask",
34 "MatchedTractBaseTask",
35)
38class MatchedBaseConnections(
39 pipeBase.PipelineTaskConnections,
40 dimensions=(),
41 defaultTemplates={
42 "coaddName": "deep",
43 "photoCalibName": "calexp.photoCalib",
44 "wcsName": "calexp.wcs",
45 "externalPhotoCalibName": "fgcm",
46 "externalWcsName": "jointcal",
47 },
48):
49 sourceCatalogs = pipeBase.connectionTypes.Input(
50 doc="Source catalogs to match up.",
51 dimensions=("instrument", "visit", "detector", "band"),
52 storageClass="SourceCatalog",
53 name="src",
54 multiple=True,
55 )
56 photoCalibs = pipeBase.connectionTypes.Input(
57 doc="Photometric calibration object.",
58 dimensions=("instrument", "visit", "detector", "band"),
59 storageClass="PhotoCalib",
60 name="{photoCalibName}",
61 multiple=True,
62 )
63 astromCalibs = pipeBase.connectionTypes.Input(
64 doc="WCS for the catalog.",
65 dimensions=("instrument", "visit", "detector", "band"),
66 storageClass="Wcs",
67 name="{wcsName}",
68 multiple=True,
69 )
70 externalSkyWcsTractCatalog = pipeBase.connectionTypes.Input(
71 doc=(
72 "Per-tract, per-visit wcs calibrations. These catalogs use the detector "
73 "id for the catalog id, sorted on id for fast lookup."
74 ),
75 name="{externalWcsName}SkyWcsCatalog",
76 storageClass="ExposureCatalog",
77 dimensions=("instrument", "visit", "tract", "band"),
78 multiple=True,
79 )
80 externalSkyWcsGlobalCatalog = pipeBase.connectionTypes.Input(
81 doc=(
82 "Per-visit wcs calibrations computed globally (with no tract information). "
83 "These catalogs use the detector id for the catalog id, sorted on id for "
84 "fast lookup."
85 ),
86 name="{externalWcsName}SkyWcsCatalog",
87 storageClass="ExposureCatalog",
88 dimensions=("instrument", "visit", "band"),
89 multiple=True,
90 )
91 externalPhotoCalibTractCatalog = pipeBase.connectionTypes.Input(
92 doc=(
93 "Per-tract, per-visit photometric calibrations. These catalogs use the "
94 "detector id for the catalog id, sorted on id for fast lookup."
95 ),
96 name="{externalPhotoCalibName}PhotoCalibCatalog",
97 storageClass="ExposureCatalog",
98 dimensions=("instrument", "visit", "tract", "band"),
99 multiple=True,
100 )
101 externalPhotoCalibGlobalCatalog = pipeBase.connectionTypes.Input(
102 doc=(
103 "Per-visit photometric calibrations computed globally (with no tract "
104 "information). These catalogs use the detector id for the catalog id, "
105 "sorted on id for fast lookup."
106 ),
107 name="{externalPhotoCalibName}PhotoCalibCatalog",
108 storageClass="ExposureCatalog",
109 dimensions=("instrument", "visit", "band"),
110 multiple=True,
111 )
112 skyMap = pipeBase.connectionTypes.Input(
113 doc="Input definition of geometry/bbox and projection/wcs for warped exposures",
114 name="skyMap",
115 storageClass="SkyMap",
116 dimensions=("skymap",),
117 )
119 def __init__(self, *, config=None):
120 super().__init__(config=config)
121 if config.doApplyExternalSkyWcs:
122 if config.useGlobalExternalSkyWcs:
123 self.inputs.remove("externalSkyWcsTractCatalog")
124 else:
125 self.inputs.remove("externalSkyWcsGlobalCatalog")
126 else:
127 self.inputs.remove("externalSkyWcsTractCatalog")
128 self.inputs.remove("externalSkyWcsGlobalCatalog")
129 if config.doApplyExternalPhotoCalib:
130 if config.useGlobalExternalPhotoCalib:
131 self.inputs.remove("externalPhotoCalibTractCatalog")
132 else:
133 self.inputs.remove("externalPhotoCalibGlobalCatalog")
134 else:
135 self.inputs.remove("externalPhotoCalibTractCatalog")
136 self.inputs.remove("externalPhotoCalibGlobalCatalog")
139class MatchedBaseConfig(
140 pipeBase.PipelineTaskConfig, pipelineConnections=MatchedBaseConnections
141):
142 match_radius = pexConfig.Field(
143 doc="Match radius in arcseconds.", dtype=float, default=1
144 )
145 doApplyExternalSkyWcs = pexConfig.Field(
146 doc="Whether or not to use the external wcs.", dtype=bool, default=False
147 )
148 useGlobalExternalSkyWcs = pexConfig.Field(
149 doc="Whether or not to use the global external wcs.", dtype=bool, default=False
150 )
151 doApplyExternalPhotoCalib = pexConfig.Field(
152 doc="Whether or not to use the external photoCalib.", dtype=bool, default=False
153 )
154 useGlobalExternalPhotoCalib = pexConfig.Field(
155 doc="Whether or not to use the global external photoCalib.",
156 dtype=bool,
157 default=False,
158 )
161class MatchedBaseTask(pipeBase.PipelineTask):
163 ConfigClass = MatchedBaseConfig
164 _DefaultName = "matchedBaseTask"
166 def __init__(self, config: MatchedBaseConfig, *args, **kwargs):
167 super().__init__(*args, config=config, **kwargs)
168 self.radius = self.config.match_radius
169 self.level = "patch"
171 def run(
172 self,
173 sourceCatalogs,
174 photoCalibs,
175 astromCalibs,
176 dataIds,
177 wcs,
178 box,
179 doApplyExternalSkyWcs=False,
180 doApplyExternalPhotoCalib=False,
181 ):
182 self.log.info("Running catalog matching")
183 radius = geom.Angle(self.radius, geom.arcseconds)
184 if len(sourceCatalogs) < 2:
185 self.log.warning("%s valid input catalogs: ", len(sourceCatalogs))
186 out_matched = afwTable.SimpleCatalog()
187 else:
188 srcvis, matched = matchCatalogs(
189 sourceCatalogs, photoCalibs, astromCalibs, dataIds, radius, logger=self.log
190 )
191 # Trim the output to the patch bounding box
192 out_matched = type(matched)(matched.schema)
193 self.log.info("%s sources in matched catalog.", len(matched))
194 for record in matched:
195 if box.contains(wcs.skyToPixel(record.getCoord())):
196 out_matched.append(record)
197 self.log.info(
198 "%s sources when trimmed to %s boundaries.", len(out_matched), self.level
199 )
200 return pipeBase.Struct(outputCatalog=out_matched)
202 def get_box_wcs(self, skymap, oid):
203 tract_info = skymap.generateTract(oid["tract"])
204 wcs = tract_info.getWcs()
205 patch_info = tract_info.getPatchInfo(oid["patch"])
206 patch_box = patch_info.getInnerBBox()
207 self.log.info("Running tract: %s and patch: %s", oid["tract"], oid["patch"])
208 return patch_box, wcs
210 def runQuantum(self, butlerQC, inputRefs, outputRefs):
211 inputs = butlerQC.get(inputRefs)
212 oid = outputRefs.outputCatalog.dataId.byName()
213 skymap = inputs["skyMap"]
214 del inputs["skyMap"]
215 box, wcs = self.get_box_wcs(skymap, oid)
216 # Cast to float to handle fractional pixels
217 box = geom.Box2D(box)
218 inputs["dataIds"] = [
219 butlerQC.registry.expandDataId(el.dataId) for el in inputRefs.sourceCatalogs
220 ]
221 inputs["wcs"] = wcs
222 inputs["box"] = box
223 inputs["doApplyExternalSkyWcs"] = self.config.doApplyExternalSkyWcs
224 inputs["doApplyExternalPhotoCalib"] = self.config.doApplyExternalPhotoCalib
226 if self.config.doApplyExternalPhotoCalib:
227 if self.config.useGlobalExternalPhotoCalib:
228 externalPhotoCalibCatalog = inputs.pop(
229 "externalPhotoCalibGlobalCatalog"
230 )
231 else:
232 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog")
234 flatPhotoCalibList = np.hstack(externalPhotoCalibCatalog)
235 visitPhotoCalibList = np.array(
236 [calib["visit"] for calib in flatPhotoCalibList]
237 )
238 detectorPhotoCalibList = np.array(
239 [calib["id"] for calib in flatPhotoCalibList]
240 )
242 if self.config.doApplyExternalSkyWcs:
243 if self.config.useGlobalExternalSkyWcs:
244 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog")
245 else:
246 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog")
248 flatSkyWcsList = np.hstack(externalSkyWcsCatalog)
249 visitSkyWcsList = np.array([calib["visit"] for calib in flatSkyWcsList])
250 detectorSkyWcsList = np.array([calib["id"] for calib in flatSkyWcsList])
252 remove_indices = []
254 if self.config.doApplyExternalPhotoCalib:
255 for i in range(len(inputs["dataIds"])):
256 dataId = inputs["dataIds"][i]
257 detector = dataId["detector"]
258 visit = dataId["visit"]
259 calib_find = (visitPhotoCalibList == visit) & (
260 detectorPhotoCalibList == detector
261 )
262 if np.sum(calib_find) < 1:
263 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog "
264 "for visit %s and will not be used.",
265 detector, visit)
266 inputs["photoCalibs"][i] = None
267 remove_indices.append(i)
268 else:
269 row = flatPhotoCalibList[calib_find]
270 externalPhotoCalib = row[0].getPhotoCalib()
271 inputs["photoCalibs"][i] = externalPhotoCalib
273 if self.config.doApplyExternalSkyWcs:
274 for i in range(len(inputs["dataIds"])):
275 dataId = inputs["dataIds"][i]
276 detector = dataId["detector"]
277 visit = dataId["visit"]
278 calib_find = (visitSkyWcsList == visit) & (
279 detectorSkyWcsList == detector
280 )
281 if np.sum(calib_find) < 1:
282 self.log.warning("Detector id %s not found in externalSkyWcsCatalog "
283 "for visit %s and will not be used.",
284 detector, visit)
285 inputs["astromCalibs"][i] = None
286 remove_indices.append(i)
287 else:
288 row = flatSkyWcsList[calib_find]
289 externalSkyWcs = row[0].getWcs()
290 inputs["astromCalibs"][i] = externalSkyWcs
292 # Remove datasets that didn't have matching external calibs
293 remove_indices = np.unique(np.array(remove_indices))
294 if len(remove_indices) > 0:
295 for ind in sorted(remove_indices, reverse=True):
296 del inputs['sourceCatalogs'][ind]
297 del inputs['dataIds'][ind]
298 del inputs['photoCalibs'][ind]
299 del inputs['astromCalibs'][ind]
301 outputs = self.run(**inputs)
302 butlerQC.put(outputs, outputRefs)
305class MatchedTractBaseTask(MatchedBaseTask):
307 ConfigClass = MatchedBaseConfig
308 _DefaultName = "matchedTractBaseTask"
310 def __init__(self, config: MatchedBaseConfig, *args, **kwargs):
311 super().__init__(*args, config=config, **kwargs)
312 self.radius = self.config.match_radius
313 self.level = "tract"
315 def get_box_wcs(self, skymap, oid):
316 tract_info = skymap.generateTract(oid["tract"])
317 wcs = tract_info.getWcs()
318 tract_box = tract_info.getBBox()
319 self.log.info("Running tract: %s", oid["tract"])
320 return tract_box, wcs