Coverage for python/lsst/analysis/tools/tasks/catalogMatch.py: 24%
137 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-23 02:33 -0700
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-23 02:33 -0700
1# This file is part of analysis_tools.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ("CatalogMatchConfig", "CatalogMatchTask")
25import lsst.geom
26import lsst.pex.config as pexConfig
27import lsst.pipe.base as pipeBase
28import numpy as np
29from astropy.table import Table, hstack, vstack
30from astropy.time import Time
31from lsst.pex.config.configurableActions import ConfigurableActionStructField
32from lsst.pipe.tasks.loadReferenceCatalog import LoadReferenceCatalogTask
33from lsst.skymap import BaseSkyMap
34from smatch import Matcher
36from ..actions.vector import (
37 CoaddPlotFlagSelector,
38 GalaxySelector,
39 MatchingFlagSelector,
40 SnSelector,
41 StarSelector,
42)
43from ..interfaces import VectorAction
46class CatalogMatchConnections(
47 pipeBase.PipelineTaskConnections,
48 dimensions=("tract", "skymap"),
49 defaultTemplates={"targetCatalog": "objectTable_tract", "refCatalog": "ps1_pv3_3pi_20170110"},
50):
51 catalog = pipeBase.connectionTypes.Input(
52 doc="The tract-wide catalog to make plots from.",
53 storageClass="ArrowAstropy",
54 name="{targetCatalog}",
55 dimensions=("tract", "skymap"),
56 deferLoad=True,
57 )
59 refCat = pipeBase.connectionTypes.PrerequisiteInput(
60 doc="The reference catalog to match to loaded input catalog sources.",
61 name="{refCatalog}",
62 storageClass="SimpleCatalog",
63 dimensions=("skypix",),
64 deferLoad=True,
65 multiple=True,
66 )
68 skymap = pipeBase.connectionTypes.Input(
69 doc="The skymap for the tract",
70 storageClass="SkyMap",
71 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
72 dimensions=("skymap",),
73 )
75 matchedCatalog = pipeBase.connectionTypes.Output(
76 doc="Catalog with matched target and reference objects with separations",
77 name="{targetCatalog}_{refCatalog}_match",
78 storageClass="ArrowAstropy",
79 dimensions=("tract", "skymap"),
80 )
83class CatalogMatchConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CatalogMatchConnections):
84 referenceCatalogLoader = pexConfig.ConfigurableField(
85 target=LoadReferenceCatalogTask,
86 doc="Reference catalog loader",
87 )
89 epoch = pexConfig.Field[float](
90 doc="Epoch to which reference objects are shifted.",
91 default=2015.0,
92 )
94 filterNames = pexConfig.ListField[str](
95 doc="Physical filter names to persist downstream.",
96 default=["u", "g", "r", "i", "z", "y"],
97 )
99 selectorBands = pexConfig.ListField[str](
100 doc="Band to use when selecting objects, primarily for extendedness.",
101 default=["i"],
102 )
104 selectorActions = ConfigurableActionStructField[VectorAction](
105 doc="Which selectors to use to narrow down the data for QA plotting.",
106 default={"flagSelector": MatchingFlagSelector()},
107 )
109 sourceSelectorActions = ConfigurableActionStructField[VectorAction](
110 doc="What types of sources to use.",
111 default={},
112 )
114 extraColumnSelectors = ConfigurableActionStructField[VectorAction](
115 doc="Other selectors that are not used in this task, but whose columns" "may be needed downstream",
116 default={
117 "selector1": SnSelector(),
118 "selector2": StarSelector(),
119 "selector3": GalaxySelector(),
120 "selector4": CoaddPlotFlagSelector(),
121 },
122 )
124 extraColumns = pexConfig.ListField[str](
125 doc="Other catalog columns to persist to downstream tasks",
126 default=["x", "y", "patch", "ebv"],
127 )
129 extraPerBandColumns = pexConfig.ListField[str](
130 doc="Other columns to load that should be loaded for each band individually.",
131 default=["cModelFlux"],
132 )
134 matchRadius = pexConfig.Field[float](
135 doc="The radius to use for matching, in arcsecs.",
136 default=1.0,
137 )
139 targetRaColumn = pexConfig.Field[str](
140 doc="RA column name for the target (being matched) catalog.",
141 default="coord_ra",
142 )
144 targetDecColumn = pexConfig.Field[str](
145 doc="Dec column name for the target (being matched) catalog.",
146 default="coord_dec",
147 )
149 refRaColumn = pexConfig.Field[str](
150 doc="RA column name for the reference (being matched to) catalog.",
151 default="ra",
152 )
154 refDecColumn = pexConfig.Field[str](
155 doc="Dec column name for the reference (being matched to) catalog.",
156 default="dec",
157 )
159 raColumn = pexConfig.Field[str](
160 doc="RA column.",
161 default="coord_ra",
162 deprecated="This field was replaced with targetRaColumn and is unused. Will be removed after v27.",
163 )
165 decColumn = pexConfig.Field[str](
166 doc="Dec column.",
167 default="coord_dec",
168 deprecated="This field was replaced with targetDecColumn and is unused. Will be removed after v27.",
169 )
171 patchColumn = pexConfig.Field[str](doc="Patch column.", default="patch")
173 matchesRefCat = pexConfig.Field[bool](
174 doc="Is the catalog being matched to stored as a reference catalog?",
175 default=False,
176 )
178 returnNonMatches = pexConfig.Field[bool](
179 doc="Return the rows of the reference catalog that didn't get matched?",
180 default=False,
181 )
183 def setDefaults(self):
184 super().setDefaults()
185 self.referenceCatalogLoader.doReferenceSelection = False
186 self.referenceCatalogLoader.doApplyColorTerms = False
189class CatalogMatchTask(pipeBase.PipelineTask):
190 """The base task for matching catalogs. Figures out which columns
191 it needs to grab for the downstream tasks and then matches the
192 two tables together and returns the matched and joined table
193 including the extra columns.
194 """
196 ConfigClass = CatalogMatchConfig
197 _DefaultName = "analysisToolsCatalogMatch"
199 def runQuantum(self, butlerQC, inputRefs, outputRefs):
200 """Implemented in the inherited tasks"""
201 pass
203 def run(self, *, targetCatalog, refCatalog, bands):
204 """Takes the two catalogs and returns the matched one.
206 Parameters
207 ----------
208 `targetCatalog` : astropy.table.Table
209 The catalog to be matched
210 `refCatalog` : astropy.table.Table
211 The catalog to be matched to
212 `bands` : list
213 A list of bands to apply the selectors in
215 Returns
216 -------
217 `matchedCatalog` : astropy.table.Table
219 Notes
220 -----
221 Performs an RA/Dec match that returns the closest match
222 within the match radius which defaults to 1.0 arcsecond.
223 Applies the suffix, _target, to the catalog being matched
224 and _ref to the reference catalog being matched to.
225 """
226 # Apply the selectors to the catalog
227 mask = np.ones(len(targetCatalog), dtype=bool)
228 for selector in self.config.sourceSelectorActions:
229 for band in self.config.selectorBands:
230 mask &= selector(targetCatalog, band=band).astype(bool)
232 targetCatalog = targetCatalog[mask]
234 if (len(targetCatalog) == 0) or (len(refCatalog)) == 0:
235 refMatchIndices = np.array([], dtype=np.int64)
236 targetMatchIndices = np.array([], dtype=np.int64)
237 dists = np.array([], dtype=np.float64)
238 else:
239 # Run the matcher.
241 # This all assumes that everything is in degrees.
242 # Which I think is okay, but the current task
243 # allows different units. Need to configure match
244 # radius, either in this task or a subtask.
246 # Get rid of entries in the refCat with non-finite RA/Dec values.
247 refRas = refCatalog[self.config.refRaColumn]
248 refDecs = refCatalog[self.config.refDecColumn]
249 refRaDecFiniteMask = np.isfinite(refRas) & np.isfinite(refDecs)
250 refCatalog = refCatalog[refRaDecFiniteMask]
251 with Matcher(refCatalog[self.config.refRaColumn], refCatalog[self.config.refDecColumn]) as m:
252 idx, refMatchIndices, targetMatchIndices, dists = m.query_radius(
253 targetCatalog[self.config.targetRaColumn],
254 targetCatalog[self.config.targetDecColumn],
255 self.config.matchRadius / 3600.0,
256 return_indices=True,
257 )
259 # Convert degrees to arcseconds.
260 dists *= 3600.0
262 targetCatalogMatched = targetCatalog[targetMatchIndices]
263 refCatalogMatched = refCatalog[refMatchIndices]
265 targetCols = targetCatalogMatched.columns.copy()
266 for col in targetCols:
267 targetCatalogMatched.rename_column(col, col + "_target")
268 refCols = refCatalogMatched.columns.copy()
269 for col in refCols:
270 refCatalogMatched.rename_column(col, col + "_ref")
272 if self.config.returnNonMatches:
273 unmatchedIndices = list(set(np.arange(0, len(refCatalog))) - set(refMatchIndices))
274 refCatalogNotMatched = refCatalog[unmatchedIndices]
275 # We need to set the relevant flag columns to
276 # true or false so that they make it through the
277 # selectors even though the none matched sources
278 # don't have values for those columns.
279 trueFlagCols = []
280 falseFlagCols = []
281 for selectorAction in [self.config.selectorActions, self.config.extraColumnSelectors]:
282 for selector in selectorAction:
283 try:
284 for flag in selector.selectWhenTrue:
285 trueFlagCols.append(flag)
286 for flag in selector.selectWhenFalse:
287 falseFlagCols.append(flag)
288 except AttributeError:
289 continue
290 for col in refCols:
291 refCatalogNotMatched.rename_column(col, col + "_ref")
292 for col in targetCols:
293 refCatalogNotMatched[col] = [np.nan] * len(refCatalogNotMatched)
294 for col in trueFlagCols:
295 refCatalogNotMatched[col] = [True] * len(refCatalogNotMatched)
296 for col in falseFlagCols:
297 refCatalogNotMatched[col] = [False] * len(refCatalogNotMatched)
299 if self.config.matchesRefCat:
300 for i, band in enumerate(bands):
301 refCatalogMatched[band + "_mag_ref"] = refCatalogMatched["refMag_ref"][:, i]
302 refCatalogMatched[band + "_magErr_ref"] = refCatalogMatched["refMagErr_ref"][:, i]
303 refCatalogMatched.remove_column("refMag_ref")
304 refCatalogMatched.remove_column("refMagErr_ref")
306 if self.config.returnNonMatches:
307 for i, band in enumerate(bands):
308 refCatalogNotMatched[band + "_mag_ref"] = refCatalogNotMatched["refMag_ref"][:, i]
309 refCatalogNotMatched[band + "_magErr_ref"] = refCatalogNotMatched["refMagErr_ref"][:, i]
310 refCatalogNotMatched.remove_column("refMag_ref")
311 refCatalogNotMatched.remove_column("refMagErr_ref")
313 tMatched = hstack([targetCatalogMatched, refCatalogMatched], join_type="exact")
314 tMatched["matchDistance"] = dists
316 if self.config.returnNonMatches:
317 refCatalogNotMatched["matchDistance"] = [np.nan] * len(refCatalogNotMatched)
318 tMatched = vstack([tMatched, refCatalogNotMatched])
320 return pipeBase.Struct(matchedCatalog=tMatched)
322 def prepColumns(self, bands):
323 """Get all the columns needed for downstream tasks.
324 Both those from the selectors and those specified in the
325 config options.
326 """
328 bandColumns = []
329 for band in bands:
330 for col in self.config.extraPerBandColumns:
331 bandColumns.append(band + "_" + col)
333 columns = (
334 [
335 self.config.targetRaColumn,
336 self.config.targetDecColumn,
337 ]
338 + self.config.extraColumns.list()
339 + bandColumns
340 )
342 if self.config.patchColumn != "":
343 columns.append(self.config.patchColumn)
345 selectorBands = list(set(list(bands) + self.config.selectorBands.list()))
346 for selectorAction in [
347 self.config.selectorActions,
348 self.config.sourceSelectorActions,
349 self.config.extraColumnSelectors,
350 ]:
351 for selector in selectorAction:
352 for band in selectorBands:
353 selectorSchema = selector.getFormattedInputSchema(band=band)
354 columns += [s[0] for s in selectorSchema]
356 return columns
358 def _loadRefCat(self, loaderTask, tractInfo):
359 """Load the reference catalog that covers the
360 catalog that is to be matched to.
362 Parameters
363 ----------
364 `loaderTask` :
365 lsst.pipe.tasks.loadReferenceCatalog.loadReferenceCatalogTask
366 `tractInfo` : lsst.skymap.tractInfo.ExplicitTractInfo
367 The tract information to get the sky location from
369 Returns
370 -------
371 `loadedRefCat` : astropy.table.Table
372 The reference catalog that covers the input catalog.
373 """
374 boundingCircle = tractInfo.getOuterSkyPolygon().getBoundingCircle()
375 center = lsst.geom.SpherePoint(boundingCircle.getCenter())
376 radius = boundingCircle.getOpeningAngle()
378 epoch = Time(self.config.epoch, format="decimalyear")
380 # This is always going to return degrees.
381 try:
382 loadedRefCat = loaderTask.getSkyCircleCatalog(
383 center, radius, self.config.filterNames, epoch=epoch
384 )
385 except RuntimeError as e:
386 raise pipeBase.NoWorkFound(e)
388 return Table(loadedRefCat)