lsst.pipe.tasks gb1d6de0934+c320316d7a
propagateSourceFlags.py
Go to the documentation of this file.
1# LSST Data Management System
2# Copyright 2022 LSST
3#
4# This product includes software developed by the
5# LSST Project (http://www.lsst.org/).
6#
7# This program is free software: you can redistribute it and/or modify
8# it under the terms of the GNU General Public License as published by
9# the Free Software Foundation, either version 3 of the License, or
10# (at your option) any later version.
11#
12# This program is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU General Public License for more details.
16#
17# You should have received a copy of the LSST License Statement and
18# the GNU General Public License along with this program. If not,
19# see <http://www.lsstcorp.org/LegalNotices/>.
20#
21__all__ = ["PropagateSourceFlagsConfig", "PropagateSourceFlagsTask"]
22
23import numpy as np
24
25from smatch.matcher import Matcher
26
27import lsst.pex.config as pexConfig
28import lsst.pipe.base as pipeBase
29
30
31class PropagateSourceFlagsConfig(pexConfig.Config):
32 """Configuration for propagating source flags to coadd objects."""
33 source_flags = pexConfig.DictField(
34 keytype=str,
35 itemtype=float,
36 default={
37 # TODO: DM-34391: when doApplyFinalizedPsf is the default, these flags
38 # should be set below and not here.
39 "calib_psf_candidate": 0.2,
40 "calib_psf_used": 0.2,
41 "calib_psf_reserved": 0.2,
42 "calib_astrometry_used": 0.2,
43 "calib_photometry_used": 0.2,
44 "calib_photometry_reserved": 0.2
45 },
46 doc=("Source flags to propagate, with the threshold of relative occurrence "
47 "(valid range: [0-1]). Coadd object will have flag set if fraction "
48 "of input visits in which it is flagged is greater than the threshold."),
49 )
50 finalized_source_flags = pexConfig.DictField(
51 keytype=str,
52 itemtype=float,
53 default={
54 # TODO: DM-34391: when doApplyFinalizedPsf is the default, these flags
55 # should be set here and not above.
56 # "calib_psf_candidate": 0.2,
57 # "calib_psf_used": 0.2,
58 # "calib_psf_reserved": 0.2
59 },
60 doc=("Finalized source flags to propagate, with the threshold of relative "
61 "occurrence (valid range: [0-1]). Coadd object will have flag set if "
62 "fraction of input visits in which it is flagged is greater than the "
63 "threshold."),
64 )
65 x_column = pexConfig.Field(
66 doc="Name of column with source x position (sourceTable_visit).",
67 dtype=str,
68 default="x",
69 )
70 y_column = pexConfig.Field(
71 doc="Name of column with source y position (sourceTable_visit).",
72 dtype=str,
73 default="y",
74 )
75 finalized_x_column = pexConfig.Field(
76 doc="Name of column with source x position (finalized_src_table).",
77 dtype=str,
78 default="slot_Centroid_x",
79 )
80 finalized_y_column = pexConfig.Field(
81 doc="Name of column with source y position (finalized_src_table).",
82 dtype=str,
83 default="slot_Centroid_y",
84 )
85 match_radius = pexConfig.Field(
86 dtype=float,
87 default=0.2,
88 doc="Source matching radius (arcsec)"
89 )
90
91 def validate(self):
92 super().validate()
93
94 if set(self.source_flagssource_flags).intersection(set(self.finalized_source_flagsfinalized_source_flags)):
95 source_flags = self.source_flagssource_flags.keys()
96 finalized_source_flags = self.finalized_source_flagsfinalized_source_flags.keys()
97 raise ValueError(f"The set of source_flags {source_flags} must not overlap "
98 f"with the finalized_source_flags {finalized_source_flags}")
99
100
101class PropagateSourceFlagsTask(pipeBase.Task):
102 """Task to propagate source flags to coadd objects.
103
104 Flagged sources may come from a mix of two different types of source catalogs.
105 The source_table catalogs from ``CalibrateTask`` contain flags for the first
106 round of astromety/photometry/psf fits.
107 The finalized_source_table catalogs from ``FinalizeCalibrationTask`` contain
108 flags from the second round of psf fitting.
109 """
110 ConfigClass = PropagateSourceFlagsConfig
111
112 def __init__(self, schema, **kwargs):
113 pipeBase.Task.__init__(self, **kwargs)
114
115 self.schemaschema = schema
116 for f in self.config.source_flags:
117 self.schemaschema.addField(f, type="Flag", doc="Propagated from sources")
118 for f in self.config.finalized_source_flags:
119 self.schemaschema.addField(f, type="Flag", doc="Propagated from finalized sources")
120
121 def run(self, coadd_object_cat, ccd_inputs,
122 source_table_handle_dict=None, finalized_source_table_handle_dict=None):
123 """Propagate flags from single-frame sources to coadd objects.
124
125 Flags are only propagated if a configurable percentage of the sources
126 are matched to the coadd objects. This task will match both "plain"
127 source flags and "finalized" source flags.
128
129 Parameters
130 ----------
131 coadd_object_cat : `lsst.afw.table.SourceCatalog`
132 Table of coadd objects.
133 ccd_inputs : `lsst.afw.table.ExposureCatalog`
134 Table of single-frame inputs to coadd.
135 source_table_handle_dict : `dict` [`int`: `lsst.daf.butler.DeferredDatasetHandle`]
136 Dict for sourceTable_visit handles (key is visit). May be None if
137 ``config.source_flags`` has no entries.
138 finalized_source_table_handle_dict : `dict` [`int`:
139 `lsst.daf.butler.DeferredDatasetHandle`]
140 Dict for finalized_src_table handles (key is visit). May be None if
141 ``config.finalized_source_flags`` has no entries.
142 """
143 if len(self.config.source_flags) == 0 and len(self.config.finalized_source_flags) == 0:
144 return
145
146 source_columns = self._get_source_table_column_names_get_source_table_column_names(
147 self.config.x_column,
148 self.config.y_column,
149 self.config.source_flags.keys()
150 )
151 finalized_columns = self._get_source_table_column_names_get_source_table_column_names(
152 self.config.finalized_x_column,
153 self.config.finalized_y_column,
154 self.config.finalized_source_flags.keys(),
155 )
156
157 # We need the number of overlaps of individual detectors for each coadd source.
158 # The following code is slow and inefficient, but can be made simpler in the future
159 # case of cell-based coadds and so optimizing usage in afw is not a priority.
160 num_overlaps = np.zeros(len(coadd_object_cat), dtype=np.int32)
161 for i, obj in enumerate(coadd_object_cat):
162 num_overlaps[i] = len(ccd_inputs.subsetContaining(obj.getCoord(), True))
163
164 visits = np.unique(ccd_inputs["visit"])
165
166 matcher = Matcher(np.rad2deg(coadd_object_cat["coord_ra"]),
167 np.rad2deg(coadd_object_cat["coord_dec"]))
168
169 source_flag_counts = {f: np.zeros(len(coadd_object_cat), dtype=np.int32)
170 for f in self.config.source_flags}
171 finalized_source_flag_counts = {f: np.zeros(len(coadd_object_cat), dtype=np.int32)
172 for f in self.config.finalized_source_flags}
173
174 handles_list = [source_table_handle_dict, finalized_source_table_handle_dict]
175 columns_list = [source_columns, finalized_columns]
176 counts_list = [source_flag_counts, finalized_source_flag_counts]
177 x_column_list = [self.config.x_column, self.config.finalized_x_column]
178 y_column_list = [self.config.y_column, self.config.finalized_y_column]
179
180 for handle_dict, columns, flag_counts, x_col, y_col in zip(handles_list,
181 columns_list,
182 counts_list,
183 x_column_list,
184 y_column_list):
185 if handle_dict is not None and len(columns) > 0:
186 for visit in visits:
187 handle = handle_dict[visit]
188 df = handle.get(parameters={"columns": columns})
189
190 # Loop over all ccd_inputs rows for this visit.
191 for row in ccd_inputs[ccd_inputs["visit"] == visit]:
192 detector = row["ccd"]
193 wcs = row.getWcs()
194
195 df_det = df[df["detector"] == detector]
196
197 if len(df_det) == 0:
198 continue
199
200 ra, dec = wcs.pixelToSkyArray(df_det[x_col].values,
201 df_det[y_col].values,
202 degrees=True)
203
204 try:
205 # The output from the matcher links
206 # coadd_object_cat[i1] <-> df_det[i2]
207 # All objects within the match radius are matched.
208 idx, i1, i2, d = matcher.query_radius(
209 ra,
210 dec,
211 self.config.match_radius/3600.,
212 return_indices=True
213 )
214 except IndexError:
215 # No matches. Workaround a bug in smatch.
216 continue
217
218 for flag in flag_counts:
219 flag_values = df_det[flag].values
220 flag_counts[flag][i1] += flag_values[i2].astype(np.int32)
221
222 for flag in source_flag_counts:
223 thresh = num_overlaps*self.config.source_flags[flag]
224 object_flag = (source_flag_counts[flag] > thresh)
225 coadd_object_cat[flag] = object_flag
226 self.log.info("Propagated %d sources with flag %s", object_flag.sum(), flag)
227
228 for flag in finalized_source_flag_counts:
229 thresh = num_overlaps*self.config.finalized_source_flags[flag]
230 object_flag = (finalized_source_flag_counts[flag] > thresh)
231 coadd_object_cat[flag] = object_flag
232 self.log.info("Propagated %d finalized sources with flag %s", object_flag.sum(), flag)
233
234 def _get_source_table_column_names(self, x_column, y_column, flags):
235 """Get the list of source table columns from the config.
236
237 Parameters
238 ----------
239 x_column : `str`
240 Name of column with x centroid.
241 y_column : `str`
242 Name of column with y centroid.
243 flags : `list` [`str`]
244 List of flags to retrieve.
245
246 Returns
247 -------
248 columns : [`list`] [`str`]
249 Columns to read.
250 """
251 columns = ["visit", "detector",
252 x_column, y_column]
253 columns.extend(flags)
254
255 return columns
def run(self, coadd_object_cat, ccd_inputs, source_table_handle_dict=None, finalized_source_table_handle_dict=None)