Coverage for python/lsst/pipe/tasks/propagateSourceFlags.py: 21%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

78 statements  

1# LSST Data Management System 

2# Copyright 2022 LSST 

3# 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# 

7# This program is free software: you can redistribute it and/or modify 

8# it under the terms of the GNU General Public License as published by 

9# the Free Software Foundation, either version 3 of the License, or 

10# (at your option) any later version. 

11# 

12# This program is distributed in the hope that it will be useful, 

13# but WITHOUT ANY WARRANTY; without even the implied warranty of 

14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

15# GNU General Public License for more details. 

16# 

17# You should have received a copy of the LSST License Statement and 

18# the GNU General Public License along with this program. If not, 

19# see <http://www.lsstcorp.org/LegalNotices/>. 

20# 

21__all__ = ["PropagateSourceFlagsConfig", "PropagateSourceFlagsTask"] 

22 

23import numpy as np 

24 

25from smatch.matcher import Matcher 

26 

27import lsst.pex.config as pexConfig 

28import lsst.pipe.base as pipeBase 

29 

30 

31class PropagateSourceFlagsConfig(pexConfig.Config): 

32 """Configuration for propagating source flags to coadd objects.""" 

33 source_flags = pexConfig.DictField( 

34 keytype=str, 

35 itemtype=float, 

36 default={ 

37 # TODO: DM-34391: when doApplyFinalizedPsf is the default, these flags 

38 # should be set below and not here. 

39 "calib_psf_candidate": 0.2, 

40 "calib_psf_used": 0.2, 

41 "calib_psf_reserved": 0.2, 

42 "calib_astrometry_used": 0.2, 

43 "calib_photometry_used": 0.2, 

44 "calib_photometry_reserved": 0.2 

45 }, 

46 doc=("Source flags to propagate, with the threshold of relative occurrence " 

47 "(valid range: [0-1]). Coadd object will have flag set if fraction " 

48 "of input visits in which it is flagged is greater than the threshold."), 

49 ) 

50 finalized_source_flags = pexConfig.DictField( 

51 keytype=str, 

52 itemtype=float, 

53 default={ 

54 # TODO: DM-34391: when doApplyFinalizedPsf is the default, these flags 

55 # should be set here and not above. 

56 # "calib_psf_candidate": 0.2, 

57 # "calib_psf_used": 0.2, 

58 # "calib_psf_reserved": 0.2 

59 }, 

60 doc=("Finalized source flags to propagate, with the threshold of relative " 

61 "occurrence (valid range: [0-1]). Coadd object will have flag set if " 

62 "fraction of input visits in which it is flagged is greater than the " 

63 "threshold."), 

64 ) 

65 x_column = pexConfig.Field( 

66 doc="Name of column with source x position (sourceTable_visit).", 

67 dtype=str, 

68 default="x", 

69 ) 

70 y_column = pexConfig.Field( 

71 doc="Name of column with source y position (sourceTable_visit).", 

72 dtype=str, 

73 default="y", 

74 ) 

75 finalized_x_column = pexConfig.Field( 

76 doc="Name of column with source x position (finalized_src_table).", 

77 dtype=str, 

78 default="slot_Centroid_x", 

79 ) 

80 finalized_y_column = pexConfig.Field( 

81 doc="Name of column with source y position (finalized_src_table).", 

82 dtype=str, 

83 default="slot_Centroid_y", 

84 ) 

85 match_radius = pexConfig.Field( 

86 dtype=float, 

87 default=0.2, 

88 doc="Source matching radius (arcsec)" 

89 ) 

90 

91 def validate(self): 

92 super().validate() 

93 

94 if set(self.source_flags).intersection(set(self.finalized_source_flags)): 

95 source_flags = self.source_flags.keys() 

96 finalized_source_flags = self.finalized_source_flags.keys() 

97 raise ValueError(f"The set of source_flags {source_flags} must not overlap " 

98 f"with the finalized_source_flags {finalized_source_flags}") 

99 

100 

101class PropagateSourceFlagsTask(pipeBase.Task): 

102 """Task to propagate source flags to coadd objects. 

103 

104 Flagged sources may come from a mix of two different types of source catalogs. 

105 The source_table catalogs from ``CalibrateTask`` contain flags for the first 

106 round of astromety/photometry/psf fits. 

107 The finalized_source_table catalogs from ``FinalizeCalibrationTask`` contain 

108 flags from the second round of psf fitting. 

109 """ 

110 ConfigClass = PropagateSourceFlagsConfig 

111 

112 def __init__(self, schema, **kwargs): 

113 pipeBase.Task.__init__(self, **kwargs) 

114 

115 self.schema = schema 

116 for f in self.config.source_flags: 

117 self.schema.addField(f, type="Flag", doc="Propagated from sources") 

118 for f in self.config.finalized_source_flags: 

119 self.schema.addField(f, type="Flag", doc="Propagated from finalized sources") 

120 

121 def run(self, coadd_object_cat, ccd_inputs, 

122 source_table_handle_dict=None, finalized_source_table_handle_dict=None): 

123 """Propagate flags from single-frame sources to coadd objects. 

124 

125 Flags are only propagated if a configurable percentage of the sources 

126 are matched to the coadd objects. This task will match both "plain" 

127 source flags and "finalized" source flags. 

128 

129 Parameters 

130 ---------- 

131 coadd_object_cat : `lsst.afw.table.SourceCatalog` 

132 Table of coadd objects. 

133 ccd_inputs : `lsst.afw.table.ExposureCatalog` 

134 Table of single-frame inputs to coadd. 

135 source_table_handle_dict : `dict` [`int`: `lsst.daf.butler.DeferredDatasetHandle`] 

136 Dict for sourceTable_visit handles (key is visit). May be None if 

137 ``config.source_flags`` has no entries. 

138 finalized_source_table_handle_dict : `dict` [`int`: 

139 `lsst.daf.butler.DeferredDatasetHandle`] 

140 Dict for finalized_src_table handles (key is visit). May be None if 

141 ``config.finalized_source_flags`` has no entries. 

142 """ 

143 if len(self.config.source_flags) == 0 and len(self.config.finalized_source_flags) == 0: 

144 return 

145 

146 source_columns = self._get_source_table_column_names( 

147 self.config.x_column, 

148 self.config.y_column, 

149 self.config.source_flags.keys() 

150 ) 

151 finalized_columns = self._get_source_table_column_names( 

152 self.config.finalized_x_column, 

153 self.config.finalized_y_column, 

154 self.config.finalized_source_flags.keys(), 

155 ) 

156 

157 # We need the number of overlaps of individual detectors for each coadd source. 

158 # The following code is slow and inefficient, but can be made simpler in the future 

159 # case of cell-based coadds and so optimizing usage in afw is not a priority. 

160 num_overlaps = np.zeros(len(coadd_object_cat), dtype=np.int32) 

161 for i, obj in enumerate(coadd_object_cat): 

162 num_overlaps[i] = len(ccd_inputs.subsetContaining(obj.getCoord(), True)) 

163 

164 visits = np.unique(ccd_inputs["visit"]) 

165 

166 matcher = Matcher(np.rad2deg(coadd_object_cat["coord_ra"]), 

167 np.rad2deg(coadd_object_cat["coord_dec"])) 

168 

169 source_flag_counts = {f: np.zeros(len(coadd_object_cat), dtype=np.int32) 

170 for f in self.config.source_flags} 

171 finalized_source_flag_counts = {f: np.zeros(len(coadd_object_cat), dtype=np.int32) 

172 for f in self.config.finalized_source_flags} 

173 

174 handles_list = [source_table_handle_dict, finalized_source_table_handle_dict] 

175 columns_list = [source_columns, finalized_columns] 

176 counts_list = [source_flag_counts, finalized_source_flag_counts] 

177 x_column_list = [self.config.x_column, self.config.finalized_x_column] 

178 y_column_list = [self.config.y_column, self.config.finalized_y_column] 

179 

180 for handle_dict, columns, flag_counts, x_col, y_col in zip(handles_list, 

181 columns_list, 

182 counts_list, 

183 x_column_list, 

184 y_column_list): 

185 if handle_dict is not None and len(columns) > 0: 

186 for visit in visits: 

187 handle = handle_dict[visit] 

188 df = handle.get(parameters={"columns": columns}) 

189 

190 # Loop over all ccd_inputs rows for this visit. 

191 for row in ccd_inputs[ccd_inputs["visit"] == visit]: 

192 detector = row["ccd"] 

193 wcs = row.getWcs() 

194 

195 df_det = df[df["detector"] == detector] 

196 

197 if len(df_det) == 0: 

198 continue 

199 

200 ra, dec = wcs.pixelToSkyArray(df_det[x_col].values, 

201 df_det[y_col].values, 

202 degrees=True) 

203 

204 try: 

205 # The output from the matcher links 

206 # coadd_object_cat[i1] <-> df_det[i2] 

207 # All objects within the match radius are matched. 

208 idx, i1, i2, d = matcher.query_radius( 

209 ra, 

210 dec, 

211 self.config.match_radius/3600., 

212 return_indices=True 

213 ) 

214 except IndexError: 

215 # No matches. Workaround a bug in smatch. 

216 continue 

217 

218 for flag in flag_counts: 

219 flag_values = df_det[flag].values 

220 flag_counts[flag][i1] += flag_values[i2].astype(np.int32) 

221 

222 for flag in source_flag_counts: 

223 thresh = num_overlaps*self.config.source_flags[flag] 

224 object_flag = (source_flag_counts[flag] > thresh) 

225 coadd_object_cat[flag] = object_flag 

226 self.log.info("Propagated %d sources with flag %s", object_flag.sum(), flag) 

227 

228 for flag in finalized_source_flag_counts: 

229 thresh = num_overlaps*self.config.finalized_source_flags[flag] 

230 object_flag = (finalized_source_flag_counts[flag] > thresh) 

231 coadd_object_cat[flag] = object_flag 

232 self.log.info("Propagated %d finalized sources with flag %s", object_flag.sum(), flag) 

233 

234 def _get_source_table_column_names(self, x_column, y_column, flags): 

235 """Get the list of source table columns from the config. 

236 

237 Parameters 

238 ---------- 

239 x_column : `str` 

240 Name of column with x centroid. 

241 y_column : `str` 

242 Name of column with y centroid. 

243 flags : `list` [`str`] 

244 List of flags to retrieve. 

245 

246 Returns 

247 ------- 

248 columns : [`list`] [`str`] 

249 Columns to read. 

250 """ 

251 columns = ["visit", "detector", 

252 x_column, y_column] 

253 columns.extend(flags) 

254 

255 return columns