Coverage for python/lsst/pipe/tasks/propagateSourceFlags.py: 19%

86 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-08-26 03:17 -0700

1# LSST Data Management System 

2# Copyright 2022 LSST 

3# 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# 

7# This program is free software: you can redistribute it and/or modify 

8# it under the terms of the GNU General Public License as published by 

9# the Free Software Foundation, either version 3 of the License, or 

10# (at your option) any later version. 

11# 

12# This program is distributed in the hope that it will be useful, 

13# but WITHOUT ANY WARRANTY; without even the implied warranty of 

14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

15# GNU General Public License for more details. 

16# 

17# You should have received a copy of the LSST License Statement and 

18# the GNU General Public License along with this program. If not, 

19# see <http://www.lsstcorp.org/LegalNotices/>. 

20# 

21__all__ = ["PropagateSourceFlagsConfig", "PropagateSourceFlagsTask"] 

22 

23import numpy as np 

24 

25from smatch.matcher import Matcher 

26 

27import lsst.pex.config as pexConfig 

28import lsst.pipe.base as pipeBase 

29 

30 

31class PropagateSourceFlagsConfig(pexConfig.Config): 

32 """Configuration for propagating source flags to coadd objects.""" 

33 source_flags = pexConfig.DictField( 

34 keytype=str, 

35 itemtype=float, 

36 default={ 

37 "calib_astrometry_used": 0.2, 

38 "calib_photometry_used": 0.2, 

39 "calib_photometry_reserved": 0.2 

40 }, 

41 doc=("Source flags to propagate, with the threshold of relative occurrence " 

42 "(valid range: [0-1]). Coadd object will have flag set if fraction " 

43 "of input visits in which it is flagged is greater than the threshold."), 

44 ) 

45 finalized_source_flags = pexConfig.DictField( 

46 keytype=str, 

47 itemtype=float, 

48 default={ 

49 "calib_psf_candidate": 0.2, 

50 "calib_psf_used": 0.2, 

51 "calib_psf_reserved": 0.2 

52 }, 

53 doc=("Finalized source flags to propagate, with the threshold of relative " 

54 "occurrence (valid range: [0-1]). Coadd object will have flag set if " 

55 "fraction of input visits in which it is flagged is greater than the " 

56 "threshold."), 

57 ) 

58 x_column = pexConfig.Field( 

59 doc="Name of column with source x position (sourceTable_visit).", 

60 dtype=str, 

61 default="x", 

62 ) 

63 y_column = pexConfig.Field( 

64 doc="Name of column with source y position (sourceTable_visit).", 

65 dtype=str, 

66 default="y", 

67 ) 

68 finalized_x_column = pexConfig.Field( 

69 doc="Name of column with source x position (finalized_src_table).", 

70 dtype=str, 

71 default="slot_Centroid_x", 

72 ) 

73 finalized_y_column = pexConfig.Field( 

74 doc="Name of column with source y position (finalized_src_table).", 

75 dtype=str, 

76 default="slot_Centroid_y", 

77 ) 

78 match_radius = pexConfig.Field( 

79 dtype=float, 

80 default=0.2, 

81 doc="Source matching radius (arcsec)" 

82 ) 

83 

84 def validate(self): 

85 super().validate() 

86 

87 if set(self.source_flags).intersection(set(self.finalized_source_flags)): 

88 source_flags = self.source_flags.keys() 

89 finalized_source_flags = self.finalized_source_flags.keys() 

90 raise ValueError(f"The set of source_flags {source_flags} must not overlap " 

91 f"with the finalized_source_flags {finalized_source_flags}") 

92 

93 

94class PropagateSourceFlagsTask(pipeBase.Task): 

95 """Task to propagate source flags to coadd objects. 

96 

97 Flagged sources may come from a mix of two different types of source catalogs. 

98 The source_table catalogs from ``CalibrateTask`` contain flags for the first 

99 round of astromety/photometry/psf fits. 

100 The finalized_source_table catalogs from ``FinalizeCalibrationTask`` contain 

101 flags from the second round of psf fitting. 

102 """ 

103 ConfigClass = PropagateSourceFlagsConfig 

104 

105 def __init__(self, schema, **kwargs): 

106 pipeBase.Task.__init__(self, **kwargs) 

107 

108 self.schema = schema 

109 for f in self.config.source_flags: 

110 self.schema.addField(f, type="Flag", doc="Propagated from sources") 

111 for f in self.config.finalized_source_flags: 

112 self.schema.addField(f, type="Flag", doc="Propagated from finalized sources") 

113 

114 def run(self, coadd_object_cat, ccd_inputs, 

115 source_table_handle_dict=None, finalized_source_table_handle_dict=None): 

116 """Propagate flags from single-frame sources to coadd objects. 

117 

118 Flags are only propagated if a configurable percentage of the sources 

119 are matched to the coadd objects. This task will match both "plain" 

120 source flags and "finalized" source flags. 

121 

122 Parameters 

123 ---------- 

124 coadd_object_cat : `lsst.afw.table.SourceCatalog` 

125 Table of coadd objects. 

126 ccd_inputs : `lsst.afw.table.ExposureCatalog` 

127 Table of single-frame inputs to coadd. 

128 source_table_handle_dict : `dict` [`int`: `lsst.daf.butler.DeferredDatasetHandle`] 

129 Dict for sourceTable_visit handles (key is visit). May be None if 

130 ``config.source_flags`` has no entries. 

131 finalized_source_table_handle_dict : `dict` [`int`: 

132 `lsst.daf.butler.DeferredDatasetHandle`] 

133 Dict for finalized_src_table handles (key is visit). May be None if 

134 ``config.finalized_source_flags`` has no entries. 

135 """ 

136 if len(self.config.source_flags) == 0 and len(self.config.finalized_source_flags) == 0: 

137 return 

138 

139 source_columns = self._get_source_table_column_names( 

140 self.config.x_column, 

141 self.config.y_column, 

142 self.config.source_flags.keys() 

143 ) 

144 finalized_columns = self._get_source_table_column_names( 

145 self.config.finalized_x_column, 

146 self.config.finalized_y_column, 

147 self.config.finalized_source_flags.keys(), 

148 ) 

149 

150 # We need the number of overlaps of individual detectors for each coadd source. 

151 # The following code is slow and inefficient, but can be made simpler in the future 

152 # case of cell-based coadds and so optimizing usage in afw is not a priority. 

153 num_overlaps = np.zeros(len(coadd_object_cat), dtype=np.int32) 

154 for i, obj in enumerate(coadd_object_cat): 

155 num_overlaps[i] = len(ccd_inputs.subsetContaining(obj.getCoord(), True)) 

156 

157 visits = np.unique(ccd_inputs["visit"]) 

158 

159 matcher = Matcher(np.rad2deg(coadd_object_cat["coord_ra"]), 

160 np.rad2deg(coadd_object_cat["coord_dec"])) 

161 

162 source_flag_counts = {f: np.zeros(len(coadd_object_cat), dtype=np.int32) 

163 for f in self.config.source_flags} 

164 finalized_source_flag_counts = {f: np.zeros(len(coadd_object_cat), dtype=np.int32) 

165 for f in self.config.finalized_source_flags} 

166 

167 handles_list = [source_table_handle_dict, finalized_source_table_handle_dict] 

168 columns_list = [source_columns, finalized_columns] 

169 counts_list = [source_flag_counts, finalized_source_flag_counts] 

170 x_column_list = [self.config.x_column, self.config.finalized_x_column] 

171 y_column_list = [self.config.y_column, self.config.finalized_y_column] 

172 name_list = ["sources", "finalized_sources"] 

173 

174 for handle_dict, columns, flag_counts, x_col, y_col, name in zip(handles_list, 

175 columns_list, 

176 counts_list, 

177 x_column_list, 

178 y_column_list, 

179 name_list): 

180 if handle_dict is not None and len(columns) > 0: 

181 for visit in visits: 

182 if visit not in handle_dict: 

183 self.log.info("Visit %d not in input handle dict for %s", visit, name) 

184 continue 

185 handle = handle_dict[visit] 

186 df = handle.get(parameters={"columns": columns}) 

187 

188 # Loop over all ccd_inputs rows for this visit. 

189 for row in ccd_inputs[ccd_inputs["visit"] == visit]: 

190 detector = row["ccd"] 

191 wcs = row.getWcs() 

192 

193 df_det = df[df["detector"] == detector] 

194 

195 if len(df_det) == 0: 

196 continue 

197 

198 ra, dec = wcs.pixelToSkyArray(df_det[x_col].values, 

199 df_det[y_col].values, 

200 degrees=True) 

201 

202 try: 

203 # The output from the matcher links 

204 # coadd_object_cat[i1] <-> df_det[i2] 

205 # All objects within the match radius are matched. 

206 idx, i1, i2, d = matcher.query_radius( 

207 ra, 

208 dec, 

209 self.config.match_radius/3600., 

210 return_indices=True 

211 ) 

212 except IndexError: 

213 # No matches. Workaround a bug in older version of smatch. 

214 self.log.info("Visit %d has no overlapping objects", visit) 

215 continue 

216 

217 if len(i1) == 0: 

218 # No matches (usually because detector does not overlap patch). 

219 self.log.info("Visit %d has no overlapping objects", visit) 

220 continue 

221 

222 for flag in flag_counts: 

223 flag_values = df_det[flag].values 

224 flag_counts[flag][i1] += flag_values[i2].astype(np.int32) 

225 

226 for flag in source_flag_counts: 

227 thresh = num_overlaps*self.config.source_flags[flag] 

228 object_flag = (source_flag_counts[flag] > thresh) 

229 coadd_object_cat[flag] = object_flag 

230 self.log.info("Propagated %d sources with flag %s", object_flag.sum(), flag) 

231 

232 for flag in finalized_source_flag_counts: 

233 thresh = num_overlaps*self.config.finalized_source_flags[flag] 

234 object_flag = (finalized_source_flag_counts[flag] > thresh) 

235 coadd_object_cat[flag] = object_flag 

236 self.log.info("Propagated %d finalized sources with flag %s", object_flag.sum(), flag) 

237 

238 def _get_source_table_column_names(self, x_column, y_column, flags): 

239 """Get the list of source table columns from the config. 

240 

241 Parameters 

242 ---------- 

243 x_column : `str` 

244 Name of column with x centroid. 

245 y_column : `str` 

246 Name of column with y centroid. 

247 flags : `list` [`str`] 

248 List of flags to retrieve. 

249 

250 Returns 

251 ------- 

252 columns : [`list`] [`str`] 

253 Columns to read. 

254 """ 

255 columns = ["visit", "detector", 

256 x_column, y_column] 

257 columns.extend(flags) 

258 

259 return columns