Coverage for python/lsst/analysis/drp/dataSelectors.py: 44%

154 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-06-21 08:11 +0000

1__all__ = ("FlagSelector", "PsfFlagSelector", "BaseSNRSelector", "SnSelector", 

2 "StarIdentifier", "GalaxyIdentifier", "UnknownIdentifier", 

3 "VisitPlotFlagSelector", "CoaddPlotFlagSelector") 

4 

5from abc import abstractmethod 

6from lsst.pex.config import ListField, Field 

7from lsst.pipe.tasks.dataFrameActions import DataFrameAction 

8import numpy as np 

9 

10 

11class FlagSelector(DataFrameAction): 

12 """The base flag selector to use to select valid sources for QA""" 

13 

14 selectWhenFalse = ListField(doc="Names of the flag columns to select on when False", 

15 dtype=str, 

16 optional=False, 

17 default=[]) 

18 

19 selectWhenTrue = ListField(doc="Names of the flag columns to select on when True", 

20 dtype=str, 

21 optional=False, 

22 default=[]) 

23 

24 @property 

25 def columns(self): 

26 allCols = list(self.selectWhenFalse) + list(self.selectWhenTrue) 

27 yield from allCols 

28 

29 def __call__(self, df, **kwargs): 

30 """Select on the given flags 

31 

32 Parameters 

33 ---------- 

34 df : `pandas.core.frame.DataFrame` 

35 

36 Returns 

37 ------- 

38 result : `numpy.ndarray` 

39 A mask of the objects that satisfy the given 

40 flag cuts. 

41 

42 Notes 

43 ----- 

44 Uses the columns in selectWhenFalse and 

45 selectWhenTrue to decide which columns to 

46 select on in each circumstance. 

47 """ 

48 result = np.ones(len(df), dtype=bool) 

49 for flag in self.selectWhenFalse: 

50 result &= (df[flag].values == 0) 

51 for flag in self.selectWhenTrue: 

52 result &= (df[flag].values == 1) 

53 return result 

54 

55 

56class PsfFlagSelector(FlagSelector): 

57 """Remove sources with bad flags set for PSF measurements """ 

58 

59 bands = ListField(doc="The bands to apply the flags in", 

60 dtype=str, 

61 default=["g", "r", "i", "z", "y"]) 

62 

63 @property 

64 def columns(self): 

65 filterColumns = ["xy_flag", "detect_isPatchInner", "detect_isDebelendedSource"] 

66 flagCols = ["psfFlux_flag", "psfFlux_flag_apCorr", "psfFlux_flag_edge", 

67 "psfFlux_flag_noGoodPixels"] 

68 filterColumns += [band + "_" + flag if len(band) > 0 else band + flag 

69 for flag in flagCols for band in self.bands] 

70 yield from filterColumns 

71 

72 def __call__(self, df, **kwargs): 

73 """Make a mask of objects with bad PSF flags 

74 

75 Parameters 

76 ---------- 

77 df : `pandas.core.frame.DataFrame` 

78 

79 Returns 

80 ------- 

81 result : `numpy.ndarray` 

82 A mask of the objects that satisfy the given 

83 flag cuts. 

84 

85 Notes 

86 ----- 

87 Uses the PSF flags and some general quality 

88 control flags to make a mask of the data that 

89 satisfies these criteria. 

90 """ 

91 

92 result = None 

93 flagCols = ["psfFlux_flag", "pixelFlags_saturatedCenter", "extendedness_flag"] 

94 filterColumns = ["xy_flag"] 

95 filterColumns += [band + "_" + flag if len(band) > 0 else band + flag 

96 for flag in flagCols for band in self.bands] 

97 for flag in filterColumns: 

98 selected = (df[flag].values == 0) 

99 if result is None: 

100 result = selected 

101 else: 

102 result &= selected 

103 result &= (df["detect_isPatchInner"] == 1) 

104 result &= (df["detect_isDeblendedSource"] == 1) 

105 return result 

106 

107 

108class BaseSNRSelector(DataFrameAction): 

109 """Selects sources that have a S/N greater than the 

110 given threshold""" 

111 

112 fluxField = Field(doc="Flux field to use in SNR calculation", dtype=str, 

113 default="psfFlux", optional=False) 

114 errField = Field(doc="Flux err field to use in SNR calculation", dtype=str, 

115 default="psfFluxErr", optional=False) 

116 threshold = Field(doc="The signal to noise threshold to select sources", 

117 dtype=float, 

118 optional=False) 

119 band = Field(doc="The band to make the selection in.", 

120 default="i", 

121 dtype=str) 

122 

123 @property 

124 def columns(self): 

125 if len(self.band) > 0: 

126 band = self.band + "_" 

127 else: 

128 band = self.band 

129 return (band + self.fluxField, band + self.errField) 

130 

131 

132class SnSelector(DataFrameAction): 

133 """Selects points that have S/N > threshold in the given flux type""" 

134 fluxType = Field(doc="Flux type to calculate the S/N in.", 

135 dtype=str, 

136 default="psfFlux") 

137 threshold = Field(doc="The S/N threshold to remove sources with.", 

138 dtype=float, 

139 default=500.0) 

140 bands = ListField(doc="The bands to apply the signal to noise cut in.", 

141 dtype=str, 

142 default=["i"]) 

143 

144 @property 

145 def columns(self): 

146 cols = [] 

147 for band in self.bands: 

148 if len(band) > 0: 

149 band = band + "_" 

150 cols += [band + self.fluxType, band + self.fluxType + "Err"] 

151 

152 return cols 

153 

154 def __call__(self, df): 

155 """Makes a mask of objects that have S/N greater than 

156 self.threshold in self.fluxType 

157 

158 Parameters 

159 ---------- 

160 df : `pandas.core.frame.DataFrame` 

161 

162 Returns 

163 ------- 

164 result : `numpy.ndarray` 

165 A mask of the objects that satisfy the given 

166 S/N cut. 

167 """ 

168 

169 mask = np.ones(len(df), dtype=bool) 

170 for band in self.bands: 

171 if len(band) > 0: 

172 band = band + "_" 

173 mask &= ((df[band + self.fluxType] / df[band + self.fluxType + "Err"]) > self.threshold) 

174 

175 return mask 

176 

177 

178def format_extendedness(prefix): 

179 return "refExtendedness" if prefix == "ref" else ( 

180 f"{prefix}{'_' if len(prefix) > 0 else ''}extendedness") 

181 

182 

183class ExtendedIdentifier(DataFrameAction): 

184 band = Field(doc="The band the object is to be classified as a star in.", 

185 default="i", 

186 dtype=str) 

187 

188 @property 

189 def columns(self): 

190 return [self.extendedness] 

191 

192 @property 

193 def extendedness(self): 

194 return format_extendedness(self.band) 

195 

196 # TODO: Add classmethod in py3.9 

197 @property 

198 @abstractmethod 

199 def sourceType(self): 

200 raise NotImplementedError("This method should be overloaded in subclasses") 

201 

202 @abstractmethod 

203 def identified(self, df): 

204 raise NotImplementedError("This method should be overloaded in subclasses") 

205 

206 def __call__(self, df, **kwargs): 

207 """Identifies sources classified as stars 

208 

209 Parameters 

210 ---------- 

211 df : `pandas.core.frame.DataFrame` 

212 

213 Returns 

214 ------- 

215 result : `numpy.ndarray` 

216 An array with the objects that are classified as 

217 stars marked with a 1. 

218 """ 

219 sourceType = np.zeros(len(df)) 

220 sourceType[self.identified(df)] = self.sourceType 

221 return sourceType 

222 

223 

224class StarIdentifier(ExtendedIdentifier): 

225 """Identifies stars from the dataFrame and marks them as a 1 

226 in the added sourceType column""" 

227 extendedness_maximum = Field(doc="Maximum extendedness to qualify as unresolved, inclusive.", 

228 default=0.5, 

229 dtype=float) 

230 

231 @property 

232 @abstractmethod 

233 def sourceType(self): 

234 return 1 

235 

236 @abstractmethod 

237 def identified(self, df): 

238 extendedness = df[self.extendedness] 

239 return (extendedness >= 0) & (extendedness < self.extendedness_maximum) 

240 

241 

242class GalaxyIdentifier(ExtendedIdentifier): 

243 """Identifies galaxies from the dataFrame and marks them as a 2 

244 in the added sourceType column""" 

245 extendedness_minimum = Field(doc="Minimum extendedness to qualify as resolved, not inclusive.", 

246 default=0.5, 

247 dtype=float) 

248 

249 @property 

250 @abstractmethod 

251 def sourceType(self): 

252 return 2 

253 

254 @abstractmethod 

255 def identified(self, df): 

256 extendedness = df[self.extendedness] 

257 return (extendedness > self.extendedness_minimum) & (extendedness <= 1) 

258 

259 

260class UnknownIdentifier(ExtendedIdentifier): 

261 """Identifies un classified objects from the dataFrame and marks them as a 

262 9 in the added sourceType column""" 

263 

264 @property 

265 @abstractmethod 

266 def sourceType(self): 

267 return 9 

268 

269 @abstractmethod 

270 def identified(self, df): 

271 return df[self.extendedness] == 9 

272 

273 

274class VisitPlotFlagSelector(DataFrameAction): 

275 

276 @property 

277 def columns(self): 

278 flagCols = ["psfFlux_flag", "pixelFlags_saturatedCenter", "extendedness_flag", "centroid_flag"] 

279 yield from flagCols 

280 

281 def __call__(self, df, **kwargs): 

282 """The flags to use for selecting sources for visit QA 

283 

284 Parameters 

285 ---------- 

286 df : `pandas.core.frame.DataFrame` 

287 

288 Returns 

289 ------- 

290 result : `numpy.ndarray` 

291 A mask of the objects that satisfy the given 

292 flag cuts. 

293 

294 Notes 

295 ----- 

296 These flags are taken from pipe_analysis and are considered to 

297 be the standard flags for general QA plots. Some of the plots 

298 will require a different set of flags, or additional ones on 

299 top of the ones specified here. These should be specifed in 

300 an additional selector rather than adding to this one. 

301 """ 

302 

303 result = None 

304 flagCols = ["psfFlux_flag", "pixelFlags_saturatedCenter", "extendedness_flag", "centroid_flag"] 

305 for flag in flagCols: 

306 selected = (df[flag].values == 0) 

307 if result is None: 

308 result = selected 

309 else: 

310 result &= selected 

311 return result 

312 

313 

314class CoaddPlotFlagSelector(DataFrameAction): 

315 """The flags to use for selecting sources for coadd QA 

316 

317 Parameters 

318 ---------- 

319 df : `pandas.core.frame.DataFrame` 

320 

321 Returns 

322 ------- 

323 result : `numpy.ndarray` 

324 A mask of the objects that satisfy the given 

325 flag cuts. 

326 

327 Notes 

328 ----- 

329 These flags are taken from pipe_analysis and are considered to 

330 be the standard flags for general QA plots. Some of the plots 

331 will require a different set of flags, or additional ones on 

332 top of the ones specified here. These should be specifed in 

333 an additional selector rather than adding to this one. 

334 """ 

335 

336 bands = ListField(doc="The bands to apply the flags in", 

337 dtype=str, 

338 default=["g", "r", "i", "z", "y"]) 

339 

340 @property 

341 def columns(self): 

342 flagCols = ["psfFlux_flag", "pixelFlags_saturatedCenter", "extendedness_flag"] 

343 filterColumns = ["xy_flag", "detect_isPatchInner", "detect_isDeblendedSource"] 

344 filterColumns += [band + "_" + flag if len(band) > 0 else band + flag 

345 for flag in flagCols for band in self.bands] 

346 yield from filterColumns 

347 

348 def __call__(self, df, **kwargs): 

349 """The flags to use for selecting sources for coadd QA 

350 

351 Parameters 

352 ---------- 

353 df : `pandas.core.frame.DataFrame` 

354 

355 Returns 

356 ------- 

357 result : `numpy.ndarray` 

358 A mask of the objects that satisfy the given 

359 flag cuts. 

360 

361 Notes 

362 ----- 

363 These flags are taken from pipe_analysis and are considered to 

364 be the standard flags for general QA plots. Some of the plots 

365 will require a different set of flags, or additional ones on 

366 top of the ones specified here. These should be specifed in 

367 an additional selector rather than adding to this one. 

368 """ 

369 

370 result = None 

371 flagCols = ["psfFlux_flag", "pixelFlags_saturatedCenter", "extendedness_flag"] 

372 filterColumns = ["xy_flag"] 

373 filterColumns += [band + "_" + flag if len(band) > 0 else band + flag 

374 for flag in flagCols for band in self.bands] 

375 for flag in filterColumns: 

376 selected = (df[flag].values == 0) 

377 if result is None: 

378 result = selected 

379 else: 

380 result &= selected 

381 result &= (df["detect_isPatchInner"] == 1) 

382 result &= (df["detect_isDeblendedSource"] == 1) 

383 return result