Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of faro. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22from lsst.afw.table import ( 

23 SchemaMapper, 

24 Field, 

25 MultiMatch, 

26 SimpleRecord, 

27 SourceCatalog, 

28 updateSourceCoords, 

29) 

30 

31import numpy as np 

32from astropy.table import join, Table 

33 

34__all__ = ( 

35 "matchCatalogs", 

36 "ellipticityFromCat", 

37 "ellipticity", 

38 "makeMatchedPhotom", 

39 "mergeCatalogs", 

40) 

41 

42 

43def matchCatalogs(inputs, photoCalibs, astromCalibs, dataIds, matchRadius, logger=None): 

44 schema = inputs[0].schema 

45 mapper = SchemaMapper(schema) 

46 mapper.addMinimalSchema(schema) 

47 mapper.addOutputField(Field[float]("base_PsfFlux_snr", "PSF flux SNR")) 

48 mapper.addOutputField(Field[float]("base_PsfFlux_mag", "PSF magnitude")) 

49 mapper.addOutputField( 

50 Field[float]("base_PsfFlux_magErr", "PSF magnitude uncertainty") 

51 ) 

52 # Needed because addOutputField(... 'slot_ModelFlux_mag') will add a field with that literal name 

53 aliasMap = schema.getAliasMap() 

54 # Possibly not needed since base_GaussianFlux is the default, but this ought to be safe 

55 modelName = ( 

56 aliasMap["slot_ModelFlux"] 

57 if "slot_ModelFlux" in aliasMap.keys() 

58 else "base_GaussianFlux" 

59 ) 

60 mapper.addOutputField(Field[float](f"{modelName}_mag", "Model magnitude")) 

61 mapper.addOutputField( 

62 Field[float](f"{modelName}_magErr", "Model magnitude uncertainty") 

63 ) 

64 mapper.addOutputField(Field[float](f"{modelName}_snr", "Model flux snr")) 

65 mapper.addOutputField(Field[float]("e1", "Source Ellipticity 1")) 

66 mapper.addOutputField(Field[float]("e2", "Source Ellipticity 1")) 

67 mapper.addOutputField(Field[float]("psf_e1", "PSF Ellipticity 1")) 

68 mapper.addOutputField(Field[float]("psf_e2", "PSF Ellipticity 1")) 

69 mapper.addOutputField(Field[np.int32]("filt", "filter code")) 

70 newSchema = mapper.getOutputSchema() 

71 newSchema.setAliasMap(schema.getAliasMap()) 

72 

73 # Create an object that matches multiple catalogs with same schema 

74 mmatch = MultiMatch( 

75 newSchema, 

76 dataIdFormat={"visit": np.int32, "detector": np.int32}, 

77 radius=matchRadius, 

78 RecordClass=SimpleRecord, 

79 ) 

80 

81 # create the new extended source catalog 

82 srcVis = SourceCatalog(newSchema) 

83 

84 filter_dict = { 

85 "u": 1, 

86 "g": 2, 

87 "r": 3, 

88 "i": 4, 

89 "z": 5, 

90 "y": 6, 

91 "HSC-U": 1, 

92 "HSC-G": 2, 

93 "HSC-R": 3, 

94 "HSC-I": 4, 

95 "HSC-Z": 5, 

96 "HSC-Y": 6, 

97 } 

98 

99 # Sort by visit, detector, then filter 

100 vislist = [v["visit"] for v in dataIds] 

101 ccdlist = [v["detector"] for v in dataIds] 

102 filtlist = [v["band"] for v in dataIds] 

103 tab_vids = Table([vislist, ccdlist, filtlist], names=["vis", "ccd", "filt"]) 

104 sortinds = np.argsort(tab_vids, order=("vis", "ccd", "filt")) 

105 

106 for ind in sortinds: 

107 oldSrc = inputs[ind] 

108 photoCalib = photoCalibs[ind] 

109 wcs = astromCalibs[ind] 

110 dataId = dataIds[ind] 

111 

112 if logger: 

113 logger.debug( 

114 "%d sources in ccd %s visit %s", 

115 len(oldSrc), 

116 dataId["detector"], 

117 dataId["visit"], 

118 ) 

119 

120 # create temporary catalog 

121 tmpCat = SourceCatalog(SourceCatalog(newSchema).table) 

122 tmpCat.extend(oldSrc, mapper=mapper) 

123 

124 filtnum = filter_dict[dataId["band"]] 

125 tmpCat["filt"] = np.repeat(filtnum, len(oldSrc)) 

126 

127 tmpCat["base_PsfFlux_snr"][:] = ( 

128 tmpCat["base_PsfFlux_instFlux"] / tmpCat["base_PsfFlux_instFluxErr"] 

129 ) 

130 

131 updateSourceCoords(wcs, tmpCat) 

132 

133 photoCalib.instFluxToMagnitude(tmpCat, "base_PsfFlux", "base_PsfFlux") 

134 tmpCat["slot_ModelFlux_snr"][:] = ( 

135 tmpCat["slot_ModelFlux_instFlux"] / tmpCat["slot_ModelFlux_instFluxErr"] 

136 ) 

137 photoCalib.instFluxToMagnitude(tmpCat, "slot_ModelFlux", "slot_ModelFlux") 

138 

139 _, psf_e1, psf_e2 = ellipticityFromCat(oldSrc, slot_shape="slot_PsfShape") 

140 _, star_e1, star_e2 = ellipticityFromCat(oldSrc, slot_shape="slot_Shape") 

141 tmpCat["e1"][:] = star_e1 

142 tmpCat["e2"][:] = star_e2 

143 tmpCat["psf_e1"][:] = psf_e1 

144 tmpCat["psf_e2"][:] = psf_e2 

145 

146 srcVis.extend(tmpCat, False) 

147 mmatch.add(catalog=tmpCat, dataId=dataId) 

148 

149 # Complete the match, returning a catalog that includes 

150 # all matched sources with object IDs that can be used to group them. 

151 matchCat = mmatch.finish() 

152 

153 # Create a mapping object that allows the matches to be manipulated 

154 # as a mapping of object ID to catalog of sources. 

155 

156 # I don't think I can persist a group view, so this may need to be called in a subsequent task 

157 # allMatches = GroupView.build(matchCat) 

158 

159 return srcVis, matchCat 

160 

161 

162def ellipticityFromCat(cat, slot_shape="slot_Shape"): 

163 """Calculate the ellipticity of the Shapes in a catalog from the 2nd moments. 

164 Parameters 

165 ---------- 

166 cat : `lsst.afw.table.BaseCatalog` 

167 A catalog with 'slot_Shape' defined and '_xx', '_xy', '_yy' 

168 entries for the target of 'slot_Shape'. 

169 E.g., 'slot_shape' defined as 'base_SdssShape' 

170 And 'base_SdssShape_xx', 'base_SdssShape_xy', 'base_SdssShape_yy' defined. 

171 slot_shape : str, optional 

172 Specify what slot shape requested. Intended use is to get the PSF shape 

173 estimates by specifying 'slot_shape=slot_PsfShape' 

174 instead of the default 'slot_shape=slot_Shape'. 

175 Returns 

176 ------- 

177 e, e1, e2 : complex, float, float 

178 Complex ellipticity, real part, imaginary part 

179 """ 

180 i_xx, i_xy, i_yy = ( 

181 cat.get(slot_shape + "_xx"), 

182 cat.get(slot_shape + "_xy"), 

183 cat.get(slot_shape + "_yy"), 

184 ) 

185 return ellipticity(i_xx, i_xy, i_yy) 

186 

187 

188def ellipticity(i_xx, i_xy, i_yy): 

189 """Calculate ellipticity from second moments. 

190 Parameters 

191 ---------- 

192 i_xx : float or `numpy.array` 

193 i_xy : float or `numpy.array` 

194 i_yy : float or `numpy.array` 

195 Returns 

196 ------- 

197 e, e1, e2 : (float, float, float) or (numpy.array, numpy.array, numpy.array) 

198 Complex ellipticity, real component, imaginary component 

199 """ 

200 e = (i_xx - i_yy + 2j * i_xy) / (i_xx + i_yy) 

201 e1 = np.real(e) 

202 e2 = np.imag(e) 

203 return e, e1, e2 

204 

205 

206def makeMatchedPhotom(dataIds, catalogs, photoCalibs): 

207 # inputs: dataIds, catalogs, photoCalibs 

208 

209 # Match all input bands: 

210 bands = list(set([f["band"] for f in dataIds])) 

211 

212 # Should probably add an "assert" that requires bands>1... 

213 

214 empty_cat = catalogs[0].copy() 

215 empty_cat.clear() 

216 

217 cat_dict = {} 

218 mags_dict = {} 

219 magerrs_dict = {} 

220 for band in bands: 

221 cat_dict[band] = empty_cat.copy() 

222 mags_dict[band] = [] 

223 magerrs_dict[band] = [] 

224 

225 for i in range(len(catalogs)): 

226 for band in bands: 

227 if dataIds[i]["band"] in band: 

228 cat_dict[band].extend(catalogs[i].copy(deep=True)) 

229 mags = photoCalibs[i].instFluxToMagnitude(catalogs[i], "base_PsfFlux") 

230 mags_dict[band] = np.append(mags_dict[band], mags[:, 0]) 

231 magerrs_dict[band] = np.append(magerrs_dict[band], mags[:, 1]) 

232 

233 for band in bands: 

234 cat_tmp = cat_dict[band] 

235 if cat_tmp: 

236 if not cat_tmp.isContiguous(): 

237 cat_tmp = cat_tmp.copy(deep=True) 

238 cat_tmp_final = cat_tmp.asAstropy() 

239 cat_tmp_final["base_PsfFlux_mag"] = mags_dict[band] 

240 cat_tmp_final["base_PsfFlux_magErr"] = magerrs_dict[band] 

241 # Put the bandpass name in the column names: 

242 for c in cat_tmp_final.colnames: 

243 if c not in "id": 

244 cat_tmp_final[c].name = c + "_" + str(band) 

245 # Write the new catalog to the dict of catalogs: 

246 cat_dict[band] = cat_tmp_final 

247 

248 cat_combined = join(cat_dict[bands[1]], cat_dict[bands[0]], keys="id") 

249 if len(bands) > 2: 

250 for i in range(2, len(bands)): 

251 cat_combined = join(cat_combined, cat_dict[bands[i]], keys="id") 

252 

253 qual_cuts = ( 

254 (cat_combined["base_ClassificationExtendedness_value_g"] < 0.5) 

255 & (cat_combined["base_PixelFlags_flag_saturated_g"] is False) 

256 & (cat_combined["base_PixelFlags_flag_cr_g"] is False) 

257 & (cat_combined["base_PixelFlags_flag_bad_g"] is False) 

258 & (cat_combined["base_PixelFlags_flag_edge_g"] is False) 

259 & (cat_combined["base_ClassificationExtendedness_value_r"] < 0.5) 

260 & (cat_combined["base_PixelFlags_flag_saturated_r"] is False) 

261 & (cat_combined["base_PixelFlags_flag_cr_r"] is False) 

262 & (cat_combined["base_PixelFlags_flag_bad_r"] is False) 

263 & (cat_combined["base_PixelFlags_flag_edge_r"] is False) 

264 & (cat_combined["base_ClassificationExtendedness_value_i"] < 0.5) 

265 & (cat_combined["base_PixelFlags_flag_saturated_i"] is False) 

266 & (cat_combined["base_PixelFlags_flag_cr_i"] is False) 

267 & (cat_combined["base_PixelFlags_flag_bad_i"] is False) 

268 & (cat_combined["base_PixelFlags_flag_edge_i"] is False) 

269 ) # noqa: E712 

270 

271 # Return the astropy table of matched catalogs: 

272 return cat_combined[qual_cuts] 

273 

274 

275def mergeCatalogs( 

276 catalogs, 

277 photoCalibs=None, 

278 astromCalibs=None, 

279 models=["slot_PsfFlux"], 

280 applyExternalWcs=False, 

281): 

282 """Merge catalogs and optionally apply photometric and astrometric calibrations. 

283 """ 

284 

285 schema = catalogs[0].schema 

286 mapper = SchemaMapper(schema) 

287 mapper.addMinimalSchema(schema) 

288 aliasMap = schema.getAliasMap() 

289 for model in models: 

290 modelName = aliasMap[model] if model in aliasMap.keys() else model 

291 mapper.addOutputField( 

292 Field[float](f"{modelName}_mag", f"{modelName} magnitude") 

293 ) 

294 mapper.addOutputField( 

295 Field[float](f"{modelName}_magErr", f"{modelName} magnitude uncertainty") 

296 ) 

297 newSchema = mapper.getOutputSchema() 

298 newSchema.setAliasMap(schema.getAliasMap()) 

299 

300 size = sum([len(cat) for cat in catalogs]) 

301 catalog = SourceCatalog(newSchema) 

302 catalog.reserve(size) 

303 

304 for ii in range(0, len(catalogs)): 

305 cat = catalogs[ii] 

306 

307 # Create temporary catalog. Is this step needed? 

308 tempCat = SourceCatalog(SourceCatalog(newSchema).table) 

309 tempCat.extend(cat, mapper=mapper) 

310 

311 if applyExternalWcs and astromCalibs is not None: 

312 wcs = astromCalibs[ii] 

313 updateSourceCoords(wcs, tempCat) 

314 

315 if photoCalibs is not None: 

316 photoCalib = photoCalibs[ii] 

317 for model in models: 

318 modelName = aliasMap[model] if model in aliasMap.keys() else model 

319 photoCalib.instFluxToMagnitude(tempCat, modelName, modelName) 

320 

321 catalog.extend(tempCat) 

322 

323 return catalog