Coverage for python/lsst/faro/utils/matcher.py: 8%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

120 statements  

1# This file is part of faro. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22from lsst.afw.geom import SkyWcs 

23from lsst.afw.image import PhotoCalib 

24from lsst.afw.table import ( 

25 SchemaMapper, 

26 Field, 

27 MultiMatch, 

28 SimpleRecord, 

29 SourceCatalog, 

30 updateSourceCoords, 

31) 

32from lsst.faro.utils.calibrated_catalog import CalibratedCatalog 

33 

34import numpy as np 

35from astropy.table import join, Table 

36from typing import Dict, List 

37 

38__all__ = ( 

39 "matchCatalogs", 

40 "ellipticityFromCat", 

41 "ellipticity", 

42 "makeMatchedPhotom", 

43 "mergeCatalogs", 

44) 

45 

46 

47def matchCatalogs( 

48 inputs: List[SourceCatalog], 

49 photoCalibs: List[PhotoCalib], 

50 astromCalibs: List[SkyWcs], 

51 dataIds, 

52 matchRadius: float, 

53 logger=None, 

54): 

55 schema = inputs[0].schema 

56 mapper = SchemaMapper(schema) 

57 mapper.addMinimalSchema(schema) 

58 mapper.addOutputField(Field[float]("base_PsfFlux_snr", "PSF flux SNR")) 

59 mapper.addOutputField(Field[float]("base_PsfFlux_mag", "PSF magnitude")) 

60 mapper.addOutputField( 

61 Field[float]("base_PsfFlux_magErr", "PSF magnitude uncertainty") 

62 ) 

63 # Needed because addOutputField(... 'slot_ModelFlux_mag') will add a field with that literal name 

64 aliasMap = schema.getAliasMap() 

65 # Possibly not needed since base_GaussianFlux is the default, but this ought to be safe 

66 modelName = ( 

67 aliasMap["slot_ModelFlux"] 

68 if "slot_ModelFlux" in aliasMap.keys() 

69 else "base_GaussianFlux" 

70 ) 

71 mapper.addOutputField(Field[float](f"{modelName}_mag", "Model magnitude")) 

72 mapper.addOutputField( 

73 Field[float](f"{modelName}_magErr", "Model magnitude uncertainty") 

74 ) 

75 mapper.addOutputField(Field[float](f"{modelName}_snr", "Model flux snr")) 

76 mapper.addOutputField(Field[float]("e1", "Source Ellipticity 1")) 

77 mapper.addOutputField(Field[float]("e2", "Source Ellipticity 1")) 

78 mapper.addOutputField(Field[float]("psf_e1", "PSF Ellipticity 1")) 

79 mapper.addOutputField(Field[float]("psf_e2", "PSF Ellipticity 1")) 

80 mapper.addOutputField(Field[np.int32]("filt", "filter code")) 

81 newSchema = mapper.getOutputSchema() 

82 newSchema.setAliasMap(schema.getAliasMap()) 

83 

84 # Create an object that matches multiple catalogs with same schema 

85 mmatch = MultiMatch( 

86 newSchema, 

87 dataIdFormat={"visit": np.int32, "detector": np.int32}, 

88 radius=matchRadius, 

89 RecordClass=SimpleRecord, 

90 ) 

91 

92 # create the new extended source catalog 

93 srcVis = SourceCatalog(newSchema) 

94 

95 filter_dict = { 

96 "u": 1, 

97 "g": 2, 

98 "r": 3, 

99 "i": 4, 

100 "z": 5, 

101 "y": 6, 

102 "HSC-U": 1, 

103 "HSC-G": 2, 

104 "HSC-R": 3, 

105 "HSC-I": 4, 

106 "HSC-Z": 5, 

107 "HSC-Y": 6, 

108 } 

109 

110 # Sort by visit, detector, then filter 

111 vislist = [v["visit"] for v in dataIds] 

112 ccdlist = [v["detector"] for v in dataIds] 

113 filtlist = [v["band"] for v in dataIds] 

114 tab_vids = Table([vislist, ccdlist, filtlist], names=["vis", "ccd", "filt"]) 

115 sortinds = np.argsort(tab_vids, order=("vis", "ccd", "filt")) 

116 

117 for ind in sortinds: 

118 oldSrc = inputs[ind] 

119 photoCalib = photoCalibs[ind] 

120 wcs = astromCalibs[ind] 

121 dataId = dataIds[ind] 

122 

123 if logger: 

124 logger.debug( 

125 "%d sources in ccd %s visit %s", 

126 len(oldSrc), 

127 dataId["detector"], 

128 dataId["visit"], 

129 ) 

130 

131 # create temporary catalog 

132 tmpCat = SourceCatalog(SourceCatalog(newSchema).table) 

133 tmpCat.extend(oldSrc, mapper=mapper) 

134 

135 filtnum = filter_dict[dataId["band"]] 

136 tmpCat["filt"] = np.repeat(filtnum, len(oldSrc)) 

137 

138 tmpCat["base_PsfFlux_snr"][:] = ( 

139 tmpCat["base_PsfFlux_instFlux"] / tmpCat["base_PsfFlux_instFluxErr"] 

140 ) 

141 

142 updateSourceCoords(wcs, tmpCat) 

143 

144 photoCalib.instFluxToMagnitude(tmpCat, "base_PsfFlux", "base_PsfFlux") 

145 tmpCat["slot_ModelFlux_snr"][:] = ( 

146 tmpCat["slot_ModelFlux_instFlux"] / tmpCat["slot_ModelFlux_instFluxErr"] 

147 ) 

148 photoCalib.instFluxToMagnitude(tmpCat, "slot_ModelFlux", "slot_ModelFlux") 

149 

150 _, psf_e1, psf_e2 = ellipticityFromCat(oldSrc, slot_shape="slot_PsfShape") 

151 _, star_e1, star_e2 = ellipticityFromCat(oldSrc, slot_shape="slot_Shape") 

152 tmpCat["e1"][:] = star_e1 

153 tmpCat["e2"][:] = star_e2 

154 tmpCat["psf_e1"][:] = psf_e1 

155 tmpCat["psf_e2"][:] = psf_e2 

156 

157 srcVis.extend(tmpCat, False) 

158 mmatch.add(catalog=tmpCat, dataId=dataId) 

159 

160 # Complete the match, returning a catalog that includes 

161 # all matched sources with object IDs that can be used to group them. 

162 matchCat = mmatch.finish() 

163 

164 # Create a mapping object that allows the matches to be manipulated 

165 # as a mapping of object ID to catalog of sources. 

166 

167 # I don't think I can persist a group view, so this may need to be called in a subsequent task 

168 # allMatches = GroupView.build(matchCat) 

169 

170 return srcVis, matchCat 

171 

172 

173def ellipticityFromCat(cat, slot_shape="slot_Shape"): 

174 """Calculate the ellipticity of the Shapes in a catalog from the 2nd moments. 

175 Parameters 

176 ---------- 

177 cat : `lsst.afw.table.BaseCatalog` 

178 A catalog with 'slot_Shape' defined and '_xx', '_xy', '_yy' 

179 entries for the target of 'slot_Shape'. 

180 E.g., 'slot_shape' defined as 'base_SdssShape' 

181 And 'base_SdssShape_xx', 'base_SdssShape_xy', 'base_SdssShape_yy' defined. 

182 slot_shape : str, optional 

183 Specify what slot shape requested. Intended use is to get the PSF shape 

184 estimates by specifying 'slot_shape=slot_PsfShape' 

185 instead of the default 'slot_shape=slot_Shape'. 

186 Returns 

187 ------- 

188 e, e1, e2 : complex, float, float 

189 Complex ellipticity, real part, imaginary part 

190 """ 

191 i_xx, i_xy, i_yy = ( 

192 cat.get(slot_shape + "_xx"), 

193 cat.get(slot_shape + "_xy"), 

194 cat.get(slot_shape + "_yy"), 

195 ) 

196 return ellipticity(i_xx, i_xy, i_yy) 

197 

198 

199def ellipticity(i_xx, i_xy, i_yy): 

200 """Calculate ellipticity from second moments. 

201 Parameters 

202 ---------- 

203 i_xx : float or `numpy.array` 

204 i_xy : float or `numpy.array` 

205 i_yy : float or `numpy.array` 

206 Returns 

207 ------- 

208 e, e1, e2 : (float, float, float) or (numpy.array, numpy.array, numpy.array) 

209 Complex ellipticity, real component, imaginary component 

210 """ 

211 e = (i_xx - i_yy + 2j * i_xy) / (i_xx + i_yy) 

212 e1 = np.real(e) 

213 e2 = np.imag(e) 

214 return e, e1, e2 

215 

216 

217def makeMatchedPhotom(data: Dict[str, List[CalibratedCatalog]]): 

218 """ Merge catalogs in multiple bands into a single shared catalog. 

219 """ 

220 

221 cat_all = None 

222 

223 for band, cat_list in data.items(): 

224 cat_tmp = [] 

225 calibs_photo = [] 

226 for cat_calib in cat_list: 

227 cat_tmp_i = cat_calib.catalog 

228 qual_cuts = ( 

229 (cat_tmp_i["base_ClassificationExtendedness_value"] < 0.5) 

230 & ~cat_tmp_i["base_PixelFlags_flag_saturated"] 

231 & ~cat_tmp_i["base_PixelFlags_flag_cr"] 

232 & ~cat_tmp_i["base_PixelFlags_flag_bad"] 

233 & ~cat_tmp_i["base_PixelFlags_flag_edge"] 

234 ) 

235 cat_tmp.append(cat_tmp_i[qual_cuts]) 

236 calibs_photo.append(cat_calib.photoCalib) 

237 

238 cat_tmp = mergeCatalogs(cat_tmp, calibs_photo, models=['base_PsfFlux']) 

239 if cat_tmp: 

240 if not cat_tmp.isContiguous(): 

241 cat_tmp = cat_tmp.copy(deep=True) 

242 

243 cat_tmp = cat_tmp.asAstropy() 

244 

245 # Put the bandpass name in the column names: 

246 for c in cat_tmp.colnames: 

247 if c != "id": 

248 cat_tmp[c].name = f"{c}_{band}" 

249 

250 if cat_all: 

251 cat_all = join(cat_all, cat_tmp, keys="id") 

252 else: 

253 cat_all = cat_tmp 

254 

255 # Return the astropy table of matched catalogs: 

256 return cat_all 

257 

258 

259def mergeCatalogs( 

260 catalogs, 

261 photoCalibs=None, 

262 astromCalibs=None, 

263 models=["slot_PsfFlux"], 

264 applyExternalWcs=False, 

265): 

266 """Merge catalogs and optionally apply photometric and astrometric calibrations. 

267 """ 

268 

269 schema = catalogs[0].schema 

270 mapper = SchemaMapper(schema) 

271 mapper.addMinimalSchema(schema) 

272 aliasMap = schema.getAliasMap() 

273 for model in models: 

274 modelName = aliasMap[model] if model in aliasMap.keys() else model 

275 mapper.addOutputField( 

276 Field[float](f"{modelName}_mag", f"{modelName} magnitude") 

277 ) 

278 mapper.addOutputField( 

279 Field[float](f"{modelName}_magErr", f"{modelName} magnitude uncertainty") 

280 ) 

281 newSchema = mapper.getOutputSchema() 

282 newSchema.setAliasMap(schema.getAliasMap()) 

283 

284 size = sum([len(cat) for cat in catalogs]) 

285 catalog = SourceCatalog(newSchema) 

286 catalog.reserve(size) 

287 

288 for ii in range(0, len(catalogs)): 

289 cat = catalogs[ii] 

290 

291 # Create temporary catalog. Is this step needed? 

292 tempCat = SourceCatalog(SourceCatalog(newSchema).table) 

293 tempCat.extend(cat, mapper=mapper) 

294 

295 if applyExternalWcs and astromCalibs is not None: 

296 wcs = astromCalibs[ii] 

297 updateSourceCoords(wcs, tempCat) 

298 

299 if photoCalibs is not None: 

300 photoCalib = photoCalibs[ii] 

301 if photoCalib is not None: 

302 for model in models: 

303 modelName = aliasMap[model] if model in aliasMap.keys() else model 

304 photoCalib.instFluxToMagnitude(tempCat, modelName, modelName) 

305 

306 catalog.extend(tempCat) 

307 

308 return catalog