Coverage for python/lsst/analysis/tools/tasks/catalogMatch.py: 24%

137 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-15 02:38 -0700

1# This file is part of analysis_tools. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ("CatalogMatchConfig", "CatalogMatchTask") 

23 

24 

25import lsst.geom 

26import lsst.pex.config as pexConfig 

27import lsst.pipe.base as pipeBase 

28import numpy as np 

29from astropy.table import Table, hstack, vstack 

30from astropy.time import Time 

31from lsst.pex.config.configurableActions import ConfigurableActionStructField 

32from lsst.pipe.tasks.loadReferenceCatalog import LoadReferenceCatalogTask 

33from lsst.skymap import BaseSkyMap 

34from smatch import Matcher 

35 

36from ..actions.vector import ( 

37 CoaddPlotFlagSelector, 

38 GalaxySelector, 

39 MatchingFlagSelector, 

40 SnSelector, 

41 StarSelector, 

42) 

43from ..interfaces import VectorAction 

44 

45 

46class CatalogMatchConnections( 

47 pipeBase.PipelineTaskConnections, 

48 dimensions=("tract", "skymap"), 

49 defaultTemplates={"targetCatalog": "objectTable_tract", "refCatalog": "ps1_pv3_3pi_20170110"}, 

50): 

51 catalog = pipeBase.connectionTypes.Input( 

52 doc="The tract-wide catalog to make plots from.", 

53 storageClass="ArrowAstropy", 

54 name="{targetCatalog}", 

55 dimensions=("tract", "skymap"), 

56 deferLoad=True, 

57 ) 

58 

59 refCat = pipeBase.connectionTypes.PrerequisiteInput( 

60 doc="The reference catalog to match to loaded input catalog sources.", 

61 name="{refCatalog}", 

62 storageClass="SimpleCatalog", 

63 dimensions=("skypix",), 

64 deferLoad=True, 

65 multiple=True, 

66 ) 

67 

68 skymap = pipeBase.connectionTypes.Input( 

69 doc="The skymap for the tract", 

70 storageClass="SkyMap", 

71 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

72 dimensions=("skymap",), 

73 ) 

74 

75 matchedCatalog = pipeBase.connectionTypes.Output( 

76 doc="Catalog with matched target and reference objects with separations", 

77 name="{targetCatalog}_{refCatalog}_match", 

78 storageClass="ArrowAstropy", 

79 dimensions=("tract", "skymap"), 

80 ) 

81 

82 

83class CatalogMatchConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CatalogMatchConnections): 

84 referenceCatalogLoader = pexConfig.ConfigurableField( 

85 target=LoadReferenceCatalogTask, 

86 doc="Reference catalog loader", 

87 ) 

88 

89 epoch = pexConfig.Field[float]( 

90 doc="Epoch to which reference objects are shifted.", 

91 default=2015.0, 

92 ) 

93 

94 filterNames = pexConfig.ListField[str]( 

95 doc="Physical filter names to persist downstream.", 

96 default=["u", "g", "r", "i", "z", "y"], 

97 ) 

98 

99 selectorBands = pexConfig.ListField[str]( 

100 doc="Band to use when selecting objects, primarily for extendedness.", 

101 default=["i"], 

102 ) 

103 

104 selectorActions = ConfigurableActionStructField[VectorAction]( 

105 doc="Which selectors to use to narrow down the data for QA plotting.", 

106 default={"flagSelector": MatchingFlagSelector()}, 

107 ) 

108 

109 sourceSelectorActions = ConfigurableActionStructField[VectorAction]( 

110 doc="What types of sources to use.", 

111 default={}, 

112 ) 

113 

114 extraColumnSelectors = ConfigurableActionStructField[VectorAction]( 

115 doc="Other selectors that are not used in this task, but whose columns" "may be needed downstream", 

116 default={ 

117 "selector1": SnSelector(), 

118 "selector2": StarSelector(), 

119 "selector3": GalaxySelector(), 

120 "selector4": CoaddPlotFlagSelector(), 

121 }, 

122 ) 

123 

124 extraColumns = pexConfig.ListField[str]( 

125 doc="Other catalog columns to persist to downstream tasks", 

126 default=["x", "y", "patch", "ebv"], 

127 ) 

128 

129 extraPerBandColumns = pexConfig.ListField[str]( 

130 doc="Other columns to load that should be loaded for each band individually.", 

131 default=["cModelFlux"], 

132 ) 

133 

134 matchRadius = pexConfig.Field[float]( 

135 doc="The radius to use for matching, in arcsecs.", 

136 default=1.0, 

137 ) 

138 

139 targetRaColumn = pexConfig.Field[str]( 

140 doc="RA column name for the target (being matched) catalog.", 

141 default="coord_ra", 

142 ) 

143 

144 targetDecColumn = pexConfig.Field[str]( 

145 doc="Dec column name for the target (being matched) catalog.", 

146 default="coord_dec", 

147 ) 

148 

149 refRaColumn = pexConfig.Field[str]( 

150 doc="RA column name for the reference (being matched to) catalog.", 

151 default="ra", 

152 ) 

153 

154 refDecColumn = pexConfig.Field[str]( 

155 doc="Dec column name for the reference (being matched to) catalog.", 

156 default="dec", 

157 ) 

158 

159 raColumn = pexConfig.Field[str]( 

160 doc="RA column.", 

161 default="coord_ra", 

162 deprecated="This field was replaced with targetRaColumn and is unused. Will be removed after v27.", 

163 ) 

164 

165 decColumn = pexConfig.Field[str]( 

166 doc="Dec column.", 

167 default="coord_dec", 

168 deprecated="This field was replaced with targetDecColumn and is unused. Will be removed after v27.", 

169 ) 

170 

171 patchColumn = pexConfig.Field[str](doc="Patch column.", default="patch") 

172 

173 matchesRefCat = pexConfig.Field[bool]( 

174 doc="Is the catalog being matched to stored as a reference catalog?", 

175 default=False, 

176 ) 

177 

178 returnNonMatches = pexConfig.Field[bool]( 

179 doc="Return the rows of the reference catalog that didn't get matched?", 

180 default=False, 

181 ) 

182 

183 def setDefaults(self): 

184 super().setDefaults() 

185 self.referenceCatalogLoader.doReferenceSelection = False 

186 self.referenceCatalogLoader.doApplyColorTerms = False 

187 

188 

189class CatalogMatchTask(pipeBase.PipelineTask): 

190 """The base task for matching catalogs. Figures out which columns 

191 it needs to grab for the downstream tasks and then matches the 

192 two tables together and returns the matched and joined table 

193 including the extra columns. 

194 """ 

195 

196 ConfigClass = CatalogMatchConfig 

197 _DefaultName = "analysisToolsCatalogMatch" 

198 

199 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

200 """Implemented in the inherited tasks""" 

201 pass 

202 

203 def run(self, *, targetCatalog, refCatalog, bands): 

204 """Takes the two catalogs and returns the matched one. 

205 

206 Parameters 

207 ---------- 

208 `targetCatalog` : astropy.table.Table 

209 The catalog to be matched 

210 `refCatalog` : astropy.table.Table 

211 The catalog to be matched to 

212 `bands` : list 

213 A list of bands to apply the selectors in 

214 

215 Returns 

216 ------- 

217 `matchedCatalog` : astropy.table.Table 

218 

219 Notes 

220 ----- 

221 Performs an RA/Dec match that returns the closest match 

222 within the match radius which defaults to 1.0 arcsecond. 

223 Applies the suffix, _target, to the catalog being matched 

224 and _ref to the reference catalog being matched to. 

225 """ 

226 # Apply the selectors to the catalog 

227 mask = np.ones(len(targetCatalog), dtype=bool) 

228 for selector in self.config.sourceSelectorActions: 

229 for band in self.config.selectorBands: 

230 mask &= selector(targetCatalog, band=band).astype(bool) 

231 

232 targetCatalog = targetCatalog[mask] 

233 

234 if (len(targetCatalog) == 0) or (len(refCatalog)) == 0: 

235 refMatchIndices = np.array([], dtype=np.int64) 

236 targetMatchIndices = np.array([], dtype=np.int64) 

237 dists = np.array([], dtype=np.float64) 

238 else: 

239 # Run the matcher. 

240 

241 # This all assumes that everything is in degrees. 

242 # Which I think is okay, but the current task 

243 # allows different units. Need to configure match 

244 # radius, either in this task or a subtask. 

245 

246 # Get rid of entries in the refCat with non-finite RA/Dec values. 

247 refRas = refCatalog[self.config.refRaColumn] 

248 refDecs = refCatalog[self.config.refDecColumn] 

249 refRaDecFiniteMask = np.isfinite(refRas) & np.isfinite(refDecs) 

250 refCatalog = refCatalog[refRaDecFiniteMask] 

251 with Matcher(refCatalog[self.config.refRaColumn], refCatalog[self.config.refDecColumn]) as m: 

252 idx, refMatchIndices, targetMatchIndices, dists = m.query_radius( 

253 targetCatalog[self.config.targetRaColumn], 

254 targetCatalog[self.config.targetDecColumn], 

255 self.config.matchRadius / 3600.0, 

256 return_indices=True, 

257 ) 

258 

259 # Convert degrees to arcseconds. 

260 dists *= 3600.0 

261 

262 targetCatalogMatched = targetCatalog[targetMatchIndices] 

263 refCatalogMatched = refCatalog[refMatchIndices] 

264 

265 targetCols = targetCatalogMatched.columns.copy() 

266 for col in targetCols: 

267 targetCatalogMatched.rename_column(col, col + "_target") 

268 refCols = refCatalogMatched.columns.copy() 

269 for col in refCols: 

270 refCatalogMatched.rename_column(col, col + "_ref") 

271 

272 if self.config.returnNonMatches: 

273 unmatchedIndices = list(set(np.arange(0, len(refCatalog))) - set(refMatchIndices)) 

274 refCatalogNotMatched = refCatalog[unmatchedIndices] 

275 # We need to set the relevant flag columns to 

276 # true or false so that they make it through the 

277 # selectors even though the none matched sources 

278 # don't have values for those columns. 

279 trueFlagCols = [] 

280 falseFlagCols = [] 

281 for selectorAction in [self.config.selectorActions, self.config.extraColumnSelectors]: 

282 for selector in selectorAction: 

283 try: 

284 for flag in selector.selectWhenTrue: 

285 trueFlagCols.append(flag) 

286 for flag in selector.selectWhenFalse: 

287 falseFlagCols.append(flag) 

288 except AttributeError: 

289 continue 

290 for col in refCols: 

291 refCatalogNotMatched.rename_column(col, col + "_ref") 

292 for col in targetCols: 

293 refCatalogNotMatched[col] = [np.nan] * len(refCatalogNotMatched) 

294 for col in trueFlagCols: 

295 refCatalogNotMatched[col] = [True] * len(refCatalogNotMatched) 

296 for col in falseFlagCols: 

297 refCatalogNotMatched[col] = [False] * len(refCatalogNotMatched) 

298 

299 if self.config.matchesRefCat: 

300 for i, band in enumerate(bands): 

301 refCatalogMatched[band + "_mag_ref"] = refCatalogMatched["refMag_ref"][:, i] 

302 refCatalogMatched[band + "_magErr_ref"] = refCatalogMatched["refMagErr_ref"][:, i] 

303 refCatalogMatched.remove_column("refMag_ref") 

304 refCatalogMatched.remove_column("refMagErr_ref") 

305 

306 if self.config.returnNonMatches: 

307 for i, band in enumerate(bands): 

308 refCatalogNotMatched[band + "_mag_ref"] = refCatalogNotMatched["refMag_ref"][:, i] 

309 refCatalogNotMatched[band + "_magErr_ref"] = refCatalogNotMatched["refMagErr_ref"][:, i] 

310 refCatalogNotMatched.remove_column("refMag_ref") 

311 refCatalogNotMatched.remove_column("refMagErr_ref") 

312 

313 tMatched = hstack([targetCatalogMatched, refCatalogMatched], join_type="exact") 

314 tMatched["matchDistance"] = dists 

315 

316 if self.config.returnNonMatches: 

317 refCatalogNotMatched["matchDistance"] = [np.nan] * len(refCatalogNotMatched) 

318 tMatched = vstack([tMatched, refCatalogNotMatched]) 

319 

320 return pipeBase.Struct(matchedCatalog=tMatched) 

321 

322 def prepColumns(self, bands): 

323 """Get all the columns needed for downstream tasks. 

324 Both those from the selectors and those specified in the 

325 config options. 

326 """ 

327 

328 bandColumns = [] 

329 for band in bands: 

330 for col in self.config.extraPerBandColumns: 

331 bandColumns.append(band + "_" + col) 

332 

333 columns = ( 

334 [ 

335 self.config.targetRaColumn, 

336 self.config.targetDecColumn, 

337 ] 

338 + self.config.extraColumns.list() 

339 + bandColumns 

340 ) 

341 

342 if self.config.patchColumn != "": 

343 columns.append(self.config.patchColumn) 

344 

345 selectorBands = list(set(list(bands) + self.config.selectorBands.list())) 

346 for selectorAction in [ 

347 self.config.selectorActions, 

348 self.config.sourceSelectorActions, 

349 self.config.extraColumnSelectors, 

350 ]: 

351 for selector in selectorAction: 

352 for band in selectorBands: 

353 selectorSchema = selector.getFormattedInputSchema(band=band) 

354 columns += [s[0] for s in selectorSchema] 

355 

356 return columns 

357 

358 def _loadRefCat(self, loaderTask, tractInfo): 

359 """Load the reference catalog that covers the 

360 catalog that is to be matched to. 

361 

362 Parameters 

363 ---------- 

364 `loaderTask` : 

365 lsst.pipe.tasks.loadReferenceCatalog.loadReferenceCatalogTask 

366 `tractInfo` : lsst.skymap.tractInfo.ExplicitTractInfo 

367 The tract information to get the sky location from 

368 

369 Returns 

370 ------- 

371 `loadedRefCat` : astropy.table.Table 

372 The reference catalog that covers the input catalog. 

373 """ 

374 boundingCircle = tractInfo.getOuterSkyPolygon().getBoundingCircle() 

375 center = lsst.geom.SpherePoint(boundingCircle.getCenter()) 

376 radius = boundingCircle.getOpeningAngle() 

377 

378 epoch = Time(self.config.epoch, format="decimalyear") 

379 

380 # This is always going to return degrees. 

381 try: 

382 loadedRefCat = loaderTask.getSkyCircleCatalog( 

383 center, radius, self.config.filterNames, epoch=epoch 

384 ) 

385 except RuntimeError as e: 

386 raise pipeBase.NoWorkFound(e) 

387 

388 return Table(loadedRefCat)