Coverage for python/lsst/faro/base/MatchedCatalogBase.py: 21%

163 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-03-16 09:50 +0000

1# This file is part of faro. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import warnings 

23 

24from lsst.utils.introspection import find_outside_stacklevel 

25import lsst.afw.table as afwTable 

26import lsst.pipe.base as pipeBase 

27import lsst.pex.config as pexConfig 

28import lsst.geom as geom 

29from lsst.utils.logging import PeriodicLogger 

30import numpy as np 

31 

32from lsst.faro.utils.matcher import matchCatalogs 

33 

34__all__ = ( 

35 "MatchedBaseConnections", 

36 "MatchedBaseConfig", 

37 "MatchedBaseTask", 

38 "MatchedTractBaseTask", 

39) 

40 

41 

42class MatchedBaseConnections( 

43 pipeBase.PipelineTaskConnections, 

44 dimensions=(), 

45 defaultTemplates={ 

46 "coaddName": "deep", 

47 "photoCalibName": "calexp.photoCalib", 

48 "wcsName": "calexp.wcs", 

49 "externalPhotoCalibName": "fgcm", 

50 "externalWcsName": "gbdesAstrometricFit", 

51 }, 

52 # TODO: remove on DM-39854. 

53 deprecatedTemplates={ 

54 "photoCalibName": "Deprecated in favor of visitSummary; will be removed after v26.", 

55 "wcsName": "Deprecated in favor of visitSummary; will be removed after v26.", 

56 "externalPhotoCalibName": "Deprecated in favor of visitSummary; will be removed after v26.", 

57 "externalWcsName": "Deprecated in favor of visitSummary; will be removed after v26.", 

58 }, 

59): 

60 sourceCatalogs = pipeBase.connectionTypes.Input( 

61 doc="Source catalogs to match up.", 

62 dimensions=("instrument", "visit", "detector", "band"), 

63 storageClass="SourceCatalog", 

64 name="src", 

65 multiple=True, 

66 ) 

67 visitSummary = pipeBase.connectionTypes.Input( 

68 doc="Exposure catalog with WCS and PhotoCalib this detector+visit combination.", 

69 dimensions=("instrument", "visit"), 

70 storageClass="ExposureCatalog", 

71 name="finalVisitSummary", 

72 multiple=True, 

73 ) 

74 photoCalibs = pipeBase.connectionTypes.Input( 

75 doc="Photometric calibration object.", 

76 dimensions=("instrument", "visit", "detector", "band"), 

77 storageClass="PhotoCalib", 

78 name="{photoCalibName}", 

79 multiple=True, 

80 # TODO: remove on DM-39854. 

81 deprecated="Deprecated in favor of visitSummary and already ignored; will be removed after v26." 

82 ) 

83 astromCalibs = pipeBase.connectionTypes.Input( 

84 doc="WCS for the catalog.", 

85 dimensions=("instrument", "visit", "detector", "band"), 

86 storageClass="Wcs", 

87 name="{wcsName}", 

88 multiple=True, 

89 # TODO: remove on DM-39854. 

90 deprecated="Deprecated in favor of visitSummary and already ignored; will be removed after v26." 

91 ) 

92 externalSkyWcsTractCatalog = pipeBase.connectionTypes.Input( 

93 doc=( 

94 "Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

95 "id for the catalog id, sorted on id for fast lookup." 

96 ), 

97 name="{externalWcsName}SkyWcsCatalog", 

98 storageClass="ExposureCatalog", 

99 dimensions=("instrument", "visit", "tract", "band"), 

100 multiple=True, 

101 # TODO: remove on DM-39854. 

102 deprecated="Deprecated in favor of visitSummary; will be removed after v26." 

103 ) 

104 externalSkyWcsGlobalCatalog = pipeBase.connectionTypes.Input( 

105 doc=( 

106 "Per-visit wcs calibrations computed globally (with no tract information). " 

107 "These catalogs use the detector id for the catalog id, sorted on id for " 

108 "fast lookup." 

109 ), 

110 name="{externalWcsName}SkyWcsCatalog", 

111 storageClass="ExposureCatalog", 

112 dimensions=("instrument", "visit", "band"), 

113 multiple=True, 

114 # TODO: remove on DM-39854. 

115 deprecated="Deprecated in favor of visitSummary; will be removed after v26." 

116 ) 

117 externalPhotoCalibTractCatalog = pipeBase.connectionTypes.Input( 

118 doc=( 

119 "Per-tract, per-visit photometric calibrations. These catalogs use the " 

120 "detector id for the catalog id, sorted on id for fast lookup." 

121 ), 

122 name="{externalPhotoCalibName}PhotoCalibCatalog", 

123 storageClass="ExposureCatalog", 

124 dimensions=("instrument", "visit", "tract", "band"), 

125 multiple=True, 

126 # TODO: remove on DM-39854. 

127 deprecated="Deprecated in favor of visitSummary; will be removed after v26." 

128 ) 

129 externalPhotoCalibGlobalCatalog = pipeBase.connectionTypes.Input( 

130 doc=( 

131 "Per-visit photometric calibrations computed globally (with no tract " 

132 "information). These catalogs use the detector id for the catalog id, " 

133 "sorted on id for fast lookup." 

134 ), 

135 name="{externalPhotoCalibName}PhotoCalibCatalog", 

136 storageClass="ExposureCatalog", 

137 dimensions=("instrument", "visit", "band"), 

138 multiple=True, 

139 # TODO: remove on DM-39854. 

140 deprecated="Deprecated in favor of visitSummary; will be removed after v26." 

141 ) 

142 skyMap = pipeBase.connectionTypes.Input( 

143 doc="Input definition of geometry/bbox and projection/wcs for warped exposures", 

144 name="skyMap", 

145 storageClass="SkyMap", 

146 dimensions=("skymap",), 

147 ) 

148 

149 def __init__(self, *, config=None): 

150 super().__init__(config=config) 

151 if config.doApplyExternalSkyWcs: 

152 if config.useGlobalExternalSkyWcs: 

153 self.inputs.remove("externalSkyWcsTractCatalog") 

154 else: 

155 self.inputs.remove("externalSkyWcsGlobalCatalog") 

156 else: 

157 self.inputs.remove("externalSkyWcsTractCatalog") 

158 self.inputs.remove("externalSkyWcsGlobalCatalog") 

159 if config.doApplyExternalPhotoCalib: 

160 if config.useGlobalExternalPhotoCalib: 

161 self.inputs.remove("externalPhotoCalibTractCatalog") 

162 else: 

163 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

164 else: 

165 self.inputs.remove("externalPhotoCalibTractCatalog") 

166 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

167 del self.photoCalibs 

168 del self.astromCalibs 

169 

170 

171class MatchedBaseConfig( 

172 pipeBase.PipelineTaskConfig, pipelineConnections=MatchedBaseConnections 

173): 

174 match_radius = pexConfig.Field( 

175 doc="Match radius in arcseconds.", dtype=float, default=1 

176 ) 

177 snrMin = pexConfig.Field( 

178 doc="Minimum SNR for a source to be included.", 

179 dtype=float, default=200 

180 ) 

181 snrMax = pexConfig.Field( 

182 doc="Maximum SNR for a source to be included.", 

183 dtype=float, default=np.Inf 

184 ) 

185 brightMagCut = pexConfig.Field( 

186 doc="Bright limit of catalog entries to include.", dtype=float, default=10.0 

187 ) 

188 faintMagCut = pexConfig.Field( 

189 doc="Faint limit of catalog entries to include.", dtype=float, default=30.0 

190 ) 

191 selectExtended = pexConfig.Field( 

192 doc="Whether to select extended sources", dtype=bool, default=False 

193 ) 

194 doApplyExternalSkyWcs = pexConfig.Field( 

195 doc="Whether or not to use the external wcs.", dtype=bool, default=False, 

196 # TODO: remove on DM-39854. 

197 deprecated="Deprecated in favor of the visitSummary connection; will be removed after v26." 

198 ) 

199 useGlobalExternalSkyWcs = pexConfig.Field( 

200 doc="Whether or not to use the global external wcs.", dtype=bool, default=False, 

201 # TODO: remove on DM-39854. 

202 deprecated="Deprecated in favor of the visitSummary connection; will be removed after v26." 

203 ) 

204 doApplyExternalPhotoCalib = pexConfig.Field( 

205 doc="Whether or not to use the external photoCalib.", dtype=bool, default=False, 

206 # TODO: remove on DM-39854. 

207 deprecated="Deprecated in favor of the visitSummary connection; will be removed after v26." 

208 ) 

209 useGlobalExternalPhotoCalib = pexConfig.Field( 

210 doc="Whether or not to use the global external photoCalib.", 

211 dtype=bool, 

212 default=False, 

213 # TODO: remove on DM-39854. 

214 deprecated="Deprecated in favor of the visitSummary connection; will be removed after v26." 

215 ) 

216 

217 

218class MatchedBaseTask(pipeBase.PipelineTask): 

219 

220 ConfigClass = MatchedBaseConfig 

221 _DefaultName = "matchedBaseTask" 

222 

223 def __init__(self, config: MatchedBaseConfig, *args, **kwargs): 

224 super().__init__(*args, config=config, **kwargs) 

225 self.radius = self.config.match_radius 

226 self.level = "patch" 

227 

228 def run( 

229 self, 

230 sourceCatalogs, 

231 photoCalibs, 

232 astromCalibs, 

233 dataIds, 

234 wcs, 

235 box, 

236 doApplyExternalSkyWcs=None, 

237 doApplyExternalPhotoCalib=None, 

238 ): 

239 # TODO: remove these arguments on DM-39854. 

240 if doApplyExternalPhotoCalib is not None: 

241 warnings.warn( 

242 "The doApplyExternalPhotoCalib argument is deprecated and will be removed after v26.", 

243 category=FutureWarning, stacklevel=find_outside_stacklevel("lsst.faro"), 

244 ) 

245 else: 

246 doApplyExternalPhotoCalib = False 

247 if doApplyExternalSkyWcs is not None: 

248 warnings.warn( 

249 "The doApplyExternalSkyWcs argument is deprecated and will be removed after v26.", 

250 category=FutureWarning, stacklevel=find_outside_stacklevel("lsst.faro"), 

251 ) 

252 doApplyExternalSkyWcs = False 

253 self.log.info("Running catalog matching") 

254 periodicLog = PeriodicLogger(self.log) 

255 radius = geom.Angle(self.radius, geom.arcseconds) 

256 if len(sourceCatalogs) < 2: 

257 self.log.warning("%s valid input catalogs: ", len(sourceCatalogs)) 

258 out_matched = afwTable.SimpleCatalog() 

259 else: 

260 srcvis, matched = matchCatalogs( 

261 sourceCatalogs, photoCalibs, astromCalibs, dataIds, radius, 

262 self.config, logger=self.log 

263 ) 

264 self.log.verbose("Finished matching catalogs.") 

265 

266 # Trim the output to the patch bounding box 

267 out_matched = type(matched)(matched.schema) 

268 self.log.info("%s sources in matched catalog.", len(matched)) 

269 for record_index, record in enumerate(matched): 

270 if box.contains(wcs.skyToPixel(record.getCoord())): 

271 out_matched.append(record) 

272 periodicLog.log("Checked %d records for trimming out of %d.", record_index + 1, len(matched)) 

273 

274 self.log.info( 

275 "%s sources when trimmed to %s boundaries.", len(out_matched), self.level 

276 ) 

277 return pipeBase.Struct(outputCatalog=out_matched) 

278 

279 def get_box_wcs(self, skymap, oid): 

280 tract_info = skymap.generateTract(oid["tract"]) 

281 wcs = tract_info.getWcs() 

282 patch_info = tract_info.getPatchInfo(oid["patch"]) 

283 patch_box = patch_info.getInnerBBox() 

284 self.log.info("Running tract: %s and patch: %s", oid["tract"], oid["patch"]) 

285 return patch_box, wcs 

286 

287 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

288 inputs = butlerQC.get(inputRefs) 

289 self.log.verbose("Inputs obtained from the butler.") 

290 oid = dict(outputRefs.outputCatalog.dataId.required) 

291 skymap = inputs["skyMap"] 

292 del inputs["skyMap"] 

293 box, wcs = self.get_box_wcs(skymap, oid) 

294 # Cast to float to handle fractional pixels 

295 box = geom.Box2D(box) 

296 inputs["dataIds"] = [el.dataId for el in inputRefs.sourceCatalogs] 

297 inputs["wcs"] = wcs 

298 inputs["box"] = box 

299 inputs["doApplyExternalSkyWcs"] = self.config.doApplyExternalSkyWcs 

300 inputs["doApplyExternalPhotoCalib"] = self.config.doApplyExternalPhotoCalib 

301 visitSummary = inputs.pop("visitSummary") 

302 

303 # TODO: significant simplification should be possible here on DM-39854. 

304 if self.config.doApplyExternalPhotoCalib: 

305 if self.config.useGlobalExternalPhotoCalib: 

306 externalPhotoCalibCatalog = inputs.pop( 

307 "externalPhotoCalibGlobalCatalog" 

308 ) 

309 else: 

310 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog") 

311 else: 

312 externalPhotoCalibCatalog = visitSummary 

313 

314 flatPhotoCalibList = np.hstack(externalPhotoCalibCatalog) 

315 visitPhotoCalibList = np.array( 

316 [calib["visit"] for calib in flatPhotoCalibList] 

317 ) 

318 detectorPhotoCalibList = np.array( 

319 [calib["id"] for calib in flatPhotoCalibList] 

320 ) 

321 

322 if self.config.doApplyExternalSkyWcs: 

323 if self.config.useGlobalExternalSkyWcs: 

324 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog") 

325 else: 

326 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog") 

327 else: 

328 externalSkyWcsCatalog = visitSummary 

329 

330 flatSkyWcsList = np.hstack(externalSkyWcsCatalog) 

331 visitSkyWcsList = np.array([calib["visit"] for calib in flatSkyWcsList]) 

332 detectorSkyWcsList = np.array([calib["id"] for calib in flatSkyWcsList]) 

333 

334 remove_indices = [] 

335 inputs.setdefault("photoCalibs", [None] * len(inputs["dataIds"])) 

336 inputs.setdefault("astromCalibs", [None] * len(inputs["dataIds"])) 

337 

338 for i in range(len(inputs["dataIds"])): 

339 dataId = inputs["dataIds"][i] 

340 detector = dataId["detector"] 

341 visit = dataId["visit"] 

342 calib_find = (visitPhotoCalibList == visit) & ( 

343 detectorPhotoCalibList == detector 

344 ) 

345 if np.sum(calib_find) < 1: 

346 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog " 

347 "for visit %s and will not be used.", 

348 detector, visit) 

349 inputs["photoCalibs"][i] = None 

350 remove_indices.append(i) 

351 else: 

352 row = flatPhotoCalibList[calib_find] 

353 externalPhotoCalib = row[0].getPhotoCalib() 

354 inputs["photoCalibs"][i] = externalPhotoCalib 

355 

356 for i in range(len(inputs["dataIds"])): 

357 dataId = inputs["dataIds"][i] 

358 detector = dataId["detector"] 

359 visit = dataId["visit"] 

360 calib_find = (visitSkyWcsList == visit) & ( 

361 detectorSkyWcsList == detector 

362 ) 

363 if np.sum(calib_find) < 1: 

364 self.log.warning("Detector id %s not found in externalSkyWcsCatalog " 

365 "for visit %s and will not be used.", 

366 detector, visit) 

367 inputs["astromCalibs"][i] = None 

368 remove_indices.append(i) 

369 else: 

370 row = flatSkyWcsList[calib_find] 

371 externalSkyWcs = row[0].getWcs() 

372 inputs["astromCalibs"][i] = externalSkyWcs 

373 

374 # Remove datasets that didn't have matching external calibs 

375 remove_indices = np.unique(np.array(remove_indices)) 

376 if len(remove_indices) > 0: 

377 for ind in sorted(remove_indices, reverse=True): 

378 del inputs['sourceCatalogs'][ind] 

379 del inputs['dataIds'][ind] 

380 del inputs['photoCalibs'][ind] 

381 del inputs['astromCalibs'][ind] 

382 

383 outputs = self.run(**inputs) 

384 butlerQC.put(outputs, outputRefs) 

385 

386 

387class MatchedTractBaseTask(MatchedBaseTask): 

388 

389 ConfigClass = MatchedBaseConfig 

390 _DefaultName = "matchedTractBaseTask" 

391 

392 def __init__(self, config: MatchedBaseConfig, *args, **kwargs): 

393 super().__init__(*args, config=config, **kwargs) 

394 self.radius = self.config.match_radius 

395 self.level = "tract" 

396 

397 def get_box_wcs(self, skymap, oid): 

398 tract_info = skymap.generateTract(oid["tract"]) 

399 wcs = tract_info.getWcs() 

400 tract_box = tract_info.getBBox() 

401 self.log.info("Running tract: %s", oid["tract"]) 

402 return tract_box, wcs