Coverage for python/lsst/faro/base/MatchedCatalogBase.py: 21%

149 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-03-07 02:22 -0800

1# This file is part of faro. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import lsst.afw.table as afwTable 

23import lsst.pipe.base as pipeBase 

24import lsst.pex.config as pexConfig 

25import lsst.geom as geom 

26from lsst.utils.logging import PeriodicLogger 

27import numpy as np 

28 

29from lsst.faro.utils.matcher import matchCatalogs 

30 

31__all__ = ( 

32 "MatchedBaseConnections", 

33 "MatchedBaseConfig", 

34 "MatchedBaseTask", 

35 "MatchedTractBaseTask", 

36) 

37 

38 

39class MatchedBaseConnections( 

40 pipeBase.PipelineTaskConnections, 

41 dimensions=(), 

42 defaultTemplates={ 

43 "coaddName": "deep", 

44 "photoCalibName": "calexp.photoCalib", 

45 "wcsName": "calexp.wcs", 

46 "externalPhotoCalibName": "fgcm", 

47 "externalWcsName": "jointcal", 

48 }, 

49): 

50 sourceCatalogs = pipeBase.connectionTypes.Input( 

51 doc="Source catalogs to match up.", 

52 dimensions=("instrument", "visit", "detector", "band"), 

53 storageClass="SourceCatalog", 

54 name="src", 

55 multiple=True, 

56 ) 

57 photoCalibs = pipeBase.connectionTypes.Input( 

58 doc="Photometric calibration object.", 

59 dimensions=("instrument", "visit", "detector", "band"), 

60 storageClass="PhotoCalib", 

61 name="{photoCalibName}", 

62 multiple=True, 

63 ) 

64 astromCalibs = pipeBase.connectionTypes.Input( 

65 doc="WCS for the catalog.", 

66 dimensions=("instrument", "visit", "detector", "band"), 

67 storageClass="Wcs", 

68 name="{wcsName}", 

69 multiple=True, 

70 ) 

71 externalSkyWcsTractCatalog = pipeBase.connectionTypes.Input( 

72 doc=( 

73 "Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

74 "id for the catalog id, sorted on id for fast lookup." 

75 ), 

76 name="{externalWcsName}SkyWcsCatalog", 

77 storageClass="ExposureCatalog", 

78 dimensions=("instrument", "visit", "tract", "band"), 

79 multiple=True, 

80 ) 

81 externalSkyWcsGlobalCatalog = pipeBase.connectionTypes.Input( 

82 doc=( 

83 "Per-visit wcs calibrations computed globally (with no tract information). " 

84 "These catalogs use the detector id for the catalog id, sorted on id for " 

85 "fast lookup." 

86 ), 

87 name="{externalWcsName}SkyWcsCatalog", 

88 storageClass="ExposureCatalog", 

89 dimensions=("instrument", "visit", "band"), 

90 multiple=True, 

91 ) 

92 externalPhotoCalibTractCatalog = pipeBase.connectionTypes.Input( 

93 doc=( 

94 "Per-tract, per-visit photometric calibrations. These catalogs use the " 

95 "detector id for the catalog id, sorted on id for fast lookup." 

96 ), 

97 name="{externalPhotoCalibName}PhotoCalibCatalog", 

98 storageClass="ExposureCatalog", 

99 dimensions=("instrument", "visit", "tract", "band"), 

100 multiple=True, 

101 ) 

102 externalPhotoCalibGlobalCatalog = pipeBase.connectionTypes.Input( 

103 doc=( 

104 "Per-visit photometric calibrations computed globally (with no tract " 

105 "information). These catalogs use the detector id for the catalog id, " 

106 "sorted on id for fast lookup." 

107 ), 

108 name="{externalPhotoCalibName}PhotoCalibCatalog", 

109 storageClass="ExposureCatalog", 

110 dimensions=("instrument", "visit", "band"), 

111 multiple=True, 

112 ) 

113 skyMap = pipeBase.connectionTypes.Input( 

114 doc="Input definition of geometry/bbox and projection/wcs for warped exposures", 

115 name="skyMap", 

116 storageClass="SkyMap", 

117 dimensions=("skymap",), 

118 ) 

119 

120 def __init__(self, *, config=None): 

121 super().__init__(config=config) 

122 if config.doApplyExternalSkyWcs: 

123 if config.useGlobalExternalSkyWcs: 

124 self.inputs.remove("externalSkyWcsTractCatalog") 

125 else: 

126 self.inputs.remove("externalSkyWcsGlobalCatalog") 

127 else: 

128 self.inputs.remove("externalSkyWcsTractCatalog") 

129 self.inputs.remove("externalSkyWcsGlobalCatalog") 

130 if config.doApplyExternalPhotoCalib: 

131 if config.useGlobalExternalPhotoCalib: 

132 self.inputs.remove("externalPhotoCalibTractCatalog") 

133 else: 

134 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

135 else: 

136 self.inputs.remove("externalPhotoCalibTractCatalog") 

137 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

138 

139 

140class MatchedBaseConfig( 

141 pipeBase.PipelineTaskConfig, pipelineConnections=MatchedBaseConnections 

142): 

143 match_radius = pexConfig.Field( 

144 doc="Match radius in arcseconds.", dtype=float, default=1 

145 ) 

146 snrMin = pexConfig.Field( 

147 doc="Minimum SNR for a source to be included.", 

148 dtype=float, default=200 

149 ) 

150 snrMax = pexConfig.Field( 

151 doc="Maximum SNR for a source to be included.", 

152 dtype=float, default=np.Inf 

153 ) 

154 brightMagCut = pexConfig.Field( 

155 doc="Bright limit of catalog entries to include.", dtype=float, default=10.0 

156 ) 

157 faintMagCut = pexConfig.Field( 

158 doc="Faint limit of catalog entries to include.", dtype=float, default=30.0 

159 ) 

160 selectExtended = pexConfig.Field( 

161 doc="Whether to select extended sources", dtype=bool, default=False 

162 ) 

163 doApplyExternalSkyWcs = pexConfig.Field( 

164 doc="Whether or not to use the external wcs.", dtype=bool, default=False 

165 ) 

166 useGlobalExternalSkyWcs = pexConfig.Field( 

167 doc="Whether or not to use the global external wcs.", dtype=bool, default=False 

168 ) 

169 doApplyExternalPhotoCalib = pexConfig.Field( 

170 doc="Whether or not to use the external photoCalib.", dtype=bool, default=False 

171 ) 

172 useGlobalExternalPhotoCalib = pexConfig.Field( 

173 doc="Whether or not to use the global external photoCalib.", 

174 dtype=bool, 

175 default=False, 

176 ) 

177 

178 

179class MatchedBaseTask(pipeBase.PipelineTask): 

180 

181 ConfigClass = MatchedBaseConfig 

182 _DefaultName = "matchedBaseTask" 

183 

184 def __init__(self, config: MatchedBaseConfig, *args, **kwargs): 

185 super().__init__(*args, config=config, **kwargs) 

186 self.radius = self.config.match_radius 

187 self.level = "patch" 

188 

189 def run( 

190 self, 

191 sourceCatalogs, 

192 photoCalibs, 

193 astromCalibs, 

194 dataIds, 

195 wcs, 

196 box, 

197 doApplyExternalSkyWcs=False, 

198 doApplyExternalPhotoCalib=False, 

199 ): 

200 self.log.info("Running catalog matching") 

201 periodicLog = PeriodicLogger(self.log) 

202 radius = geom.Angle(self.radius, geom.arcseconds) 

203 if len(sourceCatalogs) < 2: 

204 self.log.warning("%s valid input catalogs: ", len(sourceCatalogs)) 

205 out_matched = afwTable.SimpleCatalog() 

206 else: 

207 srcvis, matched = matchCatalogs( 

208 sourceCatalogs, photoCalibs, astromCalibs, dataIds, radius, 

209 self.config, logger=self.log 

210 ) 

211 self.log.verbose("Finished matching catalogs.") 

212 

213 # Trim the output to the patch bounding box 

214 out_matched = type(matched)(matched.schema) 

215 self.log.info("%s sources in matched catalog.", len(matched)) 

216 for record_index, record in enumerate(matched): 

217 if box.contains(wcs.skyToPixel(record.getCoord())): 

218 out_matched.append(record) 

219 periodicLog.log("Checked %d records for trimming out of %d.", record_index + 1, len(matched)) 

220 

221 self.log.info( 

222 "%s sources when trimmed to %s boundaries.", len(out_matched), self.level 

223 ) 

224 return pipeBase.Struct(outputCatalog=out_matched) 

225 

226 def get_box_wcs(self, skymap, oid): 

227 tract_info = skymap.generateTract(oid["tract"]) 

228 wcs = tract_info.getWcs() 

229 patch_info = tract_info.getPatchInfo(oid["patch"]) 

230 patch_box = patch_info.getInnerBBox() 

231 self.log.info("Running tract: %s and patch: %s", oid["tract"], oid["patch"]) 

232 return patch_box, wcs 

233 

234 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

235 inputs = butlerQC.get(inputRefs) 

236 self.log.verbose("Inputs obtained from the butler.") 

237 oid = outputRefs.outputCatalog.dataId.byName() 

238 skymap = inputs["skyMap"] 

239 del inputs["skyMap"] 

240 box, wcs = self.get_box_wcs(skymap, oid) 

241 # Cast to float to handle fractional pixels 

242 box = geom.Box2D(box) 

243 inputs["dataIds"] = [el.dataId for el in inputRefs.sourceCatalogs] 

244 inputs["wcs"] = wcs 

245 inputs["box"] = box 

246 inputs["doApplyExternalSkyWcs"] = self.config.doApplyExternalSkyWcs 

247 inputs["doApplyExternalPhotoCalib"] = self.config.doApplyExternalPhotoCalib 

248 

249 if self.config.doApplyExternalPhotoCalib: 

250 if self.config.useGlobalExternalPhotoCalib: 

251 externalPhotoCalibCatalog = inputs.pop( 

252 "externalPhotoCalibGlobalCatalog" 

253 ) 

254 else: 

255 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog") 

256 

257 flatPhotoCalibList = np.hstack(externalPhotoCalibCatalog) 

258 visitPhotoCalibList = np.array( 

259 [calib["visit"] for calib in flatPhotoCalibList] 

260 ) 

261 detectorPhotoCalibList = np.array( 

262 [calib["id"] for calib in flatPhotoCalibList] 

263 ) 

264 

265 if self.config.doApplyExternalSkyWcs: 

266 if self.config.useGlobalExternalSkyWcs: 

267 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog") 

268 else: 

269 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog") 

270 

271 flatSkyWcsList = np.hstack(externalSkyWcsCatalog) 

272 visitSkyWcsList = np.array([calib["visit"] for calib in flatSkyWcsList]) 

273 detectorSkyWcsList = np.array([calib["id"] for calib in flatSkyWcsList]) 

274 

275 remove_indices = [] 

276 

277 if self.config.doApplyExternalPhotoCalib: 

278 for i in range(len(inputs["dataIds"])): 

279 dataId = inputs["dataIds"][i] 

280 detector = dataId["detector"] 

281 visit = dataId["visit"] 

282 calib_find = (visitPhotoCalibList == visit) & ( 

283 detectorPhotoCalibList == detector 

284 ) 

285 if np.sum(calib_find) < 1: 

286 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog " 

287 "for visit %s and will not be used.", 

288 detector, visit) 

289 inputs["photoCalibs"][i] = None 

290 remove_indices.append(i) 

291 else: 

292 row = flatPhotoCalibList[calib_find] 

293 externalPhotoCalib = row[0].getPhotoCalib() 

294 inputs["photoCalibs"][i] = externalPhotoCalib 

295 

296 if self.config.doApplyExternalSkyWcs: 

297 for i in range(len(inputs["dataIds"])): 

298 dataId = inputs["dataIds"][i] 

299 detector = dataId["detector"] 

300 visit = dataId["visit"] 

301 calib_find = (visitSkyWcsList == visit) & ( 

302 detectorSkyWcsList == detector 

303 ) 

304 if np.sum(calib_find) < 1: 

305 self.log.warning("Detector id %s not found in externalSkyWcsCatalog " 

306 "for visit %s and will not be used.", 

307 detector, visit) 

308 inputs["astromCalibs"][i] = None 

309 remove_indices.append(i) 

310 else: 

311 row = flatSkyWcsList[calib_find] 

312 externalSkyWcs = row[0].getWcs() 

313 inputs["astromCalibs"][i] = externalSkyWcs 

314 

315 # Remove datasets that didn't have matching external calibs 

316 remove_indices = np.unique(np.array(remove_indices)) 

317 if len(remove_indices) > 0: 

318 for ind in sorted(remove_indices, reverse=True): 

319 del inputs['sourceCatalogs'][ind] 

320 del inputs['dataIds'][ind] 

321 del inputs['photoCalibs'][ind] 

322 del inputs['astromCalibs'][ind] 

323 

324 outputs = self.run(**inputs) 

325 butlerQC.put(outputs, outputRefs) 

326 

327 

328class MatchedTractBaseTask(MatchedBaseTask): 

329 

330 ConfigClass = MatchedBaseConfig 

331 _DefaultName = "matchedTractBaseTask" 

332 

333 def __init__(self, config: MatchedBaseConfig, *args, **kwargs): 

334 super().__init__(*args, config=config, **kwargs) 

335 self.radius = self.config.match_radius 

336 self.level = "tract" 

337 

338 def get_box_wcs(self, skymap, oid): 

339 tract_info = skymap.generateTract(oid["tract"]) 

340 wcs = tract_info.getWcs() 

341 tract_box = tract_info.getBBox() 

342 self.log.info("Running tract: %s", oid["tract"]) 

343 return tract_box, wcs