Coverage for python/lsst/faro/base/MatchedCatalogBase.py: 24%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

149 statements  

1# This file is part of faro. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import lsst.afw.table as afwTable 

23import lsst.pipe.base as pipeBase 

24import lsst.pex.config as pexConfig 

25import lsst.geom as geom 

26from lsst.utils.logging import PeriodicLogger 

27import numpy as np 

28 

29from lsst.faro.utils.matcher import matchCatalogs 

30 

31__all__ = ( 

32 "MatchedBaseConnections", 

33 "MatchedBaseConfig", 

34 "MatchedBaseTask", 

35 "MatchedTractBaseTask", 

36) 

37 

38 

39class MatchedBaseConnections( 

40 pipeBase.PipelineTaskConnections, 

41 dimensions=(), 

42 defaultTemplates={ 

43 "coaddName": "deep", 

44 "photoCalibName": "calexp.photoCalib", 

45 "wcsName": "calexp.wcs", 

46 "externalPhotoCalibName": "fgcm", 

47 "externalWcsName": "jointcal", 

48 }, 

49): 

50 sourceCatalogs = pipeBase.connectionTypes.Input( 

51 doc="Source catalogs to match up.", 

52 dimensions=("instrument", "visit", "detector", "band"), 

53 storageClass="SourceCatalog", 

54 name="src", 

55 multiple=True, 

56 ) 

57 photoCalibs = pipeBase.connectionTypes.Input( 

58 doc="Photometric calibration object.", 

59 dimensions=("instrument", "visit", "detector", "band"), 

60 storageClass="PhotoCalib", 

61 name="{photoCalibName}", 

62 multiple=True, 

63 ) 

64 astromCalibs = pipeBase.connectionTypes.Input( 

65 doc="WCS for the catalog.", 

66 dimensions=("instrument", "visit", "detector", "band"), 

67 storageClass="Wcs", 

68 name="{wcsName}", 

69 multiple=True, 

70 ) 

71 externalSkyWcsTractCatalog = pipeBase.connectionTypes.Input( 

72 doc=( 

73 "Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

74 "id for the catalog id, sorted on id for fast lookup." 

75 ), 

76 name="{externalWcsName}SkyWcsCatalog", 

77 storageClass="ExposureCatalog", 

78 dimensions=("instrument", "visit", "tract", "band"), 

79 multiple=True, 

80 ) 

81 externalSkyWcsGlobalCatalog = pipeBase.connectionTypes.Input( 

82 doc=( 

83 "Per-visit wcs calibrations computed globally (with no tract information). " 

84 "These catalogs use the detector id for the catalog id, sorted on id for " 

85 "fast lookup." 

86 ), 

87 name="{externalWcsName}SkyWcsCatalog", 

88 storageClass="ExposureCatalog", 

89 dimensions=("instrument", "visit", "band"), 

90 multiple=True, 

91 ) 

92 externalPhotoCalibTractCatalog = pipeBase.connectionTypes.Input( 

93 doc=( 

94 "Per-tract, per-visit photometric calibrations. These catalogs use the " 

95 "detector id for the catalog id, sorted on id for fast lookup." 

96 ), 

97 name="{externalPhotoCalibName}PhotoCalibCatalog", 

98 storageClass="ExposureCatalog", 

99 dimensions=("instrument", "visit", "tract", "band"), 

100 multiple=True, 

101 ) 

102 externalPhotoCalibGlobalCatalog = pipeBase.connectionTypes.Input( 

103 doc=( 

104 "Per-visit photometric calibrations computed globally (with no tract " 

105 "information). These catalogs use the detector id for the catalog id, " 

106 "sorted on id for fast lookup." 

107 ), 

108 name="{externalPhotoCalibName}PhotoCalibCatalog", 

109 storageClass="ExposureCatalog", 

110 dimensions=("instrument", "visit", "band"), 

111 multiple=True, 

112 ) 

113 skyMap = pipeBase.connectionTypes.Input( 

114 doc="Input definition of geometry/bbox and projection/wcs for warped exposures", 

115 name="skyMap", 

116 storageClass="SkyMap", 

117 dimensions=("skymap",), 

118 ) 

119 

120 def __init__(self, *, config=None): 

121 super().__init__(config=config) 

122 if config.doApplyExternalSkyWcs: 

123 if config.useGlobalExternalSkyWcs: 

124 self.inputs.remove("externalSkyWcsTractCatalog") 

125 else: 

126 self.inputs.remove("externalSkyWcsGlobalCatalog") 

127 else: 

128 self.inputs.remove("externalSkyWcsTractCatalog") 

129 self.inputs.remove("externalSkyWcsGlobalCatalog") 

130 if config.doApplyExternalPhotoCalib: 

131 if config.useGlobalExternalPhotoCalib: 

132 self.inputs.remove("externalPhotoCalibTractCatalog") 

133 else: 

134 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

135 else: 

136 self.inputs.remove("externalPhotoCalibTractCatalog") 

137 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

138 

139 

140class MatchedBaseConfig( 

141 pipeBase.PipelineTaskConfig, pipelineConnections=MatchedBaseConnections 

142): 

143 match_radius = pexConfig.Field( 

144 doc="Match radius in arcseconds.", dtype=float, default=1 

145 ) 

146 snrMin = pexConfig.Field( 

147 doc="Minimum SNR for a source to be included.", 

148 dtype=float, default=200 

149 ) 

150 snrMax = pexConfig.Field( 

151 doc="Maximum SNR for a source to be included.", 

152 dtype=float, default=np.Inf 

153 ) 

154 brightMagCut = pexConfig.Field( 

155 doc="Bright limit of catalog entries to include.", dtype=float, default=10.0 

156 ) 

157 faintMagCut = pexConfig.Field( 

158 doc="Faint limit of catalog entries to include.", dtype=float, default=30.0 

159 ) 

160 selectExtended = pexConfig.Field( 

161 doc="Whether to select extended sources", dtype=bool, default=False 

162 ) 

163 doApplyExternalSkyWcs = pexConfig.Field( 

164 doc="Whether or not to use the external wcs.", dtype=bool, default=False 

165 ) 

166 useGlobalExternalSkyWcs = pexConfig.Field( 

167 doc="Whether or not to use the global external wcs.", dtype=bool, default=False 

168 ) 

169 doApplyExternalPhotoCalib = pexConfig.Field( 

170 doc="Whether or not to use the external photoCalib.", dtype=bool, default=False 

171 ) 

172 useGlobalExternalPhotoCalib = pexConfig.Field( 

173 doc="Whether or not to use the global external photoCalib.", 

174 dtype=bool, 

175 default=False, 

176 ) 

177 

178 

179class MatchedBaseTask(pipeBase.PipelineTask): 

180 

181 ConfigClass = MatchedBaseConfig 

182 _DefaultName = "matchedBaseTask" 

183 

184 def __init__(self, config: MatchedBaseConfig, *args, **kwargs): 

185 super().__init__(*args, config=config, **kwargs) 

186 self.radius = self.config.match_radius 

187 self.level = "patch" 

188 

189 def run( 

190 self, 

191 sourceCatalogs, 

192 photoCalibs, 

193 astromCalibs, 

194 dataIds, 

195 wcs, 

196 box, 

197 doApplyExternalSkyWcs=False, 

198 doApplyExternalPhotoCalib=False, 

199 ): 

200 self.log.info("Running catalog matching") 

201 periodicLog = PeriodicLogger(self.log) 

202 radius = geom.Angle(self.radius, geom.arcseconds) 

203 if len(sourceCatalogs) < 2: 

204 self.log.warning("%s valid input catalogs: ", len(sourceCatalogs)) 

205 out_matched = afwTable.SimpleCatalog() 

206 else: 

207 srcvis, matched = matchCatalogs( 

208 sourceCatalogs, photoCalibs, astromCalibs, dataIds, radius, 

209 self.config, logger=self.log 

210 ) 

211 self.log.verbose("Finished matching catalogs.") 

212 

213 # Trim the output to the patch bounding box 

214 out_matched = type(matched)(matched.schema) 

215 self.log.info("%s sources in matched catalog.", len(matched)) 

216 for record_index, record in enumerate(matched): 

217 if box.contains(wcs.skyToPixel(record.getCoord())): 

218 out_matched.append(record) 

219 periodicLog.log("Checked %d records for trimming out of %d.", record_index + 1, len(matched)) 

220 

221 self.log.info( 

222 "%s sources when trimmed to %s boundaries.", len(out_matched), self.level 

223 ) 

224 return pipeBase.Struct(outputCatalog=out_matched) 

225 

226 def get_box_wcs(self, skymap, oid): 

227 tract_info = skymap.generateTract(oid["tract"]) 

228 wcs = tract_info.getWcs() 

229 patch_info = tract_info.getPatchInfo(oid["patch"]) 

230 patch_box = patch_info.getInnerBBox() 

231 self.log.info("Running tract: %s and patch: %s", oid["tract"], oid["patch"]) 

232 return patch_box, wcs 

233 

234 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

235 inputs = butlerQC.get(inputRefs) 

236 self.log.verbose("Inputs obtained from the butler.") 

237 oid = outputRefs.outputCatalog.dataId.byName() 

238 skymap = inputs["skyMap"] 

239 del inputs["skyMap"] 

240 box, wcs = self.get_box_wcs(skymap, oid) 

241 # Cast to float to handle fractional pixels 

242 box = geom.Box2D(box) 

243 inputs["dataIds"] = [ 

244 butlerQC.registry.expandDataId(el.dataId) for el in inputRefs.sourceCatalogs 

245 ] 

246 inputs["wcs"] = wcs 

247 inputs["box"] = box 

248 inputs["doApplyExternalSkyWcs"] = self.config.doApplyExternalSkyWcs 

249 inputs["doApplyExternalPhotoCalib"] = self.config.doApplyExternalPhotoCalib 

250 

251 if self.config.doApplyExternalPhotoCalib: 

252 if self.config.useGlobalExternalPhotoCalib: 

253 externalPhotoCalibCatalog = inputs.pop( 

254 "externalPhotoCalibGlobalCatalog" 

255 ) 

256 else: 

257 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog") 

258 

259 flatPhotoCalibList = np.hstack(externalPhotoCalibCatalog) 

260 visitPhotoCalibList = np.array( 

261 [calib["visit"] for calib in flatPhotoCalibList] 

262 ) 

263 detectorPhotoCalibList = np.array( 

264 [calib["id"] for calib in flatPhotoCalibList] 

265 ) 

266 

267 if self.config.doApplyExternalSkyWcs: 

268 if self.config.useGlobalExternalSkyWcs: 

269 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog") 

270 else: 

271 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog") 

272 

273 flatSkyWcsList = np.hstack(externalSkyWcsCatalog) 

274 visitSkyWcsList = np.array([calib["visit"] for calib in flatSkyWcsList]) 

275 detectorSkyWcsList = np.array([calib["id"] for calib in flatSkyWcsList]) 

276 

277 remove_indices = [] 

278 

279 if self.config.doApplyExternalPhotoCalib: 

280 for i in range(len(inputs["dataIds"])): 

281 dataId = inputs["dataIds"][i] 

282 detector = dataId["detector"] 

283 visit = dataId["visit"] 

284 calib_find = (visitPhotoCalibList == visit) & ( 

285 detectorPhotoCalibList == detector 

286 ) 

287 if np.sum(calib_find) < 1: 

288 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog " 

289 "for visit %s and will not be used.", 

290 detector, visit) 

291 inputs["photoCalibs"][i] = None 

292 remove_indices.append(i) 

293 else: 

294 row = flatPhotoCalibList[calib_find] 

295 externalPhotoCalib = row[0].getPhotoCalib() 

296 inputs["photoCalibs"][i] = externalPhotoCalib 

297 

298 if self.config.doApplyExternalSkyWcs: 

299 for i in range(len(inputs["dataIds"])): 

300 dataId = inputs["dataIds"][i] 

301 detector = dataId["detector"] 

302 visit = dataId["visit"] 

303 calib_find = (visitSkyWcsList == visit) & ( 

304 detectorSkyWcsList == detector 

305 ) 

306 if np.sum(calib_find) < 1: 

307 self.log.warning("Detector id %s not found in externalSkyWcsCatalog " 

308 "for visit %s and will not be used.", 

309 detector, visit) 

310 inputs["astromCalibs"][i] = None 

311 remove_indices.append(i) 

312 else: 

313 row = flatSkyWcsList[calib_find] 

314 externalSkyWcs = row[0].getWcs() 

315 inputs["astromCalibs"][i] = externalSkyWcs 

316 

317 # Remove datasets that didn't have matching external calibs 

318 remove_indices = np.unique(np.array(remove_indices)) 

319 if len(remove_indices) > 0: 

320 for ind in sorted(remove_indices, reverse=True): 

321 del inputs['sourceCatalogs'][ind] 

322 del inputs['dataIds'][ind] 

323 del inputs['photoCalibs'][ind] 

324 del inputs['astromCalibs'][ind] 

325 

326 outputs = self.run(**inputs) 

327 butlerQC.put(outputs, outputRefs) 

328 

329 

330class MatchedTractBaseTask(MatchedBaseTask): 

331 

332 ConfigClass = MatchedBaseConfig 

333 _DefaultName = "matchedTractBaseTask" 

334 

335 def __init__(self, config: MatchedBaseConfig, *args, **kwargs): 

336 super().__init__(*args, config=config, **kwargs) 

337 self.radius = self.config.match_radius 

338 self.level = "tract" 

339 

340 def get_box_wcs(self, skymap, oid): 

341 tract_info = skymap.generateTract(oid["tract"]) 

342 wcs = tract_info.getWcs() 

343 tract_box = tract_info.getBBox() 

344 self.log.info("Running tract: %s", oid["tract"]) 

345 return tract_box, wcs