Coverage for python/lsst/faro/base/MatchedCatalogBase.py: 24%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

144 statements  

1# This file is part of faro. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import lsst.afw.table as afwTable 

23import lsst.pipe.base as pipeBase 

24import lsst.pex.config as pexConfig 

25import lsst.geom as geom 

26import numpy as np 

27 

28from lsst.faro.utils.matcher import matchCatalogs 

29 

30__all__ = ( 

31 "MatchedBaseConnections", 

32 "MatchedBaseConfig", 

33 "MatchedBaseTask", 

34 "MatchedTractBaseTask", 

35) 

36 

37 

38class MatchedBaseConnections( 

39 pipeBase.PipelineTaskConnections, 

40 dimensions=(), 

41 defaultTemplates={ 

42 "coaddName": "deep", 

43 "photoCalibName": "calexp.photoCalib", 

44 "wcsName": "calexp.wcs", 

45 "externalPhotoCalibName": "fgcm", 

46 "externalWcsName": "jointcal", 

47 }, 

48): 

49 sourceCatalogs = pipeBase.connectionTypes.Input( 

50 doc="Source catalogs to match up.", 

51 dimensions=("instrument", "visit", "detector", "band"), 

52 storageClass="SourceCatalog", 

53 name="src", 

54 multiple=True, 

55 ) 

56 photoCalibs = pipeBase.connectionTypes.Input( 

57 doc="Photometric calibration object.", 

58 dimensions=("instrument", "visit", "detector", "band"), 

59 storageClass="PhotoCalib", 

60 name="{photoCalibName}", 

61 multiple=True, 

62 ) 

63 astromCalibs = pipeBase.connectionTypes.Input( 

64 doc="WCS for the catalog.", 

65 dimensions=("instrument", "visit", "detector", "band"), 

66 storageClass="Wcs", 

67 name="{wcsName}", 

68 multiple=True, 

69 ) 

70 externalSkyWcsTractCatalog = pipeBase.connectionTypes.Input( 

71 doc=( 

72 "Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

73 "id for the catalog id, sorted on id for fast lookup." 

74 ), 

75 name="{externalWcsName}SkyWcsCatalog", 

76 storageClass="ExposureCatalog", 

77 dimensions=("instrument", "visit", "tract", "band"), 

78 multiple=True, 

79 ) 

80 externalSkyWcsGlobalCatalog = pipeBase.connectionTypes.Input( 

81 doc=( 

82 "Per-visit wcs calibrations computed globally (with no tract information). " 

83 "These catalogs use the detector id for the catalog id, sorted on id for " 

84 "fast lookup." 

85 ), 

86 name="{externalWcsName}SkyWcsCatalog", 

87 storageClass="ExposureCatalog", 

88 dimensions=("instrument", "visit", "band"), 

89 multiple=True, 

90 ) 

91 externalPhotoCalibTractCatalog = pipeBase.connectionTypes.Input( 

92 doc=( 

93 "Per-tract, per-visit photometric calibrations. These catalogs use the " 

94 "detector id for the catalog id, sorted on id for fast lookup." 

95 ), 

96 name="{externalPhotoCalibName}PhotoCalibCatalog", 

97 storageClass="ExposureCatalog", 

98 dimensions=("instrument", "visit", "tract", "band"), 

99 multiple=True, 

100 ) 

101 externalPhotoCalibGlobalCatalog = pipeBase.connectionTypes.Input( 

102 doc=( 

103 "Per-visit photometric calibrations computed globally (with no tract " 

104 "information). These catalogs use the detector id for the catalog id, " 

105 "sorted on id for fast lookup." 

106 ), 

107 name="{externalPhotoCalibName}PhotoCalibCatalog", 

108 storageClass="ExposureCatalog", 

109 dimensions=("instrument", "visit", "band"), 

110 multiple=True, 

111 ) 

112 skyMap = pipeBase.connectionTypes.Input( 

113 doc="Input definition of geometry/bbox and projection/wcs for warped exposures", 

114 name="skyMap", 

115 storageClass="SkyMap", 

116 dimensions=("skymap",), 

117 ) 

118 

119 def __init__(self, *, config=None): 

120 super().__init__(config=config) 

121 if config.doApplyExternalSkyWcs: 

122 if config.useGlobalExternalSkyWcs: 

123 self.inputs.remove("externalSkyWcsTractCatalog") 

124 else: 

125 self.inputs.remove("externalSkyWcsGlobalCatalog") 

126 else: 

127 self.inputs.remove("externalSkyWcsTractCatalog") 

128 self.inputs.remove("externalSkyWcsGlobalCatalog") 

129 if config.doApplyExternalPhotoCalib: 

130 if config.useGlobalExternalPhotoCalib: 

131 self.inputs.remove("externalPhotoCalibTractCatalog") 

132 else: 

133 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

134 else: 

135 self.inputs.remove("externalPhotoCalibTractCatalog") 

136 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

137 

138 

139class MatchedBaseConfig( 

140 pipeBase.PipelineTaskConfig, pipelineConnections=MatchedBaseConnections 

141): 

142 match_radius = pexConfig.Field( 

143 doc="Match radius in arcseconds.", dtype=float, default=1 

144 ) 

145 snrMin = pexConfig.Field( 

146 doc="Minimum SNR for a source to be included.", 

147 dtype=float, default=200 

148 ) 

149 snrMax = pexConfig.Field( 

150 doc="Maximum SNR for a source to be included.", 

151 dtype=float, default=np.Inf 

152 ) 

153 brightMagCut = pexConfig.Field( 

154 doc="Bright limit of catalog entries to include.", dtype=float, default=10.0 

155 ) 

156 faintMagCut = pexConfig.Field( 

157 doc="Faint limit of catalog entries to include.", dtype=float, default=30.0 

158 ) 

159 selectExtended = pexConfig.Field( 

160 doc="Whether to select extended sources", dtype=bool, default=False 

161 ) 

162 doApplyExternalSkyWcs = pexConfig.Field( 

163 doc="Whether or not to use the external wcs.", dtype=bool, default=False 

164 ) 

165 useGlobalExternalSkyWcs = pexConfig.Field( 

166 doc="Whether or not to use the global external wcs.", dtype=bool, default=False 

167 ) 

168 doApplyExternalPhotoCalib = pexConfig.Field( 

169 doc="Whether or not to use the external photoCalib.", dtype=bool, default=False 

170 ) 

171 useGlobalExternalPhotoCalib = pexConfig.Field( 

172 doc="Whether or not to use the global external photoCalib.", 

173 dtype=bool, 

174 default=False, 

175 ) 

176 

177 

178class MatchedBaseTask(pipeBase.PipelineTask): 

179 

180 ConfigClass = MatchedBaseConfig 

181 _DefaultName = "matchedBaseTask" 

182 

183 def __init__(self, config: MatchedBaseConfig, *args, **kwargs): 

184 super().__init__(*args, config=config, **kwargs) 

185 self.radius = self.config.match_radius 

186 self.level = "patch" 

187 

188 def run( 

189 self, 

190 sourceCatalogs, 

191 photoCalibs, 

192 astromCalibs, 

193 dataIds, 

194 wcs, 

195 box, 

196 doApplyExternalSkyWcs=False, 

197 doApplyExternalPhotoCalib=False, 

198 ): 

199 self.log.info("Running catalog matching") 

200 radius = geom.Angle(self.radius, geom.arcseconds) 

201 if len(sourceCatalogs) < 2: 

202 self.log.warning("%s valid input catalogs: ", len(sourceCatalogs)) 

203 out_matched = afwTable.SimpleCatalog() 

204 else: 

205 srcvis, matched = matchCatalogs( 

206 sourceCatalogs, photoCalibs, astromCalibs, dataIds, radius, 

207 self.config, logger=self.log 

208 ) 

209 # Trim the output to the patch bounding box 

210 out_matched = type(matched)(matched.schema) 

211 self.log.info("%s sources in matched catalog.", len(matched)) 

212 for record in matched: 

213 if box.contains(wcs.skyToPixel(record.getCoord())): 

214 out_matched.append(record) 

215 self.log.info( 

216 "%s sources when trimmed to %s boundaries.", len(out_matched), self.level 

217 ) 

218 return pipeBase.Struct(outputCatalog=out_matched) 

219 

220 def get_box_wcs(self, skymap, oid): 

221 tract_info = skymap.generateTract(oid["tract"]) 

222 wcs = tract_info.getWcs() 

223 patch_info = tract_info.getPatchInfo(oid["patch"]) 

224 patch_box = patch_info.getInnerBBox() 

225 self.log.info("Running tract: %s and patch: %s", oid["tract"], oid["patch"]) 

226 return patch_box, wcs 

227 

228 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

229 inputs = butlerQC.get(inputRefs) 

230 oid = outputRefs.outputCatalog.dataId.byName() 

231 skymap = inputs["skyMap"] 

232 del inputs["skyMap"] 

233 box, wcs = self.get_box_wcs(skymap, oid) 

234 # Cast to float to handle fractional pixels 

235 box = geom.Box2D(box) 

236 inputs["dataIds"] = [ 

237 butlerQC.registry.expandDataId(el.dataId) for el in inputRefs.sourceCatalogs 

238 ] 

239 inputs["wcs"] = wcs 

240 inputs["box"] = box 

241 inputs["doApplyExternalSkyWcs"] = self.config.doApplyExternalSkyWcs 

242 inputs["doApplyExternalPhotoCalib"] = self.config.doApplyExternalPhotoCalib 

243 

244 if self.config.doApplyExternalPhotoCalib: 

245 if self.config.useGlobalExternalPhotoCalib: 

246 externalPhotoCalibCatalog = inputs.pop( 

247 "externalPhotoCalibGlobalCatalog" 

248 ) 

249 else: 

250 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog") 

251 

252 flatPhotoCalibList = np.hstack(externalPhotoCalibCatalog) 

253 visitPhotoCalibList = np.array( 

254 [calib["visit"] for calib in flatPhotoCalibList] 

255 ) 

256 detectorPhotoCalibList = np.array( 

257 [calib["id"] for calib in flatPhotoCalibList] 

258 ) 

259 

260 if self.config.doApplyExternalSkyWcs: 

261 if self.config.useGlobalExternalSkyWcs: 

262 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog") 

263 else: 

264 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog") 

265 

266 flatSkyWcsList = np.hstack(externalSkyWcsCatalog) 

267 visitSkyWcsList = np.array([calib["visit"] for calib in flatSkyWcsList]) 

268 detectorSkyWcsList = np.array([calib["id"] for calib in flatSkyWcsList]) 

269 

270 remove_indices = [] 

271 

272 if self.config.doApplyExternalPhotoCalib: 

273 for i in range(len(inputs["dataIds"])): 

274 dataId = inputs["dataIds"][i] 

275 detector = dataId["detector"] 

276 visit = dataId["visit"] 

277 calib_find = (visitPhotoCalibList == visit) & ( 

278 detectorPhotoCalibList == detector 

279 ) 

280 if np.sum(calib_find) < 1: 

281 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog " 

282 "for visit %s and will not be used.", 

283 detector, visit) 

284 inputs["photoCalibs"][i] = None 

285 remove_indices.append(i) 

286 else: 

287 row = flatPhotoCalibList[calib_find] 

288 externalPhotoCalib = row[0].getPhotoCalib() 

289 inputs["photoCalibs"][i] = externalPhotoCalib 

290 

291 if self.config.doApplyExternalSkyWcs: 

292 for i in range(len(inputs["dataIds"])): 

293 dataId = inputs["dataIds"][i] 

294 detector = dataId["detector"] 

295 visit = dataId["visit"] 

296 calib_find = (visitSkyWcsList == visit) & ( 

297 detectorSkyWcsList == detector 

298 ) 

299 if np.sum(calib_find) < 1: 

300 self.log.warning("Detector id %s not found in externalSkyWcsCatalog " 

301 "for visit %s and will not be used.", 

302 detector, visit) 

303 inputs["astromCalibs"][i] = None 

304 remove_indices.append(i) 

305 else: 

306 row = flatSkyWcsList[calib_find] 

307 externalSkyWcs = row[0].getWcs() 

308 inputs["astromCalibs"][i] = externalSkyWcs 

309 

310 # Remove datasets that didn't have matching external calibs 

311 remove_indices = np.unique(np.array(remove_indices)) 

312 if len(remove_indices) > 0: 

313 for ind in sorted(remove_indices, reverse=True): 

314 del inputs['sourceCatalogs'][ind] 

315 del inputs['dataIds'][ind] 

316 del inputs['photoCalibs'][ind] 

317 del inputs['astromCalibs'][ind] 

318 

319 outputs = self.run(**inputs) 

320 butlerQC.put(outputs, outputRefs) 

321 

322 

323class MatchedTractBaseTask(MatchedBaseTask): 

324 

325 ConfigClass = MatchedBaseConfig 

326 _DefaultName = "matchedTractBaseTask" 

327 

328 def __init__(self, config: MatchedBaseConfig, *args, **kwargs): 

329 super().__init__(*args, config=config, **kwargs) 

330 self.radius = self.config.match_radius 

331 self.level = "tract" 

332 

333 def get_box_wcs(self, skymap, oid): 

334 tract_info = skymap.generateTract(oid["tract"]) 

335 wcs = tract_info.getWcs() 

336 tract_box = tract_info.getBBox() 

337 self.log.info("Running tract: %s", oid["tract"]) 

338 return tract_box, wcs