Coverage for python/lsst/faro/base/MatchedCatalogBase.py: 19%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

139 statements  

1# This file is part of faro. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import lsst.afw.table as afwTable 

23import lsst.pipe.base as pipeBase 

24import lsst.pex.config as pexConfig 

25import lsst.geom as geom 

26import numpy as np 

27 

28from lsst.faro.utils.matcher import matchCatalogs 

29 

30__all__ = ( 

31 "MatchedBaseConnections", 

32 "MatchedBaseConfig", 

33 "MatchedBaseTask", 

34 "MatchedTractBaseTask", 

35) 

36 

37 

38class MatchedBaseConnections( 

39 pipeBase.PipelineTaskConnections, 

40 dimensions=(), 

41 defaultTemplates={ 

42 "coaddName": "deep", 

43 "photoCalibName": "calexp.photoCalib", 

44 "wcsName": "calexp.wcs", 

45 "externalPhotoCalibName": "fgcm", 

46 "externalWcsName": "jointcal", 

47 }, 

48): 

49 sourceCatalogs = pipeBase.connectionTypes.Input( 

50 doc="Source catalogs to match up.", 

51 dimensions=("instrument", "visit", "detector", "band"), 

52 storageClass="SourceCatalog", 

53 name="src", 

54 multiple=True, 

55 ) 

56 photoCalibs = pipeBase.connectionTypes.Input( 

57 doc="Photometric calibration object.", 

58 dimensions=("instrument", "visit", "detector", "band"), 

59 storageClass="PhotoCalib", 

60 name="{photoCalibName}", 

61 multiple=True, 

62 ) 

63 astromCalibs = pipeBase.connectionTypes.Input( 

64 doc="WCS for the catalog.", 

65 dimensions=("instrument", "visit", "detector", "band"), 

66 storageClass="Wcs", 

67 name="{wcsName}", 

68 multiple=True, 

69 ) 

70 externalSkyWcsTractCatalog = pipeBase.connectionTypes.Input( 

71 doc=( 

72 "Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

73 "id for the catalog id, sorted on id for fast lookup." 

74 ), 

75 name="{externalWcsName}SkyWcsCatalog", 

76 storageClass="ExposureCatalog", 

77 dimensions=("instrument", "visit", "tract", "band"), 

78 multiple=True, 

79 ) 

80 externalSkyWcsGlobalCatalog = pipeBase.connectionTypes.Input( 

81 doc=( 

82 "Per-visit wcs calibrations computed globally (with no tract information). " 

83 "These catalogs use the detector id for the catalog id, sorted on id for " 

84 "fast lookup." 

85 ), 

86 name="{externalWcsName}SkyWcsCatalog", 

87 storageClass="ExposureCatalog", 

88 dimensions=("instrument", "visit", "band"), 

89 multiple=True, 

90 ) 

91 externalPhotoCalibTractCatalog = pipeBase.connectionTypes.Input( 

92 doc=( 

93 "Per-tract, per-visit photometric calibrations. These catalogs use the " 

94 "detector id for the catalog id, sorted on id for fast lookup." 

95 ), 

96 name="{externalPhotoCalibName}PhotoCalibCatalog", 

97 storageClass="ExposureCatalog", 

98 dimensions=("instrument", "visit", "tract", "band"), 

99 multiple=True, 

100 ) 

101 externalPhotoCalibGlobalCatalog = pipeBase.connectionTypes.Input( 

102 doc=( 

103 "Per-visit photometric calibrations computed globally (with no tract " 

104 "information). These catalogs use the detector id for the catalog id, " 

105 "sorted on id for fast lookup." 

106 ), 

107 name="{externalPhotoCalibName}PhotoCalibCatalog", 

108 storageClass="ExposureCatalog", 

109 dimensions=("instrument", "visit", "band"), 

110 multiple=True, 

111 ) 

112 skyMap = pipeBase.connectionTypes.Input( 

113 doc="Input definition of geometry/bbox and projection/wcs for warped exposures", 

114 name="skyMap", 

115 storageClass="SkyMap", 

116 dimensions=("skymap",), 

117 ) 

118 

119 def __init__(self, *, config=None): 

120 super().__init__(config=config) 

121 if config.doApplyExternalSkyWcs: 

122 if config.useGlobalExternalSkyWcs: 

123 self.inputs.remove("externalSkyWcsTractCatalog") 

124 else: 

125 self.inputs.remove("externalSkyWcsGlobalCatalog") 

126 else: 

127 self.inputs.remove("externalSkyWcsTractCatalog") 

128 self.inputs.remove("externalSkyWcsGlobalCatalog") 

129 if config.doApplyExternalPhotoCalib: 

130 if config.useGlobalExternalPhotoCalib: 

131 self.inputs.remove("externalPhotoCalibTractCatalog") 

132 else: 

133 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

134 else: 

135 self.inputs.remove("externalPhotoCalibTractCatalog") 

136 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

137 

138 

139class MatchedBaseConfig( 

140 pipeBase.PipelineTaskConfig, pipelineConnections=MatchedBaseConnections 

141): 

142 match_radius = pexConfig.Field( 

143 doc="Match radius in arcseconds.", dtype=float, default=1 

144 ) 

145 doApplyExternalSkyWcs = pexConfig.Field( 

146 doc="Whether or not to use the external wcs.", dtype=bool, default=False 

147 ) 

148 useGlobalExternalSkyWcs = pexConfig.Field( 

149 doc="Whether or not to use the global external wcs.", dtype=bool, default=False 

150 ) 

151 doApplyExternalPhotoCalib = pexConfig.Field( 

152 doc="Whether or not to use the external photoCalib.", dtype=bool, default=False 

153 ) 

154 useGlobalExternalPhotoCalib = pexConfig.Field( 

155 doc="Whether or not to use the global external photoCalib.", 

156 dtype=bool, 

157 default=False, 

158 ) 

159 

160 

161class MatchedBaseTask(pipeBase.PipelineTask): 

162 

163 ConfigClass = MatchedBaseConfig 

164 _DefaultName = "matchedBaseTask" 

165 

166 def __init__(self, config: MatchedBaseConfig, *args, **kwargs): 

167 super().__init__(*args, config=config, **kwargs) 

168 self.radius = self.config.match_radius 

169 self.level = "patch" 

170 

171 def run( 

172 self, 

173 sourceCatalogs, 

174 photoCalibs, 

175 astromCalibs, 

176 dataIds, 

177 wcs, 

178 box, 

179 doApplyExternalSkyWcs=False, 

180 doApplyExternalPhotoCalib=False, 

181 ): 

182 self.log.info("Running catalog matching") 

183 radius = geom.Angle(self.radius, geom.arcseconds) 

184 if len(sourceCatalogs) < 2: 

185 self.log.warning("%s valid input catalogs: ", len(sourceCatalogs)) 

186 out_matched = afwTable.SimpleCatalog() 

187 else: 

188 srcvis, matched = matchCatalogs( 

189 sourceCatalogs, photoCalibs, astromCalibs, dataIds, radius, logger=self.log 

190 ) 

191 # Trim the output to the patch bounding box 

192 out_matched = type(matched)(matched.schema) 

193 self.log.info("%s sources in matched catalog.", len(matched)) 

194 for record in matched: 

195 if box.contains(wcs.skyToPixel(record.getCoord())): 

196 out_matched.append(record) 

197 self.log.info( 

198 "%s sources when trimmed to %s boundaries.", len(out_matched), self.level 

199 ) 

200 return pipeBase.Struct(outputCatalog=out_matched) 

201 

202 def get_box_wcs(self, skymap, oid): 

203 tract_info = skymap.generateTract(oid["tract"]) 

204 wcs = tract_info.getWcs() 

205 patch_info = tract_info.getPatchInfo(oid["patch"]) 

206 patch_box = patch_info.getInnerBBox() 

207 self.log.info("Running tract: %s and patch: %s", oid["tract"], oid["patch"]) 

208 return patch_box, wcs 

209 

210 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

211 inputs = butlerQC.get(inputRefs) 

212 oid = outputRefs.outputCatalog.dataId.byName() 

213 skymap = inputs["skyMap"] 

214 del inputs["skyMap"] 

215 box, wcs = self.get_box_wcs(skymap, oid) 

216 # Cast to float to handle fractional pixels 

217 box = geom.Box2D(box) 

218 inputs["dataIds"] = [ 

219 butlerQC.registry.expandDataId(el.dataId) for el in inputRefs.sourceCatalogs 

220 ] 

221 inputs["wcs"] = wcs 

222 inputs["box"] = box 

223 inputs["doApplyExternalSkyWcs"] = self.config.doApplyExternalSkyWcs 

224 inputs["doApplyExternalPhotoCalib"] = self.config.doApplyExternalPhotoCalib 

225 

226 if self.config.doApplyExternalPhotoCalib: 

227 if self.config.useGlobalExternalPhotoCalib: 

228 externalPhotoCalibCatalog = inputs.pop( 

229 "externalPhotoCalibGlobalCatalog" 

230 ) 

231 else: 

232 externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog") 

233 

234 flatPhotoCalibList = np.hstack(externalPhotoCalibCatalog) 

235 visitPhotoCalibList = np.array( 

236 [calib["visit"] for calib in flatPhotoCalibList] 

237 ) 

238 detectorPhotoCalibList = np.array( 

239 [calib["id"] for calib in flatPhotoCalibList] 

240 ) 

241 

242 if self.config.doApplyExternalSkyWcs: 

243 if self.config.useGlobalExternalSkyWcs: 

244 externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog") 

245 else: 

246 externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog") 

247 

248 flatSkyWcsList = np.hstack(externalSkyWcsCatalog) 

249 visitSkyWcsList = np.array([calib["visit"] for calib in flatSkyWcsList]) 

250 detectorSkyWcsList = np.array([calib["id"] for calib in flatSkyWcsList]) 

251 

252 remove_indices = [] 

253 

254 if self.config.doApplyExternalPhotoCalib: 

255 for i in range(len(inputs["dataIds"])): 

256 dataId = inputs["dataIds"][i] 

257 detector = dataId["detector"] 

258 visit = dataId["visit"] 

259 calib_find = (visitPhotoCalibList == visit) & ( 

260 detectorPhotoCalibList == detector 

261 ) 

262 if np.sum(calib_find) < 1: 

263 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog " 

264 "for visit %s and will not be used.", 

265 detector, visit) 

266 inputs["photoCalibs"][i] = None 

267 remove_indices.append(i) 

268 else: 

269 row = flatPhotoCalibList[calib_find] 

270 externalPhotoCalib = row[0].getPhotoCalib() 

271 inputs["photoCalibs"][i] = externalPhotoCalib 

272 

273 if self.config.doApplyExternalSkyWcs: 

274 for i in range(len(inputs["dataIds"])): 

275 dataId = inputs["dataIds"][i] 

276 detector = dataId["detector"] 

277 visit = dataId["visit"] 

278 calib_find = (visitSkyWcsList == visit) & ( 

279 detectorSkyWcsList == detector 

280 ) 

281 if np.sum(calib_find) < 1: 

282 self.log.warning("Detector id %s not found in externalSkyWcsCatalog " 

283 "for visit %s and will not be used.", 

284 detector, visit) 

285 inputs["astromCalibs"][i] = None 

286 remove_indices.append(i) 

287 else: 

288 row = flatSkyWcsList[calib_find] 

289 externalSkyWcs = row[0].getWcs() 

290 inputs["astromCalibs"][i] = externalSkyWcs 

291 

292 # Remove datasets that didn't have matching external calibs 

293 remove_indices = np.unique(np.array(remove_indices)) 

294 if len(remove_indices) > 0: 

295 for ind in sorted(remove_indices, reverse=True): 

296 del inputs['sourceCatalogs'][ind] 

297 del inputs['dataIds'][ind] 

298 del inputs['photoCalibs'][ind] 

299 del inputs['astromCalibs'][ind] 

300 

301 outputs = self.run(**inputs) 

302 butlerQC.put(outputs, outputRefs) 

303 

304 

305class MatchedTractBaseTask(MatchedBaseTask): 

306 

307 ConfigClass = MatchedBaseConfig 

308 _DefaultName = "matchedTractBaseTask" 

309 

310 def __init__(self, config: MatchedBaseConfig, *args, **kwargs): 

311 super().__init__(*args, config=config, **kwargs) 

312 self.radius = self.config.match_radius 

313 self.level = "tract" 

314 

315 def get_box_wcs(self, skymap, oid): 

316 tract_info = skymap.generateTract(oid["tract"]) 

317 wcs = tract_info.getWcs() 

318 tract_box = tract_info.getBBox() 

319 self.log.info("Running tract: %s", oid["tract"]) 

320 return tract_box, wcs