Coverage for python/lsst/pipe/tasks/loadReferenceCatalog.py: 16%

123 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-03-22 02:49 -0700

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Load a full reference catalog in numpy/table/dataframe format. 

23 

24This task will load multi-band reference objects, apply a reference selector, 

25and apply color terms. 

26""" 

27 

28__all__ = ['LoadReferenceCatalogConfig', 'LoadReferenceCatalogTask'] 

29 

30import numpy as np 

31from astropy import units 

32 

33import lsst.pex.config as pexConfig 

34import lsst.pipe.base as pipeBase 

35from lsst.meas.algorithms import ReferenceSourceSelectorTask 

36from lsst.meas.algorithms import getRefFluxField 

37from lsst.pipe.tasks.colorterms import ColortermLibrary 

38from lsst.afw.image import abMagErrFromFluxErr 

39from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig 

40 

41import lsst.geom 

42 

43 

44class LoadReferenceCatalogConfig(pexConfig.Config): 

45 """Config for LoadReferenceCatalogTask""" 

46 refObjLoader = pexConfig.ConfigField( 

47 dtype=LoadReferenceObjectsConfig, 

48 doc="Configuration for the reference object loader.", 

49 ) 

50 doApplyColorTerms = pexConfig.Field( 

51 doc=("Apply photometric color terms to reference stars? " 

52 "Requires that colorterms be set to a ColorTermLibrary"), 

53 dtype=bool, 

54 default=True 

55 ) 

56 colorterms = pexConfig.ConfigField( 

57 doc="Library of photometric reference catalog name to color term dict.", 

58 dtype=ColortermLibrary, 

59 ) 

60 referenceSelector = pexConfig.ConfigurableField( 

61 target=ReferenceSourceSelectorTask, 

62 doc="Selection of reference sources", 

63 ) 

64 doReferenceSelection = pexConfig.Field( 

65 doc="Run the reference selector on the reference catalog?", 

66 dtype=bool, 

67 default=True 

68 ) 

69 

70 def validate(self): 

71 super().validate() 

72 if self.doApplyColorTerms and len(self.colorterms.data) == 0: 

73 msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary." 

74 raise pexConfig.FieldValidationError(LoadReferenceCatalogConfig.colorterms, self, msg) 

75 

76 

77class LoadReferenceCatalogTask(pipeBase.Task): 

78 """Load multi-band reference objects from a reference catalog. 

79 

80 Parameters 

81 ---------- 

82 dataIds : iterable of `lsst.daf.butler.dataId` 

83 An iterable object of dataIds which point to reference catalogs 

84 in a Gen3 repository. Required for Gen3. 

85 refCats : iterable of `lsst.daf.butler.DeferredDatasetHandle` 

86 An iterable object of dataset refs for reference catalogs in 

87 a Gen3 repository. 

88 name : `str` 

89 The name of the refcat that this object will load. This name is used 

90 for applying colorterms, for example. 

91 

92 Raises 

93 ------ 

94 RuntimeError if dataIds or refCats is None. 

95 """ 

96 ConfigClass = LoadReferenceCatalogConfig 

97 _DefaultName = "loadReferenceCatalog" 

98 

99 def __init__(self, *, dataIds, refCats, name, **kwargs): 

100 pipeBase.Task.__init__(self, **kwargs) 

101 refConfig = self.config.refObjLoader 

102 self.refObjLoader = ReferenceObjectLoader(dataIds=dataIds, 

103 refCats=refCats, 

104 name=name, 

105 config=refConfig, 

106 log=self.log) 

107 

108 if self.config.doReferenceSelection: 

109 self.makeSubtask('referenceSelector') 

110 self._fluxFilters = None 

111 self._fluxFields = None 

112 self._referenceFilter = None 

113 

114 def getPixelBoxCatalog(self, bbox, wcs, filterList, epoch=None, 

115 bboxToSpherePadding=None): 

116 """Get a multi-band reference catalog by specifying a bounding box and WCS. 

117 

118 The catalog will be in `numpy.ndarray`, with positions proper-motion 

119 corrected to "epoch" (if specified, and if the reference catalog has 

120 proper motions); sources cut on a reference selector (if 

121 "config.doReferenceSelection = True"); and color-terms applied (if 

122 "config.doApplyColorTerms = True"). 

123 

124 The format of the reference catalog will be of the format: 

125 

126 dtype = [('ra', 'np.float64'), 

127 ('dec', 'np.float64'), 

128 ('refMag', 'np.float32', (len(filterList), )), 

129 ('refMagErr', 'np.float32', (len(filterList), ))] 

130 

131 Reference magnitudes (AB) and errors will be 99 for non-detections 

132 in a given band. 

133 

134 Parameters 

135 ---------- 

136 bbox : `lsst.geom.Box2I` 

137 Box which bounds a region in pixel space. 

138 wcs : `lsst.afw.geom.SkyWcs` 

139 Wcs object defining the pixel to sky (and reverse) transform for 

140 the supplied bbox. 

141 filterList : `List` [ `str` ] 

142 List of camera physicalFilter names to retrieve magnitudes. 

143 epoch : `astropy.time.Time`, optional 

144 Epoch to which to correct proper motion and parallax 

145 (if available), or `None` to not apply such corrections. 

146 bboxToSpherePadding : `int`, optional 

147 Padding to account for translating a set of corners into a 

148 spherical (convex) boundary that is certain to encompass the 

149 entire area covered by the bbox. 

150 

151 Returns 

152 ------- 

153 refCat : `numpy.ndarray` 

154 Reference catalog. 

155 """ 

156 # Check if we have previously cached values for the fluxFields 

157 if self._fluxFilters is None or self._fluxFilters != filterList: 

158 center = wcs.pixelToSky(bbox.getCenter()) 

159 self._determineFluxFields(center, filterList) 

160 

161 skyBox = self.refObjLoader.loadPixelBox(bbox, wcs, self._referenceFilter, 

162 epoch=epoch, 

163 bboxToSpherePadding=bboxToSpherePadding) 

164 

165 if not skyBox.refCat.isContiguous(): 

166 refCat = skyBox.refCat.copy(deep=True) 

167 else: 

168 refCat = skyBox.refCat 

169 

170 return self._formatCatalog(refCat, filterList) 

171 

172 def getSkyCircleCatalog(self, center, radius, filterList, epoch=None, 

173 catalogFormat='numpy'): 

174 """Get a multi-band reference catalog by specifying a center and radius. 

175 

176 The catalog will be in `numpy.ndarray`, with positions proper-motion 

177 corrected to "epoch" (if specified, and if the reference catalog has 

178 proper motions); sources cut on a reference selector (if 

179 "config.doReferenceSelection = True"); and color-terms applied (if 

180 "config.doApplyColorTerms = True"). 

181 

182 The format of the reference catalog will be of the format: 

183 

184 dtype = [('ra', 'np.float64'), 

185 ('dec', 'np.float64'), 

186 ('refMag', 'np.float32', (len(filterList), )), 

187 ('refMagErr', 'np.float32', (len(filterList), ))] 

188 

189 Reference magnitudes (AB) and errors will be 99 for non-detections 

190 in a given band. 

191 

192 Parameters 

193 ---------- 

194 center : `lsst.geom.SpherePoint` 

195 Point defining the center of the circular region. 

196 radius : `lsst.geom.Angle` 

197 Defines the angular radius of the circular region. 

198 filterList : `List` [ `str` ] 

199 List of camera physicalFilter names to retrieve magnitudes. 

200 epoch : `astropy.time.Time`, optional 

201 Epoch to which to correct proper motion and parallax 

202 (if available), or `None` to not apply such corrections. 

203 

204 Returns 

205 ------- 

206 refCat : `numpy.ndarray` 

207 Reference catalog. 

208 """ 

209 # Check if we have previously cached values for the fluxFields 

210 if self._fluxFilters is None or self._fluxFilters != filterList: 

211 self._determineFluxFields(center, filterList) 

212 

213 skyCircle = self.refObjLoader.loadSkyCircle(center, radius, 

214 self._referenceFilter, 

215 epoch=epoch) 

216 

217 if not skyCircle.refCat.isContiguous(): 

218 refCat = skyCircle.refCat.copy(deep=True) 

219 else: 

220 refCat = skyCircle.refCat 

221 

222 return self._formatCatalog(refCat, filterList) 

223 

224 def _formatCatalog(self, refCat, filterList): 

225 """Format a reference afw table into the final format. 

226 

227 This method applies reference selections and color terms as specified 

228 by the config. 

229 

230 Parameters 

231 ---------- 

232 refCat : `lsst.afw.table.SourceCatalog` 

233 Reference catalog in afw format. 

234 filterList : `list` [`str`] 

235 List of camera physicalFilter names to apply color terms. 

236 

237 Returns 

238 ------- 

239 refCat : `numpy.ndarray` 

240 Reference catalog. 

241 """ 

242 if self.config.doReferenceSelection: 

243 goodSources = self.referenceSelector.selectSources(refCat) 

244 selected = goodSources.selected 

245 else: 

246 selected = np.ones(len(refCat), dtype=bool) 

247 

248 npRefCat = np.zeros(np.sum(selected), dtype=[('ra', 'f8'), 

249 ('dec', 'f8'), 

250 ('refMag', 'f4', (len(filterList), )), 

251 ('refMagErr', 'f4', (len(filterList), ))]) 

252 

253 if npRefCat.size == 0: 

254 # Return an empty catalog if we don't have any selected sources. 

255 return npRefCat 

256 

257 # Natively "coord_ra" and "coord_dec" are stored in radians. 

258 # Doing this as an array rather than by row with the coord access is 

259 # approximately 600x faster. 

260 npRefCat['ra'] = np.rad2deg(refCat['coord_ra'][selected]) 

261 npRefCat['dec'] = np.rad2deg(refCat['coord_dec'][selected]) 

262 

263 # Default (unset) values are 99.0 

264 npRefCat['refMag'][:, :] = 99.0 

265 npRefCat['refMagErr'][:, :] = 99.0 

266 

267 if self.config.doApplyColorTerms: 

268 refCatName = self.refObjLoader.name 

269 

270 for i, (filterName, fluxField) in enumerate(zip(self._fluxFilters, self._fluxFields)): 

271 if fluxField is None: 

272 # There is no matching reference band. 

273 # This will leave the column filled with 99s 

274 continue 

275 self.log.debug("Applying color terms for filterName='%s'", filterName) 

276 

277 colorterm = self.config.colorterms.getColorterm(filterName, refCatName, doRaise=True) 

278 

279 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat) 

280 

281 # nan_to_num replaces nans with zeros, and this ensures 

282 # that we select magnitudes that both filter out nans and are 

283 # not very large (corresponding to very small fluxes), as "99" 

284 # is a commen sentinel for illegal magnitudes. 

285 good, = np.where((np.nan_to_num(refMag[selected], nan=99.0) < 90.0) 

286 & (np.nan_to_num(refMagErr[selected], nan=99.0) < 90.0) 

287 & (np.nan_to_num(refMagErr[selected]) > 0.0)) 

288 

289 npRefCat['refMag'][good, i] = refMag[selected][good] 

290 npRefCat['refMagErr'][good, i] = refMagErr[selected][good] 

291 else: 

292 # No color terms to apply 

293 for i, (filterName, fluxField) in enumerate(zip(self._fluxFilters, self._fluxFields)): 

294 # nan_to_num replaces nans with zeros, and this ensures that 

295 # we select fluxes that both filter out nans and are positive. 

296 good, = np.where((np.nan_to_num(refCat[fluxField][selected]) > 0.0) 

297 & (np.nan_to_num(refCat[fluxField+'Err'][selected]) > 0.0)) 

298 refMag = (refCat[fluxField][selected][good]*units.nJy).to_value(units.ABmag) 

299 refMagErr = abMagErrFromFluxErr(refCat[fluxField+'Err'][selected][good], 

300 refCat[fluxField][selected][good]) 

301 npRefCat['refMag'][good, i] = refMag 

302 npRefCat['refMagErr'][good, i] = refMagErr 

303 

304 return npRefCat 

305 

306 def _determineFluxFields(self, center, filterList): 

307 """Determine the flux field names for a reference catalog. 

308 

309 This method sets self._fluxFields, self._referenceFilter. 

310 

311 Parameters 

312 ---------- 

313 center : `lsst.geom.SpherePoint` 

314 The center around which to load test sources. 

315 filterList : `list` [`str`] 

316 List of camera physicalFilter names. 

317 """ 

318 # Search for a good filter to use to load the reference catalog 

319 # via the refObjLoader task which requires a valid filterName 

320 foundReferenceFilter = False 

321 

322 # Store the original config 

323 _config = self.refObjLoader.config 

324 

325 configTemp = LoadReferenceObjectsConfig() 

326 configIntersection = {k: getattr(self.refObjLoader.config, k) 

327 for k, v in self.refObjLoader.config.toDict().items() 

328 if (k in configTemp.keys() and k != "connections")} 

329 # We must turn off the proper motion checking to find the refFilter. 

330 configIntersection['requireProperMotion'] = False 

331 configTemp.update(**configIntersection) 

332 

333 self.refObjLoader.config = configTemp 

334 

335 for filterName in filterList: 

336 if self.config.refObjLoader.anyFilterMapsToThis is not None: 

337 refFilterName = self.config.refObjLoader.anyFilterMapsToThis 

338 else: 

339 refFilterName = self.config.refObjLoader.filterMap.get(filterName) 

340 if refFilterName is None: 

341 continue 

342 try: 

343 results = self.refObjLoader.loadSkyCircle(center, 

344 0.05*lsst.geom.degrees, 

345 refFilterName) 

346 foundReferenceFilter = True 

347 self._referenceFilter = refFilterName 

348 break 

349 except RuntimeError as err: 

350 # This just means that the filterName wasn't listed 

351 # in the reference catalog. This is okay. 

352 if 'not find flux' in err.args[0]: 

353 # The filterName wasn't listed in the reference catalog. 

354 # This is not a fatal failure (yet) 

355 pass 

356 else: 

357 raise err 

358 

359 self.refObjLoader.config = _config 

360 

361 if not foundReferenceFilter: 

362 raise RuntimeError("Could not find any valid flux field(s) %s" % 

363 (", ".join(filterList))) 

364 

365 # Record self._fluxFilters for checks on subsequent calls 

366 self._fluxFilters = filterList 

367 

368 # Retrieve all the fluxField names 

369 self._fluxFields = [] 

370 for filterName in filterList: 

371 fluxField = None 

372 

373 if self.config.refObjLoader.anyFilterMapsToThis is not None: 

374 refFilterName = self.config.refObjLoader.anyFilterMapsToThis 

375 else: 

376 refFilterName = self.config.refObjLoader.filterMap.get(filterName) 

377 

378 if refFilterName is not None: 

379 try: 

380 fluxField = getRefFluxField(results.refCat.schema, filterName=refFilterName) 

381 except RuntimeError: 

382 # This flux field isn't available. Set to None 

383 fluxField = None 

384 

385 if fluxField is None: 

386 self.log.warning('No reference flux field for camera filter %s', filterName) 

387 

388 self._fluxFields.append(fluxField)