Coverage for python/lsst/pipe/tasks/loadReferenceCatalog.py: 15%

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of pipe_tasks. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Load a full reference catalog in numpy/table/dataframe format. 

24 

25This task will load multi-band reference objects, apply a reference selector, 

26and apply color terms. 

27""" 

28import numpy as np 

29from astropy import units 

30 

31import lsst.pex.config as pexConfig 

32import lsst.pipe.base as pipeBase 

33from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, ReferenceSourceSelectorTask 

34from lsst.meas.algorithms import getRefFluxField 

35from lsst.pipe.tasks.colorterms import ColortermLibrary 

36from lsst.afw.image import abMagErrFromFluxErr 

37from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig 

38 

39import lsst.geom 

40 

41__all__ = ['LoadReferenceCatalogConfig', 'LoadReferenceCatalogTask'] 

42 

43 

44class LoadReferenceCatalogConfig(pexConfig.Config): 

45 """Config for LoadReferenceCatalogTask""" 

46 refObjLoader = pexConfig.ConfigurableField( 

47 target=LoadIndexedReferenceObjectsTask, 

48 doc="Reference object loader for photometry", 

49 ) 

50 doApplyColorTerms = pexConfig.Field( 

51 doc=("Apply photometric color terms to reference stars? " 

52 "Requires that colorterms be set to a ColorTermLibrary"), 

53 dtype=bool, 

54 default=True 

55 ) 

56 colorterms = pexConfig.ConfigField( 

57 doc="Library of photometric reference catalog name to color term dict.", 

58 dtype=ColortermLibrary, 

59 ) 

60 referenceSelector = pexConfig.ConfigurableField( 

61 target=ReferenceSourceSelectorTask, 

62 doc="Selection of reference sources", 

63 ) 

64 doReferenceSelection = pexConfig.Field( 

65 doc="Run the reference selector on the reference catalog?", 

66 dtype=bool, 

67 default=True 

68 ) 

69 

70 def validate(self): 

71 super().validate() 

72 if self.doApplyColorTerms and len(self.colorterms.data) == 0: 

73 msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary." 

74 raise pexConfig.FieldValidationError(LoadReferenceCatalogConfig.colorterms, self, msg) 

75 

76 

77class LoadReferenceCatalogTask(pipeBase.Task): 

78 """Load multi-band reference objects from a reference catalog. 

79 

80 Parameters 

81 ---------- 

82 dataIds : iterable of `lsst.daf.butler.dataId`, optional 

83 An iterable object of dataIds which point to reference catalogs 

84 in a Gen3 repository. Required for Gen3. 

85 refCats : iterable of `lsst.daf.butler.DeferredDatasetHandle`, optional 

86 An iterable object of dataset refs for reference catalogs in 

87 a Gen3 repository. Required for Gen3. 

88 butler : `lsst.daf.persistence.Butler`, optional 

89 A Gen2 butler. Required for Gen2. 

90 """ 

91 ConfigClass = LoadReferenceCatalogConfig 

92 _DefaultName = "loadReferenceCatalog" 

93 

94 def __init__(self, dataIds=None, refCats=None, butler=None, **kwargs): 

95 if dataIds is not None and refCats is not None: 

96 pipeBase.Task.__init__(self, **kwargs) 

97 refConfig = self.config.refObjLoader 

98 self.refObjLoader = ReferenceObjectLoader(dataIds=dataIds, 

99 refCats=refCats, 

100 config=refConfig, 

101 log=self.log) 

102 elif butler is not None: 

103 raise RuntimeError("LoadReferenceCatalogTask does not support Gen2 Butlers.") 

104 else: 

105 raise RuntimeError("Must instantiate LoadReferenceCatalogTask with " 

106 "dataIds and refCats (Gen3)") 

107 

108 if self.config.doReferenceSelection: 

109 self.makeSubtask('referenceSelector') 

110 self._fluxFilters = None 

111 self._fluxFields = None 

112 self._referenceFilter = None 

113 

114 def getPixelBoxCatalog(self, bbox, wcs, filterList, epoch=None, 

115 bboxToSpherePadding=None): 

116 """Get a multi-band reference catalog by specifying a bounding box and WCS. 

117 

118 The catalog will be in `numpy.ndarray`, with positions proper-motion 

119 corrected to "epoch" (if specified, and if the reference catalog has 

120 proper motions); sources cut on a reference selector (if 

121 "config.doReferenceSelection = True"); and color-terms applied (if 

122 "config.doApplyColorTerms = True"). 

123 

124 The format of the reference catalog will be of the format: 

125 

126 dtype = [('ra', 'np.float64'), 

127 ('dec', 'np.float64'), 

128 ('refMag', 'np.float32', (len(filterList), )), 

129 ('refMagErr', 'np.float32', (len(filterList), ))] 

130 

131 Reference magnitudes (AB) and errors will be 99 for non-detections 

132 in a given band. 

133 

134 Parameters 

135 ---------- 

136 bbox : `lsst.geom.Box2I` 

137 Box which bounds a region in pixel space. 

138 wcs : `lsst.afw.geom.SkyWcs` 

139 Wcs object defining the pixel to sky (and reverse) transform for 

140 the supplied bbox. 

141 filterList : `List` [ `str` ] 

142 List of camera physicalFilter names to retrieve magnitudes. 

143 epoch : `astropy.time.Time`, optional 

144 Epoch to which to correct proper motion and parallax 

145 (if available), or `None` to not apply such corrections. 

146 bboxToSpherePadding : `int`, optional 

147 Padding to account for translating a set of corners into a 

148 spherical (convex) boundary that is certain to encompass the 

149 entire area covered by the bbox. 

150 

151 Returns 

152 ------- 

153 refCat : `numpy.ndarray` 

154 Reference catalog. 

155 """ 

156 # Check if we have previously cached values for the fluxFields 

157 if self._fluxFilters is None or self._fluxFilters != filterList: 

158 center = wcs.pixelToSky(bbox.getCenter()) 

159 self._determineFluxFields(center, filterList) 

160 

161 skyBox = self.refObjLoader.loadPixelBox(bbox, wcs, self._referenceFilter, 

162 epoch=epoch, 

163 bboxToSpherePadding=bboxToSpherePadding) 

164 

165 if not skyBox.refCat.isContiguous(): 

166 refCat = skyBox.refCat.copy(deep=True) 

167 else: 

168 refCat = skyBox.refCat 

169 

170 return self._formatCatalog(refCat, filterList) 

171 

172 def getSkyCircleCatalog(self, center, radius, filterList, epoch=None, 

173 catalogFormat='numpy'): 

174 """Get a multi-band reference catalog by specifying a center and radius. 

175 

176 The catalog will be in `numpy.ndarray`, with positions proper-motion 

177 corrected to "epoch" (if specified, and if the reference catalog has 

178 proper motions); sources cut on a reference selector (if 

179 "config.doReferenceSelection = True"); and color-terms applied (if 

180 "config.doApplyColorTerms = True"). 

181 

182 The format of the reference catalog will be of the format: 

183 

184 dtype = [('ra', 'np.float64'), 

185 ('dec', 'np.float64'), 

186 ('refMag', 'np.float32', (len(filterList), )), 

187 ('refMagErr', 'np.float32', (len(filterList), ))] 

188 

189 Reference magnitudes (AB) and errors will be 99 for non-detections 

190 in a given band. 

191 

192 Parameters 

193 ---------- 

194 center : `lsst.geom.SpherePoint` 

195 Point defining the center of the circular region. 

196 radius : `lsst.geom.Angle` 

197 Defines the angular radius of the circular region. 

198 filterList : `List` [ `str` ] 

199 List of camera physicalFilter names to retrieve magnitudes. 

200 epoch : `astropy.time.Time`, optional 

201 Epoch to which to correct proper motion and parallax 

202 (if available), or `None` to not apply such corrections. 

203 

204 Returns 

205 ------- 

206 refCat : `numpy.ndarray` 

207 Reference catalog. 

208 """ 

209 # Check if we have previously cached values for the fluxFields 

210 if self._fluxFilters is None or self._fluxFilters != filterList: 

211 self._determineFluxFields(center, filterList) 

212 

213 skyCircle = self.refObjLoader.loadSkyCircle(center, radius, 

214 self._referenceFilter, 

215 epoch=epoch) 

216 

217 if not skyCircle.refCat.isContiguous(): 

218 refCat = skyCircle.refCat.copy(deep=True) 

219 else: 

220 refCat = skyCircle.refCat 

221 

222 return self._formatCatalog(refCat, filterList) 

223 

224 def _formatCatalog(self, refCat, filterList): 

225 """Format a reference afw table into the final format. 

226 

227 This method applies reference selections and color terms as specified 

228 by the config. 

229 

230 Parameters 

231 ---------- 

232 refCat : `lsst.afw.table.SourceCatalog` 

233 Reference catalog in afw format. 

234 filterList : `list` [`str`] 

235 List of camera physicalFilter names to apply color terms. 

236 

237 Returns 

238 ------- 

239 refCat : `numpy.ndarray` 

240 Reference catalog. 

241 """ 

242 if self.config.doReferenceSelection: 

243 goodSources = self.referenceSelector.selectSources(refCat) 

244 selected = goodSources.selected 

245 else: 

246 selected = np.ones(len(refCat), dtype=bool) 

247 

248 npRefCat = np.zeros(np.sum(selected), dtype=[('ra', 'f8'), 

249 ('dec', 'f8'), 

250 ('refMag', 'f4', (len(filterList), )), 

251 ('refMagErr', 'f4', (len(filterList), ))]) 

252 

253 if npRefCat.size == 0: 

254 # Return an empty catalog if we don't have any selected sources. 

255 return npRefCat 

256 

257 # Natively "coord_ra" and "coord_dec" are stored in radians. 

258 # Doing this as an array rather than by row with the coord access is 

259 # approximately 600x faster. 

260 npRefCat['ra'] = np.rad2deg(refCat['coord_ra'][selected]) 

261 npRefCat['dec'] = np.rad2deg(refCat['coord_dec'][selected]) 

262 

263 # Default (unset) values are 99.0 

264 npRefCat['refMag'][:, :] = 99.0 

265 npRefCat['refMagErr'][:, :] = 99.0 

266 

267 if self.config.doApplyColorTerms: 

268 if isinstance(self.refObjLoader, ReferenceObjectLoader): 

269 # Gen3 

270 refCatName = self.refObjLoader.config.value.ref_dataset_name 

271 else: 

272 # Gen2 

273 refCatName = self.refObjLoader.ref_dataset_name 

274 

275 for i, (filterName, fluxField) in enumerate(zip(self._fluxFilters, self._fluxFields)): 

276 if fluxField is None: 

277 # There is no matching reference band. 

278 # This will leave the column filled with 99s 

279 continue 

280 self.log.debug("Applying color terms for filterName='%s'", filterName) 

281 

282 colorterm = self.config.colorterms.getColorterm(filterName, refCatName, doRaise=True) 

283 

284 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat) 

285 

286 # nan_to_num replaces nans with zeros, and this ensures 

287 # that we select magnitudes that both filter out nans and are 

288 # not very large (corresponding to very small fluxes), as "99" 

289 # is a commen sentinel for illegal magnitudes. 

290 good, = np.where((np.nan_to_num(refMag[selected], nan=99.0) < 90.0) 

291 & (np.nan_to_num(refMagErr[selected], nan=99.0) < 90.0) 

292 & (np.nan_to_num(refMagErr[selected]) > 0.0)) 

293 

294 npRefCat['refMag'][good, i] = refMag[selected][good] 

295 npRefCat['refMagErr'][good, i] = refMagErr[selected][good] 

296 else: 

297 # No color terms to apply 

298 for i, (filterName, fluxField) in enumerate(zip(self._fluxFilters, self._fluxFields)): 

299 # nan_to_num replaces nans with zeros, and this ensures that 

300 # we select fluxes that both filter out nans and are positive. 

301 good, = np.where((np.nan_to_num(refCat[fluxField][selected]) > 0.0) 

302 & (np.nan_to_num(refCat[fluxField+'Err'][selected]) > 0.0)) 

303 refMag = (refCat[fluxField][selected][good]*units.nJy).to_value(units.ABmag) 

304 refMagErr = abMagErrFromFluxErr(refCat[fluxField+'Err'][selected][good], 

305 refCat[fluxField][selected][good]) 

306 npRefCat['refMag'][good, i] = refMag 

307 npRefCat['refMagErr'][good, i] = refMagErr 

308 

309 return npRefCat 

310 

311 def _determineFluxFields(self, center, filterList): 

312 """Determine the flux field names for a reference catalog. 

313 

314 This method sets self._fluxFields, self._referenceFilter. 

315 

316 Parameters 

317 ---------- 

318 center : `lsst.geom.SpherePoint` 

319 The center around which to load test sources. 

320 filterList : `list` [`str`] 

321 List of camera physicalFilter names. 

322 """ 

323 # Search for a good filter to use to load the reference catalog 

324 # via the refObjLoader task which requires a valid filterName 

325 foundReferenceFilter = False 

326 

327 # Store the original config 

328 _config = self.refObjLoader.config 

329 

330 configTemp = LoadReferenceObjectsConfig() 

331 configIntersection = {k: getattr(self.refObjLoader.config, k) 

332 for k, v in self.refObjLoader.config.toDict().items() 

333 if (k in configTemp.keys() and k != "connections")} 

334 # We must turn off the proper motion checking to find the refFilter. 

335 configIntersection['requireProperMotion'] = False 

336 configTemp.update(**configIntersection) 

337 

338 self.refObjLoader.config = configTemp 

339 

340 for filterName in filterList: 

341 if self.config.refObjLoader.anyFilterMapsToThis is not None: 

342 refFilterName = self.config.refObjLoader.anyFilterMapsToThis 

343 else: 

344 refFilterName = self.config.refObjLoader.filterMap.get(filterName) 

345 if refFilterName is None: 

346 continue 

347 try: 

348 results = self.refObjLoader.loadSkyCircle(center, 

349 0.05*lsst.geom.degrees, 

350 refFilterName) 

351 foundReferenceFilter = True 

352 self._referenceFilter = refFilterName 

353 break 

354 except RuntimeError as err: 

355 # This just means that the filterName wasn't listed 

356 # in the reference catalog. This is okay. 

357 if 'not find flux' in err.args[0]: 

358 # The filterName wasn't listed in the reference catalog. 

359 # This is not a fatal failure (yet) 

360 pass 

361 else: 

362 raise err 

363 

364 self.refObjLoader.config = _config 

365 

366 if not foundReferenceFilter: 

367 raise RuntimeError("Could not find any valid flux field(s) %s" % 

368 (", ".join(filterList))) 

369 

370 # Record self._fluxFilters for checks on subsequent calls 

371 self._fluxFilters = filterList 

372 

373 # Retrieve all the fluxField names 

374 self._fluxFields = [] 

375 for filterName in filterList: 

376 fluxField = None 

377 

378 if self.config.refObjLoader.anyFilterMapsToThis is not None: 

379 refFilterName = self.config.refObjLoader.anyFilterMapsToThis 

380 else: 

381 refFilterName = self.config.refObjLoader.filterMap.get(filterName) 

382 

383 if refFilterName is not None: 

384 try: 

385 fluxField = getRefFluxField(results.refCat.schema, filterName=refFilterName) 

386 except RuntimeError: 

387 # This flux field isn't available. Set to None 

388 fluxField = None 

389 

390 if fluxField is None: 

391 self.log.warning('No reference flux field for camera filter %s', filterName) 

392 

393 self._fluxFields.append(fluxField)