Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of pipe_tasks. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Load a full reference catalog in numpy/table/dataframe format. 

24 

25This task will load multi-band reference objects, apply a reference selector, 

26and apply color terms. 

27""" 

28import numpy as np 

29from astropy import units 

30 

31import lsst.pex.config as pexConfig 

32import lsst.pipe.base as pipeBase 

33from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, ReferenceSourceSelectorTask 

34from lsst.meas.algorithms import getRefFluxField 

35from lsst.pipe.tasks.colorterms import ColortermLibrary 

36from lsst.afw.image import abMagErrFromFluxErr 

37from lsst.meas.algorithms import ReferenceObjectLoader 

38 

39import lsst.geom 

40 

41__all__ = ['LoadReferenceCatalogConfig', 'LoadReferenceCatalogTask'] 

42 

43 

44class LoadReferenceCatalogConfig(pexConfig.Config): 

45 """Config for LoadReferenceCatalogTask""" 

46 refObjLoader = pexConfig.ConfigurableField( 

47 target=LoadIndexedReferenceObjectsTask, 

48 doc="Reference object loader for photometry", 

49 ) 

50 doApplyColorTerms = pexConfig.Field( 

51 doc=("Apply photometric color terms to reference stars? " 

52 "Requires that colorterms be set to a ColorTermLibrary"), 

53 dtype=bool, 

54 default=True 

55 ) 

56 colorterms = pexConfig.ConfigField( 

57 doc="Library of photometric reference catalog name to color term dict.", 

58 dtype=ColortermLibrary, 

59 ) 

60 referenceSelector = pexConfig.ConfigurableField( 

61 target=ReferenceSourceSelectorTask, 

62 doc="Selection of reference sources", 

63 ) 

64 doReferenceSelection = pexConfig.Field( 

65 doc="Run the reference selector on the reference catalog?", 

66 dtype=bool, 

67 default=True 

68 ) 

69 

70 def validate(self): 

71 super().validate() 

72 if self.doApplyColorTerms and len(self.colorterms.data) == 0: 

73 msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary." 

74 raise pexConfig.FieldValidationError(LoadReferenceCatalogConfig.colorterms, self, msg) 

75 

76 

77class LoadReferenceCatalogTask(pipeBase.Task): 

78 """Load multi-band reference objects from a reference catalog. 

79 

80 Parameters 

81 ---------- 

82 dataIds : iterable of `lsst.daf.butler.dataId`, optional 

83 An iterable object of dataIds which point to reference catalogs 

84 in a Gen3 repository. Required for Gen3. 

85 refCats : iterable of `lsst.daf.butler.DeferredDatasetHandle`, optional 

86 An iterable object of dataset refs for reference catalogs in 

87 a Gen3 repository. Required for Gen3. 

88 butler : `lsst.daf.persistence.Butler`, optional 

89 A Gen2 butler. Required for Gen2. 

90 """ 

91 ConfigClass = LoadReferenceCatalogConfig 

92 _DefaultName = "loadReferenceCatalog" 

93 

94 def __init__(self, dataIds=None, refCats=None, butler=None, **kwargs): 

95 if dataIds is not None and refCats is not None: 

96 pipeBase.Task.__init__(self, **kwargs) 

97 refConfig = self.config.refObjLoader 

98 self.refObjLoader = ReferenceObjectLoader(dataIds=dataIds, 

99 refCats=refCats, 

100 config=refConfig, 

101 log=self.log) 

102 elif butler is not None: 

103 self.makeSubtask('refObjLoader', butler=butler) 

104 else: 

105 raise RuntimeError("Must instantiate LoadReferenceCatalogTask with " 

106 "dataIds/refCats (Gen3) or butler (Gen2)") 

107 

108 if self.config.doReferenceSelection: 

109 self.makeSubtask('referenceSelector') 

110 self._fluxFilters = None 

111 self._fluxFields = None 

112 self._referenceFilter = None 

113 

114 def getPixelBoxCatalog(self, bbox, wcs, filterList, epoch=None, 

115 bboxToSpherePadding=None): 

116 """Get a multi-band reference catalog by specifying a bounding box and WCS. 

117 

118 The catalog will be in `numpy.ndarray`, with positions proper-motion 

119 corrected to "epoch" (if specified, and if the reference catalog has 

120 proper motions); sources cut on a reference selector (if 

121 "config.doReferenceSelection = True"); and color-terms applied (if 

122 "config.doApplyColorTerms = True"). 

123 

124 The format of the reference catalog will be of the format: 

125 

126 dtype = [('ra', 'np.float64'), 

127 ('dec', 'np.float64'), 

128 ('refMag', 'np.float32', (len(filterList), )), 

129 ('refMagErr', 'np.float32', (len(filterList), ))] 

130 

131 Reference magnitudes (AB) and errors will be 99 for non-detections 

132 in a given band. 

133 

134 Parameters 

135 ---------- 

136 bbox : `lsst.geom.Box2I` 

137 Box which bounds a region in pixel space. 

138 wcs : `lsst.afw.geom.SkyWcs` 

139 Wcs object defining the pixel to sky (and reverse) transform for 

140 the supplied bbox. 

141 filterList : `List` [ `str` ] 

142 List of camera physicalFilter names to retrieve magnitudes. 

143 epoch : `astropy.time.Time`, optional 

144 Epoch to which to correct proper motion and parallax 

145 (if available), or `None` to not apply such corrections. 

146 bboxToSpherePadding : `int`, optional 

147 Padding to account for translating a set of corners into a 

148 spherical (convex) boundary that is certain to encompass the 

149 entire area covered by the bbox. 

150 

151 Returns 

152 ------- 

153 refCat : `numpy.ndarray` 

154 Reference catalog. 

155 """ 

156 # Check if we have previously cached values for the fluxFields 

157 if self._fluxFilters is None or self._fluxFilters != filterList: 

158 center = wcs.pixelToSky(bbox.getCenter()) 

159 self._determineFluxFields(center, filterList) 

160 

161 skyBox = self.refObjLoader.loadPixelBox(bbox, wcs, self._referenceFilter, 

162 epoch=epoch, 

163 bboxToSpherePadding=bboxToSpherePadding) 

164 

165 if not skyBox.refCat.isContiguous(): 

166 refCat = skyBox.refCat.copy(deep=True) 

167 else: 

168 refCat = skyBox.refCat 

169 

170 return self._formatCatalog(refCat, filterList) 

171 

172 def getSkyCircleCatalog(self, center, radius, filterList, epoch=None, 

173 catalogFormat='numpy'): 

174 """Get a multi-band reference catalog by specifying a center and radius. 

175 

176 The catalog will be in `numpy.ndarray`, with positions proper-motion 

177 corrected to "epoch" (if specified, and if the reference catalog has 

178 proper motions); sources cut on a reference selector (if 

179 "config.doReferenceSelection = True"); and color-terms applied (if 

180 "config.doApplyColorTerms = True"). 

181 

182 The format of the reference catalog will be of the format: 

183 

184 dtype = [('ra', 'np.float64'), 

185 ('dec', 'np.float64'), 

186 ('refMag', 'np.float32', (len(filterList), )), 

187 ('refMagErr', 'np.float32', (len(filterList), ))] 

188 

189 Reference magnitudes (AB) and errors will be 99 for non-detections 

190 in a given band. 

191 

192 Parameters 

193 ---------- 

194 center : `lsst.geom.SpherePoint` 

195 Point defining the center of the circular region. 

196 radius : `lsst.geom.Angle` 

197 Defines the angular radius of the circular region. 

198 filterList : `List` [ `str` ] 

199 List of camera physicalFilter names to retrieve magnitudes. 

200 epoch : `astropy.time.Time`, optional 

201 Epoch to which to correct proper motion and parallax 

202 (if available), or `None` to not apply such corrections. 

203 

204 Parameters 

205 ---------- 

206 refCat : `numpy.ndarray` 

207 Reference catalog. 

208 """ 

209 # Check if we have previously cached values for the fluxFields 

210 if self._fluxFilters is None or self._fluxFilters != filterList: 

211 self._determineFluxFields(center, filterList) 

212 

213 skyCircle = self.refObjLoader.loadSkyCircle(center, radius, 

214 self._referenceFilter, 

215 epoch=epoch) 

216 

217 if not skyCircle.refCat.isContiguous(): 

218 refCat = skyCircle.refCat.copy(deep=True) 

219 else: 

220 refCat = skyCircle.refCat 

221 

222 return self._formatCatalog(refCat, filterList) 

223 

224 def _formatCatalog(self, refCat, filterList): 

225 """Format a reference afw table into the final format. 

226 

227 This method applies reference selections and color terms as specified 

228 by the config. 

229 

230 Parameters 

231 ---------- 

232 refCat : `lsst.afw.table.SourceCatalog` 

233 Reference catalog in afw format. 

234 filterList : `list` [`str`] 

235 List of camera physicalFilter names to apply color terms. 

236 

237 Returns 

238 ------- 

239 refCat : `numpy.ndarray` 

240 Reference catalog. 

241 """ 

242 if self.config.doReferenceSelection: 

243 goodSources = self.referenceSelector.selectSources(refCat) 

244 selected = goodSources.selected 

245 else: 

246 selected = np.ones(len(refCat), dtype=bool) 

247 

248 npRefCat = np.zeros(np.sum(selected), dtype=[('ra', 'f8'), 

249 ('dec', 'f8'), 

250 ('refMag', 'f4', (len(filterList), )), 

251 ('refMagErr', 'f4', (len(filterList), ))]) 

252 

253 if npRefCat.size == 0: 

254 # Return an empty catalog if we don't have any selected sources. 

255 return npRefCat 

256 

257 # Natively "coord_ra" and "coord_dec" are stored in radians. 

258 # Doing this as an array rather than by row with the coord access is 

259 # approximately 600x faster. 

260 npRefCat['ra'] = np.rad2deg(refCat['coord_ra'][selected]) 

261 npRefCat['dec'] = np.rad2deg(refCat['coord_dec'][selected]) 

262 

263 # Default (unset) values are 99.0 

264 npRefCat['refMag'][:, :] = 99.0 

265 npRefCat['refMagErr'][:, :] = 99.0 

266 

267 if self.config.doApplyColorTerms: 

268 if isinstance(self.refObjLoader, ReferenceObjectLoader): 

269 # Gen3 

270 refCatName = self.refObjLoader.config.value.ref_dataset_name 

271 else: 

272 # Gen2 

273 refCatName = self.refObjLoader.ref_dataset_name 

274 

275 for i, (filterName, fluxField) in enumerate(zip(self._fluxFilters, self._fluxFields)): 

276 if fluxField is None: 

277 # There is no matching reference band. 

278 # This will leave the column filled with 99s 

279 continue 

280 self.log.debug("Applying color terms for filterName='%s'", filterName) 

281 

282 colorterm = self.config.colorterms.getColorterm(filterName, refCatName, doRaise=True) 

283 

284 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat) 

285 

286 # nan_to_num replaces nans with zeros, and this ensures 

287 # that we select magnitudes that both filter out nans and are 

288 # not very large (corresponding to very small fluxes), as "99" 

289 # is a commen sentinel for illegal magnitudes. 

290 good, = np.where((np.nan_to_num(refMag[selected], nan=99.0) < 90.0) 

291 & (np.nan_to_num(refMagErr[selected], nan=99.0) < 90.0) 

292 & (np.nan_to_num(refMagErr[selected]) > 0.0)) 

293 

294 npRefCat['refMag'][good, i] = refMag[selected][good] 

295 npRefCat['refMagErr'][good, i] = refMagErr[selected][good] 

296 else: 

297 # No color terms to apply 

298 for i, (filterName, fluxField) in enumerate(zip(self._fluxFilters, self._fluxFields)): 

299 # nan_to_num replaces nans with zeros, and this ensures that 

300 # we select fluxes that both filter out nans and are positive. 

301 good, = np.where((np.nan_to_num(refCat[fluxField][selected]) > 0.0) 

302 & (np.nan_to_num(refCat[fluxField+'Err'][selected]) > 0.0)) 

303 refMag = (refCat[fluxField][selected][good]*units.nJy).to_value(units.ABmag) 

304 refMagErr = abMagErrFromFluxErr(refCat[fluxField+'Err'][selected][good], 

305 refCat[fluxField][selected][good]) 

306 npRefCat['refMag'][good, i] = refMag 

307 npRefCat['refMagErr'][good, i] = refMagErr 

308 

309 return npRefCat 

310 

311 def _determineFluxFields(self, center, filterList): 

312 """Determine the flux field names for a reference catalog. 

313 

314 This method sets self._fluxFields, self._referenceFilter. 

315 

316 Parameters 

317 ---------- 

318 center : `lsst.geom.SpherePoint` 

319 The center around which to load test sources. 

320 filterList : `list` [`str`] 

321 List of camera physicalFilter names. 

322 """ 

323 # Search for a good filter to use to load the reference catalog 

324 # via the refObjLoader task which requires a valid filterName 

325 foundReferenceFilter = False 

326 # Temporarily turn off proper motion because we do not need it to 

327 # check the filter/flux names. 

328 _requireProperMotion = self.refObjLoader.config.requireProperMotion 

329 self.refObjLoader.config.requireProperMotion = False 

330 for filterName in filterList: 

331 if self.config.refObjLoader.anyFilterMapsToThis is not None: 

332 refFilterName = self.config.refObjLoader.anyFilterMapsToThis 

333 else: 

334 refFilterName = self.config.refObjLoader.filterMap.get(filterName) 

335 if refFilterName is None: 

336 continue 

337 try: 

338 results = self.refObjLoader.loadSkyCircle(center, 

339 0.05*lsst.geom.degrees, 

340 refFilterName) 

341 foundReferenceFilter = True 

342 self._referenceFilter = refFilterName 

343 break 

344 except RuntimeError as err: 

345 # This just means that the filterName wasn't listed 

346 # in the reference catalog. This is okay. 

347 if 'not find flux' in err.args[0]: 

348 # The filterName wasn't listed in the reference catalog. 

349 # This is not a fatal failure (yet) 

350 pass 

351 else: 

352 raise err 

353 self.refObjLoader.config.requireProperMotion = _requireProperMotion 

354 

355 if not foundReferenceFilter: 

356 raise RuntimeError("Could not find any valid flux field(s) %s" % 

357 (", ".join(filterList))) 

358 

359 # Record self._fluxFilters for checks on subsequent calls 

360 self._fluxFilters = filterList 

361 

362 # Retrieve all the fluxField names 

363 self._fluxFields = [] 

364 for filterName in filterList: 

365 fluxField = None 

366 

367 if self.config.refObjLoader.anyFilterMapsToThis is not None: 

368 refFilterName = self.config.refObjLoader.anyFilterMapsToThis 

369 else: 

370 refFilterName = self.config.refObjLoader.filterMap.get(filterName) 

371 

372 if refFilterName is not None: 

373 try: 

374 fluxField = getRefFluxField(results.refCat.schema, filterName=refFilterName) 

375 except RuntimeError: 

376 # This flux field isn't available. Set to None 

377 fluxField = None 

378 

379 if fluxField is None: 

380 self.log.warn(f'No reference flux field for camera filter {filterName}') 

381 

382 self._fluxFields.append(fluxField)