Coverage for python / lsst / pipe / tasks / loadReferenceCatalog.py: 15%
121 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-05-05 18:38 +0000
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""Load a full reference catalog in numpy/table/dataframe format.
24This task will load multi-band reference objects, apply a reference selector,
25and apply color terms.
26"""
28__all__ = ['LoadReferenceCatalogConfig', 'LoadReferenceCatalogTask']
30import numpy as np
31from astropy import units
33import lsst.pex.config as pexConfig
34import lsst.pipe.base as pipeBase
35from lsst.meas.algorithms import ReferenceSourceSelectorTask
36from lsst.meas.algorithms import getRefFluxField
37from lsst.pipe.tasks.colorterms import ColortermLibrary
38from lsst.afw.image import abMagErrFromFluxErr
39from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig
42class LoadReferenceCatalogConfig(pexConfig.Config):
43 """Config for LoadReferenceCatalogTask"""
44 refObjLoader = pexConfig.ConfigField(
45 dtype=LoadReferenceObjectsConfig,
46 doc="Configuration for the reference object loader.",
47 )
48 doApplyColorTerms = pexConfig.Field(
49 doc=("Apply photometric color terms to reference stars? "
50 "Requires that colorterms be set to a ColorTermLibrary"),
51 dtype=bool,
52 default=True
53 )
54 colorterms = pexConfig.ConfigField(
55 doc="Library of photometric reference catalog name to color term dict.",
56 dtype=ColortermLibrary,
57 )
58 referenceSelector = pexConfig.ConfigurableField(
59 target=ReferenceSourceSelectorTask,
60 doc="Selection of reference sources",
61 )
62 doReferenceSelection = pexConfig.Field(
63 doc="Run the reference selector on the reference catalog?",
64 dtype=bool,
65 default=True
66 )
68 def validate(self):
69 super().validate()
70 if self.doApplyColorTerms and len(self.colorterms.data) == 0:
71 msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary."
72 raise pexConfig.FieldValidationError(LoadReferenceCatalogConfig.colorterms, self, msg)
75class LoadReferenceCatalogTask(pipeBase.Task):
76 """Load multi-band reference objects from a reference catalog.
78 Parameters
79 ----------
80 dataIds : iterable of `lsst.daf.butler.dataId`
81 An iterable object of dataIds which point to reference catalogs
82 in a Gen3 repository. Required for Gen3.
83 refCats : iterable of `lsst.daf.butler.DeferredDatasetHandle`
84 An iterable object of dataset refs for reference catalogs in
85 a Gen3 repository.
86 name : `str`
87 The name of the refcat that this object will load. This name is used
88 for applying colorterms, for example.
90 Raises
91 ------
92 RuntimeError if dataIds or refCats is None.
93 """
94 ConfigClass = LoadReferenceCatalogConfig
95 _DefaultName = "loadReferenceCatalog"
97 def __init__(self, *, dataIds, refCats, name, **kwargs):
98 pipeBase.Task.__init__(self, **kwargs)
99 refConfig = self.config.refObjLoader
100 self.refObjLoader = ReferenceObjectLoader(dataIds=dataIds,
101 refCats=refCats,
102 name=name,
103 config=refConfig,
104 log=self.log)
106 if self.config.doReferenceSelection:
107 self.makeSubtask('referenceSelector')
108 self._fluxFilters = None
109 self._fluxFields = None
110 self._referenceFilter = None
112 def getPixelBoxCatalog(self, bbox, wcs, filterList, epoch=None,
113 bboxToSpherePadding=None):
114 """Get a multi-band reference catalog by specifying a bounding box and WCS.
116 The catalog will be in `numpy.ndarray`, with positions proper-motion
117 corrected to "epoch" (if specified, and if the reference catalog has
118 proper motions); sources cut on a reference selector (if
119 "config.doReferenceSelection = True"); and color-terms applied (if
120 "config.doApplyColorTerms = True").
122 The format of the reference catalog will be of the format:
124 dtype = [('ra', 'np.float64'),
125 ('dec', 'np.float64'),
126 ('refMag', 'np.float32', (len(filterList), )),
127 ('refMagErr', 'np.float32', (len(filterList), ))]
129 Reference magnitudes (AB) and errors will be NaN for non-detections
130 in a given band.
132 Parameters
133 ----------
134 bbox : `lsst.geom.Box2I`
135 Box which bounds a region in pixel space.
136 wcs : `lsst.afw.geom.SkyWcs`
137 Wcs object defining the pixel to sky (and reverse) transform for
138 the supplied bbox.
139 filterList : `List` [ `str` ]
140 List of camera physicalFilter names to retrieve magnitudes.
141 epoch : `astropy.time.Time`, optional
142 Epoch to which to correct proper motion and parallax
143 (if available), or `None` to not apply such corrections.
144 bboxToSpherePadding : `int`, optional
145 Padding to account for translating a set of corners into a
146 spherical (convex) boundary that is certain to encompass the
147 entire area covered by the bbox.
149 Returns
150 -------
151 refCat : `numpy.ndarray`
152 Reference catalog.
153 """
154 # Check if we have previously cached values for the fluxFields
155 if self._fluxFilters is None or self._fluxFilters != filterList:
156 center = wcs.pixelToSky(bbox.getCenter())
157 self._determineFluxFields(center, filterList)
159 skyBox = self.refObjLoader.loadPixelBox(bbox, wcs, self._referenceFilter,
160 epoch=epoch,
161 bboxToSpherePadding=bboxToSpherePadding)
163 if not skyBox.refCat.isContiguous():
164 refCat = skyBox.refCat.copy(deep=True)
165 else:
166 refCat = skyBox.refCat
168 return self._formatCatalog(refCat, filterList)
170 def getSkyCircleCatalog(self, center, radius, filterList, epoch=None,
171 catalogFormat='numpy'):
172 """Get a multi-band reference catalog by specifying a center and radius.
174 The catalog will be in `numpy.ndarray`, with positions proper-motion
175 corrected to "epoch" (if specified, and if the reference catalog has
176 proper motions); sources cut on a reference selector (if
177 "config.doReferenceSelection = True"); and color-terms applied (if
178 "config.doApplyColorTerms = True").
180 The format of the reference catalog will be of the format:
182 dtype = [('ra', 'np.float64'),
183 ('dec', 'np.float64'),
184 ('refMag', 'np.float32', (len(filterList), )),
185 ('refMagErr', 'np.float32', (len(filterList), ))]
187 Reference magnitudes (AB) and errors will be NaN for non-detections
188 in a given band.
190 Parameters
191 ----------
192 center : `lsst.geom.SpherePoint`
193 Point defining the center of the circular region.
194 radius : `lsst.geom.Angle`
195 Defines the angular radius of the circular region.
196 filterList : `List` [ `str` ]
197 List of camera physicalFilter names to retrieve magnitudes.
198 epoch : `astropy.time.Time`, optional
199 Epoch to which to correct proper motion and parallax
200 (if available), or `None` to not apply such corrections.
202 Returns
203 -------
204 refCat : `numpy.ndarray`
205 Reference catalog.
206 """
207 # Check if we have previously cached values for the fluxFields
208 if self._fluxFilters is None or self._fluxFilters != filterList:
209 self._determineFluxFields(center, filterList)
211 skyCircle = self.refObjLoader.loadSkyCircle(center, radius,
212 self._referenceFilter,
213 epoch=epoch)
215 if not skyCircle.refCat.isContiguous():
216 refCat = skyCircle.refCat.copy(deep=True)
217 else:
218 refCat = skyCircle.refCat
220 return self._formatCatalog(refCat, filterList)
222 def _formatCatalog(self, refCat, filterList):
223 """Format a reference afw table into the final format.
225 This method applies reference selections and color terms as specified
226 by the config.
228 Parameters
229 ----------
230 refCat : `lsst.afw.table.SourceCatalog`
231 Reference catalog in afw format.
232 filterList : `list` [`str`]
233 List of camera physicalFilter names to apply color terms.
235 Returns
236 -------
237 refCat : `numpy.ndarray`
238 Reference catalog.
239 """
240 if self.config.doReferenceSelection:
241 goodSources = self.referenceSelector.selectSources(refCat)
242 selected = goodSources.selected
243 else:
244 selected = np.ones(len(refCat), dtype=bool)
246 npRefCat = np.zeros(np.sum(selected), dtype=[('ra', 'f8'),
247 ('dec', 'f8'),
248 ('refMag', 'f4', (len(filterList), )),
249 ('refMagErr', 'f4', (len(filterList), ))])
251 if npRefCat.size == 0:
252 # Return an empty catalog if we don't have any selected sources.
253 return npRefCat
255 # Natively "coord_ra" and "coord_dec" are stored in radians.
256 # Doing this as an array rather than by row with the coord access is
257 # approximately 600x faster.
258 npRefCat['ra'] = np.rad2deg(refCat['coord_ra'][selected])
259 npRefCat['dec'] = np.rad2deg(refCat['coord_dec'][selected])
261 # Default (unset) values are np.nan
262 npRefCat['refMag'][:, :] = np.nan
263 npRefCat['refMagErr'][:, :] = np.nan
265 if self.config.doApplyColorTerms:
266 refCatName = self.refObjLoader.name
268 for i, (filterName, fluxField) in enumerate(zip(self._fluxFilters, self._fluxFields)):
269 if fluxField is None:
270 # There is no matching reference band.
271 # This will leave the column filled with np.nans
272 continue
273 self.log.debug("Applying color terms for filterName='%s'", filterName)
275 colorterm = self.config.colorterms.getColorterm(filterName, refCatName, doRaise=True)
277 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat)
279 # nan_to_num below replaces nans with 99, and this ensures
280 # that we select magnitudes that both filter out nans and are
281 # not very large (corresponding to very small fluxes), as "99"
282 # is a common sentinel for illegal magnitudes in reference catalogs.
283 good, = np.where((np.nan_to_num(refMag[selected], nan=99.0) < 90.0)
284 & (np.nan_to_num(refMagErr[selected], nan=99.0) < 90.0)
285 & (np.nan_to_num(refMagErr[selected]) > 0.0))
287 npRefCat['refMag'][good, i] = refMag[selected][good]
288 npRefCat['refMagErr'][good, i] = refMagErr[selected][good]
289 else:
290 # No color terms to apply
291 for i, (filterName, fluxField) in enumerate(zip(self._fluxFilters, self._fluxFields)):
292 # nan_to_num below replaces nans with zeros, and this ensures that
293 # we select fluxes that both filter out nans and are positive.
294 good, = np.where((np.nan_to_num(refCat[fluxField][selected]) > 0.0)
295 & (np.nan_to_num(refCat[fluxField+'Err'][selected]) > 0.0))
296 refMag = (refCat[fluxField][selected][good]*units.nJy).to_value(units.ABmag)
297 refMagErr = abMagErrFromFluxErr(refCat[fluxField+'Err'][selected][good],
298 refCat[fluxField][selected][good])
299 npRefCat['refMag'][good, i] = refMag
300 npRefCat['refMagErr'][good, i] = refMagErr
302 return npRefCat
304 def _determineFluxFields(self, center, filterList):
305 """Determine the flux field names for a reference catalog.
307 This method sets self._fluxFields, self._referenceFilter.
309 Parameters
310 ----------
311 center : `lsst.geom.SpherePoint`
312 The center around which to load test sources.
313 filterList : `list` [`str`]
314 List of camera physicalFilter names.
315 """
316 # Search for a good filter to use to load the reference catalog
317 # via the refObjLoader task which requires a valid filterName
318 foundReferenceFilter = False
320 # Store the original config
321 _config = self.refObjLoader.config
323 configTemp = LoadReferenceObjectsConfig()
324 configIntersection = {k: getattr(self.refObjLoader.config, k)
325 for k, v in self.refObjLoader.config.toDict().items()
326 if (k in configTemp.keys() and k != "connections")}
327 # We must turn off the proper motion checking to find the refFilter.
328 configIntersection['requireProperMotion'] = False
329 configTemp.update(**configIntersection)
331 self.refObjLoader.config = configTemp
333 for filterName in filterList:
334 if self.config.refObjLoader.anyFilterMapsToThis is not None:
335 refFilterName = self.config.refObjLoader.anyFilterMapsToThis
336 else:
337 refFilterName = self.config.refObjLoader.filterMap.get(filterName)
338 if refFilterName is None:
339 continue
340 try:
341 results = self.refObjLoader.loadSchema(refFilterName)
342 foundReferenceFilter = True
343 self._referenceFilter = refFilterName
344 break
345 except RuntimeError as err:
346 # This just means that the filterName wasn't listed
347 # in the reference catalog. This is okay.
348 if 'not find flux' in err.args[0]:
349 # The filterName wasn't listed in the reference catalog.
350 # This is not a fatal failure (yet)
351 pass
352 else:
353 raise err
355 self.refObjLoader.config = _config
357 if not foundReferenceFilter:
358 raise RuntimeError("Could not find any valid flux field(s) %s" %
359 (", ".join(filterList)))
361 # Record self._fluxFilters for checks on subsequent calls
362 self._fluxFilters = filterList
364 # Retrieve all the fluxField names
365 self._fluxFields = []
366 for filterName in filterList:
367 fluxField = None
369 if self.config.refObjLoader.anyFilterMapsToThis is not None:
370 refFilterName = self.config.refObjLoader.anyFilterMapsToThis
371 else:
372 refFilterName = self.config.refObjLoader.filterMap.get(filterName)
374 if refFilterName is not None:
375 try:
376 fluxField = getRefFluxField(results.schema, filterName=refFilterName)
377 except RuntimeError:
378 # This flux field isn't available. Set to None
379 fluxField = None
381 if fluxField is None:
382 self.log.warning('No reference flux field for camera filter %s', filterName)
384 self._fluxFields.append(fluxField)