Coverage for python/lsst/pipe/tasks/loadReferenceCatalog.py: 18%
125 statements
« prev ^ index » next coverage.py v6.4.4, created at 2022-08-18 20:10 +0000
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of pipe_tasks.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""Load a full reference catalog in numpy/table/dataframe format.
25This task will load multi-band reference objects, apply a reference selector,
26and apply color terms.
27"""
28import numpy as np
29from astropy import units
31import lsst.pex.config as pexConfig
32import lsst.pipe.base as pipeBase
33from lsst.meas.algorithms import ReferenceSourceSelectorTask
34from lsst.meas.algorithms import getRefFluxField
35from lsst.pipe.tasks.colorterms import ColortermLibrary
36from lsst.afw.image import abMagErrFromFluxErr
37from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig
39import lsst.geom
41__all__ = ['LoadReferenceCatalogConfig', 'LoadReferenceCatalogTask']
44class LoadReferenceCatalogConfig(pexConfig.Config):
45 """Config for LoadReferenceCatalogTask"""
46 refObjLoader = pexConfig.ConfigField(
47 dtype=LoadReferenceObjectsConfig,
48 doc="Configuration for the reference object loader.",
49 )
50 doApplyColorTerms = pexConfig.Field(
51 doc=("Apply photometric color terms to reference stars? "
52 "Requires that colorterms be set to a ColorTermLibrary"),
53 dtype=bool,
54 default=True
55 )
56 colorterms = pexConfig.ConfigField(
57 doc="Library of photometric reference catalog name to color term dict.",
58 dtype=ColortermLibrary,
59 )
60 referenceSelector = pexConfig.ConfigurableField(
61 target=ReferenceSourceSelectorTask,
62 doc="Selection of reference sources",
63 )
64 doReferenceSelection = pexConfig.Field(
65 doc="Run the reference selector on the reference catalog?",
66 dtype=bool,
67 default=True
68 )
70 def validate(self):
71 super().validate()
72 if self.doApplyColorTerms and len(self.colorterms.data) == 0:
73 msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary."
74 raise pexConfig.FieldValidationError(LoadReferenceCatalogConfig.colorterms, self, msg)
77class LoadReferenceCatalogTask(pipeBase.Task):
78 """Load multi-band reference objects from a reference catalog.
80 Parameters
81 ----------
82 dataIds : iterable of `lsst.daf.butler.dataId`, optional
83 An iterable object of dataIds which point to reference catalogs
84 in a Gen3 repository. Required for Gen3.
85 refCats : iterable of `lsst.daf.butler.DeferredDatasetHandle`, optional
86 An iterable object of dataset refs for reference catalogs in
87 a Gen3 repository.
89 Raises
90 ------
91 RuntimeError if dataIds or refCats is None.
92 """
93 ConfigClass = LoadReferenceCatalogConfig
94 _DefaultName = "loadReferenceCatalog"
96 def __init__(self, dataIds=None, refCats=None, **kwargs):
97 if dataIds is not None and refCats is not None:
98 pipeBase.Task.__init__(self, **kwargs)
99 refConfig = self.config.refObjLoader
100 self.refObjLoader = ReferenceObjectLoader(dataIds=dataIds,
101 refCats=refCats,
102 config=refConfig,
103 log=self.log)
104 else:
105 raise RuntimeError("Must instantiate LoadReferenceCatalogTask with "
106 "dataIds and refCats (Gen3)")
108 if self.config.doReferenceSelection:
109 self.makeSubtask('referenceSelector')
110 self._fluxFilters = None
111 self._fluxFields = None
112 self._referenceFilter = None
114 def getPixelBoxCatalog(self, bbox, wcs, filterList, epoch=None,
115 bboxToSpherePadding=None):
116 """Get a multi-band reference catalog by specifying a bounding box and WCS.
118 The catalog will be in `numpy.ndarray`, with positions proper-motion
119 corrected to "epoch" (if specified, and if the reference catalog has
120 proper motions); sources cut on a reference selector (if
121 "config.doReferenceSelection = True"); and color-terms applied (if
122 "config.doApplyColorTerms = True").
124 The format of the reference catalog will be of the format:
126 dtype = [('ra', 'np.float64'),
127 ('dec', 'np.float64'),
128 ('refMag', 'np.float32', (len(filterList), )),
129 ('refMagErr', 'np.float32', (len(filterList), ))]
131 Reference magnitudes (AB) and errors will be 99 for non-detections
132 in a given band.
134 Parameters
135 ----------
136 bbox : `lsst.geom.Box2I`
137 Box which bounds a region in pixel space.
138 wcs : `lsst.afw.geom.SkyWcs`
139 Wcs object defining the pixel to sky (and reverse) transform for
140 the supplied bbox.
141 filterList : `List` [ `str` ]
142 List of camera physicalFilter names to retrieve magnitudes.
143 epoch : `astropy.time.Time`, optional
144 Epoch to which to correct proper motion and parallax
145 (if available), or `None` to not apply such corrections.
146 bboxToSpherePadding : `int`, optional
147 Padding to account for translating a set of corners into a
148 spherical (convex) boundary that is certain to encompass the
149 entire area covered by the bbox.
151 Returns
152 -------
153 refCat : `numpy.ndarray`
154 Reference catalog.
155 """
156 # Check if we have previously cached values for the fluxFields
157 if self._fluxFilters is None or self._fluxFilters != filterList:
158 center = wcs.pixelToSky(bbox.getCenter())
159 self._determineFluxFields(center, filterList)
161 skyBox = self.refObjLoader.loadPixelBox(bbox, wcs, self._referenceFilter,
162 epoch=epoch,
163 bboxToSpherePadding=bboxToSpherePadding)
165 if not skyBox.refCat.isContiguous():
166 refCat = skyBox.refCat.copy(deep=True)
167 else:
168 refCat = skyBox.refCat
170 return self._formatCatalog(refCat, filterList)
172 def getSkyCircleCatalog(self, center, radius, filterList, epoch=None,
173 catalogFormat='numpy'):
174 """Get a multi-band reference catalog by specifying a center and radius.
176 The catalog will be in `numpy.ndarray`, with positions proper-motion
177 corrected to "epoch" (if specified, and if the reference catalog has
178 proper motions); sources cut on a reference selector (if
179 "config.doReferenceSelection = True"); and color-terms applied (if
180 "config.doApplyColorTerms = True").
182 The format of the reference catalog will be of the format:
184 dtype = [('ra', 'np.float64'),
185 ('dec', 'np.float64'),
186 ('refMag', 'np.float32', (len(filterList), )),
187 ('refMagErr', 'np.float32', (len(filterList), ))]
189 Reference magnitudes (AB) and errors will be 99 for non-detections
190 in a given band.
192 Parameters
193 ----------
194 center : `lsst.geom.SpherePoint`
195 Point defining the center of the circular region.
196 radius : `lsst.geom.Angle`
197 Defines the angular radius of the circular region.
198 filterList : `List` [ `str` ]
199 List of camera physicalFilter names to retrieve magnitudes.
200 epoch : `astropy.time.Time`, optional
201 Epoch to which to correct proper motion and parallax
202 (if available), or `None` to not apply such corrections.
204 Parameters
205 ----------
206 refCat : `numpy.ndarray`
207 Reference catalog.
208 """
209 # Check if we have previously cached values for the fluxFields
210 if self._fluxFilters is None or self._fluxFilters != filterList:
211 self._determineFluxFields(center, filterList)
213 skyCircle = self.refObjLoader.loadSkyCircle(center, radius,
214 self._referenceFilter,
215 epoch=epoch)
217 if not skyCircle.refCat.isContiguous():
218 refCat = skyCircle.refCat.copy(deep=True)
219 else:
220 refCat = skyCircle.refCat
222 return self._formatCatalog(refCat, filterList)
224 def _formatCatalog(self, refCat, filterList):
225 """Format a reference afw table into the final format.
227 This method applies reference selections and color terms as specified
228 by the config.
230 Parameters
231 ----------
232 refCat : `lsst.afw.table.SourceCatalog`
233 Reference catalog in afw format.
234 filterList : `list` [`str`]
235 List of camera physicalFilter names to apply color terms.
237 Returns
238 -------
239 refCat : `numpy.ndarray`
240 Reference catalog.
241 """
242 if self.config.doReferenceSelection:
243 goodSources = self.referenceSelector.selectSources(refCat)
244 selected = goodSources.selected
245 else:
246 selected = np.ones(len(refCat), dtype=bool)
248 npRefCat = np.zeros(np.sum(selected), dtype=[('ra', 'f8'),
249 ('dec', 'f8'),
250 ('refMag', 'f4', (len(filterList), )),
251 ('refMagErr', 'f4', (len(filterList), ))])
253 if npRefCat.size == 0:
254 # Return an empty catalog if we don't have any selected sources.
255 return npRefCat
257 # Natively "coord_ra" and "coord_dec" are stored in radians.
258 # Doing this as an array rather than by row with the coord access is
259 # approximately 600x faster.
260 npRefCat['ra'] = np.rad2deg(refCat['coord_ra'][selected])
261 npRefCat['dec'] = np.rad2deg(refCat['coord_dec'][selected])
263 # Default (unset) values are 99.0
264 npRefCat['refMag'][:, :] = 99.0
265 npRefCat['refMagErr'][:, :] = 99.0
267 if self.config.doApplyColorTerms:
268 refCatName = self.refObjLoader.config.ref_dataset_name
270 for i, (filterName, fluxField) in enumerate(zip(self._fluxFilters, self._fluxFields)):
271 if fluxField is None:
272 # There is no matching reference band.
273 # This will leave the column filled with 99s
274 continue
275 self.log.debug("Applying color terms for filterName='%s'", filterName)
277 colorterm = self.config.colorterms.getColorterm(filterName, refCatName, doRaise=True)
279 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat)
281 # nan_to_num replaces nans with zeros, and this ensures
282 # that we select magnitudes that both filter out nans and are
283 # not very large (corresponding to very small fluxes), as "99"
284 # is a commen sentinel for illegal magnitudes.
285 good, = np.where((np.nan_to_num(refMag[selected], nan=99.0) < 90.0)
286 & (np.nan_to_num(refMagErr[selected], nan=99.0) < 90.0)
287 & (np.nan_to_num(refMagErr[selected]) > 0.0))
289 npRefCat['refMag'][good, i] = refMag[selected][good]
290 npRefCat['refMagErr'][good, i] = refMagErr[selected][good]
291 else:
292 # No color terms to apply
293 for i, (filterName, fluxField) in enumerate(zip(self._fluxFilters, self._fluxFields)):
294 # nan_to_num replaces nans with zeros, and this ensures that
295 # we select fluxes that both filter out nans and are positive.
296 good, = np.where((np.nan_to_num(refCat[fluxField][selected]) > 0.0)
297 & (np.nan_to_num(refCat[fluxField+'Err'][selected]) > 0.0))
298 refMag = (refCat[fluxField][selected][good]*units.nJy).to_value(units.ABmag)
299 refMagErr = abMagErrFromFluxErr(refCat[fluxField+'Err'][selected][good],
300 refCat[fluxField][selected][good])
301 npRefCat['refMag'][good, i] = refMag
302 npRefCat['refMagErr'][good, i] = refMagErr
304 return npRefCat
306 def _determineFluxFields(self, center, filterList):
307 """Determine the flux field names for a reference catalog.
309 This method sets self._fluxFields, self._referenceFilter.
311 Parameters
312 ----------
313 center : `lsst.geom.SpherePoint`
314 The center around which to load test sources.
315 filterList : `list` [`str`]
316 List of camera physicalFilter names.
317 """
318 # Search for a good filter to use to load the reference catalog
319 # via the refObjLoader task which requires a valid filterName
320 foundReferenceFilter = False
322 # Store the original config
323 _config = self.refObjLoader.config
325 configTemp = LoadReferenceObjectsConfig()
326 configIntersection = {k: getattr(self.refObjLoader.config, k)
327 for k, v in self.refObjLoader.config.toDict().items()
328 if (k in configTemp.keys() and k != "connections")}
329 # We must turn off the proper motion checking to find the refFilter.
330 configIntersection['requireProperMotion'] = False
331 configTemp.update(**configIntersection)
333 self.refObjLoader.config = configTemp
335 for filterName in filterList:
336 if self.config.refObjLoader.anyFilterMapsToThis is not None:
337 refFilterName = self.config.refObjLoader.anyFilterMapsToThis
338 else:
339 refFilterName = self.config.refObjLoader.filterMap.get(filterName)
340 if refFilterName is None:
341 continue
342 try:
343 results = self.refObjLoader.loadSkyCircle(center,
344 0.05*lsst.geom.degrees,
345 refFilterName)
346 foundReferenceFilter = True
347 self._referenceFilter = refFilterName
348 break
349 except RuntimeError as err:
350 # This just means that the filterName wasn't listed
351 # in the reference catalog. This is okay.
352 if 'not find flux' in err.args[0]:
353 # The filterName wasn't listed in the reference catalog.
354 # This is not a fatal failure (yet)
355 pass
356 else:
357 raise err
359 self.refObjLoader.config = _config
361 if not foundReferenceFilter:
362 raise RuntimeError("Could not find any valid flux field(s) %s" %
363 (", ".join(filterList)))
365 # Record self._fluxFilters for checks on subsequent calls
366 self._fluxFilters = filterList
368 # Retrieve all the fluxField names
369 self._fluxFields = []
370 for filterName in filterList:
371 fluxField = None
373 if self.config.refObjLoader.anyFilterMapsToThis is not None:
374 refFilterName = self.config.refObjLoader.anyFilterMapsToThis
375 else:
376 refFilterName = self.config.refObjLoader.filterMap.get(filterName)
378 if refFilterName is not None:
379 try:
380 fluxField = getRefFluxField(results.refCat.schema, filterName=refFilterName)
381 except RuntimeError:
382 # This flux field isn't available. Set to None
383 fluxField = None
385 if fluxField is None:
386 self.log.warning('No reference flux field for camera filter %s', filterName)
388 self._fluxFields.append(fluxField)