lsst.meas.algorithms ge3a8c498d5+6eace70ba9
Loading...
Searching...
No Matches
loadReferenceObjects.py
Go to the documentation of this file.
2# LSST Data Management System
3#
4# Copyright 2008-2017 AURA/LSST.
5#
6# This product includes software developed by the
7# LSST Project (http://www.lsst.org/).
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the LSST License Statement and
20# the GNU General Public License along with this program. If not,
21# see <https://www.lsstcorp.org/LegalNotices/>.
22#
23
24__all__ = ["getRefFluxField", "getRefFluxKeys", "LoadReferenceObjectsTask", "LoadReferenceObjectsConfig",
25 "ReferenceObjectLoader", "ReferenceObjectLoaderBase"]
26
27import abc
28import logging
29import warnings
30from deprecated.sphinx import deprecated
31
32import astropy.time
33import astropy.units
34import numpy
35
36import lsst.geom as geom
37import lsst.afw.table as afwTable
38import lsst.pex.config as pexConfig
39import lsst.pipe.base as pipeBase
40from lsst import sphgeom
41from lsst.daf.base import PropertyList
42
43
44# TODO DM-34793: remove this function
45def isOldFluxField(name, units):
46 """Return True if this name/units combination corresponds to an
47 "old-style" reference catalog flux field.
48 """
49 unitsCheck = units != 'nJy' # (units == 'Jy' or units == '' or units == '?')
50 isFlux = name.endswith('_flux')
51 isFluxSigma = name.endswith('_fluxSigma')
52 isFluxErr = name.endswith('_fluxErr')
53 return (isFlux or isFluxSigma or isFluxErr) and unitsCheck
54
55
56# TODO DM-34793: remove this function
58 """Return True if the units of all flux and fluxErr are correct (nJy).
59 """
60 for field in schema:
61 if isOldFluxField(field.field.getName(), field.field.getUnits()):
62 return False
63 return True
64
65
67 """"Return the format version stored in a reference catalog header.
68
69 Parameters
70 ----------
72 Reference catalog to inspect.
73
74 Returns
75 -------
76 version : `int`
77 Format verison integer. Returns `0` if the catalog has no metadata
78 or the metadata does not include a "REFCAT_FORMAT_VERSION" key.
79 """
80 # TODO DM-34793: change to "Version 0 refcats are no longer supported: refcat fluxes must have nJy units."
81 # TODO DM-34793: and raise an exception instead of returning 0.
82 deprecation_msg = "Support for version 0 refcats (pre-nJy fluxes) will be removed after v25."
83 md = refCat.getMetadata()
84 if md is None:
85 warnings.warn(deprecation_msg)
86 return 0
87 try:
88 return md.getScalar("REFCAT_FORMAT_VERSION")
89 except KeyError:
90 warnings.warn(deprecation_msg)
91 return 0
92
93
94# TODO DM-34793: remove this function
95@deprecated(reason="Support for version 0 refcats (pre-nJy fluxes) will be removed after v25.",
96 version="v24.0", category=FutureWarning)
97def convertToNanojansky(catalog, log, doConvert=True):
98 """Convert fluxes in a catalog from jansky to nanojansky.
99
100 Parameters
101 ----------
103 The catalog to convert.
104 log : `lsst.log.Log` or `logging.Logger`
105 Log to send messages to.
106 doConvert : `bool`, optional
107 Return a converted catalog, or just identify the fields that need to be converted?
108 This supports the "write=False" mode of `bin/convert_to_nJy.py`.
109
110 Returns
111 -------
112 catalog : `lsst.afw.table.SimpleCatalog` or None
113 The converted catalog, or None if ``doConvert`` is False.
114
115 Notes
116 -----
117 Support for old units in reference catalogs will be removed after the
118 release of late calendar year 2019.
119 Use `meas_algorithms/bin/convert_to_nJy.py` to update your reference catalog.
120 """
121 # Do not share the AliasMap: for refcats, that gets created when the
122 # catalog is read from disk and should not be propagated.
123 mapper = afwTable.SchemaMapper(catalog.schema, shareAliasMap=False)
124 mapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
125 input_fields = []
126 output_fields = []
127 for field in catalog.schema:
128 oldName = field.field.getName()
129 oldUnits = field.field.getUnits()
130 if isOldFluxField(oldName, oldUnits):
131 units = 'nJy'
132 # remap Sigma flux fields to Err, so we can drop the alias
133 if oldName.endswith('_fluxSigma'):
134 name = oldName.replace('_fluxSigma', '_fluxErr')
135 else:
136 name = oldName
137 newField = afwTable.Field[field.dtype](name, field.field.getDoc(), units)
138 mapper.addMapping(field.getKey(), newField)
139 input_fields.append(field.field)
140 output_fields.append(newField)
141 else:
142 mapper.addMapping(field.getKey())
143
144 fluxFieldsStr = '; '.join("(%s, '%s')" % (field.getName(), field.getUnits()) for field in input_fields)
145
146 if doConvert:
147 newSchema = mapper.getOutputSchema()
148 output = afwTable.SimpleCatalog(newSchema)
149 output.reserve(len(catalog))
150 output.extend(catalog, mapper=mapper)
151 for field in output_fields:
152 output[field.getName()] *= 1e9
153 log.info("Converted refcat flux fields to nJy (name, units): %s", fluxFieldsStr)
154 return output
155 else:
156 log.info("Found old-style refcat flux fields (name, units): %s", fluxFieldsStr)
157 return None
158
159
161 """This is a private helper class which filters catalogs by
162 row based on the row being inside the region used to initialize
163 the class.
164
165 Parameters
166 ----------
167 region : `lsst.sphgeom.Region`
168 The spatial region which all objects should lie within
169 """
170 def __init__(self, region):
171 self.region = region
172
173 def __call__(self, refCat, catRegion):
174 """This call method on an instance of this class takes in a reference
175 catalog, and the region from which the catalog was generated.
176
177 If the catalog region is entirely contained within the region used to
178 initialize this class, then all the entries in the catalog must be
179 within the region and so the whole catalog is returned.
180
181 If the catalog region is not entirely contained, then the location for
182 each record is tested against the region used to initialize the class.
183 Records which fall inside this region are added to a new catalog, and
184 this catalog is then returned.
185
186 Parameters
187 ---------
189 SourceCatalog to be filtered.
190 catRegion : `lsst.sphgeom.Region`
191 Region in which the catalog was created
192 """
193 if catRegion.isWithin(self.region):
194 # no filtering needed, region completely contains refcat
195 return refCat
196
197 filteredRefCat = type(refCat)(refCat.table)
198 for record in refCat:
199 if self.region.contains(record.getCoord().getVector()):
200 filteredRefCat.append(record)
201 return filteredRefCat
202
203
204class LoadReferenceObjectsConfig(pexConfig.Config):
205 pixelMargin = pexConfig.RangeField(
206 doc="Padding to add to 4 all edges of the bounding box (pixels)",
207 dtype=int,
208 default=250,
209 min=0,
210 )
211 anyFilterMapsToThis = pexConfig.Field(
212 doc=("Always use this reference catalog filter, no matter whether or what filter name is "
213 "supplied to the loader. Effectively a trivial filterMap: map all filter names to this filter."
214 " This can be set for purely-astrometric catalogs (e.g. Gaia DR2) where there is only one "
215 "reasonable choice for every camera filter->refcat mapping, but not for refcats used for "
216 "photometry, which need a filterMap and/or colorterms/transmission corrections."),
217 dtype=str,
218 default=None,
219 optional=True
220 )
221 filterMap = pexConfig.DictField(
222 doc=("Mapping of camera filter name: reference catalog filter name; "
223 "each reference filter must exist in the refcat."
224 " Note that this does not perform any bandpass corrections: it is just a lookup."),
225 keytype=str,
226 itemtype=str,
227 default={},
228 )
229 requireProperMotion = pexConfig.Field(
230 doc="Require that the fields needed to correct proper motion "
231 "(epoch, pm_ra and pm_dec) are present?",
232 dtype=bool,
233 default=False,
234 )
235 ref_dataset_name = pexConfig.Field(
236 doc="Deprecated; do not use. Added for easier transition from LoadIndexedReferenceObjectsConfig to "
237 "LoadReferenceObjectsConfig",
238 dtype=str,
239 default='',
240 deprecated='This field is not used. It will be removed after v25.',
241 )
242
243 def validate(self):
244 super().validate()
245 if self.filterMap != {} and self.anyFilterMapsToThis is not None:
246 msg = "`filterMap` and `anyFilterMapsToThis` are mutually exclusive"
247 raise pexConfig.FieldValidationError(LoadReferenceObjectsConfig.anyFilterMapsToThis,
248 self, msg)
249
250
252 """This class facilitates loading reference catalogs.
253
254 The QuantumGraph generation will create a list of datasets that may
255 possibly overlap a given region. These datasets are then used to construct
256 an instance of this class. The class instance should then be passed into
257 a task which needs reference catalogs. These tasks should then determine
258 the exact region of the sky reference catalogs will be loaded for, and
259 call a corresponding method to load the reference objects.
260
261 Parameters
262 ----------
263 dataIds : iterable of `lsst.daf.butler.DataCoordinate`
264 An iterable object of data IDs that point to reference catalogs.
265 refCats : iterable of `lsst.daf.butler.DeferredDatasetHandle`
266 Handles to load refCats on demand.
267 name : `str`, optional
268 The name of the refcat that this object will load. This name is used
269 for applying colorterms, for example.
270 config : `LoadReferenceObjectsConfig`
271 Configuration of this reference loader.
272 log : `lsst.log.Log`, `logging.Logger` or `None`, optional
273 Logger object used to write out messages. If `None` a default
274 logger will be used.
275 """
276 ConfigClass = LoadReferenceObjectsConfig
277
278 def __init__(self, dataIds, refCats, name=None, log=None, config=None, **kwargs):
279 if kwargs:
280 warnings.warn("Instantiating ReferenceObjectLoader with additional kwargs is deprecated "
281 "and will be removed after v25.0", FutureWarning, stacklevel=2)
282
283 if config is None:
284 config = self.ConfigClass()
285 self.config = config
286 self.dataIds = dataIds
287 self.refCats = refCats
288 self.name = name
289 self.log = log or logging.getLogger(__name__).getChild("ReferenceObjectLoader")
290
291 def applyProperMotions(self, catalog, epoch):
292 """Apply proper motion correction to a reference catalog.
293
294 Adjust position and position error in the ``catalog``
295 for proper motion to the specified ``epoch``,
296 modifying the catalog in place.
297
298 Parameters
299 ----------
301 Catalog of positions, containing at least these fields:
302
303 - Coordinates, retrieved by the table's coordinate key.
304 - ``coord_raErr`` : Error in Right Ascension (rad).
305 - ``coord_decErr`` : Error in Declination (rad).
306 - ``pm_ra`` : Proper motion in Right Ascension (rad/yr,
307 East positive)
308 - ``pm_raErr`` : Error in ``pm_ra`` (rad/yr), optional.
309 - ``pm_dec`` : Proper motion in Declination (rad/yr,
310 North positive)
311 - ``pm_decErr`` : Error in ``pm_dec`` (rad/yr), optional.
312 - ``epoch`` : Mean epoch of object (an astropy.time.Time)
313 epoch : `astropy.time.Time`
314 Epoch to which to correct proper motion.
315 If None, do not apply PM corrections or raise if
316 ``config.requireProperMotion`` is True.
317
318 Raises
319 ------
320 RuntimeError
321 Raised if ``config.requireProperMotion`` is set but we cannot
322 apply the proper motion correction for some reason.
323 """
324 if epoch is None:
325 if self.config.requireProperMotion:
326 raise RuntimeError("requireProperMotion=True but epoch not provided to loader.")
327 else:
328 self.log.debug("No epoch provided: not applying proper motion corrections to refcat.")
329 return
330
331 # Warn/raise for a catalog in an incorrect format, if epoch was specified.
332 if ("pm_ra" in catalog.schema
333 and not isinstance(catalog.schema["pm_ra"].asKey(), afwTable.KeyAngle)):
334 if self.config.requireProperMotion:
335 raise RuntimeError("requireProperMotion=True but refcat pm_ra field is not an Angle.")
336 else:
337 self.log.warning("Reference catalog pm_ra field is not an Angle; cannot apply proper motion.")
338 return
339
340 if ("epoch" not in catalog.schema or "pm_ra" not in catalog.schema):
341 if self.config.requireProperMotion:
342 raise RuntimeError("requireProperMotion=True but PM data not available from catalog.")
343 else:
344 self.log.warning("Proper motion correction not available for this reference catalog.")
345 return
346
347 applyProperMotionsImpl(self.log, catalog, epoch)
348
349 @staticmethod
350 def _remapReferenceCatalogSchema(refCat, *, anyFilterMapsToThis=None,
351 filterMap=None, centroids=False):
352 """This function takes in a reference catalog and returns a new catalog
353 with additional columns defined from the remaining function arguments.
354
355 Parameters
356 ----------
358 Reference catalog to map to new catalog
359 anyFilterMapsToThis : `str`, optional
360 Always use this reference catalog filter.
361 Mutually exclusive with `filterMap`
362 filterMap : `dict` [`str`,`str`], optional
363 Mapping of camera filter name: reference catalog filter name.
364 centroids : `bool`, optional
365 Add centroid fields to the loaded Schema. ``loadPixelBox`` expects
366 these fields to exist.
367
368 Returns
369 -------
370 expandedCat : `lsst.afw.table.SimpleCatalog`
371 Deep copy of input reference catalog with additional columns added
372 """
373 if anyFilterMapsToThis or filterMap:
374 ReferenceObjectLoader._addFluxAliases(refCat.schema, anyFilterMapsToThis, filterMap)
375
376 mapper = afwTable.SchemaMapper(refCat.schema, True)
377 mapper.addMinimalSchema(refCat.schema, True)
378 mapper.editOutputSchema().disconnectAliases()
379
380 if centroids:
381 # Add and initialize centroid and hasCentroid fields (these are
382 # added after loading to avoid wasting space in the saved catalogs).
383 # The new fields are automatically initialized to (nan, nan) and
384 # False so no need to set them explicitly.
385 mapper.editOutputSchema().addField("centroid_x", type=float, doReplace=True)
386 mapper.editOutputSchema().addField("centroid_y", type=float, doReplace=True)
387 mapper.editOutputSchema().addField("hasCentroid", type="Flag", doReplace=True)
388 mapper.editOutputSchema().getAliasMap().set("slot_Centroid", "centroid")
389
390 expandedCat = afwTable.SimpleCatalog(mapper.getOutputSchema())
391 expandedCat.setMetadata(refCat.getMetadata())
392 expandedCat.extend(refCat, mapper=mapper)
393
394 return expandedCat
395
396 @staticmethod
397 def _addFluxAliases(schema, anyFilterMapsToThis=None, filterMap=None):
398 """Add aliases for camera filter fluxes to the schema.
399
400 For each camFilter: refFilter in filterMap, adds these aliases:
401 <camFilter>_camFlux: <refFilter>_flux
402 <camFilter>_camFluxErr: <refFilter>_fluxErr, if the latter exists
403 or sets `anyFilterMapsToThis` in the schema.
404
405 Parameters
406 ----------
407 schema : `lsst.afw.table.Schema`
408 Schema for reference catalog.
409 anyFilterMapsToThis : `str`, optional
410 Always use this reference catalog filter.
411 Mutually exclusive with `filterMap`.
412 filterMap : `dict` [`str`,`str`], optional
413 Mapping of camera filter name: reference catalog filter name.
414 Mutually exclusive with `anyFilterMapsToThis`.
415
416 Raises
417 ------
418 RuntimeError
419 Raised if any required reference flux field is missing from the
420 schema.
421 """
422 # Fail on any truthy value for either of these.
423 if anyFilterMapsToThis and filterMap:
424 raise ValueError("anyFilterMapsToThis and filterMap are mutually exclusive!")
425
426 aliasMap = schema.getAliasMap()
427
428 if anyFilterMapsToThis is not None:
429 refFluxName = anyFilterMapsToThis + "_flux"
430 if refFluxName not in schema:
431 msg = f"Unknown reference filter for anyFilterMapsToThis='{refFluxName}'"
432 raise RuntimeError(msg)
433 aliasMap.set("anyFilterMapsToThis", refFluxName)
434 return # this is mutually exclusive with filterMap
435
436 def addAliasesForOneFilter(filterName, refFilterName):
437 """Add aliases for a single filter
438
439 Parameters
440 ----------
441 filterName : `str` (optional)
442 Camera filter name. The resulting alias name is
443 <filterName>_camFlux
444 refFilterName : `str`
445 Reference catalog filter name; the field
446 <refFilterName>_flux must exist.
447 """
448 camFluxName = filterName + "_camFlux"
449 refFluxName = refFilterName + "_flux"
450 if refFluxName not in schema:
451 raise RuntimeError("Unknown reference filter %s" % (refFluxName,))
452 aliasMap.set(camFluxName, refFluxName)
453 refFluxErrName = refFluxName + "Err"
454 if refFluxErrName in schema:
455 camFluxErrName = camFluxName + "Err"
456 aliasMap.set(camFluxErrName, refFluxErrName)
457
458 if filterMap is not None:
459 for filterName, refFilterName in filterMap.items():
460 addAliasesForOneFilter(filterName, refFilterName)
461
462 @staticmethod
463 def _makeBoxRegion(BBox, wcs, BBoxPadding):
464 outerLocalBBox = geom.Box2D(BBox)
465 innerLocalBBox = geom.Box2D(BBox)
466
467 # Grow the bounding box to allow for effects not fully captured by the
468 # wcs provided (which represents the current best-guess wcs solution
469 # associated with the dataset for which the calibration is to be
470 # computed using the loaded and trimmed reference catalog being defined
471 # here). These effects could include pointing errors and/or an
472 # insufficient optical distorition model for the instrument. The idea
473 # is to ensure the spherical geometric region created contains the
474 # entire region covered by the bbox.
475 # Also create an inner region that is sure to be inside the bbox.
476 outerLocalBBox.grow(BBoxPadding)
477 innerLocalBBox.grow(-1*BBoxPadding)
478
479 # Handle the case where the inner bounding box shrank to a zero sized
480 # region (which will be the case if the shrunken size of either
481 # dimension is less than or equal to zero). In this case, the inner
482 # bounding box is set to the original input bounding box. This is
483 # probably not the best way to handle an empty inner bounding box, but
484 # it is what the calling code currently expects.
485 if innerLocalBBox.getDimensions() == geom.Extent2D(0, 0):
486 innerLocalBBox = geom.Box2D(BBox)
487
488 # Convert the corners of the bounding boxes to sky coordinates.
489 innerBoxCorners = innerLocalBBox.getCorners()
490 innerSphCorners = [wcs.pixelToSky(corner).getVector() for corner in innerBoxCorners]
491 innerSkyRegion = sphgeom.ConvexPolygon(innerSphCorners)
492
493 outerBoxCorners = outerLocalBBox.getCorners()
494 outerSphCorners = [wcs.pixelToSky(corner).getVector() for corner in outerBoxCorners]
495 outerSkyRegion = sphgeom.ConvexPolygon(outerSphCorners)
496
497 return innerSkyRegion, outerSkyRegion, innerSphCorners, outerSphCorners
498
499 @staticmethod
500 def _calculateCircle(bbox, wcs, pixelMargin):
501 """Compute on-sky center and radius of search region.
502
503 Parameters
504 ----------
505 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
506 Pixel bounding box.
508 WCS; used to convert pixel positions to sky coordinates.
509 pixelMargin : `int`
510 Padding to add to 4 all edges of the bounding box (pixels).
511
512 Returns
513 -------
514 results : `lsst.pipe.base.Struct`
515 A Struct containing:
516
517 - coord : `lsst.geom.SpherePoint`
518 ICRS center of the search region.
519 - radius : `lsst.geom.Angle`
520 Radius of the search region.
521 - bbox : `lsst.geom.Box2D`
522 Bounding box used to compute the circle.
523 """
524 bbox = geom.Box2D(bbox) # we modify the box, so use a copy
525 bbox.grow(pixelMargin)
526 coord = wcs.pixelToSky(bbox.getCenter())
527 radius = max(coord.separation(wcs.pixelToSky(pp)) for pp in bbox.getCorners())
528 return pipeBase.Struct(coord=coord, radius=radius, bbox=bbox)
529
530 @staticmethod
531 def getMetadataCircle(coord, radius, filterName, epoch=None):
532 """Return metadata about the loaded reference catalog, in an on-sky
533 circle.
534
535 This metadata is used for reloading the catalog (e.g. for
536 reconstituting a normalized match list).
537
538 Parameters
539 ----------
540 coord : `lsst.geom.SpherePoint`
541 ICRS center of the search region.
542 radius : `lsst.geom.Angle`
543 Radius of the search region.
544 filterName : `str`
545 Name of the camera filter.
546 epoch : `astropy.time.Time` or `None`, optional
547 Epoch that proper motion and parallax were corrected to, or `None`
548 if no such corrections were applied.
549
550 Returns
551 -------
553 Metadata about the catalog.
554 """
555 md = PropertyList()
556 md.add('RA', coord.getRa().asDegrees(), 'field center in degrees')
557 md.add('DEC', coord.getDec().asDegrees(), 'field center in degrees')
558 md.add('RADIUS', radius.asDegrees(), 'field radius in degrees, minimum')
559 # Version 1: Initial version
560 # Version 2: JEPOCH for TAI Julian Epoch year of PM/parallax correction
561 md.add('SMATCHV', 2, 'SourceMatchVector version number')
562 md.add('FILTER', filterName, 'camera filter name for photometric data')
563 md.add('TIMESYS', "TAI", "time scale of time keywords")
564 md.add('JEPOCH', None if epoch is None else epoch.tai.jyear,
565 'Julian epoch (TAI Julian Epoch year) for catalog')
566 return md
567
568 def getMetadataBox(self, bbox, wcs, filterName, epoch=None,
569 bboxToSpherePadding=100):
570 """Return metadata about the loaded reference catalog, in an
571 on-detector box.
572
573 This metadata is used for reloading the catalog (e.g., for
574 reconstituting a normalised match list).
575
576 Parameters
577 ----------
578 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
579 Bounding box for the pixels.
581 The WCS object associated with ``bbox``.
582 filterName : `str`
583 Name of the camera filter.
584 epoch : `astropy.time.Time` or `None`, optional
585 Epoch that proper motion and parallax were corrected to, or `None`
586 if no such corrections were applied.
587 bboxToSpherePadding : `int`, optional
588 Padding in pixels to account for translating a set of corners into
589 a spherical (convex) boundary that is certain to encompass the
590 enitre area covered by the box.
591
592 Returns
593 -------
595 The metadata detailing the search parameters used for this
596 dataset.
597 """
598 circle = self._calculateCircle(bbox, wcs, self.config.pixelMargin)
599 md = self.getMetadataCircle(circle.coord, circle.radius, filterName, epoch=epoch)
600
601 paddedBbox = circle.bbox
602 _, _, innerCorners, outerCorners = self._makeBoxRegion(paddedBbox, wcs, bboxToSpherePadding)
603 for box, corners in zip(("INNER", "OUTER"), (innerCorners, outerCorners)):
604 for (name, corner) in zip(("UPPER_LEFT", "UPPER_RIGHT", "LOWER_LEFT", "LOWER_RIGHT"),
605 corners):
606 md.add(f"{box}_{name}_RA", geom.SpherePoint(corner).getRa().asDegrees(), f"{box}_corner")
607 md.add(f"{box}_{name}_DEC", geom.SpherePoint(corner).getDec().asDegrees(), f"{box}_corner")
608 return md
609
610 def loadPixelBox(self, bbox, wcs, filterName, epoch=None,
611 bboxToSpherePadding=100):
612 """Load reference objects that are within a pixel-based rectangular
613 region.
614
615 This algorithm works by creating a spherical box whose corners
616 correspond to the WCS converted corners of the input bounding box
617 (possibly padded). It then defines a filtering function which looks at
618 the pixel position of the reference objects and accepts only those that
619 lie within the specified bounding box.
620
621 The spherical box region and filtering function are passed to the
622 generic loadRegion method which loads and filters the reference objects
623 from the datastore and returns a single catalog containing the filtered
624 set of reference objects.
625
626 Parameters
627 ----------
628 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
629 Box which bounds a region in pixel space.
631 Wcs object defining the pixel to sky (and inverse) transform for
632 the supplied ``bbox``.
633 filterName : `str`
634 Name of camera filter.
635 epoch : `astropy.time.Time` or `None`, optional
636 Epoch to which to correct proper motion and parallax, or `None`
637 to not apply such corrections.
638 bboxToSpherePadding : `int`, optional
639 Padding to account for translating a set of corners into a
640 spherical (convex) boundary that is certain to encompase the
641 enitre area covered by the box.
642
643 Returns
644 -------
645 output : `lsst.pipe.base.Struct`
646 Results struct with attributes:
647
648 ``refCat``
649 Catalog containing reference objects inside the specified
650 bounding box (padded by self.config.pixelMargin).
651 ``fluxField``
652 Name of the field containing the flux associated with
653 ``filterName``.
654
655 Raises
656 ------
657 RuntimeError
658 Raised if no reference catalogs could be found for the specified
659 region.
660 TypeError
661 Raised if the loaded reference catalogs do not have matching
662 schemas.
663 """
664 paddedBbox = geom.Box2D(bbox)
665 paddedBbox.grow(self.config.pixelMargin)
666 innerSkyRegion, outerSkyRegion, _, _ = self._makeBoxRegion(paddedBbox, wcs, bboxToSpherePadding)
667
668 def _filterFunction(refCat, region):
669 # Perform an initial "pre filter" step based on the refCat coords
670 # and the outerSkyRegion created from the self.config.pixelMargin-
671 # paddedBbox plus an "extra" padding of bboxToSpherePadding and the
672 # raw wcs. This should ensure a large enough projected area on the
673 # sky that accounts for any projection/distortion issues, but small
674 # enough to filter out loaded reference objects that lie well
675 # beyond the projected detector of interest. This step is required
676 # due to the very local nature of the wcs available for the
677 # sky <--> pixel conversions.
678 preFiltFunc = _FilterCatalog(outerSkyRegion)
679 refCat = preFiltFunc(refCat, region)
680
681 # Add columns to the pre-filtered reference catalog relating their
682 # coordinates to equivalent pixel positions for the wcs provided
683 # and use to populate those columns.
684 refCat = self._remapReferenceCatalogSchema(refCat, centroids=True)
685 afwTable.updateRefCentroids(wcs, refCat)
686 # No need to filter the catalog if it is entirely contained in the
687 # region defined by the inner sky region.
688 if innerSkyRegion.contains(region):
689 return refCat
690 # Create a new reference catalog, and populate it only with records
691 # that fall inside the padded bbox.
692 filteredRefCat = type(refCat)(refCat.table)
693 centroidKey = afwTable.Point2DKey(refCat.schema['centroid'])
694 for record in refCat:
695 pixCoords = record[centroidKey]
696 if paddedBbox.contains(geom.Point2D(pixCoords)):
697 filteredRefCat.append(record)
698 return filteredRefCat
699 return self.loadRegion(outerSkyRegion, filterName, filtFunc=_filterFunction, epoch=epoch)
700
701 def loadRegion(self, region, filterName, filtFunc=None, epoch=None):
702 """Load reference objects within a specified region.
703
704 This function loads the DataIds used to construct an instance of this
705 class which intersect or are contained within the specified region. The
706 reference catalogs which intersect but are not fully contained within
707 the input region are further filtered by the specified filter function.
708 This function returns a single source catalog containing all reference
709 objects inside the specified region.
710
711 Parameters
712 ----------
713 region : `lsst.sphgeom.Region`
714 This can be any type that is derived from `lsst.sphgeom.Region` and
715 should define the spatial region for which reference objects are to
716 be loaded.
717 filtFunc : callable or `None`, optional
718 This optional parameter should be a callable object that takes a
719 reference catalog and its corresponding region as parameters,
720 filters the catalog by some criteria and returns the filtered
721 reference catalog. If `None`, an internal filter function is used
722 which filters according to if a reference object falls within the
723 input region.
724 filterName : `str`
725 Name of camera filter.
726 epoch : `astropy.time.Time` or `None`, optional
727 Epoch to which to correct proper motion and parallax, or `None` to
728 not apply such corrections.
729
730 Returns
731 -------
732 output : `lsst.pipe.base.Struct`
733 Results struct with attributes:
734
735 ``refCat``
736 Catalog containing reference objects which intersect the
737 input region, filtered by the specified filter function.
738 ``fluxField``
739 Name of the field containing the flux associated with
740 ``filterName``.
741
742 Raises
743 ------
744 RuntimeError
745 Raised if no reference catalogs could be found for the specified
746 region.
747 TypeError
748 Raised if the loaded reference catalogs do not have matching
749 schemas.
750 """
751 regionLat = region.getBoundingBox().getLat()
752 regionLon = region.getBoundingBox().getLon()
753 self.log.info("Loading reference objects from %s in region bounded by "
754 "[%.8f, %.8f], [%.8f, %.8f] RA Dec",
755 self.name,
756 regionLon.getA().asDegrees(), regionLon.getB().asDegrees(),
757 regionLat.getA().asDegrees(), regionLat.getB().asDegrees())
758 if filtFunc is None:
759 filtFunc = _FilterCatalog(region)
760 # filter out all the regions supplied by the constructor that do not overlap
761 overlapList = []
762 for dataId, refCat in zip(self.dataIds, self.refCats):
763 # SphGeom supports some objects intersecting others, but is not symmetric,
764 # try the intersect operation in both directions
765 try:
766 intersects = dataId.region.intersects(region)
767 except TypeError:
768 intersects = region.intersects(dataId.region)
769
770 if intersects:
771 overlapList.append((dataId, refCat))
772
773 if len(overlapList) == 0:
774 raise RuntimeError("No reference tables could be found for input region")
775
776 firstCat = overlapList[0][1].get()
777 refCat = filtFunc(firstCat, overlapList[0][0].region)
778 trimmedAmount = len(firstCat) - len(refCat)
779
780 # Load in the remaining catalogs
781 for dataId, inputRefCat in overlapList[1:]:
782 tmpCat = inputRefCat.get()
783
784 if tmpCat.schema != firstCat.schema:
785 raise TypeError("Reference catalogs have mismatching schemas")
786
787 filteredCat = filtFunc(tmpCat, dataId.region)
788 refCat.extend(filteredCat)
789 trimmedAmount += len(tmpCat) - len(filteredCat)
790
791 self.log.debug("Trimmed %d refCat objects lying outside padded region, leaving %d",
792 trimmedAmount, len(refCat))
793 self.log.info("Loaded %d reference objects", len(refCat))
794
795 # Ensure that the loaded reference catalog is continuous in memory
796 if not refCat.isContiguous():
797 refCat = refCat.copy(deep=True)
798
799 self.applyProperMotions(refCat, epoch)
800
801 # TODO DM-34793: remove this entire if block.
802 # Verify the schema is in the correct units and has the correct version; automatically convert
803 # it with a warning if this is not the case.
804 if not hasNanojanskyFluxUnits(refCat.schema) or not getFormatVersionFromRefCat(refCat) >= 1:
805 self.log.warning("Found version 0 reference catalog with old style units in schema.")
806 self.log.warning("run `meas_algorithms/bin/convert_refcat_to_nJy.py` to convert fluxes to nJy.")
807 self.log.warning("See RFC-575 for more details.")
808 refCat = convertToNanojansky(refCat, self.log)
809
810 expandedCat = self._remapReferenceCatalogSchema(refCat,
811 anyFilterMapsToThis=self.config.anyFilterMapsToThis,
812 filterMap=self.config.filterMap)
813
814 # Ensure that the returned reference catalog is continuous in memory
815 if not expandedCat.isContiguous():
816 expandedCat = expandedCat.copy(deep=True)
817
818 fluxField = getRefFluxField(expandedCat.schema, filterName)
819 return pipeBase.Struct(refCat=expandedCat, fluxField=fluxField)
820
821 def loadSkyCircle(self, ctrCoord, radius, filterName, epoch=None):
822 """Load reference objects that lie within a circular region on the sky.
823
824 This method constructs a circular region from an input center and
825 angular radius, loads reference catalogs which are contained in or
826 intersect the circle, and filters reference catalogs which intersect
827 down to objects which lie within the defined circle.
828
829 Parameters
830 ----------
831 ctrCoord : `lsst.geom.SpherePoint`
832 Point defining the center of the circular region.
833 radius : `lsst.geom.Angle`
834 Defines the angular radius of the circular region.
835 filterName : `str`
836 Name of camera filter.
837 epoch : `astropy.time.Time` or `None`, optional
838 Epoch to which to correct proper motion and parallax, or `None` to
839 not apply such corrections.
840
841 Returns
842 -------
843 output : `lsst.pipe.base.Struct`
844 Results struct with attributes:
845
846 ``refCat``
847 Catalog containing reference objects inside the specified
848 search circle.
849 ``fluxField``
850 Name of the field containing the flux associated with
851 ``filterName``.
852 """
853 centerVector = ctrCoord.getVector()
854 sphRadius = sphgeom.Angle(radius.asRadians())
855 circularRegion = sphgeom.Circle(centerVector, sphRadius)
856 return self.loadRegion(circularRegion, filterName, epoch=epoch)
857
858
859def getRefFluxField(schema, filterName):
860 """Get the name of a flux field from a schema.
861
862 Parameters
863 ----------
864 schema : `lsst.afw.table.Schema`
865 Reference catalog schema.
866 filterName : `str`
867 Name of camera filter.
868
869 Returns
870 -------
871 fluxFieldName : `str`
872 Name of flux field.
873
874 Notes
875 -----
876 Return the alias of ``anyFilterMapsToThis``, if present
877 else, return ``*filterName*_camFlux`` if present,
878 else, return ``*filterName*_flux`` if present (camera filter name
879 matches reference filter name), else raise an exception.
880
881 Raises
882 ------
883 RuntimeError
884 Raised if an appropriate field is not found.
885 """
886 if not isinstance(schema, afwTable.Schema):
887 raise RuntimeError("schema=%s is not a schema" % (schema,))
888 try:
889 return schema.getAliasMap().get("anyFilterMapsToThis")
890 except LookupError:
891 pass # try the filterMap next
892
893 fluxFieldList = [filterName + "_camFlux", filterName + "_flux"]
894 for fluxField in fluxFieldList:
895 if fluxField in schema:
896 return fluxField
897
898 raise RuntimeError("Could not find flux field(s) %s" % (", ".join(fluxFieldList)))
899
900
901def getRefFluxKeys(schema, filterName):
902 """Return keys for flux and flux error.
903
904 Parameters
905 ----------
906 schema : `lsst.afw.table.Schema`
907 Reference catalog schema.
908 filterName : `str`
909 Name of camera filter.
910
911 Returns
912 -------
913 keys : `tuple` of (`lsst.afw.table.Key`, `lsst.afw.table.Key`)
914 Two keys:
915
916 - flux key
917 - flux error key, if present, else None
918
919 Raises
920 ------
921 RuntimeError
922 If flux field not found.
923 """
924 fluxField = getRefFluxField(schema, filterName)
925 fluxErrField = fluxField + "Err"
926 fluxKey = schema[fluxField].asKey()
927 try:
928 fluxErrKey = schema[fluxErrField].asKey()
929 except Exception:
930 fluxErrKey = None
931 return (fluxKey, fluxErrKey)
932
933
934@deprecated(reason=("This task is used in gen2 only; it will be removed after v25. "
935 "See DM-35671 for details on updating code to avoid this warning."),
936 version="v25.0", category=FutureWarning)
937class LoadReferenceObjectsTask(pipeBase.Task, ReferenceObjectLoader, metaclass=abc.ABCMeta):
938 """Abstract gen2 base class to load objects from reference catalogs.
939 """
940 _DefaultName = "LoadReferenceObjects"
941
942 @abc.abstractmethod
943 def loadSkyCircle(self, ctrCoord, radius, filterName, epoch=None, centroids=False):
944 """Load reference objects that overlap a circular sky region.
945
946 Parameters
947 ----------
948 ctrCoord : `lsst.geom.SpherePoint`
949 ICRS center of search region.
950 radius : `lsst.geom.Angle`
951 Radius of search region.
952 filterName : `str`
953 Name of filter. This can be used for flux limit comparisons.
954 epoch : `astropy.time.Time` or `None`, optional
955 Epoch to which to correct proper motion and parallax, or `None` to
956 not apply such corrections.
957 centroids : `bool`, optional
958 Add centroid fields to the loaded Schema. ``loadPixelBox`` expects
959 these fields to exist.
960
961 Returns
962 -------
963 results : `lsst.pipe.base.Struct`
964 A `~lsst.pipe.base.Struct` containing the following fields:
965
966 ``refCat``
967 A catalog of reference objects with the standard
968 schema, as documented in the main doc string for
969 `LoadReferenceObjects`.
970 The catalog is guaranteed to be contiguous.
971 (`lsst.afw.catalog.SimpleCatalog`)
972 ``fluxField``
973 Name of flux field for specified `filterName`. (`str`)
974
975 Notes
976 -----
977 Note that subclasses are responsible for performing the proper motion
978 correction, since this is the lowest-level interface for retrieving
979 the catalog.
980 """
981 return
982
983
984@deprecated(reason="Base class only used for gen2 interface, and will be removed after v25.0. "
985 "Please use ReferenceObjectLoader directly.",
986 version="v25.0", category=FutureWarning)
988 """Stub of a deprecated class.
989
990 Parameters
991 ----------
992 config : `lsst.pex.config.Config`
993 Configuration for the loader.
994 """
995 def __init__(self, config=None, *args, **kwargs):
996 pass
997
998
999def applyProperMotionsImpl(log, catalog, epoch):
1000 """Apply proper motion correction to a reference catalog.
1001
1002 Adjust position and position error in the ``catalog``
1003 for proper motion to the specified ``epoch``,
1004 modifying the catalog in place.
1005
1006 Parameters
1007 ----------
1008 log : `lsst.log.Log` or `logging.getLogger`
1009 Log object to write to.
1011 Catalog of positions, containing:
1012
1013 - Coordinates, retrieved by the table's coordinate key.
1014 - ``coord_raErr`` : Error in Right Ascension (rad).
1015 - ``coord_decErr`` : Error in Declination (rad).
1016 - ``pm_ra`` : Proper motion in Right Ascension (rad/yr,
1017 East positive)
1018 - ``pm_raErr`` : Error in ``pm_ra`` (rad/yr), optional.
1019 - ``pm_dec`` : Proper motion in Declination (rad/yr,
1020 North positive)
1021 - ``pm_decErr`` : Error in ``pm_dec`` (rad/yr), optional.
1022 - ``epoch`` : Mean epoch of object (an astropy.time.Time)
1023 epoch : `astropy.time.Time`
1024 Epoch to which to correct proper motion.
1025 """
1026 if "epoch" not in catalog.schema or "pm_ra" not in catalog.schema or "pm_dec" not in catalog.schema:
1027 log.warning("Proper motion correction not available from catalog")
1028 return
1029 if not catalog.isContiguous():
1030 raise RuntimeError("Catalog must be contiguous")
1031 catEpoch = astropy.time.Time(catalog["epoch"], scale="tai", format="mjd")
1032 log.info("Correcting reference catalog for proper motion to %r", epoch)
1033 # Use `epoch.tai` to make sure the time difference is in TAI
1034 timeDiffsYears = (epoch.tai - catEpoch).to(astropy.units.yr).value
1035 coordKey = catalog.table.getCoordKey()
1036 # Compute the offset of each object due to proper motion
1037 # as components of the arc of a great circle along RA and Dec
1038 pmRaRad = catalog["pm_ra"]
1039 pmDecRad = catalog["pm_dec"]
1040 offsetsRaRad = pmRaRad*timeDiffsYears
1041 offsetsDecRad = pmDecRad*timeDiffsYears
1042 # Compute the corresponding bearing and arc length of each offset
1043 # due to proper motion, and apply the offset
1044 # The factor of 1e6 for computing bearing is intended as
1045 # a reasonable scale for typical values of proper motion
1046 # in order to avoid large errors for small values of proper motion;
1047 # using the offsets is another option, but it can give
1048 # needlessly large errors for short duration
1049 offsetBearingsRad = numpy.arctan2(pmDecRad*1e6, pmRaRad*1e6)
1050 offsetAmountsRad = numpy.hypot(offsetsRaRad, offsetsDecRad)
1051 for record, bearingRad, amountRad in zip(catalog, offsetBearingsRad, offsetAmountsRad):
1052 record.set(coordKey,
1053 record.get(coordKey).offset(bearing=bearingRad*geom.radians,
1054 amount=amountRad*geom.radians))
1055 # TODO DM-36979: this needs to incorporate the full covariance!
1056 # Increase error in RA and Dec based on error in proper motion
1057 if "coord_raErr" in catalog.schema:
1058 catalog["coord_raErr"] = numpy.hypot(catalog["coord_raErr"],
1059 catalog["pm_raErr"]*timeDiffsYears)
1060 if "coord_decErr" in catalog.schema:
1061 catalog["coord_decErr"] = numpy.hypot(catalog["coord_decErr"],
1062 catalog["pm_decErr"]*timeDiffsYears)
def loadSkyCircle(self, ctrCoord, radius, filterName, epoch=None, centroids=False)
def __init__(self, dataIds, refCats, name=None, log=None, config=None, **kwargs)
def loadSkyCircle(self, ctrCoord, radius, filterName, epoch=None)
def _remapReferenceCatalogSchema(refCat, *anyFilterMapsToThis=None, filterMap=None, centroids=False)
def loadRegion(self, region, filterName, filtFunc=None, epoch=None)
def getMetadataBox(self, bbox, wcs, filterName, epoch=None, bboxToSpherePadding=100)
def loadPixelBox(self, bbox, wcs, filterName, epoch=None, bboxToSpherePadding=100)
def convertToNanojansky(catalog, log, doConvert=True)