24 __all__ = [
"getRefFluxField",
"getRefFluxKeys",
"LoadReferenceObjectsTask",
"LoadReferenceObjectsConfig",
25 "ReferenceObjectLoader"]
42 from lsst
import sphgeom
47 """Return True if this name/units combination corresponds to an
48 "old-style" reference catalog flux field.
50 unitsCheck = units !=
'nJy'
51 isFlux = name.endswith(
'_flux')
52 isFluxSigma = name.endswith(
'_fluxSigma')
53 isFluxErr = name.endswith(
'_fluxErr')
54 return (isFlux
or isFluxSigma
or isFluxErr)
and unitsCheck
58 """Return True if the units of all flux and fluxErr are correct (nJy).
67 """"Return the format version stored in a reference catalog header.
71 refCat : `lsst.afw.table.SimpleCatalog`
72 Reference catalog to inspect.
76 version : `int` or `None`
77 Format version integer, or `None` if the catalog has no metadata
78 or the metadata does not include a "REFCAT_FORMAT_VERSION" key.
80 md = refCat.getMetadata()
84 return md.getScalar(
"REFCAT_FORMAT_VERSION")
90 """Convert fluxes in a catalog from jansky to nanojansky.
94 catalog : `lsst.afw.table.SimpleCatalog`
95 The catalog to convert.
97 Log to send messages to.
98 doConvert : `bool`, optional
99 Return a converted catalog, or just identify the fields that need to be converted?
100 This supports the "write=False" mode of `bin/convert_to_nJy.py`.
104 catalog : `lsst.afw.table.SimpleCatalog` or None
105 The converted catalog, or None if ``doConvert`` is False.
109 Support for old units in reference catalogs will be removed after the
110 release of late calendar year 2019.
111 Use `meas_algorithms/bin/convert_to_nJy.py` to update your reference catalog.
119 for field
in catalog.schema:
120 oldName = field.field.getName()
121 oldUnits = field.field.getUnits()
125 if oldName.endswith(
'_fluxSigma'):
126 name = oldName.replace(
'_fluxSigma',
'_fluxErr')
130 mapper.addMapping(field.getKey(), newField)
131 input_fields.append(field.field)
132 output_fields.append(newField)
134 mapper.addMapping(field.getKey())
136 fluxFieldsStr =
'; '.join(
"(%s, '%s')" % (field.getName(), field.getUnits())
for field
in input_fields)
139 newSchema = mapper.getOutputSchema()
141 output.extend(catalog, mapper=mapper)
142 for field
in output_fields:
143 output[field.getName()] *= 1e9
144 log.info(f
"Converted refcat flux fields to nJy (name, units): {fluxFieldsStr}")
147 log.info(f
"Found old-style refcat flux fields (name, units): {fluxFieldsStr}")
152 """This is a private helper class which filters catalogs by
153 row based on the row being inside the region used to initialize
158 region : `lsst.sphgeom.Region`
159 The spatial region which all objects should lie within
165 """This call method on an instance of this class takes in a reference
166 catalog, and the region from which the catalog was generated.
168 If the catalog region is entirely contained within the region used to
169 initialize this class, then all the entries in the catalog must be
170 within the region and so the whole catalog is returned.
172 If the catalog region is not entirely contained, then the location for
173 each record is tested against the region used to initialize the class.
174 Records which fall inside this region are added to a new catalog, and
175 this catalog is then returned.
179 refCat : `lsst.afw.table.SourceCatalog`
180 SourceCatalog to be filtered.
181 catRegion : `lsst.sphgeom.Region`
182 Region in which the catalog was created
184 if catRegion.isWithin(self.
regionregion):
188 filteredRefCat = type(refCat)(refCat.table)
189 for record
in refCat:
190 if self.
regionregion.contains(record.getCoord().getVector()):
191 filteredRefCat.append(record)
192 return filteredRefCat
196 """Base class for reference object loaders, to facilitate gen2/gen3 code
200 """Apply proper motion correction to a reference catalog.
202 Adjust position and position error in the ``catalog``
203 for proper motion to the specified ``epoch``,
204 modifying the catalog in place.
208 catalog : `lsst.afw.table.SimpleCatalog`
209 Catalog of positions, containing at least these fields:
211 - Coordinates, retrieved by the table's coordinate key.
212 - ``coord_raErr`` : Error in Right Ascension (rad).
213 - ``coord_decErr`` : Error in Declination (rad).
214 - ``pm_ra`` : Proper motion in Right Ascension (rad/yr,
216 - ``pm_raErr`` : Error in ``pm_ra`` (rad/yr), optional.
217 - ``pm_dec`` : Proper motion in Declination (rad/yr,
219 - ``pm_decErr`` : Error in ``pm_dec`` (rad/yr), optional.
220 - ``epoch`` : Mean epoch of object (an astropy.time.Time)
221 epoch : `astropy.time.Time`
222 Epoch to which to correct proper motion.
223 If None, do not apply PM corrections or raise if
224 ``config.requireProperMotion`` is True.
229 Raised if ``config.requireProperMotion`` is set but we cannot
230 apply the proper motion correction for some reason.
233 if self.config.requireProperMotion:
234 raise RuntimeError(
"requireProperMotion=True but epoch not provided to loader.")
236 self.log.debug(
"No epoch provided: not applying proper motion corrections to refcat.")
240 if (
"pm_ra" in catalog.schema
241 and not isinstance(catalog.schema[
"pm_ra"].asKey(), lsst.afw.table.KeyAngle)):
242 if self.config.requireProperMotion:
243 raise RuntimeError(
"requireProperMotion=True but refcat pm_ra field is not an Angle.")
245 self.log.warn(
"Reference catalog pm_ra field is not an Angle; cannot apply proper motion.")
248 if (
"epoch" not in catalog.schema
or "pm_ra" not in catalog.schema):
249 if self.config.requireProperMotion:
250 raise RuntimeError(
"requireProperMotion=True but PM data not available from catalog.")
252 self.log.warn(
"Proper motion correction not available for this reference catalog.")
259 """ This class facilitates loading reference catalogs with gen 3 middleware
261 The middleware preflight solver will create a list of datarefs that may
262 possibly overlap a given region. These datarefs are then used to construct
263 and instance of this class. The class instance should then be passed into
264 a task which needs reference catalogs. These tasks should then determine
265 the exact region of the sky reference catalogs will be loaded for, and
266 call a corresponding method to load the reference objects.
268 def __init__(self, dataIds, refCats, config, log=None):
269 """ Constructs an instance of ReferenceObjectLoader
273 dataIds : iterable of `lsst.daf.butler.DataIds`
274 An iterable object of DataSetRefs which point to reference catalogs
275 in a gen 3 repository
276 refCats : Iterable of `lsst.daf.butler.DeferedDatasetHandle`
277 Handles to load refCats on demand
279 Logger object used to write out messages. If `None` (default) the default
280 lsst logger will be used
289 def _makeBoxRegion(BBox, wcs, BBoxPadding):
296 outerLocalBBox.grow(BBoxPadding)
297 innerLocalBBox.grow(-1*BBoxPadding)
305 innerBoxCorners = innerLocalBBox.getCorners()
306 innerSphCorners = [wcs.pixelToSky(corner).getVector()
for corner
in innerBoxCorners]
311 outerBoxCorners = outerLocalBBox.getCorners()
312 outerSphCorners = [wcs.pixelToSky(corner).getVector()
for corner
in outerBoxCorners]
316 return innerSkyRegion, outerSkyRegion, innerSphCorners, outerSphCorners
318 def loadPixelBox(self, bbox, wcs, filterName=None, epoch=None, photoCalib=None, bboxPadding=100):
319 """Load reference objects that are within a pixel-based rectangular region
321 This algorithm works by creating a spherical box whose corners correspond
322 to the WCS converted corners of the input bounding box (possibly padded).
323 It then defines a filtering function which will look at a reference
324 objects pixel position and accept objects that lie within the specified
327 The spherical box region and filtering function are passed to the generic
328 loadRegion method which will load and filter the reference objects from
329 the datastore and return a single catalog containing all reference objects
333 bbox : `lsst.geom.box2I`
334 Box which bounds a region in pixel space
335 wcs : `lsst.afw.geom.SkyWcs`
336 Wcs object defining the pixel to sky (and inverse) transform for the space
337 of pixels of the supplied bbox
339 Name of camera filter, or None or blank for the default filter
340 epoch : `astropy.time.Time` (optional)
341 Epoch to which to correct proper motion and parallax,
342 or None to not apply such corrections.
344 Deprecated and ignored, only included for api compatibility
346 Number describing how much to pad the input bbox by (in pixels), defaults
347 to 100. This parameter is necessary because optical distortions in telescopes
348 can cause a rectangular pixel grid to map into a non "rectangular" spherical
349 region in sky coordinates. This padding is used to create a spherical
350 "rectangle", which will for sure enclose the input box. This padding is only
351 used to determine if the reference catalog for a sky patch will be loaded from
352 the data store, this function will filter out objects which lie within the
353 padded region but fall outside the input bounding box region.
357 referenceCatalog : `lsst.afw.table.SimpleCatalog`
358 Catalog containing reference objects inside the specified bounding box
362 `lsst.pex.exception.RuntimeError`
363 Raised if no reference catalogs could be found for the specified region
365 `lsst.pex.exception.TypeError`
366 Raised if the loaded reference catalogs do not have matching schemas
368 innerSkyRegion, outerSkyRegion, _, _ = self.
_makeBoxRegion_makeBoxRegion(bbox, wcs, bboxPadding)
370 def _filterFunction(refCat, region):
374 afwTable.updateRefCentroids(wcs, refCat)
377 if innerSkyRegion.contains(region):
380 filteredRefCat = type(refCat)(refCat.table)
381 centroidKey = afwTable.Point2DKey(refCat.schema[
'centroid'])
382 for record
in refCat:
383 pixCoords = record[centroidKey]
385 filteredRefCat.append(record)
386 return filteredRefCat
387 return self.
loadRegionloadRegion(outerSkyRegion, filtFunc=_filterFunction, epoch=epoch, filterName=filterName)
389 def loadRegion(self, region, filtFunc=None, filterName=None, epoch=None):
390 """ Load reference objects within a specified region
392 This function loads the DataIds used to construct an instance of this class
393 which intersect or are contained within the specified region. The reference
394 catalogs which intersect but are not fully contained within the input region are
395 further filtered by the specified filter function. This function will return a
396 single source catalog containing all reference objects inside the specified region.
400 region : `lsst.sphgeom.Region`
401 This can be any type that is derived from `lsst.sphgeom.region` and should
402 define the spatial region for which reference objects are to be loaded.
404 This optional parameter should be a callable object that takes a reference
405 catalog and its corresponding region as parameters, filters the catalog by
406 some criteria and returns the filtered reference catalog. If the value is
407 left as the default (None) than an internal filter function is used which
408 filters according to if a reference object falls within the input region.
410 Name of camera filter, or None or blank for the default filter
411 epoch : `astropy.time.Time` (optional)
412 Epoch to which to correct proper motion and parallax,
413 or None to not apply such corrections.
417 referenceCatalog : `lsst.afw.table.SourceCatalog`
418 Catalog containing reference objects which intersect the input region,
419 filtered by the specified filter function
423 `lsst.pex.exception.RuntimeError`
424 Raised if no reference catalogs could be found for the specified region
426 `lsst.pex.exception.TypeError`
427 Raised if the loaded reference catalogs do not have matching schemas
430 regionBounding = region.getBoundingBox()
431 self.
loglog.info(
"Loading reference objects from region bounded by {}, {} lat lon".format(
432 regionBounding.getLat(), regionBounding.getLon()))
437 for dataId, refCat
in zip(self.
dataIdsdataIds, self.
refCatsrefCats):
441 intersects = dataId.region.intersects(region)
443 intersects = region.intersects(dataId.region)
446 overlapList.append((dataId, refCat))
448 if len(overlapList) == 0:
449 raise pexExceptions.RuntimeError(
"No reference tables could be found for input region")
451 firstCat = overlapList[0][1].get()
452 refCat = filtFunc(firstCat, overlapList[0][0].region)
453 trimmedAmount = len(firstCat) - len(refCat)
456 for dataId, inputRefCat
in overlapList[1:]:
457 tmpCat = inputRefCat.get()
459 if tmpCat.schema != firstCat.schema:
460 raise pexExceptions.TypeError(
"Reference catalogs have mismatching schemas")
462 filteredCat = filtFunc(tmpCat, dataId.region)
463 refCat.extend(filteredCat)
464 trimmedAmount += len(tmpCat) - len(filteredCat)
466 self.
loglog.debug(f
"Trimmed {trimmedAmount} out of region objects, leaving {len(refCat)}")
467 self.
loglog.info(f
"Loaded {len(refCat)} reference objects")
470 if not refCat.isContiguous():
471 refCat = refCat.copy(deep=
True)
478 self.
loglog.warn(
"Found version 0 reference catalog with old style units in schema.")
479 self.
loglog.warn(
"run `meas_algorithms/bin/convert_refcat_to_nJy.py` to convert fluxes to nJy.")
480 self.
loglog.warn(
"See RFC-575 for more details.")
489 if not expandedCat.isContiguous():
490 expandedCat = expandedCat.copy(deep=
True)
492 fluxField =
getRefFluxField(schema=expandedCat.schema, filterName=filterName)
493 return pipeBase.Struct(refCat=expandedCat, fluxField=fluxField)
496 """Load reference objects that lie within a circular region on the sky
498 This method constructs a circular region from an input center and angular radius,
499 loads reference catalogs which are contained in or intersect the circle, and
500 filters reference catalogs which intersect down to objects which lie within
505 ctrCoord : `lsst.geom.SpherePoint`
506 Point defining the center of the circular region
507 radius : `lsst.geom.Angle`
508 Defines the angular radius of the circular region
510 Name of camera filter, or None or blank for the default filter
511 epoch : `astropy.time.Time` (optional)
512 Epoch to which to correct proper motion and parallax,
513 or None to not apply such corrections.
517 referenceCatalog : `lsst.afw.table.SourceCatalog`
518 Catalog containing reference objects inside the specified bounding box
522 `lsst.pex.exception.RuntimeError`
523 Raised if no reference catalogs could be found for the specified region
525 `lsst.pex.exception.TypeError`
526 Raised if the loaded reference catalogs do not have matching schemas
529 centerVector = ctrCoord.getVector()
532 return self.
loadRegionloadRegion(circularRegion, filterName=filterName, epoch=epoch)
535 """Relink an unpersisted match list to sources and reference
538 A match list is persisted and unpersisted as a catalog of IDs
539 produced by afw.table.packMatches(), with match metadata
540 (as returned by the astrometry tasks) in the catalog's metadata
541 attribute. This method converts such a match catalog into a match
542 list, with links to source records and reference object records.
546 matchCat : `lsst.afw.table.BaseCatalog`
547 Unpersisted packed match list.
548 ``matchCat.table.getMetadata()`` must contain match metadata,
549 as returned by the astrometry tasks.
550 sourceCat : `lsst.afw.table.SourceCatalog`
551 Source catalog. As a side effect, the catalog will be sorted
556 matchList : `lsst.afw.table.ReferenceMatchVector`
562 def getMetadataBox(cls, bbox, wcs, filterName=None, photoCalib=None, epoch=None, bboxPadding=100):
563 """Return metadata about the load
565 This metadata is used for reloading the catalog (e.g., for
566 reconstituting a normalised match list.)
570 bbox : `lsst.geom.Box2I`
571 Bounding bos for the pixels
572 wcs : `lsst.afw.geom.SkyWcs
574 filterName : `str` or None
575 filterName of the camera filter, or None or blank for the default filter
577 Deprecated, only included for api compatibility
578 epoch : `astropy.time.Time` (optional)
579 Epoch to which to correct proper motion and parallax,
580 or None to not apply such corrections.
582 Number describing how much to pad the input bbox by (in pixels), defaults
583 to 100. This parameter is necessary because optical distortions in telescopes
584 can cause a rectangular pixel grid to map into a non "rectangular" spherical
585 region in sky coordinates. This padding is used to create a spherical
586 "rectangle", which will for sure enclose the input box. This padding is only
587 used to determine if the reference catalog for a sky patch will be loaded from
588 the data store, this function will filter out objects which lie within the
589 padded region but fall outside the input bounding box region.
592 md : `lsst.daf.base.PropertyList`
594 _, _, innerCorners, outerCorners = cls.
_makeBoxRegion_makeBoxRegion(bbox, wcs, bboxPadding)
596 for box, corners
in zip((
"INNER",
"OUTER"), (innerCorners, outerCorners)):
597 for (name, corner)
in zip((
"UPPER_LEFT",
"UPPER_RIGHT",
"LOWER_LEFT",
"LOWER_RIGHT"),
599 md.add(f
"{box}_{name}_RA",
geom.SpherePoint(corner).getRa().asDegrees(), f
"{box}_corner")
600 md.add(f
"{box}_{name}_DEC",
geom.SpherePoint(corner).getDec().asDegrees(), f
"{box}_corner")
601 md.add(
"SMATCHV", 1,
'SourceMatchVector version number')
602 filterName =
"UNKNOWN" if filterName
is None else str(filterName)
603 md.add(
'FILTER', filterName,
'filter name for photometric data')
604 md.add(
'EPOCH',
"NONE" if epoch
is None else epoch.mjd,
'Epoch (TAI MJD) for catalog')
609 """Return metadata about the load
611 This metadata is used for reloading the catalog (e.g. for reconstituting
612 a normalized match list.)
616 coord : `lsst.geom.SpherePoint`
617 ICRS center of a circle
618 radius : `lsst.geom.angle`
620 filterName : `str` or None
621 filterName of the camera filter, or None or blank for the default filter
623 Deprecated, only included for api compatibility
624 epoch : `astropy.time.Time` (optional)
625 Epoch to which to correct proper motion and parallax,
626 or None to not apply such corrections.
630 md : `lsst.daf.base.PropertyList`
633 md.add(
'RA', coord.getRa().asDegrees(),
'field center in degrees')
634 md.add(
'DEC', coord.getDec().asDegrees(),
'field center in degrees')
635 md.add(
'RADIUS', radius.asDegrees(),
'field radius in degrees, minimum')
636 md.add(
'SMATCHV', 1,
'SourceMatchVector version number')
637 filterName =
"UNKNOWN" if filterName
is None else str(filterName)
638 md.add(
'FILTER', filterName,
'filter name for photometric data')
639 md.add(
'EPOCH',
"NONE" if epoch
is None else epoch.mjd,
'Epoch (TAI MJD) for catalog')
644 """This function creates a new catalog containing the information of the input refCat
645 as well as added flux columns and aliases between camera and reference flux.
649 refCat : `lsst.afw.table.SimpleCatalog`
650 Catalog of reference objects
651 defaultFilter : `str`
652 Name of the default reference filter
653 filterReferenceMap : `dict` of `str`
654 Dictionary with keys corresponding to a filter name, and values which
655 correspond to the name of the reference filter.
659 refCat : `lsst.afw.table.SimpleCatalog`
660 Reference catalog with columns added to track reference filters
665 If specified reference filter name is not a filter specifed as a key in the
666 reference filter map.
668 refCat = ReferenceObjectLoader.remapReferenceCatalogSchema(refCat,
669 filterNameList=filterReferenceMap.keys())
670 aliasMap = refCat.schema.getAliasMap()
671 if filterReferenceMap
is None:
672 filterReferenceMap = {}
673 for filterName, refFilterName
in itertools.chain([(
None, defaultFilter)],
674 filterReferenceMap.items()):
676 camFluxName = filterName +
"_camFlux" if filterName
is not None else "camFlux"
677 refFluxName = refFilterName +
"_flux"
678 if refFluxName
not in refCat.schema:
679 raise RuntimeError(
"Unknown reference filter %s" % (refFluxName,))
680 aliasMap.set(camFluxName, refFluxName)
682 refFluxErrName = refFluxName +
"Err"
683 camFluxErrName = camFluxName +
"Err"
684 aliasMap.set(camFluxErrName, refFluxErrName)
690 """This function takes in a reference catalog and creates a new catalog with additional
691 columns defined the remaining function arguments.
695 refCat : `lsst.afw.table.SimpleCatalog`
696 Reference catalog to map to new catalog
700 expandedCat : `lsst.afw.table.SimpleCatalog`
701 Deep copy of input reference catalog with additional columns added
703 mapper = afwTable.SchemaMapper(refCat.schema,
True)
704 mapper.addMinimalSchema(refCat.schema,
True)
705 mapper.editOutputSchema().disconnectAliases()
707 for filterName
in filterNameList:
708 mapper.editOutputSchema().addField(f
"{filterName}_flux",
710 doc=f
"flux in filter {filterName}",
713 mapper.editOutputSchema().addField(f
"{filterName}_fluxErr",
715 doc=f
"flux uncertanty in filter {filterName}",
720 mapper.editOutputSchema().addField(
"centroid_x", type=float, doReplace=
True)
721 mapper.editOutputSchema().addField(
"centroid_y", type=float, doReplace=
True)
722 mapper.editOutputSchema().addField(
"hasCentroid", type=
"Flag", doReplace=
True)
723 mapper.editOutputSchema().getAliasMap().set(
"slot_Centroid",
"centroid")
726 mapper.editOutputSchema().addField(
"photometric",
728 doc=
"set if the object can be used for photometric"
731 mapper.editOutputSchema().addField(
"resolved",
733 doc=
"set if the object is spatially resolved"
735 mapper.editOutputSchema().addField(
"variable",
737 doc=
"set if the object has variable brightness"
740 expandedCat = afwTable.SimpleCatalog(mapper.getOutputSchema())
741 expandedCat.setMetadata(refCat.getMetadata())
742 expandedCat.extend(refCat, mapper=mapper)
748 """Get the name of a flux field from a schema.
750 return the alias of "anyFilterMapsToThis", if present
751 else if filterName is specified:
752 return "*filterName*_camFlux" if present
753 else return "*filterName*_flux" if present (camera filter name
754 matches reference filter name)
755 else throw RuntimeError
757 return "camFlux", if present,
758 else throw RuntimeError
762 schema : `lsst.afw.table.Schema`
763 Reference catalog schema.
764 filterName : `str`, optional
765 Name of camera filter. If not specified, ``defaultFilter`` needs to be
766 set in the refcat loader config.
770 fluxFieldName : `str`
776 If an appropriate field is not found.
778 if not isinstance(schema, afwTable.Schema):
779 raise RuntimeError(
"schema=%s is not a schema" % (schema,))
781 return schema.getAliasMap().get(
"anyFilterMapsToThis")
786 fluxFieldList = [filterName +
"_camFlux", filterName +
"_flux"]
788 fluxFieldList = [
"camFlux"]
789 for fluxField
in fluxFieldList:
790 if fluxField
in schema:
793 raise RuntimeError(
"Could not find flux field(s) %s" % (
", ".join(fluxFieldList)))
797 """Return keys for flux and flux error.
801 schema : `lsst.afw.table.Schema`
802 Reference catalog schema.
804 Name of camera filter.
808 keys : `tuple` of (`lsst.afw.table.Key`, `lsst.afw.table.Key`)
812 - flux error key, if present, else None
817 If flux field not found.
820 fluxErrField = fluxField +
"Err"
821 fluxKey = schema[fluxField].asKey()
823 fluxErrKey = schema[fluxErrField].asKey()
826 return (fluxKey, fluxErrKey)
830 pixelMargin = pexConfig.RangeField(
831 doc=
"Padding to add to 4 all edges of the bounding box (pixels)",
836 defaultFilter = pexConfig.Field(
837 doc=(
"Default reference catalog filter to use if filter not specified in exposure;"
838 " if blank then filter must be specified in exposure."),
841 deprecated=
"defaultFilter is deprecated by RFC-716. Will be removed after v22."
843 anyFilterMapsToThis = pexConfig.Field(
844 doc=(
"Always use this reference catalog filter, no matter whether or what filter name is "
845 "supplied to the loader. Effectively a trivial filterMap: map all filter names to this filter."
846 " This can be set for purely-astrometric catalogs (e.g. Gaia DR2) where there is only one "
847 "reasonable choice for every camera filter->refcat mapping, but not for refcats used for "
848 "photometry, which need a filterMap and/or colorterms/transmission corrections."),
853 filterMap = pexConfig.DictField(
854 doc=(
"Mapping of camera filter name: reference catalog filter name; "
855 "each reference filter must exist in the refcat."
856 " Note that this does not perform any bandpass corrections: it is just a lookup."),
861 requireProperMotion = pexConfig.Field(
862 doc=
"Require that the fields needed to correct proper motion "
863 "(epoch, pm_ra and pm_dec) are present?",
871 msg =
"`filterMap` and `anyFilterMapsToThis` are mutually exclusive"
872 raise pexConfig.FieldValidationError(LoadReferenceObjectsConfig.anyFilterMapsToThis,
877 """Abstract base class to load objects from reference catalogs.
879 ConfigClass = LoadReferenceObjectsConfig
880 _DefaultName =
"LoadReferenceObjects"
883 """Construct a LoadReferenceObjectsTask
887 butler : `lsst.daf.persistence.Butler`
888 Data butler, for access reference catalogs.
890 pipeBase.Task.__init__(self, *args, **kwargs)
894 def loadPixelBox(self, bbox, wcs, filterName=None, photoCalib=None, epoch=None):
895 """Load reference objects that overlap a rectangular pixel region.
899 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
900 Bounding box for pixels.
901 wcs : `lsst.afw.geom.SkyWcs`
902 WCS; used to convert pixel positions to sky coordinates
905 Name of filter, or `None` or `""` for the default filter.
906 This is used for flux values in case we have flux limits
907 (which are not yet implemented).
908 photoCalib : `lsst.afw.image.PhotoCalib` (optional)
909 Calibration, or `None` if unknown.
910 epoch : `astropy.time.Time` (optional)
911 Epoch to which to correct proper motion and parallax,
912 or None to not apply such corrections.
916 results : `lsst.pipe.base.Struct`
917 A Struct containing the following fields:
918 refCat : `lsst.afw.catalog.SimpleCatalog`
919 A catalog of reference objects with the standard
920 schema, as documented in the main doc string for
921 `LoadReferenceObjects`.
922 The catalog is guaranteed to be contiguous.
924 Name of flux field for specified `filterName`.
928 The search algorithm works by searching in a region in sky
929 coordinates whose center is the center of the bbox and radius
930 is large enough to just include all 4 corners of the bbox.
931 Stars that lie outside the bbox are then trimmed from the list.
936 self.log.info(
"Loading reference objects using center %s and radius %s deg" %
937 (circle.coord, circle.radius.asDegrees()))
938 loadRes = self.
loadSkyCircleloadSkyCircle(circle.coord, circle.radius, filterName=filterName, epoch=epoch,
940 refCat = loadRes.refCat
941 numFound = len(refCat)
944 refCat = self.
_trimToBBox_trimToBBox(refCat=refCat, bbox=circle.bbox, wcs=wcs)
945 numTrimmed = numFound - len(refCat)
946 self.log.debug(
"trimmed %d out-of-bbox objects, leaving %d", numTrimmed, len(refCat))
947 self.log.info(
"Loaded %d reference objects", len(refCat))
950 if not refCat.isContiguous():
951 loadRes.refCat = refCat.copy(deep=
True)
956 def loadSkyCircle(self, ctrCoord, radius, filterName=None, epoch=None, centroids=False):
957 """Load reference objects that overlap a circular sky region.
961 ctrCoord : `lsst.geom.SpherePoint`
962 ICRS center of search region.
963 radius : `lsst.geom.Angle`
964 Radius of search region.
965 filterName : `str` (optional)
966 Name of filter, or `None` or `""` for the default filter.
967 This is used for flux values in case we have flux limits
968 (which are not yet implemented).
969 epoch : `astropy.time.Time` (optional)
970 Epoch to which to correct proper motion and parallax,
971 or None to not apply such corrections.
972 centroids : `bool` (optional)
973 Add centroid fields to the loaded Schema. ``loadPixelBox`` expects
974 these fields to exist.
978 results : `lsst.pipe.base.Struct`
979 A Struct containing the following fields:
980 refCat : `lsst.afw.catalog.SimpleCatalog`
981 A catalog of reference objects with the standard
982 schema, as documented in the main doc string for
983 `LoadReferenceObjects`.
984 The catalog is guaranteed to be contiguous.
986 Name of flux field for specified `filterName`.
990 Note that subclasses are responsible for performing the proper motion
991 correction, since this is the lowest-level interface for retrieving
997 def _trimToBBox(refCat, bbox, wcs):
998 """Remove objects outside a given pixel bounding box and set
999 centroid and hasCentroid fields.
1003 refCat : `lsst.afw.table.SimpleCatalog`
1004 A catalog of objects. The schema must include fields
1005 "coord", "centroid" and "hasCentroid".
1006 The "coord" field is read.
1007 The "centroid" and "hasCentroid" fields are set.
1008 bbox : `lsst.geom.Box2D`
1010 wcs : `lsst.afw.geom.SkyWcs`
1011 WCS; used to convert sky coordinates to pixel positions.
1015 catalog : `lsst.afw.table.SimpleCatalog`
1016 Reference objects in the bbox, with centroid and
1017 hasCentroid fields set.
1019 afwTable.updateRefCentroids(wcs, refCat)
1020 centroidKey = afwTable.Point2DKey(refCat.schema[
"centroid"])
1021 retStarCat = type(refCat)(refCat.table)
1023 point = star.get(centroidKey)
1024 if bbox.contains(point):
1025 retStarCat.append(star)
1028 def _addFluxAliases(self, schema):
1029 """Add aliases for camera filter fluxes to the schema.
1031 If self.config.defaultFilter then adds these aliases:
1032 camFlux: <defaultFilter>_flux
1033 camFluxErr: <defaultFilter>_fluxErr, if the latter exists
1035 For each camFilter: refFilter in self.config.filterMap adds these aliases:
1036 <camFilter>_camFlux: <refFilter>_flux
1037 <camFilter>_camFluxErr: <refFilter>_fluxErr, if the latter exists
1041 schema : `lsst.afw.table.Schema`
1042 Schema for reference catalog.
1047 If any reference flux field is missing from the schema.
1049 aliasMap = schema.getAliasMap()
1051 if self.config.anyFilterMapsToThis
is not None:
1052 refFluxName = self.config.anyFilterMapsToThis +
"_flux"
1053 if refFluxName
not in schema:
1054 msg = f
"Unknown reference filter for anyFilterMapsToThis='{refFluxName}'"
1055 raise RuntimeError(msg)
1056 aliasMap.set(
"anyFilterMapsToThis", refFluxName)
1059 def addAliasesForOneFilter(filterName, refFilterName):
1060 """Add aliases for a single filter
1064 filterName : `str` (optional)
1065 Camera filter name. The resulting alias name is
1066 <filterName>_camFlux, or simply "camFlux" if `filterName`
1068 refFilterName : `str`
1069 Reference catalog filter name; the field
1070 <refFilterName>_flux must exist.
1072 camFluxName = filterName +
"_camFlux" if filterName
is not None else "camFlux"
1073 refFluxName = refFilterName +
"_flux"
1074 if refFluxName
not in schema:
1075 raise RuntimeError(
"Unknown reference filter %s" % (refFluxName,))
1076 aliasMap.set(camFluxName, refFluxName)
1077 refFluxErrName = refFluxName +
"Err"
1078 if refFluxErrName
in schema:
1079 camFluxErrName = camFluxName +
"Err"
1080 aliasMap.set(camFluxErrName, refFluxErrName)
1082 if self.config.defaultFilter:
1083 addAliasesForOneFilter(
None, self.config.defaultFilter)
1085 for filterName, refFilterName
in self.config.filterMap.items():
1086 addAliasesForOneFilter(filterName, refFilterName)
1090 addIsPhotometric=False, addIsResolved=False,
1091 addIsVariable=False, coordErrDim=2,
1092 addProperMotion=False, properMotionErrDim=2,
1094 """Make a standard schema for reference object catalogs.
1098 filterNameList : `list` of `str`
1099 List of filter names. Used to create <filterName>_flux fields.
1100 addIsPhotometric : `bool`
1101 If True then add field "photometric".
1102 addIsResolved : `bool`
1103 If True then add field "resolved".
1104 addIsVariable : `bool`
1105 If True then add field "variable".
1107 Number of coord error fields; must be one of 0, 2, 3:
1109 - If 2 or 3: add fields "coord_raErr" and "coord_decErr".
1110 - If 3: also add field "coord_radecErr".
1111 addProperMotion : `bool`
1112 If True add fields "epoch", "pm_ra", "pm_dec" and "pm_flag".
1113 properMotionErrDim : `int`
1114 Number of proper motion error fields; must be one of 0, 2, 3;
1115 ignored if addProperMotion false:
1116 - If 2 or 3: add fields "pm_raErr" and "pm_decErr".
1117 - If 3: also add field "pm_radecErr".
1118 addParallax : `bool`
1119 If True add fields "epoch", "parallax", "parallaxErr"
1120 and "parallax_flag".
1124 schema : `lsst.afw.table.Schema`
1125 Schema for reference catalog, an
1126 `lsst.afw.table.SimpleCatalog`.
1130 Reference catalogs support additional covariances, such as
1131 covariance between RA and proper motion in declination,
1132 that are not supported by this method, but can be added after
1133 calling this method.
1135 schema = afwTable.SimpleTable.makeMinimalSchema()
1137 afwTable.Point2DKey.addFields(
1140 "centroid on an exposure, if relevant",
1144 field=
"hasCentroid",
1146 doc=
"is position known?",
1148 for filterName
in filterNameList:
1150 field=
"%s_flux" % (filterName,),
1152 doc=
"flux in filter %s" % (filterName,),
1155 for filterName
in filterNameList:
1157 field=
"%s_fluxErr" % (filterName,),
1159 doc=
"flux uncertainty in filter %s" % (filterName,),
1162 if addIsPhotometric:
1164 field=
"photometric",
1166 doc=
"set if the object can be used for photometric calibration",
1172 doc=
"set if the object is spatially resolved",
1178 doc=
"set if the object has variable brightness",
1180 if coordErrDim
not in (0, 2, 3):
1181 raise ValueError(
"coordErrDim={}; must be (0, 2, 3)".format(coordErrDim))
1183 afwTable.CovarianceMatrix2fKey.addFields(
1186 names=[
"ra",
"dec"],
1187 units=[
"rad",
"rad"],
1188 diagonalOnly=(coordErrDim == 2),
1191 if addProperMotion
or addParallax:
1195 doc=
"date of observation (TAI, MJD)",
1203 doc=
"proper motion in the right ascension direction = dra/dt * cos(dec)",
1209 doc=
"proper motion in the declination direction",
1212 if properMotionErrDim
not in (0, 2, 3):
1213 raise ValueError(
"properMotionErrDim={}; must be (0, 2, 3)".format(properMotionErrDim))
1214 if properMotionErrDim > 0:
1215 afwTable.CovarianceMatrix2fKey.addFields(
1218 names=[
"ra",
"dec"],
1219 units=[
"rad/year",
"rad/year"],
1220 diagonalOnly=(properMotionErrDim == 2),
1225 doc=
"Set if proper motion or proper motion error is bad",
1236 field=
"parallaxErr",
1238 doc=
"uncertainty in parallax",
1242 field=
"parallax_flag",
1244 doc=
"Set if parallax or parallax error is bad",
1248 def _calculateCircle(self, bbox, wcs):
1249 """Compute on-sky center and radius of search region.
1253 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
1255 wcs : `lsst.afw.geom.SkyWcs`
1256 WCS; used to convert pixel positions to sky coordinates.
1260 results : `lsst.pipe.base.Struct`
1261 A Struct containing:
1263 - coord : `lsst.geom.SpherePoint`
1264 ICRS center of the search region.
1265 - radius : `lsst.geom.Angle`
1266 Radius of the search region.
1267 - bbox : `lsst.geom.Box2D`
1268 Bounding box used to compute the circle.
1271 bbox.grow(self.config.pixelMargin)
1272 coord = wcs.pixelToSky(bbox.getCenter())
1273 radius = max(coord.separation(wcs.pixelToSky(pp))
for pp
in bbox.getCorners())
1274 return pipeBase.Struct(coord=coord, radius=radius, bbox=bbox)
1277 """Return metadata about the load.
1279 This metadata is used for reloading the catalog (e.g., for
1280 reconstituting a normalised match list.
1284 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
1286 wcs : `lsst.afw.geom.SkyWcs`
1287 WCS; used to convert pixel positions to sky coordinates.
1289 Name of camera filter, or `None` or `""` for the default
1291 photoCalib : `lsst.afw.image.PhotoCalib` (optional)
1292 Calibration, or `None` if unknown.
1293 epoch : `astropy.time.Time` (optional)
1294 Epoch to which to correct proper motion and parallax,
1295 or None to not apply such corrections.
1299 metadata : lsst.daf.base.PropertyList
1300 Metadata about the load.
1303 return self.
getMetadataCirclegetMetadataCircle(circle.coord, circle.radius, filterName, photoCalib=photoCalib,
1307 """Return metadata about the load.
1309 This metadata is used for reloading the catalog (e.g., for
1310 reconstituting a normalised match list.
1314 coord : `lsst.geom.SpherePoint`
1315 ICRS center of the search region.
1316 radius : `lsst.geom.Angle`
1317 Radius of the search region.
1319 Name of camera filter, or `None` or `""` for the default
1321 photoCalib : `lsst.afw.image.PhotoCalib` (optional)
1322 Calibration, or `None` if unknown.
1323 epoch : `astropy.time.Time` (optional)
1324 Epoch to which to correct proper motion and parallax,
1325 or None to not apply such corrections.
1329 metadata : lsst.daf.base.PropertyList
1330 Metadata about the load
1333 md.add(
'RA', coord.getRa().asDegrees(),
'field center in degrees')
1334 md.add(
'DEC', coord.getDec().asDegrees(),
'field center in degrees')
1335 md.add(
'RADIUS', radius.asDegrees(),
'field radius in degrees, minimum')
1336 md.add(
'SMATCHV', 1,
'SourceMatchVector version number')
1337 filterName =
"UNKNOWN" if filterName
is None else str(filterName)
1338 md.add(
'FILTER', filterName,
'filter name for photometric data')
1339 md.add(
'EPOCH',
"NONE" if epoch
is None else epoch.mjd,
'Epoch (TAI MJD) for catalog')
1343 """Relink an unpersisted match list to sources and reference
1346 A match list is persisted and unpersisted as a catalog of IDs
1347 produced by afw.table.packMatches(), with match metadata
1348 (as returned by the astrometry tasks) in the catalog's metadata
1349 attribute. This method converts such a match catalog into a match
1350 list, with links to source records and reference object records.
1354 matchCat : `lsst.afw.table.BaseCatalog`
1355 Unperisted packed match list.
1356 ``matchCat.table.getMetadata()`` must contain match metadata,
1357 as returned by the astrometry tasks.
1358 sourceCat : `lsst.afw.table.SourceCatalog`
1359 Source catalog. As a side effect, the catalog will be sorted
1364 matchList : `lsst.afw.table.ReferenceMatchVector`
1371 """Relink an unpersisted match list to sources and reference
1374 A match list is persisted and unpersisted as a catalog of IDs
1375 produced by afw.table.packMatches(), with match metadata
1376 (as returned by the astrometry tasks) in the catalog's metadata
1377 attribute. This method converts such a match catalog into a match
1378 list, with links to source records and reference object records.
1383 Reference object loader to use in getting reference objects
1384 matchCat : `lsst.afw.table.BaseCatalog`
1385 Unperisted packed match list.
1386 ``matchCat.table.getMetadata()`` must contain match metadata,
1387 as returned by the astrometry tasks.
1388 sourceCat : `lsst.afw.table.SourceCatalog`
1389 Source catalog. As a side effect, the catalog will be sorted
1394 matchList : `lsst.afw.table.ReferenceMatchVector`
1397 matchmeta = matchCat.table.getMetadata()
1398 version = matchmeta.getInt(
'SMATCHV')
1400 raise ValueError(
'SourceMatchVector version number is %i, not 1.' % version)
1401 filterName = matchmeta.getString(
'FILTER').strip()
1403 epoch = matchmeta.getDouble(
'EPOCH')
1404 except (pexExcept.NotFoundError, pexExcept.TypeError):
1406 if 'RADIUS' in matchmeta:
1409 matchmeta.getDouble(
'DEC'), lsst.geom.degrees)
1410 rad = matchmeta.getDouble(
'RADIUS') * lsst.geom.degrees
1411 refCat = refObjLoader.loadSkyCircle(ctrCoord, rad, filterName, epoch=epoch).refCat
1412 elif "INNER_UPPER_LEFT_RA" in matchmeta:
1418 for place
in (
"UPPER_LEFT",
"UPPER_RIGHT",
"LOWER_LEFT",
"LOWER_RIGHT"):
1420 matchmeta.getDouble(f
"OUTER_{place}_DEC"),
1421 lsst.geom.degrees).getVector()
1424 refCat = refObjLoader.loadRegion(outerBox, filterName=filterName, epoch=epoch).refCat
1428 return afwTable.unpackMatches(matchCat, refCat, sourceCat)
1432 """Apply proper motion correction to a reference catalog.
1434 Adjust position and position error in the ``catalog``
1435 for proper motion to the specified ``epoch``,
1436 modifying the catalog in place.
1440 log : `lsst.log.Log`
1441 Log object to write to.
1442 catalog : `lsst.afw.table.SimpleCatalog`
1443 Catalog of positions, containing:
1445 - Coordinates, retrieved by the table's coordinate key.
1446 - ``coord_raErr`` : Error in Right Ascension (rad).
1447 - ``coord_decErr`` : Error in Declination (rad).
1448 - ``pm_ra`` : Proper motion in Right Ascension (rad/yr,
1450 - ``pm_raErr`` : Error in ``pm_ra`` (rad/yr), optional.
1451 - ``pm_dec`` : Proper motion in Declination (rad/yr,
1453 - ``pm_decErr`` : Error in ``pm_dec`` (rad/yr), optional.
1454 - ``epoch`` : Mean epoch of object (an astropy.time.Time)
1455 epoch : `astropy.time.Time`
1456 Epoch to which to correct proper motion.
1458 if "epoch" not in catalog.schema
or "pm_ra" not in catalog.schema
or "pm_dec" not in catalog.schema:
1459 log.warn(
"Proper motion correction not available from catalog")
1461 if not catalog.isContiguous():
1462 raise RuntimeError(
"Catalog must be contiguous")
1463 catEpoch = astropy.time.Time(catalog[
"epoch"], scale=
"tai", format=
"mjd")
1464 log.info(
"Correcting reference catalog for proper motion to %r", epoch)
1466 timeDiffsYears = (epoch.tai - catEpoch).to(astropy.units.yr).value
1467 coordKey = catalog.table.getCoordKey()
1470 pmRaRad = catalog[
"pm_ra"]
1471 pmDecRad = catalog[
"pm_dec"]
1472 offsetsRaRad = pmRaRad*timeDiffsYears
1473 offsetsDecRad = pmDecRad*timeDiffsYears
1481 offsetBearingsRad = numpy.arctan2(pmDecRad*1e6, pmRaRad*1e6)
1482 offsetAmountsRad = numpy.hypot(offsetsRaRad, offsetsDecRad)
1483 for record, bearingRad, amountRad
in zip(catalog, offsetBearingsRad, offsetAmountsRad):
1484 record.set(coordKey,
1485 record.get(coordKey).offset(bearing=bearingRad*lsst.geom.radians,
1486 amount=amountRad*lsst.geom.radians))
1488 if "coord_raErr" in catalog.schema:
1489 catalog[
"coord_raErr"] = numpy.hypot(catalog[
"coord_raErr"],
1490 catalog[
"pm_raErr"]*timeDiffsYears)
1491 if "coord_decErr" in catalog.schema:
1492 catalog[
"coord_decErr"] = numpy.hypot(catalog[
"coord_decErr"],
1493 catalog[
"pm_decErr"]*timeDiffsYears)
static Schema makeMinimalSchema()
static Log getDefaultLogger()
def __call__(self, refCat, catRegion)
def __init__(self, region)
def _trimToBBox(refCat, bbox, wcs)
def joinMatchListWithCatalog(self, matchCat, sourceCat)
def makeMinimalSchema(filterNameList, *addCentroid=False, addIsPhotometric=False, addIsResolved=False, addIsVariable=False, coordErrDim=2, addProperMotion=False, properMotionErrDim=2, addParallax=False)
def getMetadataCircle(self, coord, radius, filterName, photoCalib=None, epoch=None)
def __init__(self, butler=None, *args, **kwargs)
def _calculateCircle(self, bbox, wcs)
def loadPixelBox(self, bbox, wcs, filterName=None, photoCalib=None, epoch=None)
def loadSkyCircle(self, ctrCoord, radius, filterName=None, epoch=None, centroids=False)
def getMetadataBox(self, bbox, wcs, filterName=None, photoCalib=None, epoch=None)
def applyProperMotions(self, catalog, epoch)
def loadRegion(self, region, filtFunc=None, filterName=None, epoch=None)
def getMetadataCircle(coord, radius, filterName, photoCalib=None, epoch=None)
def _makeBoxRegion(BBox, wcs, BBoxPadding)
def remapReferenceCatalogSchema(refCat, *filterNameList=None, position=False, photometric=False)
def getMetadataBox(cls, bbox, wcs, filterName=None, photoCalib=None, epoch=None, bboxPadding=100)
def addFluxAliases(refCat, defaultFilter, filterReferenceMap)
def loadSkyCircle(self, ctrCoord, radius, filterName=None, epoch=None)
def joinMatchListWithCatalog(self, matchCat, sourceCat)
def __init__(self, dataIds, refCats, config, log=None)
def loadPixelBox(self, bbox, wcs, filterName=None, epoch=None, photoCalib=None, bboxPadding=100)
def getRefFluxKeys(schema, filterName=None)
def hasNanojanskyFluxUnits(schema)
def getRefFluxField(schema, filterName=None)
def convertToNanojansky(catalog, log, doConvert=True)
def joinMatchListWithCatalogImpl(refObjLoader, matchCat, sourceCat)
def isOldFluxField(name, units)
def getFormatVersionFromRefCat(refCat)
def applyProperMotionsImpl(log, catalog, epoch)