24 __all__ = [
"getRefFluxField",
"getRefFluxKeys",
"LoadReferenceObjectsTask",
"LoadReferenceObjectsConfig",
25 "ReferenceObjectLoader"]
42 from lsst
import sphgeom
47 """Return True if this name/units combination corresponds to an
48 "old-style" reference catalog flux field.
50 unitsCheck = units !=
'nJy'
51 isFlux = name.endswith(
'_flux')
52 isFluxSigma = name.endswith(
'_fluxSigma')
53 isFluxErr = name.endswith(
'_fluxErr')
54 return (isFlux
or isFluxSigma
or isFluxErr)
and unitsCheck
58 """Return True if the units of all flux and fluxErr are correct (nJy).
67 """"Return the format version stored in a reference catalog header.
71 refCat : `lsst.afw.table.SimpleCatalog`
72 Reference catalog to inspect.
76 version : `int` or `None`
77 Format version integer, or `None` if the catalog has no metadata
78 or the metadata does not include a "REFCAT_FORMAT_VERSION" key.
80 md = refCat.getMetadata()
84 return md.getScalar(
"REFCAT_FORMAT_VERSION")
90 """Convert fluxes in a catalog from jansky to nanojansky.
94 catalog : `lsst.afw.table.SimpleCatalog`
95 The catalog to convert.
97 Log to send messages to.
98 doConvert : `bool`, optional
99 Return a converted catalog, or just identify the fields that need to be converted?
100 This supports the "write=False" mode of `bin/convert_to_nJy.py`.
104 catalog : `lsst.afw.table.SimpleCatalog` or None
105 The converted catalog, or None if ``doConvert`` is False.
109 Support for old units in reference catalogs will be removed after the
110 release of late calendar year 2019.
111 Use `meas_algorithms/bin/convert_to_nJy.py` to update your reference catalog.
119 for field
in catalog.schema:
120 oldName = field.field.getName()
121 oldUnits = field.field.getUnits()
125 if oldName.endswith(
'_fluxSigma'):
126 name = oldName.replace(
'_fluxSigma',
'_fluxErr')
130 mapper.addMapping(field.getKey(), newField)
131 input_fields.append(field.field)
132 output_fields.append(newField)
134 mapper.addMapping(field.getKey())
136 fluxFieldsStr =
'; '.join(
"(%s, '%s')" % (field.getName(), field.getUnits())
for field
in input_fields)
139 newSchema = mapper.getOutputSchema()
141 output.extend(catalog, mapper=mapper)
142 for field
in output_fields:
143 output[field.getName()] *= 1e9
144 log.info(f
"Converted refcat flux fields to nJy (name, units): {fluxFieldsStr}")
147 log.info(f
"Found old-style refcat flux fields (name, units): {fluxFieldsStr}")
152 """This is a private helper class which filters catalogs by
153 row based on the row being inside the region used to initialize
158 region : `lsst.sphgeom.Region`
159 The spatial region which all objects should lie within
165 """This call method on an instance of this class takes in a reference
166 catalog, and the region from which the catalog was generated.
168 If the catalog region is entirely contained within the region used to
169 initialize this class, then all the entries in the catalog must be
170 within the region and so the whole catalog is returned.
172 If the catalog region is not entirely contained, then the location for
173 each record is tested against the region used to initialize the class.
174 Records which fall inside this region are added to a new catalog, and
175 this catalog is then returned.
179 refCat : `lsst.afw.table.SourceCatalog`
180 SourceCatalog to be filtered.
181 catRegion : `lsst.sphgeom.Region`
182 Region in which the catalog was created
184 if catRegion.isWithin(self.
regionregion):
188 filteredRefCat = type(refCat)(refCat.table)
189 for record
in refCat:
190 if self.
regionregion.contains(record.getCoord().getVector()):
191 filteredRefCat.append(record)
192 return filteredRefCat
196 """ This class facilitates loading reference catalogs with gen 3 middleware
198 The middleware preflight solver will create a list of datarefs that may
199 possibly overlap a given region. These datarefs are then used to construct
200 and instance of this class. The class instance should then be passed into
201 a task which needs reference catalogs. These tasks should then determine
202 the exact region of the sky reference catalogs will be loaded for, and
203 call a corresponding method to load the reference objects.
205 def __init__(self, dataIds, refCats, config, log=None):
206 """ Constructs an instance of ReferenceObjectLoader
210 dataIds : iterable of `lsst.daf.butler.DataIds`
211 An iterable object of DataSetRefs which point to reference catalogs
212 in a gen 3 repository
213 refCats : Iterable of `lsst.daf.butler.DeferedDatasetHandle`
214 Handles to load refCats on demand
216 Logger object used to write out messages. If `None` (default) the default
217 lsst logger will be used
226 def _makeBoxRegion(BBox, wcs, BBoxPadding):
233 outerLocalBBox.grow(BBoxPadding)
234 innerLocalBBox.grow(-1*BBoxPadding)
242 innerBoxCorners = innerLocalBBox.getCorners()
243 innerSphCorners = [wcs.pixelToSky(corner).getVector()
for corner
in innerBoxCorners]
248 outerBoxCorners = outerLocalBBox.getCorners()
249 outerSphCorners = [wcs.pixelToSky(corner).getVector()
for corner
in outerBoxCorners]
253 return innerSkyRegion, outerSkyRegion, innerSphCorners, outerSphCorners
255 def loadPixelBox(self, bbox, wcs, filterName=None, epoch=None, photoCalib=None, bboxPadding=100):
256 """Load reference objects that are within a pixel-based rectangular region
258 This algorithm works by creating a spherical box whose corners correspond
259 to the WCS converted corners of the input bounding box (possibly padded).
260 It then defines a filtering function which will look at a reference
261 objects pixel position and accept objects that lie within the specified
264 The spherical box region and filtering function are passed to the generic
265 loadRegion method which will load and filter the reference objects from
266 the datastore and return a single catalog containing all reference objects
270 bbox : `lsst.geom.box2I`
271 Box which bounds a region in pixel space
272 wcs : `lsst.afw.geom.SkyWcs`
273 Wcs object defining the pixel to sky (and inverse) transform for the space
274 of pixels of the supplied bbox
276 Name of camera filter, or None or blank for the default filter
277 epoch : `astropy.time.Time` (optional)
278 Epoch to which to correct proper motion and parallax,
279 or None to not apply such corrections.
281 Deprecated and ignored, only included for api compatibility
283 Number describing how much to pad the input bbox by (in pixels), defaults
284 to 100. This parameter is necessary because optical distortions in telescopes
285 can cause a rectangular pixel grid to map into a non "rectangular" spherical
286 region in sky coordinates. This padding is used to create a spherical
287 "rectangle", which will for sure enclose the input box. This padding is only
288 used to determine if the reference catalog for a sky patch will be loaded from
289 the data store, this function will filter out objects which lie within the
290 padded region but fall outside the input bounding box region.
294 referenceCatalog : `lsst.afw.table.SimpleCatalog`
295 Catalog containing reference objects inside the specified bounding box
299 `lsst.pex.exception.RuntimeError`
300 Raised if no reference catalogs could be found for the specified region
302 `lsst.pex.exception.TypeError`
303 Raised if the loaded reference catalogs do not have matching schemas
305 innerSkyRegion, outerSkyRegion, _, _ = self.
_makeBoxRegion_makeBoxRegion(bbox, wcs, bboxPadding)
307 def _filterFunction(refCat, region):
311 afwTable.updateRefCentroids(wcs, refCat)
314 if innerSkyRegion.contains(region):
317 filteredRefCat = type(refCat)(refCat.table)
318 centroidKey = afwTable.Point2DKey(refCat.schema[
'centroid'])
319 for record
in refCat:
320 pixCoords = record[centroidKey]
322 filteredRefCat.append(record)
323 return filteredRefCat
324 return self.
loadRegionloadRegion(outerSkyRegion, filtFunc=_filterFunction, epoch=epoch, filterName=filterName)
326 def loadRegion(self, region, filtFunc=None, filterName=None, epoch=None):
327 """ Load reference objects within a specified region
329 This function loads the DataIds used to construct an instance of this class
330 which intersect or are contained within the specified region. The reference
331 catalogs which intersect but are not fully contained within the input region are
332 further filtered by the specified filter function. This function will return a
333 single source catalog containing all reference objects inside the specified region.
337 region : `lsst.sphgeom.Region`
338 This can be any type that is derived from `lsst.sphgeom.region` and should
339 define the spatial region for which reference objects are to be loaded.
341 This optional parameter should be a callable object that takes a reference
342 catalog and its corresponding region as parameters, filters the catalog by
343 some criteria and returns the filtered reference catalog. If the value is
344 left as the default (None) than an internal filter function is used which
345 filters according to if a reference object falls within the input region.
347 Name of camera filter, or None or blank for the default filter
348 epoch : `astropy.time.Time` (optional)
349 Epoch to which to correct proper motion and parallax,
350 or None to not apply such corrections.
354 referenceCatalog : `lsst.afw.table.SourceCatalog`
355 Catalog containing reference objects which intersect the input region,
356 filtered by the specified filter function
360 `lsst.pex.exception.RuntimeError`
361 Raised if no reference catalogs could be found for the specified region
363 `lsst.pex.exception.TypeError`
364 Raised if the loaded reference catalogs do not have matching schemas
367 regionBounding = region.getBoundingBox()
368 self.
loglog.info(
"Loading reference objects from region bounded by {}, {} lat lon".format(
369 regionBounding.getLat(), regionBounding.getLon()))
374 for dataId, refCat
in zip(self.
dataIdsdataIds, self.
refCatsrefCats):
378 intersects = dataId.region.intersects(region)
380 intersects = region.intersects(dataId.region)
383 overlapList.append((dataId, refCat))
385 if len(overlapList) == 0:
386 raise pexExceptions.RuntimeError(
"No reference tables could be found for input region")
388 firstCat = overlapList[0][1].get()
389 refCat = filtFunc(firstCat, overlapList[0][0].region)
390 trimmedAmount = len(firstCat) - len(refCat)
393 for dataId, inputRefCat
in overlapList[1:]:
394 tmpCat = inputRefCat.get()
396 if tmpCat.schema != firstCat.schema:
397 raise pexExceptions.TypeError(
"Reference catalogs have mismatching schemas")
399 filteredCat = filtFunc(tmpCat, dataId.region)
400 refCat.extend(filteredCat)
401 trimmedAmount += len(tmpCat) - len(filteredCat)
403 self.
loglog.debug(f
"Trimmed {trimmedAmount} out of region objects, leaving {len(refCat)}")
404 self.
loglog.info(f
"Loaded {len(refCat)} reference objects")
407 if not refCat.isContiguous():
408 refCat = refCat.copy(deep=
True)
410 if epoch
is not None and "pm_ra" in refCat.schema:
412 if isinstance(refCat.schema[
"pm_ra"].asKey(), lsst.afw.table.KeyAngle):
415 self.
loglog.warn(
"Catalog pm_ra field is not an Angle; not applying proper motion")
420 self.
loglog.warn(
"Found version 0 reference catalog with old style units in schema.")
421 self.
loglog.warn(
"run `meas_algorithms/bin/convert_refcat_to_nJy.py` to convert fluxes to nJy.")
422 self.
loglog.warn(
"See RFC-575 for more details.")
431 if not expandedCat.isContiguous():
432 expandedCat = expandedCat.copy(deep=
True)
434 fluxField =
getRefFluxField(schema=expandedCat.schema, filterName=filterName)
435 return pipeBase.Struct(refCat=expandedCat, fluxField=fluxField)
438 """Load reference objects that lie within a circular region on the sky
440 This method constructs a circular region from an input center and angular radius,
441 loads reference catalogs which are contained in or intersect the circle, and
442 filters reference catalogs which intersect down to objects which lie within
447 ctrCoord : `lsst.geom.SpherePoint`
448 Point defining the center of the circular region
449 radius : `lsst.geom.Angle`
450 Defines the angular radius of the circular region
452 Name of camera filter, or None or blank for the default filter
453 epoch : `astropy.time.Time` (optional)
454 Epoch to which to correct proper motion and parallax,
455 or None to not apply such corrections.
459 referenceCatalog : `lsst.afw.table.SourceCatalog`
460 Catalog containing reference objects inside the specified bounding box
464 `lsst.pex.exception.RuntimeError`
465 Raised if no reference catalogs could be found for the specified region
467 `lsst.pex.exception.TypeError`
468 Raised if the loaded reference catalogs do not have matching schemas
471 centerVector = ctrCoord.getVector()
474 return self.
loadRegionloadRegion(circularRegion, filterName=filterName, epoch=epoch)
477 """Relink an unpersisted match list to sources and reference
480 A match list is persisted and unpersisted as a catalog of IDs
481 produced by afw.table.packMatches(), with match metadata
482 (as returned by the astrometry tasks) in the catalog's metadata
483 attribute. This method converts such a match catalog into a match
484 list, with links to source records and reference object records.
488 matchCat : `lsst.afw.table.BaseCatalog`
489 Unpersisted packed match list.
490 ``matchCat.table.getMetadata()`` must contain match metadata,
491 as returned by the astrometry tasks.
492 sourceCat : `lsst.afw.table.SourceCatalog`
493 Source catalog. As a side effect, the catalog will be sorted
498 matchList : `lsst.afw.table.ReferenceMatchVector`
504 def getMetadataBox(cls, bbox, wcs, filterName=None, photoCalib=None, epoch=None, bboxPadding=100):
505 """Return metadata about the load
507 This metadata is used for reloading the catalog (e.g., for
508 reconstituting a normalised match list.)
512 bbox : `lsst.geom.Box2I`
513 Bounding bos for the pixels
514 wcs : `lsst.afw.geom.SkyWcs
516 filterName : `str` or None
517 filterName of the camera filter, or None or blank for the default filter
519 Deprecated, only included for api compatibility
520 epoch : `astropy.time.Time` (optional)
521 Epoch to which to correct proper motion and parallax,
522 or None to not apply such corrections.
524 Number describing how much to pad the input bbox by (in pixels), defaults
525 to 100. This parameter is necessary because optical distortions in telescopes
526 can cause a rectangular pixel grid to map into a non "rectangular" spherical
527 region in sky coordinates. This padding is used to create a spherical
528 "rectangle", which will for sure enclose the input box. This padding is only
529 used to determine if the reference catalog for a sky patch will be loaded from
530 the data store, this function will filter out objects which lie within the
531 padded region but fall outside the input bounding box region.
534 md : `lsst.daf.base.PropertyList`
536 _, _, innerCorners, outerCorners = cls.
_makeBoxRegion_makeBoxRegion(bbox, wcs, bboxPadding)
538 for box, corners
in zip((
"INNER",
"OUTER"), (innerCorners, outerCorners)):
539 for (name, corner)
in zip((
"UPPER_LEFT",
"UPPER_RIGHT",
"LOWER_LEFT",
"LOWER_RIGHT"),
541 md.add(f
"{box}_{name}_RA",
geom.SpherePoint(corner).getRa().asDegrees(), f
"{box}_corner")
542 md.add(f
"{box}_{name}_DEC",
geom.SpherePoint(corner).getDec().asDegrees(), f
"{box}_corner")
543 md.add(
"SMATCHV", 1,
'SourceMatchVector version number')
544 filterName =
"UNKNOWN" if filterName
is None else str(filterName)
545 md.add(
'FILTER', filterName,
'filter name for photometric data')
546 md.add(
'EPOCH',
"NONE" if epoch
is None else epoch.mjd,
'Epoch (TAI MJD) for catalog')
551 """Return metadata about the load
553 This metadata is used for reloading the catalog (e.g. for reconstituting
554 a normalized match list.)
558 coord : `lsst.geom.SpherePoint`
559 ICRS center of a circle
560 radius : `lsst.geom.angle`
562 filterName : `str` or None
563 filterName of the camera filter, or None or blank for the default filter
565 Deprecated, only included for api compatibility
566 epoch : `astropy.time.Time` (optional)
567 Epoch to which to correct proper motion and parallax,
568 or None to not apply such corrections.
572 md : `lsst.daf.base.PropertyList`
575 md.add(
'RA', coord.getRa().asDegrees(),
'field center in degrees')
576 md.add(
'DEC', coord.getDec().asDegrees(),
'field center in degrees')
577 md.add(
'RADIUS', radius.asDegrees(),
'field radius in degrees, minimum')
578 md.add(
'SMATCHV', 1,
'SourceMatchVector version number')
579 filterName =
"UNKNOWN" if filterName
is None else str(filterName)
580 md.add(
'FILTER', filterName,
'filter name for photometric data')
581 md.add(
'EPOCH',
"NONE" if epoch
is None else epoch.mjd,
'Epoch (TAI MJD) for catalog')
586 """This function creates a new catalog containing the information of the input refCat
587 as well as added flux columns and aliases between camera and reference flux.
591 refCat : `lsst.afw.table.SimpleCatalog`
592 Catalog of reference objects
593 defaultFilter : `str`
594 Name of the default reference filter
595 filterReferenceMap : `dict` of `str`
596 Dictionary with keys corresponding to a filter name, and values which
597 correspond to the name of the reference filter.
601 refCat : `lsst.afw.table.SimpleCatalog`
602 Reference catalog with columns added to track reference filters
607 If specified reference filter name is not a filter specifed as a key in the
608 reference filter map.
610 refCat = ReferenceObjectLoader.remapReferenceCatalogSchema(refCat,
611 filterNameList=filterReferenceMap.keys())
612 aliasMap = refCat.schema.getAliasMap()
613 if filterReferenceMap
is None:
614 filterReferenceMap = {}
615 for filterName, refFilterName
in itertools.chain([(
None, defaultFilter)],
616 filterReferenceMap.items()):
618 camFluxName = filterName +
"_camFlux" if filterName
is not None else "camFlux"
619 refFluxName = refFilterName +
"_flux"
620 if refFluxName
not in refCat.schema:
621 raise RuntimeError(
"Unknown reference filter %s" % (refFluxName,))
622 aliasMap.set(camFluxName, refFluxName)
624 refFluxErrName = refFluxName +
"Err"
625 camFluxErrName = camFluxName +
"Err"
626 aliasMap.set(camFluxErrName, refFluxErrName)
632 """This function takes in a reference catalog and creates a new catalog with additional
633 columns defined the remaining function arguments.
637 refCat : `lsst.afw.table.SimpleCatalog`
638 Reference catalog to map to new catalog
642 expandedCat : `lsst.afw.table.SimpleCatalog`
643 Deep copy of input reference catalog with additional columns added
645 mapper = afwTable.SchemaMapper(refCat.schema,
True)
646 mapper.addMinimalSchema(refCat.schema,
True)
647 mapper.editOutputSchema().disconnectAliases()
649 for filterName
in filterNameList:
650 mapper.editOutputSchema().addField(f
"{filterName}_flux",
652 doc=f
"flux in filter {filterName}",
655 mapper.editOutputSchema().addField(f
"{filterName}_fluxErr",
657 doc=f
"flux uncertanty in filter {filterName}",
662 mapper.editOutputSchema().addField(
"centroid_x", type=float, doReplace=
True)
663 mapper.editOutputSchema().addField(
"centroid_y", type=float, doReplace=
True)
664 mapper.editOutputSchema().addField(
"hasCentroid", type=
"Flag", doReplace=
True)
665 mapper.editOutputSchema().getAliasMap().set(
"slot_Centroid",
"centroid")
668 mapper.editOutputSchema().addField(
"photometric",
670 doc=
"set if the object can be used for photometric"
673 mapper.editOutputSchema().addField(
"resolved",
675 doc=
"set if the object is spatially resolved"
677 mapper.editOutputSchema().addField(
"variable",
679 doc=
"set if the object has variable brightness"
682 expandedCat = afwTable.SimpleCatalog(mapper.getOutputSchema())
683 expandedCat.setMetadata(refCat.getMetadata())
684 expandedCat.extend(refCat, mapper=mapper)
690 """Get the name of a flux field from a schema.
692 return the alias of "anyFilterMapsToThis", if present
693 else if filterName is specified:
694 return "*filterName*_camFlux" if present
695 else return "*filterName*_flux" if present (camera filter name
696 matches reference filter name)
697 else throw RuntimeError
699 return "camFlux", if present,
700 else throw RuntimeError
704 schema : `lsst.afw.table.Schema`
705 Reference catalog schema.
706 filterName : `str`, optional
707 Name of camera filter. If not specified, ``defaultFilter`` needs to be
708 set in the refcat loader config.
712 fluxFieldName : `str`
718 If an appropriate field is not found.
720 if not isinstance(schema, afwTable.Schema):
721 raise RuntimeError(
"schema=%s is not a schema" % (schema,))
723 return schema.getAliasMap().get(
"anyFilterMapsToThis")
728 fluxFieldList = [filterName +
"_camFlux", filterName +
"_flux"]
730 fluxFieldList = [
"camFlux"]
731 for fluxField
in fluxFieldList:
732 if fluxField
in schema:
735 raise RuntimeError(
"Could not find flux field(s) %s" % (
", ".join(fluxFieldList)))
739 """Return keys for flux and flux error.
743 schema : `lsst.afw.table.Schema`
744 Reference catalog schema.
746 Name of camera filter.
750 keys : `tuple` of (`lsst.afw.table.Key`, `lsst.afw.table.Key`)
754 - flux error key, if present, else None
759 If flux field not found.
762 fluxErrField = fluxField +
"Err"
763 fluxKey = schema[fluxField].asKey()
765 fluxErrKey = schema[fluxErrField].asKey()
768 return (fluxKey, fluxErrKey)
772 pixelMargin = pexConfig.RangeField(
773 doc=
"Padding to add to 4 all edges of the bounding box (pixels)",
778 defaultFilter = pexConfig.Field(
779 doc=(
"Default reference catalog filter to use if filter not specified in exposure;"
780 " if blank then filter must be specified in exposure."),
783 deprecated=
"defaultFilter is deprecated by RFC-716. Will be removed after v22."
785 anyFilterMapsToThis = pexConfig.Field(
786 doc=(
"Always use this reference catalog filter, no matter whether or what filter name is "
787 "supplied to the loader. Effectively a trivial filterMap: map all filter names to this filter."
788 " This can be set for purely-astrometric catalogs (e.g. Gaia DR2) where there is only one "
789 "reasonable choice for every camera filter->refcat mapping, but not for refcats used for "
790 "photometry, which need a filterMap and/or colorterms/transmission corrections."),
795 filterMap = pexConfig.DictField(
796 doc=(
"Mapping of camera filter name: reference catalog filter name; "
797 "each reference filter must exist in the refcat."
798 " Note that this does not perform any bandpass corrections: it is just a lookup."),
803 requireProperMotion = pexConfig.Field(
804 doc=
"Require that the fields needed to correct proper motion "
805 "(epoch, pm_ra and pm_dec) are present?",
813 msg =
"`filterMap` and `anyFilterMapsToThis` are mutually exclusive"
814 raise pexConfig.FieldValidationError(LoadReferenceObjectsConfig.anyFilterMapsToThis,
819 r"""Abstract base class to load objects from reference catalogs
821 ConfigClass = LoadReferenceObjectsConfig
822 _DefaultName =
"LoadReferenceObjects"
825 """Construct a LoadReferenceObjectsTask
829 butler : `lsst.daf.persistence.Butler`
830 Data butler, for access reference catalogs.
832 pipeBase.Task.__init__(self, *args, **kwargs)
836 def loadPixelBox(self, bbox, wcs, filterName=None, photoCalib=None, epoch=None):
837 """Load reference objects that overlap a rectangular pixel region.
841 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
842 Bounding box for pixels.
843 wcs : `lsst.afw.geom.SkyWcs`
844 WCS; used to convert pixel positions to sky coordinates
847 Name of filter, or `None` or `""` for the default filter.
848 This is used for flux values in case we have flux limits
849 (which are not yet implemented).
850 photoCalib : `lsst.afw.image.PhotoCalib` (optional)
851 Calibration, or `None` if unknown.
852 epoch : `astropy.time.Time` (optional)
853 Epoch to which to correct proper motion and parallax,
854 or None to not apply such corrections.
858 results : `lsst.pipe.base.Struct`
859 A Struct containing the following fields:
860 refCat : `lsst.afw.catalog.SimpleCatalog`
861 A catalog of reference objects with the standard
862 schema, as documented in the main doc string for
863 `LoadReferenceObjects`.
864 The catalog is guaranteed to be contiguous.
866 Name of flux field for specified `filterName`.
870 The search algorithm works by searching in a region in sky
871 coordinates whose center is the center of the bbox and radius
872 is large enough to just include all 4 corners of the bbox.
873 Stars that lie outside the bbox are then trimmed from the list.
878 self.log.info(
"Loading reference objects using center %s and radius %s deg" %
879 (circle.coord, circle.radius.asDegrees()))
880 loadRes = self.
loadSkyCircleloadSkyCircle(circle.coord, circle.radius, filterName=filterName, epoch=epoch,
882 refCat = loadRes.refCat
883 numFound = len(refCat)
886 refCat = self.
_trimToBBox_trimToBBox(refCat=refCat, bbox=circle.bbox, wcs=wcs)
887 numTrimmed = numFound - len(refCat)
888 self.log.debug(
"trimmed %d out-of-bbox objects, leaving %d", numTrimmed, len(refCat))
889 self.log.info(
"Loaded %d reference objects", len(refCat))
892 if not refCat.isContiguous():
893 loadRes.refCat = refCat.copy(deep=
True)
898 def loadSkyCircle(self, ctrCoord, radius, filterName=None, epoch=None, centroids=False):
899 """Load reference objects that overlap a circular sky region.
903 ctrCoord : `lsst.geom.SpherePoint`
904 ICRS center of search region.
905 radius : `lsst.geom.Angle`
906 Radius of search region.
907 filterName : `str` (optional)
908 Name of filter, or `None` or `""` for the default filter.
909 This is used for flux values in case we have flux limits
910 (which are not yet implemented).
911 epoch : `astropy.time.Time` (optional)
912 Epoch to which to correct proper motion and parallax,
913 or None to not apply such corrections.
914 centroids : `bool` (optional)
915 Add centroid fields to the loaded Schema. ``loadPixelBox`` expects
916 these fields to exist.
920 results : `lsst.pipe.base.Struct`
921 A Struct containing the following fields:
922 refCat : `lsst.afw.catalog.SimpleCatalog`
923 A catalog of reference objects with the standard
924 schema, as documented in the main doc string for
925 `LoadReferenceObjects`.
926 The catalog is guaranteed to be contiguous.
928 Name of flux field for specified `filterName`.
932 Note that subclasses are responsible for performing the proper motion
933 correction, since this is the lowest-level interface for retrieving
939 def _trimToBBox(refCat, bbox, wcs):
940 """Remove objects outside a given pixel bounding box and set
941 centroid and hasCentroid fields.
945 refCat : `lsst.afw.table.SimpleCatalog`
946 A catalog of objects. The schema must include fields
947 "coord", "centroid" and "hasCentroid".
948 The "coord" field is read.
949 The "centroid" and "hasCentroid" fields are set.
950 bbox : `lsst.geom.Box2D`
952 wcs : `lsst.afw.geom.SkyWcs`
953 WCS; used to convert sky coordinates to pixel positions.
957 catalog : `lsst.afw.table.SimpleCatalog`
958 Reference objects in the bbox, with centroid and
959 hasCentroid fields set.
961 afwTable.updateRefCentroids(wcs, refCat)
962 centroidKey = afwTable.Point2DKey(refCat.schema[
"centroid"])
963 retStarCat = type(refCat)(refCat.table)
965 point = star.get(centroidKey)
966 if bbox.contains(point):
967 retStarCat.append(star)
970 def _addFluxAliases(self, schema):
971 """Add aliases for camera filter fluxes to the schema.
973 If self.config.defaultFilter then adds these aliases:
974 camFlux: <defaultFilter>_flux
975 camFluxErr: <defaultFilter>_fluxErr, if the latter exists
977 For each camFilter: refFilter in self.config.filterMap adds these aliases:
978 <camFilter>_camFlux: <refFilter>_flux
979 <camFilter>_camFluxErr: <refFilter>_fluxErr, if the latter exists
983 schema : `lsst.afw.table.Schema`
984 Schema for reference catalog.
989 If any reference flux field is missing from the schema.
991 aliasMap = schema.getAliasMap()
993 if self.config.anyFilterMapsToThis
is not None:
994 refFluxName = self.config.anyFilterMapsToThis +
"_flux"
995 if refFluxName
not in schema:
996 msg = f
"Unknown reference filter for anyFilterMapsToThis='{refFluxName}'"
997 raise RuntimeError(msg)
998 aliasMap.set(
"anyFilterMapsToThis", refFluxName)
1001 def addAliasesForOneFilter(filterName, refFilterName):
1002 """Add aliases for a single filter
1006 filterName : `str` (optional)
1007 Camera filter name. The resulting alias name is
1008 <filterName>_camFlux, or simply "camFlux" if `filterName`
1010 refFilterName : `str`
1011 Reference catalog filter name; the field
1012 <refFilterName>_flux must exist.
1014 camFluxName = filterName +
"_camFlux" if filterName
is not None else "camFlux"
1015 refFluxName = refFilterName +
"_flux"
1016 if refFluxName
not in schema:
1017 raise RuntimeError(
"Unknown reference filter %s" % (refFluxName,))
1018 aliasMap.set(camFluxName, refFluxName)
1019 refFluxErrName = refFluxName +
"Err"
1020 if refFluxErrName
in schema:
1021 camFluxErrName = camFluxName +
"Err"
1022 aliasMap.set(camFluxErrName, refFluxErrName)
1024 if self.config.defaultFilter:
1025 addAliasesForOneFilter(
None, self.config.defaultFilter)
1027 for filterName, refFilterName
in self.config.filterMap.items():
1028 addAliasesForOneFilter(filterName, refFilterName)
1032 addIsPhotometric=False, addIsResolved=False,
1033 addIsVariable=False, coordErrDim=2,
1034 addProperMotion=False, properMotionErrDim=2,
1036 """Make a standard schema for reference object catalogs.
1040 filterNameList : `list` of `str`
1041 List of filter names. Used to create <filterName>_flux fields.
1042 addIsPhotometric : `bool`
1043 If True then add field "photometric".
1044 addIsResolved : `bool`
1045 If True then add field "resolved".
1046 addIsVariable : `bool`
1047 If True then add field "variable".
1049 Number of coord error fields; must be one of 0, 2, 3:
1051 - If 2 or 3: add fields "coord_raErr" and "coord_decErr".
1052 - If 3: also add field "coord_radecErr".
1053 addProperMotion : `bool`
1054 If True add fields "epoch", "pm_ra", "pm_dec" and "pm_flag".
1055 properMotionErrDim : `int`
1056 Number of proper motion error fields; must be one of 0, 2, 3;
1057 ignored if addProperMotion false:
1058 - If 2 or 3: add fields "pm_raErr" and "pm_decErr".
1059 - If 3: also add field "pm_radecErr".
1060 addParallax : `bool`
1061 If True add fields "epoch", "parallax", "parallaxErr"
1062 and "parallax_flag".
1066 schema : `lsst.afw.table.Schema`
1067 Schema for reference catalog, an
1068 `lsst.afw.table.SimpleCatalog`.
1072 Reference catalogs support additional covariances, such as
1073 covariance between RA and proper motion in declination,
1074 that are not supported by this method, but can be added after
1075 calling this method.
1077 schema = afwTable.SimpleTable.makeMinimalSchema()
1079 afwTable.Point2DKey.addFields(
1082 "centroid on an exposure, if relevant",
1086 field=
"hasCentroid",
1088 doc=
"is position known?",
1090 for filterName
in filterNameList:
1092 field=
"%s_flux" % (filterName,),
1094 doc=
"flux in filter %s" % (filterName,),
1097 for filterName
in filterNameList:
1099 field=
"%s_fluxErr" % (filterName,),
1101 doc=
"flux uncertainty in filter %s" % (filterName,),
1104 if addIsPhotometric:
1106 field=
"photometric",
1108 doc=
"set if the object can be used for photometric calibration",
1114 doc=
"set if the object is spatially resolved",
1120 doc=
"set if the object has variable brightness",
1122 if coordErrDim
not in (0, 2, 3):
1123 raise ValueError(
"coordErrDim={}; must be (0, 2, 3)".format(coordErrDim))
1125 afwTable.CovarianceMatrix2fKey.addFields(
1128 names=[
"ra",
"dec"],
1129 units=[
"rad",
"rad"],
1130 diagonalOnly=(coordErrDim == 2),
1133 if addProperMotion
or addParallax:
1137 doc=
"date of observation (TAI, MJD)",
1145 doc=
"proper motion in the right ascension direction = dra/dt * cos(dec)",
1151 doc=
"proper motion in the declination direction",
1154 if properMotionErrDim
not in (0, 2, 3):
1155 raise ValueError(
"properMotionErrDim={}; must be (0, 2, 3)".format(properMotionErrDim))
1156 if properMotionErrDim > 0:
1157 afwTable.CovarianceMatrix2fKey.addFields(
1160 names=[
"ra",
"dec"],
1161 units=[
"rad/year",
"rad/year"],
1162 diagonalOnly=(properMotionErrDim == 2),
1167 doc=
"Set if proper motion or proper motion error is bad",
1178 field=
"parallaxErr",
1180 doc=
"uncertainty in parallax",
1184 field=
"parallax_flag",
1186 doc=
"Set if parallax or parallax error is bad",
1190 def _calculateCircle(self, bbox, wcs):
1191 """Compute on-sky center and radius of search region.
1195 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
1197 wcs : `lsst.afw.geom.SkyWcs`
1198 WCS; used to convert pixel positions to sky coordinates.
1202 results : `lsst.pipe.base.Struct`
1203 A Struct containing:
1205 - coord : `lsst.geom.SpherePoint`
1206 ICRS center of the search region.
1207 - radius : `lsst.geom.Angle`
1208 Radius of the search region.
1209 - bbox : `lsst.geom.Box2D`
1210 Bounding box used to compute the circle.
1213 bbox.grow(self.config.pixelMargin)
1214 coord = wcs.pixelToSky(bbox.getCenter())
1215 radius = max(coord.separation(wcs.pixelToSky(pp))
for pp
in bbox.getCorners())
1216 return pipeBase.Struct(coord=coord, radius=radius, bbox=bbox)
1219 """Return metadata about the load.
1221 This metadata is used for reloading the catalog (e.g., for
1222 reconstituting a normalised match list.
1226 bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
1228 wcs : `lsst.afw.geom.SkyWcs`
1229 WCS; used to convert pixel positions to sky coordinates.
1231 Name of camera filter, or `None` or `""` for the default
1233 photoCalib : `lsst.afw.image.PhotoCalib` (optional)
1234 Calibration, or `None` if unknown.
1235 epoch : `astropy.time.Time` (optional)
1236 Epoch to which to correct proper motion and parallax,
1237 or None to not apply such corrections.
1241 metadata : lsst.daf.base.PropertyList
1242 Metadata about the load.
1245 return self.
getMetadataCirclegetMetadataCircle(circle.coord, circle.radius, filterName, photoCalib=photoCalib,
1249 """Return metadata about the load.
1251 This metadata is used for reloading the catalog (e.g., for
1252 reconstituting a normalised match list.
1256 coord : `lsst.geom.SpherePoint`
1257 ICRS center of the search region.
1258 radius : `lsst.geom.Angle`
1259 Radius of the search region.
1261 Name of camera filter, or `None` or `""` for the default
1263 photoCalib : `lsst.afw.image.PhotoCalib` (optional)
1264 Calibration, or `None` if unknown.
1265 epoch : `astropy.time.Time` (optional)
1266 Epoch to which to correct proper motion and parallax,
1267 or None to not apply such corrections.
1271 metadata : lsst.daf.base.PropertyList
1272 Metadata about the load
1275 md.add(
'RA', coord.getRa().asDegrees(),
'field center in degrees')
1276 md.add(
'DEC', coord.getDec().asDegrees(),
'field center in degrees')
1277 md.add(
'RADIUS', radius.asDegrees(),
'field radius in degrees, minimum')
1278 md.add(
'SMATCHV', 1,
'SourceMatchVector version number')
1279 filterName =
"UNKNOWN" if filterName
is None else str(filterName)
1280 md.add(
'FILTER', filterName,
'filter name for photometric data')
1281 md.add(
'EPOCH',
"NONE" if epoch
is None else epoch.mjd,
'Epoch (TAI MJD) for catalog')
1285 """Relink an unpersisted match list to sources and reference
1288 A match list is persisted and unpersisted as a catalog of IDs
1289 produced by afw.table.packMatches(), with match metadata
1290 (as returned by the astrometry tasks) in the catalog's metadata
1291 attribute. This method converts such a match catalog into a match
1292 list, with links to source records and reference object records.
1296 matchCat : `lsst.afw.table.BaseCatalog`
1297 Unperisted packed match list.
1298 ``matchCat.table.getMetadata()`` must contain match metadata,
1299 as returned by the astrometry tasks.
1300 sourceCat : `lsst.afw.table.SourceCatalog`
1301 Source catalog. As a side effect, the catalog will be sorted
1306 matchList : `lsst.afw.table.ReferenceMatchVector`
1312 """Apply proper motion correction to a reference catalog.
1314 Adjust position and position error in the ``catalog``
1315 for proper motion to the specified ``epoch``,
1316 modifying the catalog in place.
1320 catalog : `lsst.afw.table.SimpleCatalog`
1321 Catalog of positions, containing:
1323 - Coordinates, retrieved by the table's coordinate key.
1324 - ``coord_raErr`` : Error in Right Ascension (rad).
1325 - ``coord_decErr`` : Error in Declination (rad).
1326 - ``pm_ra`` : Proper motion in Right Ascension (rad/yr,
1328 - ``pm_raErr`` : Error in ``pm_ra`` (rad/yr), optional.
1329 - ``pm_dec`` : Proper motion in Declination (rad/yr,
1331 - ``pm_decErr`` : Error in ``pm_dec`` (rad/yr), optional.
1332 - ``epoch`` : Mean epoch of object (an astropy.time.Time)
1333 epoch : `astropy.time.Time`
1334 Epoch to which to correct proper motion,
1336 if (
"epoch" not in catalog.schema
or "pm_ra" not in catalog.schema
or "pm_dec" not in catalog.schema):
1337 if self.config.requireProperMotion:
1338 raise RuntimeError(
"Proper motion correction required but not available from catalog")
1339 self.log.warn(
"Proper motion correction not available from catalog")
1345 """Relink an unpersisted match list to sources and reference
1348 A match list is persisted and unpersisted as a catalog of IDs
1349 produced by afw.table.packMatches(), with match metadata
1350 (as returned by the astrometry tasks) in the catalog's metadata
1351 attribute. This method converts such a match catalog into a match
1352 list, with links to source records and reference object records.
1357 Reference object loader to use in getting reference objects
1358 matchCat : `lsst.afw.table.BaseCatalog`
1359 Unperisted packed match list.
1360 ``matchCat.table.getMetadata()`` must contain match metadata,
1361 as returned by the astrometry tasks.
1362 sourceCat : `lsst.afw.table.SourceCatalog`
1363 Source catalog. As a side effect, the catalog will be sorted
1368 matchList : `lsst.afw.table.ReferenceMatchVector`
1371 matchmeta = matchCat.table.getMetadata()
1372 version = matchmeta.getInt(
'SMATCHV')
1374 raise ValueError(
'SourceMatchVector version number is %i, not 1.' % version)
1375 filterName = matchmeta.getString(
'FILTER').strip()
1377 epoch = matchmeta.getDouble(
'EPOCH')
1378 except (pexExcept.NotFoundError, pexExcept.TypeError):
1380 if 'RADIUS' in matchmeta:
1383 matchmeta.getDouble(
'DEC'), lsst.geom.degrees)
1384 rad = matchmeta.getDouble(
'RADIUS') * lsst.geom.degrees
1385 refCat = refObjLoader.loadSkyCircle(ctrCoord, rad, filterName, epoch=epoch).refCat
1386 elif "INNER_UPPER_LEFT_RA" in matchmeta:
1392 for place
in (
"UPPER_LEFT",
"UPPER_RIGHT",
"LOWER_LEFT",
"LOWER_RIGHT"):
1394 matchmeta.getDouble(f
"OUTER_{place}_DEC"),
1395 lsst.geom.degrees).getVector()
1398 refCat = refObjLoader.loadRegion(outerBox, filterName=filterName, epoch=epoch).refCat
1402 return afwTable.unpackMatches(matchCat, refCat, sourceCat)
1406 """Apply proper motion correction to a reference catalog.
1408 Adjust position and position error in the ``catalog``
1409 for proper motion to the specified ``epoch``,
1410 modifying the catalog in place.
1414 log : `lsst.log.Log`
1415 Log object to write to.
1416 catalog : `lsst.afw.table.SimpleCatalog`
1417 Catalog of positions, containing:
1419 - Coordinates, retrieved by the table's coordinate key.
1420 - ``coord_raErr`` : Error in Right Ascension (rad).
1421 - ``coord_decErr`` : Error in Declination (rad).
1422 - ``pm_ra`` : Proper motion in Right Ascension (rad/yr,
1424 - ``pm_raErr`` : Error in ``pm_ra`` (rad/yr), optional.
1425 - ``pm_dec`` : Proper motion in Declination (rad/yr,
1427 - ``pm_decErr`` : Error in ``pm_dec`` (rad/yr), optional.
1428 - ``epoch`` : Mean epoch of object (an astropy.time.Time)
1429 epoch : `astropy.time.Time`
1430 Epoch to which to correct proper motion.
1432 if "epoch" not in catalog.schema
or "pm_ra" not in catalog.schema
or "pm_dec" not in catalog.schema:
1433 log.warn(
"Proper motion correction not available from catalog")
1435 if not catalog.isContiguous():
1436 raise RuntimeError(
"Catalog must be contiguous")
1437 catEpoch = astropy.time.Time(catalog[
"epoch"], scale=
"tai", format=
"mjd")
1438 log.info(
"Correcting reference catalog for proper motion to %r", epoch)
1440 timeDiffsYears = (epoch.tai - catEpoch).to(astropy.units.yr).value
1441 coordKey = catalog.table.getCoordKey()
1444 pmRaRad = catalog[
"pm_ra"]
1445 pmDecRad = catalog[
"pm_dec"]
1446 offsetsRaRad = pmRaRad*timeDiffsYears
1447 offsetsDecRad = pmDecRad*timeDiffsYears
1455 offsetBearingsRad = numpy.arctan2(pmDecRad*1e6, pmRaRad*1e6)
1456 offsetAmountsRad = numpy.hypot(offsetsRaRad, offsetsDecRad)
1457 for record, bearingRad, amountRad
in zip(catalog, offsetBearingsRad, offsetAmountsRad):
1458 record.set(coordKey,
1459 record.get(coordKey).offset(bearing=bearingRad*lsst.geom.radians,
1460 amount=amountRad*lsst.geom.radians))
1462 if "coord_raErr" in catalog.schema:
1463 catalog[
"coord_raErr"] = numpy.hypot(catalog[
"coord_raErr"],
1464 catalog[
"pm_raErr"]*timeDiffsYears)
1465 if "coord_decErr" in catalog.schema:
1466 catalog[
"coord_decErr"] = numpy.hypot(catalog[
"coord_decErr"],
1467 catalog[
"pm_decErr"]*timeDiffsYears)
static Schema makeMinimalSchema()
static Log getDefaultLogger()
def __call__(self, refCat, catRegion)
def __init__(self, region)
def _trimToBBox(refCat, bbox, wcs)
def joinMatchListWithCatalog(self, matchCat, sourceCat)
def makeMinimalSchema(filterNameList, *addCentroid=False, addIsPhotometric=False, addIsResolved=False, addIsVariable=False, coordErrDim=2, addProperMotion=False, properMotionErrDim=2, addParallax=False)
def getMetadataCircle(self, coord, radius, filterName, photoCalib=None, epoch=None)
def applyProperMotions(self, catalog, epoch)
def __init__(self, butler=None, *args, **kwargs)
def _calculateCircle(self, bbox, wcs)
def loadPixelBox(self, bbox, wcs, filterName=None, photoCalib=None, epoch=None)
def loadSkyCircle(self, ctrCoord, radius, filterName=None, epoch=None, centroids=False)
def getMetadataBox(self, bbox, wcs, filterName=None, photoCalib=None, epoch=None)
def loadRegion(self, region, filtFunc=None, filterName=None, epoch=None)
def getMetadataCircle(coord, radius, filterName, photoCalib=None, epoch=None)
def _makeBoxRegion(BBox, wcs, BBoxPadding)
def remapReferenceCatalogSchema(refCat, *filterNameList=None, position=False, photometric=False)
def getMetadataBox(cls, bbox, wcs, filterName=None, photoCalib=None, epoch=None, bboxPadding=100)
def addFluxAliases(refCat, defaultFilter, filterReferenceMap)
def loadSkyCircle(self, ctrCoord, radius, filterName=None, epoch=None)
def joinMatchListWithCatalog(self, matchCat, sourceCat)
def __init__(self, dataIds, refCats, config, log=None)
def loadPixelBox(self, bbox, wcs, filterName=None, epoch=None, photoCalib=None, bboxPadding=100)
def getRefFluxKeys(schema, filterName=None)
def hasNanojanskyFluxUnits(schema)
def getRefFluxField(schema, filterName=None)
def convertToNanojansky(catalog, log, doConvert=True)
def joinMatchListWithCatalogImpl(refObjLoader, matchCat, sourceCat)
def isOldFluxField(name, units)
def getFormatVersionFromRefCat(refCat)
def applyProperMotionsImpl(log, catalog, epoch)