24__all__ = [
"getRefFluxField",
"getRefFluxKeys",
"LoadReferenceObjectsTask",
"LoadReferenceObjectsConfig",
25 "ReferenceObjectLoader"]
38import lsst.pipe.base
as pipeBase
39from lsst
import sphgeom
41from lsst.utils.timer
import timeMethod
45 """Return True if this name/units combination corresponds to an
46 "old-style" reference catalog flux field.
48 unitsCheck = units != 'nJy'
49 isFlux = name.endswith(
'_flux')
50 isFluxSigma = name.endswith(
'_fluxSigma')
51 isFluxErr = name.endswith(
'_fluxErr')
52 return (isFlux
or isFluxSigma
or isFluxErr)
and unitsCheck
56 """Return True if the units of all flux and fluxErr are correct (nJy).
65 """"Return the format version stored in a reference catalog header.
70 Reference catalog to inspect.
75 Format verison integer. Returns `0` if the catalog has no metadata
76 or the metadata does
not include a
"REFCAT_FORMAT_VERSION" key.
78 md = refCat.getMetadata()
82 return md.getScalar(
"REFCAT_FORMAT_VERSION")
88 """Convert fluxes in a catalog from jansky to nanojansky.
93 The catalog to convert.
95 Log to send messages to.
96 doConvert : `bool`, optional
97 Return a converted catalog,
or just identify the fields that need to be converted?
98 This supports the
"write=False" mode of `bin/convert_to_nJy.py`.
103 The converted catalog,
or None if ``doConvert``
is False.
107 Support
for old units
in reference catalogs will be removed after the
108 release of late calendar year 2019.
109 Use `meas_algorithms/bin/convert_to_nJy.py` to update your reference catalog.
113 mapper = afwTable.SchemaMapper(catalog.schema, shareAliasMap=
False)
114 mapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema())
117 for field
in catalog.schema:
118 oldName = field.field.getName()
119 oldUnits = field.field.getUnits()
123 if oldName.endswith(
'_fluxSigma'):
124 name = oldName.replace(
'_fluxSigma',
'_fluxErr')
127 newField = afwTable.Field[field.dtype](name, field.field.getDoc(), units)
128 mapper.addMapping(field.getKey(), newField)
129 input_fields.append(field.field)
130 output_fields.append(newField)
132 mapper.addMapping(field.getKey())
134 fluxFieldsStr =
'; '.join(
"(%s, '%s')" % (field.getName(), field.getUnits())
for field
in input_fields)
137 newSchema = mapper.getOutputSchema()
138 output = afwTable.SimpleCatalog(newSchema)
139 output.extend(catalog, mapper=mapper)
140 for field
in output_fields:
141 output[field.getName()] *= 1e9
142 log.info(
"Converted refcat flux fields to nJy (name, units): %s", fluxFieldsStr)
145 log.info(
"Found old-style refcat flux fields (name, units): %s", fluxFieldsStr)
150 """This is a private helper class which filters catalogs by
151 row based on the row being inside the region used to initialize
157 The spatial region which all objects should lie within
163 """This call method on an instance of this class takes in a reference
164 catalog, and the region
from which the catalog was generated.
166 If the catalog region
is entirely contained within the region used to
167 initialize this
class, then all the entries
in the catalog must be
168 within the region
and so the whole catalog
is returned.
170 If the catalog region
is not entirely contained, then the location
for
171 each record
is tested against the region used to initialize the
class.
172 Records which fall inside this region are added to a new catalog,
and
173 this catalog
is then returned.
178 SourceCatalog to be filtered.
180 Region
in which the catalog was created
182 if catRegion.isWithin(self.
regionregion):
186 filteredRefCat = type(refCat)(refCat.table)
187 for record
in refCat:
188 if self.
regionregion.contains(record.getCoord().getVector()):
189 filteredRefCat.append(record)
190 return filteredRefCat
194 pixelMargin = pexConfig.RangeField(
195 doc=
"Padding to add to 4 all edges of the bounding box (pixels)",
200 defaultFilter = pexConfig.Field(
201 doc=(
"Default reference catalog filter to use if filter not specified in exposure;"
202 " if blank then filter must be specified in exposure."),
205 deprecated=
"defaultFilter is deprecated by RFC-716. Will be removed after v22."
207 anyFilterMapsToThis = pexConfig.Field(
208 doc=(
"Always use this reference catalog filter, no matter whether or what filter name is "
209 "supplied to the loader. Effectively a trivial filterMap: map all filter names to this filter."
210 " This can be set for purely-astrometric catalogs (e.g. Gaia DR2) where there is only one "
211 "reasonable choice for every camera filter->refcat mapping, but not for refcats used for "
212 "photometry, which need a filterMap and/or colorterms/transmission corrections."),
217 filterMap = pexConfig.DictField(
218 doc=(
"Mapping of camera filter name: reference catalog filter name; "
219 "each reference filter must exist in the refcat."
220 " Note that this does not perform any bandpass corrections: it is just a lookup."),
225 requireProperMotion = pexConfig.Field(
226 doc=
"Require that the fields needed to correct proper motion "
227 "(epoch, pm_ra and pm_dec) are present?",
235 msg =
"`filterMap` and `anyFilterMapsToThis` are mutually exclusive"
236 raise pexConfig.FieldValidationError(LoadReferenceObjectsConfig.anyFilterMapsToThis,
241 """Base class for reference object loaders, to facilitate gen2/gen3 code
244 ConfigClass = LoadReferenceObjectsConfig
247 """Apply proper motion correction to a reference catalog.
249 Adjust position and position error
in the ``catalog``
250 for proper motion to the specified ``epoch``,
251 modifying the catalog
in place.
256 Catalog of positions, containing at least these fields:
258 - Coordinates, retrieved by the table
's coordinate key.
259 - ``coord_raErr`` : Error in Right Ascension (rad).
260 - ``coord_decErr`` : Error
in Declination (rad).
261 - ``pm_ra`` : Proper motion
in Right Ascension (rad/yr,
263 - ``pm_raErr`` : Error
in ``pm_ra`` (rad/yr), optional.
264 - ``pm_dec`` : Proper motion
in Declination (rad/yr,
266 - ``pm_decErr`` : Error
in ``pm_dec`` (rad/yr), optional.
267 - ``epoch`` : Mean epoch of object (an astropy.time.Time)
268 epoch : `astropy.time.Time`
269 Epoch to which to correct proper motion.
270 If
None, do
not apply PM corrections
or raise if
271 ``config.requireProperMotion``
is True.
276 Raised
if ``config.requireProperMotion``
is set but we cannot
277 apply the proper motion correction
for some reason.
280 if self.config.requireProperMotion:
281 raise RuntimeError(
"requireProperMotion=True but epoch not provided to loader.")
283 self.log.debug(
"No epoch provided: not applying proper motion corrections to refcat.")
287 if (
"pm_ra" in catalog.schema
288 and not isinstance(catalog.schema[
"pm_ra"].asKey(), afwTable.KeyAngle)):
289 if self.config.requireProperMotion:
290 raise RuntimeError(
"requireProperMotion=True but refcat pm_ra field is not an Angle.")
292 self.log.warning(
"Reference catalog pm_ra field is not an Angle; cannot apply proper motion.")
295 if (
"epoch" not in catalog.schema
or "pm_ra" not in catalog.schema):
296 if self.config.requireProperMotion:
297 raise RuntimeError(
"requireProperMotion=True but PM data not available from catalog.")
299 self.log.warning(
"Proper motion correction not available for this reference catalog.")
306 """This class facilitates loading reference catalogs with gen 3 middleware
308 The middleware preflight solver will create a list of datarefs that may
309 possibly overlap a given region. These datarefs are then used to construct
310 and instance of this
class. The
class instance should then be passed into
311 a task which needs reference catalogs. These tasks should then determine
312 the exact region of the sky reference catalogs will be loaded
for,
and
313 call a corresponding method to load the reference objects.
315 def __init__(self, dataIds, refCats, config, log=None):
316 """ Constructs an instance of ReferenceObjectLoader
320 dataIds : iterable of `lsst.daf.butler.DataIds`
321 An iterable object of DataSetRefs which point to reference catalogs
322 in a gen 3 repository.
323 refCats : iterable of `lsst.daf.butler.DeferedDatasetHandle`
324 Handles to load refCats on demand
326 Configuration
for the loader.
327 log : `
lsst.log.Log`, `logging.Logger`
or `
None`, optional
328 Logger object used to write out messages. If `
None` a default
333 self.loglog = log or logging.getLogger(__name__).getChild(
"ReferenceObjectLoader")
337 def _makeBoxRegion(BBox, wcs, BBoxPadding):
350 outerLocalBBox.grow(BBoxPadding)
351 innerLocalBBox.grow(-1*BBoxPadding)
363 innerBoxCorners = innerLocalBBox.getCorners()
364 innerSphCorners = [wcs.pixelToSky(corner).getVector()
for corner
in innerBoxCorners]
367 outerBoxCorners = outerLocalBBox.getCorners()
368 outerSphCorners = [wcs.pixelToSky(corner).getVector()
for corner
in outerBoxCorners]
371 return innerSkyRegion, outerSkyRegion, innerSphCorners, outerSphCorners
373 def loadPixelBox(self, bbox, wcs, filterName=None, epoch=None, photoCalib=None,
374 bboxToSpherePadding=100):
375 """Load reference objects that are within a pixel-based rectangular
378 This algorithm works by creating a spherical box whose corners
379 correspond to the WCS converted corners of the input bounding box
380 (possibly padded). It then defines a filtering function which looks at
381 the pixel position of the reference objects and accepts only those that
382 lie within the specified bounding box.
384 The spherical box region
and filtering function are passed to the
385 generic loadRegion method which loads
and filters the reference objects
386 from the datastore
and returns a single catalog containing the filtered
387 set of reference objects.
392 Box which bounds a region
in pixel space.
394 Wcs object defining the pixel to sky (
and inverse) transform
for
395 the supplied ``bbox``.
396 filterName : `str`
or `
None`, optional
397 Name of camera filter,
or `
None`
or blank
for the default filter.
398 epoch : `astropy.time.Time`
or `
None`, optional
399 Epoch to which to correct proper motion
and parallax,
or `
None`
400 to
not apply such corrections.
402 Deprecated
and ignored, only included
for api compatibility.
403 bboxToSpherePadding : `int`, optional
404 Padding to account
for translating a set of corners into a
405 spherical (convex) boundary that
is certain to encompase the
406 enitre area covered by the box.
411 Catalog containing reference objects inside the specified bounding
412 box (padded by self.
configconfig.pixelMargin).
417 Raised
if no reference catalogs could be found
for the specified
420 Raised
if the loaded reference catalogs do
not have matching
424 paddedBbox.grow(self.configconfig.pixelMargin)
425 innerSkyRegion, outerSkyRegion, _, _ = self._makeBoxRegion_makeBoxRegion(paddedBbox, wcs, bboxToSpherePadding)
427 def _filterFunction(refCat, region):
438 refCat = preFiltFunc(refCat, region)
444 afwTable.updateRefCentroids(wcs, refCat)
447 if innerSkyRegion.contains(region):
451 filteredRefCat = type(refCat)(refCat.table)
452 centroidKey = afwTable.Point2DKey(refCat.schema[
'centroid'])
453 for record
in refCat:
454 pixCoords = record[centroidKey]
456 filteredRefCat.append(record)
457 return filteredRefCat
458 return self.
loadRegionloadRegion(outerSkyRegion, filtFunc=_filterFunction, epoch=epoch, filterName=filterName)
460 def loadRegion(self, region, filtFunc=None, filterName=None, epoch=None):
461 """Load reference objects within a specified region.
463 This function loads the DataIds used to construct an instance of this
464 class which intersect or are contained within the specified region. The
465 reference catalogs which intersect but are
not fully contained within
466 the input region are further filtered by the specified filter function.
467 This function returns a single source catalog containing all reference
468 objects inside the specified region.
474 should define the spatial region
for which reference objects are to
476 filtFunc : callable
or `
None`, optional
477 This optional parameter should be a callable object that takes a
478 reference catalog
and its corresponding region
as parameters,
479 filters the catalog by some criteria
and returns the filtered
480 reference catalog. If `
None`, an internal filter function
is used
481 which filters according to
if a reference object falls within the
483 filterName : `str`
or `
None`, optional
484 Name of camera filter,
or `
None`
or blank
for the default filter.
485 epoch : `astropy.time.Time`
or `
None`, optional
486 Epoch to which to correct proper motion
and parallax,
or `
None` to
487 not apply such corrections.
492 Catalog containing reference objects which intersect the input region,
493 filtered by the specified filter function.
498 Raised
if no reference catalogs could be found
for the specified
501 Raised
if the loaded reference catalogs do
not have matching
504 regionLat = region.getBoundingBox().getLat()
505 regionLon = region.getBoundingBox().getLon()
506 self.loglog.info("Loading reference objects from %s in region bounded by "
507 "[%.8f, %.8f], [%.8f, %.8f] RA Dec",
509 self.
refCatsrefCats[0].ref.datasetType.name,
510 regionLon.getA().asDegrees(), regionLon.getB().asDegrees(),
511 regionLat.getA().asDegrees(), regionLat.getB().asDegrees())
516 for dataId, refCat
in zip(self.
dataIdsdataIds, self.
refCatsrefCats):
520 intersects = dataId.region.intersects(region)
522 intersects = region.intersects(dataId.region)
525 overlapList.append((dataId, refCat))
527 if len(overlapList) == 0:
528 raise RuntimeError(
"No reference tables could be found for input region")
530 firstCat = overlapList[0][1].get()
531 refCat = filtFunc(firstCat, overlapList[0][0].region)
532 trimmedAmount = len(firstCat) - len(refCat)
535 for dataId, inputRefCat
in overlapList[1:]:
536 tmpCat = inputRefCat.get()
538 if tmpCat.schema != firstCat.schema:
539 raise TypeError(
"Reference catalogs have mismatching schemas")
541 filteredCat = filtFunc(tmpCat, dataId.region)
542 refCat.extend(filteredCat)
543 trimmedAmount += len(tmpCat) - len(filteredCat)
545 self.
loglog.debug(
"Trimmed %d refCat objects lying outside padded region, leaving %d",
546 trimmedAmount, len(refCat))
547 self.
loglog.info(
"Loaded %d reference objects", len(refCat))
550 if not refCat.isContiguous():
551 refCat = refCat.copy(deep=
True)
558 self.
loglog.warning(
"Found version 0 reference catalog with old style units in schema.")
559 self.
loglog.warning(
"run `meas_algorithms/bin/convert_refcat_to_nJy.py` to convert fluxes to nJy.")
560 self.
loglog.warning(
"See RFC-575 for more details.")
569 if not expandedCat.isContiguous():
570 expandedCat = expandedCat.copy(deep=
True)
572 fluxField =
getRefFluxField(schema=expandedCat.schema, filterName=filterName)
573 return pipeBase.Struct(refCat=expandedCat, fluxField=fluxField)
576 """Load reference objects that lie within a circular region on the sky.
578 This method constructs a circular region from an input center
and
579 angular radius, loads reference catalogs which are contained
in or
580 intersect the circle,
and filters reference catalogs which intersect
581 down to objects which lie within the defined circle.
586 Point defining the center of the circular region.
588 Defines the angular radius of the circular region.
589 filterName : `str`
or `
None`, optional
590 Name of camera filter,
or `
None`
or blank
for the default filter.
591 epoch : `astropy.time.Time`
or `
None`, optional
592 Epoch to which to correct proper motion
and parallax,
or `
None` to
593 not apply such corrections.
598 Catalog containing reference objects inside the specified search
601 centerVector = ctrCoord.getVector()
604 return self.
loadRegionloadRegion(circularRegion, filterName=filterName, epoch=epoch)
607 """Relink an unpersisted match list to sources and reference objects.
609 A match list is persisted
and unpersisted
as a catalog of IDs
610 produced by afw.table.packMatches(),
with match metadata
611 (
as returned by the astrometry tasks)
in the catalog
's metadata
612 attribute. This method converts such a match catalog into a match
613 list, with links to source records
and reference object records.
618 Unpersisted packed match list.
619 ``matchCat.table.getMetadata()`` must contain match metadata,
620 as returned by the astrometry tasks.
622 Source catalog. As a side effect, the catalog will be sorted
632 def getMetadataBox(self, bbox, wcs, filterName=None, photoCalib=None, epoch=None,
633 bboxToSpherePadding=100):
634 """Return metadata about the load
636 This metadata is used
for reloading the catalog (e.g.,
for
637 reconstituting a normalised match list.)
642 Bounding box
for the pixels.
644 The WCS object associated
with ``bbox``.
645 filterName : `str`
or `
None`, optional
646 Name of the camera filter,
or `
None`
or blank
for the default
649 Deprecated, only included
for api compatibility.
650 epoch : `astropy.time.Time`
or `
None`, optional
651 Epoch to which to correct proper motion
and parallax,
or `
None` to
652 not apply such corrections.
653 bboxToSpherePadding : `int`, optional
654 Padding to account
for translating a set of corners into a
655 spherical (convex) boundary that
is certain to encompase the
656 enitre area covered by the box.
661 The metadata detailing the search parameters used
for this
665 paddedBbox.grow(self.configconfig.pixelMargin)
666 _, _, innerCorners, outerCorners = self._makeBoxRegion_makeBoxRegion(paddedBbox, wcs, bboxToSpherePadding)
668 for box, corners
in zip((
"INNER",
"OUTER"), (innerCorners, outerCorners)):
669 for (name, corner)
in zip((
"UPPER_LEFT",
"UPPER_RIGHT",
"LOWER_LEFT",
"LOWER_RIGHT"),
671 md.add(f
"{box}_{name}_RA",
geom.SpherePoint(corner).getRa().asDegrees(), f
"{box}_corner")
672 md.add(f
"{box}_{name}_DEC",
geom.SpherePoint(corner).getDec().asDegrees(), f
"{box}_corner")
673 md.add(
"SMATCHV", 1,
'SourceMatchVector version number')
674 filterName =
"UNKNOWN" if filterName
is None else str(filterName)
675 md.add(
'FILTER', filterName,
'filter name for photometric data')
676 md.add(
'EPOCH',
"NONE" if epoch
is None else epoch.mjd,
'Epoch (TAI MJD) for catalog')
681 """Return metadata about the load.
683 This metadata is used
for reloading the catalog (e.g.
for
684 reconstituting a normalized match list.)
689 ICRS center of the search region.
691 Radius of the search region.
692 filterName : `str`
or `
None`
693 Name of the camera filter,
or `
None`
or blank
for the default
696 Deprecated, only included
for api compatibility.
697 epoch : `astropy.time.Time`
or `
None`, optional
698 Epoch to which to correct proper motion
and parallax,
or `
None` to
699 not apply such corrections.
706 md.add('RA', coord.getRa().asDegrees(),
'field center in degrees')
707 md.add(
'DEC', coord.getDec().asDegrees(),
'field center in degrees')
708 md.add(
'RADIUS', radius.asDegrees(),
'field radius in degrees, minimum')
709 md.add(
'SMATCHV', 1,
'SourceMatchVector version number')
710 filterName =
"UNKNOWN" if filterName
is None else str(filterName)
711 md.add(
'FILTER', filterName,
'filter name for photometric data')
712 md.add(
'EPOCH',
"NONE" if epoch
is None else epoch.mjd,
'Epoch (TAI MJD) for catalog')
717 """Add flux columns and aliases for camera to reference mapping.
719 Creates a new catalog containing the information of the input refCat
720 as well
as added flux columns
and aliases between camera
and reference
726 Catalog of reference objects
727 defaultFilter : `str`
728 Name of the default reference filter
729 filterReferenceMap : `dict` of `str`
730 Dictionary
with keys corresponding to a filter name
and values
731 which correspond to the name of the reference filter.
736 Reference catalog
with columns added to track reference filters.
741 If the specified reference filter name
is not specifed
as a
742 key
in the reference filter map.
744 refCat = ReferenceObjectLoader.remapReferenceCatalogSchema(refCat,
745 filterNameList=filterReferenceMap.keys())
746 aliasMap = refCat.schema.getAliasMap()
747 if filterReferenceMap
is None:
748 filterReferenceMap = {}
749 for filterName, refFilterName
in itertools.chain([(
None, defaultFilter)],
750 filterReferenceMap.items()):
752 camFluxName = filterName +
"_camFlux" if filterName
is not None else "camFlux"
753 refFluxName = refFilterName +
"_flux"
754 if refFluxName
not in refCat.schema:
755 raise RuntimeError(
"Unknown reference filter %s" % (refFluxName,))
756 aliasMap.set(camFluxName, refFluxName)
758 refFluxErrName = refFluxName +
"Err"
759 camFluxErrName = camFluxName +
"Err"
760 aliasMap.set(camFluxErrName, refFluxErrName)
766 """This function takes in a reference catalog and creates a new catalog with additional
767 columns defined the remaining function arguments.
772 Reference catalog to map to new catalog
777 Deep copy of input reference catalog with additional columns added
779 mapper = afwTable.SchemaMapper(refCat.schema, True)
780 mapper.addMinimalSchema(refCat.schema,
True)
781 mapper.editOutputSchema().disconnectAliases()
783 for filterName
in filterNameList:
784 mapper.editOutputSchema().addField(f
"{filterName}_flux",
786 doc=f
"flux in filter {filterName}",
789 mapper.editOutputSchema().addField(f
"{filterName}_fluxErr",
791 doc=f
"flux uncertanty in filter {filterName}",
796 mapper.editOutputSchema().addField(
"centroid_x", type=float, doReplace=
True)
797 mapper.editOutputSchema().addField(
"centroid_y", type=float, doReplace=
True)
798 mapper.editOutputSchema().addField(
"hasCentroid", type=
"Flag", doReplace=
True)
799 mapper.editOutputSchema().getAliasMap().set(
"slot_Centroid",
"centroid")
802 mapper.editOutputSchema().addField(
"photometric",
804 doc=
"set if the object can be used for photometric"
807 mapper.editOutputSchema().addField(
"resolved",
809 doc=
"set if the object is spatially resolved"
811 mapper.editOutputSchema().addField(
"variable",
813 doc=
"set if the object has variable brightness"
816 expandedCat = afwTable.SimpleCatalog(mapper.getOutputSchema())
817 expandedCat.setMetadata(refCat.getMetadata())
818 expandedCat.extend(refCat, mapper=mapper)
824 """Get the name of a flux field from a schema.
826 return the alias of
"anyFilterMapsToThis",
if present
827 else if filterName
is specified:
828 return "*filterName*_camFlux" if present
829 else return "*filterName*_flux" if present (camera filter name
830 matches reference filter name)
831 else throw RuntimeError
833 return "camFlux",
if present,
834 else throw RuntimeError
839 Reference catalog schema.
840 filterName : `str`, optional
841 Name of camera filter. If
not specified, ``defaultFilter`` needs to be
842 set
in the refcat loader config.
846 fluxFieldName : `str`
852 If an appropriate field
is not found.
854 if not isinstance(schema, afwTable.Schema):
855 raise RuntimeError(
"schema=%s is not a schema" % (schema,))
857 return schema.getAliasMap().get(
"anyFilterMapsToThis")
862 fluxFieldList = [filterName +
"_camFlux", filterName +
"_flux"]
864 fluxFieldList = [
"camFlux"]
865 for fluxField
in fluxFieldList:
866 if fluxField
in schema:
869 raise RuntimeError(
"Could not find flux field(s) %s" % (
", ".join(fluxFieldList)))
873 """Return keys for flux and flux error.
878 Reference catalog schema.
880 Name of camera filter.
888 - flux error key, if present,
else None
893 If flux field
not found.
896 fluxErrField = fluxField + "Err"
897 fluxKey = schema[fluxField].asKey()
899 fluxErrKey = schema[fluxErrField].asKey()
902 return (fluxKey, fluxErrKey)
906 """Abstract base class to load objects from reference catalogs.
908 _DefaultName = "LoadReferenceObjects"
911 """Construct a LoadReferenceObjectsTask
915 butler : `lsst.daf.persistence.Butler`
916 Data butler, for access reference catalogs.
918 pipeBase.Task.__init__(self, *args, **kwargs)
922 def loadPixelBox(self, bbox, wcs, filterName=None, photoCalib=None, epoch=None):
923 """Load reference objects that overlap a rectangular pixel region.
928 Bounding box
for pixels.
930 WCS; used to convert pixel positions to sky coordinates
932 filterName : `str`
or `
None`, optional
933 Name of filter,
or `
None`
or `
""`
for the default filter.
934 This
is used
for flux values
in case we have flux limits
935 (which are
not yet implemented).
937 Deprecated, only included
for api compatibility.
938 epoch : `astropy.time.Time`
or `
None`, optional
939 Epoch to which to correct proper motion
and parallax,
or `
None` to
940 not apply such corrections.
944 results : `lsst.pipe.base.Struct`
945 A Struct containing the following fields:
946 refCat : `lsst.afw.catalog.SimpleCatalog`
947 A catalog of reference objects
with the standard
948 schema,
as documented
in the main doc string
for
949 `LoadReferenceObjects`.
950 The catalog
is guaranteed to be contiguous.
952 Name of flux field
for specified `filterName`.
956 The search algorithm works by searching
in a region
in sky
957 coordinates whose center
is the center of the bbox
and radius
958 is large enough to just include all 4 corners of the bbox.
959 Stars that lie outside the bbox are then trimmed
from the list.
964 self.log.info(
"Loading reference objects from %s using center %s and radius %s deg",
965 self.config.ref_dataset_name, circle.coord, circle.radius.asDegrees())
966 loadRes = self.
loadSkyCircleloadSkyCircle(circle.coord, circle.radius, filterName=filterName, epoch=epoch,
968 refCat = loadRes.refCat
969 numFound = len(refCat)
972 refCat = self.
_trimToBBox_trimToBBox(refCat=refCat, bbox=circle.bbox, wcs=wcs)
973 numTrimmed = numFound - len(refCat)
974 self.log.debug(
"trimmed %d out-of-bbox objects, leaving %d", numTrimmed, len(refCat))
975 self.log.info(
"Loaded %d reference objects", len(refCat))
978 if not refCat.isContiguous():
979 loadRes.refCat = refCat.copy(deep=
True)
984 def loadSkyCircle(self, ctrCoord, radius, filterName=None, epoch=None, centroids=False):
985 """Load reference objects that overlap a circular sky region.
990 ICRS center of search region.
992 Radius of search region.
993 filterName : `str` or `
None`, optional
994 Name of filter,
or `
None`
or `
""`
for the default filter.
995 This
is used
for flux values
in case we have flux limits
996 (which are
not yet implemented).
997 epoch : `astropy.time.Time`
or `
None`, optional
998 Epoch to which to correct proper motion
and parallax,
or `
None` to
999 not apply such corrections.
1000 centroids : `bool`, optional
1001 Add centroid fields to the loaded Schema. ``loadPixelBox`` expects
1002 these fields to exist.
1006 results : `lsst.pipe.base.Struct`
1007 A Struct containing the following fields:
1008 refCat : `lsst.afw.catalog.SimpleCatalog`
1009 A catalog of reference objects
with the standard
1010 schema,
as documented
in the main doc string
for
1011 `LoadReferenceObjects`.
1012 The catalog
is guaranteed to be contiguous.
1014 Name of flux field
for specified `filterName`.
1018 Note that subclasses are responsible
for performing the proper motion
1019 correction, since this
is the lowest-level interface
for retrieving
1025 def _trimToBBox(refCat, bbox, wcs):
1026 """Remove objects outside a given pixel bounding box and set
1027 centroid and hasCentroid fields.
1032 A catalog of objects. The schema must include fields
1033 "coord",
"centroid" and "hasCentroid".
1034 The
"coord" field
is read.
1035 The
"centroid" and "hasCentroid" fields are set.
1039 WCS; used to convert sky coordinates to pixel positions.
1044 Reference objects
in the bbox,
with centroid
and
1045 hasCentroid fields set.
1047 afwTable.updateRefCentroids(wcs, refCat)
1048 centroidKey = afwTable.Point2DKey(refCat.schema["centroid"])
1049 retStarCat = type(refCat)(refCat.table)
1051 point = star.get(centroidKey)
1052 if bbox.contains(point):
1053 retStarCat.append(star)
1056 def _addFluxAliases(self, schema):
1057 """Add aliases for camera filter fluxes to the schema.
1059 If self.config.defaultFilter then adds these aliases:
1060 camFlux: <defaultFilter>_flux
1061 camFluxErr: <defaultFilter>_fluxErr, if the latter exists
1063 For each camFilter: refFilter
in self.config.filterMap adds these aliases:
1064 <camFilter>_camFlux: <refFilter>_flux
1065 <camFilter>_camFluxErr: <refFilter>_fluxErr,
if the latter exists
1070 Schema
for reference catalog.
1075 If any reference flux field
is missing
from the schema.
1077 aliasMap = schema.getAliasMap()
1079 if self.config.anyFilterMapsToThis
is not None:
1080 refFluxName = self.config.anyFilterMapsToThis +
"_flux"
1081 if refFluxName
not in schema:
1082 msg = f
"Unknown reference filter for anyFilterMapsToThis='{refFluxName}'"
1083 raise RuntimeError(msg)
1084 aliasMap.set(
"anyFilterMapsToThis", refFluxName)
1087 def addAliasesForOneFilter(filterName, refFilterName):
1088 """Add aliases for a single filter
1092 filterName : `str` (optional)
1093 Camera filter name. The resulting alias name is
1094 <filterName>_camFlux,
or simply
"camFlux" if `filterName`
1096 refFilterName : `str`
1097 Reference catalog filter name; the field
1098 <refFilterName>_flux must exist.
1100 camFluxName = filterName + "_camFlux" if filterName
is not None else "camFlux"
1101 refFluxName = refFilterName +
"_flux"
1102 if refFluxName
not in schema:
1103 raise RuntimeError(
"Unknown reference filter %s" % (refFluxName,))
1104 aliasMap.set(camFluxName, refFluxName)
1105 refFluxErrName = refFluxName +
"Err"
1106 if refFluxErrName
in schema:
1107 camFluxErrName = camFluxName +
"Err"
1108 aliasMap.set(camFluxErrName, refFluxErrName)
1110 if self.config.defaultFilter:
1111 addAliasesForOneFilter(
None, self.config.defaultFilter)
1113 for filterName, refFilterName
in self.config.filterMap.items():
1114 addAliasesForOneFilter(filterName, refFilterName)
1118 addIsPhotometric=False, addIsResolved=False,
1119 addIsVariable=False, coordErrDim=2,
1120 addProperMotion=False, properMotionErrDim=2,
1122 """Make a standard schema for reference object catalogs.
1126 filterNameList : `list` of `str`
1127 List of filter names. Used to create <filterName>_flux fields.
1128 addIsPhotometric : `bool`
1129 If True then add field
"photometric".
1130 addIsResolved : `bool`
1131 If
True then add field
"resolved".
1132 addIsVariable : `bool`
1133 If
True then add field
"variable".
1135 Number of coord error fields; must be one of 0, 2, 3:
1137 - If 2
or 3: add fields
"coord_raErr" and "coord_decErr".
1138 - If 3: also add field
"coord_radecErr".
1139 addProperMotion : `bool`
1140 If
True add fields
"epoch",
"pm_ra",
"pm_dec" and "pm_flag".
1141 properMotionErrDim : `int`
1142 Number of proper motion error fields; must be one of 0, 2, 3;
1143 ignored
if addProperMotion false:
1144 - If 2
or 3: add fields
"pm_raErr" and "pm_decErr".
1145 - If 3: also add field
"pm_radecErr".
1146 addParallax : `bool`
1147 If
True add fields
"epoch",
"parallax",
"parallaxErr"
1148 and "parallax_flag".
1153 Schema
for reference catalog, an
1158 Reference catalogs support additional covariances, such
as
1159 covariance between RA
and proper motion
in declination,
1160 that are
not supported by this method, but can be added after
1161 calling this method.
1163 schema = afwTable.SimpleTable.makeMinimalSchema()
1165 afwTable.Point2DKey.addFields(
1168 "centroid on an exposure, if relevant",
1172 field=
"hasCentroid",
1174 doc=
"is position known?",
1176 for filterName
in filterNameList:
1178 field=
"%s_flux" % (filterName,),
1180 doc=
"flux in filter %s" % (filterName,),
1183 for filterName
in filterNameList:
1185 field=
"%s_fluxErr" % (filterName,),
1187 doc=
"flux uncertainty in filter %s" % (filterName,),
1190 if addIsPhotometric:
1192 field=
"photometric",
1194 doc=
"set if the object can be used for photometric calibration",
1200 doc=
"set if the object is spatially resolved",
1206 doc=
"set if the object has variable brightness",
1208 if coordErrDim
not in (0, 2, 3):
1209 raise ValueError(
"coordErrDim={}; must be (0, 2, 3)".format(coordErrDim))
1211 afwTable.CovarianceMatrix2fKey.addFields(
1214 names=[
"ra",
"dec"],
1215 units=[
"rad",
"rad"],
1216 diagonalOnly=(coordErrDim == 2),
1219 if addProperMotion
or addParallax:
1223 doc=
"date of observation (TAI, MJD)",
1231 doc=
"proper motion in the right ascension direction = dra/dt * cos(dec)",
1237 doc=
"proper motion in the declination direction",
1240 if properMotionErrDim
not in (0, 2, 3):
1241 raise ValueError(
"properMotionErrDim={}; must be (0, 2, 3)".format(properMotionErrDim))
1242 if properMotionErrDim > 0:
1243 afwTable.CovarianceMatrix2fKey.addFields(
1246 names=[
"ra",
"dec"],
1247 units=[
"rad/year",
"rad/year"],
1248 diagonalOnly=(properMotionErrDim == 2),
1253 doc=
"Set if proper motion or proper motion error is bad",
1264 field=
"parallaxErr",
1266 doc=
"uncertainty in parallax",
1270 field=
"parallax_flag",
1272 doc=
"Set if parallax or parallax error is bad",
1276 def _calculateCircle(self, bbox, wcs):
1277 """Compute on-sky center and radius of search region.
1284 WCS; used to convert pixel positions to sky coordinates.
1288 results : `lsst.pipe.base.Struct`
1289 A Struct containing:
1292 ICRS center of the search region.
1294 Radius of the search region.
1296 Bounding box used to compute the circle.
1299 bbox.grow(self.config.pixelMargin)
1300 coord = wcs.pixelToSky(bbox.getCenter())
1301 radius = max(coord.separation(wcs.pixelToSky(pp))
for pp
in bbox.getCorners())
1302 return pipeBase.Struct(coord=coord, radius=radius, bbox=bbox)
1305 """Return metadata about the load.
1307 This metadata is used
for reloading the catalog (e.g.,
for
1308 reconstituting a normalised match list.
1315 WCS; used to convert pixel positions to sky coordinates.
1316 filterName : `str`
or `
None`, optional
1317 Name of camera filter,
or `
None`
or `
""`
for the default
1320 Deprecated, only included
for api compatibility.
1321 epoch : `astropy.time.Time`
or `
None`, optional
1322 Epoch to which to correct proper motion
and parallax,
1323 or None to
not apply such corrections.
1328 Metadata about the load.
1331 return self.
getMetadataCirclegetMetadataCircle(circle.coord, circle.radius, filterName, epoch=epoch)
1334 """Return metadata about the load.
1336 This metadata is used
for reloading the catalog (e.g.,
for
1337 reconstituting a normalised match list.
1342 ICRS center of the search region.
1344 Radius of the search region.
1346 Name of camera filter,
or `
None`
or `
""`
for the default
1349 Deprecated, only included
for api compatibility.
1350 epoch : `astropy.time.Time` (optional)
1351 Epoch to which to correct proper motion
and parallax,
or `
None` to
1352 not apply such corrections.
1357 Metadata about the load
1360 md.add('RA', coord.getRa().asDegrees(),
'field center in degrees')
1361 md.add(
'DEC', coord.getDec().asDegrees(),
'field center in degrees')
1362 md.add(
'RADIUS', radius.asDegrees(),
'field radius in degrees, minimum')
1363 md.add(
'SMATCHV', 1,
'SourceMatchVector version number')
1364 filterName =
"UNKNOWN" if filterName
is None else str(filterName)
1365 md.add(
'FILTER', filterName,
'filter name for photometric data')
1366 md.add(
'EPOCH',
"NONE" if epoch
is None else epoch.mjd,
'Epoch (TAI MJD) for catalog')
1370 """Relink an unpersisted match list to sources and reference
1373 A match list is persisted
and unpersisted
as a catalog of IDs
1374 produced by afw.table.packMatches(),
with match metadata
1375 (
as returned by the astrometry tasks)
in the catalog
's metadata
1376 attribute. This method converts such a match catalog into a match
1377 list, with links to source records
and reference object records.
1382 Unperisted packed match list.
1383 ``matchCat.table.getMetadata()`` must contain match metadata,
1384 as returned by the astrometry tasks.
1386 Source catalog. As a side effect, the catalog will be sorted
1398 """Relink an unpersisted match list to sources and reference
1401 A match list is persisted
and unpersisted
as a catalog of IDs
1402 produced by afw.table.packMatches(),
with match metadata
1403 (
as returned by the astrometry tasks)
in the catalog
's metadata
1404 attribute. This method converts such a match catalog into a match
1405 list, with links to source records
and reference object records.
1410 Reference object loader to use
in getting reference objects
1412 Unperisted packed match list.
1413 ``matchCat.table.getMetadata()`` must contain match metadata,
1414 as returned by the astrometry tasks.
1416 Source catalog. As a side effect, the catalog will be sorted
1424 matchmeta = matchCat.table.getMetadata()
1425 version = matchmeta.getInt('SMATCHV')
1427 raise ValueError(
'SourceMatchVector version number is %i, not 1.' % version)
1428 filterName = matchmeta.getString(
'FILTER').strip()
1430 epoch = matchmeta.getDouble(
'EPOCH')
1431 except (LookupError, TypeError):
1433 if 'RADIUS' in matchmeta:
1436 matchmeta.getDouble(
'DEC'), geom.degrees)
1437 rad = matchmeta.getDouble(
'RADIUS')*geom.degrees
1438 refCat = refObjLoader.loadSkyCircle(ctrCoord, rad, filterName, epoch=epoch).refCat
1439 elif "INNER_UPPER_LEFT_RA" in matchmeta:
1445 for place
in (
"UPPER_LEFT",
"UPPER_RIGHT",
"LOWER_LEFT",
"LOWER_RIGHT"):
1447 matchmeta.getDouble(f
"OUTER_{place}_DEC"),
1448 geom.degrees).getVector()
1451 refCat = refObjLoader.loadRegion(outerBox, filterName=filterName, epoch=epoch).refCat
1455 return afwTable.unpackMatches(matchCat, refCat, sourceCat)
1459 """Apply proper motion correction to a reference catalog.
1461 Adjust position and position error
in the ``catalog``
1462 for proper motion to the specified ``epoch``,
1463 modifying the catalog
in place.
1468 Log object to write to.
1470 Catalog of positions, containing:
1472 - Coordinates, retrieved by the table
's coordinate key.
1473 - ``coord_raErr`` : Error in Right Ascension (rad).
1474 - ``coord_decErr`` : Error
in Declination (rad).
1475 - ``pm_ra`` : Proper motion
in Right Ascension (rad/yr,
1477 - ``pm_raErr`` : Error
in ``pm_ra`` (rad/yr), optional.
1478 - ``pm_dec`` : Proper motion
in Declination (rad/yr,
1480 - ``pm_decErr`` : Error
in ``pm_dec`` (rad/yr), optional.
1481 - ``epoch`` : Mean epoch of object (an astropy.time.Time)
1482 epoch : `astropy.time.Time`
1483 Epoch to which to correct proper motion.
1485 if "epoch" not in catalog.schema
or "pm_ra" not in catalog.schema
or "pm_dec" not in catalog.schema:
1486 log.warning(
"Proper motion correction not available from catalog")
1488 if not catalog.isContiguous():
1489 raise RuntimeError(
"Catalog must be contiguous")
1490 catEpoch = astropy.time.Time(catalog[
"epoch"], scale=
"tai", format=
"mjd")
1491 log.info(
"Correcting reference catalog for proper motion to %r", epoch)
1493 timeDiffsYears = (epoch.tai - catEpoch).to(astropy.units.yr).value
1494 coordKey = catalog.table.getCoordKey()
1497 pmRaRad = catalog[
"pm_ra"]
1498 pmDecRad = catalog[
"pm_dec"]
1499 offsetsRaRad = pmRaRad*timeDiffsYears
1500 offsetsDecRad = pmDecRad*timeDiffsYears
1508 offsetBearingsRad = numpy.arctan2(pmDecRad*1e6, pmRaRad*1e6)
1509 offsetAmountsRad = numpy.hypot(offsetsRaRad, offsetsDecRad)
1510 for record, bearingRad, amountRad
in zip(catalog, offsetBearingsRad, offsetAmountsRad):
1511 record.set(coordKey,
1512 record.get(coordKey).offset(bearing=bearingRad*geom.radians,
1513 amount=amountRad*geom.radians))
1515 if "coord_raErr" in catalog.schema:
1516 catalog[
"coord_raErr"] = numpy.hypot(catalog[
"coord_raErr"],
1517 catalog[
"pm_raErr"]*timeDiffsYears)
1518 if "coord_decErr" in catalog.schema:
1519 catalog[
"coord_decErr"] = numpy.hypot(catalog[
"coord_decErr"],
1520 catalog[
"pm_decErr"]*timeDiffsYears)
def __call__(self, refCat, catRegion)
def __init__(self, region)
def _trimToBBox(refCat, bbox, wcs)
def joinMatchListWithCatalog(self, matchCat, sourceCat)
def makeMinimalSchema(filterNameList, *addCentroid=False, addIsPhotometric=False, addIsResolved=False, addIsVariable=False, coordErrDim=2, addProperMotion=False, properMotionErrDim=2, addParallax=False)
def getMetadataCircle(self, coord, radius, filterName, photoCalib=None, epoch=None)
def __init__(self, butler=None, *args, **kwargs)
def _calculateCircle(self, bbox, wcs)
def loadPixelBox(self, bbox, wcs, filterName=None, photoCalib=None, epoch=None)
def loadSkyCircle(self, ctrCoord, radius, filterName=None, epoch=None, centroids=False)
def getMetadataBox(self, bbox, wcs, filterName=None, photoCalib=None, epoch=None)
def applyProperMotions(self, catalog, epoch)
def loadRegion(self, region, filtFunc=None, filterName=None, epoch=None)
def getMetadataCircle(coord, radius, filterName, photoCalib=None, epoch=None)
def _makeBoxRegion(BBox, wcs, BBoxPadding)
def remapReferenceCatalogSchema(refCat, *filterNameList=None, position=False, photometric=False)
def addFluxAliases(refCat, defaultFilter, filterReferenceMap)
def getMetadataBox(self, bbox, wcs, filterName=None, photoCalib=None, epoch=None, bboxToSpherePadding=100)
def loadPixelBox(self, bbox, wcs, filterName=None, epoch=None, photoCalib=None, bboxToSpherePadding=100)
def loadSkyCircle(self, ctrCoord, radius, filterName=None, epoch=None)
def joinMatchListWithCatalog(self, matchCat, sourceCat)
def __init__(self, dataIds, refCats, config, log=None)
def getRefFluxKeys(schema, filterName=None)
def hasNanojanskyFluxUnits(schema)
def getRefFluxField(schema, filterName=None)
def convertToNanojansky(catalog, log, doConvert=True)
def joinMatchListWithCatalogImpl(refObjLoader, matchCat, sourceCat)
def isOldFluxField(name, units)
def getFormatVersionFromRefCat(refCat)
def applyProperMotionsImpl(log, catalog, epoch)