lsst.meas.algorithms  21.0.0-12-g63909ac9+643a1044a5
loadReferenceObjects.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 #
4 # Copyright 2008-2017 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 
24 __all__ = ["getRefFluxField", "getRefFluxKeys", "LoadReferenceObjectsTask", "LoadReferenceObjectsConfig",
25  "ReferenceObjectLoader"]
26 
27 import abc
28 import itertools
29 
30 import astropy.time
31 import astropy.units
32 import numpy
33 
34 import lsst.geom
35 import lsst.afw.table as afwTable
36 import lsst.pex.config as pexConfig
37 import lsst.pex.exceptions as pexExceptions
38 import lsst.pipe.base as pipeBase
39 import lsst.pex.exceptions as pexExcept
40 import lsst.log
41 from lsst import geom
42 from lsst import sphgeom
43 from lsst.daf.base import PropertyList
44 
45 
46 def isOldFluxField(name, units):
47  """Return True if this name/units combination corresponds to an
48  "old-style" reference catalog flux field.
49  """
50  unitsCheck = units != 'nJy' # (units == 'Jy' or units == '' or units == '?')
51  isFlux = name.endswith('_flux')
52  isFluxSigma = name.endswith('_fluxSigma')
53  isFluxErr = name.endswith('_fluxErr')
54  return (isFlux or isFluxSigma or isFluxErr) and unitsCheck
55 
56 
58  """Return True if the units of all flux and fluxErr are correct (nJy).
59  """
60  for field in schema:
61  if isOldFluxField(field.field.getName(), field.field.getUnits()):
62  return False
63  return True
64 
65 
67  """"Return the format version stored in a reference catalog header.
68 
69  Parameters
70  ----------
71  refCat : `lsst.afw.table.SimpleCatalog`
72  Reference catalog to inspect.
73 
74  Returns
75  -------
76  version : `int` or `None`
77  Format version integer, or `None` if the catalog has no metadata
78  or the metadata does not include a "REFCAT_FORMAT_VERSION" key.
79  """
80  md = refCat.getMetadata()
81  if md is None:
82  return None
83  try:
84  return md.getScalar("REFCAT_FORMAT_VERSION")
85  except KeyError:
86  return None
87 
88 
89 def convertToNanojansky(catalog, log, doConvert=True):
90  """Convert fluxes in a catalog from jansky to nanojansky.
91 
92  Parameters
93  ----------
94  catalog : `lsst.afw.table.SimpleCatalog`
95  The catalog to convert.
96  log : `lsst.log.Log`
97  Log to send messages to.
98  doConvert : `bool`, optional
99  Return a converted catalog, or just identify the fields that need to be converted?
100  This supports the "write=False" mode of `bin/convert_to_nJy.py`.
101 
102  Returns
103  -------
104  catalog : `lsst.afw.table.SimpleCatalog` or None
105  The converted catalog, or None if ``doConvert`` is False.
106 
107  Notes
108  -----
109  Support for old units in reference catalogs will be removed after the
110  release of late calendar year 2019.
111  Use `meas_algorithms/bin/convert_to_nJy.py` to update your reference catalog.
112  """
113  # Do not share the AliasMap: for refcats, that gets created when the
114  # catalog is read from disk and should not be propagated.
115  mapper = lsst.afw.table.SchemaMapper(catalog.schema, shareAliasMap=False)
116  mapper.addMinimalSchema(lsst.afw.table.SimpleTable.makeMinimalSchema())
117  input_fields = []
118  output_fields = []
119  for field in catalog.schema:
120  oldName = field.field.getName()
121  oldUnits = field.field.getUnits()
122  if isOldFluxField(oldName, oldUnits):
123  units = 'nJy'
124  # remap Sigma flux fields to Err, so we can drop the alias
125  if oldName.endswith('_fluxSigma'):
126  name = oldName.replace('_fluxSigma', '_fluxErr')
127  else:
128  name = oldName
129  newField = lsst.afw.table.Field[field.dtype](name, field.field.getDoc(), units)
130  mapper.addMapping(field.getKey(), newField)
131  input_fields.append(field.field)
132  output_fields.append(newField)
133  else:
134  mapper.addMapping(field.getKey())
135 
136  fluxFieldsStr = '; '.join("(%s, '%s')" % (field.getName(), field.getUnits()) for field in input_fields)
137 
138  if doConvert:
139  newSchema = mapper.getOutputSchema()
140  output = lsst.afw.table.SimpleCatalog(newSchema)
141  output.extend(catalog, mapper=mapper)
142  for field in output_fields:
143  output[field.getName()] *= 1e9
144  log.info(f"Converted refcat flux fields to nJy (name, units): {fluxFieldsStr}")
145  return output
146  else:
147  log.info(f"Found old-style refcat flux fields (name, units): {fluxFieldsStr}")
148  return None
149 
150 
152  """This is a private helper class which filters catalogs by
153  row based on the row being inside the region used to initialize
154  the class.
155 
156  Parameters
157  ----------
158  region : `lsst.sphgeom.Region`
159  The spatial region which all objects should lie within
160  """
161  def __init__(self, region):
162  self.regionregion = region
163 
164  def __call__(self, refCat, catRegion):
165  """This call method on an instance of this class takes in a reference
166  catalog, and the region from which the catalog was generated.
167 
168  If the catalog region is entirely contained within the region used to
169  initialize this class, then all the entries in the catalog must be
170  within the region and so the whole catalog is returned.
171 
172  If the catalog region is not entirely contained, then the location for
173  each record is tested against the region used to initialize the class.
174  Records which fall inside this region are added to a new catalog, and
175  this catalog is then returned.
176 
177  Parameters
178  ---------
179  refCat : `lsst.afw.table.SourceCatalog`
180  SourceCatalog to be filtered.
181  catRegion : `lsst.sphgeom.Region`
182  Region in which the catalog was created
183  """
184  if catRegion.isWithin(self.regionregion):
185  # no filtering needed, region completely contains refcat
186  return refCat
187 
188  filteredRefCat = type(refCat)(refCat.table)
189  for record in refCat:
190  if self.regionregion.contains(record.getCoord().getVector()):
191  filteredRefCat.append(record)
192  return filteredRefCat
193 
194 
196  """ This class facilitates loading reference catalogs with gen 3 middleware
197 
198  The middleware preflight solver will create a list of datarefs that may
199  possibly overlap a given region. These datarefs are then used to construct
200  and instance of this class. The class instance should then be passed into
201  a task which needs reference catalogs. These tasks should then determine
202  the exact region of the sky reference catalogs will be loaded for, and
203  call a corresponding method to load the reference objects.
204  """
205  def __init__(self, dataIds, refCats, config, log=None):
206  """ Constructs an instance of ReferenceObjectLoader
207 
208  Parameters
209  ----------
210  dataIds : iterable of `lsst.daf.butler.DataIds`
211  An iterable object of DataSetRefs which point to reference catalogs
212  in a gen 3 repository
213  refCats : Iterable of `lsst.daf.butler.DeferedDatasetHandle`
214  Handles to load refCats on demand
215  log : `lsst.log.Log`
216  Logger object used to write out messages. If `None` (default) the default
217  lsst logger will be used
218 
219  """
220  self.dataIdsdataIds = dataIds
221  self.refCatsrefCats = refCats
223  self.configconfig = config
224 
225  @staticmethod
226  def _makeBoxRegion(BBox, wcs, BBoxPadding):
227  outerLocalBBox = geom.Box2D(BBox)
228  innerLocalBBox = geom.Box2D(BBox)
229 
230  # Grow the bounding box to make sure the spherical geometry bbox will contain
231  # the same region, as non-padded boxes may contain different regions because of optical distortion.
232  # Also create an inner region that is sure to be inside the bbox
233  outerLocalBBox.grow(BBoxPadding)
234  innerLocalBBox.grow(-1*BBoxPadding)
235 
236  # Handle the fact that the inner bounding box shrunk to a zero sized region in at least one
237  # dimension, in which case all reference catalogs must be checked fully against the input
238  # bounding box
239  if innerLocalBBox.getDimensions() == geom.Extent2D(0, 0):
240  innerSkyRegion = sphgeom.Box()
241  else:
242  innerBoxCorners = innerLocalBBox.getCorners()
243  innerSphCorners = [wcs.pixelToSky(corner).getVector() for corner in innerBoxCorners]
244 
245  innerSkyRegion = sphgeom.ConvexPolygon(innerSphCorners)
246 
247  # Convert the corners of the box to sky coordinates
248  outerBoxCorners = outerLocalBBox.getCorners()
249  outerSphCorners = [wcs.pixelToSky(corner).getVector() for corner in outerBoxCorners]
250 
251  outerSkyRegion = sphgeom.ConvexPolygon(outerSphCorners)
252 
253  return innerSkyRegion, outerSkyRegion, innerSphCorners, outerSphCorners
254 
255  def loadPixelBox(self, bbox, wcs, filterName=None, epoch=None, photoCalib=None, bboxPadding=100):
256  """Load reference objects that are within a pixel-based rectangular region
257 
258  This algorithm works by creating a spherical box whose corners correspond
259  to the WCS converted corners of the input bounding box (possibly padded).
260  It then defines a filtering function which will look at a reference
261  objects pixel position and accept objects that lie within the specified
262  bounding box.
263 
264  The spherical box region and filtering function are passed to the generic
265  loadRegion method which will load and filter the reference objects from
266  the datastore and return a single catalog containing all reference objects
267 
268  Parameters
269  ----------
270  bbox : `lsst.geom.box2I`
271  Box which bounds a region in pixel space
272  wcs : `lsst.afw.geom.SkyWcs`
273  Wcs object defining the pixel to sky (and inverse) transform for the space
274  of pixels of the supplied bbox
275  filterName : `str`
276  Name of camera filter, or None or blank for the default filter
277  epoch : `astropy.time.Time` (optional)
278  Epoch to which to correct proper motion and parallax,
279  or None to not apply such corrections.
280  photoCalib : None
281  Deprecated and ignored, only included for api compatibility
282  bboxPadding : `int`
283  Number describing how much to pad the input bbox by (in pixels), defaults
284  to 100. This parameter is necessary because optical distortions in telescopes
285  can cause a rectangular pixel grid to map into a non "rectangular" spherical
286  region in sky coordinates. This padding is used to create a spherical
287  "rectangle", which will for sure enclose the input box. This padding is only
288  used to determine if the reference catalog for a sky patch will be loaded from
289  the data store, this function will filter out objects which lie within the
290  padded region but fall outside the input bounding box region.
291 
292  Returns
293  -------
294  referenceCatalog : `lsst.afw.table.SimpleCatalog`
295  Catalog containing reference objects inside the specified bounding box
296 
297  Raises
298  ------
299  `lsst.pex.exception.RuntimeError`
300  Raised if no reference catalogs could be found for the specified region
301 
302  `lsst.pex.exception.TypeError`
303  Raised if the loaded reference catalogs do not have matching schemas
304  """
305  innerSkyRegion, outerSkyRegion, _, _ = self._makeBoxRegion_makeBoxRegion(bbox, wcs, bboxPadding)
306 
307  def _filterFunction(refCat, region):
308  # Add columns to the reference catalog relating to center positions and use afwTable
309  # to populate those columns
310  refCat = self.remapReferenceCatalogSchemaremapReferenceCatalogSchema(refCat, position=True)
311  afwTable.updateRefCentroids(wcs, refCat)
312  # no need to filter the catalog if it is sure that it is entirely contained in the region
313  # defined by given bbox
314  if innerSkyRegion.contains(region):
315  return refCat
316  # Create a new reference catalog, and populate it with records which fall inside the bbox
317  filteredRefCat = type(refCat)(refCat.table)
318  centroidKey = afwTable.Point2DKey(refCat.schema['centroid'])
319  for record in refCat:
320  pixCoords = record[centroidKey]
321  if bbox.contains(geom.Point2I(pixCoords)):
322  filteredRefCat.append(record)
323  return filteredRefCat
324  return self.loadRegionloadRegion(outerSkyRegion, filtFunc=_filterFunction, epoch=epoch, filterName=filterName)
325 
326  def loadRegion(self, region, filtFunc=None, filterName=None, epoch=None):
327  """ Load reference objects within a specified region
328 
329  This function loads the DataIds used to construct an instance of this class
330  which intersect or are contained within the specified region. The reference
331  catalogs which intersect but are not fully contained within the input region are
332  further filtered by the specified filter function. This function will return a
333  single source catalog containing all reference objects inside the specified region.
334 
335  Parameters
336  ----------
337  region : `lsst.sphgeom.Region`
338  This can be any type that is derived from `lsst.sphgeom.region` and should
339  define the spatial region for which reference objects are to be loaded.
340  filtFunc : callable
341  This optional parameter should be a callable object that takes a reference
342  catalog and its corresponding region as parameters, filters the catalog by
343  some criteria and returns the filtered reference catalog. If the value is
344  left as the default (None) than an internal filter function is used which
345  filters according to if a reference object falls within the input region.
346  filterName : `str`
347  Name of camera filter, or None or blank for the default filter
348  epoch : `astropy.time.Time` (optional)
349  Epoch to which to correct proper motion and parallax,
350  or None to not apply such corrections.
351 
352  Returns
353  -------
354  referenceCatalog : `lsst.afw.table.SourceCatalog`
355  Catalog containing reference objects which intersect the input region,
356  filtered by the specified filter function
357 
358  Raises
359  ------
360  `lsst.pex.exception.RuntimeError`
361  Raised if no reference catalogs could be found for the specified region
362 
363  `lsst.pex.exception.TypeError`
364  Raised if the loaded reference catalogs do not have matching schemas
365 
366  """
367  regionBounding = region.getBoundingBox()
368  self.loglog.info("Loading reference objects from region bounded by {}, {} lat lon".format(
369  regionBounding.getLat(), regionBounding.getLon()))
370  if filtFunc is None:
371  filtFunc = _FilterCatalog(region)
372  # filter out all the regions supplied by the constructor that do not overlap
373  overlapList = []
374  for dataId, refCat in zip(self.dataIdsdataIds, self.refCatsrefCats):
375  # SphGeom supports some objects intersecting others, but is not symmetric,
376  # try the intersect operation in both directions
377  try:
378  intersects = dataId.region.intersects(region)
379  except TypeError:
380  intersects = region.intersects(dataId.region)
381 
382  if intersects:
383  overlapList.append((dataId, refCat))
384 
385  if len(overlapList) == 0:
386  raise pexExceptions.RuntimeError("No reference tables could be found for input region")
387 
388  firstCat = overlapList[0][1].get()
389  refCat = filtFunc(firstCat, overlapList[0][0].region)
390  trimmedAmount = len(firstCat) - len(refCat)
391 
392  # Load in the remaining catalogs
393  for dataId, inputRefCat in overlapList[1:]:
394  tmpCat = inputRefCat.get()
395 
396  if tmpCat.schema != firstCat.schema:
397  raise pexExceptions.TypeError("Reference catalogs have mismatching schemas")
398 
399  filteredCat = filtFunc(tmpCat, dataId.region)
400  refCat.extend(filteredCat)
401  trimmedAmount += len(tmpCat) - len(filteredCat)
402 
403  self.loglog.debug(f"Trimmed {trimmedAmount} out of region objects, leaving {len(refCat)}")
404  self.loglog.info(f"Loaded {len(refCat)} reference objects")
405 
406  # Ensure that the loaded reference catalog is continuous in memory
407  if not refCat.isContiguous():
408  refCat = refCat.copy(deep=True)
409 
410  if epoch is not None and "pm_ra" in refCat.schema:
411  # check for a catalog in a non-standard format
412  if isinstance(refCat.schema["pm_ra"].asKey(), lsst.afw.table.KeyAngle):
413  applyProperMotionsImpl(self.loglog, refCat, epoch)
414  else:
415  self.loglog.warn("Catalog pm_ra field is not an Angle; not applying proper motion")
416 
417  # Verify the schema is in the correct units and has the correct version; automatically convert
418  # it with a warning if this is not the case.
419  if not hasNanojanskyFluxUnits(refCat.schema) or not getFormatVersionFromRefCat(refCat) >= 1:
420  self.loglog.warn("Found version 0 reference catalog with old style units in schema.")
421  self.loglog.warn("run `meas_algorithms/bin/convert_refcat_to_nJy.py` to convert fluxes to nJy.")
422  self.loglog.warn("See RFC-575 for more details.")
423  refCat = convertToNanojansky(refCat, self.loglog)
424 
425  expandedCat = self.remapReferenceCatalogSchemaremapReferenceCatalogSchema(refCat, position=True)
426 
427  # Add flux aliases
428  expandedCat = self.addFluxAliasesaddFluxAliases(expandedCat, self.configconfig.defaultFilter, self.configconfig.filterMap)
429 
430  # Ensure that the returned reference catalog is continuous in memory
431  if not expandedCat.isContiguous():
432  expandedCat = expandedCat.copy(deep=True)
433 
434  fluxField = getRefFluxField(schema=expandedCat.schema, filterName=filterName)
435  return pipeBase.Struct(refCat=expandedCat, fluxField=fluxField)
436 
437  def loadSkyCircle(self, ctrCoord, radius, filterName=None, epoch=None):
438  """Load reference objects that lie within a circular region on the sky
439 
440  This method constructs a circular region from an input center and angular radius,
441  loads reference catalogs which are contained in or intersect the circle, and
442  filters reference catalogs which intersect down to objects which lie within
443  the defined circle.
444 
445  Parameters
446  ----------
447  ctrCoord : `lsst.geom.SpherePoint`
448  Point defining the center of the circular region
449  radius : `lsst.geom.Angle`
450  Defines the angular radius of the circular region
451  filterName : `str`
452  Name of camera filter, or None or blank for the default filter
453  epoch : `astropy.time.Time` (optional)
454  Epoch to which to correct proper motion and parallax,
455  or None to not apply such corrections.
456 
457  Returns
458  -------
459  referenceCatalog : `lsst.afw.table.SourceCatalog`
460  Catalog containing reference objects inside the specified bounding box
461 
462  Raises
463  ------
464  `lsst.pex.exception.RuntimeError`
465  Raised if no reference catalogs could be found for the specified region
466 
467  `lsst.pex.exception.TypeError`
468  Raised if the loaded reference catalogs do not have matching schemas
469 
470  """
471  centerVector = ctrCoord.getVector()
472  sphRadius = sphgeom.Angle(radius.asRadians())
473  circularRegion = sphgeom.Circle(centerVector, sphRadius)
474  return self.loadRegionloadRegion(circularRegion, filterName=filterName, epoch=epoch)
475 
476  def joinMatchListWithCatalog(self, matchCat, sourceCat):
477  """Relink an unpersisted match list to sources and reference
478  objects.
479 
480  A match list is persisted and unpersisted as a catalog of IDs
481  produced by afw.table.packMatches(), with match metadata
482  (as returned by the astrometry tasks) in the catalog's metadata
483  attribute. This method converts such a match catalog into a match
484  list, with links to source records and reference object records.
485 
486  Parameters
487  ----------
488  matchCat : `lsst.afw.table.BaseCatalog`
489  Unpersisted packed match list.
490  ``matchCat.table.getMetadata()`` must contain match metadata,
491  as returned by the astrometry tasks.
492  sourceCat : `lsst.afw.table.SourceCatalog`
493  Source catalog. As a side effect, the catalog will be sorted
494  by ID.
495 
496  Returns
497  -------
498  matchList : `lsst.afw.table.ReferenceMatchVector`
499  Match list.
500  """
501  return joinMatchListWithCatalogImpl(self, matchCat, sourceCat)
502 
503  @classmethod
504  def getMetadataBox(cls, bbox, wcs, filterName=None, photoCalib=None, epoch=None, bboxPadding=100):
505  """Return metadata about the load
506 
507  This metadata is used for reloading the catalog (e.g., for
508  reconstituting a normalised match list.)
509 
510  Parameters
511  ----------
512  bbox : `lsst.geom.Box2I`
513  Bounding bos for the pixels
514  wcs : `lsst.afw.geom.SkyWcs
515  WCS object
516  filterName : `str` or None
517  filterName of the camera filter, or None or blank for the default filter
518  photoCalib : None
519  Deprecated, only included for api compatibility
520  epoch : `astropy.time.Time` (optional)
521  Epoch to which to correct proper motion and parallax,
522  or None to not apply such corrections.
523  bboxPadding : `int`
524  Number describing how much to pad the input bbox by (in pixels), defaults
525  to 100. This parameter is necessary because optical distortions in telescopes
526  can cause a rectangular pixel grid to map into a non "rectangular" spherical
527  region in sky coordinates. This padding is used to create a spherical
528  "rectangle", which will for sure enclose the input box. This padding is only
529  used to determine if the reference catalog for a sky patch will be loaded from
530  the data store, this function will filter out objects which lie within the
531  padded region but fall outside the input bounding box region.
532  Returns
533  -------
534  md : `lsst.daf.base.PropertyList`
535  """
536  _, _, innerCorners, outerCorners = cls._makeBoxRegion_makeBoxRegion(bbox, wcs, bboxPadding)
537  md = PropertyList()
538  for box, corners in zip(("INNER", "OUTER"), (innerCorners, outerCorners)):
539  for (name, corner) in zip(("UPPER_LEFT", "UPPER_RIGHT", "LOWER_LEFT", "LOWER_RIGHT"),
540  corners):
541  md.add(f"{box}_{name}_RA", geom.SpherePoint(corner).getRa().asDegrees(), f"{box}_corner")
542  md.add(f"{box}_{name}_DEC", geom.SpherePoint(corner).getDec().asDegrees(), f"{box}_corner")
543  md.add("SMATCHV", 1, 'SourceMatchVector version number')
544  filterName = "UNKNOWN" if filterName is None else str(filterName)
545  md.add('FILTER', filterName, 'filter name for photometric data')
546  md.add('EPOCH', "NONE" if epoch is None else epoch.mjd, 'Epoch (TAI MJD) for catalog')
547  return md
548 
549  @staticmethod
550  def getMetadataCircle(coord, radius, filterName, photoCalib=None, epoch=None):
551  """Return metadata about the load
552 
553  This metadata is used for reloading the catalog (e.g. for reconstituting
554  a normalized match list.)
555 
556  Parameters
557  ----------
558  coord : `lsst.geom.SpherePoint`
559  ICRS center of a circle
560  radius : `lsst.geom.angle`
561  radius of a circle
562  filterName : `str` or None
563  filterName of the camera filter, or None or blank for the default filter
564  photoCalib : None
565  Deprecated, only included for api compatibility
566  epoch : `astropy.time.Time` (optional)
567  Epoch to which to correct proper motion and parallax,
568  or None to not apply such corrections.
569 
570  Returns
571  -------
572  md : `lsst.daf.base.PropertyList`
573  """
574  md = PropertyList()
575  md.add('RA', coord.getRa().asDegrees(), 'field center in degrees')
576  md.add('DEC', coord.getDec().asDegrees(), 'field center in degrees')
577  md.add('RADIUS', radius.asDegrees(), 'field radius in degrees, minimum')
578  md.add('SMATCHV', 1, 'SourceMatchVector version number')
579  filterName = "UNKNOWN" if filterName is None else str(filterName)
580  md.add('FILTER', filterName, 'filter name for photometric data')
581  md.add('EPOCH', "NONE" if epoch is None else epoch.mjd, 'Epoch (TAI MJD) for catalog')
582  return md
583 
584  @staticmethod
585  def addFluxAliases(refCat, defaultFilter, filterReferenceMap):
586  """This function creates a new catalog containing the information of the input refCat
587  as well as added flux columns and aliases between camera and reference flux.
588 
589  Parameters
590  ----------
591  refCat : `lsst.afw.table.SimpleCatalog`
592  Catalog of reference objects
593  defaultFilter : `str`
594  Name of the default reference filter
595  filterReferenceMap : `dict` of `str`
596  Dictionary with keys corresponding to a filter name, and values which
597  correspond to the name of the reference filter.
598 
599  Returns
600  -------
601  refCat : `lsst.afw.table.SimpleCatalog`
602  Reference catalog with columns added to track reference filters
603 
604  Raises
605  ------
606  `RuntimeError`
607  If specified reference filter name is not a filter specifed as a key in the
608  reference filter map.
609  """
610  refCat = ReferenceObjectLoader.remapReferenceCatalogSchema(refCat,
611  filterNameList=filterReferenceMap.keys())
612  aliasMap = refCat.schema.getAliasMap()
613  if filterReferenceMap is None:
614  filterReferenceMap = {}
615  for filterName, refFilterName in itertools.chain([(None, defaultFilter)],
616  filterReferenceMap.items()):
617  if refFilterName:
618  camFluxName = filterName + "_camFlux" if filterName is not None else "camFlux"
619  refFluxName = refFilterName + "_flux"
620  if refFluxName not in refCat.schema:
621  raise RuntimeError("Unknown reference filter %s" % (refFluxName,))
622  aliasMap.set(camFluxName, refFluxName)
623 
624  refFluxErrName = refFluxName + "Err"
625  camFluxErrName = camFluxName + "Err"
626  aliasMap.set(camFluxErrName, refFluxErrName)
627 
628  return refCat
629 
630  @staticmethod
631  def remapReferenceCatalogSchema(refCat, *, filterNameList=None, position=False, photometric=False):
632  """This function takes in a reference catalog and creates a new catalog with additional
633  columns defined the remaining function arguments.
634 
635  Parameters
636  ----------
637  refCat : `lsst.afw.table.SimpleCatalog`
638  Reference catalog to map to new catalog
639 
640  Returns
641  -------
642  expandedCat : `lsst.afw.table.SimpleCatalog`
643  Deep copy of input reference catalog with additional columns added
644  """
645  mapper = afwTable.SchemaMapper(refCat.schema, True)
646  mapper.addMinimalSchema(refCat.schema, True)
647  mapper.editOutputSchema().disconnectAliases()
648  if filterNameList:
649  for filterName in filterNameList:
650  mapper.editOutputSchema().addField(f"{filterName}_flux",
651  type=numpy.float64,
652  doc=f"flux in filter {filterName}",
653  units="Jy"
654  )
655  mapper.editOutputSchema().addField(f"{filterName}_fluxErr",
656  type=numpy.float64,
657  doc=f"flux uncertanty in filter {filterName}",
658  units="Jy"
659  )
660 
661  if position:
662  mapper.editOutputSchema().addField("centroid_x", type=float, doReplace=True)
663  mapper.editOutputSchema().addField("centroid_y", type=float, doReplace=True)
664  mapper.editOutputSchema().addField("hasCentroid", type="Flag", doReplace=True)
665  mapper.editOutputSchema().getAliasMap().set("slot_Centroid", "centroid")
666 
667  if photometric:
668  mapper.editOutputSchema().addField("photometric",
669  type="Flag",
670  doc="set if the object can be used for photometric"
671  "calibration",
672  )
673  mapper.editOutputSchema().addField("resolved",
674  type="Flag",
675  doc="set if the object is spatially resolved"
676  )
677  mapper.editOutputSchema().addField("variable",
678  type="Flag",
679  doc="set if the object has variable brightness"
680  )
681 
682  expandedCat = afwTable.SimpleCatalog(mapper.getOutputSchema())
683  expandedCat.setMetadata(refCat.getMetadata())
684  expandedCat.extend(refCat, mapper=mapper)
685 
686  return expandedCat
687 
688 
689 def getRefFluxField(schema, filterName=None):
690  """Get the name of a flux field from a schema.
691 
692  return the alias of "anyFilterMapsToThis", if present
693  else if filterName is specified:
694  return "*filterName*_camFlux" if present
695  else return "*filterName*_flux" if present (camera filter name
696  matches reference filter name)
697  else throw RuntimeError
698  else:
699  return "camFlux", if present,
700  else throw RuntimeError
701 
702  Parameters
703  ----------
704  schema : `lsst.afw.table.Schema`
705  Reference catalog schema.
706  filterName : `str`, optional
707  Name of camera filter. If not specified, ``defaultFilter`` needs to be
708  set in the refcat loader config.
709 
710  Returns
711  -------
712  fluxFieldName : `str`
713  Name of flux field.
714 
715  Raises
716  ------
717  RuntimeError
718  If an appropriate field is not found.
719  """
720  if not isinstance(schema, afwTable.Schema):
721  raise RuntimeError("schema=%s is not a schema" % (schema,))
722  try:
723  return schema.getAliasMap().get("anyFilterMapsToThis")
724  except LookupError:
725  pass # try the filterMap next
726 
727  if filterName:
728  fluxFieldList = [filterName + "_camFlux", filterName + "_flux"]
729  else:
730  fluxFieldList = ["camFlux"]
731  for fluxField in fluxFieldList:
732  if fluxField in schema:
733  return fluxField
734 
735  raise RuntimeError("Could not find flux field(s) %s" % (", ".join(fluxFieldList)))
736 
737 
738 def getRefFluxKeys(schema, filterName=None):
739  """Return keys for flux and flux error.
740 
741  Parameters
742  ----------
743  schema : `lsst.afw.table.Schema`
744  Reference catalog schema.
745  filterName : `str`
746  Name of camera filter.
747 
748  Returns
749  -------
750  keys : `tuple` of (`lsst.afw.table.Key`, `lsst.afw.table.Key`)
751  Two keys:
752 
753  - flux key
754  - flux error key, if present, else None
755 
756  Raises
757  ------
758  RuntimeError
759  If flux field not found.
760  """
761  fluxField = getRefFluxField(schema, filterName)
762  fluxErrField = fluxField + "Err"
763  fluxKey = schema[fluxField].asKey()
764  try:
765  fluxErrKey = schema[fluxErrField].asKey()
766  except Exception:
767  fluxErrKey = None
768  return (fluxKey, fluxErrKey)
769 
770 
771 class LoadReferenceObjectsConfig(pexConfig.Config):
772  pixelMargin = pexConfig.RangeField(
773  doc="Padding to add to 4 all edges of the bounding box (pixels)",
774  dtype=int,
775  default=300,
776  min=0,
777  )
778  defaultFilter = pexConfig.Field(
779  doc=("Default reference catalog filter to use if filter not specified in exposure;"
780  " if blank then filter must be specified in exposure."),
781  dtype=str,
782  default="",
783  )
784  anyFilterMapsToThis = pexConfig.Field(
785  doc=("Always use this reference catalog filter, no matter whether or what filter name is "
786  "supplied to the loader. Effectively a trivial filterMap: map all filter names to this filter."
787  " This can be set for purely-astrometric catalogs (e.g. Gaia DR2) where there is only one "
788  "reasonable choice for every camera filter->refcat mapping, but not for refcats used for "
789  "photometry, which need a filterMap and/or colorterms/transmission corrections."),
790  dtype=str,
791  default=None,
792  optional=True
793  )
794  filterMap = pexConfig.DictField(
795  doc=("Mapping of camera filter name: reference catalog filter name; "
796  "each reference filter must exist in the refcat."
797  " Note that this does not perform any bandpass corrections: it is just a lookup."),
798  keytype=str,
799  itemtype=str,
800  default={},
801  )
802  requireProperMotion = pexConfig.Field(
803  doc="Require that the fields needed to correct proper motion "
804  "(epoch, pm_ra and pm_dec) are present?",
805  dtype=bool,
806  default=False,
807  )
808 
809  def validate(self):
810  super().validate()
811  if self.filterMapfilterMap != {} and self.anyFilterMapsToThisanyFilterMapsToThis is not None:
812  msg = "`filterMap` and `anyFilterMapsToThis` are mutually exclusive"
813  raise pexConfig.FieldValidationError(LoadReferenceObjectsConfig.anyFilterMapsToThis,
814  self, msg)
815 
816 
817 class LoadReferenceObjectsTask(pipeBase.Task, metaclass=abc.ABCMeta):
818  r"""Abstract base class to load objects from reference catalogs
819  """
820  ConfigClass = LoadReferenceObjectsConfig
821  _DefaultName = "LoadReferenceObjects"
822 
823  def __init__(self, butler=None, *args, **kwargs):
824  """Construct a LoadReferenceObjectsTask
825 
826  Parameters
827  ----------
828  butler : `lsst.daf.persistence.Butler`
829  Data butler, for access reference catalogs.
830  """
831  pipeBase.Task.__init__(self, *args, **kwargs)
832  self.butlerbutler = butler
833 
834  @pipeBase.timeMethod
835  def loadPixelBox(self, bbox, wcs, filterName=None, photoCalib=None, epoch=None):
836  """Load reference objects that overlap a rectangular pixel region.
837 
838  Parameters
839  ----------
840  bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
841  Bounding box for pixels.
842  wcs : `lsst.afw.geom.SkyWcs`
843  WCS; used to convert pixel positions to sky coordinates
844  and vice-versa.
845  filterName : `str`
846  Name of filter, or `None` or `""` for the default filter.
847  This is used for flux values in case we have flux limits
848  (which are not yet implemented).
849  photoCalib : `lsst.afw.image.PhotoCalib` (optional)
850  Calibration, or `None` if unknown.
851  epoch : `astropy.time.Time` (optional)
852  Epoch to which to correct proper motion and parallax,
853  or None to not apply such corrections.
854 
855  Returns
856  -------
857  results : `lsst.pipe.base.Struct`
858  A Struct containing the following fields:
859  refCat : `lsst.afw.catalog.SimpleCatalog`
860  A catalog of reference objects with the standard
861  schema, as documented in the main doc string for
862  `LoadReferenceObjects`.
863  The catalog is guaranteed to be contiguous.
864  fluxField : `str`
865  Name of flux field for specified `filterName`.
866 
867  Notes
868  -----
869  The search algorithm works by searching in a region in sky
870  coordinates whose center is the center of the bbox and radius
871  is large enough to just include all 4 corners of the bbox.
872  Stars that lie outside the bbox are then trimmed from the list.
873  """
874  circle = self._calculateCircle_calculateCircle(bbox, wcs)
875 
876  # find objects in circle
877  self.log.info("Loading reference objects using center %s and radius %s deg" %
878  (circle.coord, circle.radius.asDegrees()))
879  loadRes = self.loadSkyCircleloadSkyCircle(circle.coord, circle.radius, filterName=filterName, epoch=epoch,
880  centroids=True)
881  refCat = loadRes.refCat
882  numFound = len(refCat)
883 
884  # trim objects outside bbox
885  refCat = self._trimToBBox_trimToBBox(refCat=refCat, bbox=circle.bbox, wcs=wcs)
886  numTrimmed = numFound - len(refCat)
887  self.log.debug("trimmed %d out-of-bbox objects, leaving %d", numTrimmed, len(refCat))
888  self.log.info("Loaded %d reference objects", len(refCat))
889 
890  # make sure catalog is contiguous
891  if not refCat.isContiguous():
892  loadRes.refCat = refCat.copy(deep=True)
893 
894  return loadRes
895 
896  @abc.abstractmethod
897  def loadSkyCircle(self, ctrCoord, radius, filterName=None, epoch=None, centroids=False):
898  """Load reference objects that overlap a circular sky region.
899 
900  Parameters
901  ----------
902  ctrCoord : `lsst.geom.SpherePoint`
903  ICRS center of search region.
904  radius : `lsst.geom.Angle`
905  Radius of search region.
906  filterName : `str` (optional)
907  Name of filter, or `None` or `""` for the default filter.
908  This is used for flux values in case we have flux limits
909  (which are not yet implemented).
910  epoch : `astropy.time.Time` (optional)
911  Epoch to which to correct proper motion and parallax,
912  or None to not apply such corrections.
913  centroids : `bool` (optional)
914  Add centroid fields to the loaded Schema. ``loadPixelBox`` expects
915  these fields to exist.
916 
917  Returns
918  -------
919  results : `lsst.pipe.base.Struct`
920  A Struct containing the following fields:
921  refCat : `lsst.afw.catalog.SimpleCatalog`
922  A catalog of reference objects with the standard
923  schema, as documented in the main doc string for
924  `LoadReferenceObjects`.
925  The catalog is guaranteed to be contiguous.
926  fluxField : `str`
927  Name of flux field for specified `filterName`.
928 
929  Notes
930  -----
931  Note that subclasses are responsible for performing the proper motion
932  correction, since this is the lowest-level interface for retrieving
933  the catalog.
934  """
935  return
936 
937  @staticmethod
938  def _trimToBBox(refCat, bbox, wcs):
939  """Remove objects outside a given pixel bounding box and set
940  centroid and hasCentroid fields.
941 
942  Parameters
943  ----------
944  refCat : `lsst.afw.table.SimpleCatalog`
945  A catalog of objects. The schema must include fields
946  "coord", "centroid" and "hasCentroid".
947  The "coord" field is read.
948  The "centroid" and "hasCentroid" fields are set.
949  bbox : `lsst.geom.Box2D`
950  Pixel region
951  wcs : `lsst.afw.geom.SkyWcs`
952  WCS; used to convert sky coordinates to pixel positions.
953 
954  Returns
955  -------
956  catalog : `lsst.afw.table.SimpleCatalog`
957  Reference objects in the bbox, with centroid and
958  hasCentroid fields set.
959  """
960  afwTable.updateRefCentroids(wcs, refCat)
961  centroidKey = afwTable.Point2DKey(refCat.schema["centroid"])
962  retStarCat = type(refCat)(refCat.table)
963  for star in refCat:
964  point = star.get(centroidKey)
965  if bbox.contains(point):
966  retStarCat.append(star)
967  return retStarCat
968 
969  def _addFluxAliases(self, schema):
970  """Add aliases for camera filter fluxes to the schema.
971 
972  If self.config.defaultFilter then adds these aliases:
973  camFlux: <defaultFilter>_flux
974  camFluxErr: <defaultFilter>_fluxErr, if the latter exists
975 
976  For each camFilter: refFilter in self.config.filterMap adds these aliases:
977  <camFilter>_camFlux: <refFilter>_flux
978  <camFilter>_camFluxErr: <refFilter>_fluxErr, if the latter exists
979 
980  Parameters
981  ----------
982  schema : `lsst.afw.table.Schema`
983  Schema for reference catalog.
984 
985  Raises
986  ------
987  RuntimeError
988  If any reference flux field is missing from the schema.
989  """
990  aliasMap = schema.getAliasMap()
991 
992  if self.config.anyFilterMapsToThis is not None:
993  refFluxName = self.config.anyFilterMapsToThis + "_flux"
994  if refFluxName not in schema:
995  msg = f"Unknown reference filter for anyFilterMapsToThis='{refFluxName}'"
996  raise RuntimeError(msg)
997  aliasMap.set("anyFilterMapsToThis", refFluxName)
998  return # this is mutually exclusive with filterMap
999 
1000  def addAliasesForOneFilter(filterName, refFilterName):
1001  """Add aliases for a single filter
1002 
1003  Parameters
1004  ----------
1005  filterName : `str` (optional)
1006  Camera filter name. The resulting alias name is
1007  <filterName>_camFlux, or simply "camFlux" if `filterName`
1008  is `None` or `""`.
1009  refFilterName : `str`
1010  Reference catalog filter name; the field
1011  <refFilterName>_flux must exist.
1012  """
1013  camFluxName = filterName + "_camFlux" if filterName is not None else "camFlux"
1014  refFluxName = refFilterName + "_flux"
1015  if refFluxName not in schema:
1016  raise RuntimeError("Unknown reference filter %s" % (refFluxName,))
1017  aliasMap.set(camFluxName, refFluxName)
1018  refFluxErrName = refFluxName + "Err"
1019  if refFluxErrName in schema:
1020  camFluxErrName = camFluxName + "Err"
1021  aliasMap.set(camFluxErrName, refFluxErrName)
1022 
1023  if self.config.defaultFilter:
1024  addAliasesForOneFilter(None, self.config.defaultFilter)
1025 
1026  for filterName, refFilterName in self.config.filterMap.items():
1027  addAliasesForOneFilter(filterName, refFilterName)
1028 
1029  @staticmethod
1030  def makeMinimalSchema(filterNameList, *, addCentroid=False,
1031  addIsPhotometric=False, addIsResolved=False,
1032  addIsVariable=False, coordErrDim=2,
1033  addProperMotion=False, properMotionErrDim=2,
1034  addParallax=False):
1035  """Make a standard schema for reference object catalogs.
1036 
1037  Parameters
1038  ----------
1039  filterNameList : `list` of `str`
1040  List of filter names. Used to create <filterName>_flux fields.
1041  addIsPhotometric : `bool`
1042  If True then add field "photometric".
1043  addIsResolved : `bool`
1044  If True then add field "resolved".
1045  addIsVariable : `bool`
1046  If True then add field "variable".
1047  coordErrDim : `int`
1048  Number of coord error fields; must be one of 0, 2, 3:
1049 
1050  - If 2 or 3: add fields "coord_raErr" and "coord_decErr".
1051  - If 3: also add field "coord_radecErr".
1052  addProperMotion : `bool`
1053  If True add fields "epoch", "pm_ra", "pm_dec" and "pm_flag".
1054  properMotionErrDim : `int`
1055  Number of proper motion error fields; must be one of 0, 2, 3;
1056  ignored if addProperMotion false:
1057  - If 2 or 3: add fields "pm_raErr" and "pm_decErr".
1058  - If 3: also add field "pm_radecErr".
1059  addParallax : `bool`
1060  If True add fields "epoch", "parallax", "parallaxErr"
1061  and "parallax_flag".
1062 
1063  Returns
1064  -------
1065  schema : `lsst.afw.table.Schema`
1066  Schema for reference catalog, an
1067  `lsst.afw.table.SimpleCatalog`.
1068 
1069  Notes
1070  -----
1071  Reference catalogs support additional covariances, such as
1072  covariance between RA and proper motion in declination,
1073  that are not supported by this method, but can be added after
1074  calling this method.
1075  """
1076  schema = afwTable.SimpleTable.makeMinimalSchema()
1077  if addCentroid:
1078  afwTable.Point2DKey.addFields(
1079  schema,
1080  "centroid",
1081  "centroid on an exposure, if relevant",
1082  "pixel",
1083  )
1084  schema.addField(
1085  field="hasCentroid",
1086  type="Flag",
1087  doc="is position known?",
1088  )
1089  for filterName in filterNameList:
1090  schema.addField(
1091  field="%s_flux" % (filterName,),
1092  type=numpy.float64,
1093  doc="flux in filter %s" % (filterName,),
1094  units="nJy",
1095  )
1096  for filterName in filterNameList:
1097  schema.addField(
1098  field="%s_fluxErr" % (filterName,),
1099  type=numpy.float64,
1100  doc="flux uncertainty in filter %s" % (filterName,),
1101  units="nJy",
1102  )
1103  if addIsPhotometric:
1104  schema.addField(
1105  field="photometric",
1106  type="Flag",
1107  doc="set if the object can be used for photometric calibration",
1108  )
1109  if addIsResolved:
1110  schema.addField(
1111  field="resolved",
1112  type="Flag",
1113  doc="set if the object is spatially resolved",
1114  )
1115  if addIsVariable:
1116  schema.addField(
1117  field="variable",
1118  type="Flag",
1119  doc="set if the object has variable brightness",
1120  )
1121  if coordErrDim not in (0, 2, 3):
1122  raise ValueError("coordErrDim={}; must be (0, 2, 3)".format(coordErrDim))
1123  if coordErrDim > 0:
1124  afwTable.CovarianceMatrix2fKey.addFields(
1125  schema=schema,
1126  prefix="coord",
1127  names=["ra", "dec"],
1128  units=["rad", "rad"],
1129  diagonalOnly=(coordErrDim == 2),
1130  )
1131 
1132  if addProperMotion or addParallax:
1133  schema.addField(
1134  field="epoch",
1135  type=numpy.float64,
1136  doc="date of observation (TAI, MJD)",
1137  units="day",
1138  )
1139 
1140  if addProperMotion:
1141  schema.addField(
1142  field="pm_ra",
1143  type="Angle",
1144  doc="proper motion in the right ascension direction = dra/dt * cos(dec)",
1145  units="rad/year",
1146  )
1147  schema.addField(
1148  field="pm_dec",
1149  type="Angle",
1150  doc="proper motion in the declination direction",
1151  units="rad/year",
1152  )
1153  if properMotionErrDim not in (0, 2, 3):
1154  raise ValueError("properMotionErrDim={}; must be (0, 2, 3)".format(properMotionErrDim))
1155  if properMotionErrDim > 0:
1156  afwTable.CovarianceMatrix2fKey.addFields(
1157  schema=schema,
1158  prefix="pm",
1159  names=["ra", "dec"],
1160  units=["rad/year", "rad/year"],
1161  diagonalOnly=(properMotionErrDim == 2),
1162  )
1163  schema.addField(
1164  field="pm_flag",
1165  type="Flag",
1166  doc="Set if proper motion or proper motion error is bad",
1167  )
1168 
1169  if addParallax:
1170  schema.addField(
1171  field="parallax",
1172  type="Angle",
1173  doc="parallax",
1174  units="rad",
1175  )
1176  schema.addField(
1177  field="parallaxErr",
1178  type="Angle",
1179  doc="uncertainty in parallax",
1180  units="rad",
1181  )
1182  schema.addField(
1183  field="parallax_flag",
1184  type="Flag",
1185  doc="Set if parallax or parallax error is bad",
1186  )
1187  return schema
1188 
1189  def _calculateCircle(self, bbox, wcs):
1190  """Compute on-sky center and radius of search region.
1191 
1192  Parameters
1193  ----------
1194  bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
1195  Pixel bounding box.
1196  wcs : `lsst.afw.geom.SkyWcs`
1197  WCS; used to convert pixel positions to sky coordinates.
1198 
1199  Returns
1200  -------
1201  results : `lsst.pipe.base.Struct`
1202  A Struct containing:
1203 
1204  - coord : `lsst.geom.SpherePoint`
1205  ICRS center of the search region.
1206  - radius : `lsst.geom.Angle`
1207  Radius of the search region.
1208  - bbox : `lsst.geom.Box2D`
1209  Bounding box used to compute the circle.
1210  """
1211  bbox = lsst.geom.Box2D(bbox) # make sure bbox is double and that we have a copy
1212  bbox.grow(self.config.pixelMargin)
1213  coord = wcs.pixelToSky(bbox.getCenter())
1214  radius = max(coord.separation(wcs.pixelToSky(pp)) for pp in bbox.getCorners())
1215  return pipeBase.Struct(coord=coord, radius=radius, bbox=bbox)
1216 
1217  def getMetadataBox(self, bbox, wcs, filterName=None, photoCalib=None, epoch=None):
1218  """Return metadata about the load.
1219 
1220  This metadata is used for reloading the catalog (e.g., for
1221  reconstituting a normalised match list.
1222 
1223  Parameters
1224  ----------
1225  bbox : `lsst.geom.Box2I` or `lsst.geom.Box2D`
1226  Pixel bounding box.
1227  wcs : `lsst.afw.geom.SkyWcs`
1228  WCS; used to convert pixel positions to sky coordinates.
1229  filterName : `str`
1230  Name of camera filter, or `None` or `""` for the default
1231  filter.
1232  photoCalib : `lsst.afw.image.PhotoCalib` (optional)
1233  Calibration, or `None` if unknown.
1234  epoch : `astropy.time.Time` (optional)
1235  Epoch to which to correct proper motion and parallax,
1236  or None to not apply such corrections.
1237 
1238  Returns
1239  -------
1240  metadata : lsst.daf.base.PropertyList
1241  Metadata about the load.
1242  """
1243  circle = self._calculateCircle_calculateCircle(bbox, wcs)
1244  return self.getMetadataCirclegetMetadataCircle(circle.coord, circle.radius, filterName, photoCalib=photoCalib,
1245  epoch=epoch)
1246 
1247  def getMetadataCircle(self, coord, radius, filterName, photoCalib=None, epoch=None):
1248  """Return metadata about the load.
1249 
1250  This metadata is used for reloading the catalog (e.g., for
1251  reconstituting a normalised match list.
1252 
1253  Parameters
1254  ----------
1255  coord : `lsst.geom.SpherePoint`
1256  ICRS center of the search region.
1257  radius : `lsst.geom.Angle`
1258  Radius of the search region.
1259  filterName : `str`
1260  Name of camera filter, or `None` or `""` for the default
1261  filter.
1262  photoCalib : `lsst.afw.image.PhotoCalib` (optional)
1263  Calibration, or `None` if unknown.
1264  epoch : `astropy.time.Time` (optional)
1265  Epoch to which to correct proper motion and parallax,
1266  or None to not apply such corrections.
1267 
1268  Returns
1269  -------
1270  metadata : lsst.daf.base.PropertyList
1271  Metadata about the load
1272  """
1273  md = PropertyList()
1274  md.add('RA', coord.getRa().asDegrees(), 'field center in degrees')
1275  md.add('DEC', coord.getDec().asDegrees(), 'field center in degrees')
1276  md.add('RADIUS', radius.asDegrees(), 'field radius in degrees, minimum')
1277  md.add('SMATCHV', 1, 'SourceMatchVector version number')
1278  filterName = "UNKNOWN" if filterName is None else str(filterName)
1279  md.add('FILTER', filterName, 'filter name for photometric data')
1280  md.add('EPOCH', "NONE" if epoch is None else epoch.mjd, 'Epoch (TAI MJD) for catalog')
1281  return md
1282 
1283  def joinMatchListWithCatalog(self, matchCat, sourceCat):
1284  """Relink an unpersisted match list to sources and reference
1285  objects.
1286 
1287  A match list is persisted and unpersisted as a catalog of IDs
1288  produced by afw.table.packMatches(), with match metadata
1289  (as returned by the astrometry tasks) in the catalog's metadata
1290  attribute. This method converts such a match catalog into a match
1291  list, with links to source records and reference object records.
1292 
1293  Parameters
1294  ----------
1295  matchCat : `lsst.afw.table.BaseCatalog`
1296  Unperisted packed match list.
1297  ``matchCat.table.getMetadata()`` must contain match metadata,
1298  as returned by the astrometry tasks.
1299  sourceCat : `lsst.afw.table.SourceCatalog`
1300  Source catalog. As a side effect, the catalog will be sorted
1301  by ID.
1302 
1303  Returns
1304  -------
1305  matchList : `lsst.afw.table.ReferenceMatchVector`
1306  Match list.
1307  """
1308  return joinMatchListWithCatalogImpl(self, matchCat, sourceCat)
1309 
1310  def applyProperMotions(self, catalog, epoch):
1311  """Apply proper motion correction to a reference catalog.
1312 
1313  Adjust position and position error in the ``catalog``
1314  for proper motion to the specified ``epoch``,
1315  modifying the catalog in place.
1316 
1317  Parameters
1318  ----------
1319  catalog : `lsst.afw.table.SimpleCatalog`
1320  Catalog of positions, containing:
1321 
1322  - Coordinates, retrieved by the table's coordinate key.
1323  - ``coord_raErr`` : Error in Right Ascension (rad).
1324  - ``coord_decErr`` : Error in Declination (rad).
1325  - ``pm_ra`` : Proper motion in Right Ascension (rad/yr,
1326  East positive)
1327  - ``pm_raErr`` : Error in ``pm_ra`` (rad/yr), optional.
1328  - ``pm_dec`` : Proper motion in Declination (rad/yr,
1329  North positive)
1330  - ``pm_decErr`` : Error in ``pm_dec`` (rad/yr), optional.
1331  - ``epoch`` : Mean epoch of object (an astropy.time.Time)
1332  epoch : `astropy.time.Time`
1333  Epoch to which to correct proper motion,
1334  """
1335  if ("epoch" not in catalog.schema or "pm_ra" not in catalog.schema or "pm_dec" not in catalog.schema):
1336  if self.config.requireProperMotion:
1337  raise RuntimeError("Proper motion correction required but not available from catalog")
1338  self.log.warn("Proper motion correction not available from catalog")
1339  return
1340  applyProperMotionsImpl(self.log, catalog, epoch)
1341 
1342 
1343 def joinMatchListWithCatalogImpl(refObjLoader, matchCat, sourceCat):
1344  """Relink an unpersisted match list to sources and reference
1345  objects.
1346 
1347  A match list is persisted and unpersisted as a catalog of IDs
1348  produced by afw.table.packMatches(), with match metadata
1349  (as returned by the astrometry tasks) in the catalog's metadata
1350  attribute. This method converts such a match catalog into a match
1351  list, with links to source records and reference object records.
1352 
1353  Parameters
1354  ----------
1355  refObjLoader
1356  Reference object loader to use in getting reference objects
1357  matchCat : `lsst.afw.table.BaseCatalog`
1358  Unperisted packed match list.
1359  ``matchCat.table.getMetadata()`` must contain match metadata,
1360  as returned by the astrometry tasks.
1361  sourceCat : `lsst.afw.table.SourceCatalog`
1362  Source catalog. As a side effect, the catalog will be sorted
1363  by ID.
1364 
1365  Returns
1366  -------
1367  matchList : `lsst.afw.table.ReferenceMatchVector`
1368  Match list.
1369  """
1370  matchmeta = matchCat.table.getMetadata()
1371  version = matchmeta.getInt('SMATCHV')
1372  if version != 1:
1373  raise ValueError('SourceMatchVector version number is %i, not 1.' % version)
1374  filterName = matchmeta.getString('FILTER').strip()
1375  try:
1376  epoch = matchmeta.getDouble('EPOCH')
1377  except (pexExcept.NotFoundError, pexExcept.TypeError):
1378  epoch = None # Not present, or not correct type means it's not set
1379  if 'RADIUS' in matchmeta:
1380  # This is a circle style metadata, call loadSkyCircle
1381  ctrCoord = lsst.geom.SpherePoint(matchmeta.getDouble('RA'),
1382  matchmeta.getDouble('DEC'), lsst.geom.degrees)
1383  rad = matchmeta.getDouble('RADIUS') * lsst.geom.degrees
1384  refCat = refObjLoader.loadSkyCircle(ctrCoord, rad, filterName, epoch=epoch).refCat
1385  elif "INNER_UPPER_LEFT_RA" in matchmeta:
1386  # This is the sky box type (only triggers in the LoadReferenceObject class, not task)
1387  # Only the outer box is required to be loaded to get the maximum region, all filtering
1388  # will be done by the unpackMatches function, and no spatial filtering needs to be done
1389  # by the refObjLoader
1390  box = []
1391  for place in ("UPPER_LEFT", "UPPER_RIGHT", "LOWER_LEFT", "LOWER_RIGHT"):
1392  coord = lsst.geom.SpherePoint(matchmeta.getDouble(f"OUTER_{place}_RA"),
1393  matchmeta.getDouble(f"OUTER_{place}_DEC"),
1394  lsst.geom.degrees).getVector()
1395  box.append(coord)
1396  outerBox = sphgeom.ConvexPolygon(box)
1397  refCat = refObjLoader.loadRegion(outerBox, filterName=filterName, epoch=epoch).refCat
1398 
1399  refCat.sort()
1400  sourceCat.sort()
1401  return afwTable.unpackMatches(matchCat, refCat, sourceCat)
1402 
1403 
1404 def applyProperMotionsImpl(log, catalog, epoch):
1405  """Apply proper motion correction to a reference catalog.
1406 
1407  Adjust position and position error in the ``catalog``
1408  for proper motion to the specified ``epoch``,
1409  modifying the catalog in place.
1410 
1411  Parameters
1412  ----------
1413  log : `lsst.log.Log`
1414  Log object to write to.
1415  catalog : `lsst.afw.table.SimpleCatalog`
1416  Catalog of positions, containing:
1417 
1418  - Coordinates, retrieved by the table's coordinate key.
1419  - ``coord_raErr`` : Error in Right Ascension (rad).
1420  - ``coord_decErr`` : Error in Declination (rad).
1421  - ``pm_ra`` : Proper motion in Right Ascension (rad/yr,
1422  East positive)
1423  - ``pm_raErr`` : Error in ``pm_ra`` (rad/yr), optional.
1424  - ``pm_dec`` : Proper motion in Declination (rad/yr,
1425  North positive)
1426  - ``pm_decErr`` : Error in ``pm_dec`` (rad/yr), optional.
1427  - ``epoch`` : Mean epoch of object (an astropy.time.Time)
1428  epoch : `astropy.time.Time`
1429  Epoch to which to correct proper motion.
1430  """
1431  if "epoch" not in catalog.schema or "pm_ra" not in catalog.schema or "pm_dec" not in catalog.schema:
1432  log.warn("Proper motion correction not available from catalog")
1433  return
1434  if not catalog.isContiguous():
1435  raise RuntimeError("Catalog must be contiguous")
1436  catEpoch = astropy.time.Time(catalog["epoch"], scale="tai", format="mjd")
1437  log.info("Correcting reference catalog for proper motion to %r", epoch)
1438  # Use `epoch.tai` to make sure the time difference is in TAI
1439  timeDiffsYears = (epoch.tai - catEpoch).to(astropy.units.yr).value
1440  coordKey = catalog.table.getCoordKey()
1441  # Compute the offset of each object due to proper motion
1442  # as components of the arc of a great circle along RA and Dec
1443  pmRaRad = catalog["pm_ra"]
1444  pmDecRad = catalog["pm_dec"]
1445  offsetsRaRad = pmRaRad*timeDiffsYears
1446  offsetsDecRad = pmDecRad*timeDiffsYears
1447  # Compute the corresponding bearing and arc length of each offset
1448  # due to proper motion, and apply the offset
1449  # The factor of 1e6 for computing bearing is intended as
1450  # a reasonable scale for typical values of proper motion
1451  # in order to avoid large errors for small values of proper motion;
1452  # using the offsets is another option, but it can give
1453  # needlessly large errors for short duration
1454  offsetBearingsRad = numpy.arctan2(pmDecRad*1e6, pmRaRad*1e6)
1455  offsetAmountsRad = numpy.hypot(offsetsRaRad, offsetsDecRad)
1456  for record, bearingRad, amountRad in zip(catalog, offsetBearingsRad, offsetAmountsRad):
1457  record.set(coordKey,
1458  record.get(coordKey).offset(bearing=bearingRad*lsst.geom.radians,
1459  amount=amountRad*lsst.geom.radians))
1460  # Increase error in RA and Dec based on error in proper motion
1461  if "coord_raErr" in catalog.schema:
1462  catalog["coord_raErr"] = numpy.hypot(catalog["coord_raErr"],
1463  catalog["pm_raErr"]*timeDiffsYears)
1464  if "coord_decErr" in catalog.schema:
1465  catalog["coord_decErr"] = numpy.hypot(catalog["coord_decErr"],
1466  catalog["pm_decErr"]*timeDiffsYears)
static Schema makeMinimalSchema()
static Log getDefaultLogger()
def makeMinimalSchema(filterNameList, *addCentroid=False, addIsPhotometric=False, addIsResolved=False, addIsVariable=False, coordErrDim=2, addProperMotion=False, properMotionErrDim=2, addParallax=False)
def getMetadataCircle(self, coord, radius, filterName, photoCalib=None, epoch=None)
def loadPixelBox(self, bbox, wcs, filterName=None, photoCalib=None, epoch=None)
def loadSkyCircle(self, ctrCoord, radius, filterName=None, epoch=None, centroids=False)
def getMetadataBox(self, bbox, wcs, filterName=None, photoCalib=None, epoch=None)
def loadRegion(self, region, filtFunc=None, filterName=None, epoch=None)
def getMetadataCircle(coord, radius, filterName, photoCalib=None, epoch=None)
def remapReferenceCatalogSchema(refCat, *filterNameList=None, position=False, photometric=False)
def getMetadataBox(cls, bbox, wcs, filterName=None, photoCalib=None, epoch=None, bboxPadding=100)
def addFluxAliases(refCat, defaultFilter, filterReferenceMap)
def loadSkyCircle(self, ctrCoord, radius, filterName=None, epoch=None)
def loadPixelBox(self, bbox, wcs, filterName=None, epoch=None, photoCalib=None, bboxPadding=100)
def convertToNanojansky(catalog, log, doConvert=True)
def joinMatchListWithCatalogImpl(refObjLoader, matchCat, sourceCat)