lsst.obs.base  20.0.0-36-g2de6156+993169450c
_instrument.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 from __future__ import annotations
23 
24 __all__ = ("Instrument", "makeExposureRecordFromObsInfo", "addUnboundedCalibrationLabel", "loadCamera")
25 
26 import os.path
27 from abc import ABCMeta, abstractmethod
28 from typing import Any, Tuple, TYPE_CHECKING
29 import astropy.time
30 
31 from lsst.afw.cameraGeom import Camera
32 from lsst.daf.butler import (
33  Butler,
34  CollectionType,
35  DataCoordinate,
36  DataId,
37  DatasetType,
38  Timespan,
39 )
40 from lsst.utils import getPackageDir, doImport
41 
42 if TYPE_CHECKING:
43  from .gen2to3 import TranslatorFactory
44  from lsst.daf.butler import Registry
45 
46 # To be a standard text curated calibration means that we use a
47 # standard definition for the corresponding DatasetType.
48 StandardCuratedCalibrationDatasetTypes = {
49  "defects": {"dimensions": ("instrument", "detector", "calibration_label"),
50  "storageClass": "Defects"},
51  "qe_curve": {"dimensions": ("instrument", "detector", "calibration_label"),
52  "storageClass": "QECurve"},
53  "crosstalk": {"dimensions": ("instrument", "detector", "calibration_label"),
54  "storageClass": "CrosstalkCalib"},
55 }
56 
57 
58 class Instrument(metaclass=ABCMeta):
59  """Base class for instrument-specific logic for the Gen3 Butler.
60 
61  Concrete instrument subclasses should be directly constructable with no
62  arguments.
63  """
64 
65  configPaths = ()
66  """Paths to config files to read for specific Tasks.
67 
68  The paths in this list should contain files of the form `task.py`, for
69  each of the Tasks that requires special configuration.
70  """
71 
72  policyName = None
73  """Instrument specific name to use when locating a policy or configuration
74  file in the file system."""
75 
76  obsDataPackage = None
77  """Name of the package containing the text curated calibration files.
78  Usually a obs _data package. If `None` no curated calibration files
79  will be read. (`str`)"""
80 
81  standardCuratedDatasetTypes = tuple(StandardCuratedCalibrationDatasetTypes)
82  """The dataset types expected to be obtained from the obsDataPackage.
83  These dataset types are all required to have standard definitions and
84  must be known to the base class. Clearing this list will prevent
85  any of these calibrations from being stored. If a dataset type is not
86  known to a specific instrument it can still be included in this list
87  since the data package is the source of truth.
88  """
89 
90  @property
91  @abstractmethod
92  def filterDefinitions(self):
93  """`~lsst.obs.base.FilterDefinitionCollection`, defining the filters
94  for this instrument.
95  """
96  return None
97 
98  def __init__(self):
99  self.filterDefinitions.reset()
100  self.filterDefinitions.defineFilters()
101  self._obsDataPackageDir = None
102 
103  @classmethod
104  @abstractmethod
105  def getName(cls):
106  """Return the short (dimension) name for this instrument.
107 
108  This is not (in general) the same as the class name - it's what is used
109  as the value of the "instrument" field in data IDs, and is usually an
110  abbreviation of the full name.
111  """
112  raise NotImplementedError()
113 
114  @abstractmethod
115  def getCamera(self):
116  """Retrieve the cameraGeom representation of this instrument.
117 
118  This is a temporary API that should go away once ``obs_`` packages have
119  a standardized approach to writing versioned cameras to a Gen3 repo.
120  """
121  raise NotImplementedError()
122 
123  @abstractmethod
124  def register(self, registry):
125  """Insert instrument, physical_filter, and detector entries into a
126  `Registry`.
127  """
128  raise NotImplementedError()
129 
130  @property
131  def obsDataPackageDir(self):
132  """The root of the obs package that provides specializations for
133  this instrument (`str`).
134  """
135  if self.obsDataPackage is None:
136  return None
137  if self._obsDataPackageDir is None:
138  # Defer any problems with locating the package until
139  # we need to find it.
140  self._obsDataPackageDir = getPackageDir(self.obsDataPackage)
141  return self._obsDataPackageDir
142 
143  @staticmethod
144  def fromName(name: str, registry: Registry) -> Instrument:
145  """Given an instrument name and a butler, retrieve a corresponding
146  instantiated instrument object.
147 
148  Parameters
149  ----------
150  name : `str`
151  Name of the instrument (must match the return value of `getName`).
152  registry : `lsst.daf.butler.Registry`
153  Butler registry to query to find the information.
154 
155  Returns
156  -------
157  instrument : `Instrument`
158  An instance of the relevant `Instrument`.
159 
160  Notes
161  -----
162  The instrument must be registered in the corresponding butler.
163 
164  Raises
165  ------
166  LookupError
167  Raised if the instrument is not known to the supplied registry.
168  ModuleNotFoundError
169  Raised if the class could not be imported. This could mean
170  that the relevant obs package has not been setup.
171  TypeError
172  Raised if the class name retrieved is not a string.
173  """
174  records = list(registry.queryDimensionRecords("instrument", instrument=name))
175  if not records:
176  raise LookupError(f"No registered instrument with name '{name}'.")
177  cls = records[0].class_name
178  if not isinstance(cls, str):
179  raise TypeError(f"Unexpected class name retrieved from {name} instrument dimension (got {cls})")
180  instrument = doImport(cls)
181  return instrument()
182 
183  @staticmethod
184  def importAll(registry: Registry) -> None:
185  """Import all the instruments known to this registry.
186 
187  This will ensure that all metadata translators have been registered.
188 
189  Parameters
190  ----------
191  registry : `lsst.daf.butler.Registry`
192  Butler registry to query to find the information.
193 
194  Notes
195  -----
196  It is allowed for a particular instrument class to fail on import.
197  This might simply indicate that a particular obs package has
198  not been setup.
199  """
200  records = list(registry.queryDimensionRecords("instrument"))
201  for record in records:
202  cls = record.class_name
203  try:
204  doImport(cls)
205  except Exception:
206  pass
207 
208  def _registerFilters(self, registry):
209  """Register the physical and abstract filter Dimension relationships.
210  This should be called in the ``register`` implementation.
211 
212  Parameters
213  ----------
214  registry : `lsst.daf.butler.core.Registry`
215  The registry to add dimensions to.
216  """
217  for filter in self.filterDefinitions:
218  # fix for undefined abstract filters causing trouble in the registry:
219  if filter.abstract_filter is None:
220  abstract_filter = filter.physical_filter
221  else:
222  abstract_filter = filter.abstract_filter
223 
224  registry.insertDimensionData("physical_filter",
225  {"instrument": self.getName(),
226  "name": filter.physical_filter,
227  "abstract_filter": abstract_filter
228  })
229 
230  @abstractmethod
231  def getRawFormatter(self, dataId):
232  """Return the Formatter class that should be used to read a particular
233  raw file.
234 
235  Parameters
236  ----------
237  dataId : `DataCoordinate`
238  Dimension-based ID for the raw file or files being ingested.
239 
240  Returns
241  -------
242  formatter : `Formatter` class
243  Class to be used that reads the file into an
244  `lsst.afw.image.Exposure` instance.
245  """
246  raise NotImplementedError()
247 
248  def writeCuratedCalibrations(self, butler, run=None):
249  """Write human-curated calibration Datasets to the given Butler with
250  the appropriate validity ranges.
251 
252  Parameters
253  ----------
254  butler : `lsst.daf.butler.Butler`
255  Butler to use to store these calibrations.
256  run : `str`
257  Run to use for this collection of calibrations. If `None` the
258  collection name is worked out automatically from the instrument
259  name and other metadata.
260 
261  Notes
262  -----
263  Expected to be called from subclasses. The base method calls
264  ``writeCameraGeom`` and ``writeStandardTextCuratedCalibrations``.
265  """
266  # Need to determine the run for ingestion based on the instrument
267  # name and eventually the data package version. The camera geom
268  # is currently special in that it is not in the _data package.
269  if run is None:
270  run = self.makeCollectionName("calib")
271  butler.registry.registerCollection(run, type=CollectionType.RUN)
272  self.writeCameraGeom(butler, run=run)
273  self.writeStandardTextCuratedCalibrations(butler, run=run)
274  self.writeAdditionalCuratedCalibrations(butler, run=run)
275 
276  def writeAdditionalCuratedCalibrations(self, butler, run=None):
277  """Write additional curated calibrations that might be instrument
278  specific and are not part of the standard set.
279 
280  Default implementation does nothing.
281 
282  Parameters
283  ----------
284  butler : `lsst.daf.butler.Butler`
285  Butler to use to store these calibrations.
286  run : `str`, optional
287  Name of the run to use to override the default run associated
288  with this Butler.
289  """
290  return
291 
292  def applyConfigOverrides(self, name, config):
293  """Apply instrument-specific overrides for a task config.
294 
295  Parameters
296  ----------
297  name : `str`
298  Name of the object being configured; typically the _DefaultName
299  of a Task.
300  config : `lsst.pex.config.Config`
301  Config instance to which overrides should be applied.
302  """
303  for root in self.configPaths:
304  path = os.path.join(root, f"{name}.py")
305  if os.path.exists(path):
306  config.load(path)
307 
308  def writeCameraGeom(self, butler, run=None):
309  """Write the default camera geometry to the butler repository
310  with an infinite validity range.
311 
312  Parameters
313  ----------
314  butler : `lsst.daf.butler.Butler`
315  Butler to receive these calibration datasets.
316  run : `str`, optional
317  Name of the run to use to override the default run associated
318  with this Butler.
319  """
320 
321  datasetType = DatasetType("camera", ("instrument", "calibration_label"), "Camera",
322  universe=butler.registry.dimensions)
323  butler.registry.registerDatasetType(datasetType)
324  unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName())
325  camera = self.getCamera()
326  butler.put(camera, datasetType, unboundedDataId, run=run)
327 
328  def writeStandardTextCuratedCalibrations(self, butler, run=None):
329  """Write the set of standardized curated text calibrations to
330  the repository.
331 
332  Parameters
333  ----------
334  butler : `lsst.daf.butler.Butler`
335  Butler to receive these calibration datasets.
336  run : `str`, optional
337  Name of the run to use to override the default run associated
338  with this Butler.
339  """
340 
341  for datasetTypeName in self.standardCuratedDatasetTypes:
342  # We need to define the dataset types.
343  if datasetTypeName not in StandardCuratedCalibrationDatasetTypes:
344  raise ValueError(f"DatasetType {datasetTypeName} not in understood list"
345  f" [{'.'.join(StandardCuratedCalibrationDatasetTypes)}]")
346  definition = StandardCuratedCalibrationDatasetTypes[datasetTypeName]
347  datasetType = DatasetType(datasetTypeName,
348  universe=butler.registry.dimensions,
349  **definition)
350  self._writeSpecificCuratedCalibrationDatasets(butler, datasetType, run=run)
351 
352  def _writeSpecificCuratedCalibrationDatasets(self, butler, datasetType, run=None):
353  """Write standardized curated calibration datasets for this specific
354  dataset type from an obs data package.
355 
356  Parameters
357  ----------
358  butler : `lsst.daf.butler.Butler`
359  Gen3 butler in which to put the calibrations.
360  datasetType : `lsst.daf.butler.DatasetType`
361  Dataset type to be put.
362  run : `str`, optional
363  Name of the run to use to override the default run associated
364  with this Butler.
365 
366  Notes
367  -----
368  This method scans the location defined in the ``obsDataPackageDir``
369  class attribute for curated calibrations corresponding to the
370  supplied dataset type. The directory name in the data package must
371  match the name of the dataset type. They are assumed to use the
372  standard layout and can be read by
373  `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard
374  metadata.
375  """
376  if self.obsDataPackageDir is None:
377  # if there is no data package then there can't be datasets
378  return
379 
380  calibPath = os.path.join(self.obsDataPackageDir, self.policyName,
381  datasetType.name)
382 
383  if not os.path.exists(calibPath):
384  return
385 
386  # Register the dataset type
387  butler.registry.registerDatasetType(datasetType)
388 
389  # obs_base can't depend on pipe_tasks but concrete obs packages
390  # can -- we therefore have to defer import
391  from lsst.pipe.tasks.read_curated_calibs import read_all
392 
393  camera = self.getCamera()
394  calibsDict = read_all(calibPath, camera)[0] # second return is calib type
395  dimensionRecords = []
396  datasetRecords = []
397  for det in calibsDict:
398  times = sorted([k for k in calibsDict[det]])
399  calibs = [calibsDict[det][time] for time in times]
400  times = [astropy.time.Time(t, format="datetime", scale="utc") for t in times]
401  times += [None]
402  for calib, beginTime, endTime in zip(calibs, times[:-1], times[1:]):
403  md = calib.getMetadata()
404  calibrationLabel = f"{datasetType.name}/{md['CALIBDATE']}/{md['DETECTOR']}"
405  dataId = DataCoordinate.standardize(
406  universe=butler.registry.dimensions,
407  instrument=self.getName(),
408  calibration_label=calibrationLabel,
409  detector=md["DETECTOR"],
410  )
411  datasetRecords.append((calib, dataId))
412  dimensionRecords.append({
413  "instrument": self.getName(),
414  "name": calibrationLabel,
415  "timespan": Timespan(beginTime, endTime),
416  })
417 
418  # Second loop actually does the inserts and filesystem writes.
419  with butler.transaction():
420  butler.registry.insertDimensionData("calibration_label", *dimensionRecords)
421  # TODO: vectorize these puts, once butler APIs for that become
422  # available.
423  for calib, dataId in datasetRecords:
424  butler.put(calib, datasetType, dataId, run=run)
425 
426  @abstractmethod
427  def makeDataIdTranslatorFactory(self) -> TranslatorFactory:
428  """Return a factory for creating Gen2->Gen3 data ID translators,
429  specialized for this instrument.
430 
431  Derived class implementations should generally call
432  `TranslatorFactory.addGenericInstrumentRules` with appropriate
433  arguments, but are not required to (and may not be able to if their
434  Gen2 raw data IDs are sufficiently different from the HSC/DECam/CFHT
435  norm).
436 
437  Returns
438  -------
439  factory : `TranslatorFactory`.
440  Factory for `Translator` objects.
441  """
442  raise NotImplementedError("Must be implemented by derived classes.")
443 
444  @classmethod
446  """Make the default instrument-specific run collection string for raw
447  data ingest.
448 
449  Returns
450  -------
451  coll : `str`
452  Run collection name to be used as the default for ingestion of
453  raws.
454  """
455  return cls.makeCollectionName("raw/all")
456 
457  @classmethod
458  def makeCollectionName(cls, label: str) -> str:
459  """Get the instrument-specific collection string to use as derived
460  from the supplied label.
461 
462  Parameters
463  ----------
464  label : `str`
465  String to be combined with the instrument name to form a
466  collection name.
467 
468  Returns
469  -------
470  name : `str`
471  Collection name to use that includes the instrument name.
472  """
473  return f"{cls.getName()}/{label}"
474 
475 
476 def makeExposureRecordFromObsInfo(obsInfo, universe):
477  """Construct an exposure DimensionRecord from
478  `astro_metadata_translator.ObservationInfo`.
479 
480  Parameters
481  ----------
482  obsInfo : `astro_metadata_translator.ObservationInfo`
483  A `~astro_metadata_translator.ObservationInfo` object corresponding to
484  the exposure.
485  universe : `DimensionUniverse`
486  Set of all known dimensions.
487 
488  Returns
489  -------
490  record : `DimensionRecord`
491  A record containing exposure metadata, suitable for insertion into
492  a `Registry`.
493  """
494  dimension = universe["exposure"]
495 
496  ra, dec, sky_angle, zenith_angle = (None, None, None, None)
497  if obsInfo.tracking_radec is not None:
498  icrs = obsInfo.tracking_radec.icrs
499  ra = icrs.ra.degree
500  dec = icrs.dec.degree
501  if obsInfo.boresight_rotation_coord == "sky":
502  sky_angle = obsInfo.boresight_rotation_angle.degree
503  if obsInfo.altaz_begin is not None:
504  zenith_angle = obsInfo.altaz_begin.zen.degree
505 
506  return dimension.RecordClass(
507  instrument=obsInfo.instrument,
508  id=obsInfo.exposure_id,
509  name=obsInfo.observation_id,
510  group_name=obsInfo.exposure_group,
511  group_id=obsInfo.visit_id,
512  datetime_begin=obsInfo.datetime_begin,
513  datetime_end=obsInfo.datetime_end,
514  exposure_time=obsInfo.exposure_time.to_value("s"),
515  dark_time=obsInfo.dark_time.to_value("s"),
516  observation_type=obsInfo.observation_type,
517  physical_filter=obsInfo.physical_filter,
518  science_program=obsInfo.science_program,
519  target_name=obsInfo.object,
520  tracking_ra=ra,
521  tracking_dec=dec,
522  sky_angle=sky_angle,
523  zenith_angle=zenith_angle,
524  )
525 
526 
527 def addUnboundedCalibrationLabel(registry, instrumentName):
528  """Add a special 'unbounded' calibration_label dimension entry for the
529  given camera that is valid for any exposure.
530 
531  If such an entry already exists, this function just returns a `DataId`
532  for the existing entry.
533 
534  Parameters
535  ----------
536  registry : `Registry`
537  Registry object in which to insert the dimension entry.
538  instrumentName : `str`
539  Name of the instrument this calibration label is associated with.
540 
541  Returns
542  -------
543  dataId : `DataId`
544  New or existing data ID for the unbounded calibration.
545  """
546  d = dict(instrument=instrumentName, calibration_label="unbounded")
547  try:
548  return registry.expandDataId(d)
549  except LookupError:
550  pass
551  entry = d.copy()
552  entry["timespan"] = Timespan(None, None)
553  registry.insertDimensionData("calibration_label", entry)
554  return registry.expandDataId(d)
555 
556 
557 def loadCamera(butler: Butler, dataId: DataId, *, collections: Any = None) -> Tuple[Camera, bool]:
558  """Attempt to load versioned camera geometry from a butler, but fall back
559  to obtaining a nominal camera from the `Instrument` class if that fails.
560 
561  Parameters
562  ----------
563  butler : `lsst.daf.butler.Butler`
564  Butler instance to attempt to query for and load a ``camera`` dataset
565  from.
566  dataId : `dict` or `DataCoordinate`
567  Data ID that identifies at least the ``instrument`` and ``exposure``
568  dimensions.
569  collections : Any, optional
570  Collections to be searched, overriding ``self.butler.collections``.
571  Can be any of the types supported by the ``collections`` argument
572  to butler construction.
573 
574  Returns
575  -------
576  camera : `lsst.afw.cameraGeom.Camera`
577  Camera object.
578  versioned : `bool`
579  If `True`, the camera was obtained from the butler and should represent
580  a versioned camera from a calibration repository. If `False`, no
581  camera datasets were found, and the returned camera was produced by
582  instantiating the appropriate `Instrument` class and calling
583  `Instrument.getCamera`.
584  """
585  if collections is None:
586  collections = butler.collections
587  # Registry would do data ID expansion internally if we didn't do it first,
588  # but we might want an expanded data ID ourselves later, so we do it here
589  # to ensure it only happens once.
590  # This will also catch problems with the data ID not having keys we need.
591  dataId = butler.registry.expandDataId(dataId, graph=butler.registry.dimensions["exposure"].graph)
592  cameraRefs = list(butler.registry.queryDatasets("camera", dataId=dataId, collections=collections,
593  deduplicate=True))
594  if cameraRefs:
595  assert len(cameraRefs) == 1, "Should be guaranteed by deduplicate=True above."
596  return butler.getDirect(cameraRefs[0]), True
597  instrument = Instrument.fromName(dataId["instrument"], butler.registry)
598  return instrument.getCamera(), False
lsst.obs.base._instrument.Instrument.applyConfigOverrides
def applyConfigOverrides(self, name, config)
Definition: _instrument.py:292
lsst.obs.base._instrument.loadCamera
Tuple[Camera, bool] loadCamera(Butler butler, DataId dataId, *Any collections=None)
Definition: _instrument.py:557
lsst.obs.base._instrument.Instrument.getName
def getName(cls)
Definition: _instrument.py:105
lsst.obs.base._instrument.Instrument.obsDataPackage
obsDataPackage
Definition: _instrument.py:76
lsst.obs.base._instrument.Instrument.importAll
None importAll(Registry registry)
Definition: _instrument.py:184
lsst.obs.base._instrument.Instrument._writeSpecificCuratedCalibrationDatasets
def _writeSpecificCuratedCalibrationDatasets(self, butler, datasetType, run=None)
Definition: _instrument.py:352
lsst.obs.base._instrument.Instrument.getCamera
def getCamera(self)
Definition: _instrument.py:115
lsst.obs.base._instrument.Instrument.filterDefinitions
def filterDefinitions(self)
Definition: _instrument.py:92
lsst.obs.base._instrument.Instrument.fromName
Instrument fromName(str name, Registry registry)
Definition: _instrument.py:144
lsst.obs.base._instrument.Instrument.policyName
policyName
Definition: _instrument.py:72
lsst.obs.base._instrument.Instrument.makeCollectionName
str makeCollectionName(cls, str label)
Definition: _instrument.py:458
lsst::utils
lsst.obs.base._instrument.Instrument
Definition: _instrument.py:58
lsst.obs.base._instrument.Instrument.writeAdditionalCuratedCalibrations
def writeAdditionalCuratedCalibrations(self, butler, run=None)
Definition: _instrument.py:276
lsst.obs.base._instrument.Instrument.writeCuratedCalibrations
def writeCuratedCalibrations(self, butler, run=None)
Definition: _instrument.py:248
lsst.obs.base._instrument.addUnboundedCalibrationLabel
def addUnboundedCalibrationLabel(registry, instrumentName)
Definition: _instrument.py:527
lsst.obs.base._instrument.makeExposureRecordFromObsInfo
def makeExposureRecordFromObsInfo(obsInfo, universe)
Definition: _instrument.py:476
lsst.obs.base._instrument.Instrument.register
def register(self, registry)
Definition: _instrument.py:124
lsst.obs.base._instrument.Instrument.writeCameraGeom
def writeCameraGeom(self, butler, run=None)
Definition: _instrument.py:308
lsst.obs.base._instrument.Instrument.getRawFormatter
def getRawFormatter(self, dataId)
Definition: _instrument.py:231
lsst.obs.base._instrument.Instrument._obsDataPackageDir
_obsDataPackageDir
Definition: _instrument.py:101
lsst.obs.base._instrument.Instrument.standardCuratedDatasetTypes
standardCuratedDatasetTypes
Definition: _instrument.py:81
lsst.obs.base._instrument.Instrument.makeDataIdTranslatorFactory
TranslatorFactory makeDataIdTranslatorFactory(self)
Definition: _instrument.py:427
lsst.obs.base._instrument.Instrument.makeDefaultRawIngestRunName
str makeDefaultRawIngestRunName(cls)
Definition: _instrument.py:445
lsst.obs.base._instrument.Instrument.writeStandardTextCuratedCalibrations
def writeStandardTextCuratedCalibrations(self, butler, run=None)
Definition: _instrument.py:328
lsst.obs.base._instrument.Instrument.__init__
def __init__(self)
Definition: _instrument.py:98
lsst.obs.base._instrument.Instrument.obsDataPackageDir
def obsDataPackageDir(self)
Definition: _instrument.py:131