lsst.obs.base  20.0.0-22-g8ce909b
_instrument.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 from __future__ import annotations
23 
24 __all__ = ("Instrument", "makeExposureRecordFromObsInfo", "addUnboundedCalibrationLabel", "loadCamera")
25 
26 import os.path
27 from abc import ABCMeta, abstractmethod
28 from typing import Any, Tuple, TYPE_CHECKING
29 import astropy.time
30 
31 from lsst.afw.cameraGeom import Camera
32 from lsst.daf.butler import (
33  Butler,
34  CollectionType,
35  DataCoordinate,
36  DataId,
37  DatasetType,
38  TIMESPAN_MIN,
39  TIMESPAN_MAX,
40 )
41 from lsst.utils import getPackageDir, doImport
42 
43 if TYPE_CHECKING:
44  from .gen2to3 import TranslatorFactory
45  from lsst.daf.butler import Registry
46 
47 # To be a standard text curated calibration means that we use a
48 # standard definition for the corresponding DatasetType.
49 StandardCuratedCalibrationDatasetTypes = {
50  "defects": {"dimensions": ("instrument", "detector", "calibration_label"),
51  "storageClass": "Defects"},
52  "qe_curve": {"dimensions": ("instrument", "detector", "calibration_label"),
53  "storageClass": "QECurve"},
54  "crosstalk": {"dimensions": ("instrument", "detector", "calibration_label"),
55  "storageClass": "CrosstalkCalib"},
56 }
57 
58 
59 class Instrument(metaclass=ABCMeta):
60  """Base class for instrument-specific logic for the Gen3 Butler.
61 
62  Concrete instrument subclasses should be directly constructable with no
63  arguments.
64  """
65 
66  configPaths = ()
67  """Paths to config files to read for specific Tasks.
68 
69  The paths in this list should contain files of the form `task.py`, for
70  each of the Tasks that requires special configuration.
71  """
72 
73  policyName = None
74  """Instrument specific name to use when locating a policy or configuration
75  file in the file system."""
76 
77  obsDataPackage = None
78  """Name of the package containing the text curated calibration files.
79  Usually a obs _data package. If `None` no curated calibration files
80  will be read. (`str`)"""
81 
82  standardCuratedDatasetTypes = tuple(StandardCuratedCalibrationDatasetTypes)
83  """The dataset types expected to be obtained from the obsDataPackage.
84  These dataset types are all required to have standard definitions and
85  must be known to the base class. Clearing this list will prevent
86  any of these calibrations from being stored. If a dataset type is not
87  known to a specific instrument it can still be included in this list
88  since the data package is the source of truth.
89  """
90 
91  @property
92  @abstractmethod
93  def filterDefinitions(self):
94  """`~lsst.obs.base.FilterDefinitionCollection`, defining the filters
95  for this instrument.
96  """
97  return None
98 
99  def __init__(self):
100  self.filterDefinitions.reset()
101  self.filterDefinitions.defineFilters()
102  self._obsDataPackageDir = None
103 
104  @classmethod
105  @abstractmethod
106  def getName(cls):
107  """Return the short (dimension) name for this instrument.
108 
109  This is not (in general) the same as the class name - it's what is used
110  as the value of the "instrument" field in data IDs, and is usually an
111  abbreviation of the full name.
112  """
113  raise NotImplementedError()
114 
115  @abstractmethod
116  def getCamera(self):
117  """Retrieve the cameraGeom representation of this instrument.
118 
119  This is a temporary API that should go away once ``obs_`` packages have
120  a standardized approach to writing versioned cameras to a Gen3 repo.
121  """
122  raise NotImplementedError()
123 
124  @abstractmethod
125  def register(self, registry):
126  """Insert instrument, physical_filter, and detector entries into a
127  `Registry`.
128  """
129  raise NotImplementedError()
130 
131  @property
132  def obsDataPackageDir(self):
133  """The root of the obs package that provides specializations for
134  this instrument (`str`).
135  """
136  if self.obsDataPackage is None:
137  return None
138  if self._obsDataPackageDir is None:
139  # Defer any problems with locating the package until
140  # we need to find it.
141  self._obsDataPackageDir = getPackageDir(self.obsDataPackage)
142  return self._obsDataPackageDir
143 
144  @staticmethod
145  def fromName(name: str, registry: Registry) -> Instrument:
146  """Given an instrument name and a butler, retrieve a corresponding
147  instantiated instrument object.
148 
149  Parameters
150  ----------
151  name : `str`
152  Name of the instrument (must match the return value of `getName`).
153  registry : `lsst.daf.butler.Registry`
154  Butler registry to query to find the information.
155 
156  Returns
157  -------
158  instrument : `Instrument`
159  An instance of the relevant `Instrument`.
160 
161  Notes
162  -----
163  The instrument must be registered in the corresponding butler.
164 
165  Raises
166  ------
167  LookupError
168  Raised if the instrument is not known to the supplied registry.
169  ModuleNotFoundError
170  Raised if the class could not be imported. This could mean
171  that the relevant obs package has not been setup.
172  TypeError
173  Raised if the class name retrieved is not a string.
174  """
175  dimensions = list(registry.queryDimensions("instrument", dataId={"instrument": name}))
176  cls = dimensions[0].records["instrument"].class_name
177  if not isinstance(cls, str):
178  raise TypeError(f"Unexpected class name retrieved from {name} instrument dimension (got {cls})")
179  instrument = doImport(cls)
180  return instrument()
181 
182  @staticmethod
183  def importAll(registry: Registry) -> None:
184  """Import all the instruments known to this registry.
185 
186  This will ensure that all metadata translators have been registered.
187 
188  Parameters
189  ----------
190  registry : `lsst.daf.butler.Registry`
191  Butler registry to query to find the information.
192 
193  Notes
194  -----
195  It is allowed for a particular instrument class to fail on import.
196  This might simply indicate that a particular obs package has
197  not been setup.
198  """
199  dimensions = list(registry.queryDimensions("instrument"))
200  for dim in dimensions:
201  cls = dim.records["instrument"].class_name
202  try:
203  doImport(cls)
204  except Exception:
205  pass
206 
207  def _registerFilters(self, registry):
208  """Register the physical and abstract filter Dimension relationships.
209  This should be called in the ``register`` implementation.
210 
211  Parameters
212  ----------
213  registry : `lsst.daf.butler.core.Registry`
214  The registry to add dimensions to.
215  """
216  for filter in self.filterDefinitions:
217  # fix for undefined abstract filters causing trouble in the registry:
218  if filter.abstract_filter is None:
219  abstract_filter = filter.physical_filter
220  else:
221  abstract_filter = filter.abstract_filter
222 
223  registry.insertDimensionData("physical_filter",
224  {"instrument": self.getName(),
225  "name": filter.physical_filter,
226  "abstract_filter": abstract_filter
227  })
228 
229  @abstractmethod
230  def getRawFormatter(self, dataId):
231  """Return the Formatter class that should be used to read a particular
232  raw file.
233 
234  Parameters
235  ----------
236  dataId : `DataCoordinate`
237  Dimension-based ID for the raw file or files being ingested.
238 
239  Returns
240  -------
241  formatter : `Formatter` class
242  Class to be used that reads the file into an
243  `lsst.afw.image.Exposure` instance.
244  """
245  raise NotImplementedError()
246 
247  def writeCuratedCalibrations(self, butler, run=None):
248  """Write human-curated calibration Datasets to the given Butler with
249  the appropriate validity ranges.
250 
251  Parameters
252  ----------
253  butler : `lsst.daf.butler.Butler`
254  Butler to use to store these calibrations.
255  run : `str`
256  Run to use for this collection of calibrations. If `None` the
257  collection name is worked out automatically from the instrument
258  name and other metadata.
259 
260  Notes
261  -----
262  Expected to be called from subclasses. The base method calls
263  ``writeCameraGeom`` and ``writeStandardTextCuratedCalibrations``.
264  """
265  # Need to determine the run for ingestion based on the instrument
266  # name and eventually the data package version. The camera geom
267  # is currently special in that it is not in the _data package.
268  if run is None:
269  run = self.makeCollectionName("calib")
270  butler.registry.registerCollection(run, type=CollectionType.RUN)
271  self.writeCameraGeom(butler, run=run)
272  self.writeStandardTextCuratedCalibrations(butler, run=run)
273  self.writeAdditionalCuratedCalibrations(butler, run=run)
274 
275  def writeAdditionalCuratedCalibrations(self, butler, run=None):
276  """Write additional curated calibrations that might be instrument
277  specific and are not part of the standard set.
278 
279  Default implementation does nothing.
280 
281  Parameters
282  ----------
283  butler : `lsst.daf.butler.Butler`
284  Butler to use to store these calibrations.
285  run : `str`, optional
286  Name of the run to use to override the default run associated
287  with this Butler.
288  """
289  return
290 
291  def applyConfigOverrides(self, name, config):
292  """Apply instrument-specific overrides for a task config.
293 
294  Parameters
295  ----------
296  name : `str`
297  Name of the object being configured; typically the _DefaultName
298  of a Task.
299  config : `lsst.pex.config.Config`
300  Config instance to which overrides should be applied.
301  """
302  for root in self.configPaths:
303  path = os.path.join(root, f"{name}.py")
304  if os.path.exists(path):
305  config.load(path)
306 
307  def writeCameraGeom(self, butler, run=None):
308  """Write the default camera geometry to the butler repository
309  with an infinite validity range.
310 
311  Parameters
312  ----------
313  butler : `lsst.daf.butler.Butler`
314  Butler to receive these calibration datasets.
315  run : `str`, optional
316  Name of the run to use to override the default run associated
317  with this Butler.
318  """
319 
320  datasetType = DatasetType("camera", ("instrument", "calibration_label"), "Camera",
321  universe=butler.registry.dimensions)
322  butler.registry.registerDatasetType(datasetType)
323  unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName())
324  camera = self.getCamera()
325  butler.put(camera, datasetType, unboundedDataId, run=run)
326 
327  def writeStandardTextCuratedCalibrations(self, butler, run=None):
328  """Write the set of standardized curated text calibrations to
329  the repository.
330 
331  Parameters
332  ----------
333  butler : `lsst.daf.butler.Butler`
334  Butler to receive these calibration datasets.
335  run : `str`, optional
336  Name of the run to use to override the default run associated
337  with this Butler.
338  """
339 
340  for datasetTypeName in self.standardCuratedDatasetTypes:
341  # We need to define the dataset types.
342  if datasetTypeName not in StandardCuratedCalibrationDatasetTypes:
343  raise ValueError(f"DatasetType {datasetTypeName} not in understood list"
344  f" [{'.'.join(StandardCuratedCalibrationDatasetTypes)}]")
345  definition = StandardCuratedCalibrationDatasetTypes[datasetTypeName]
346  datasetType = DatasetType(datasetTypeName,
347  universe=butler.registry.dimensions,
348  **definition)
349  self._writeSpecificCuratedCalibrationDatasets(butler, datasetType, run=run)
350 
351  def _writeSpecificCuratedCalibrationDatasets(self, butler, datasetType, run=None):
352  """Write standardized curated calibration datasets for this specific
353  dataset type from an obs data package.
354 
355  Parameters
356  ----------
357  butler : `lsst.daf.butler.Butler`
358  Gen3 butler in which to put the calibrations.
359  datasetType : `lsst.daf.butler.DatasetType`
360  Dataset type to be put.
361  run : `str`, optional
362  Name of the run to use to override the default run associated
363  with this Butler.
364 
365  Notes
366  -----
367  This method scans the location defined in the ``obsDataPackageDir``
368  class attribute for curated calibrations corresponding to the
369  supplied dataset type. The directory name in the data package must
370  match the name of the dataset type. They are assumed to use the
371  standard layout and can be read by
372  `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard
373  metadata.
374  """
375  if self.obsDataPackageDir is None:
376  # if there is no data package then there can't be datasets
377  return
378 
379  calibPath = os.path.join(self.obsDataPackageDir, self.policyName,
380  datasetType.name)
381 
382  if not os.path.exists(calibPath):
383  return
384 
385  # Register the dataset type
386  butler.registry.registerDatasetType(datasetType)
387 
388  # obs_base can't depend on pipe_tasks but concrete obs packages
389  # can -- we therefore have to defer import
390  from lsst.pipe.tasks.read_curated_calibs import read_all
391 
392  camera = self.getCamera()
393  calibsDict = read_all(calibPath, camera)[0] # second return is calib type
394  endOfTime = TIMESPAN_MAX
395  dimensionRecords = []
396  datasetRecords = []
397  for det in calibsDict:
398  times = sorted([k for k in calibsDict[det]])
399  calibs = [calibsDict[det][time] for time in times]
400  times = [astropy.time.Time(t, format="datetime", scale="utc") for t in times]
401  times += [endOfTime]
402  for calib, beginTime, endTime in zip(calibs, times[:-1], times[1:]):
403  md = calib.getMetadata()
404  calibrationLabel = f"{datasetType.name}/{md['CALIBDATE']}/{md['DETECTOR']}"
405  dataId = DataCoordinate.standardize(
406  universe=butler.registry.dimensions,
407  instrument=self.getName(),
408  calibration_label=calibrationLabel,
409  detector=md["DETECTOR"],
410  )
411  datasetRecords.append((calib, dataId))
412  dimensionRecords.append({
413  "instrument": self.getName(),
414  "name": calibrationLabel,
415  "datetime_begin": beginTime,
416  "datetime_end": endTime,
417  })
418 
419  # Second loop actually does the inserts and filesystem writes.
420  with butler.transaction():
421  butler.registry.insertDimensionData("calibration_label", *dimensionRecords)
422  # TODO: vectorize these puts, once butler APIs for that become
423  # available.
424  for calib, dataId in datasetRecords:
425  butler.put(calib, datasetType, dataId, run=run)
426 
427  @abstractmethod
428  def makeDataIdTranslatorFactory(self) -> TranslatorFactory:
429  """Return a factory for creating Gen2->Gen3 data ID translators,
430  specialized for this instrument.
431 
432  Derived class implementations should generally call
433  `TranslatorFactory.addGenericInstrumentRules` with appropriate
434  arguments, but are not required to (and may not be able to if their
435  Gen2 raw data IDs are sufficiently different from the HSC/DECam/CFHT
436  norm).
437 
438  Returns
439  -------
440  factory : `TranslatorFactory`.
441  Factory for `Translator` objects.
442  """
443  raise NotImplementedError("Must be implemented by derived classes.")
444 
445  @classmethod
447  """Make the default instrument-specific run collection string for raw
448  data ingest.
449 
450  Returns
451  -------
452  coll : `str`
453  Run collection name to be used as the default for ingestion of
454  raws.
455  """
456  return cls.makeCollectionName("raw/all")
457 
458  @classmethod
459  def makeCollectionName(cls, label: str) -> str:
460  """Get the instrument-specific collection string to use as derived
461  from the supplied label.
462 
463  Parameters
464  ----------
465  label : `str`
466  String to be combined with the instrument name to form a
467  collection name.
468 
469  Returns
470  -------
471  name : `str`
472  Collection name to use that includes the instrument name.
473  """
474  return f"{cls.getName()}/{label}"
475 
476 
477 def makeExposureRecordFromObsInfo(obsInfo, universe):
478  """Construct an exposure DimensionRecord from
479  `astro_metadata_translator.ObservationInfo`.
480 
481  Parameters
482  ----------
483  obsInfo : `astro_metadata_translator.ObservationInfo`
484  A `~astro_metadata_translator.ObservationInfo` object corresponding to
485  the exposure.
486  universe : `DimensionUniverse`
487  Set of all known dimensions.
488 
489  Returns
490  -------
491  record : `DimensionRecord`
492  A record containing exposure metadata, suitable for insertion into
493  a `Registry`.
494  """
495  dimension = universe["exposure"]
496  return dimension.RecordClass.fromDict({
497  "instrument": obsInfo.instrument,
498  "id": obsInfo.exposure_id,
499  "name": obsInfo.observation_id,
500  "group_name": obsInfo.exposure_group,
501  "group_id": obsInfo.visit_id,
502  "datetime_begin": obsInfo.datetime_begin,
503  "datetime_end": obsInfo.datetime_end,
504  "exposure_time": obsInfo.exposure_time.to_value("s"),
505  "dark_time": obsInfo.dark_time.to_value("s"),
506  "observation_type": obsInfo.observation_type,
507  "physical_filter": obsInfo.physical_filter,
508  })
509 
510 
511 def addUnboundedCalibrationLabel(registry, instrumentName):
512  """Add a special 'unbounded' calibration_label dimension entry for the
513  given camera that is valid for any exposure.
514 
515  If such an entry already exists, this function just returns a `DataId`
516  for the existing entry.
517 
518  Parameters
519  ----------
520  registry : `Registry`
521  Registry object in which to insert the dimension entry.
522  instrumentName : `str`
523  Name of the instrument this calibration label is associated with.
524 
525  Returns
526  -------
527  dataId : `DataId`
528  New or existing data ID for the unbounded calibration.
529  """
530  d = dict(instrument=instrumentName, calibration_label="unbounded")
531  try:
532  return registry.expandDataId(d)
533  except LookupError:
534  pass
535  entry = d.copy()
536  entry["datetime_begin"] = TIMESPAN_MIN
537  entry["datetime_end"] = TIMESPAN_MAX
538  registry.insertDimensionData("calibration_label", entry)
539  return registry.expandDataId(d)
540 
541 
542 def loadCamera(butler: Butler, dataId: DataId, *, collections: Any = None) -> Tuple[Camera, bool]:
543  """Attempt to load versioned camera geometry from a butler, but fall back
544  to obtaining a nominal camera from the `Instrument` class if that fails.
545 
546  Parameters
547  ----------
548  butler : `lsst.daf.butler.Butler`
549  Butler instance to attempt to query for and load a ``camera`` dataset
550  from.
551  dataId : `dict` or `DataCoordinate`
552  Data ID that identifies at least the ``instrument`` and ``exposure``
553  dimensions.
554  collections : Any, optional
555  Collections to be searched, overriding ``self.butler.collections``.
556  Can be any of the types supported by the ``collections`` argument
557  to butler construction.
558 
559  Returns
560  -------
561  camera : `lsst.afw.cameraGeom.Camera`
562  Camera object.
563  versioned : `bool`
564  If `True`, the camera was obtained from the butler and should represent
565  a versioned camera from a calibration repository. If `False`, no
566  camera datasets were found, and the returned camera was produced by
567  instantiating the appropriate `Instrument` class and calling
568  `Instrument.getCamera`.
569  """
570  if collections is None:
571  collections = butler.collections
572  # Registry would do data ID expansion internally if we didn't do it first,
573  # but we might want an expanded data ID ourselves later, so we do it here
574  # to ensure it only happens once.
575  # This will also catch problems with the data ID not having keys we need.
576  dataId = butler.registry.expandDataId(dataId, graph=butler.registry.dimensions["exposure"].graph)
577  cameraRefs = list(butler.registry.queryDatasets("camera", dataId=dataId, collections=collections,
578  deduplicate=True))
579  if cameraRefs:
580  assert len(cameraRefs) == 1, "Should be guaranteed by deduplicate=True above."
581  return butler.getDirect(cameraRefs[0]), True
582  instrument = Instrument.fromName(dataId["instrument"], butler.registry)
583  return instrument.getCamera(), False
lsst.obs.base._instrument.Instrument.applyConfigOverrides
def applyConfigOverrides(self, name, config)
Definition: _instrument.py:291
lsst.obs.base._instrument.loadCamera
Tuple[Camera, bool] loadCamera(Butler butler, DataId dataId, *Any collections=None)
Definition: _instrument.py:542
lsst.obs.base._instrument.Instrument.getName
def getName(cls)
Definition: _instrument.py:106
lsst.obs.base._instrument.Instrument.obsDataPackage
obsDataPackage
Definition: _instrument.py:77
lsst.obs.base._instrument.Instrument.importAll
None importAll(Registry registry)
Definition: _instrument.py:183
lsst.obs.base._instrument.Instrument._writeSpecificCuratedCalibrationDatasets
def _writeSpecificCuratedCalibrationDatasets(self, butler, datasetType, run=None)
Definition: _instrument.py:351
lsst.obs.base._instrument.Instrument.getCamera
def getCamera(self)
Definition: _instrument.py:116
lsst.obs.base._instrument.Instrument.filterDefinitions
def filterDefinitions(self)
Definition: _instrument.py:93
lsst.obs.base._instrument.Instrument.fromName
Instrument fromName(str name, Registry registry)
Definition: _instrument.py:145
lsst.obs.base._instrument.Instrument.policyName
policyName
Definition: _instrument.py:73
lsst.obs.base._instrument.Instrument.makeCollectionName
str makeCollectionName(cls, str label)
Definition: _instrument.py:459
lsst::utils
lsst.obs.base._instrument.Instrument
Definition: _instrument.py:59
lsst.obs.base._instrument.Instrument.writeAdditionalCuratedCalibrations
def writeAdditionalCuratedCalibrations(self, butler, run=None)
Definition: _instrument.py:275
lsst.obs.base._instrument.Instrument.writeCuratedCalibrations
def writeCuratedCalibrations(self, butler, run=None)
Definition: _instrument.py:247
lsst.obs.base._instrument.addUnboundedCalibrationLabel
def addUnboundedCalibrationLabel(registry, instrumentName)
Definition: _instrument.py:511
lsst.obs.base._instrument.makeExposureRecordFromObsInfo
def makeExposureRecordFromObsInfo(obsInfo, universe)
Definition: _instrument.py:477
lsst.obs.base._instrument.Instrument.register
def register(self, registry)
Definition: _instrument.py:125
lsst.obs.base._instrument.Instrument.writeCameraGeom
def writeCameraGeom(self, butler, run=None)
Definition: _instrument.py:307
lsst.obs.base._instrument.Instrument.getRawFormatter
def getRawFormatter(self, dataId)
Definition: _instrument.py:230
lsst.obs.base._instrument.Instrument._obsDataPackageDir
_obsDataPackageDir
Definition: _instrument.py:102
lsst.obs.base._instrument.Instrument.standardCuratedDatasetTypes
standardCuratedDatasetTypes
Definition: _instrument.py:82
lsst.obs.base._instrument.Instrument.makeDataIdTranslatorFactory
TranslatorFactory makeDataIdTranslatorFactory(self)
Definition: _instrument.py:428
lsst.obs.base._instrument.Instrument.makeDefaultRawIngestRunName
str makeDefaultRawIngestRunName(cls)
Definition: _instrument.py:446
lsst.obs.base._instrument.Instrument.writeStandardTextCuratedCalibrations
def writeStandardTextCuratedCalibrations(self, butler, run=None)
Definition: _instrument.py:327
lsst.obs.base._instrument.Instrument.__init__
def __init__(self)
Definition: _instrument.py:99
lsst.obs.base._instrument.Instrument.obsDataPackageDir
def obsDataPackageDir(self)
Definition: _instrument.py:132