lsst.obs.base  19.0.0-79-gada4096
_instrument.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 from __future__ import annotations
23 
24 __all__ = ("Instrument", "makeExposureRecordFromObsInfo", "addUnboundedCalibrationLabel", "loadCamera")
25 
26 import os.path
27 from abc import ABCMeta, abstractmethod
28 from typing import Any, Tuple, TYPE_CHECKING
29 import astropy.time
30 
31 from lsst.afw.cameraGeom import Camera
32 from lsst.daf.butler import Butler, DataId, TIMESPAN_MIN, TIMESPAN_MAX, DatasetType, DataCoordinate
33 from lsst.utils import getPackageDir, doImport
34 
35 if TYPE_CHECKING:
36  from .gen2to3 import TranslatorFactory
37  from lsst.daf.butler import Registry
38 
39 # To be a standard text curated calibration means that we use a
40 # standard definition for the corresponding DatasetType.
41 StandardCuratedCalibrationDatasetTypes = {
42  "defects": {"dimensions": ("instrument", "detector", "calibration_label"),
43  "storageClass": "Defects"},
44  "qe_curve": {"dimensions": ("instrument", "detector", "calibration_label"),
45  "storageClass": "QECurve"},
46  "crosstalk": {"dimensions": ("instrument", "detector", "calibration_label"),
47  "storageClass": "CrosstalkCalib"},
48 }
49 
50 
51 class Instrument(metaclass=ABCMeta):
52  """Base class for instrument-specific logic for the Gen3 Butler.
53 
54  Concrete instrument subclasses should be directly constructable with no
55  arguments.
56  """
57 
58  configPaths = ()
59  """Paths to config files to read for specific Tasks.
60 
61  The paths in this list should contain files of the form `task.py`, for
62  each of the Tasks that requires special configuration.
63  """
64 
65  policyName = None
66  """Instrument specific name to use when locating a policy or configuration
67  file in the file system."""
68 
69  obsDataPackage = None
70  """Name of the package containing the text curated calibration files.
71  Usually a obs _data package. If `None` no curated calibration files
72  will be read. (`str`)"""
73 
74  standardCuratedDatasetTypes = tuple(StandardCuratedCalibrationDatasetTypes)
75  """The dataset types expected to be obtained from the obsDataPackage.
76  These dataset types are all required to have standard definitions and
77  must be known to the base class. Clearing this list will prevent
78  any of these calibrations from being stored. If a dataset type is not
79  known to a specific instrument it can still be included in this list
80  since the data package is the source of truth.
81  """
82 
83  @property
84  @abstractmethod
85  def filterDefinitions(self):
86  """`~lsst.obs.base.FilterDefinitionCollection`, defining the filters
87  for this instrument.
88  """
89  return None
90 
91  def __init__(self):
92  self.filterDefinitions.reset()
93  self.filterDefinitions.defineFilters()
94  self._obsDataPackageDir = None
95 
96  @classmethod
97  @abstractmethod
98  def getName(cls):
99  """Return the short (dimension) name for this instrument.
100 
101  This is not (in general) the same as the class name - it's what is used
102  as the value of the "instrument" field in data IDs, and is usually an
103  abbreviation of the full name.
104  """
105  raise NotImplementedError()
106 
107  @abstractmethod
108  def getCamera(self):
109  """Retrieve the cameraGeom representation of this instrument.
110 
111  This is a temporary API that should go away once ``obs_`` packages have
112  a standardized approach to writing versioned cameras to a Gen3 repo.
113  """
114  raise NotImplementedError()
115 
116  @abstractmethod
117  def register(self, registry):
118  """Insert instrument, physical_filter, and detector entries into a
119  `Registry`.
120  """
121  raise NotImplementedError()
122 
123  @property
124  def obsDataPackageDir(self):
125  """The root of the obs package that provides specializations for
126  this instrument (`str`).
127  """
128  if self.obsDataPackage is None:
129  return None
130  if self._obsDataPackageDir is None:
131  # Defer any problems with locating the package until
132  # we need to find it.
133  self._obsDataPackageDir = getPackageDir(self.obsDataPackage)
134  return self._obsDataPackageDir
135 
136  @staticmethod
137  def fromName(name: str, registry: Registry) -> Instrument:
138  """Given an instrument name and a butler, retrieve a corresponding
139  instantiated instrument object.
140 
141  Parameters
142  ----------
143  name : `str`
144  Name of the instrument (must match the return value of `getName`).
145  registry : `lsst.daf.butler.Registry`
146  Butler registry to query to find the information.
147 
148  Returns
149  -------
150  instrument : `Instrument`
151  An instance of the relevant `Instrument`.
152 
153  Notes
154  -----
155  The instrument must be registered in the corresponding butler.
156 
157  Raises
158  ------
159  LookupError
160  Raised if the instrument is not known to the supplied registry.
161  ModuleNotFoundError
162  Raised if the class could not be imported. This could mean
163  that the relevant obs package has not been setup.
164  TypeError
165  Raised if the class name retrieved is not a string.
166  """
167  dimensions = list(registry.queryDimensions("instrument", dataId={"instrument": name}))
168  cls = dimensions[0].records["instrument"].class_name
169  if not isinstance(cls, str):
170  raise TypeError(f"Unexpected class name retrieved from {name} instrument dimension (got {cls})")
171  instrument = doImport(cls)
172  return instrument()
173 
174  @staticmethod
175  def importAll(registry: Registry) -> None:
176  """Import all the instruments known to this registry.
177 
178  This will ensure that all metadata translators have been registered.
179 
180  Parameters
181  ----------
182  registry : `lsst.daf.butler.Registry`
183  Butler registry to query to find the information.
184 
185  Notes
186  -----
187  It is allowed for a particular instrument class to fail on import.
188  This might simply indicate that a particular obs package has
189  not been setup.
190  """
191  dimensions = list(registry.queryDimensions("instrument"))
192  for dim in dimensions:
193  cls = dim.records["instrument"].class_name
194  try:
195  doImport(cls)
196  except Exception:
197  pass
198 
199  def _registerFilters(self, registry):
200  """Register the physical and abstract filter Dimension relationships.
201  This should be called in the ``register`` implementation.
202 
203  Parameters
204  ----------
205  registry : `lsst.daf.butler.core.Registry`
206  The registry to add dimensions to.
207  """
208  for filter in self.filterDefinitions:
209  # fix for undefined abstract filters causing trouble in the registry:
210  if filter.abstract_filter is None:
211  abstract_filter = filter.physical_filter
212  else:
213  abstract_filter = filter.abstract_filter
214 
215  registry.insertDimensionData("physical_filter",
216  {"instrument": self.getName(),
217  "name": filter.physical_filter,
218  "abstract_filter": abstract_filter
219  })
220 
221  @abstractmethod
222  def getRawFormatter(self, dataId):
223  """Return the Formatter class that should be used to read a particular
224  raw file.
225 
226  Parameters
227  ----------
228  dataId : `DataCoordinate`
229  Dimension-based ID for the raw file or files being ingested.
230 
231  Returns
232  -------
233  formatter : `Formatter` class
234  Class to be used that reads the file into an
235  `lsst.afw.image.Exposure` instance.
236  """
237  raise NotImplementedError()
238 
239  def writeCuratedCalibrations(self, butler):
240  """Write human-curated calibration Datasets to the given Butler with
241  the appropriate validity ranges.
242 
243  Parameters
244  ----------
245  butler : `lsst.daf.butler.Butler`
246  Butler to use to store these calibrations.
247 
248  Notes
249  -----
250  Expected to be called from subclasses. The base method calls
251  ``writeCameraGeom`` and ``writeStandardTextCuratedCalibrations``.
252  """
253  self.writeCameraGeom(butler)
255 
256  def applyConfigOverrides(self, name, config):
257  """Apply instrument-specific overrides for a task config.
258 
259  Parameters
260  ----------
261  name : `str`
262  Name of the object being configured; typically the _DefaultName
263  of a Task.
264  config : `lsst.pex.config.Config`
265  Config instance to which overrides should be applied.
266  """
267  for root in self.configPaths:
268  path = os.path.join(root, f"{name}.py")
269  if os.path.exists(path):
270  config.load(path)
271 
272  def writeCameraGeom(self, butler):
273  """Write the default camera geometry to the butler repository
274  with an infinite validity range.
275 
276  Parameters
277  ----------
278  butler : `lsst.daf.butler.Butler`
279  Butler to receive these calibration datasets.
280  """
281 
282  datasetType = DatasetType("camera", ("instrument", "calibration_label"), "Camera",
283  universe=butler.registry.dimensions)
284  butler.registry.registerDatasetType(datasetType)
285  unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName())
286  camera = self.getCamera()
287  butler.put(camera, datasetType, unboundedDataId)
288 
290  """Write the set of standardized curated text calibrations to
291  the repository.
292 
293  Parameters
294  ----------
295  butler : `lsst.daf.butler.Butler`
296  Butler to receive these calibration datasets.
297  """
298 
299  for datasetTypeName in self.standardCuratedDatasetTypes:
300  # We need to define the dataset types.
301  if datasetTypeName not in StandardCuratedCalibrationDatasetTypes:
302  raise ValueError(f"DatasetType {datasetTypeName} not in understood list"
303  f" [{'.'.join(StandardCuratedCalibrationDatasetTypes)}]")
304  definition = StandardCuratedCalibrationDatasetTypes[datasetTypeName]
305  datasetType = DatasetType(datasetTypeName,
306  universe=butler.registry.dimensions,
307  **definition)
308  self._writeSpecificCuratedCalibrationDatasets(butler, datasetType)
309 
310  def _writeSpecificCuratedCalibrationDatasets(self, butler, datasetType):
311  """Write standardized curated calibration datasets for this specific
312  dataset type from an obs data package.
313 
314  Parameters
315  ----------
316  butler : `lsst.daf.butler.Butler`
317  Gen3 butler in which to put the calibrations.
318  datasetType : `lsst.daf.butler.DatasetType`
319  Dataset type to be put.
320 
321  Notes
322  -----
323  This method scans the location defined in the ``obsDataPackageDir``
324  class attribute for curated calibrations corresponding to the
325  supplied dataset type. The directory name in the data package must
326  match the name of the dataset type. They are assumed to use the
327  standard layout and can be read by
328  `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard
329  metadata.
330  """
331  if self.obsDataPackageDir is None:
332  # if there is no data package then there can't be datasets
333  return
334 
335  calibPath = os.path.join(self.obsDataPackageDir, self.policyName,
336  datasetType.name)
337 
338  if not os.path.exists(calibPath):
339  return
340 
341  # Register the dataset type
342  butler.registry.registerDatasetType(datasetType)
343 
344  # obs_base can't depend on pipe_tasks but concrete obs packages
345  # can -- we therefore have to defer import
346  from lsst.pipe.tasks.read_curated_calibs import read_all
347 
348  camera = self.getCamera()
349  calibsDict = read_all(calibPath, camera)[0] # second return is calib type
350  endOfTime = TIMESPAN_MAX
351  dimensionRecords = []
352  datasetRecords = []
353  for det in calibsDict:
354  times = sorted([k for k in calibsDict[det]])
355  calibs = [calibsDict[det][time] for time in times]
356  times = [astropy.time.Time(t, format="datetime", scale="utc") for t in times]
357  times += [endOfTime]
358  for calib, beginTime, endTime in zip(calibs, times[:-1], times[1:]):
359  md = calib.getMetadata()
360  calibrationLabel = f"{datasetType.name}/{md['CALIBDATE']}/{md['DETECTOR']}"
361  dataId = DataCoordinate.standardize(
362  universe=butler.registry.dimensions,
363  instrument=self.getName(),
364  calibration_label=calibrationLabel,
365  detector=md["DETECTOR"],
366  )
367  datasetRecords.append((calib, dataId))
368  dimensionRecords.append({
369  "instrument": self.getName(),
370  "name": calibrationLabel,
371  "datetime_begin": beginTime,
372  "datetime_end": endTime,
373  })
374 
375  # Second loop actually does the inserts and filesystem writes.
376  with butler.transaction():
377  butler.registry.insertDimensionData("calibration_label", *dimensionRecords)
378  # TODO: vectorize these puts, once butler APIs for that become
379  # available.
380  for calib, dataId in datasetRecords:
381  butler.put(calib, datasetType, dataId)
382 
383  @abstractmethod
384  def makeDataIdTranslatorFactory(self) -> TranslatorFactory:
385  """Return a factory for creating Gen2->Gen3 data ID translators,
386  specialized for this instrument.
387 
388  Derived class implementations should generally call
389  `TranslatorFactory.addGenericInstrumentRules` with appropriate
390  arguments, but are not required to (and may not be able to if their
391  Gen2 raw data IDs are sufficiently different from the HSC/DECam/CFHT
392  norm).
393 
394  Returns
395  -------
396  factory : `TranslatorFactory`.
397  Factory for `Translator` objects.
398  """
399  raise NotImplementedError("Must be implemented by derived classes.")
400 
401 
402 def makeExposureRecordFromObsInfo(obsInfo, universe):
403  """Construct an exposure DimensionRecord from
404  `astro_metadata_translator.ObservationInfo`.
405 
406  Parameters
407  ----------
408  obsInfo : `astro_metadata_translator.ObservationInfo`
409  A `~astro_metadata_translator.ObservationInfo` object corresponding to
410  the exposure.
411  universe : `DimensionUniverse`
412  Set of all known dimensions.
413 
414  Returns
415  -------
416  record : `DimensionRecord`
417  A record containing exposure metadata, suitable for insertion into
418  a `Registry`.
419  """
420  dimension = universe["exposure"]
421  return dimension.RecordClass.fromDict({
422  "instrument": obsInfo.instrument,
423  "id": obsInfo.exposure_id,
424  "name": obsInfo.observation_id,
425  "group_name": obsInfo.exposure_group,
426  "group_id": obsInfo.visit_id,
427  "datetime_begin": obsInfo.datetime_begin,
428  "datetime_end": obsInfo.datetime_end,
429  "exposure_time": obsInfo.exposure_time.to_value("s"),
430  "dark_time": obsInfo.dark_time.to_value("s"),
431  "observation_type": obsInfo.observation_type,
432  "physical_filter": obsInfo.physical_filter,
433  })
434 
435 
436 def addUnboundedCalibrationLabel(registry, instrumentName):
437  """Add a special 'unbounded' calibration_label dimension entry for the
438  given camera that is valid for any exposure.
439 
440  If such an entry already exists, this function just returns a `DataId`
441  for the existing entry.
442 
443  Parameters
444  ----------
445  registry : `Registry`
446  Registry object in which to insert the dimension entry.
447  instrumentName : `str`
448  Name of the instrument this calibration label is associated with.
449 
450  Returns
451  -------
452  dataId : `DataId`
453  New or existing data ID for the unbounded calibration.
454  """
455  d = dict(instrument=instrumentName, calibration_label="unbounded")
456  try:
457  return registry.expandDataId(d)
458  except LookupError:
459  pass
460  entry = d.copy()
461  entry["datetime_begin"] = TIMESPAN_MIN
462  entry["datetime_end"] = TIMESPAN_MAX
463  registry.insertDimensionData("calibration_label", entry)
464  return registry.expandDataId(d)
465 
466 
467 def loadCamera(butler: Butler, dataId: DataId, *, collections: Any = None) -> Tuple[Camera, bool]:
468  """Attempt to load versioned camera geometry from a butler, but fall back
469  to obtaining a nominal camera from the `Instrument` class if that fails.
470 
471  Parameters
472  ----------
473  butler : `lsst.daf.butler.Butler`
474  Butler instance to attempt to query for and load a ``camera`` dataset
475  from.
476  dataId : `dict` or `DataCoordinate`
477  Data ID that identifies at least the ``instrument`` and ``exposure``
478  dimensions.
479  collections : Any, optional
480  Collections to be searched, overriding ``self.butler.collections``.
481  Can be any of the types supported by the ``collections`` argument
482  to butler construction.
483 
484  Returns
485  -------
486  camera : `lsst.afw.cameraGeom.Camera`
487  Camera object.
488  versioned : `bool`
489  If `True`, the camera was obtained from the butler and should represent
490  a versioned camera from a calibration repository. If `False`, no
491  camera datasets were found, and the returned camera was produced by
492  instantiating the appropriate `Instrument` class and calling
493  `Instrument.getCamera`.
494  """
495  if collections is None:
496  collections = butler.collections
497  # Registry would do data ID expansion internally if we didn't do it first,
498  # but we might want an expanded data ID ourselves later, so we do it here
499  # to ensure it only happens once.
500  # This will also catch problems with the data ID not having keys we need.
501  dataId = butler.registry.expandDataId(dataId, graph=butler.registry.dimensions["exposure"].graph)
502  cameraRefs = list(butler.registry.queryDatasets("camera", dataId=dataId, collections=collections,
503  deduplicate=True))
504  if cameraRefs:
505  assert len(cameraRefs) == 1, "Should be guaranteed by deduplicate=True above."
506  return butler.getDirect(cameraRefs[0]), True
507  instrument = Instrument.fromName(dataId["instrument"], butler.registry)
508  return instrument.getCamera(), False
lsst.obs.base._instrument.Instrument.applyConfigOverrides
def applyConfigOverrides(self, name, config)
Definition: _instrument.py:256
lsst.obs.base._instrument.loadCamera
Tuple[Camera, bool] loadCamera(Butler butler, DataId dataId, *Any collections=None)
Definition: _instrument.py:467
lsst.obs.base._instrument.Instrument.getName
def getName(cls)
Definition: _instrument.py:98
lsst.obs.base._instrument.Instrument.obsDataPackage
obsDataPackage
Definition: _instrument.py:69
lsst.obs.base._instrument.Instrument.importAll
None importAll(Registry registry)
Definition: _instrument.py:175
lsst.obs.base._instrument.Instrument.getCamera
def getCamera(self)
Definition: _instrument.py:108
lsst.obs.base._instrument.Instrument.filterDefinitions
def filterDefinitions(self)
Definition: _instrument.py:85
lsst.obs.base._instrument.Instrument.fromName
Instrument fromName(str name, Registry registry)
Definition: _instrument.py:137
lsst.obs.base._instrument.Instrument.policyName
policyName
Definition: _instrument.py:65
lsst::utils
lsst.obs.base._instrument.Instrument
Definition: _instrument.py:51
lsst.obs.base._instrument.Instrument._writeSpecificCuratedCalibrationDatasets
def _writeSpecificCuratedCalibrationDatasets(self, butler, datasetType)
Definition: _instrument.py:310
lsst.obs.base._instrument.Instrument.writeCameraGeom
def writeCameraGeom(self, butler)
Definition: _instrument.py:272
lsst.obs.base._instrument.addUnboundedCalibrationLabel
def addUnboundedCalibrationLabel(registry, instrumentName)
Definition: _instrument.py:436
lsst.obs.base._instrument.makeExposureRecordFromObsInfo
def makeExposureRecordFromObsInfo(obsInfo, universe)
Definition: _instrument.py:402
lsst.obs.base._instrument.Instrument.register
def register(self, registry)
Definition: _instrument.py:117
lsst.obs.base._instrument.Instrument.getRawFormatter
def getRawFormatter(self, dataId)
Definition: _instrument.py:222
lsst.obs.base._instrument.Instrument._obsDataPackageDir
_obsDataPackageDir
Definition: _instrument.py:94
lsst.obs.base._instrument.Instrument.configPaths
tuple configPaths
Definition: _instrument.py:58
lsst.obs.base._instrument.Instrument.standardCuratedDatasetTypes
standardCuratedDatasetTypes
Definition: _instrument.py:74
lsst.obs.base._instrument.Instrument.makeDataIdTranslatorFactory
TranslatorFactory makeDataIdTranslatorFactory(self)
Definition: _instrument.py:384
lsst.obs.base._instrument.Instrument.writeCuratedCalibrations
def writeCuratedCalibrations(self, butler)
Definition: _instrument.py:239
lsst.obs.base._instrument.Instrument.__init__
def __init__(self)
Definition: _instrument.py:91
lsst.obs.base._instrument.Instrument.obsDataPackageDir
def obsDataPackageDir(self)
Definition: _instrument.py:124
lsst.obs.base._instrument.Instrument.writeStandardTextCuratedCalibrations
def writeStandardTextCuratedCalibrations(self, butler)
Definition: _instrument.py:289