lsst.obs.base  19.0.0-33-g58bbfa5
instrument.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 __all__ = ("Instrument", "makeExposureRecordFromObsInfo", "makeVisitRecordFromObsInfo",
23  "addUnboundedCalibrationLabel")
24 
25 import os.path
26 from abc import ABCMeta, abstractmethod
27 import astropy.time
28 
29 from lsst.daf.butler import TIMESPAN_MIN, TIMESPAN_MAX, DatasetType, DataCoordinate
30 from lsst.utils import getPackageDir
31 
32 # To be a standard text curated calibration means that we use a
33 # standard definition for the corresponding DatasetType.
34 StandardCuratedCalibrationDatasetTypes = {
35  "defects": {"dimensions": ("instrument", "detector", "calibration_label"),
36  "storageClass": "Defects"},
37  "qe_curve": {"dimensions": ("instrument", "detector", "calibration_label"),
38  "storageClass": "QECurve"},
39 }
40 
41 
42 class Instrument(metaclass=ABCMeta):
43  """Base class for instrument-specific logic for the Gen3 Butler.
44 
45  Concrete instrument subclasses should be directly constructable with no
46  arguments.
47  """
48 
49  configPaths = ()
50  """Paths to config files to read for specific Tasks.
51 
52  The paths in this list should contain files of the form `task.py`, for
53  each of the Tasks that requires special configuration.
54  """
55 
56  policyName = None
57  """Instrument specific name to use when locating a policy or configuration
58  file in the file system."""
59 
60  obsDataPackage = None
61  """Name of the package containing the text curated calibration files.
62  Usually a obs _data package. If `None` no curated calibration files
63  will be read. (`str`)"""
64 
65  standardCuratedDatasetTypes = tuple(StandardCuratedCalibrationDatasetTypes)
66  """The dataset types expected to be obtained from the obsDataPackage.
67  These dataset types are all required to have standard definitions and
68  must be known to the base class. Clearing this list will prevent
69  any of these calibrations from being stored. If a dataset type is not
70  known to a specific instrument it can still be included in this list
71  since the data package is the source of truth.
72  """
73 
74  @property
75  @abstractmethod
76  def filterDefinitions(self):
77  """`~lsst.obs.base.FilterDefinitionCollection`, defining the filters
78  for this instrument.
79  """
80  return None
81 
82  def __init__(self, *args, **kwargs):
83  self.filterDefinitions.reset()
84  self.filterDefinitions.defineFilters()
85  self._obsDataPackageDir = None
86 
87  @classmethod
88  @abstractmethod
89  def getName(cls):
90  raise NotImplementedError()
91 
92  @abstractmethod
93  def getCamera(self):
94  """Retrieve the cameraGeom representation of this instrument.
95 
96  This is a temporary API that should go away once obs_ packages have
97  a standardized approach to writing versioned cameras to a Gen3 repo.
98  """
99  raise NotImplementedError()
100 
101  @abstractmethod
102  def register(self, registry):
103  """Insert instrument, physical_filter, and detector entries into a
104  `Registry`.
105  """
106  raise NotImplementedError()
107 
108  @property
109  def obsDataPackageDir(self):
110  if self.obsDataPackage is None:
111  return None
112  if self._obsDataPackageDir is None:
113  # Defer any problems with locating the package until
114  # we need to find it.
115  self._obsDataPackageDir = getPackageDir(self.obsDataPackage)
116  return self._obsDataPackageDir
117 
118  def _registerFilters(self, registry):
119  """Register the physical and abstract filter Dimension relationships.
120  This should be called in the ``register`` implementation.
121 
122  Parameters
123  ----------
124  registry : `lsst.daf.butler.core.Registry`
125  The registry to add dimensions to.
126  """
127  for filter in self.filterDefinitions:
128  # fix for undefined abstract filters causing trouble in the registry:
129  if filter.abstract_filter is None:
130  abstract_filter = filter.physical_filter
131  else:
132  abstract_filter = filter.abstract_filter
133 
134  registry.insertDimensionData("physical_filter",
135  {"instrument": self.getName(),
136  "name": filter.physical_filter,
137  "abstract_filter": abstract_filter
138  })
139 
140  @abstractmethod
141  def getRawFormatter(self, dataId):
142  """Return the Formatter class that should be used to read a particular
143  raw file.
144 
145  Parameters
146  ----------
147  dataId : `DataCoordinate`
148  Dimension-based ID for the raw file or files being ingested.
149 
150  Returns
151  -------
152  formatter : `Formatter` class
153  Class to be used that reads the file into an
154  `lsst.afw.image.Exposure` instance.
155  """
156  raise NotImplementedError()
157 
158  def writeCuratedCalibrations(self, butler):
159  """Write human-curated calibration Datasets to the given Butler with
160  the appropriate validity ranges.
161 
162  Parameters
163  ----------
164  butler : `lsst.daf.butler.Butler`
165  Butler to use to store these calibrations.
166 
167  Notes
168  -----
169  Expected to be called from subclasses. The base method calls
170  ``writeCameraGeom`` and ``writeStandardTextCuratedCalibrations``.
171  """
172  self.writeCameraGeom(butler)
174 
175  def applyConfigOverrides(self, name, config):
176  """Apply instrument-specific overrides for a task config.
177 
178  Parameters
179  ----------
180  name : `str`
181  Name of the object being configured; typically the _DefaultName
182  of a Task.
183  config : `lsst.pex.config.Config`
184  Config instance to which overrides should be applied.
185  """
186  for root in self.configPaths:
187  path = os.path.join(root, f"{name}.py")
188  if os.path.exists(path):
189  config.load(path)
190 
191  def writeCameraGeom(self, butler):
192  """Write the default camera geometry to the butler repository
193  with an infinite validity range.
194 
195  Parameters
196  ----------
197  butler : `lsst.daf.butler.Butler`
198  Butler to receive these calibration datasets.
199  """
200 
201  datasetType = DatasetType("camera", ("instrument", "calibration_label"), "Camera",
202  universe=butler.registry.dimensions)
203  butler.registry.registerDatasetType(datasetType)
204  unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName())
205  camera = self.getCamera()
206  butler.put(camera, datasetType, unboundedDataId)
207 
209  """Write the set of standardized curated text calibrations to
210  the repository.
211 
212  Parameters
213  ----------
214  butler : `lsst.daf.butler.Butler`
215  Butler to receive these calibration datasets.
216  """
217 
218  for datasetTypeName in self.standardCuratedDatasetTypes:
219  # We need to define the dataset types.
220  if datasetTypeName not in StandardCuratedCalibrationDatasetTypes:
221  raise ValueError(f"DatasetType {datasetTypeName} not in understood list"
222  f" [{'.'.join(StandardCuratedCalibrationDatasetTypes)}]")
223  definition = StandardCuratedCalibrationDatasetTypes[datasetTypeName]
224  datasetType = DatasetType(datasetTypeName,
225  universe=butler.registry.dimensions,
226  **definition)
227  self._writeSpecificCuratedCalibrationDatasets(butler, datasetType)
228 
229  def _writeSpecificCuratedCalibrationDatasets(self, butler, datasetType):
230  """Write standardized curated calibration datasets for this specific
231  dataset type from an obs data package.
232 
233  Parameters
234  ----------
235  butler : `lsst.daf.butler.Butler`
236  Gen3 butler in which to put the calibrations.
237  datasetType : `lsst.daf.butler.DatasetType`
238  Dataset type to be put.
239 
240  Notes
241  -----
242  This method scans the location defined in the ``obsDataPackageDir``
243  class attribute for curated calibrations corresponding to the
244  supplied dataset type. The directory name in the data package must
245  match the name of the dataset type. They are assumed to use the
246  standard layout and can be read by
247  `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard
248  metadata.
249  """
250  if self.obsDataPackageDir is None:
251  # if there is no data package then there can't be datasets
252  return
253 
254  calibPath = os.path.join(self.obsDataPackageDir, self.policyName,
255  datasetType.name)
256 
257  if not os.path.exists(calibPath):
258  return
259 
260  # Register the dataset type
261  butler.registry.registerDatasetType(datasetType)
262 
263  # obs_base can't depend on pipe_tasks but concrete obs packages
264  # can -- we therefore have to defer import
265  from lsst.pipe.tasks.read_curated_calibs import read_all
266 
267  camera = self.getCamera()
268  calibsDict = read_all(calibPath, camera)[0] # second return is calib type
269  endOfTime = TIMESPAN_MAX
270  dimensionRecords = []
271  datasetRecords = []
272  for det in calibsDict:
273  times = sorted([k for k in calibsDict[det]])
274  calibs = [calibsDict[det][time] for time in times]
275  times = [astropy.time.Time(t, format="datetime", scale="utc") for t in times]
276  times += [endOfTime]
277  for calib, beginTime, endTime in zip(calibs, times[:-1], times[1:]):
278  md = calib.getMetadata()
279  calibrationLabel = f"{datasetType.name}/{md['CALIBDATE']}/{md['DETECTOR']}"
280  dataId = DataCoordinate.standardize(
281  universe=butler.registry.dimensions,
282  instrument=self.getName(),
283  calibration_label=calibrationLabel,
284  detector=md["DETECTOR"],
285  )
286  datasetRecords.append((calib, dataId))
287  dimensionRecords.append({
288  "instrument": self.getName(),
289  "name": calibrationLabel,
290  "datetime_begin": beginTime,
291  "datetime_end": endTime,
292  })
293 
294  # Second loop actually does the inserts and filesystem writes.
295  with butler.transaction():
296  butler.registry.insertDimensionData("calibration_label", *dimensionRecords)
297  # TODO: vectorize these puts, once butler APIs for that become
298  # available.
299  for calib, dataId in datasetRecords:
300  butler.put(calib, datasetType, dataId)
301 
302 
303 def makeExposureRecordFromObsInfo(obsInfo, universe):
304  """Construct an exposure DimensionRecord from
305  `astro_metadata_translator.ObservationInfo`.
306 
307  Parameters
308  ----------
309  obsInfo : `astro_metadata_translator.ObservationInfo`
310  A `~astro_metadata_translator.ObservationInfo` object corresponding to
311  the exposure.
312  universe : `DimensionUniverse`
313  Set of all known dimensions.
314 
315  Returns
316  -------
317  record : `DimensionRecord`
318  A record containing exposure metadata, suitable for insertion into
319  a `Registry`.
320  """
321  dimension = universe["exposure"]
322  return dimension.RecordClass.fromDict({
323  "instrument": obsInfo.instrument,
324  "id": obsInfo.exposure_id,
325  "name": obsInfo.observation_id,
326  "group": obsInfo.exposure_group,
327  "datetime_begin": obsInfo.datetime_begin,
328  "datetime_end": obsInfo.datetime_end,
329  "exposure_time": obsInfo.exposure_time.to_value("s"),
330  "dark_time": obsInfo.dark_time.to_value("s"),
331  "observation_type": obsInfo.observation_type,
332  "physical_filter": obsInfo.physical_filter,
333  "visit": obsInfo.visit_id,
334  })
335 
336 
337 def makeVisitRecordFromObsInfo(obsInfo, universe, *, region=None):
338  """Construct a visit `DimensionRecord` from
339  `astro_metadata_translator.ObservationInfo`.
340 
341  Parameters
342  ----------
343  obsInfo : `astro_metadata_translator.ObservationInfo`
344  A `~astro_metadata_translator.ObservationInfo` object corresponding to
345  the exposure.
346  universe : `DimensionUniverse`
347  Set of all known dimensions.
348  region : `lsst.sphgeom.Region`, optional
349  Spatial region for the visit.
350 
351  Returns
352  -------
353  record : `DimensionRecord`
354  A record containing visit metadata, suitable for insertion into a
355  `Registry`.
356  """
357  dimension = universe["visit"]
358  return dimension.RecordClass.fromDict({
359  "instrument": obsInfo.instrument,
360  "id": obsInfo.visit_id,
361  "name": obsInfo.observation_id,
362  "datetime_begin": obsInfo.datetime_begin,
363  "datetime_end": obsInfo.datetime_end,
364  "exposure_time": obsInfo.exposure_time.to_value("s"),
365  "physical_filter": obsInfo.physical_filter,
366  "region": region,
367  })
368 
369 
370 def addUnboundedCalibrationLabel(registry, instrumentName):
371  """Add a special 'unbounded' calibration_label dimension entry for the
372  given camera that is valid for any exposure.
373 
374  If such an entry already exists, this function just returns a `DataId`
375  for the existing entry.
376 
377  Parameters
378  ----------
379  registry : `Registry`
380  Registry object in which to insert the dimension entry.
381  instrumentName : `str`
382  Name of the instrument this calibration label is associated with.
383 
384  Returns
385  -------
386  dataId : `DataId`
387  New or existing data ID for the unbounded calibration.
388  """
389  d = dict(instrument=instrumentName, calibration_label="unbounded")
390  try:
391  return registry.expandDataId(d)
392  except LookupError:
393  pass
394  entry = d.copy()
395  entry["datetime_begin"] = TIMESPAN_MIN
396  entry["datetime_end"] = TIMESPAN_MAX
397  registry.insertDimensionData("calibration_label", entry)
398  return registry.expandDataId(d)
def applyConfigOverrides(self, name, config)
Definition: instrument.py:175
def makeExposureRecordFromObsInfo(obsInfo, universe)
Definition: instrument.py:303
def writeStandardTextCuratedCalibrations(self, butler)
Definition: instrument.py:208
def makeVisitRecordFromObsInfo(obsInfo, universe, region=None)
Definition: instrument.py:337
def __init__(self, args, kwargs)
Definition: instrument.py:82
def addUnboundedCalibrationLabel(registry, instrumentName)
Definition: instrument.py:370
def _writeSpecificCuratedCalibrationDatasets(self, butler, datasetType)
Definition: instrument.py:229
def writeCuratedCalibrations(self, butler)
Definition: instrument.py:158