Coverage for python/lsst/obs/base/_instrument.py : 23%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ("Instrument", "makeExposureRecordFromObsInfo", "addUnboundedCalibrationLabel", "loadCamera")
26import os.path
27from abc import ABCMeta, abstractmethod
28from typing import Any, Tuple, TYPE_CHECKING
29import astropy.time
31from lsst.afw.cameraGeom import Camera
32from lsst.daf.butler import Butler, DataId, TIMESPAN_MIN, TIMESPAN_MAX, DatasetType, DataCoordinate
33from lsst.utils import getPackageDir, doImport
35if TYPE_CHECKING: 35 ↛ 36line 35 didn't jump to line 36, because the condition on line 35 was never true
36 from .gen2to3 import TranslatorFactory
37 from lsst.daf.butler import Registry
39# To be a standard text curated calibration means that we use a
40# standard definition for the corresponding DatasetType.
41StandardCuratedCalibrationDatasetTypes = {
42 "defects": {"dimensions": ("instrument", "detector", "calibration_label"),
43 "storageClass": "Defects"},
44 "qe_curve": {"dimensions": ("instrument", "detector", "calibration_label"),
45 "storageClass": "QECurve"},
46}
49class Instrument(metaclass=ABCMeta):
50 """Base class for instrument-specific logic for the Gen3 Butler.
52 Concrete instrument subclasses should be directly constructable with no
53 arguments.
54 """
56 configPaths = ()
57 """Paths to config files to read for specific Tasks.
59 The paths in this list should contain files of the form `task.py`, for
60 each of the Tasks that requires special configuration.
61 """
63 policyName = None
64 """Instrument specific name to use when locating a policy or configuration
65 file in the file system."""
67 obsDataPackage = None
68 """Name of the package containing the text curated calibration files.
69 Usually a obs _data package. If `None` no curated calibration files
70 will be read. (`str`)"""
72 standardCuratedDatasetTypes = tuple(StandardCuratedCalibrationDatasetTypes)
73 """The dataset types expected to be obtained from the obsDataPackage.
74 These dataset types are all required to have standard definitions and
75 must be known to the base class. Clearing this list will prevent
76 any of these calibrations from being stored. If a dataset type is not
77 known to a specific instrument it can still be included in this list
78 since the data package is the source of truth.
79 """
81 @property
82 @abstractmethod
83 def filterDefinitions(self):
84 """`~lsst.obs.base.FilterDefinitionCollection`, defining the filters
85 for this instrument.
86 """
87 return None
89 def __init__(self):
90 self.filterDefinitions.reset()
91 self.filterDefinitions.defineFilters()
92 self._obsDataPackageDir = None
94 @classmethod
95 @abstractmethod
96 def getName(cls):
97 """Return the short (dimension) name for this instrument.
99 This is not (in general) the same as the class name - it's what is used
100 as the value of the "instrument" field in data IDs, and is usually an
101 abbreviation of the full name.
102 """
103 raise NotImplementedError()
105 @abstractmethod
106 def getCamera(self):
107 """Retrieve the cameraGeom representation of this instrument.
109 This is a temporary API that should go away once obs_ packages have
110 a standardized approach to writing versioned cameras to a Gen3 repo.
111 """
112 raise NotImplementedError()
114 @abstractmethod
115 def register(self, registry):
116 """Insert instrument, physical_filter, and detector entries into a
117 `Registry`.
118 """
119 raise NotImplementedError()
121 @property
122 def obsDataPackageDir(self):
123 """The root of the obs package that provides specializations for
124 this instrument (`str`).
125 """
126 if self.obsDataPackage is None:
127 return None
128 if self._obsDataPackageDir is None:
129 # Defer any problems with locating the package until
130 # we need to find it.
131 self._obsDataPackageDir = getPackageDir(self.obsDataPackage)
132 return self._obsDataPackageDir
134 @staticmethod
135 def fromName(name: str, registry: Registry) -> Instrument:
136 """Given an instrument name and a butler, retrieve a corresponding
137 instantiated instrument object.
139 Parameters
140 ----------
141 name : `str`
142 Name of the instrument (must match the return value of `getName`).
143 registry : `lsst.daf.butler.Registry`
144 Butler registry to query to find the information.
146 Returns
147 -------
148 instrument : `Instrument`
149 An instance of the relevant `Instrument`.
151 Notes
152 -----
153 The instrument must be registered in the corresponding butler.
155 Raises
156 ------
157 LookupError
158 Raised if the instrument is not known to the supplied registry.
159 ModuleNotFoundError
160 Raised if the class could not be imported. This could mean
161 that the relevant obs package has not been setup.
162 TypeError
163 Raised if the class name retrieved is not a string.
164 """
165 dimensions = list(registry.queryDimensions("instrument", dataId={"instrument": name}))
166 cls = dimensions[0].records["instrument"].class_name
167 if not isinstance(cls, str):
168 raise TypeError(f"Unexpected class name retrieved from {name} instrument dimension (got {cls})")
169 instrument = doImport(cls)
170 return instrument()
172 @staticmethod
173 def importAll(registry: Registry) -> None:
174 """Import all the instruments known to this registry.
176 This will ensure that all metadata translators have been registered.
178 Parameters
179 ----------
180 registry : `lsst.daf.butler.Registry`
181 Butler registry to query to find the information.
183 Notes
184 -----
185 It is allowed for a particular instrument class to fail on import.
186 This might simply indicate that a particular obs package has
187 not been setup.
188 """
189 dimensions = list(registry.queryDimensions("instrument"))
190 for dim in dimensions:
191 cls = dim.records["instrument"].class_name
192 try:
193 doImport(cls)
194 except Exception:
195 pass
197 def _registerFilters(self, registry):
198 """Register the physical and abstract filter Dimension relationships.
199 This should be called in the ``register`` implementation.
201 Parameters
202 ----------
203 registry : `lsst.daf.butler.core.Registry`
204 The registry to add dimensions to.
205 """
206 for filter in self.filterDefinitions:
207 # fix for undefined abstract filters causing trouble in the registry:
208 if filter.abstract_filter is None:
209 abstract_filter = filter.physical_filter
210 else:
211 abstract_filter = filter.abstract_filter
213 registry.insertDimensionData("physical_filter",
214 {"instrument": self.getName(),
215 "name": filter.physical_filter,
216 "abstract_filter": abstract_filter
217 })
219 @abstractmethod
220 def getRawFormatter(self, dataId):
221 """Return the Formatter class that should be used to read a particular
222 raw file.
224 Parameters
225 ----------
226 dataId : `DataCoordinate`
227 Dimension-based ID for the raw file or files being ingested.
229 Returns
230 -------
231 formatter : `Formatter` class
232 Class to be used that reads the file into an
233 `lsst.afw.image.Exposure` instance.
234 """
235 raise NotImplementedError()
237 def writeCuratedCalibrations(self, butler):
238 """Write human-curated calibration Datasets to the given Butler with
239 the appropriate validity ranges.
241 Parameters
242 ----------
243 butler : `lsst.daf.butler.Butler`
244 Butler to use to store these calibrations.
246 Notes
247 -----
248 Expected to be called from subclasses. The base method calls
249 ``writeCameraGeom`` and ``writeStandardTextCuratedCalibrations``.
250 """
251 self.writeCameraGeom(butler)
252 self.writeStandardTextCuratedCalibrations(butler)
254 def applyConfigOverrides(self, name, config):
255 """Apply instrument-specific overrides for a task config.
257 Parameters
258 ----------
259 name : `str`
260 Name of the object being configured; typically the _DefaultName
261 of a Task.
262 config : `lsst.pex.config.Config`
263 Config instance to which overrides should be applied.
264 """
265 for root in self.configPaths:
266 path = os.path.join(root, f"{name}.py")
267 if os.path.exists(path):
268 config.load(path)
270 def writeCameraGeom(self, butler):
271 """Write the default camera geometry to the butler repository
272 with an infinite validity range.
274 Parameters
275 ----------
276 butler : `lsst.daf.butler.Butler`
277 Butler to receive these calibration datasets.
278 """
280 datasetType = DatasetType("camera", ("instrument", "calibration_label"), "Camera",
281 universe=butler.registry.dimensions)
282 butler.registry.registerDatasetType(datasetType)
283 unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName())
284 camera = self.getCamera()
285 butler.put(camera, datasetType, unboundedDataId)
287 def writeStandardTextCuratedCalibrations(self, butler):
288 """Write the set of standardized curated text calibrations to
289 the repository.
291 Parameters
292 ----------
293 butler : `lsst.daf.butler.Butler`
294 Butler to receive these calibration datasets.
295 """
297 for datasetTypeName in self.standardCuratedDatasetTypes:
298 # We need to define the dataset types.
299 if datasetTypeName not in StandardCuratedCalibrationDatasetTypes:
300 raise ValueError(f"DatasetType {datasetTypeName} not in understood list"
301 f" [{'.'.join(StandardCuratedCalibrationDatasetTypes)}]")
302 definition = StandardCuratedCalibrationDatasetTypes[datasetTypeName]
303 datasetType = DatasetType(datasetTypeName,
304 universe=butler.registry.dimensions,
305 **definition)
306 self._writeSpecificCuratedCalibrationDatasets(butler, datasetType)
308 def _writeSpecificCuratedCalibrationDatasets(self, butler, datasetType):
309 """Write standardized curated calibration datasets for this specific
310 dataset type from an obs data package.
312 Parameters
313 ----------
314 butler : `lsst.daf.butler.Butler`
315 Gen3 butler in which to put the calibrations.
316 datasetType : `lsst.daf.butler.DatasetType`
317 Dataset type to be put.
319 Notes
320 -----
321 This method scans the location defined in the ``obsDataPackageDir``
322 class attribute for curated calibrations corresponding to the
323 supplied dataset type. The directory name in the data package must
324 match the name of the dataset type. They are assumed to use the
325 standard layout and can be read by
326 `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard
327 metadata.
328 """
329 if self.obsDataPackageDir is None:
330 # if there is no data package then there can't be datasets
331 return
333 calibPath = os.path.join(self.obsDataPackageDir, self.policyName,
334 datasetType.name)
336 if not os.path.exists(calibPath):
337 return
339 # Register the dataset type
340 butler.registry.registerDatasetType(datasetType)
342 # obs_base can't depend on pipe_tasks but concrete obs packages
343 # can -- we therefore have to defer import
344 from lsst.pipe.tasks.read_curated_calibs import read_all
346 camera = self.getCamera()
347 calibsDict = read_all(calibPath, camera)[0] # second return is calib type
348 endOfTime = TIMESPAN_MAX
349 dimensionRecords = []
350 datasetRecords = []
351 for det in calibsDict:
352 times = sorted([k for k in calibsDict[det]])
353 calibs = [calibsDict[det][time] for time in times]
354 times = [astropy.time.Time(t, format="datetime", scale="utc") for t in times]
355 times += [endOfTime]
356 for calib, beginTime, endTime in zip(calibs, times[:-1], times[1:]):
357 md = calib.getMetadata()
358 calibrationLabel = f"{datasetType.name}/{md['CALIBDATE']}/{md['DETECTOR']}"
359 dataId = DataCoordinate.standardize(
360 universe=butler.registry.dimensions,
361 instrument=self.getName(),
362 calibration_label=calibrationLabel,
363 detector=md["DETECTOR"],
364 )
365 datasetRecords.append((calib, dataId))
366 dimensionRecords.append({
367 "instrument": self.getName(),
368 "name": calibrationLabel,
369 "datetime_begin": beginTime,
370 "datetime_end": endTime,
371 })
373 # Second loop actually does the inserts and filesystem writes.
374 with butler.transaction():
375 butler.registry.insertDimensionData("calibration_label", *dimensionRecords)
376 # TODO: vectorize these puts, once butler APIs for that become
377 # available.
378 for calib, dataId in datasetRecords:
379 butler.put(calib, datasetType, dataId)
381 @abstractmethod
382 def makeDataIdTranslatorFactory(self) -> TranslatorFactory:
383 """Return a factory for creating Gen2->Gen3 data ID translators,
384 specialized for this instrument.
386 Derived class implementations should generally call
387 `TranslatorFactory.addGenericInstrumentRules` with appropriate
388 arguments, but are not required to (and may not be able to if their
389 Gen2 raw data IDs are sufficiently different from the HSC/DECam/CFHT
390 norm).
392 Returns
393 -------
394 factory : `TranslatorFactory`.
395 Factory for `Translator` objects.
396 """
397 raise NotImplementedError("Must be implemented by derived classes.")
400def makeExposureRecordFromObsInfo(obsInfo, universe):
401 """Construct an exposure DimensionRecord from
402 `astro_metadata_translator.ObservationInfo`.
404 Parameters
405 ----------
406 obsInfo : `astro_metadata_translator.ObservationInfo`
407 A `~astro_metadata_translator.ObservationInfo` object corresponding to
408 the exposure.
409 universe : `DimensionUniverse`
410 Set of all known dimensions.
412 Returns
413 -------
414 record : `DimensionRecord`
415 A record containing exposure metadata, suitable for insertion into
416 a `Registry`.
417 """
418 dimension = universe["exposure"]
419 return dimension.RecordClass.fromDict({
420 "instrument": obsInfo.instrument,
421 "id": obsInfo.exposure_id,
422 "name": obsInfo.observation_id,
423 "group_name": obsInfo.exposure_group,
424 "group_id": obsInfo.visit_id,
425 "datetime_begin": obsInfo.datetime_begin,
426 "datetime_end": obsInfo.datetime_end,
427 "exposure_time": obsInfo.exposure_time.to_value("s"),
428 "dark_time": obsInfo.dark_time.to_value("s"),
429 "observation_type": obsInfo.observation_type,
430 "physical_filter": obsInfo.physical_filter,
431 })
434def addUnboundedCalibrationLabel(registry, instrumentName):
435 """Add a special 'unbounded' calibration_label dimension entry for the
436 given camera that is valid for any exposure.
438 If such an entry already exists, this function just returns a `DataId`
439 for the existing entry.
441 Parameters
442 ----------
443 registry : `Registry`
444 Registry object in which to insert the dimension entry.
445 instrumentName : `str`
446 Name of the instrument this calibration label is associated with.
448 Returns
449 -------
450 dataId : `DataId`
451 New or existing data ID for the unbounded calibration.
452 """
453 d = dict(instrument=instrumentName, calibration_label="unbounded")
454 try:
455 return registry.expandDataId(d)
456 except LookupError:
457 pass
458 entry = d.copy()
459 entry["datetime_begin"] = TIMESPAN_MIN
460 entry["datetime_end"] = TIMESPAN_MAX
461 registry.insertDimensionData("calibration_label", entry)
462 return registry.expandDataId(d)
465def loadCamera(butler: Butler, dataId: DataId, *, collections: Any = None) -> Tuple[Camera, bool]:
466 """Attempt to load versioned camera geometry from a butler, but fall back
467 to obtaining a nominal camera from the `Instrument` class if that fails.
469 Parameters
470 ----------
471 butler : `lsst.daf.butler.Butler`
472 Butler instance to attempt to query for and load a ``camera`` dataset
473 from.
474 dataId : `dict` or `DataCoordinate`
475 Data ID that identifies at least the ``instrument`` and ``exposure``
476 dimensions.
477 collections : Any, optional
478 Collections to be searched, overriding ``self.butler.collections``.
479 Can be any of the types supported by the ``collections`` argument
480 to butler construction.
482 Returns
483 -------
484 camera : `lsst.afw.cameraGeom.Camera`
485 Camera object.
486 versioned : `bool`
487 If `True`, the camera was obtained from the butler and should represent
488 a versioned camera from a calibration repository. If `False`, no
489 camera datasets were found, and the returned camera was produced by
490 instantiating the appropriate `Instrument` class and calling
491 `Instrument.getCamera`.
492 """
493 if collections is None:
494 collections = butler.collections
495 # Registry would do data ID expansion internally if we didn't do it first,
496 # but we might want an expanded data ID ourselves later, so we do it here
497 # to ensure it only happens once.
498 # This will also catch problems with the data ID not having keys we need.
499 dataId = butler.registry.expandDataId(dataId, graph=butler.registry.dimensions["exposure"].graph)
500 cameraRefs = list(butler.registry.queryDatasets("camera", dataId=dataId, collections=collections,
501 deduplicate=True))
502 if cameraRefs:
503 assert len(cameraRefs) == 1, "Should be guaranteed by deduplicate=True above."
504 return butler.getDirect(cameraRefs[0]), True
505 instrument = Instrument.fromName(dataId["instrument"], butler.registry)
506 return instrument.getCamera(), False