22 from __future__
import annotations
24 __all__ = (
"Instrument",
"makeExposureRecordFromObsInfo",
"addUnboundedCalibrationLabel",
"loadCamera")
27 from abc
import ABCMeta, abstractmethod
28 from typing
import Any, Tuple, TYPE_CHECKING
31 from lsst.afw.cameraGeom
import Camera
32 from lsst.daf.butler
import (
44 from .gen2to3
import TranslatorFactory
45 from lsst.daf.butler
import Registry
49 StandardCuratedCalibrationDatasetTypes = {
50 "defects": {
"dimensions": (
"instrument",
"detector",
"calibration_label"),
51 "storageClass":
"Defects"},
52 "qe_curve": {
"dimensions": (
"instrument",
"detector",
"calibration_label"),
53 "storageClass":
"QECurve"},
54 "crosstalk": {
"dimensions": (
"instrument",
"detector",
"calibration_label"),
55 "storageClass":
"CrosstalkCalib"},
60 """Base class for instrument-specific logic for the Gen3 Butler.
62 Concrete instrument subclasses should be directly constructable with no
67 """Paths to config files to read for specific Tasks.
69 The paths in this list should contain files of the form `task.py`, for
70 each of the Tasks that requires special configuration.
74 """Instrument specific name to use when locating a policy or configuration
75 file in the file system."""
78 """Name of the package containing the text curated calibration files.
79 Usually a obs _data package. If `None` no curated calibration files
80 will be read. (`str`)"""
82 standardCuratedDatasetTypes = tuple(StandardCuratedCalibrationDatasetTypes)
83 """The dataset types expected to be obtained from the obsDataPackage.
84 These dataset types are all required to have standard definitions and
85 must be known to the base class. Clearing this list will prevent
86 any of these calibrations from being stored. If a dataset type is not
87 known to a specific instrument it can still be included in this list
88 since the data package is the source of truth.
94 """`~lsst.obs.base.FilterDefinitionCollection`, defining the filters
107 """Return the short (dimension) name for this instrument.
109 This is not (in general) the same as the class name - it's what is used
110 as the value of the "instrument" field in data IDs, and is usually an
111 abbreviation of the full name.
113 raise NotImplementedError()
117 """Retrieve the cameraGeom representation of this instrument.
119 This is a temporary API that should go away once ``obs_`` packages have
120 a standardized approach to writing versioned cameras to a Gen3 repo.
122 raise NotImplementedError()
126 """Insert instrument, physical_filter, and detector entries into a
129 raise NotImplementedError()
133 """The root of the obs package that provides specializations for
134 this instrument (`str`).
145 def fromName(name: str, registry: Registry) -> Instrument:
146 """Given an instrument name and a butler, retrieve a corresponding
147 instantiated instrument object.
152 Name of the instrument (must match the return value of `getName`).
153 registry : `lsst.daf.butler.Registry`
154 Butler registry to query to find the information.
158 instrument : `Instrument`
159 An instance of the relevant `Instrument`.
163 The instrument must be registered in the corresponding butler.
168 Raised if the instrument is not known to the supplied registry.
170 Raised if the class could not be imported. This could mean
171 that the relevant obs package has not been setup.
173 Raised if the class name retrieved is not a string.
175 dimensions = list(registry.queryDimensions(
"instrument", dataId={
"instrument": name}))
176 cls = dimensions[0].records[
"instrument"].class_name
177 if not isinstance(cls, str):
178 raise TypeError(f
"Unexpected class name retrieved from {name} instrument dimension (got {cls})")
179 instrument = doImport(cls)
184 """Import all the instruments known to this registry.
186 This will ensure that all metadata translators have been registered.
190 registry : `lsst.daf.butler.Registry`
191 Butler registry to query to find the information.
195 It is allowed for a particular instrument class to fail on import.
196 This might simply indicate that a particular obs package has
199 dimensions = list(registry.queryDimensions(
"instrument"))
200 for dim
in dimensions:
201 cls = dim.records[
"instrument"].class_name
207 def _registerFilters(self, registry):
208 """Register the physical and abstract filter Dimension relationships.
209 This should be called in the ``register`` implementation.
213 registry : `lsst.daf.butler.core.Registry`
214 The registry to add dimensions to.
218 if filter.abstract_filter
is None:
219 abstract_filter = filter.physical_filter
221 abstract_filter = filter.abstract_filter
223 registry.insertDimensionData(
"physical_filter",
225 "name": filter.physical_filter,
226 "abstract_filter": abstract_filter
231 """Return the Formatter class that should be used to read a particular
236 dataId : `DataCoordinate`
237 Dimension-based ID for the raw file or files being ingested.
241 formatter : `Formatter` class
242 Class to be used that reads the file into an
243 `lsst.afw.image.Exposure` instance.
245 raise NotImplementedError()
248 """Write human-curated calibration Datasets to the given Butler with
249 the appropriate validity ranges.
253 butler : `lsst.daf.butler.Butler`
254 Butler to use to store these calibrations.
256 Run to use for this collection of calibrations. If `None` the
257 collection name is worked out automatically from the instrument
258 name and other metadata.
262 Expected to be called from subclasses. The base method calls
263 ``writeCameraGeom`` and ``writeStandardTextCuratedCalibrations``.
270 butler.registry.registerCollection(run, type=CollectionType.RUN)
276 """Write additional curated calibrations that might be instrument
277 specific and are not part of the standard set.
279 Default implementation does nothing.
283 butler : `lsst.daf.butler.Butler`
284 Butler to use to store these calibrations.
285 run : `str`, optional
286 Name of the run to use to override the default run associated
292 """Apply instrument-specific overrides for a task config.
297 Name of the object being configured; typically the _DefaultName
299 config : `lsst.pex.config.Config`
300 Config instance to which overrides should be applied.
302 for root
in self.configPaths:
303 path = os.path.join(root, f
"{name}.py")
304 if os.path.exists(path):
308 """Write the default camera geometry to the butler repository
309 with an infinite validity range.
313 butler : `lsst.daf.butler.Butler`
314 Butler to receive these calibration datasets.
315 run : `str`, optional
316 Name of the run to use to override the default run associated
320 datasetType = DatasetType(
"camera", (
"instrument",
"calibration_label"),
"Camera",
321 universe=butler.registry.dimensions)
322 butler.registry.registerDatasetType(datasetType)
325 butler.put(camera, datasetType, unboundedDataId, run=run)
328 """Write the set of standardized curated text calibrations to
333 butler : `lsst.daf.butler.Butler`
334 Butler to receive these calibration datasets.
335 run : `str`, optional
336 Name of the run to use to override the default run associated
342 if datasetTypeName
not in StandardCuratedCalibrationDatasetTypes:
343 raise ValueError(f
"DatasetType {datasetTypeName} not in understood list"
344 f
" [{'.'.join(StandardCuratedCalibrationDatasetTypes)}]")
345 definition = StandardCuratedCalibrationDatasetTypes[datasetTypeName]
346 datasetType = DatasetType(datasetTypeName,
347 universe=butler.registry.dimensions,
351 def _writeSpecificCuratedCalibrationDatasets(self, butler, datasetType, run=None):
352 """Write standardized curated calibration datasets for this specific
353 dataset type from an obs data package.
357 butler : `lsst.daf.butler.Butler`
358 Gen3 butler in which to put the calibrations.
359 datasetType : `lsst.daf.butler.DatasetType`
360 Dataset type to be put.
361 run : `str`, optional
362 Name of the run to use to override the default run associated
367 This method scans the location defined in the ``obsDataPackageDir``
368 class attribute for curated calibrations corresponding to the
369 supplied dataset type. The directory name in the data package must
370 match the name of the dataset type. They are assumed to use the
371 standard layout and can be read by
372 `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard
382 if not os.path.exists(calibPath):
386 butler.registry.registerDatasetType(datasetType)
390 from lsst.pipe.tasks.read_curated_calibs
import read_all
393 calibsDict = read_all(calibPath, camera)[0]
394 endOfTime = TIMESPAN_MAX
395 dimensionRecords = []
397 for det
in calibsDict:
398 times = sorted([k
for k
in calibsDict[det]])
399 calibs = [calibsDict[det][time]
for time
in times]
400 times = [astropy.time.Time(t, format=
"datetime", scale=
"utc")
for t
in times]
402 for calib, beginTime, endTime
in zip(calibs, times[:-1], times[1:]):
403 md = calib.getMetadata()
404 calibrationLabel = f
"{datasetType.name}/{md['CALIBDATE']}/{md['DETECTOR']}"
405 dataId = DataCoordinate.standardize(
406 universe=butler.registry.dimensions,
408 calibration_label=calibrationLabel,
409 detector=md[
"DETECTOR"],
411 datasetRecords.append((calib, dataId))
412 dimensionRecords.append({
414 "name": calibrationLabel,
415 "datetime_begin": beginTime,
416 "datetime_end": endTime,
420 with butler.transaction():
421 butler.registry.insertDimensionData(
"calibration_label", *dimensionRecords)
424 for calib, dataId
in datasetRecords:
425 butler.put(calib, datasetType, dataId, run=run)
429 """Return a factory for creating Gen2->Gen3 data ID translators,
430 specialized for this instrument.
432 Derived class implementations should generally call
433 `TranslatorFactory.addGenericInstrumentRules` with appropriate
434 arguments, but are not required to (and may not be able to if their
435 Gen2 raw data IDs are sufficiently different from the HSC/DECam/CFHT
440 factory : `TranslatorFactory`.
441 Factory for `Translator` objects.
443 raise NotImplementedError(
"Must be implemented by derived classes.")
447 """Make the default instrument-specific run collection string for raw
453 Run collection name to be used as the default for ingestion of
460 """Get the instrument-specific collection string to use as derived
461 from the supplied label.
466 String to be combined with the instrument name to form a
472 Collection name to use that includes the instrument name.
474 return f
"{cls.getName()}/{label}"
478 """Construct an exposure DimensionRecord from
479 `astro_metadata_translator.ObservationInfo`.
483 obsInfo : `astro_metadata_translator.ObservationInfo`
484 A `~astro_metadata_translator.ObservationInfo` object corresponding to
486 universe : `DimensionUniverse`
487 Set of all known dimensions.
491 record : `DimensionRecord`
492 A record containing exposure metadata, suitable for insertion into
495 dimension = universe[
"exposure"]
496 return dimension.RecordClass.fromDict({
497 "instrument": obsInfo.instrument,
498 "id": obsInfo.exposure_id,
499 "name": obsInfo.observation_id,
500 "group_name": obsInfo.exposure_group,
501 "group_id": obsInfo.visit_id,
502 "datetime_begin": obsInfo.datetime_begin,
503 "datetime_end": obsInfo.datetime_end,
504 "exposure_time": obsInfo.exposure_time.to_value(
"s"),
505 "dark_time": obsInfo.dark_time.to_value(
"s"),
506 "observation_type": obsInfo.observation_type,
507 "physical_filter": obsInfo.physical_filter,
512 """Add a special 'unbounded' calibration_label dimension entry for the
513 given camera that is valid for any exposure.
515 If such an entry already exists, this function just returns a `DataId`
516 for the existing entry.
520 registry : `Registry`
521 Registry object in which to insert the dimension entry.
522 instrumentName : `str`
523 Name of the instrument this calibration label is associated with.
528 New or existing data ID for the unbounded calibration.
530 d = dict(instrument=instrumentName, calibration_label=
"unbounded")
532 return registry.expandDataId(d)
536 entry[
"datetime_begin"] = TIMESPAN_MIN
537 entry[
"datetime_end"] = TIMESPAN_MAX
538 registry.insertDimensionData(
"calibration_label", entry)
539 return registry.expandDataId(d)
542 def loadCamera(butler: Butler, dataId: DataId, *, collections: Any =
None) -> Tuple[Camera, bool]:
543 """Attempt to load versioned camera geometry from a butler, but fall back
544 to obtaining a nominal camera from the `Instrument` class if that fails.
548 butler : `lsst.daf.butler.Butler`
549 Butler instance to attempt to query for and load a ``camera`` dataset
551 dataId : `dict` or `DataCoordinate`
552 Data ID that identifies at least the ``instrument`` and ``exposure``
554 collections : Any, optional
555 Collections to be searched, overriding ``self.butler.collections``.
556 Can be any of the types supported by the ``collections`` argument
557 to butler construction.
561 camera : `lsst.afw.cameraGeom.Camera`
564 If `True`, the camera was obtained from the butler and should represent
565 a versioned camera from a calibration repository. If `False`, no
566 camera datasets were found, and the returned camera was produced by
567 instantiating the appropriate `Instrument` class and calling
568 `Instrument.getCamera`.
570 if collections
is None:
571 collections = butler.collections
576 dataId = butler.registry.expandDataId(dataId, graph=butler.registry.dimensions[
"exposure"].graph)
577 cameraRefs = list(butler.registry.queryDatasets(
"camera", dataId=dataId, collections=collections,
580 assert len(cameraRefs) == 1,
"Should be guaranteed by deduplicate=True above."
581 return butler.getDirect(cameraRefs[0]),
True
582 instrument = Instrument.fromName(dataId[
"instrument"], butler.registry)
583 return instrument.getCamera(),
False