22 from __future__
import annotations
24 __all__ = (
"Instrument",
"makeExposureRecordFromObsInfo",
"addUnboundedCalibrationLabel",
"loadCamera")
27 from abc
import ABCMeta, abstractmethod
28 from typing
import Any, Optional, Set, Sequence, Tuple, TYPE_CHECKING
30 from functools
import lru_cache
32 from lsst.afw.cameraGeom
import Camera
33 from lsst.daf.butler
import (
44 from .gen2to3
import TranslatorFactory
45 from lsst.daf.butler
import Registry
49 StandardCuratedCalibrationDatasetTypes = {
50 "defects": {
"dimensions": (
"instrument",
"detector",
"calibration_label"),
51 "storageClass":
"Defects"},
52 "qe_curve": {
"dimensions": (
"instrument",
"detector",
"calibration_label"),
53 "storageClass":
"QECurve"},
54 "crosstalk": {
"dimensions": (
"instrument",
"detector",
"calibration_label"),
55 "storageClass":
"CrosstalkCalib"},
60 """Base class for instrument-specific logic for the Gen3 Butler.
62 Concrete instrument subclasses should be directly constructable with no
66 configPaths: Sequence[str] = ()
67 """Paths to config files to read for specific Tasks.
69 The paths in this list should contain files of the form `task.py`, for
70 each of the Tasks that requires special configuration.
73 policyName: Optional[str] =
None
74 """Instrument specific name to use when locating a policy or configuration
75 file in the file system."""
77 obsDataPackage: Optional[str] =
None
78 """Name of the package containing the text curated calibration files.
79 Usually a obs _data package. If `None` no curated calibration files
80 will be read. (`str`)"""
82 standardCuratedDatasetTypes: Set[str] = frozenset(StandardCuratedCalibrationDatasetTypes)
83 """The dataset types expected to be obtained from the obsDataPackage.
85 These dataset types are all required to have standard definitions and
86 must be known to the base class. Clearing this list will prevent
87 any of these calibrations from being stored. If a dataset type is not
88 known to a specific instrument it can still be included in this list
89 since the data package is the source of truth. (`set` of `str`)
92 additionalCuratedDatasetTypes: Set[str] = frozenset()
93 """Curated dataset types specific to this particular instrument that do
94 not follow the standard organization found in obs data packages.
96 These are the instrument-specific dataset types written by
97 `writeAdditionalCuratedCalibrations` in addition to the calibrations
98 found in obs data packages that follow the standard scheme.
104 """`~lsst.obs.base.FilterDefinitionCollection`, defining the filters
116 """Return the short (dimension) name for this instrument.
118 This is not (in general) the same as the class name - it's what is used
119 as the value of the "instrument" field in data IDs, and is usually an
120 abbreviation of the full name.
122 raise NotImplementedError()
127 """Return the names of all the curated calibration dataset types.
131 names : `set` of `str`
132 The dataset type names of all curated calibrations. This will
133 include the standard curated calibrations even if the particular
134 instrument does not support them.
138 The returned list does not indicate whether a particular dataset
139 is present in the Butler repository, simply that these are the
140 dataset types that are handled by ``writeCuratedCalibrations``.
149 for datasetTypeName
in cls.standardCuratedDatasetTypes:
151 if calibPath
is not None:
152 curated.add(datasetTypeName)
154 curated.update(cls.additionalCuratedDatasetTypes)
155 return frozenset(curated)
159 """Retrieve the cameraGeom representation of this instrument.
161 This is a temporary API that should go away once ``obs_`` packages have
162 a standardized approach to writing versioned cameras to a Gen3 repo.
164 raise NotImplementedError()
168 """Insert instrument, physical_filter, and detector entries into a
171 raise NotImplementedError()
176 """The root of the obs data package that provides specializations for
182 The root of the relevat obs data package.
184 if cls.obsDataPackage
is None:
186 return getPackageDir(cls.obsDataPackage)
189 def fromName(name: str, registry: Registry) -> Instrument:
190 """Given an instrument name and a butler, retrieve a corresponding
191 instantiated instrument object.
196 Name of the instrument (must match the return value of `getName`).
197 registry : `lsst.daf.butler.Registry`
198 Butler registry to query to find the information.
202 instrument : `Instrument`
203 An instance of the relevant `Instrument`.
207 The instrument must be registered in the corresponding butler.
212 Raised if the instrument is not known to the supplied registry.
214 Raised if the class could not be imported. This could mean
215 that the relevant obs package has not been setup.
217 Raised if the class name retrieved is not a string.
219 records = list(registry.queryDimensionRecords(
"instrument", instrument=name))
221 raise LookupError(f
"No registered instrument with name '{name}'.")
222 cls = records[0].class_name
223 if not isinstance(cls, str):
224 raise TypeError(f
"Unexpected class name retrieved from {name} instrument dimension (got {cls})")
225 instrument = doImport(cls)
230 """Import all the instruments known to this registry.
232 This will ensure that all metadata translators have been registered.
236 registry : `lsst.daf.butler.Registry`
237 Butler registry to query to find the information.
241 It is allowed for a particular instrument class to fail on import.
242 This might simply indicate that a particular obs package has
245 records = list(registry.queryDimensionRecords(
"instrument"))
246 for record
in records:
247 cls = record.class_name
253 def _registerFilters(self, registry):
254 """Register the physical and abstract filter Dimension relationships.
255 This should be called in the ``register`` implementation.
259 registry : `lsst.daf.butler.core.Registry`
260 The registry to add dimensions to.
264 if filter.band
is None:
265 band = filter.physical_filter
269 registry.insertDimensionData(
"physical_filter",
271 "name": filter.physical_filter,
277 """Return the Formatter class that should be used to read a particular
282 dataId : `DataCoordinate`
283 Dimension-based ID for the raw file or files being ingested.
287 formatter : `Formatter` class
288 Class to be used that reads the file into an
289 `lsst.afw.image.Exposure` instance.
291 raise NotImplementedError()
294 """Write human-curated calibration Datasets to the given Butler with
295 the appropriate validity ranges.
299 butler : `lsst.daf.butler.Butler`
300 Butler to use to store these calibrations.
302 Run to use for this collection of calibrations. If `None` the
303 collection name is worked out automatically from the instrument
304 name and other metadata.
308 Expected to be called from subclasses. The base method calls
309 ``writeCameraGeom`` and ``writeStandardTextCuratedCalibrations``.
316 butler.registry.registerCollection(run, type=CollectionType.RUN)
322 """Write additional curated calibrations that might be instrument
323 specific and are not part of the standard set.
325 Default implementation does nothing.
329 butler : `lsst.daf.butler.Butler`
330 Butler to use to store these calibrations.
331 run : `str`, optional
332 Name of the run to use to override the default run associated
338 """Apply instrument-specific overrides for a task config.
343 Name of the object being configured; typically the _DefaultName
345 config : `lsst.pex.config.Config`
346 Config instance to which overrides should be applied.
348 for root
in self.configPaths:
349 path = os.path.join(root, f
"{name}.py")
350 if os.path.exists(path):
354 """Write the default camera geometry to the butler repository
355 with an infinite validity range.
359 butler : `lsst.daf.butler.Butler`
360 Butler to receive these calibration datasets.
361 run : `str`, optional
362 Name of the run to use to override the default run associated
366 datasetType = DatasetType(
"camera", (
"instrument",
"calibration_label"),
"Camera",
367 universe=butler.registry.dimensions)
368 butler.registry.registerDatasetType(datasetType)
371 butler.put(camera, datasetType, unboundedDataId, run=run)
374 """Write the set of standardized curated text calibrations to
379 butler : `lsst.daf.butler.Butler`
380 Butler to receive these calibration datasets.
381 run : `str`, optional
382 Name of the run to use to override the default run associated
386 for datasetTypeName
in self.standardCuratedDatasetTypes:
388 if datasetTypeName
not in StandardCuratedCalibrationDatasetTypes:
389 raise ValueError(f
"DatasetType {datasetTypeName} not in understood list"
390 f
" [{'.'.join(StandardCuratedCalibrationDatasetTypes)}]")
391 definition = StandardCuratedCalibrationDatasetTypes[datasetTypeName]
392 datasetType = DatasetType(datasetTypeName,
393 universe=butler.registry.dimensions,
398 def _getSpecificCuratedCalibrationPath(cls, datasetTypeName):
399 """Return the path of the curated calibration directory.
403 datasetTypeName : `str`
404 The name of the standard dataset type to find.
409 The path to the standard curated data directory. `None` if the
410 dataset type is not found or the obs data package is not
420 if os.path.exists(calibPath):
425 def _writeSpecificCuratedCalibrationDatasets(self, butler, datasetType, run=None):
426 """Write standardized curated calibration datasets for this specific
427 dataset type from an obs data package.
431 butler : `lsst.daf.butler.Butler`
432 Gen3 butler in which to put the calibrations.
433 datasetType : `lsst.daf.butler.DatasetType`
434 Dataset type to be put.
435 run : `str`, optional
436 Name of the run to use to override the default run associated
441 This method scans the location defined in the ``obsDataPackageDir``
442 class attribute for curated calibrations corresponding to the
443 supplied dataset type. The directory name in the data package must
444 match the name of the dataset type. They are assumed to use the
445 standard layout and can be read by
446 `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard
450 if calibPath
is None:
454 butler.registry.registerDatasetType(datasetType)
458 from lsst.pipe.tasks.read_curated_calibs
import read_all
461 calibsDict = read_all(calibPath, camera)[0]
462 dimensionRecords = []
464 for det
in calibsDict:
465 times = sorted([k
for k
in calibsDict[det]])
466 calibs = [calibsDict[det][time]
for time
in times]
467 times = [astropy.time.Time(t, format=
"datetime", scale=
"utc")
for t
in times]
469 for calib, beginTime, endTime
in zip(calibs, times[:-1], times[1:]):
470 md = calib.getMetadata()
471 calibrationLabel = f
"{datasetType.name}/{md['CALIBDATE']}/{md['DETECTOR']}"
472 dataId = DataCoordinate.standardize(
473 universe=butler.registry.dimensions,
475 calibration_label=calibrationLabel,
476 detector=md[
"DETECTOR"],
478 datasetRecords.append((calib, dataId))
479 dimensionRecords.append({
481 "name": calibrationLabel,
482 "timespan": Timespan(beginTime, endTime),
486 with butler.transaction():
487 butler.registry.insertDimensionData(
"calibration_label", *dimensionRecords)
490 for calib, dataId
in datasetRecords:
491 butler.put(calib, datasetType, dataId, run=run)
495 """Return a factory for creating Gen2->Gen3 data ID translators,
496 specialized for this instrument.
498 Derived class implementations should generally call
499 `TranslatorFactory.addGenericInstrumentRules` with appropriate
500 arguments, but are not required to (and may not be able to if their
501 Gen2 raw data IDs are sufficiently different from the HSC/DECam/CFHT
506 factory : `TranslatorFactory`.
507 Factory for `Translator` objects.
509 raise NotImplementedError(
"Must be implemented by derived classes.")
513 """Make the default instrument-specific run collection string for raw
519 Run collection name to be used as the default for ingestion of
526 """Get the instrument-specific collection string to use as derived
527 from the supplied label.
532 String to be combined with the instrument name to form a
538 Collection name to use that includes the instrument name.
540 return f
"{cls.getName()}/{label}"
544 """Construct an exposure DimensionRecord from
545 `astro_metadata_translator.ObservationInfo`.
549 obsInfo : `astro_metadata_translator.ObservationInfo`
550 A `~astro_metadata_translator.ObservationInfo` object corresponding to
552 universe : `DimensionUniverse`
553 Set of all known dimensions.
557 record : `DimensionRecord`
558 A record containing exposure metadata, suitable for insertion into
561 dimension = universe[
"exposure"]
563 ra, dec, sky_angle, zenith_angle = (
None,
None,
None,
None)
564 if obsInfo.tracking_radec
is not None:
565 icrs = obsInfo.tracking_radec.icrs
567 dec = icrs.dec.degree
568 if obsInfo.boresight_rotation_coord ==
"sky":
569 sky_angle = obsInfo.boresight_rotation_angle.degree
570 if obsInfo.altaz_begin
is not None:
571 zenith_angle = obsInfo.altaz_begin.zen.degree
573 return dimension.RecordClass(
574 instrument=obsInfo.instrument,
575 id=obsInfo.exposure_id,
576 name=obsInfo.observation_id,
577 group_name=obsInfo.exposure_group,
578 group_id=obsInfo.visit_id,
579 datetime_begin=obsInfo.datetime_begin,
580 datetime_end=obsInfo.datetime_end,
581 exposure_time=obsInfo.exposure_time.to_value(
"s"),
582 dark_time=obsInfo.dark_time.to_value(
"s"),
583 observation_type=obsInfo.observation_type,
584 physical_filter=obsInfo.physical_filter,
585 science_program=obsInfo.science_program,
586 target_name=obsInfo.object,
590 zenith_angle=zenith_angle,
595 """Add a special 'unbounded' calibration_label dimension entry for the
596 given camera that is valid for any exposure.
598 If such an entry already exists, this function just returns a `DataId`
599 for the existing entry.
603 registry : `Registry`
604 Registry object in which to insert the dimension entry.
605 instrumentName : `str`
606 Name of the instrument this calibration label is associated with.
611 New or existing data ID for the unbounded calibration.
613 d = dict(instrument=instrumentName, calibration_label=
"unbounded")
615 return registry.expandDataId(d)
619 entry[
"timespan"] = Timespan(
None,
None)
620 registry.insertDimensionData(
"calibration_label", entry)
621 return registry.expandDataId(d)
624 def loadCamera(butler: Butler, dataId: DataId, *, collections: Any =
None) -> Tuple[Camera, bool]:
625 """Attempt to load versioned camera geometry from a butler, but fall back
626 to obtaining a nominal camera from the `Instrument` class if that fails.
630 butler : `lsst.daf.butler.Butler`
631 Butler instance to attempt to query for and load a ``camera`` dataset
633 dataId : `dict` or `DataCoordinate`
634 Data ID that identifies at least the ``instrument`` and ``exposure``
636 collections : Any, optional
637 Collections to be searched, overriding ``self.butler.collections``.
638 Can be any of the types supported by the ``collections`` argument
639 to butler construction.
643 camera : `lsst.afw.cameraGeom.Camera`
646 If `True`, the camera was obtained from the butler and should represent
647 a versioned camera from a calibration repository. If `False`, no
648 camera datasets were found, and the returned camera was produced by
649 instantiating the appropriate `Instrument` class and calling
650 `Instrument.getCamera`.
652 if collections
is None:
653 collections = butler.collections
658 dataId = butler.registry.expandDataId(dataId, graph=butler.registry.dimensions[
"exposure"].graph)
659 cameraRefs = list(butler.registry.queryDatasets(
"camera", dataId=dataId, collections=collections,
662 assert len(cameraRefs) == 1,
"Should be guaranteed by deduplicate=True above."
663 return butler.getDirect(cameraRefs[0]),
True
664 instrument = Instrument.fromName(dataId[
"instrument"], butler.registry)
665 return instrument.getCamera(),
False