22 from __future__
import annotations
24 __all__ = (
"Instrument",
"makeExposureRecordFromObsInfo",
"loadCamera")
27 from abc
import ABCMeta, abstractmethod
28 from collections
import defaultdict
29 from typing
import Any, Optional, Set, Sequence, Tuple, TYPE_CHECKING
30 from functools
import lru_cache
34 from lsst.afw.cameraGeom
import Camera
35 from lsst.daf.butler
import (
46 from .gen2to3
import TranslatorFactory
47 from lsst.daf.butler
import Registry
51 StandardCuratedCalibrationDatasetTypes = {
52 "defects": {
"dimensions": (
"instrument",
"detector"),
"storageClass":
"Defects"},
53 "qe_curve": {
"dimensions": (
"instrument",
"detector"),
"storageClass":
"QECurve"},
54 "crosstalk": {
"dimensions": (
"instrument",
"detector"),
"storageClass":
"CrosstalkCalib"},
59 """Base class for instrument-specific logic for the Gen3 Butler.
61 Concrete instrument subclasses should be directly constructable with no
65 configPaths: Sequence[str] = ()
66 """Paths to config files to read for specific Tasks.
68 The paths in this list should contain files of the form `task.py`, for
69 each of the Tasks that requires special configuration.
72 policyName: Optional[str] =
None
73 """Instrument specific name to use when locating a policy or configuration
74 file in the file system."""
76 obsDataPackage: Optional[str] =
None
77 """Name of the package containing the text curated calibration files.
78 Usually a obs _data package. If `None` no curated calibration files
79 will be read. (`str`)"""
81 standardCuratedDatasetTypes: Set[str] = frozenset(StandardCuratedCalibrationDatasetTypes)
82 """The dataset types expected to be obtained from the obsDataPackage.
84 These dataset types are all required to have standard definitions and
85 must be known to the base class. Clearing this list will prevent
86 any of these calibrations from being stored. If a dataset type is not
87 known to a specific instrument it can still be included in this list
88 since the data package is the source of truth. (`set` of `str`)
91 additionalCuratedDatasetTypes: Set[str] = frozenset()
92 """Curated dataset types specific to this particular instrument that do
93 not follow the standard organization found in obs data packages.
95 These are the instrument-specific dataset types written by
96 `writeAdditionalCuratedCalibrations` in addition to the calibrations
97 found in obs data packages that follow the standard scheme.
103 """`~lsst.obs.base.FilterDefinitionCollection`, defining the filters
115 """Return the short (dimension) name for this instrument.
117 This is not (in general) the same as the class name - it's what is used
118 as the value of the "instrument" field in data IDs, and is usually an
119 abbreviation of the full name.
121 raise NotImplementedError()
126 """Return the names of all the curated calibration dataset types.
130 names : `set` of `str`
131 The dataset type names of all curated calibrations. This will
132 include the standard curated calibrations even if the particular
133 instrument does not support them.
137 The returned list does not indicate whether a particular dataset
138 is present in the Butler repository, simply that these are the
139 dataset types that are handled by ``writeCuratedCalibrations``.
148 for datasetTypeName
in cls.standardCuratedDatasetTypes:
150 if calibPath
is not None:
151 curated.add(datasetTypeName)
153 curated.update(cls.additionalCuratedDatasetTypes)
154 return frozenset(curated)
158 """Retrieve the cameraGeom representation of this instrument.
160 This is a temporary API that should go away once ``obs_`` packages have
161 a standardized approach to writing versioned cameras to a Gen3 repo.
163 raise NotImplementedError()
167 """Insert instrument, physical_filter, and detector entries into a
170 Implementations should guarantee that registration is atomic (the
171 registry should not be modified if any error occurs) and idempotent at
172 the level of individual dimension entries; new detectors and filters
173 should be added, but changes to any existing record should not be.
174 This can generally be achieved via a block like::
176 with registry.transaction():
177 registry.syncDimensionData("instrument", ...)
178 registry.syncDimensionData("detector", ...)
179 self.registerFilters(registry)
183 lsst.daf.butler.registry.ConflictingDefinitionError
184 Raised if any existing record has the same key but a different
185 definition as one being registered.
187 raise NotImplementedError()
192 """The root of the obs data package that provides specializations for
198 The root of the relevat obs data package.
200 if cls.obsDataPackage
is None:
202 return getPackageDir(cls.obsDataPackage)
205 def fromName(name: str, registry: Registry) -> Instrument:
206 """Given an instrument name and a butler, retrieve a corresponding
207 instantiated instrument object.
212 Name of the instrument (must match the return value of `getName`).
213 registry : `lsst.daf.butler.Registry`
214 Butler registry to query to find the information.
218 instrument : `Instrument`
219 An instance of the relevant `Instrument`.
223 The instrument must be registered in the corresponding butler.
228 Raised if the instrument is not known to the supplied registry.
230 Raised if the class could not be imported. This could mean
231 that the relevant obs package has not been setup.
233 Raised if the class name retrieved is not a string.
235 records = list(registry.queryDimensionRecords(
"instrument", instrument=name))
237 raise LookupError(f
"No registered instrument with name '{name}'.")
238 cls = records[0].class_name
239 if not isinstance(cls, str):
240 raise TypeError(f
"Unexpected class name retrieved from {name} instrument dimension (got {cls})")
241 instrument = doImport(cls)
246 """Import all the instruments known to this registry.
248 This will ensure that all metadata translators have been registered.
252 registry : `lsst.daf.butler.Registry`
253 Butler registry to query to find the information.
257 It is allowed for a particular instrument class to fail on import.
258 This might simply indicate that a particular obs package has
261 records = list(registry.queryDimensionRecords(
"instrument"))
262 for record
in records:
263 cls = record.class_name
269 def _registerFilters(self, registry):
270 """Register the physical and abstract filter Dimension relationships.
271 This should be called in the `register` implementation, within
272 a transaction context manager block.
276 registry : `lsst.daf.butler.core.Registry`
277 The registry to add dimensions to.
282 if filter.band
is None:
283 band = filter.physical_filter
287 registry.syncDimensionData(
"physical_filter",
289 "name": filter.physical_filter,
295 """Return the Formatter class that should be used to read a particular
300 dataId : `DataCoordinate`
301 Dimension-based ID for the raw file or files being ingested.
305 formatter : `Formatter` class
306 Class to be used that reads the file into an
307 `lsst.afw.image.Exposure` instance.
309 raise NotImplementedError()
312 """Apply instrument-specific overrides for a task config.
317 Name of the object being configured; typically the _DefaultName
319 config : `lsst.pex.config.Config`
320 Config instance to which overrides should be applied.
322 for root
in self.configPaths:
323 path = os.path.join(root, f
"{name}.py")
324 if os.path.exists(path):
328 suffixes: Sequence[str] = ()) ->
None:
329 """Write human-curated calibration Datasets to the given Butler with
330 the appropriate validity ranges.
334 butler : `lsst.daf.butler.Butler`
335 Butler to use to store these calibrations.
336 collection : `str`, optional
337 Name to use for the calibration collection that associates all
338 datasets with a validity range. If this collection already exists,
339 it must be a `~CollectionType.CALIBRATION` collection, and it must
340 not have any datasets that would conflict with those inserted by
341 this method. If `None`, a collection name is worked out
342 automatically from the instrument name and other metadata by
343 calling ``makeCalibrationCollectionName``, but this
344 default name may not work well for long-lived repositories unless
345 one or more ``suffixes`` are also provided (and changed every time
346 curated calibrations are ingested).
347 suffixes : `Sequence` [ `str` ], optional
348 Name suffixes to append to collection names, after concatenating
349 them with the standard collection name delimeter. If provided,
350 these are appended to the names of the `~CollectionType.RUN`
351 collections that datasets are inserted directly into, as well the
352 `~CollectionType.CALIBRATION` collection if it is generated
353 automatically (i.e. if ``collection is None``).
357 Expected to be called from subclasses. The base method calls
358 ``writeCameraGeom``, ``writeStandardTextCuratedCalibrations``,
359 and ``writeAdditionalCuratdCalibrations``.
372 suffixes: Sequence[str] = ()) ->
None:
373 """Write additional curated calibrations that might be instrument
374 specific and are not part of the standard set.
376 Default implementation does nothing.
380 butler : `lsst.daf.butler.Butler`
381 Butler to use to store these calibrations.
382 collection : `str`, optional
383 Name to use for the calibration collection that associates all
384 datasets with a validity range. If this collection already exists,
385 it must be a `~CollectionType.CALIBRATION` collection, and it must
386 not have any datasets that would conflict with those inserted by
387 this method. If `None`, a collection name is worked out
388 automatically from the instrument name and other metadata by
389 calling ``makeCalibrationCollectionName``, but this
390 default name may not work well for long-lived repositories unless
391 one or more ``suffixes`` are also provided (and changed every time
392 curated calibrations are ingested).
393 suffixes : `Sequence` [ `str` ], optional
394 Name suffixes to append to collection names, after concatenating
395 them with the standard collection name delimeter. If provided,
396 these are appended to the names of the `~CollectionType.RUN`
397 collections that datasets are inserted directly into, as well the
398 `~CollectionType.CALIBRATION` collection if it is generated
399 automatically (i.e. if ``collection is None``).
404 suffixes: Sequence[str] = ()) ->
None:
405 """Write the default camera geometry to the butler repository and
406 associate it with the appropriate validity range in a calibration
411 butler : `lsst.daf.butler.Butler`
412 Butler to use to store these calibrations.
413 collection : `str`, optional
414 Name to use for the calibration collection that associates all
415 datasets with a validity range. If this collection already exists,
416 it must be a `~CollectionType.CALIBRATION` collection, and it must
417 not have any datasets that would conflict with those inserted by
418 this method. If `None`, a collection name is worked out
419 automatically from the instrument name and other metadata by
420 calling ``makeCalibrationCollectionName``, but this
421 default name may not work well for long-lived repositories unless
422 one or more ``suffixes`` are also provided (and changed every time
423 curated calibrations are ingested).
424 suffixes : `Sequence` [ `str` ], optional
425 Name suffixes to append to collection names, after concatenating
426 them with the standard collection name delimeter. If provided,
427 these are appended to the names of the `~CollectionType.RUN`
428 collections that datasets are inserted directly into, as well the
429 `~CollectionType.CALIBRATION` collection if it is generated
430 automatically (i.e. if ``collection is None``).
432 if collection
is None:
434 butler.registry.registerCollection(collection, type=CollectionType.CALIBRATION)
436 butler.registry.registerRun(run)
437 datasetType = DatasetType(
"camera", (
"instrument",),
"Camera", isCalibration=
True,
438 universe=butler.registry.dimensions)
439 butler.registry.registerDatasetType(datasetType)
441 ref = butler.put(camera, datasetType, {
"instrument": self.
getName()}, run=run)
442 butler.registry.certify(collection, [ref], Timespan(begin=
None, end=
None))
445 suffixes: Sequence[str] = ()) ->
None:
446 """Write the set of standardized curated text calibrations to
451 butler : `lsst.daf.butler.Butler`
452 Butler to receive these calibration datasets.
453 collection : `str`, optional
454 Name to use for the calibration collection that associates all
455 datasets with a validity range. If this collection already exists,
456 it must be a `~CollectionType.CALIBRATION` collection, and it must
457 not have any datasets that would conflict with those inserted by
458 this method. If `None`, a collection name is worked out
459 automatically from the instrument name and other metadata by
460 calling ``makeCalibrationCollectionName``, but this
461 default name may not work well for long-lived repositories unless
462 one or more ``suffixes`` are also provided (and changed every time
463 curated calibrations are ingested).
464 suffixes : `Sequence` [ `str` ], optional
465 Name suffixes to append to collection names, after concatenating
466 them with the standard collection name delimeter. If provided,
467 these are appended to the names of the `~CollectionType.RUN`
468 collections that datasets are inserted directly into, as well the
469 `~CollectionType.CALIBRATION` collection if it is generated
470 automatically (i.e. if ``collection is None``).
472 if collection
is None:
474 butler.registry.registerCollection(collection, type=CollectionType.CALIBRATION)
476 for datasetTypeName
in self.standardCuratedDatasetTypes:
478 if datasetTypeName
not in StandardCuratedCalibrationDatasetTypes:
479 raise ValueError(f
"DatasetType {datasetTypeName} not in understood list"
480 f
" [{'.'.join(StandardCuratedCalibrationDatasetTypes)}]")
481 definition = StandardCuratedCalibrationDatasetTypes[datasetTypeName]
482 datasetType = DatasetType(datasetTypeName,
483 universe=butler.registry.dimensions,
490 def _getSpecificCuratedCalibrationPath(cls, datasetTypeName):
491 """Return the path of the curated calibration directory.
495 datasetTypeName : `str`
496 The name of the standard dataset type to find.
501 The path to the standard curated data directory. `None` if the
502 dataset type is not found or the obs data package is not
512 if os.path.exists(calibPath):
517 def _writeSpecificCuratedCalibrationDatasets(self, butler: Butler, datasetType: DatasetType,
518 collection: str, runs: Set[str], suffixes: Sequence[str]):
519 """Write standardized curated calibration datasets for this specific
520 dataset type from an obs data package.
524 butler : `lsst.daf.butler.Butler`
525 Gen3 butler in which to put the calibrations.
526 datasetType : `lsst.daf.butler.DatasetType`
527 Dataset type to be put.
529 Name of the `~CollectionType.CALIBRATION` collection that
530 associates all datasets with validity ranges. Must have been
531 registered prior to this call.
532 runs : `set` [ `str` ]
533 Names of runs that have already been registered by previous calls
534 and need not be registered again. Should be updated by this
535 method as new runs are registered.
536 suffixes : `Sequence` [ `str` ]
537 Suffixes to append to run names when creating them from
538 ``CALIBDATE`` metadata, via calls to `makeCuratedCalibrationName`.
542 This method scans the location defined in the ``obsDataPackageDir``
543 class attribute for curated calibrations corresponding to the
544 supplied dataset type. The directory name in the data package must
545 match the name of the dataset type. They are assumed to use the
546 standard layout and can be read by
547 `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard
551 if calibPath
is None:
555 butler.registry.registerDatasetType(datasetType)
559 from lsst.pipe.tasks.read_curated_calibs
import read_all
565 calibsDict = read_all(calibPath, camera)[0]
567 for det
in calibsDict:
568 times = sorted([k
for k
in calibsDict[det]])
569 calibs = [calibsDict[det][time]
for time
in times]
570 times = [astropy.time.Time(t, format=
"datetime", scale=
"utc")
for t
in times]
572 for calib, beginTime, endTime
in zip(calibs, times[:-1], times[1:]):
573 md = calib.getMetadata()
576 butler.registry.registerRun(run)
578 dataId = DataCoordinate.standardize(
579 universe=butler.registry.dimensions,
581 detector=md[
"DETECTOR"],
583 datasetRecords.append((calib, dataId, run, Timespan(beginTime, endTime)))
589 refsByTimespan = defaultdict(list)
590 with butler.transaction():
591 for calib, dataId, run, timespan
in datasetRecords:
592 refsByTimespan[timespan].append(butler.put(calib, datasetType, dataId, run=run))
593 for timespan, refs
in refsByTimespan.items():
594 butler.registry.certify(collection, refs, timespan)
598 """Return a factory for creating Gen2->Gen3 data ID translators,
599 specialized for this instrument.
601 Derived class implementations should generally call
602 `TranslatorFactory.addGenericInstrumentRules` with appropriate
603 arguments, but are not required to (and may not be able to if their
604 Gen2 raw data IDs are sufficiently different from the HSC/DECam/CFHT
609 factory : `TranslatorFactory`.
610 Factory for `Translator` objects.
612 raise NotImplementedError(
"Must be implemented by derived classes.")
616 """Make the default instrument-specific run collection string for raw
622 Run collection name to be used as the default for ingestion of
629 """Make a RUN collection name appropriate for inserting calibration
630 datasets whose validity ranges are unbounded.
635 Strings to be appended to the base name, using the default
636 delimiter for collection names.
647 """Make a RUN collection name appropriate for inserting curated
648 calibration datasets with the given ``CALIBDATE`` metadata value.
653 The ``CALIBDATE`` metadata value.
655 Strings to be appended to the base name, using the default
656 delimiter for collection names.
667 """Make a CALIBRATION collection name appropriate for associating
668 calibration datasets with validity ranges.
673 Strings to be appended to the base name, using the default
674 delimiter for collection names.
679 Calibration collection name.
685 """Get the instrument-specific collection string to use as derived
686 from the supplied labels.
691 Strings to be combined with the instrument name to form a
697 Collection name to use that includes the instrument name.
699 return "/".join((cls.
getName(),) + labels)
703 """Construct an exposure DimensionRecord from
704 `astro_metadata_translator.ObservationInfo`.
708 obsInfo : `astro_metadata_translator.ObservationInfo`
709 A `~astro_metadata_translator.ObservationInfo` object corresponding to
711 universe : `DimensionUniverse`
712 Set of all known dimensions.
716 record : `DimensionRecord`
717 A record containing exposure metadata, suitable for insertion into
720 dimension = universe[
"exposure"]
722 ra, dec, sky_angle, zenith_angle = (
None,
None,
None,
None)
723 if obsInfo.tracking_radec
is not None:
724 icrs = obsInfo.tracking_radec.icrs
726 dec = icrs.dec.degree
727 if obsInfo.boresight_rotation_coord ==
"sky":
728 sky_angle = obsInfo.boresight_rotation_angle.degree
729 if obsInfo.altaz_begin
is not None:
730 zenith_angle = obsInfo.altaz_begin.zen.degree
732 return dimension.RecordClass(
733 instrument=obsInfo.instrument,
734 id=obsInfo.exposure_id,
735 name=obsInfo.observation_id,
736 group_name=obsInfo.exposure_group,
737 group_id=obsInfo.visit_id,
738 datetime_begin=obsInfo.datetime_begin,
739 datetime_end=obsInfo.datetime_end,
740 exposure_time=obsInfo.exposure_time.to_value(
"s"),
741 dark_time=obsInfo.dark_time.to_value(
"s"),
742 observation_type=obsInfo.observation_type,
743 observation_reason=obsInfo.observation_reason,
744 physical_filter=obsInfo.physical_filter,
745 science_program=obsInfo.science_program,
746 target_name=obsInfo.object,
750 zenith_angle=zenith_angle,
754 def loadCamera(butler: Butler, dataId: DataId, *, collections: Any =
None) -> Tuple[Camera, bool]:
755 """Attempt to load versioned camera geometry from a butler, but fall back
756 to obtaining a nominal camera from the `Instrument` class if that fails.
760 butler : `lsst.daf.butler.Butler`
761 Butler instance to attempt to query for and load a ``camera`` dataset
763 dataId : `dict` or `DataCoordinate`
764 Data ID that identifies at least the ``instrument`` and ``exposure``
766 collections : Any, optional
767 Collections to be searched, overriding ``self.butler.collections``.
768 Can be any of the types supported by the ``collections`` argument
769 to butler construction.
773 camera : `lsst.afw.cameraGeom.Camera`
776 If `True`, the camera was obtained from the butler and should represent
777 a versioned camera from a calibration repository. If `False`, no
778 camera datasets were found, and the returned camera was produced by
779 instantiating the appropriate `Instrument` class and calling
780 `Instrument.getCamera`.
782 if collections
is None:
783 collections = butler.collections
788 dataId = butler.registry.expandDataId(dataId, graph=butler.registry.dimensions[
"exposure"].graph)
790 cameraRef = butler.get(
"camera", dataId=dataId, collections=collections)
791 return cameraRef,
True
794 instrument = Instrument.fromName(dataId[
"instrument"], butler.registry)
795 return instrument.getCamera(),
False