22 from __future__
import annotations
24 __all__ = (
"Instrument",
"makeExposureRecordFromObsInfo",
"loadCamera")
27 from abc
import ABCMeta, abstractmethod
28 from collections
import defaultdict
29 from typing
import Any, Optional, Set, Sequence, Tuple, TYPE_CHECKING
30 from functools
import lru_cache
34 from lsst.afw.cameraGeom
import Camera
35 from lsst.daf.butler
import (
46 from .gen2to3
import TranslatorFactory
47 from lsst.daf.butler
import Registry
51 StandardCuratedCalibrationDatasetTypes = {
52 "defects": {
"dimensions": (
"instrument",
"detector"),
"storageClass":
"Defects"},
53 "qe_curve": {
"dimensions": (
"instrument",
"detector"),
"storageClass":
"QECurve"},
54 "crosstalk": {
"dimensions": (
"instrument",
"detector"),
"storageClass":
"CrosstalkCalib"},
59 """Base class for instrument-specific logic for the Gen3 Butler.
61 Concrete instrument subclasses should be directly constructable with no
65 configPaths: Sequence[str] = ()
66 """Paths to config files to read for specific Tasks.
68 The paths in this list should contain files of the form `task.py`, for
69 each of the Tasks that requires special configuration.
72 policyName: Optional[str] =
None
73 """Instrument specific name to use when locating a policy or configuration
74 file in the file system."""
76 obsDataPackage: Optional[str] =
None
77 """Name of the package containing the text curated calibration files.
78 Usually a obs _data package. If `None` no curated calibration files
79 will be read. (`str`)"""
81 standardCuratedDatasetTypes: Set[str] = frozenset(StandardCuratedCalibrationDatasetTypes)
82 """The dataset types expected to be obtained from the obsDataPackage.
84 These dataset types are all required to have standard definitions and
85 must be known to the base class. Clearing this list will prevent
86 any of these calibrations from being stored. If a dataset type is not
87 known to a specific instrument it can still be included in this list
88 since the data package is the source of truth. (`set` of `str`)
91 additionalCuratedDatasetTypes: Set[str] = frozenset()
92 """Curated dataset types specific to this particular instrument that do
93 not follow the standard organization found in obs data packages.
95 These are the instrument-specific dataset types written by
96 `writeAdditionalCuratedCalibrations` in addition to the calibrations
97 found in obs data packages that follow the standard scheme.
103 """`~lsst.obs.base.FilterDefinitionCollection`, defining the filters
115 """Return the short (dimension) name for this instrument.
117 This is not (in general) the same as the class name - it's what is used
118 as the value of the "instrument" field in data IDs, and is usually an
119 abbreviation of the full name.
121 raise NotImplementedError()
126 """Return the names of all the curated calibration dataset types.
130 names : `set` of `str`
131 The dataset type names of all curated calibrations. This will
132 include the standard curated calibrations even if the particular
133 instrument does not support them.
137 The returned list does not indicate whether a particular dataset
138 is present in the Butler repository, simply that these are the
139 dataset types that are handled by ``writeCuratedCalibrations``.
148 for datasetTypeName
in cls.standardCuratedDatasetTypes:
150 if calibPath
is not None:
151 curated.add(datasetTypeName)
153 curated.update(cls.additionalCuratedDatasetTypes)
154 return frozenset(curated)
158 """Retrieve the cameraGeom representation of this instrument.
160 This is a temporary API that should go away once ``obs_`` packages have
161 a standardized approach to writing versioned cameras to a Gen3 repo.
163 raise NotImplementedError()
167 """Insert instrument, physical_filter, and detector entries into a
170 raise NotImplementedError()
175 """The root of the obs data package that provides specializations for
181 The root of the relevat obs data package.
183 if cls.obsDataPackage
is None:
185 return getPackageDir(cls.obsDataPackage)
188 def fromName(name: str, registry: Registry) -> Instrument:
189 """Given an instrument name and a butler, retrieve a corresponding
190 instantiated instrument object.
195 Name of the instrument (must match the return value of `getName`).
196 registry : `lsst.daf.butler.Registry`
197 Butler registry to query to find the information.
201 instrument : `Instrument`
202 An instance of the relevant `Instrument`.
206 The instrument must be registered in the corresponding butler.
211 Raised if the instrument is not known to the supplied registry.
213 Raised if the class could not be imported. This could mean
214 that the relevant obs package has not been setup.
216 Raised if the class name retrieved is not a string.
218 records = list(registry.queryDimensionRecords(
"instrument", instrument=name))
220 raise LookupError(f
"No registered instrument with name '{name}'.")
221 cls = records[0].class_name
222 if not isinstance(cls, str):
223 raise TypeError(f
"Unexpected class name retrieved from {name} instrument dimension (got {cls})")
224 instrument = doImport(cls)
229 """Import all the instruments known to this registry.
231 This will ensure that all metadata translators have been registered.
235 registry : `lsst.daf.butler.Registry`
236 Butler registry to query to find the information.
240 It is allowed for a particular instrument class to fail on import.
241 This might simply indicate that a particular obs package has
244 records = list(registry.queryDimensionRecords(
"instrument"))
245 for record
in records:
246 cls = record.class_name
252 def _registerFilters(self, registry):
253 """Register the physical and abstract filter Dimension relationships.
254 This should be called in the ``register`` implementation.
258 registry : `lsst.daf.butler.core.Registry`
259 The registry to add dimensions to.
264 if filter.band
is None:
265 band = filter.physical_filter
269 registry.insertDimensionData(
"physical_filter",
271 "name": filter.physical_filter,
277 """Return the Formatter class that should be used to read a particular
282 dataId : `DataCoordinate`
283 Dimension-based ID for the raw file or files being ingested.
287 formatter : `Formatter` class
288 Class to be used that reads the file into an
289 `lsst.afw.image.Exposure` instance.
291 raise NotImplementedError()
294 """Apply instrument-specific overrides for a task config.
299 Name of the object being configured; typically the _DefaultName
301 config : `lsst.pex.config.Config`
302 Config instance to which overrides should be applied.
304 for root
in self.configPaths:
305 path = os.path.join(root, f
"{name}.py")
306 if os.path.exists(path):
310 suffixes: Sequence[str] = ()) ->
None:
311 """Write human-curated calibration Datasets to the given Butler with
312 the appropriate validity ranges.
316 butler : `lsst.daf.butler.Butler`
317 Butler to use to store these calibrations.
318 collection : `str`, optional
319 Name to use for the calibration collection that associates all
320 datasets with a validity range. If this collection already exists,
321 it must be a `~CollectionType.CALIBRATION` collection, and it must
322 not have any datasets that would conflict with those inserted by
323 this method. If `None`, a collection name is worked out
324 automatically from the instrument name and other metadata by
325 calling ``makeCalibrationCollectionName``, but this
326 default name may not work well for long-lived repositories unless
327 one or more ``suffixes`` are also provided (and changed every time
328 curated calibrations are ingested).
329 suffixes : `Sequence` [ `str` ], optional
330 Name suffixes to append to collection names, after concatenating
331 them with the standard collection name delimeter. If provided,
332 these are appended to the names of the `~CollectionType.RUN`
333 collections that datasets are inserted directly into, as well the
334 `~CollectionType.CALIBRATION` collection if it is generated
335 automatically (i.e. if ``collection is None``).
339 Expected to be called from subclasses. The base method calls
340 ``writeCameraGeom``, ``writeStandardTextCuratedCalibrations``,
341 and ``writeAdditionalCuratdCalibrations``.
354 suffixes: Sequence[str] = ()) ->
None:
355 """Write additional curated calibrations that might be instrument
356 specific and are not part of the standard set.
358 Default implementation does nothing.
362 butler : `lsst.daf.butler.Butler`
363 Butler to use to store these calibrations.
364 collection : `str`, optional
365 Name to use for the calibration collection that associates all
366 datasets with a validity range. If this collection already exists,
367 it must be a `~CollectionType.CALIBRATION` collection, and it must
368 not have any datasets that would conflict with those inserted by
369 this method. If `None`, a collection name is worked out
370 automatically from the instrument name and other metadata by
371 calling ``makeCalibrationCollectionName``, but this
372 default name may not work well for long-lived repositories unless
373 one or more ``suffixes`` are also provided (and changed every time
374 curated calibrations are ingested).
375 suffixes : `Sequence` [ `str` ], optional
376 Name suffixes to append to collection names, after concatenating
377 them with the standard collection name delimeter. If provided,
378 these are appended to the names of the `~CollectionType.RUN`
379 collections that datasets are inserted directly into, as well the
380 `~CollectionType.CALIBRATION` collection if it is generated
381 automatically (i.e. if ``collection is None``).
386 suffixes: Sequence[str] = ()) ->
None:
387 """Write the default camera geometry to the butler repository and
388 associate it with the appropriate validity range in a calibration
393 butler : `lsst.daf.butler.Butler`
394 Butler to use to store these calibrations.
395 collection : `str`, optional
396 Name to use for the calibration collection that associates all
397 datasets with a validity range. If this collection already exists,
398 it must be a `~CollectionType.CALIBRATION` collection, and it must
399 not have any datasets that would conflict with those inserted by
400 this method. If `None`, a collection name is worked out
401 automatically from the instrument name and other metadata by
402 calling ``makeCalibrationCollectionName``, but this
403 default name may not work well for long-lived repositories unless
404 one or more ``suffixes`` are also provided (and changed every time
405 curated calibrations are ingested).
406 suffixes : `Sequence` [ `str` ], optional
407 Name suffixes to append to collection names, after concatenating
408 them with the standard collection name delimeter. If provided,
409 these are appended to the names of the `~CollectionType.RUN`
410 collections that datasets are inserted directly into, as well the
411 `~CollectionType.CALIBRATION` collection if it is generated
412 automatically (i.e. if ``collection is None``).
414 if collection
is None:
416 butler.registry.registerCollection(collection, type=CollectionType.CALIBRATION)
418 butler.registry.registerRun(run)
419 datasetType = DatasetType(
"camera", (
"instrument",),
"Camera", isCalibration=
True,
420 universe=butler.registry.dimensions)
421 butler.registry.registerDatasetType(datasetType)
423 ref = butler.put(camera, datasetType, {
"instrument": self.
getName()}, run=run)
424 butler.registry.certify(collection, [ref], Timespan(begin=
None, end=
None))
427 suffixes: Sequence[str] = ()) ->
None:
428 """Write the set of standardized curated text calibrations to
433 butler : `lsst.daf.butler.Butler`
434 Butler to receive these calibration datasets.
435 collection : `str`, optional
436 Name to use for the calibration collection that associates all
437 datasets with a validity range. If this collection already exists,
438 it must be a `~CollectionType.CALIBRATION` collection, and it must
439 not have any datasets that would conflict with those inserted by
440 this method. If `None`, a collection name is worked out
441 automatically from the instrument name and other metadata by
442 calling ``makeCalibrationCollectionName``, but this
443 default name may not work well for long-lived repositories unless
444 one or more ``suffixes`` are also provided (and changed every time
445 curated calibrations are ingested).
446 suffixes : `Sequence` [ `str` ], optional
447 Name suffixes to append to collection names, after concatenating
448 them with the standard collection name delimeter. If provided,
449 these are appended to the names of the `~CollectionType.RUN`
450 collections that datasets are inserted directly into, as well the
451 `~CollectionType.CALIBRATION` collection if it is generated
452 automatically (i.e. if ``collection is None``).
454 if collection
is None:
456 butler.registry.registerCollection(collection, type=CollectionType.CALIBRATION)
458 for datasetTypeName
in self.standardCuratedDatasetTypes:
460 if datasetTypeName
not in StandardCuratedCalibrationDatasetTypes:
461 raise ValueError(f
"DatasetType {datasetTypeName} not in understood list"
462 f
" [{'.'.join(StandardCuratedCalibrationDatasetTypes)}]")
463 definition = StandardCuratedCalibrationDatasetTypes[datasetTypeName]
464 datasetType = DatasetType(datasetTypeName,
465 universe=butler.registry.dimensions,
472 def _getSpecificCuratedCalibrationPath(cls, datasetTypeName):
473 """Return the path of the curated calibration directory.
477 datasetTypeName : `str`
478 The name of the standard dataset type to find.
483 The path to the standard curated data directory. `None` if the
484 dataset type is not found or the obs data package is not
494 if os.path.exists(calibPath):
499 def _writeSpecificCuratedCalibrationDatasets(self, butler: Butler, datasetType: DatasetType,
500 collection: str, runs: Set[str], suffixes: Sequence[str]):
501 """Write standardized curated calibration datasets for this specific
502 dataset type from an obs data package.
506 butler : `lsst.daf.butler.Butler`
507 Gen3 butler in which to put the calibrations.
508 datasetType : `lsst.daf.butler.DatasetType`
509 Dataset type to be put.
511 Name of the `~CollectionType.CALIBRATION` collection that
512 associates all datasets with validity ranges. Must have been
513 registered prior to this call.
514 runs : `set` [ `str` ]
515 Names of runs that have already been registered by previous calls
516 and need not be registered again. Should be updated by this
517 method as new runs are registered.
518 suffixes : `Sequence` [ `str` ]
519 Suffixes to append to run names when creating them from
520 ``CALIBDATE`` metadata, via calls to `makeCuratedCalibrationName`.
524 This method scans the location defined in the ``obsDataPackageDir``
525 class attribute for curated calibrations corresponding to the
526 supplied dataset type. The directory name in the data package must
527 match the name of the dataset type. They are assumed to use the
528 standard layout and can be read by
529 `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard
533 if calibPath
is None:
537 butler.registry.registerDatasetType(datasetType)
541 from lsst.pipe.tasks.read_curated_calibs
import read_all
547 calibsDict = read_all(calibPath, camera)[0]
549 for det
in calibsDict:
550 times = sorted([k
for k
in calibsDict[det]])
551 calibs = [calibsDict[det][time]
for time
in times]
552 times = [astropy.time.Time(t, format=
"datetime", scale=
"utc")
for t
in times]
554 for calib, beginTime, endTime
in zip(calibs, times[:-1], times[1:]):
555 md = calib.getMetadata()
558 butler.registry.registerRun(run)
560 dataId = DataCoordinate.standardize(
561 universe=butler.registry.dimensions,
563 detector=md[
"DETECTOR"],
565 datasetRecords.append((calib, dataId, run, Timespan(beginTime, endTime)))
571 refsByTimespan = defaultdict(list)
572 with butler.transaction():
573 for calib, dataId, run, timespan
in datasetRecords:
574 refsByTimespan[timespan].append(butler.put(calib, datasetType, dataId, run=run))
575 for timespan, refs
in refsByTimespan.items():
576 butler.registry.certify(collection, refs, timespan)
580 """Return a factory for creating Gen2->Gen3 data ID translators,
581 specialized for this instrument.
583 Derived class implementations should generally call
584 `TranslatorFactory.addGenericInstrumentRules` with appropriate
585 arguments, but are not required to (and may not be able to if their
586 Gen2 raw data IDs are sufficiently different from the HSC/DECam/CFHT
591 factory : `TranslatorFactory`.
592 Factory for `Translator` objects.
594 raise NotImplementedError(
"Must be implemented by derived classes.")
598 """Make the default instrument-specific run collection string for raw
604 Run collection name to be used as the default for ingestion of
611 """Make a RUN collection name appropriate for inserting calibration
612 datasets whose validity ranges are unbounded.
617 Strings to be appended to the base name, using the default
618 delimiter for collection names.
629 """Make a RUN collection name appropriate for inserting curated
630 calibration datasets with the given ``CALIBDATE`` metadata value.
635 The ``CALIBDATE`` metadata value.
637 Strings to be appended to the base name, using the default
638 delimiter for collection names.
649 """Make a CALIBRATION collection name appropriate for associating
650 calibration datasets with validity ranges.
655 Strings to be appended to the base name, using the default
656 delimiter for collection names.
661 Calibration collection name.
667 """Get the instrument-specific collection string to use as derived
668 from the supplied labels.
673 Strings to be combined with the instrument name to form a
679 Collection name to use that includes the instrument name.
681 return "/".join((cls.
getName(),) + labels)
685 """Construct an exposure DimensionRecord from
686 `astro_metadata_translator.ObservationInfo`.
690 obsInfo : `astro_metadata_translator.ObservationInfo`
691 A `~astro_metadata_translator.ObservationInfo` object corresponding to
693 universe : `DimensionUniverse`
694 Set of all known dimensions.
698 record : `DimensionRecord`
699 A record containing exposure metadata, suitable for insertion into
702 dimension = universe[
"exposure"]
704 ra, dec, sky_angle, zenith_angle = (
None,
None,
None,
None)
705 if obsInfo.tracking_radec
is not None:
706 icrs = obsInfo.tracking_radec.icrs
708 dec = icrs.dec.degree
709 if obsInfo.boresight_rotation_coord ==
"sky":
710 sky_angle = obsInfo.boresight_rotation_angle.degree
711 if obsInfo.altaz_begin
is not None:
712 zenith_angle = obsInfo.altaz_begin.zen.degree
714 return dimension.RecordClass(
715 instrument=obsInfo.instrument,
716 id=obsInfo.exposure_id,
717 name=obsInfo.observation_id,
718 group_name=obsInfo.exposure_group,
719 group_id=obsInfo.visit_id,
720 datetime_begin=obsInfo.datetime_begin,
721 datetime_end=obsInfo.datetime_end,
722 exposure_time=obsInfo.exposure_time.to_value(
"s"),
723 dark_time=obsInfo.dark_time.to_value(
"s"),
724 observation_type=obsInfo.observation_type,
725 observation_reason=obsInfo.observation_reason,
726 physical_filter=obsInfo.physical_filter,
727 science_program=obsInfo.science_program,
728 target_name=obsInfo.object,
732 zenith_angle=zenith_angle,
736 def loadCamera(butler: Butler, dataId: DataId, *, collections: Any =
None) -> Tuple[Camera, bool]:
737 """Attempt to load versioned camera geometry from a butler, but fall back
738 to obtaining a nominal camera from the `Instrument` class if that fails.
742 butler : `lsst.daf.butler.Butler`
743 Butler instance to attempt to query for and load a ``camera`` dataset
745 dataId : `dict` or `DataCoordinate`
746 Data ID that identifies at least the ``instrument`` and ``exposure``
748 collections : Any, optional
749 Collections to be searched, overriding ``self.butler.collections``.
750 Can be any of the types supported by the ``collections`` argument
751 to butler construction.
755 camera : `lsst.afw.cameraGeom.Camera`
758 If `True`, the camera was obtained from the butler and should represent
759 a versioned camera from a calibration repository. If `False`, no
760 camera datasets were found, and the returned camera was produced by
761 instantiating the appropriate `Instrument` class and calling
762 `Instrument.getCamera`.
764 if collections
is None:
765 collections = butler.collections
770 dataId = butler.registry.expandDataId(dataId, graph=butler.registry.dimensions[
"exposure"].graph)
772 cameraRef = butler.get(
"camera", dataId=dataId, collections=collections)
773 return cameraRef,
True
776 instrument = Instrument.fromName(dataId[
"instrument"], butler.registry)
777 return instrument.getCamera(),
False