22 from __future__
import annotations
24 __all__ = (
"Instrument",
"makeExposureRecordFromObsInfo",
"loadCamera")
27 from abc
import ABCMeta, abstractmethod
28 from collections
import defaultdict
29 from typing
import Any, Optional, Set, Sequence, Tuple, TYPE_CHECKING
30 from functools
import lru_cache
34 from lsst.afw.cameraGeom
import Camera
35 from lsst.daf.butler
import (
46 from .gen2to3
import TranslatorFactory
47 from lsst.daf.butler
import Registry
51 StandardCuratedCalibrationDatasetTypes = {
52 "defects": {
"dimensions": (
"instrument",
"detector"),
"storageClass":
"Defects"},
53 "qe_curve": {
"dimensions": (
"instrument",
"detector"),
"storageClass":
"QECurve"},
54 "crosstalk": {
"dimensions": (
"instrument",
"detector"),
"storageClass":
"CrosstalkCalib"},
59 """Base class for instrument-specific logic for the Gen3 Butler.
61 Concrete instrument subclasses should be directly constructable with no
65 configPaths: Sequence[str] = ()
66 """Paths to config files to read for specific Tasks.
68 The paths in this list should contain files of the form `task.py`, for
69 each of the Tasks that requires special configuration.
72 policyName: Optional[str] =
None
73 """Instrument specific name to use when locating a policy or configuration
74 file in the file system."""
76 obsDataPackage: Optional[str] =
None
77 """Name of the package containing the text curated calibration files.
78 Usually a obs _data package. If `None` no curated calibration files
79 will be read. (`str`)"""
81 standardCuratedDatasetTypes: Set[str] = frozenset(StandardCuratedCalibrationDatasetTypes)
82 """The dataset types expected to be obtained from the obsDataPackage.
84 These dataset types are all required to have standard definitions and
85 must be known to the base class. Clearing this list will prevent
86 any of these calibrations from being stored. If a dataset type is not
87 known to a specific instrument it can still be included in this list
88 since the data package is the source of truth. (`set` of `str`)
91 additionalCuratedDatasetTypes: Set[str] = frozenset()
92 """Curated dataset types specific to this particular instrument that do
93 not follow the standard organization found in obs data packages.
95 These are the instrument-specific dataset types written by
96 `writeAdditionalCuratedCalibrations` in addition to the calibrations
97 found in obs data packages that follow the standard scheme.
103 """`~lsst.obs.base.FilterDefinitionCollection`, defining the filters
115 """Return the short (dimension) name for this instrument.
117 This is not (in general) the same as the class name - it's what is used
118 as the value of the "instrument" field in data IDs, and is usually an
119 abbreviation of the full name.
121 raise NotImplementedError()
126 """Return the names of all the curated calibration dataset types.
130 names : `set` of `str`
131 The dataset type names of all curated calibrations. This will
132 include the standard curated calibrations even if the particular
133 instrument does not support them.
137 The returned list does not indicate whether a particular dataset
138 is present in the Butler repository, simply that these are the
139 dataset types that are handled by ``writeCuratedCalibrations``.
148 for datasetTypeName
in cls.standardCuratedDatasetTypes:
150 if calibPath
is not None:
151 curated.add(datasetTypeName)
153 curated.update(cls.additionalCuratedDatasetTypes)
154 return frozenset(curated)
158 """Retrieve the cameraGeom representation of this instrument.
160 This is a temporary API that should go away once ``obs_`` packages have
161 a standardized approach to writing versioned cameras to a Gen3 repo.
163 raise NotImplementedError()
167 """Insert instrument, physical_filter, and detector entries into a
170 raise NotImplementedError()
175 """The root of the obs data package that provides specializations for
181 The root of the relevat obs data package.
183 if cls.obsDataPackage
is None:
185 return getPackageDir(cls.obsDataPackage)
188 def fromName(name: str, registry: Registry) -> Instrument:
189 """Given an instrument name and a butler, retrieve a corresponding
190 instantiated instrument object.
195 Name of the instrument (must match the return value of `getName`).
196 registry : `lsst.daf.butler.Registry`
197 Butler registry to query to find the information.
201 instrument : `Instrument`
202 An instance of the relevant `Instrument`.
206 The instrument must be registered in the corresponding butler.
211 Raised if the instrument is not known to the supplied registry.
213 Raised if the class could not be imported. This could mean
214 that the relevant obs package has not been setup.
216 Raised if the class name retrieved is not a string.
218 records = list(registry.queryDimensionRecords(
"instrument", instrument=name))
220 raise LookupError(f
"No registered instrument with name '{name}'.")
221 cls = records[0].class_name
222 if not isinstance(cls, str):
223 raise TypeError(f
"Unexpected class name retrieved from {name} instrument dimension (got {cls})")
224 instrument = doImport(cls)
229 """Import all the instruments known to this registry.
231 This will ensure that all metadata translators have been registered.
235 registry : `lsst.daf.butler.Registry`
236 Butler registry to query to find the information.
240 It is allowed for a particular instrument class to fail on import.
241 This might simply indicate that a particular obs package has
244 records = list(registry.queryDimensionRecords(
"instrument"))
245 for record
in records:
246 cls = record.class_name
252 def _registerFilters(self, registry):
253 """Register the physical and abstract filter Dimension relationships.
254 This should be called in the ``register`` implementation.
258 registry : `lsst.daf.butler.core.Registry`
259 The registry to add dimensions to.
263 if filter.band
is None:
264 band = filter.physical_filter
268 registry.insertDimensionData(
"physical_filter",
270 "name": filter.physical_filter,
276 """Return the Formatter class that should be used to read a particular
281 dataId : `DataCoordinate`
282 Dimension-based ID for the raw file or files being ingested.
286 formatter : `Formatter` class
287 Class to be used that reads the file into an
288 `lsst.afw.image.Exposure` instance.
290 raise NotImplementedError()
293 """Apply instrument-specific overrides for a task config.
298 Name of the object being configured; typically the _DefaultName
300 config : `lsst.pex.config.Config`
301 Config instance to which overrides should be applied.
303 for root
in self.configPaths:
304 path = os.path.join(root, f
"{name}.py")
305 if os.path.exists(path):
309 suffixes: Sequence[str] = ()) ->
None:
310 """Write human-curated calibration Datasets to the given Butler with
311 the appropriate validity ranges.
315 butler : `lsst.daf.butler.Butler`
316 Butler to use to store these calibrations.
317 collection : `str`, optional
318 Name to use for the calibration collection that associates all
319 datasets with a validity range. If this collection already exists,
320 it must be a `~CollectionType.CALIBRATION` collection, and it must
321 not have any datasets that would conflict with those inserted by
322 this method. If `None`, a collection name is worked out
323 automatically from the instrument name and other metadata by
324 calling ``makeCalibrationCollectionName``, but this
325 default name may not work well for long-lived repositories unless
326 one or more ``suffixes`` are also provided (and changed every time
327 curated calibrations are ingested).
328 suffixes : `Sequence` [ `str` ], optional
329 Name suffixes to append to collection names, after concatenating
330 them with the standard collection name delimeter. If provided,
331 these are appended to the names of the `~CollectionType.RUN`
332 collections that datasets are inserted directly into, as well the
333 `~CollectionType.CALIBRATION` collection if it is generated
334 automatically (i.e. if ``collection is None``).
338 Expected to be called from subclasses. The base method calls
339 ``writeCameraGeom``, ``writeStandardTextCuratedCalibrations``,
340 and ``writeAdditionalCuratdCalibrations``.
353 suffixes: Sequence[str] = ()) ->
None:
354 """Write additional curated calibrations that might be instrument
355 specific and are not part of the standard set.
357 Default implementation does nothing.
361 butler : `lsst.daf.butler.Butler`
362 Butler to use to store these calibrations.
363 collection : `str`, optional
364 Name to use for the calibration collection that associates all
365 datasets with a validity range. If this collection already exists,
366 it must be a `~CollectionType.CALIBRATION` collection, and it must
367 not have any datasets that would conflict with those inserted by
368 this method. If `None`, a collection name is worked out
369 automatically from the instrument name and other metadata by
370 calling ``makeCalibrationCollectionName``, but this
371 default name may not work well for long-lived repositories unless
372 one or more ``suffixes`` are also provided (and changed every time
373 curated calibrations are ingested).
374 suffixes : `Sequence` [ `str` ], optional
375 Name suffixes to append to collection names, after concatenating
376 them with the standard collection name delimeter. If provided,
377 these are appended to the names of the `~CollectionType.RUN`
378 collections that datasets are inserted directly into, as well the
379 `~CollectionType.CALIBRATION` collection if it is generated
380 automatically (i.e. if ``collection is None``).
385 suffixes: Sequence[str] = ()) ->
None:
386 """Write the default camera geometry to the butler repository and
387 associate it with the appropriate validity range in a calibration
392 butler : `lsst.daf.butler.Butler`
393 Butler to use to store these calibrations.
394 collection : `str`, optional
395 Name to use for the calibration collection that associates all
396 datasets with a validity range. If this collection already exists,
397 it must be a `~CollectionType.CALIBRATION` collection, and it must
398 not have any datasets that would conflict with those inserted by
399 this method. If `None`, a collection name is worked out
400 automatically from the instrument name and other metadata by
401 calling ``makeCalibrationCollectionName``, but this
402 default name may not work well for long-lived repositories unless
403 one or more ``suffixes`` are also provided (and changed every time
404 curated calibrations are ingested).
405 suffixes : `Sequence` [ `str` ], optional
406 Name suffixes to append to collection names, after concatenating
407 them with the standard collection name delimeter. If provided,
408 these are appended to the names of the `~CollectionType.RUN`
409 collections that datasets are inserted directly into, as well the
410 `~CollectionType.CALIBRATION` collection if it is generated
411 automatically (i.e. if ``collection is None``).
413 if collection
is None:
415 butler.registry.registerCollection(collection, type=CollectionType.CALIBRATION)
417 butler.registry.registerRun(run)
418 datasetType = DatasetType(
"camera", (
"instrument",),
"Camera", isCalibration=
True,
419 universe=butler.registry.dimensions)
420 butler.registry.registerDatasetType(datasetType)
422 ref = butler.put(camera, datasetType, {
"instrument": self.
getName()}, run=run)
423 butler.registry.certify(collection, [ref], Timespan(begin=
None, end=
None))
426 suffixes: Sequence[str] = ()) ->
None:
427 """Write the set of standardized curated text calibrations to
432 butler : `lsst.daf.butler.Butler`
433 Butler to receive these calibration datasets.
434 collection : `str`, optional
435 Name to use for the calibration collection that associates all
436 datasets with a validity range. If this collection already exists,
437 it must be a `~CollectionType.CALIBRATION` collection, and it must
438 not have any datasets that would conflict with those inserted by
439 this method. If `None`, a collection name is worked out
440 automatically from the instrument name and other metadata by
441 calling ``makeCalibrationCollectionName``, but this
442 default name may not work well for long-lived repositories unless
443 one or more ``suffixes`` are also provided (and changed every time
444 curated calibrations are ingested).
445 suffixes : `Sequence` [ `str` ], optional
446 Name suffixes to append to collection names, after concatenating
447 them with the standard collection name delimeter. If provided,
448 these are appended to the names of the `~CollectionType.RUN`
449 collections that datasets are inserted directly into, as well the
450 `~CollectionType.CALIBRATION` collection if it is generated
451 automatically (i.e. if ``collection is None``).
453 if collection
is None:
455 butler.registry.registerCollection(collection, type=CollectionType.CALIBRATION)
457 for datasetTypeName
in self.standardCuratedDatasetTypes:
459 if datasetTypeName
not in StandardCuratedCalibrationDatasetTypes:
460 raise ValueError(f
"DatasetType {datasetTypeName} not in understood list"
461 f
" [{'.'.join(StandardCuratedCalibrationDatasetTypes)}]")
462 definition = StandardCuratedCalibrationDatasetTypes[datasetTypeName]
463 datasetType = DatasetType(datasetTypeName,
464 universe=butler.registry.dimensions,
471 def _getSpecificCuratedCalibrationPath(cls, datasetTypeName):
472 """Return the path of the curated calibration directory.
476 datasetTypeName : `str`
477 The name of the standard dataset type to find.
482 The path to the standard curated data directory. `None` if the
483 dataset type is not found or the obs data package is not
493 if os.path.exists(calibPath):
498 def _writeSpecificCuratedCalibrationDatasets(self, butler: Butler, datasetType: DatasetType,
499 collection: str, runs: Set[str], suffixes: Sequence[str]):
500 """Write standardized curated calibration datasets for this specific
501 dataset type from an obs data package.
505 butler : `lsst.daf.butler.Butler`
506 Gen3 butler in which to put the calibrations.
507 datasetType : `lsst.daf.butler.DatasetType`
508 Dataset type to be put.
510 Name of the `~CollectionType.CALIBRATION` collection that
511 associates all datasets with validity ranges. Must have been
512 registered prior to this call.
513 runs : `set` [ `str` ]
514 Names of runs that have already been registered by previous calls
515 and need not be registered again. Should be updated by this
516 method as new runs are registered.
517 suffixes : `Sequence` [ `str` ]
518 Suffixes to append to run names when creating them from
519 ``CALIBDATE`` metadata, via calls to `makeCuratedCalibrationName`.
523 This method scans the location defined in the ``obsDataPackageDir``
524 class attribute for curated calibrations corresponding to the
525 supplied dataset type. The directory name in the data package must
526 match the name of the dataset type. They are assumed to use the
527 standard layout and can be read by
528 `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard
532 if calibPath
is None:
536 butler.registry.registerDatasetType(datasetType)
540 from lsst.pipe.tasks.read_curated_calibs
import read_all
546 calibsDict = read_all(calibPath, camera)[0]
548 for det
in calibsDict:
549 times = sorted([k
for k
in calibsDict[det]])
550 calibs = [calibsDict[det][time]
for time
in times]
551 times = [astropy.time.Time(t, format=
"datetime", scale=
"utc")
for t
in times]
553 for calib, beginTime, endTime
in zip(calibs, times[:-1], times[1:]):
554 md = calib.getMetadata()
557 butler.registry.registerRun(run)
559 dataId = DataCoordinate.standardize(
560 universe=butler.registry.dimensions,
562 detector=md[
"DETECTOR"],
564 datasetRecords.append((calib, dataId, run, Timespan(beginTime, endTime)))
570 refsByTimespan = defaultdict(list)
571 with butler.transaction():
572 for calib, dataId, run, timespan
in datasetRecords:
573 refsByTimespan[timespan].append(butler.put(calib, datasetType, dataId, run=run))
574 for timespan, refs
in refsByTimespan.items():
575 butler.registry.certify(collection, refs, timespan)
579 """Return a factory for creating Gen2->Gen3 data ID translators,
580 specialized for this instrument.
582 Derived class implementations should generally call
583 `TranslatorFactory.addGenericInstrumentRules` with appropriate
584 arguments, but are not required to (and may not be able to if their
585 Gen2 raw data IDs are sufficiently different from the HSC/DECam/CFHT
590 factory : `TranslatorFactory`.
591 Factory for `Translator` objects.
593 raise NotImplementedError(
"Must be implemented by derived classes.")
597 """Make the default instrument-specific run collection string for raw
603 Run collection name to be used as the default for ingestion of
610 """Make a RUN collection name appropriate for inserting calibration
611 datasets whose validity ranges are unbounded.
616 Strings to be appended to the base name, using the default
617 delimiter for collection names.
628 """Make a RUN collection name appropriate for inserting curated
629 calibration datasets with the given ``CALIBDATE`` metadata value.
634 The ``CALIBDATE`` metadata value.
636 Strings to be appended to the base name, using the default
637 delimiter for collection names.
648 """Make a CALIBRATION collection name appropriate for associating
649 calibration datasets with validity ranges.
654 Strings to be appended to the base name, using the default
655 delimiter for collection names.
660 Calibration collection name.
666 """Get the instrument-specific collection string to use as derived
667 from the supplied labels.
672 Strings to be combined with the instrument name to form a
678 Collection name to use that includes the instrument name.
680 return "/".join((cls.
getName(),) + labels)
684 """Construct an exposure DimensionRecord from
685 `astro_metadata_translator.ObservationInfo`.
689 obsInfo : `astro_metadata_translator.ObservationInfo`
690 A `~astro_metadata_translator.ObservationInfo` object corresponding to
692 universe : `DimensionUniverse`
693 Set of all known dimensions.
697 record : `DimensionRecord`
698 A record containing exposure metadata, suitable for insertion into
701 dimension = universe[
"exposure"]
703 ra, dec, sky_angle, zenith_angle = (
None,
None,
None,
None)
704 if obsInfo.tracking_radec
is not None:
705 icrs = obsInfo.tracking_radec.icrs
707 dec = icrs.dec.degree
708 if obsInfo.boresight_rotation_coord ==
"sky":
709 sky_angle = obsInfo.boresight_rotation_angle.degree
710 if obsInfo.altaz_begin
is not None:
711 zenith_angle = obsInfo.altaz_begin.zen.degree
713 return dimension.RecordClass(
714 instrument=obsInfo.instrument,
715 id=obsInfo.exposure_id,
716 name=obsInfo.observation_id,
717 group_name=obsInfo.exposure_group,
718 group_id=obsInfo.visit_id,
719 datetime_begin=obsInfo.datetime_begin,
720 datetime_end=obsInfo.datetime_end,
721 exposure_time=obsInfo.exposure_time.to_value(
"s"),
722 dark_time=obsInfo.dark_time.to_value(
"s"),
723 observation_type=obsInfo.observation_type,
724 physical_filter=obsInfo.physical_filter,
725 science_program=obsInfo.science_program,
726 target_name=obsInfo.object,
730 zenith_angle=zenith_angle,
734 def loadCamera(butler: Butler, dataId: DataId, *, collections: Any =
None) -> Tuple[Camera, bool]:
735 """Attempt to load versioned camera geometry from a butler, but fall back
736 to obtaining a nominal camera from the `Instrument` class if that fails.
740 butler : `lsst.daf.butler.Butler`
741 Butler instance to attempt to query for and load a ``camera`` dataset
743 dataId : `dict` or `DataCoordinate`
744 Data ID that identifies at least the ``instrument`` and ``exposure``
746 collections : Any, optional
747 Collections to be searched, overriding ``self.butler.collections``.
748 Can be any of the types supported by the ``collections`` argument
749 to butler construction.
753 camera : `lsst.afw.cameraGeom.Camera`
756 If `True`, the camera was obtained from the butler and should represent
757 a versioned camera from a calibration repository. If `False`, no
758 camera datasets were found, and the returned camera was produced by
759 instantiating the appropriate `Instrument` class and calling
760 `Instrument.getCamera`.
762 if collections
is None:
763 collections = butler.collections
768 dataId = butler.registry.expandDataId(dataId, graph=butler.registry.dimensions[
"exposure"].graph)
770 cameraRef = butler.get(
"camera", dataId=dataId, collections=collections)
771 return cameraRef,
True
774 instrument = Instrument.fromName(dataId[
"instrument"], butler.registry)
775 return instrument.getCamera(),
False