22 from __future__
import annotations
24 __all__ = (
"Instrument",
"makeExposureRecordFromObsInfo",
"addUnboundedCalibrationLabel",
"loadCamera")
27 from abc
import ABCMeta, abstractmethod
28 from typing
import Any, Tuple, TYPE_CHECKING
31 from lsst.afw.cameraGeom
import Camera
32 from lsst.daf.butler
import Butler, DataId, TIMESPAN_MIN, TIMESPAN_MAX, DatasetType, DataCoordinate
36 from .gen2to3
import TranslatorFactory
37 from lsst.daf.butler
import Registry
41 StandardCuratedCalibrationDatasetTypes = {
42 "defects": {
"dimensions": (
"instrument",
"detector",
"calibration_label"),
43 "storageClass":
"Defects"},
44 "qe_curve": {
"dimensions": (
"instrument",
"detector",
"calibration_label"),
45 "storageClass":
"QECurve"},
46 "crosstalk": {
"dimensions": (
"instrument",
"detector",
"calibration_label"),
47 "storageClass":
"CrosstalkCalib"},
52 """Base class for instrument-specific logic for the Gen3 Butler.
54 Concrete instrument subclasses should be directly constructable with no
59 """Paths to config files to read for specific Tasks.
61 The paths in this list should contain files of the form `task.py`, for
62 each of the Tasks that requires special configuration.
66 """Instrument specific name to use when locating a policy or configuration
67 file in the file system."""
70 """Name of the package containing the text curated calibration files.
71 Usually a obs _data package. If `None` no curated calibration files
72 will be read. (`str`)"""
74 standardCuratedDatasetTypes = tuple(StandardCuratedCalibrationDatasetTypes)
75 """The dataset types expected to be obtained from the obsDataPackage.
76 These dataset types are all required to have standard definitions and
77 must be known to the base class. Clearing this list will prevent
78 any of these calibrations from being stored. If a dataset type is not
79 known to a specific instrument it can still be included in this list
80 since the data package is the source of truth.
86 """`~lsst.obs.base.FilterDefinitionCollection`, defining the filters
99 """Return the short (dimension) name for this instrument.
101 This is not (in general) the same as the class name - it's what is used
102 as the value of the "instrument" field in data IDs, and is usually an
103 abbreviation of the full name.
105 raise NotImplementedError()
109 """Retrieve the cameraGeom representation of this instrument.
111 This is a temporary API that should go away once ``obs_`` packages have
112 a standardized approach to writing versioned cameras to a Gen3 repo.
114 raise NotImplementedError()
118 """Insert instrument, physical_filter, and detector entries into a
121 raise NotImplementedError()
125 """The root of the obs package that provides specializations for
126 this instrument (`str`).
137 def fromName(name: str, registry: Registry) -> Instrument:
138 """Given an instrument name and a butler, retrieve a corresponding
139 instantiated instrument object.
144 Name of the instrument (must match the return value of `getName`).
145 registry : `lsst.daf.butler.Registry`
146 Butler registry to query to find the information.
150 instrument : `Instrument`
151 An instance of the relevant `Instrument`.
155 The instrument must be registered in the corresponding butler.
160 Raised if the instrument is not known to the supplied registry.
162 Raised if the class could not be imported. This could mean
163 that the relevant obs package has not been setup.
165 Raised if the class name retrieved is not a string.
167 dimensions = list(registry.queryDimensions(
"instrument", dataId={
"instrument": name}))
168 cls = dimensions[0].records[
"instrument"].class_name
169 if not isinstance(cls, str):
170 raise TypeError(f
"Unexpected class name retrieved from {name} instrument dimension (got {cls})")
171 instrument = doImport(cls)
176 """Import all the instruments known to this registry.
178 This will ensure that all metadata translators have been registered.
182 registry : `lsst.daf.butler.Registry`
183 Butler registry to query to find the information.
187 It is allowed for a particular instrument class to fail on import.
188 This might simply indicate that a particular obs package has
191 dimensions = list(registry.queryDimensions(
"instrument"))
192 for dim
in dimensions:
193 cls = dim.records[
"instrument"].class_name
199 def _registerFilters(self, registry):
200 """Register the physical and abstract filter Dimension relationships.
201 This should be called in the ``register`` implementation.
205 registry : `lsst.daf.butler.core.Registry`
206 The registry to add dimensions to.
210 if filter.abstract_filter
is None:
211 abstract_filter = filter.physical_filter
213 abstract_filter = filter.abstract_filter
215 registry.insertDimensionData(
"physical_filter",
217 "name": filter.physical_filter,
218 "abstract_filter": abstract_filter
223 """Return the Formatter class that should be used to read a particular
228 dataId : `DataCoordinate`
229 Dimension-based ID for the raw file or files being ingested.
233 formatter : `Formatter` class
234 Class to be used that reads the file into an
235 `lsst.afw.image.Exposure` instance.
237 raise NotImplementedError()
240 """Write human-curated calibration Datasets to the given Butler with
241 the appropriate validity ranges.
245 butler : `lsst.daf.butler.Butler`
246 Butler to use to store these calibrations.
250 Expected to be called from subclasses. The base method calls
251 ``writeCameraGeom`` and ``writeStandardTextCuratedCalibrations``.
257 """Apply instrument-specific overrides for a task config.
262 Name of the object being configured; typically the _DefaultName
264 config : `lsst.pex.config.Config`
265 Config instance to which overrides should be applied.
268 path = os.path.join(root, f
"{name}.py")
269 if os.path.exists(path):
273 """Write the default camera geometry to the butler repository
274 with an infinite validity range.
278 butler : `lsst.daf.butler.Butler`
279 Butler to receive these calibration datasets.
282 datasetType = DatasetType(
"camera", (
"instrument",
"calibration_label"),
"Camera",
283 universe=butler.registry.dimensions)
284 butler.registry.registerDatasetType(datasetType)
287 butler.put(camera, datasetType, unboundedDataId)
290 """Write the set of standardized curated text calibrations to
295 butler : `lsst.daf.butler.Butler`
296 Butler to receive these calibration datasets.
301 if datasetTypeName
not in StandardCuratedCalibrationDatasetTypes:
302 raise ValueError(f
"DatasetType {datasetTypeName} not in understood list"
303 f
" [{'.'.join(StandardCuratedCalibrationDatasetTypes)}]")
304 definition = StandardCuratedCalibrationDatasetTypes[datasetTypeName]
305 datasetType = DatasetType(datasetTypeName,
306 universe=butler.registry.dimensions,
310 def _writeSpecificCuratedCalibrationDatasets(self, butler, datasetType):
311 """Write standardized curated calibration datasets for this specific
312 dataset type from an obs data package.
316 butler : `lsst.daf.butler.Butler`
317 Gen3 butler in which to put the calibrations.
318 datasetType : `lsst.daf.butler.DatasetType`
319 Dataset type to be put.
323 This method scans the location defined in the ``obsDataPackageDir``
324 class attribute for curated calibrations corresponding to the
325 supplied dataset type. The directory name in the data package must
326 match the name of the dataset type. They are assumed to use the
327 standard layout and can be read by
328 `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard
338 if not os.path.exists(calibPath):
342 butler.registry.registerDatasetType(datasetType)
346 from lsst.pipe.tasks.read_curated_calibs
import read_all
349 calibsDict = read_all(calibPath, camera)[0]
350 endOfTime = TIMESPAN_MAX
351 dimensionRecords = []
353 for det
in calibsDict:
354 times = sorted([k
for k
in calibsDict[det]])
355 calibs = [calibsDict[det][time]
for time
in times]
356 times = [astropy.time.Time(t, format=
"datetime", scale=
"utc")
for t
in times]
358 for calib, beginTime, endTime
in zip(calibs, times[:-1], times[1:]):
359 md = calib.getMetadata()
360 calibrationLabel = f
"{datasetType.name}/{md['CALIBDATE']}/{md['DETECTOR']}"
361 dataId = DataCoordinate.standardize(
362 universe=butler.registry.dimensions,
364 calibration_label=calibrationLabel,
365 detector=md[
"DETECTOR"],
367 datasetRecords.append((calib, dataId))
368 dimensionRecords.append({
370 "name": calibrationLabel,
371 "datetime_begin": beginTime,
372 "datetime_end": endTime,
376 with butler.transaction():
377 butler.registry.insertDimensionData(
"calibration_label", *dimensionRecords)
380 for calib, dataId
in datasetRecords:
381 butler.put(calib, datasetType, dataId)
385 """Return a factory for creating Gen2->Gen3 data ID translators,
386 specialized for this instrument.
388 Derived class implementations should generally call
389 `TranslatorFactory.addGenericInstrumentRules` with appropriate
390 arguments, but are not required to (and may not be able to if their
391 Gen2 raw data IDs are sufficiently different from the HSC/DECam/CFHT
396 factory : `TranslatorFactory`.
397 Factory for `Translator` objects.
399 raise NotImplementedError(
"Must be implemented by derived classes.")
403 """Construct an exposure DimensionRecord from
404 `astro_metadata_translator.ObservationInfo`.
408 obsInfo : `astro_metadata_translator.ObservationInfo`
409 A `~astro_metadata_translator.ObservationInfo` object corresponding to
411 universe : `DimensionUniverse`
412 Set of all known dimensions.
416 record : `DimensionRecord`
417 A record containing exposure metadata, suitable for insertion into
420 dimension = universe[
"exposure"]
421 return dimension.RecordClass.fromDict({
422 "instrument": obsInfo.instrument,
423 "id": obsInfo.exposure_id,
424 "name": obsInfo.observation_id,
425 "group_name": obsInfo.exposure_group,
426 "group_id": obsInfo.visit_id,
427 "datetime_begin": obsInfo.datetime_begin,
428 "datetime_end": obsInfo.datetime_end,
429 "exposure_time": obsInfo.exposure_time.to_value(
"s"),
430 "dark_time": obsInfo.dark_time.to_value(
"s"),
431 "observation_type": obsInfo.observation_type,
432 "physical_filter": obsInfo.physical_filter,
437 """Add a special 'unbounded' calibration_label dimension entry for the
438 given camera that is valid for any exposure.
440 If such an entry already exists, this function just returns a `DataId`
441 for the existing entry.
445 registry : `Registry`
446 Registry object in which to insert the dimension entry.
447 instrumentName : `str`
448 Name of the instrument this calibration label is associated with.
453 New or existing data ID for the unbounded calibration.
455 d = dict(instrument=instrumentName, calibration_label=
"unbounded")
457 return registry.expandDataId(d)
461 entry[
"datetime_begin"] = TIMESPAN_MIN
462 entry[
"datetime_end"] = TIMESPAN_MAX
463 registry.insertDimensionData(
"calibration_label", entry)
464 return registry.expandDataId(d)
467 def loadCamera(butler: Butler, dataId: DataId, *, collections: Any =
None) -> Tuple[Camera, bool]:
468 """Attempt to load versioned camera geometry from a butler, but fall back
469 to obtaining a nominal camera from the `Instrument` class if that fails.
473 butler : `lsst.daf.butler.Butler`
474 Butler instance to attempt to query for and load a ``camera`` dataset
476 dataId : `dict` or `DataCoordinate`
477 Data ID that identifies at least the ``instrument`` and ``exposure``
479 collections : Any, optional
480 Collections to be searched, overriding ``self.butler.collections``.
481 Can be any of the types supported by the ``collections`` argument
482 to butler construction.
486 camera : `lsst.afw.cameraGeom.Camera`
489 If `True`, the camera was obtained from the butler and should represent
490 a versioned camera from a calibration repository. If `False`, no
491 camera datasets were found, and the returned camera was produced by
492 instantiating the appropriate `Instrument` class and calling
493 `Instrument.getCamera`.
495 if collections
is None:
496 collections = butler.collections
501 dataId = butler.registry.expandDataId(dataId, graph=butler.registry.dimensions[
"exposure"].graph)
502 cameraRefs = list(butler.registry.queryDatasets(
"camera", dataId=dataId, collections=collections,
505 assert len(cameraRefs) == 1,
"Should be guaranteed by deduplicate=True above."
506 return butler.getDirect(cameraRefs[0]),
True
507 instrument = Instrument.fromName(dataId[
"instrument"], butler.registry)
508 return instrument.getCamera(),
False