Coverage for python/lsst/obs/base/_instrument.py : 22%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ("Instrument", "makeExposureRecordFromObsInfo", "addUnboundedCalibrationLabel", "loadCamera")
26import os.path
27from abc import ABCMeta, abstractmethod
28from typing import Any, Optional, Set, Sequence, Tuple, TYPE_CHECKING
29import astropy.time
30from functools import lru_cache
32from lsst.afw.cameraGeom import Camera
33from lsst.daf.butler import (
34 Butler,
35 CollectionType,
36 DataCoordinate,
37 DataId,
38 DatasetType,
39 Timespan,
40)
41from lsst.utils import getPackageDir, doImport
43if TYPE_CHECKING: 43 ↛ 44line 43 didn't jump to line 44, because the condition on line 43 was never true
44 from .gen2to3 import TranslatorFactory
45 from lsst.daf.butler import Registry
47# To be a standard text curated calibration means that we use a
48# standard definition for the corresponding DatasetType.
49StandardCuratedCalibrationDatasetTypes = {
50 "defects": {"dimensions": ("instrument", "detector", "calibration_label"),
51 "storageClass": "Defects"},
52 "qe_curve": {"dimensions": ("instrument", "detector", "calibration_label"),
53 "storageClass": "QECurve"},
54 "crosstalk": {"dimensions": ("instrument", "detector", "calibration_label"),
55 "storageClass": "CrosstalkCalib"},
56}
59class Instrument(metaclass=ABCMeta):
60 """Base class for instrument-specific logic for the Gen3 Butler.
62 Concrete instrument subclasses should be directly constructable with no
63 arguments.
64 """
66 configPaths: Sequence[str] = ()
67 """Paths to config files to read for specific Tasks.
69 The paths in this list should contain files of the form `task.py`, for
70 each of the Tasks that requires special configuration.
71 """
73 policyName: Optional[str] = None
74 """Instrument specific name to use when locating a policy or configuration
75 file in the file system."""
77 obsDataPackage: Optional[str] = None
78 """Name of the package containing the text curated calibration files.
79 Usually a obs _data package. If `None` no curated calibration files
80 will be read. (`str`)"""
82 standardCuratedDatasetTypes: Set[str] = frozenset(StandardCuratedCalibrationDatasetTypes)
83 """The dataset types expected to be obtained from the obsDataPackage.
85 These dataset types are all required to have standard definitions and
86 must be known to the base class. Clearing this list will prevent
87 any of these calibrations from being stored. If a dataset type is not
88 known to a specific instrument it can still be included in this list
89 since the data package is the source of truth. (`set` of `str`)
90 """
92 additionalCuratedDatasetTypes: Set[str] = frozenset()
93 """Curated dataset types specific to this particular instrument that do
94 not follow the standard organization found in obs data packages.
96 These are the instrument-specific dataset types written by
97 `writeAdditionalCuratedCalibrations` in addition to the calibrations
98 found in obs data packages that follow the standard scheme.
99 (`set` of `str`)"""
101 @property
102 @abstractmethod
103 def filterDefinitions(self):
104 """`~lsst.obs.base.FilterDefinitionCollection`, defining the filters
105 for this instrument.
106 """
107 return None
109 def __init__(self):
110 self.filterDefinitions.reset()
111 self.filterDefinitions.defineFilters()
113 @classmethod
114 @abstractmethod
115 def getName(cls):
116 """Return the short (dimension) name for this instrument.
118 This is not (in general) the same as the class name - it's what is used
119 as the value of the "instrument" field in data IDs, and is usually an
120 abbreviation of the full name.
121 """
122 raise NotImplementedError()
124 @classmethod
125 @lru_cache()
126 def getCuratedCalibrationNames(cls) -> Set[str]:
127 """Return the names of all the curated calibration dataset types.
129 Returns
130 -------
131 names : `set` of `str`
132 The dataset type names of all curated calibrations. This will
133 include the standard curated calibrations even if the particular
134 instrument does not support them.
136 Notes
137 -----
138 The returned list does not indicate whether a particular dataset
139 is present in the Butler repository, simply that these are the
140 dataset types that are handled by ``writeCuratedCalibrations``.
141 """
143 # Camera is a special dataset type that is also handled as a
144 # curated calibration.
145 curated = {"camera"}
147 # Make a cursory attempt to filter out curated dataset types
148 # that are not present for this instrument
149 for datasetTypeName in cls.standardCuratedDatasetTypes:
150 calibPath = cls._getSpecificCuratedCalibrationPath(datasetTypeName)
151 if calibPath is not None:
152 curated.add(datasetTypeName)
154 curated.update(cls.additionalCuratedDatasetTypes)
155 return frozenset(curated)
157 @abstractmethod
158 def getCamera(self):
159 """Retrieve the cameraGeom representation of this instrument.
161 This is a temporary API that should go away once ``obs_`` packages have
162 a standardized approach to writing versioned cameras to a Gen3 repo.
163 """
164 raise NotImplementedError()
166 @abstractmethod
167 def register(self, registry):
168 """Insert instrument, physical_filter, and detector entries into a
169 `Registry`.
170 """
171 raise NotImplementedError()
173 @classmethod
174 @lru_cache()
175 def getObsDataPackageDir(cls):
176 """The root of the obs data package that provides specializations for
177 this instrument.
179 returns
180 -------
181 dir : `str`
182 The root of the relevat obs data package.
183 """
184 if cls.obsDataPackage is None:
185 return None
186 return getPackageDir(cls.obsDataPackage)
188 @staticmethod
189 def fromName(name: str, registry: Registry) -> Instrument:
190 """Given an instrument name and a butler, retrieve a corresponding
191 instantiated instrument object.
193 Parameters
194 ----------
195 name : `str`
196 Name of the instrument (must match the return value of `getName`).
197 registry : `lsst.daf.butler.Registry`
198 Butler registry to query to find the information.
200 Returns
201 -------
202 instrument : `Instrument`
203 An instance of the relevant `Instrument`.
205 Notes
206 -----
207 The instrument must be registered in the corresponding butler.
209 Raises
210 ------
211 LookupError
212 Raised if the instrument is not known to the supplied registry.
213 ModuleNotFoundError
214 Raised if the class could not be imported. This could mean
215 that the relevant obs package has not been setup.
216 TypeError
217 Raised if the class name retrieved is not a string.
218 """
219 records = list(registry.queryDimensionRecords("instrument", instrument=name))
220 if not records:
221 raise LookupError(f"No registered instrument with name '{name}'.")
222 cls = records[0].class_name
223 if not isinstance(cls, str):
224 raise TypeError(f"Unexpected class name retrieved from {name} instrument dimension (got {cls})")
225 instrument = doImport(cls)
226 return instrument()
228 @staticmethod
229 def importAll(registry: Registry) -> None:
230 """Import all the instruments known to this registry.
232 This will ensure that all metadata translators have been registered.
234 Parameters
235 ----------
236 registry : `lsst.daf.butler.Registry`
237 Butler registry to query to find the information.
239 Notes
240 -----
241 It is allowed for a particular instrument class to fail on import.
242 This might simply indicate that a particular obs package has
243 not been setup.
244 """
245 records = list(registry.queryDimensionRecords("instrument"))
246 for record in records:
247 cls = record.class_name
248 try:
249 doImport(cls)
250 except Exception:
251 pass
253 def _registerFilters(self, registry):
254 """Register the physical and abstract filter Dimension relationships.
255 This should be called in the ``register`` implementation.
257 Parameters
258 ----------
259 registry : `lsst.daf.butler.core.Registry`
260 The registry to add dimensions to.
261 """
262 for filter in self.filterDefinitions:
263 # fix for undefined abstract filters causing trouble in the registry:
264 if filter.abstract_filter is None:
265 abstract_filter = filter.physical_filter
266 else:
267 abstract_filter = filter.abstract_filter
269 registry.insertDimensionData("physical_filter",
270 {"instrument": self.getName(),
271 "name": filter.physical_filter,
272 "abstract_filter": abstract_filter
273 })
275 @abstractmethod
276 def getRawFormatter(self, dataId):
277 """Return the Formatter class that should be used to read a particular
278 raw file.
280 Parameters
281 ----------
282 dataId : `DataCoordinate`
283 Dimension-based ID for the raw file or files being ingested.
285 Returns
286 -------
287 formatter : `Formatter` class
288 Class to be used that reads the file into an
289 `lsst.afw.image.Exposure` instance.
290 """
291 raise NotImplementedError()
293 def writeCuratedCalibrations(self, butler, run=None):
294 """Write human-curated calibration Datasets to the given Butler with
295 the appropriate validity ranges.
297 Parameters
298 ----------
299 butler : `lsst.daf.butler.Butler`
300 Butler to use to store these calibrations.
301 run : `str`
302 Run to use for this collection of calibrations. If `None` the
303 collection name is worked out automatically from the instrument
304 name and other metadata.
306 Notes
307 -----
308 Expected to be called from subclasses. The base method calls
309 ``writeCameraGeom`` and ``writeStandardTextCuratedCalibrations``.
310 """
311 # Need to determine the run for ingestion based on the instrument
312 # name and eventually the data package version. The camera geom
313 # is currently special in that it is not in the _data package.
314 if run is None:
315 run = self.makeCollectionName("calib")
316 butler.registry.registerCollection(run, type=CollectionType.RUN)
317 self.writeCameraGeom(butler, run=run)
318 self.writeStandardTextCuratedCalibrations(butler, run=run)
319 self.writeAdditionalCuratedCalibrations(butler, run=run)
321 def writeAdditionalCuratedCalibrations(self, butler, run=None):
322 """Write additional curated calibrations that might be instrument
323 specific and are not part of the standard set.
325 Default implementation does nothing.
327 Parameters
328 ----------
329 butler : `lsst.daf.butler.Butler`
330 Butler to use to store these calibrations.
331 run : `str`, optional
332 Name of the run to use to override the default run associated
333 with this Butler.
334 """
335 return
337 def applyConfigOverrides(self, name, config):
338 """Apply instrument-specific overrides for a task config.
340 Parameters
341 ----------
342 name : `str`
343 Name of the object being configured; typically the _DefaultName
344 of a Task.
345 config : `lsst.pex.config.Config`
346 Config instance to which overrides should be applied.
347 """
348 for root in self.configPaths:
349 path = os.path.join(root, f"{name}.py")
350 if os.path.exists(path):
351 config.load(path)
353 def writeCameraGeom(self, butler, run=None):
354 """Write the default camera geometry to the butler repository
355 with an infinite validity range.
357 Parameters
358 ----------
359 butler : `lsst.daf.butler.Butler`
360 Butler to receive these calibration datasets.
361 run : `str`, optional
362 Name of the run to use to override the default run associated
363 with this Butler.
364 """
366 datasetType = DatasetType("camera", ("instrument", "calibration_label"), "Camera",
367 universe=butler.registry.dimensions)
368 butler.registry.registerDatasetType(datasetType)
369 unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName())
370 camera = self.getCamera()
371 butler.put(camera, datasetType, unboundedDataId, run=run)
373 def writeStandardTextCuratedCalibrations(self, butler, run=None):
374 """Write the set of standardized curated text calibrations to
375 the repository.
377 Parameters
378 ----------
379 butler : `lsst.daf.butler.Butler`
380 Butler to receive these calibration datasets.
381 run : `str`, optional
382 Name of the run to use to override the default run associated
383 with this Butler.
384 """
386 for datasetTypeName in self.standardCuratedDatasetTypes:
387 # We need to define the dataset types.
388 if datasetTypeName not in StandardCuratedCalibrationDatasetTypes:
389 raise ValueError(f"DatasetType {datasetTypeName} not in understood list"
390 f" [{'.'.join(StandardCuratedCalibrationDatasetTypes)}]")
391 definition = StandardCuratedCalibrationDatasetTypes[datasetTypeName]
392 datasetType = DatasetType(datasetTypeName,
393 universe=butler.registry.dimensions,
394 **definition)
395 self._writeSpecificCuratedCalibrationDatasets(butler, datasetType, run=run)
397 @classmethod
398 def _getSpecificCuratedCalibrationPath(cls, datasetTypeName):
399 """Return the path of the curated calibration directory.
401 Parameters
402 ----------
403 datasetTypeName : `str`
404 The name of the standard dataset type to find.
406 Returns
407 -------
408 path : `str`
409 The path to the standard curated data directory. `None` if the
410 dataset type is not found or the obs data package is not
411 available.
412 """
413 if cls.getObsDataPackageDir() is None:
414 # if there is no data package then there can't be datasets
415 return None
417 calibPath = os.path.join(cls.getObsDataPackageDir(), cls.policyName,
418 datasetTypeName)
420 if os.path.exists(calibPath):
421 return calibPath
423 return None
425 def _writeSpecificCuratedCalibrationDatasets(self, butler, datasetType, run=None):
426 """Write standardized curated calibration datasets for this specific
427 dataset type from an obs data package.
429 Parameters
430 ----------
431 butler : `lsst.daf.butler.Butler`
432 Gen3 butler in which to put the calibrations.
433 datasetType : `lsst.daf.butler.DatasetType`
434 Dataset type to be put.
435 run : `str`, optional
436 Name of the run to use to override the default run associated
437 with this Butler.
439 Notes
440 -----
441 This method scans the location defined in the ``obsDataPackageDir``
442 class attribute for curated calibrations corresponding to the
443 supplied dataset type. The directory name in the data package must
444 match the name of the dataset type. They are assumed to use the
445 standard layout and can be read by
446 `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard
447 metadata.
448 """
449 calibPath = self._getSpecificCuratedCalibrationPath(datasetType.name)
450 if calibPath is None:
451 return
453 # Register the dataset type
454 butler.registry.registerDatasetType(datasetType)
456 # obs_base can't depend on pipe_tasks but concrete obs packages
457 # can -- we therefore have to defer import
458 from lsst.pipe.tasks.read_curated_calibs import read_all
460 camera = self.getCamera()
461 calibsDict = read_all(calibPath, camera)[0] # second return is calib type
462 dimensionRecords = []
463 datasetRecords = []
464 for det in calibsDict:
465 times = sorted([k for k in calibsDict[det]])
466 calibs = [calibsDict[det][time] for time in times]
467 times = [astropy.time.Time(t, format="datetime", scale="utc") for t in times]
468 times += [None]
469 for calib, beginTime, endTime in zip(calibs, times[:-1], times[1:]):
470 md = calib.getMetadata()
471 calibrationLabel = f"{datasetType.name}/{md['CALIBDATE']}/{md['DETECTOR']}"
472 dataId = DataCoordinate.standardize(
473 universe=butler.registry.dimensions,
474 instrument=self.getName(),
475 calibration_label=calibrationLabel,
476 detector=md["DETECTOR"],
477 )
478 datasetRecords.append((calib, dataId))
479 dimensionRecords.append({
480 "instrument": self.getName(),
481 "name": calibrationLabel,
482 "timespan": Timespan(beginTime, endTime),
483 })
485 # Second loop actually does the inserts and filesystem writes.
486 with butler.transaction():
487 butler.registry.insertDimensionData("calibration_label", *dimensionRecords)
488 # TODO: vectorize these puts, once butler APIs for that become
489 # available.
490 for calib, dataId in datasetRecords:
491 butler.put(calib, datasetType, dataId, run=run)
493 @abstractmethod
494 def makeDataIdTranslatorFactory(self) -> TranslatorFactory:
495 """Return a factory for creating Gen2->Gen3 data ID translators,
496 specialized for this instrument.
498 Derived class implementations should generally call
499 `TranslatorFactory.addGenericInstrumentRules` with appropriate
500 arguments, but are not required to (and may not be able to if their
501 Gen2 raw data IDs are sufficiently different from the HSC/DECam/CFHT
502 norm).
504 Returns
505 -------
506 factory : `TranslatorFactory`.
507 Factory for `Translator` objects.
508 """
509 raise NotImplementedError("Must be implemented by derived classes.")
511 @classmethod
512 def makeDefaultRawIngestRunName(cls) -> str:
513 """Make the default instrument-specific run collection string for raw
514 data ingest.
516 Returns
517 -------
518 coll : `str`
519 Run collection name to be used as the default for ingestion of
520 raws.
521 """
522 return cls.makeCollectionName("raw/all")
524 @classmethod
525 def makeCollectionName(cls, label: str) -> str:
526 """Get the instrument-specific collection string to use as derived
527 from the supplied label.
529 Parameters
530 ----------
531 label : `str`
532 String to be combined with the instrument name to form a
533 collection name.
535 Returns
536 -------
537 name : `str`
538 Collection name to use that includes the instrument name.
539 """
540 return f"{cls.getName()}/{label}"
543def makeExposureRecordFromObsInfo(obsInfo, universe):
544 """Construct an exposure DimensionRecord from
545 `astro_metadata_translator.ObservationInfo`.
547 Parameters
548 ----------
549 obsInfo : `astro_metadata_translator.ObservationInfo`
550 A `~astro_metadata_translator.ObservationInfo` object corresponding to
551 the exposure.
552 universe : `DimensionUniverse`
553 Set of all known dimensions.
555 Returns
556 -------
557 record : `DimensionRecord`
558 A record containing exposure metadata, suitable for insertion into
559 a `Registry`.
560 """
561 dimension = universe["exposure"]
563 ra, dec, sky_angle, zenith_angle = (None, None, None, None)
564 if obsInfo.tracking_radec is not None:
565 icrs = obsInfo.tracking_radec.icrs
566 ra = icrs.ra.degree
567 dec = icrs.dec.degree
568 if obsInfo.boresight_rotation_coord == "sky":
569 sky_angle = obsInfo.boresight_rotation_angle.degree
570 if obsInfo.altaz_begin is not None:
571 zenith_angle = obsInfo.altaz_begin.zen.degree
573 return dimension.RecordClass(
574 instrument=obsInfo.instrument,
575 id=obsInfo.exposure_id,
576 name=obsInfo.observation_id,
577 group_name=obsInfo.exposure_group,
578 group_id=obsInfo.visit_id,
579 datetime_begin=obsInfo.datetime_begin,
580 datetime_end=obsInfo.datetime_end,
581 exposure_time=obsInfo.exposure_time.to_value("s"),
582 dark_time=obsInfo.dark_time.to_value("s"),
583 observation_type=obsInfo.observation_type,
584 physical_filter=obsInfo.physical_filter,
585 science_program=obsInfo.science_program,
586 target_name=obsInfo.object,
587 tracking_ra=ra,
588 tracking_dec=dec,
589 sky_angle=sky_angle,
590 zenith_angle=zenith_angle,
591 )
594def addUnboundedCalibrationLabel(registry, instrumentName):
595 """Add a special 'unbounded' calibration_label dimension entry for the
596 given camera that is valid for any exposure.
598 If such an entry already exists, this function just returns a `DataId`
599 for the existing entry.
601 Parameters
602 ----------
603 registry : `Registry`
604 Registry object in which to insert the dimension entry.
605 instrumentName : `str`
606 Name of the instrument this calibration label is associated with.
608 Returns
609 -------
610 dataId : `DataId`
611 New or existing data ID for the unbounded calibration.
612 """
613 d = dict(instrument=instrumentName, calibration_label="unbounded")
614 try:
615 return registry.expandDataId(d)
616 except LookupError:
617 pass
618 entry = d.copy()
619 entry["timespan"] = Timespan(None, None)
620 registry.insertDimensionData("calibration_label", entry)
621 return registry.expandDataId(d)
624def loadCamera(butler: Butler, dataId: DataId, *, collections: Any = None) -> Tuple[Camera, bool]:
625 """Attempt to load versioned camera geometry from a butler, but fall back
626 to obtaining a nominal camera from the `Instrument` class if that fails.
628 Parameters
629 ----------
630 butler : `lsst.daf.butler.Butler`
631 Butler instance to attempt to query for and load a ``camera`` dataset
632 from.
633 dataId : `dict` or `DataCoordinate`
634 Data ID that identifies at least the ``instrument`` and ``exposure``
635 dimensions.
636 collections : Any, optional
637 Collections to be searched, overriding ``self.butler.collections``.
638 Can be any of the types supported by the ``collections`` argument
639 to butler construction.
641 Returns
642 -------
643 camera : `lsst.afw.cameraGeom.Camera`
644 Camera object.
645 versioned : `bool`
646 If `True`, the camera was obtained from the butler and should represent
647 a versioned camera from a calibration repository. If `False`, no
648 camera datasets were found, and the returned camera was produced by
649 instantiating the appropriate `Instrument` class and calling
650 `Instrument.getCamera`.
651 """
652 if collections is None:
653 collections = butler.collections
654 # Registry would do data ID expansion internally if we didn't do it first,
655 # but we might want an expanded data ID ourselves later, so we do it here
656 # to ensure it only happens once.
657 # This will also catch problems with the data ID not having keys we need.
658 dataId = butler.registry.expandDataId(dataId, graph=butler.registry.dimensions["exposure"].graph)
659 cameraRefs = list(butler.registry.queryDatasets("camera", dataId=dataId, collections=collections,
660 deduplicate=True))
661 if cameraRefs:
662 assert len(cameraRefs) == 1, "Should be guaranteed by deduplicate=True above."
663 return butler.getDirect(cameraRefs[0]), True
664 instrument = Instrument.fromName(dataId["instrument"], butler.registry)
665 return instrument.getCamera(), False