Coverage for python/lsst/obs/base/_instrument.py : 21%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ("Instrument", "makeExposureRecordFromObsInfo", "addUnboundedCalibrationLabel", "loadCamera")
26import os.path
27from abc import ABCMeta, abstractmethod
28from typing import Any, Tuple, TYPE_CHECKING
29import astropy.time
31from lsst.afw.cameraGeom import Camera
32from lsst.daf.butler import (
33 Butler,
34 CollectionType,
35 DataCoordinate,
36 DataId,
37 DatasetType,
38 TIMESPAN_MIN,
39 TIMESPAN_MAX,
40)
41from lsst.utils import getPackageDir, doImport
43if TYPE_CHECKING: 43 ↛ 44line 43 didn't jump to line 44, because the condition on line 43 was never true
44 from .gen2to3 import TranslatorFactory
45 from lsst.daf.butler import Registry
47# To be a standard text curated calibration means that we use a
48# standard definition for the corresponding DatasetType.
49StandardCuratedCalibrationDatasetTypes = {
50 "defects": {"dimensions": ("instrument", "detector", "calibration_label"),
51 "storageClass": "Defects"},
52 "qe_curve": {"dimensions": ("instrument", "detector", "calibration_label"),
53 "storageClass": "QECurve"},
54 "crosstalk": {"dimensions": ("instrument", "detector", "calibration_label"),
55 "storageClass": "CrosstalkCalib"},
56}
59class Instrument(metaclass=ABCMeta):
60 """Base class for instrument-specific logic for the Gen3 Butler.
62 Concrete instrument subclasses should be directly constructable with no
63 arguments.
64 """
66 configPaths = ()
67 """Paths to config files to read for specific Tasks.
69 The paths in this list should contain files of the form `task.py`, for
70 each of the Tasks that requires special configuration.
71 """
73 policyName = None
74 """Instrument specific name to use when locating a policy or configuration
75 file in the file system."""
77 obsDataPackage = None
78 """Name of the package containing the text curated calibration files.
79 Usually a obs _data package. If `None` no curated calibration files
80 will be read. (`str`)"""
82 standardCuratedDatasetTypes = tuple(StandardCuratedCalibrationDatasetTypes)
83 """The dataset types expected to be obtained from the obsDataPackage.
84 These dataset types are all required to have standard definitions and
85 must be known to the base class. Clearing this list will prevent
86 any of these calibrations from being stored. If a dataset type is not
87 known to a specific instrument it can still be included in this list
88 since the data package is the source of truth.
89 """
91 @property
92 @abstractmethod
93 def filterDefinitions(self):
94 """`~lsst.obs.base.FilterDefinitionCollection`, defining the filters
95 for this instrument.
96 """
97 return None
99 def __init__(self):
100 self.filterDefinitions.reset()
101 self.filterDefinitions.defineFilters()
102 self._obsDataPackageDir = None
104 @classmethod
105 @abstractmethod
106 def getName(cls):
107 """Return the short (dimension) name for this instrument.
109 This is not (in general) the same as the class name - it's what is used
110 as the value of the "instrument" field in data IDs, and is usually an
111 abbreviation of the full name.
112 """
113 raise NotImplementedError()
115 @abstractmethod
116 def getCamera(self):
117 """Retrieve the cameraGeom representation of this instrument.
119 This is a temporary API that should go away once ``obs_`` packages have
120 a standardized approach to writing versioned cameras to a Gen3 repo.
121 """
122 raise NotImplementedError()
124 @abstractmethod
125 def register(self, registry):
126 """Insert instrument, physical_filter, and detector entries into a
127 `Registry`.
128 """
129 raise NotImplementedError()
131 @property
132 def obsDataPackageDir(self):
133 """The root of the obs package that provides specializations for
134 this instrument (`str`).
135 """
136 if self.obsDataPackage is None:
137 return None
138 if self._obsDataPackageDir is None:
139 # Defer any problems with locating the package until
140 # we need to find it.
141 self._obsDataPackageDir = getPackageDir(self.obsDataPackage)
142 return self._obsDataPackageDir
144 @staticmethod
145 def fromName(name: str, registry: Registry) -> Instrument:
146 """Given an instrument name and a butler, retrieve a corresponding
147 instantiated instrument object.
149 Parameters
150 ----------
151 name : `str`
152 Name of the instrument (must match the return value of `getName`).
153 registry : `lsst.daf.butler.Registry`
154 Butler registry to query to find the information.
156 Returns
157 -------
158 instrument : `Instrument`
159 An instance of the relevant `Instrument`.
161 Notes
162 -----
163 The instrument must be registered in the corresponding butler.
165 Raises
166 ------
167 LookupError
168 Raised if the instrument is not known to the supplied registry.
169 ModuleNotFoundError
170 Raised if the class could not be imported. This could mean
171 that the relevant obs package has not been setup.
172 TypeError
173 Raised if the class name retrieved is not a string.
174 """
175 records = list(registry.queryDimensionRecords("instrument", instrument=name))
176 if not records:
177 raise LookupError(f"No registered instrument with name '{name}'.")
178 cls = records[0].class_name
179 if not isinstance(cls, str):
180 raise TypeError(f"Unexpected class name retrieved from {name} instrument dimension (got {cls})")
181 instrument = doImport(cls)
182 return instrument()
184 @staticmethod
185 def importAll(registry: Registry) -> None:
186 """Import all the instruments known to this registry.
188 This will ensure that all metadata translators have been registered.
190 Parameters
191 ----------
192 registry : `lsst.daf.butler.Registry`
193 Butler registry to query to find the information.
195 Notes
196 -----
197 It is allowed for a particular instrument class to fail on import.
198 This might simply indicate that a particular obs package has
199 not been setup.
200 """
201 records = list(registry.queryDimensionRecords("instrument"))
202 for record in records:
203 cls = record.class_name
204 try:
205 doImport(cls)
206 except Exception:
207 pass
209 def _registerFilters(self, registry):
210 """Register the physical and abstract filter Dimension relationships.
211 This should be called in the ``register`` implementation.
213 Parameters
214 ----------
215 registry : `lsst.daf.butler.core.Registry`
216 The registry to add dimensions to.
217 """
218 for filter in self.filterDefinitions:
219 # fix for undefined abstract filters causing trouble in the registry:
220 if filter.abstract_filter is None:
221 abstract_filter = filter.physical_filter
222 else:
223 abstract_filter = filter.abstract_filter
225 registry.insertDimensionData("physical_filter",
226 {"instrument": self.getName(),
227 "name": filter.physical_filter,
228 "abstract_filter": abstract_filter
229 })
231 @abstractmethod
232 def getRawFormatter(self, dataId):
233 """Return the Formatter class that should be used to read a particular
234 raw file.
236 Parameters
237 ----------
238 dataId : `DataCoordinate`
239 Dimension-based ID for the raw file or files being ingested.
241 Returns
242 -------
243 formatter : `Formatter` class
244 Class to be used that reads the file into an
245 `lsst.afw.image.Exposure` instance.
246 """
247 raise NotImplementedError()
249 def writeCuratedCalibrations(self, butler, run=None):
250 """Write human-curated calibration Datasets to the given Butler with
251 the appropriate validity ranges.
253 Parameters
254 ----------
255 butler : `lsst.daf.butler.Butler`
256 Butler to use to store these calibrations.
257 run : `str`
258 Run to use for this collection of calibrations. If `None` the
259 collection name is worked out automatically from the instrument
260 name and other metadata.
262 Notes
263 -----
264 Expected to be called from subclasses. The base method calls
265 ``writeCameraGeom`` and ``writeStandardTextCuratedCalibrations``.
266 """
267 # Need to determine the run for ingestion based on the instrument
268 # name and eventually the data package version. The camera geom
269 # is currently special in that it is not in the _data package.
270 if run is None:
271 run = self.makeCollectionName("calib")
272 butler.registry.registerCollection(run, type=CollectionType.RUN)
273 self.writeCameraGeom(butler, run=run)
274 self.writeStandardTextCuratedCalibrations(butler, run=run)
275 self.writeAdditionalCuratedCalibrations(butler, run=run)
277 def writeAdditionalCuratedCalibrations(self, butler, run=None):
278 """Write additional curated calibrations that might be instrument
279 specific and are not part of the standard set.
281 Default implementation does nothing.
283 Parameters
284 ----------
285 butler : `lsst.daf.butler.Butler`
286 Butler to use to store these calibrations.
287 run : `str`, optional
288 Name of the run to use to override the default run associated
289 with this Butler.
290 """
291 return
293 def applyConfigOverrides(self, name, config):
294 """Apply instrument-specific overrides for a task config.
296 Parameters
297 ----------
298 name : `str`
299 Name of the object being configured; typically the _DefaultName
300 of a Task.
301 config : `lsst.pex.config.Config`
302 Config instance to which overrides should be applied.
303 """
304 for root in self.configPaths:
305 path = os.path.join(root, f"{name}.py")
306 if os.path.exists(path):
307 config.load(path)
309 def writeCameraGeom(self, butler, run=None):
310 """Write the default camera geometry to the butler repository
311 with an infinite validity range.
313 Parameters
314 ----------
315 butler : `lsst.daf.butler.Butler`
316 Butler to receive these calibration datasets.
317 run : `str`, optional
318 Name of the run to use to override the default run associated
319 with this Butler.
320 """
322 datasetType = DatasetType("camera", ("instrument", "calibration_label"), "Camera",
323 universe=butler.registry.dimensions)
324 butler.registry.registerDatasetType(datasetType)
325 unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName())
326 camera = self.getCamera()
327 butler.put(camera, datasetType, unboundedDataId, run=run)
329 def writeStandardTextCuratedCalibrations(self, butler, run=None):
330 """Write the set of standardized curated text calibrations to
331 the repository.
333 Parameters
334 ----------
335 butler : `lsst.daf.butler.Butler`
336 Butler to receive these calibration datasets.
337 run : `str`, optional
338 Name of the run to use to override the default run associated
339 with this Butler.
340 """
342 for datasetTypeName in self.standardCuratedDatasetTypes:
343 # We need to define the dataset types.
344 if datasetTypeName not in StandardCuratedCalibrationDatasetTypes:
345 raise ValueError(f"DatasetType {datasetTypeName} not in understood list"
346 f" [{'.'.join(StandardCuratedCalibrationDatasetTypes)}]")
347 definition = StandardCuratedCalibrationDatasetTypes[datasetTypeName]
348 datasetType = DatasetType(datasetTypeName,
349 universe=butler.registry.dimensions,
350 **definition)
351 self._writeSpecificCuratedCalibrationDatasets(butler, datasetType, run=run)
353 def _writeSpecificCuratedCalibrationDatasets(self, butler, datasetType, run=None):
354 """Write standardized curated calibration datasets for this specific
355 dataset type from an obs data package.
357 Parameters
358 ----------
359 butler : `lsst.daf.butler.Butler`
360 Gen3 butler in which to put the calibrations.
361 datasetType : `lsst.daf.butler.DatasetType`
362 Dataset type to be put.
363 run : `str`, optional
364 Name of the run to use to override the default run associated
365 with this Butler.
367 Notes
368 -----
369 This method scans the location defined in the ``obsDataPackageDir``
370 class attribute for curated calibrations corresponding to the
371 supplied dataset type. The directory name in the data package must
372 match the name of the dataset type. They are assumed to use the
373 standard layout and can be read by
374 `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard
375 metadata.
376 """
377 if self.obsDataPackageDir is None:
378 # if there is no data package then there can't be datasets
379 return
381 calibPath = os.path.join(self.obsDataPackageDir, self.policyName,
382 datasetType.name)
384 if not os.path.exists(calibPath):
385 return
387 # Register the dataset type
388 butler.registry.registerDatasetType(datasetType)
390 # obs_base can't depend on pipe_tasks but concrete obs packages
391 # can -- we therefore have to defer import
392 from lsst.pipe.tasks.read_curated_calibs import read_all
394 camera = self.getCamera()
395 calibsDict = read_all(calibPath, camera)[0] # second return is calib type
396 endOfTime = TIMESPAN_MAX
397 dimensionRecords = []
398 datasetRecords = []
399 for det in calibsDict:
400 times = sorted([k for k in calibsDict[det]])
401 calibs = [calibsDict[det][time] for time in times]
402 times = [astropy.time.Time(t, format="datetime", scale="utc") for t in times]
403 times += [endOfTime]
404 for calib, beginTime, endTime in zip(calibs, times[:-1], times[1:]):
405 md = calib.getMetadata()
406 calibrationLabel = f"{datasetType.name}/{md['CALIBDATE']}/{md['DETECTOR']}"
407 dataId = DataCoordinate.standardize(
408 universe=butler.registry.dimensions,
409 instrument=self.getName(),
410 calibration_label=calibrationLabel,
411 detector=md["DETECTOR"],
412 )
413 datasetRecords.append((calib, dataId))
414 dimensionRecords.append({
415 "instrument": self.getName(),
416 "name": calibrationLabel,
417 "datetime_begin": beginTime,
418 "datetime_end": endTime,
419 })
421 # Second loop actually does the inserts and filesystem writes.
422 with butler.transaction():
423 butler.registry.insertDimensionData("calibration_label", *dimensionRecords)
424 # TODO: vectorize these puts, once butler APIs for that become
425 # available.
426 for calib, dataId in datasetRecords:
427 butler.put(calib, datasetType, dataId, run=run)
429 @abstractmethod
430 def makeDataIdTranslatorFactory(self) -> TranslatorFactory:
431 """Return a factory for creating Gen2->Gen3 data ID translators,
432 specialized for this instrument.
434 Derived class implementations should generally call
435 `TranslatorFactory.addGenericInstrumentRules` with appropriate
436 arguments, but are not required to (and may not be able to if their
437 Gen2 raw data IDs are sufficiently different from the HSC/DECam/CFHT
438 norm).
440 Returns
441 -------
442 factory : `TranslatorFactory`.
443 Factory for `Translator` objects.
444 """
445 raise NotImplementedError("Must be implemented by derived classes.")
447 @classmethod
448 def makeDefaultRawIngestRunName(cls) -> str:
449 """Make the default instrument-specific run collection string for raw
450 data ingest.
452 Returns
453 -------
454 coll : `str`
455 Run collection name to be used as the default for ingestion of
456 raws.
457 """
458 return cls.makeCollectionName("raw/all")
460 @classmethod
461 def makeCollectionName(cls, label: str) -> str:
462 """Get the instrument-specific collection string to use as derived
463 from the supplied label.
465 Parameters
466 ----------
467 label : `str`
468 String to be combined with the instrument name to form a
469 collection name.
471 Returns
472 -------
473 name : `str`
474 Collection name to use that includes the instrument name.
475 """
476 return f"{cls.getName()}/{label}"
479def makeExposureRecordFromObsInfo(obsInfo, universe):
480 """Construct an exposure DimensionRecord from
481 `astro_metadata_translator.ObservationInfo`.
483 Parameters
484 ----------
485 obsInfo : `astro_metadata_translator.ObservationInfo`
486 A `~astro_metadata_translator.ObservationInfo` object corresponding to
487 the exposure.
488 universe : `DimensionUniverse`
489 Set of all known dimensions.
491 Returns
492 -------
493 record : `DimensionRecord`
494 A record containing exposure metadata, suitable for insertion into
495 a `Registry`.
496 """
497 dimension = universe["exposure"]
499 ra, dec, sky_angle, zenith_angle = (None, None, None, None)
500 if obsInfo.tracking_radec is not None:
501 icrs = obsInfo.tracking_radec.icrs
502 ra = icrs.ra.degree
503 dec = icrs.dec.degree
504 if obsInfo.boresight_rotation_coord == "sky":
505 sky_angle = obsInfo.boresight_rotation_angle.degree
506 if obsInfo.altaz_begin is not None:
507 zenith_angle = obsInfo.altaz_begin.zen.degree
509 return dimension.RecordClass.fromDict({
510 "instrument": obsInfo.instrument,
511 "id": obsInfo.exposure_id,
512 "name": obsInfo.observation_id,
513 "group_name": obsInfo.exposure_group,
514 "group_id": obsInfo.visit_id,
515 "datetime_begin": obsInfo.datetime_begin,
516 "datetime_end": obsInfo.datetime_end,
517 "exposure_time": obsInfo.exposure_time.to_value("s"),
518 "dark_time": obsInfo.dark_time.to_value("s"),
519 "observation_type": obsInfo.observation_type,
520 "physical_filter": obsInfo.physical_filter,
521 "science_program": obsInfo.science_program,
522 "target_name": obsInfo.object,
523 "tracking_ra": ra,
524 "tracking_dec": dec,
525 "sky_angle": sky_angle,
526 "zenith_angle": zenith_angle,
527 })
530def addUnboundedCalibrationLabel(registry, instrumentName):
531 """Add a special 'unbounded' calibration_label dimension entry for the
532 given camera that is valid for any exposure.
534 If such an entry already exists, this function just returns a `DataId`
535 for the existing entry.
537 Parameters
538 ----------
539 registry : `Registry`
540 Registry object in which to insert the dimension entry.
541 instrumentName : `str`
542 Name of the instrument this calibration label is associated with.
544 Returns
545 -------
546 dataId : `DataId`
547 New or existing data ID for the unbounded calibration.
548 """
549 d = dict(instrument=instrumentName, calibration_label="unbounded")
550 try:
551 return registry.expandDataId(d)
552 except LookupError:
553 pass
554 entry = d.copy()
555 entry["datetime_begin"] = TIMESPAN_MIN
556 entry["datetime_end"] = TIMESPAN_MAX
557 registry.insertDimensionData("calibration_label", entry)
558 return registry.expandDataId(d)
561def loadCamera(butler: Butler, dataId: DataId, *, collections: Any = None) -> Tuple[Camera, bool]:
562 """Attempt to load versioned camera geometry from a butler, but fall back
563 to obtaining a nominal camera from the `Instrument` class if that fails.
565 Parameters
566 ----------
567 butler : `lsst.daf.butler.Butler`
568 Butler instance to attempt to query for and load a ``camera`` dataset
569 from.
570 dataId : `dict` or `DataCoordinate`
571 Data ID that identifies at least the ``instrument`` and ``exposure``
572 dimensions.
573 collections : Any, optional
574 Collections to be searched, overriding ``self.butler.collections``.
575 Can be any of the types supported by the ``collections`` argument
576 to butler construction.
578 Returns
579 -------
580 camera : `lsst.afw.cameraGeom.Camera`
581 Camera object.
582 versioned : `bool`
583 If `True`, the camera was obtained from the butler and should represent
584 a versioned camera from a calibration repository. If `False`, no
585 camera datasets were found, and the returned camera was produced by
586 instantiating the appropriate `Instrument` class and calling
587 `Instrument.getCamera`.
588 """
589 if collections is None:
590 collections = butler.collections
591 # Registry would do data ID expansion internally if we didn't do it first,
592 # but we might want an expanded data ID ourselves later, so we do it here
593 # to ensure it only happens once.
594 # This will also catch problems with the data ID not having keys we need.
595 dataId = butler.registry.expandDataId(dataId, graph=butler.registry.dimensions["exposure"].graph)
596 cameraRefs = list(butler.registry.queryDatasets("camera", dataId=dataId, collections=collections,
597 deduplicate=True))
598 if cameraRefs:
599 assert len(cameraRefs) == 1, "Should be guaranteed by deduplicate=True above."
600 return butler.getDirect(cameraRefs[0]), True
601 instrument = Instrument.fromName(dataId["instrument"], butler.registry)
602 return instrument.getCamera(), False