Coverage for python/lsst/obs/base/_instrument.py: 25%
158 statements
« prev ^ index » next coverage.py v6.4, created at 2022-05-27 11:22 +0000
« prev ^ index » next coverage.py v6.4, created at 2022-05-27 11:22 +0000
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ("Instrument", "makeExposureRecordFromObsInfo", "loadCamera")
26import os.path
27from abc import abstractmethod
28from collections import defaultdict
29from functools import lru_cache
30from typing import TYPE_CHECKING, AbstractSet, Any, FrozenSet, Optional, Sequence, Set, Tuple
32import astropy.time
33from lsst.afw.cameraGeom import Camera
34from lsst.daf.butler import (
35 Butler,
36 CollectionType,
37 DataCoordinate,
38 DataId,
39 DatasetType,
40 DimensionRecord,
41 DimensionUniverse,
42 Timespan,
43)
44from lsst.daf.butler.registry import DataIdError
45from lsst.pipe.base import Instrument as InstrumentBase
46from lsst.utils import getPackageDir
48if TYPE_CHECKING: 48 ↛ 49line 48 didn't jump to line 49, because the condition on line 48 was never true
49 from astro_metadata_translator import ObservationInfo
50 from lsst.daf.butler import Registry
52 from .filters import FilterDefinitionCollection
53 from .gen2to3 import TranslatorFactory
55# To be a standard text curated calibration means that we use a
56# standard definition for the corresponding DatasetType.
57StandardCuratedCalibrationDatasetTypes = {
58 "defects": {"dimensions": ("instrument", "detector"), "storageClass": "Defects"},
59 "qe_curve": {"dimensions": ("instrument", "detector"), "storageClass": "QECurve"},
60 "crosstalk": {"dimensions": ("instrument", "detector"), "storageClass": "CrosstalkCalib"},
61 "linearizer": {"dimensions": ("instrument", "detector"), "storageClass": "Linearizer"},
62 "bfk": {"dimensions": ("instrument", "detector"), "storageClass": "BrighterFatterKernel"},
63}
66class Instrument(InstrumentBase):
67 """Rubin-specified base for instrument-specific logic for the Gen3 Butler.
69 Parameters
70 ----------
71 collection_prefix : `str`, optional
72 Prefix for collection names to use instead of the intrument's own name.
73 This is primarily for use in simulated-data repositories, where the
74 instrument name may not be necessary and/or sufficient to distinguish
75 between collections.
77 Notes
78 -----
79 Concrete instrument subclasses must have the same construction signature as
80 the base class.
81 """
83 policyName: Optional[str] = None
84 """Instrument specific name to use when locating a policy or configuration
85 file in the file system."""
87 obsDataPackage: Optional[str] = None
88 """Name of the package containing the text curated calibration files.
89 Usually a obs _data package. If `None` no curated calibration files
90 will be read. (`str`)"""
92 standardCuratedDatasetTypes: AbstractSet[str] = frozenset(StandardCuratedCalibrationDatasetTypes)
93 """The dataset types expected to be obtained from the obsDataPackage.
95 These dataset types are all required to have standard definitions and
96 must be known to the base class. Clearing this list will prevent
97 any of these calibrations from being stored. If a dataset type is not
98 known to a specific instrument it can still be included in this list
99 since the data package is the source of truth. (`set` of `str`)
100 """
102 additionalCuratedDatasetTypes: AbstractSet[str] = frozenset()
103 """Curated dataset types specific to this particular instrument that do
104 not follow the standard organization found in obs data packages.
106 These are the instrument-specific dataset types written by
107 `writeAdditionalCuratedCalibrations` in addition to the calibrations
108 found in obs data packages that follow the standard scheme.
109 (`set` of `str`)"""
111 @property
112 @abstractmethod
113 def filterDefinitions(self) -> FilterDefinitionCollection:
114 """`~lsst.obs.base.FilterDefinitionCollection`, defining the filters
115 for this instrument.
116 """
117 raise NotImplementedError()
119 def __init__(self, collection_prefix: Optional[str] = None):
120 super().__init__(collection_prefix=collection_prefix)
121 self.filterDefinitions.reset()
122 self.filterDefinitions.defineFilters()
124 @classmethod
125 @lru_cache()
126 def getCuratedCalibrationNames(cls) -> FrozenSet[str]:
127 """Return the names of all the curated calibration dataset types.
129 Returns
130 -------
131 names : `frozenset` of `str`
132 The dataset type names of all curated calibrations. This will
133 include the standard curated calibrations even if the particular
134 instrument does not support them.
136 Notes
137 -----
138 The returned list does not indicate whether a particular dataset
139 is present in the Butler repository, simply that these are the
140 dataset types that are handled by ``writeCuratedCalibrations``.
141 """
143 # Camera is a special dataset type that is also handled as a
144 # curated calibration.
145 curated = {"camera"}
147 # Make a cursory attempt to filter out curated dataset types
148 # that are not present for this instrument
149 for datasetTypeName in cls.standardCuratedDatasetTypes:
150 calibPath = cls._getSpecificCuratedCalibrationPath(datasetTypeName)
151 if calibPath is not None:
152 curated.add(datasetTypeName)
154 curated.update(cls.additionalCuratedDatasetTypes)
155 return frozenset(curated)
157 @abstractmethod
158 def getCamera(self) -> Camera:
159 """Retrieve the cameraGeom representation of this instrument.
161 This is a temporary API that should go away once ``obs`` packages have
162 a standardized approach to writing versioned cameras to a Gen3 repo.
163 """
164 raise NotImplementedError()
166 @classmethod
167 @lru_cache()
168 def getObsDataPackageDir(cls) -> Optional[str]:
169 """The root of the obs data package that provides specializations for
170 this instrument.
172 returns
173 -------
174 dir : `str` or `None`
175 The root of the relevant obs data package, or `None` if this
176 instrument does not have one.
177 """
178 if cls.obsDataPackage is None:
179 return None
180 return getPackageDir(cls.obsDataPackage)
182 def _registerFilters(self, registry: Registry, update: bool = False) -> None:
183 """Register the physical and abstract filter Dimension relationships.
184 This should be called in the `register` implementation, within
185 a transaction context manager block.
187 Parameters
188 ----------
189 registry : `lsst.daf.butler.core.Registry`
190 The registry to add dimensions to.
191 update : `bool`, optional
192 If `True` (`False` is default), update existing records if they
193 differ from the new ones.
194 """
195 for filter in self.filterDefinitions:
196 # fix for undefined abstract filters causing trouble in the
197 # registry:
198 if filter.band is None:
199 band = filter.physical_filter
200 else:
201 band = filter.band
203 registry.syncDimensionData(
204 "physical_filter",
205 {"instrument": self.getName(), "name": filter.physical_filter, "band": band},
206 update=update,
207 )
209 def writeCuratedCalibrations(
210 self, butler: Butler, collection: Optional[str] = None, labels: Sequence[str] = ()
211 ) -> None:
212 """Write human-curated calibration Datasets to the given Butler with
213 the appropriate validity ranges.
215 Parameters
216 ----------
217 butler : `lsst.daf.butler.Butler`
218 Butler to use to store these calibrations.
219 collection : `str`, optional
220 Name to use for the calibration collection that associates all
221 datasets with a validity range. If this collection already exists,
222 it must be a `~CollectionType.CALIBRATION` collection, and it must
223 not have any datasets that would conflict with those inserted by
224 this method. If `None`, a collection name is worked out
225 automatically from the instrument name and other metadata by
226 calling ``makeCalibrationCollectionName``, but this
227 default name may not work well for long-lived repositories unless
228 ``labels`` is also provided (and changed every time curated
229 calibrations are ingested).
230 labels : `Sequence` [ `str` ], optional
231 Extra strings to include in collection names, after concatenating
232 them with the standard collection name delimeter. If provided,
233 these are inserted into the names of the `~CollectionType.RUN`
234 collections that datasets are inserted directly into, as well the
235 `~CollectionType.CALIBRATION` collection if it is generated
236 automatically (i.e. if ``collection is None``). Usually this is
237 just the name of the ticket on which the calibration collection is
238 being created.
240 Notes
241 -----
242 Expected to be called from subclasses. The base method calls
243 ``writeCameraGeom``, ``writeStandardTextCuratedCalibrations``,
244 and ``writeAdditionalCuratdCalibrations``.
245 """
246 # Delegate registration of collections (and creating names for them)
247 # to other methods so they can be called independently with the same
248 # preconditions. Collection registration is idempotent, so this is
249 # safe, and while it adds a bit of overhead, as long as it's one
250 # registration attempt per method (not per dataset or dataset type),
251 # that's negligible.
252 self.writeCameraGeom(butler, collection, labels=labels)
253 self.writeStandardTextCuratedCalibrations(butler, collection, labels=labels)
254 self.writeAdditionalCuratedCalibrations(butler, collection, labels=labels)
256 def writeAdditionalCuratedCalibrations(
257 self, butler: Butler, collection: Optional[str] = None, labels: Sequence[str] = ()
258 ) -> None:
259 """Write additional curated calibrations that might be instrument
260 specific and are not part of the standard set.
262 Default implementation does nothing.
264 Parameters
265 ----------
266 butler : `lsst.daf.butler.Butler`
267 Butler to use to store these calibrations.
268 collection : `str`, optional
269 Name to use for the calibration collection that associates all
270 datasets with a validity range. If this collection already exists,
271 it must be a `~CollectionType.CALIBRATION` collection, and it must
272 not have any datasets that would conflict with those inserted by
273 this method. If `None`, a collection name is worked out
274 automatically from the instrument name and other metadata by
275 calling ``makeCalibrationCollectionName``, but this
276 default name may not work well for long-lived repositories unless
277 ``labels`` is also provided (and changed every time curated
278 calibrations are ingested).
279 labels : `Sequence` [ `str` ], optional
280 Extra strings to include in collection names, after concatenating
281 them with the standard collection name delimeter. If provided,
282 these are inserted into the names of the `~CollectionType.RUN`
283 collections that datasets are inserted directly into, as well the
284 `~CollectionType.CALIBRATION` collection if it is generated
285 automatically (i.e. if ``collection is None``). Usually this is
286 just the name of the ticket on which the calibration collection is
287 being created.
288 """
289 return
291 def writeCameraGeom(
292 self, butler: Butler, collection: Optional[str] = None, labels: Sequence[str] = ()
293 ) -> None:
294 """Write the default camera geometry to the butler repository and
295 associate it with the appropriate validity range in a calibration
296 collection.
298 Parameters
299 ----------
300 butler : `lsst.daf.butler.Butler`
301 Butler to use to store these calibrations.
302 collection : `str`, optional
303 Name to use for the calibration collection that associates all
304 datasets with a validity range. If this collection already exists,
305 it must be a `~CollectionType.CALIBRATION` collection, and it must
306 not have any datasets that would conflict with those inserted by
307 this method. If `None`, a collection name is worked out
308 automatically from the instrument name and other metadata by
309 calling ``makeCalibrationCollectionName``, but this
310 default name may not work well for long-lived repositories unless
311 ``labels`` is also provided (and changed every time curated
312 calibrations are ingested).
313 labels : `Sequence` [ `str` ], optional
314 Extra strings to include in collection names, after concatenating
315 them with the standard collection name delimeter. If provided,
316 these are inserted into the names of the `~CollectionType.RUN`
317 collections that datasets are inserted directly into, as well the
318 `~CollectionType.CALIBRATION` collection if it is generated
319 automatically (i.e. if ``collection is None``). Usually this is
320 just the name of the ticket on which the calibration collection is
321 being created.
322 """
323 if collection is None:
324 collection = self.makeCalibrationCollectionName(*labels)
325 butler.registry.registerCollection(collection, type=CollectionType.CALIBRATION)
326 run = self.makeUnboundedCalibrationRunName(*labels)
327 butler.registry.registerRun(run)
328 datasetType = DatasetType(
329 "camera", ("instrument",), "Camera", isCalibration=True, universe=butler.registry.dimensions
330 )
331 butler.registry.registerDatasetType(datasetType)
332 camera = self.getCamera()
333 ref = butler.put(camera, datasetType, {"instrument": self.getName()}, run=run)
334 butler.registry.certify(collection, [ref], Timespan(begin=None, end=None))
336 def writeStandardTextCuratedCalibrations(
337 self, butler: Butler, collection: Optional[str] = None, labels: Sequence[str] = ()
338 ) -> None:
339 """Write the set of standardized curated text calibrations to
340 the repository.
342 Parameters
343 ----------
344 butler : `lsst.daf.butler.Butler`
345 Butler to receive these calibration datasets.
346 collection : `str`, optional
347 Name to use for the calibration collection that associates all
348 datasets with a validity range. If this collection already exists,
349 it must be a `~CollectionType.CALIBRATION` collection, and it must
350 not have any datasets that would conflict with those inserted by
351 this method. If `None`, a collection name is worked out
352 automatically from the instrument name and other metadata by
353 calling ``makeCalibrationCollectionName``, but this
354 default name may not work well for long-lived repositories unless
355 ``labels`` is also provided (and changed every time curated
356 calibrations are ingested).
357 labels : `Sequence` [ `str` ], optional
358 Extra strings to include in collection names, after concatenating
359 them with the standard collection name delimeter. If provided,
360 these are inserted into the names of the `~CollectionType.RUN`
361 collections that datasets are inserted directly into, as well the
362 `~CollectionType.CALIBRATION` collection if it is generated
363 automatically (i.e. if ``collection is None``). Usually this is
364 just the name of the ticket on which the calibration collection is
365 being created.
366 """
367 if collection is None:
368 collection = self.makeCalibrationCollectionName(*labels)
369 butler.registry.registerCollection(collection, type=CollectionType.CALIBRATION)
370 runs: Set[str] = set()
371 for datasetTypeName in self.standardCuratedDatasetTypes:
372 # We need to define the dataset types.
373 if datasetTypeName not in StandardCuratedCalibrationDatasetTypes:
374 raise ValueError(
375 f"DatasetType {datasetTypeName} not in understood list"
376 f" [{'.'.join(StandardCuratedCalibrationDatasetTypes)}]"
377 )
378 definition = StandardCuratedCalibrationDatasetTypes[datasetTypeName]
379 datasetType = DatasetType(
380 datasetTypeName,
381 universe=butler.registry.dimensions,
382 isCalibration=True,
383 # MyPy should be able to figure out that the kwargs here have
384 # the right types, but it can't.
385 **definition, # type: ignore
386 )
387 self._writeSpecificCuratedCalibrationDatasets(
388 butler, datasetType, collection, runs=runs, labels=labels
389 )
391 @classmethod
392 def _getSpecificCuratedCalibrationPath(cls, datasetTypeName: str) -> Optional[str]:
393 """Return the path of the curated calibration directory.
395 Parameters
396 ----------
397 datasetTypeName : `str`
398 The name of the standard dataset type to find.
400 Returns
401 -------
402 path : `str` or `None`
403 The path to the standard curated data directory. `None` if the
404 dataset type is not found or the obs data package is not
405 available.
406 """
407 data_package_dir = cls.getObsDataPackageDir()
408 if data_package_dir is None:
409 # if there is no data package then there can't be datasets
410 return None
412 if cls.policyName is None:
413 raise TypeError(f"Instrument {cls.getName()} has an obs data package but no policy name.")
415 calibPath = os.path.join(data_package_dir, cls.policyName, datasetTypeName)
417 if os.path.exists(calibPath):
418 return calibPath
420 return None
422 def _writeSpecificCuratedCalibrationDatasets(
423 self, butler: Butler, datasetType: DatasetType, collection: str, runs: Set[str], labels: Sequence[str]
424 ) -> None:
425 """Write standardized curated calibration datasets for this specific
426 dataset type from an obs data package.
428 Parameters
429 ----------
430 butler : `lsst.daf.butler.Butler`
431 Gen3 butler in which to put the calibrations.
432 datasetType : `lsst.daf.butler.DatasetType`
433 Dataset type to be put.
434 collection : `str`
435 Name of the `~CollectionType.CALIBRATION` collection that
436 associates all datasets with validity ranges. Must have been
437 registered prior to this call.
438 runs : `set` [ `str` ]
439 Names of runs that have already been registered by previous calls
440 and need not be registered again. Should be updated by this
441 method as new runs are registered.
442 labels : `Sequence` [ `str` ]
443 Extra strings to include in run names when creating them from
444 ``CALIBDATE`` metadata, via calls to `makeCuratedCalibrationName`.
445 Usually this is the name of the ticket on which the calibration
446 collection is being created.
448 Notes
449 -----
450 This method scans the location defined in the ``obsDataPackageDir``
451 class attribute for curated calibrations corresponding to the
452 supplied dataset type. The directory name in the data package must
453 match the name of the dataset type. They are assumed to use the
454 standard layout and can be read by
455 `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard
456 metadata.
457 """
458 calibPath = self._getSpecificCuratedCalibrationPath(datasetType.name)
459 if calibPath is None:
460 return
462 # Register the dataset type
463 butler.registry.registerDatasetType(datasetType)
465 # obs_base can't depend on pipe_tasks but concrete obs packages
466 # can -- we therefore have to defer import
467 from lsst.pipe.tasks.read_curated_calibs import read_all
469 # Read calibs, registering a new run for each CALIBDATE as needed.
470 # We try to avoid registering runs multiple times as an optimization
471 # by putting them in the ``runs`` set that was passed in.
472 camera = self.getCamera()
473 calibsDict = read_all(calibPath, camera)[0] # second return is calib type
474 datasetRecords = []
475 for det in calibsDict:
476 times = sorted([k for k in calibsDict[det]])
477 calibs = [calibsDict[det][time] for time in times]
478 times = [astropy.time.Time(t, format="datetime", scale="utc") for t in times]
479 times += [None]
480 for calib, beginTime, endTime in zip(calibs, times[:-1], times[1:]):
481 md = calib.getMetadata()
482 run = self.makeCuratedCalibrationRunName(md["CALIBDATE"], *labels)
483 if run not in runs:
484 butler.registry.registerRun(run)
485 runs.add(run)
486 dataId = DataCoordinate.standardize(
487 universe=butler.registry.dimensions,
488 instrument=self.getName(),
489 detector=md["DETECTOR"],
490 )
491 datasetRecords.append((calib, dataId, run, Timespan(beginTime, endTime)))
493 # Second loop actually does the inserts and filesystem writes. We
494 # first do a butler.put on each dataset, inserting it into the run for
495 # its calibDate. We remember those refs and group them by timespan, so
496 # we can vectorize the certify calls as much as possible.
497 refsByTimespan = defaultdict(list)
498 with butler.transaction():
499 for calib, dataId, run, timespan in datasetRecords:
500 refsByTimespan[timespan].append(butler.put(calib, datasetType, dataId, run=run))
501 for timespan, refs in refsByTimespan.items():
502 butler.registry.certify(collection, refs, timespan)
504 @abstractmethod
505 def makeDataIdTranslatorFactory(self) -> TranslatorFactory:
506 """Return a factory for creating Gen2->Gen3 data ID translators,
507 specialized for this instrument.
509 Derived class implementations should generally call
510 `TranslatorFactory.addGenericInstrumentRules` with appropriate
511 arguments, but are not required to (and may not be able to if their
512 Gen2 raw data IDs are sufficiently different from the HSC/DECam/CFHT
513 norm).
515 Returns
516 -------
517 factory : `TranslatorFactory`.
518 Factory for `Translator` objects.
519 """
520 raise NotImplementedError("Must be implemented by derived classes.")
523def makeExposureRecordFromObsInfo(
524 obsInfo: ObservationInfo, universe: DimensionUniverse, **kwargs: Any
525) -> DimensionRecord:
526 """Construct an exposure DimensionRecord from
527 `astro_metadata_translator.ObservationInfo`.
529 Parameters
530 ----------
531 obsInfo : `astro_metadata_translator.ObservationInfo`
532 A `~astro_metadata_translator.ObservationInfo` object corresponding to
533 the exposure.
534 universe : `DimensionUniverse`
535 Set of all known dimensions.
536 **kwargs
537 Additional field values for this record.
539 Returns
540 -------
541 record : `DimensionRecord`
542 A record containing exposure metadata, suitable for insertion into
543 a `Registry`.
544 """
545 dimension = universe["exposure"]
547 ra, dec, sky_angle, zenith_angle = (None, None, None, None)
548 if obsInfo.tracking_radec is not None:
549 icrs = obsInfo.tracking_radec.icrs
550 ra = icrs.ra.degree
551 dec = icrs.dec.degree
552 if obsInfo.boresight_rotation_coord == "sky":
553 sky_angle = obsInfo.boresight_rotation_angle.degree
554 if obsInfo.altaz_begin is not None:
555 zenith_angle = obsInfo.altaz_begin.zen.degree
557 return dimension.RecordClass(
558 instrument=obsInfo.instrument,
559 id=obsInfo.exposure_id,
560 obs_id=obsInfo.observation_id,
561 group_name=obsInfo.exposure_group,
562 group_id=obsInfo.visit_id,
563 datetime_begin=obsInfo.datetime_begin,
564 datetime_end=obsInfo.datetime_end,
565 exposure_time=obsInfo.exposure_time.to_value("s"),
566 # we are not mandating that dark_time be calculable
567 dark_time=obsInfo.dark_time.to_value("s") if obsInfo.dark_time is not None else None,
568 observation_type=obsInfo.observation_type,
569 observation_reason=obsInfo.observation_reason,
570 day_obs=obsInfo.observing_day,
571 seq_num=obsInfo.observation_counter,
572 physical_filter=obsInfo.physical_filter,
573 science_program=obsInfo.science_program,
574 target_name=obsInfo.object,
575 tracking_ra=ra,
576 tracking_dec=dec,
577 sky_angle=sky_angle,
578 zenith_angle=zenith_angle,
579 **kwargs,
580 )
583def loadCamera(butler: Butler, dataId: DataId, *, collections: Any = None) -> Tuple[Camera, bool]:
584 """Attempt to load versioned camera geometry from a butler, but fall back
585 to obtaining a nominal camera from the `Instrument` class if that fails.
587 Parameters
588 ----------
589 butler : `lsst.daf.butler.Butler`
590 Butler instance to attempt to query for and load a ``camera`` dataset
591 from.
592 dataId : `dict` or `DataCoordinate`
593 Data ID that identifies at least the ``instrument`` and ``exposure``
594 dimensions.
595 collections : Any, optional
596 Collections to be searched, overriding ``self.butler.collections``.
597 Can be any of the types supported by the ``collections`` argument
598 to butler construction.
600 Returns
601 -------
602 camera : `lsst.afw.cameraGeom.Camera`
603 Camera object.
604 versioned : `bool`
605 If `True`, the camera was obtained from the butler and should represent
606 a versioned camera from a calibration repository. If `False`, no
607 camera datasets were found, and the returned camera was produced by
608 instantiating the appropriate `Instrument` class and calling
609 `Instrument.getCamera`.
611 Raises
612 ------
613 LookupError
614 Raised when ``dataId`` does not specify a valid data ID.
615 """
616 if collections is None:
617 collections = butler.collections
618 # Registry would do data ID expansion internally if we didn't do it first,
619 # but we might want an expanded data ID ourselves later, so we do it here
620 # to ensure it only happens once.
621 # This will also catch problems with the data ID not having keys we need.
622 try:
623 dataId = butler.registry.expandDataId(dataId, graph=butler.registry.dimensions["exposure"].graph)
624 except DataIdError as exc:
625 raise LookupError(str(exc)) from exc
626 try:
627 cameraRef = butler.get("camera", dataId=dataId, collections=collections)
628 return cameraRef, True
629 except LookupError:
630 pass
631 # We know an instrument data ID is a value, but MyPy doesn't.
632 instrument = Instrument.fromName(dataId["instrument"], butler.registry) # type: ignore
633 assert isinstance(instrument, Instrument) # for mypy
634 return instrument.getCamera(), False