Coverage for python/lsst/obs/base/_instrument.py : 27%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ("Instrument", "makeExposureRecordFromObsInfo", "loadCamera")
26import os.path
27from abc import ABCMeta, abstractmethod
28from collections import defaultdict
29import datetime
30from typing import Any, Optional, Set, Sequence, Tuple, TYPE_CHECKING, Union
31from functools import lru_cache
33import astropy.time
35from lsst.afw.cameraGeom import Camera
36from lsst.daf.butler import (
37 Butler,
38 CollectionType,
39 DataCoordinate,
40 DataId,
41 DatasetType,
42 Timespan,
43)
44from lsst.utils import getPackageDir, doImport
46if TYPE_CHECKING: 46 ↛ 47line 46 didn't jump to line 47, because the condition on line 46 was never true
47 from .gen2to3 import TranslatorFactory
48 from lsst.daf.butler import Registry
50# To be a standard text curated calibration means that we use a
51# standard definition for the corresponding DatasetType.
52StandardCuratedCalibrationDatasetTypes = {
53 "defects": {"dimensions": ("instrument", "detector"), "storageClass": "Defects"},
54 "qe_curve": {"dimensions": ("instrument", "detector"), "storageClass": "QECurve"},
55 "crosstalk": {"dimensions": ("instrument", "detector"), "storageClass": "CrosstalkCalib"},
56 "linearizer": {"dimensions": ("instrument", "detector"), "storageClass": "Linearizer"},
57 "bfk": {"dimensions": ("instrument", "detector"), "storageClass": "BrighterFatterKernel"},
58}
61class Instrument(metaclass=ABCMeta):
62 """Base class for instrument-specific logic for the Gen3 Butler.
64 Parameters
65 ----------
66 collection_prefix : `str`, optional
67 Prefix for collection names to use instead of the intrument's own name.
68 This is primarily for use in simulated-data repositories, where the
69 instrument name may not be necessary and/or sufficient to distinguish
70 between collections.
72 Notes
73 -----
74 Concrete instrument subclasses must have the same construction signature as
75 the base class.
76 """
78 configPaths: Sequence[str] = ()
79 """Paths to config files to read for specific Tasks.
81 The paths in this list should contain files of the form `task.py`, for
82 each of the Tasks that requires special configuration.
83 """
85 policyName: Optional[str] = None
86 """Instrument specific name to use when locating a policy or configuration
87 file in the file system."""
89 obsDataPackage: Optional[str] = None
90 """Name of the package containing the text curated calibration files.
91 Usually a obs _data package. If `None` no curated calibration files
92 will be read. (`str`)"""
94 standardCuratedDatasetTypes: Set[str] = frozenset(StandardCuratedCalibrationDatasetTypes)
95 """The dataset types expected to be obtained from the obsDataPackage.
97 These dataset types are all required to have standard definitions and
98 must be known to the base class. Clearing this list will prevent
99 any of these calibrations from being stored. If a dataset type is not
100 known to a specific instrument it can still be included in this list
101 since the data package is the source of truth. (`set` of `str`)
102 """
104 additionalCuratedDatasetTypes: Set[str] = frozenset()
105 """Curated dataset types specific to this particular instrument that do
106 not follow the standard organization found in obs data packages.
108 These are the instrument-specific dataset types written by
109 `writeAdditionalCuratedCalibrations` in addition to the calibrations
110 found in obs data packages that follow the standard scheme.
111 (`set` of `str`)"""
113 @property
114 @abstractmethod
115 def filterDefinitions(self):
116 """`~lsst.obs.base.FilterDefinitionCollection`, defining the filters
117 for this instrument.
118 """
119 return None
121 def __init__(self, collection_prefix: Optional[str] = None):
122 self.filterDefinitions.reset()
123 self.filterDefinitions.defineFilters()
124 if collection_prefix is None: 124 ↛ 126line 124 didn't jump to line 126, because the condition on line 124 was never false
125 collection_prefix = self.getName()
126 self.collection_prefix = collection_prefix
128 @classmethod
129 @abstractmethod
130 def getName(cls):
131 """Return the short (dimension) name for this instrument.
133 This is not (in general) the same as the class name - it's what is used
134 as the value of the "instrument" field in data IDs, and is usually an
135 abbreviation of the full name.
136 """
137 raise NotImplementedError()
139 @classmethod
140 @lru_cache()
141 def getCuratedCalibrationNames(cls) -> Set[str]:
142 """Return the names of all the curated calibration dataset types.
144 Returns
145 -------
146 names : `set` of `str`
147 The dataset type names of all curated calibrations. This will
148 include the standard curated calibrations even if the particular
149 instrument does not support them.
151 Notes
152 -----
153 The returned list does not indicate whether a particular dataset
154 is present in the Butler repository, simply that these are the
155 dataset types that are handled by ``writeCuratedCalibrations``.
156 """
158 # Camera is a special dataset type that is also handled as a
159 # curated calibration.
160 curated = {"camera"}
162 # Make a cursory attempt to filter out curated dataset types
163 # that are not present for this instrument
164 for datasetTypeName in cls.standardCuratedDatasetTypes:
165 calibPath = cls._getSpecificCuratedCalibrationPath(datasetTypeName)
166 if calibPath is not None:
167 curated.add(datasetTypeName)
169 curated.update(cls.additionalCuratedDatasetTypes)
170 return frozenset(curated)
172 @abstractmethod
173 def getCamera(self):
174 """Retrieve the cameraGeom representation of this instrument.
176 This is a temporary API that should go away once ``obs`` packages have
177 a standardized approach to writing versioned cameras to a Gen3 repo.
178 """
179 raise NotImplementedError()
181 @abstractmethod
182 def register(self, registry):
183 """Insert instrument, physical_filter, and detector entries into a
184 `Registry`.
186 Implementations should guarantee that registration is atomic (the
187 registry should not be modified if any error occurs) and idempotent at
188 the level of individual dimension entries; new detectors and filters
189 should be added, but changes to any existing record should not be.
190 This can generally be achieved via a block like::
192 with registry.transaction():
193 registry.syncDimensionData("instrument", ...)
194 registry.syncDimensionData("detector", ...)
195 self.registerFilters(registry)
197 Raises
198 ------
199 lsst.daf.butler.registry.ConflictingDefinitionError
200 Raised if any existing record has the same key but a different
201 definition as one being registered.
202 """
203 raise NotImplementedError()
205 @classmethod
206 @lru_cache()
207 def getObsDataPackageDir(cls):
208 """The root of the obs data package that provides specializations for
209 this instrument.
211 returns
212 -------
213 dir : `str`
214 The root of the relevat obs data package.
215 """
216 if cls.obsDataPackage is None:
217 return None
218 return getPackageDir(cls.obsDataPackage)
220 @staticmethod
221 def fromName(name: str, registry: Registry, collection_prefix: Optional[str] = None) -> Instrument:
222 """Given an instrument name and a butler, retrieve a corresponding
223 instantiated instrument object.
225 Parameters
226 ----------
227 name : `str`
228 Name of the instrument (must match the return value of `getName`).
229 registry : `lsst.daf.butler.Registry`
230 Butler registry to query to find the information.
231 collection_prefix : `str`, optional
232 Prefix for collection names to use instead of the intrument's own
233 name. This is primarily for use in simulated-data repositories,
234 where the instrument name may not be necessary and/or sufficient to
235 distinguish between collections.
237 Returns
238 -------
239 instrument : `Instrument`
240 An instance of the relevant `Instrument`.
242 Notes
243 -----
244 The instrument must be registered in the corresponding butler.
246 Raises
247 ------
248 LookupError
249 Raised if the instrument is not known to the supplied registry.
250 ModuleNotFoundError
251 Raised if the class could not be imported. This could mean
252 that the relevant obs package has not been setup.
253 TypeError
254 Raised if the class name retrieved is not a string.
255 """
256 records = list(registry.queryDimensionRecords("instrument", instrument=name))
257 if not records:
258 raise LookupError(f"No registered instrument with name '{name}'.")
259 cls = records[0].class_name
260 if not isinstance(cls, str):
261 raise TypeError(f"Unexpected class name retrieved from {name} instrument dimension (got {cls})")
262 instrument = doImport(cls)
263 return instrument(collection_prefix=collection_prefix)
265 @staticmethod
266 def importAll(registry: Registry) -> None:
267 """Import all the instruments known to this registry.
269 This will ensure that all metadata translators have been registered.
271 Parameters
272 ----------
273 registry : `lsst.daf.butler.Registry`
274 Butler registry to query to find the information.
276 Notes
277 -----
278 It is allowed for a particular instrument class to fail on import.
279 This might simply indicate that a particular obs package has
280 not been setup.
281 """
282 records = list(registry.queryDimensionRecords("instrument"))
283 for record in records:
284 cls = record.class_name
285 try:
286 doImport(cls)
287 except Exception:
288 pass
290 def _registerFilters(self, registry):
291 """Register the physical and abstract filter Dimension relationships.
292 This should be called in the `register` implementation, within
293 a transaction context manager block.
295 Parameters
296 ----------
297 registry : `lsst.daf.butler.core.Registry`
298 The registry to add dimensions to.
299 """
300 for filter in self.filterDefinitions:
301 # fix for undefined abstract filters causing trouble in the
302 # registry:
303 if filter.band is None:
304 band = filter.physical_filter
305 else:
306 band = filter.band
308 registry.syncDimensionData("physical_filter",
309 {"instrument": self.getName(),
310 "name": filter.physical_filter,
311 "band": band
312 })
314 @abstractmethod
315 def getRawFormatter(self, dataId):
316 """Return the Formatter class that should be used to read a particular
317 raw file.
319 Parameters
320 ----------
321 dataId : `DataCoordinate`
322 Dimension-based ID for the raw file or files being ingested.
324 Returns
325 -------
326 formatter : `Formatter` class
327 Class to be used that reads the file into an
328 `lsst.afw.image.Exposure` instance.
329 """
330 raise NotImplementedError()
332 def applyConfigOverrides(self, name, config):
333 """Apply instrument-specific overrides for a task config.
335 Parameters
336 ----------
337 name : `str`
338 Name of the object being configured; typically the _DefaultName
339 of a Task.
340 config : `lsst.pex.config.Config`
341 Config instance to which overrides should be applied.
342 """
343 for root in self.configPaths:
344 path = os.path.join(root, f"{name}.py")
345 if os.path.exists(path):
346 config.load(path)
348 def writeCuratedCalibrations(self, butler: Butler, collection: Optional[str] = None,
349 labels: Sequence[str] = ()) -> None:
350 """Write human-curated calibration Datasets to the given Butler with
351 the appropriate validity ranges.
353 Parameters
354 ----------
355 butler : `lsst.daf.butler.Butler`
356 Butler to use to store these calibrations.
357 collection : `str`, optional
358 Name to use for the calibration collection that associates all
359 datasets with a validity range. If this collection already exists,
360 it must be a `~CollectionType.CALIBRATION` collection, and it must
361 not have any datasets that would conflict with those inserted by
362 this method. If `None`, a collection name is worked out
363 automatically from the instrument name and other metadata by
364 calling ``makeCalibrationCollectionName``, but this
365 default name may not work well for long-lived repositories unless
366 ``labels`` is also provided (and changed every time curated
367 calibrations are ingested).
368 labels : `Sequence` [ `str` ], optional
369 Extra strings to include in collection names, after concatenating
370 them with the standard collection name delimeter. If provided,
371 these are inserted into the names of the `~CollectionType.RUN`
372 collections that datasets are inserted directly into, as well the
373 `~CollectionType.CALIBRATION` collection if it is generated
374 automatically (i.e. if ``collection is None``). Usually this is
375 just the name of the ticket on which the calibration collection is
376 being created.
378 Notes
379 -----
380 Expected to be called from subclasses. The base method calls
381 ``writeCameraGeom``, ``writeStandardTextCuratedCalibrations``,
382 and ``writeAdditionalCuratdCalibrations``.
383 """
384 # Delegate registration of collections (and creating names for them)
385 # to other methods so they can be called independently with the same
386 # preconditions. Collection registration is idempotent, so this is
387 # safe, and while it adds a bit of overhead, as long as it's one
388 # registration attempt per method (not per dataset or dataset type),
389 # that's negligible.
390 self.writeCameraGeom(butler, collection, labels=labels)
391 self.writeStandardTextCuratedCalibrations(butler, collection, labels=labels)
392 self.writeAdditionalCuratedCalibrations(butler, collection, labels=labels)
394 def writeAdditionalCuratedCalibrations(self, butler: Butler, collection: Optional[str] = None,
395 labels: Sequence[str] = ()) -> None:
396 """Write additional curated calibrations that might be instrument
397 specific and are not part of the standard set.
399 Default implementation does nothing.
401 Parameters
402 ----------
403 butler : `lsst.daf.butler.Butler`
404 Butler to use to store these calibrations.
405 collection : `str`, optional
406 Name to use for the calibration collection that associates all
407 datasets with a validity range. If this collection already exists,
408 it must be a `~CollectionType.CALIBRATION` collection, and it must
409 not have any datasets that would conflict with those inserted by
410 this method. If `None`, a collection name is worked out
411 automatically from the instrument name and other metadata by
412 calling ``makeCalibrationCollectionName``, but this
413 default name may not work well for long-lived repositories unless
414 ``labels`` is also provided (and changed every time curated
415 calibrations are ingested).
416 labels : `Sequence` [ `str` ], optional
417 Extra strings to include in collection names, after concatenating
418 them with the standard collection name delimeter. If provided,
419 these are inserted into the names of the `~CollectionType.RUN`
420 collections that datasets are inserted directly into, as well the
421 `~CollectionType.CALIBRATION` collection if it is generated
422 automatically (i.e. if ``collection is None``). Usually this is
423 just the name of the ticket on which the calibration collection is
424 being created.
425 """
426 return
428 def writeCameraGeom(self, butler: Butler, collection: Optional[str] = None,
429 labels: Sequence[str] = ()) -> None:
430 """Write the default camera geometry to the butler repository and
431 associate it with the appropriate validity range in a calibration
432 collection.
434 Parameters
435 ----------
436 butler : `lsst.daf.butler.Butler`
437 Butler to use to store these calibrations.
438 collection : `str`, optional
439 Name to use for the calibration collection that associates all
440 datasets with a validity range. If this collection already exists,
441 it must be a `~CollectionType.CALIBRATION` collection, and it must
442 not have any datasets that would conflict with those inserted by
443 this method. If `None`, a collection name is worked out
444 automatically from the instrument name and other metadata by
445 calling ``makeCalibrationCollectionName``, but this
446 default name may not work well for long-lived repositories unless
447 ``labels`` is also provided (and changed every time curated
448 calibrations are ingested).
449 labels : `Sequence` [ `str` ], optional
450 Extra strings to include in collection names, after concatenating
451 them with the standard collection name delimeter. If provided,
452 these are inserted into the names of the `~CollectionType.RUN`
453 collections that datasets are inserted directly into, as well the
454 `~CollectionType.CALIBRATION` collection if it is generated
455 automatically (i.e. if ``collection is None``). Usually this is
456 just the name of the ticket on which the calibration collection is
457 being created.
458 """
459 if collection is None:
460 collection = self.makeCalibrationCollectionName(*labels)
461 butler.registry.registerCollection(collection, type=CollectionType.CALIBRATION)
462 run = self.makeUnboundedCalibrationRunName(*labels)
463 butler.registry.registerRun(run)
464 datasetType = DatasetType("camera", ("instrument",), "Camera", isCalibration=True,
465 universe=butler.registry.dimensions)
466 butler.registry.registerDatasetType(datasetType)
467 camera = self.getCamera()
468 ref = butler.put(camera, datasetType, {"instrument": self.getName()}, run=run)
469 butler.registry.certify(collection, [ref], Timespan(begin=None, end=None))
471 def writeStandardTextCuratedCalibrations(self, butler: Butler, collection: Optional[str] = None,
472 labels: Sequence[str] = ()) -> None:
473 """Write the set of standardized curated text calibrations to
474 the repository.
476 Parameters
477 ----------
478 butler : `lsst.daf.butler.Butler`
479 Butler to receive these calibration datasets.
480 collection : `str`, optional
481 Name to use for the calibration collection that associates all
482 datasets with a validity range. If this collection already exists,
483 it must be a `~CollectionType.CALIBRATION` collection, and it must
484 not have any datasets that would conflict with those inserted by
485 this method. If `None`, a collection name is worked out
486 automatically from the instrument name and other metadata by
487 calling ``makeCalibrationCollectionName``, but this
488 default name may not work well for long-lived repositories unless
489 ``labels`` is also provided (and changed every time curated
490 calibrations are ingested).
491 labels : `Sequence` [ `str` ], optional
492 Extra strings to include in collection names, after concatenating
493 them with the standard collection name delimeter. If provided,
494 these are inserted into the names of the `~CollectionType.RUN`
495 collections that datasets are inserted directly into, as well the
496 `~CollectionType.CALIBRATION` collection if it is generated
497 automatically (i.e. if ``collection is None``). Usually this is
498 just the name of the ticket on which the calibration collection is
499 being created.
500 """
501 if collection is None:
502 collection = self.makeCalibrationCollectionName(*labels)
503 butler.registry.registerCollection(collection, type=CollectionType.CALIBRATION)
504 runs = set()
505 for datasetTypeName in self.standardCuratedDatasetTypes:
506 # We need to define the dataset types.
507 if datasetTypeName not in StandardCuratedCalibrationDatasetTypes:
508 raise ValueError(f"DatasetType {datasetTypeName} not in understood list"
509 f" [{'.'.join(StandardCuratedCalibrationDatasetTypes)}]")
510 definition = StandardCuratedCalibrationDatasetTypes[datasetTypeName]
511 datasetType = DatasetType(datasetTypeName,
512 universe=butler.registry.dimensions,
513 isCalibration=True,
514 **definition)
515 self._writeSpecificCuratedCalibrationDatasets(butler, datasetType, collection, runs=runs,
516 labels=labels)
518 @classmethod
519 def _getSpecificCuratedCalibrationPath(cls, datasetTypeName):
520 """Return the path of the curated calibration directory.
522 Parameters
523 ----------
524 datasetTypeName : `str`
525 The name of the standard dataset type to find.
527 Returns
528 -------
529 path : `str`
530 The path to the standard curated data directory. `None` if the
531 dataset type is not found or the obs data package is not
532 available.
533 """
534 if cls.getObsDataPackageDir() is None:
535 # if there is no data package then there can't be datasets
536 return None
538 calibPath = os.path.join(cls.getObsDataPackageDir(), cls.policyName,
539 datasetTypeName)
541 if os.path.exists(calibPath):
542 return calibPath
544 return None
546 def _writeSpecificCuratedCalibrationDatasets(self, butler: Butler, datasetType: DatasetType,
547 collection: str, runs: Set[str], labels: Sequence[str]):
548 """Write standardized curated calibration datasets for this specific
549 dataset type from an obs data package.
551 Parameters
552 ----------
553 butler : `lsst.daf.butler.Butler`
554 Gen3 butler in which to put the calibrations.
555 datasetType : `lsst.daf.butler.DatasetType`
556 Dataset type to be put.
557 collection : `str`
558 Name of the `~CollectionType.CALIBRATION` collection that
559 associates all datasets with validity ranges. Must have been
560 registered prior to this call.
561 runs : `set` [ `str` ]
562 Names of runs that have already been registered by previous calls
563 and need not be registered again. Should be updated by this
564 method as new runs are registered.
565 labels : `Sequence` [ `str` ]
566 Extra strings to include in run names when creating them from
567 ``CALIBDATE`` metadata, via calls to `makeCuratedCalibrationName`.
568 Usually this is the name of the ticket on which the calibration
569 collection is being created.
571 Notes
572 -----
573 This method scans the location defined in the ``obsDataPackageDir``
574 class attribute for curated calibrations corresponding to the
575 supplied dataset type. The directory name in the data package must
576 match the name of the dataset type. They are assumed to use the
577 standard layout and can be read by
578 `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard
579 metadata.
580 """
581 calibPath = self._getSpecificCuratedCalibrationPath(datasetType.name)
582 if calibPath is None:
583 return
585 # Register the dataset type
586 butler.registry.registerDatasetType(datasetType)
588 # obs_base can't depend on pipe_tasks but concrete obs packages
589 # can -- we therefore have to defer import
590 from lsst.pipe.tasks.read_curated_calibs import read_all
592 # Read calibs, registering a new run for each CALIBDATE as needed.
593 # We try to avoid registering runs multiple times as an optimization
594 # by putting them in the ``runs`` set that was passed in.
595 camera = self.getCamera()
596 calibsDict = read_all(calibPath, camera)[0] # second return is calib type
597 datasetRecords = []
598 for det in calibsDict:
599 times = sorted([k for k in calibsDict[det]])
600 calibs = [calibsDict[det][time] for time in times]
601 times = [astropy.time.Time(t, format="datetime", scale="utc") for t in times]
602 times += [None]
603 for calib, beginTime, endTime in zip(calibs, times[:-1], times[1:]):
604 md = calib.getMetadata()
605 run = self.makeCuratedCalibrationRunName(md['CALIBDATE'], *labels)
606 if run not in runs:
607 butler.registry.registerRun(run)
608 runs.add(run)
609 dataId = DataCoordinate.standardize(
610 universe=butler.registry.dimensions,
611 instrument=self.getName(),
612 detector=md["DETECTOR"],
613 )
614 datasetRecords.append((calib, dataId, run, Timespan(beginTime, endTime)))
616 # Second loop actually does the inserts and filesystem writes. We
617 # first do a butler.put on each dataset, inserting it into the run for
618 # its calibDate. We remember those refs and group them by timespan, so
619 # we can vectorize the certify calls as much as possible.
620 refsByTimespan = defaultdict(list)
621 with butler.transaction():
622 for calib, dataId, run, timespan in datasetRecords:
623 refsByTimespan[timespan].append(butler.put(calib, datasetType, dataId, run=run))
624 for timespan, refs in refsByTimespan.items():
625 butler.registry.certify(collection, refs, timespan)
627 @abstractmethod
628 def makeDataIdTranslatorFactory(self) -> TranslatorFactory:
629 """Return a factory for creating Gen2->Gen3 data ID translators,
630 specialized for this instrument.
632 Derived class implementations should generally call
633 `TranslatorFactory.addGenericInstrumentRules` with appropriate
634 arguments, but are not required to (and may not be able to if their
635 Gen2 raw data IDs are sufficiently different from the HSC/DECam/CFHT
636 norm).
638 Returns
639 -------
640 factory : `TranslatorFactory`.
641 Factory for `Translator` objects.
642 """
643 raise NotImplementedError("Must be implemented by derived classes.")
645 @staticmethod
646 def formatCollectionTimestamp(timestamp: Union[str, datetime.datetime]) -> str:
647 """Format a timestamp for use in a collection name.
649 Parameters
650 ----------
651 timestamp : `str` or `datetime.datetime`
652 Timestamp to format. May be a date or datetime string in extended
653 ISO format (assumed UTC), with or without a timezone specifier, a
654 datetime string in basic ISO format with a timezone specifier, a
655 naive `datetime.datetime` instance (assumed UTC) or a
656 timezone-aware `datetime.datetime` instance (converted to UTC).
657 This is intended to cover all forms that string ``CALIBDATE``
658 metadata values have taken in the past, as well as the format this
659 method itself writes out (to enable round-tripping).
661 Returns
662 -------
663 formatted : `str`
664 Standardized string form for the timestamp.
665 """
666 if isinstance(timestamp, str):
667 if "-" in timestamp:
668 # extended ISO format, with - and : delimiters
669 timestamp = datetime.datetime.fromisoformat(timestamp)
670 else:
671 # basic ISO format, with no delimiters (what this method
672 # returns)
673 timestamp = datetime.datetime.strptime(timestamp, "%Y%m%dT%H%M%S%z")
674 if not isinstance(timestamp, datetime.datetime):
675 raise TypeError(f"Unexpected date/time object: {timestamp!r}.")
676 if timestamp.tzinfo is not None:
677 timestamp = timestamp.astimezone(datetime.timezone.utc)
678 return f"{timestamp:%Y%m%dT%H%M%S}Z"
680 @staticmethod
681 def makeCollectionTimestamp() -> str:
682 """Create a timestamp string for use in a collection name from the
683 current time.
685 Returns
686 -------
687 formatted : `str`
688 Standardized string form of the current time.
689 """
690 return Instrument.formatCollectionTimestamp(datetime.datetime.now(tz=datetime.timezone.utc))
692 def makeDefaultRawIngestRunName(self) -> str:
693 """Make the default instrument-specific run collection string for raw
694 data ingest.
696 Returns
697 -------
698 coll : `str`
699 Run collection name to be used as the default for ingestion of
700 raws.
701 """
702 return self.makeCollectionName("raw", "all")
704 def makeUnboundedCalibrationRunName(self, *labels: str) -> str:
705 """Make a RUN collection name appropriate for inserting calibration
706 datasets whose validity ranges are unbounded.
708 Parameters
709 ----------
710 *labels : `str`
711 Extra strings to be included in the base name, using the default
712 delimiter for collection names. Usually this is the name of the
713 ticket on which the calibration collection is being created.
715 Returns
716 -------
717 name : `str`
718 Run collection name.
719 """
720 return self.makeCollectionName("calib", *labels, "unbounded")
722 def makeCuratedCalibrationRunName(self, calibDate: str, *labels: str) -> str:
723 """Make a RUN collection name appropriate for inserting curated
724 calibration datasets with the given ``CALIBDATE`` metadata value.
726 Parameters
727 ----------
728 calibDate : `str`
729 The ``CALIBDATE`` metadata value.
730 *labels : `str`
731 Strings to be included in the collection name (before
732 ``calibDate``, but after all other terms), using the default
733 delimiter for collection names. Usually this is the name of the
734 ticket on which the calibration collection is being created.
736 Returns
737 -------
738 name : `str`
739 Run collection name.
740 """
741 return self.makeCollectionName("calib", *labels, "curated", self.formatCollectionTimestamp(calibDate))
743 def makeCalibrationCollectionName(self, *labels: str) -> str:
744 """Make a CALIBRATION collection name appropriate for associating
745 calibration datasets with validity ranges.
747 Parameters
748 ----------
749 *labels : `str`
750 Strings to be appended to the base name, using the default
751 delimiter for collection names. Usually this is the name of the
752 ticket on which the calibration collection is being created.
754 Returns
755 -------
756 name : `str`
757 Calibration collection name.
758 """
759 return self.makeCollectionName("calib", *labels)
761 @staticmethod
762 def makeRefCatCollectionName(*labels: str) -> str:
763 """Return a global (not instrument-specific) name for a collection that
764 holds reference catalogs.
766 With no arguments, this returns the name of the collection that holds
767 all reference catalogs (usually a ``CHAINED`` collection, at least in
768 long-lived repos that may contain more than one reference catalog).
770 Parameters
771 ----------
772 *labels : `str`
773 Strings to be added to the global collection name, in order to
774 define a collection name for one or more reference catalogs being
775 ingested at the same time.
777 Returns
778 -------
779 name : `str`
780 Collection name.
782 Notes
783 -----
784 This is a ``staticmethod``, not a ``classmethod``, because it should
785 be the same for all instruments.
786 """
787 return "/".join(("refcats",) + labels)
789 def makeUmbrellaCollectionName(self) -> str:
790 """Return the name of the umbrella ``CHAINED`` collection for this
791 instrument that combines all standard recommended input collections.
793 This method should almost never be overridden by derived classes.
795 Returns
796 -------
797 name : `str`
798 Name for the umbrella collection.
799 """
800 return self.makeCollectionName("defaults")
802 def makeCollectionName(self, *labels: str) -> str:
803 """Get the instrument-specific collection string to use as derived
804 from the supplied labels.
806 Parameters
807 ----------
808 *labels : `str`
809 Strings to be combined with the instrument name to form a
810 collection name.
812 Returns
813 -------
814 name : `str`
815 Collection name to use that includes the instrument's recommended
816 prefix.
817 """
818 return "/".join((self.collection_prefix,) + labels)
821def makeExposureRecordFromObsInfo(obsInfo, universe):
822 """Construct an exposure DimensionRecord from
823 `astro_metadata_translator.ObservationInfo`.
825 Parameters
826 ----------
827 obsInfo : `astro_metadata_translator.ObservationInfo`
828 A `~astro_metadata_translator.ObservationInfo` object corresponding to
829 the exposure.
830 universe : `DimensionUniverse`
831 Set of all known dimensions.
833 Returns
834 -------
835 record : `DimensionRecord`
836 A record containing exposure metadata, suitable for insertion into
837 a `Registry`.
838 """
839 dimension = universe["exposure"]
841 ra, dec, sky_angle, zenith_angle = (None, None, None, None)
842 if obsInfo.tracking_radec is not None:
843 icrs = obsInfo.tracking_radec.icrs
844 ra = icrs.ra.degree
845 dec = icrs.dec.degree
846 if obsInfo.boresight_rotation_coord == "sky":
847 sky_angle = obsInfo.boresight_rotation_angle.degree
848 if obsInfo.altaz_begin is not None:
849 zenith_angle = obsInfo.altaz_begin.zen.degree
851 return dimension.RecordClass(
852 instrument=obsInfo.instrument,
853 id=obsInfo.exposure_id,
854 obs_id=obsInfo.observation_id,
855 group_name=obsInfo.exposure_group,
856 group_id=obsInfo.visit_id,
857 datetime_begin=obsInfo.datetime_begin,
858 datetime_end=obsInfo.datetime_end,
859 exposure_time=obsInfo.exposure_time.to_value("s"),
860 # we are not mandating that dark_time be calculable
861 dark_time=obsInfo.dark_time.to_value("s") if obsInfo.dark_time is not None else None,
862 observation_type=obsInfo.observation_type,
863 observation_reason=obsInfo.observation_reason,
864 day_obs=obsInfo.observing_day,
865 seq_num=obsInfo.observation_counter,
866 physical_filter=obsInfo.physical_filter,
867 science_program=obsInfo.science_program,
868 target_name=obsInfo.object,
869 tracking_ra=ra,
870 tracking_dec=dec,
871 sky_angle=sky_angle,
872 zenith_angle=zenith_angle,
873 )
876def loadCamera(butler: Butler, dataId: DataId, *, collections: Any = None) -> Tuple[Camera, bool]:
877 """Attempt to load versioned camera geometry from a butler, but fall back
878 to obtaining a nominal camera from the `Instrument` class if that fails.
880 Parameters
881 ----------
882 butler : `lsst.daf.butler.Butler`
883 Butler instance to attempt to query for and load a ``camera`` dataset
884 from.
885 dataId : `dict` or `DataCoordinate`
886 Data ID that identifies at least the ``instrument`` and ``exposure``
887 dimensions.
888 collections : Any, optional
889 Collections to be searched, overriding ``self.butler.collections``.
890 Can be any of the types supported by the ``collections`` argument
891 to butler construction.
893 Returns
894 -------
895 camera : `lsst.afw.cameraGeom.Camera`
896 Camera object.
897 versioned : `bool`
898 If `True`, the camera was obtained from the butler and should represent
899 a versioned camera from a calibration repository. If `False`, no
900 camera datasets were found, and the returned camera was produced by
901 instantiating the appropriate `Instrument` class and calling
902 `Instrument.getCamera`.
903 """
904 if collections is None:
905 collections = butler.collections
906 # Registry would do data ID expansion internally if we didn't do it first,
907 # but we might want an expanded data ID ourselves later, so we do it here
908 # to ensure it only happens once.
909 # This will also catch problems with the data ID not having keys we need.
910 dataId = butler.registry.expandDataId(dataId, graph=butler.registry.dimensions["exposure"].graph)
911 try:
912 cameraRef = butler.get("camera", dataId=dataId, collections=collections)
913 return cameraRef, True
914 except LookupError:
915 pass
916 instrument = Instrument.fromName(dataId["instrument"], butler.registry)
917 return instrument.getCamera(), False