Coverage for python/lsst/pipe/base/_instrument.py: 33%
104 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-13 16:29 -0700
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-13 16:29 -0700
1# This file is part of pipe_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ("Instrument",)
26import datetime
27import os.path
28from abc import ABCMeta, abstractmethod
29from typing import TYPE_CHECKING, Optional, Sequence, Type, Union
31from lsst.daf.butler import DataId, Formatter
32from lsst.daf.butler.registry import DataIdError
33from lsst.utils import doImportType
35if TYPE_CHECKING: 35 ↛ 36line 35 didn't jump to line 36, because the condition on line 35 was never true
36 from lsst.daf.butler import Registry
37 from lsst.pex.config import Config
40class Instrument(metaclass=ABCMeta):
41 """Base class for instrument-specific logic for the Gen3 Butler.
43 Parameters
44 ----------
45 collection_prefix : `str`, optional
46 Prefix for collection names to use instead of the intrument's own name.
47 This is primarily for use in simulated-data repositories, where the
48 instrument name may not be necessary and/or sufficient to distinguish
49 between collections.
51 Notes
52 -----
53 Concrete instrument subclasses must have the same construction signature as
54 the base class.
55 """
57 configPaths: Sequence[str] = ()
58 """Paths to config files to read for specific Tasks.
60 The paths in this list should contain files of the form `task.py`, for
61 each of the Tasks that requires special configuration.
62 """
64 policyName: Optional[str] = None
65 """Instrument specific name to use when locating a policy or configuration
66 file in the file system."""
68 def __init__(self, collection_prefix: Optional[str] = None):
69 if collection_prefix is None:
70 collection_prefix = self.getName()
71 self.collection_prefix = collection_prefix
73 @classmethod
74 @abstractmethod
75 def getName(cls) -> str:
76 """Return the short (dimension) name for this instrument.
78 This is not (in general) the same as the class name - it's what is used
79 as the value of the "instrument" field in data IDs, and is usually an
80 abbreviation of the full name.
81 """
82 raise NotImplementedError()
84 @abstractmethod
85 def register(self, registry: Registry, *, update: bool = False) -> None:
86 """Insert instrument, and other relevant records into `Registry`.
88 Parameters
89 ----------
90 registry : `lsst.daf.butler.Registry`
91 Registry client for the data repository to modify.
92 update : `bool`, optional
93 If `True` (`False` is default), update existing records if they
94 differ from the new ones.
96 Raises
97 ------
98 lsst.daf.butler.registry.ConflictingDefinitionError
99 Raised if any existing record has the same key but a different
100 definition as one being registered.
102 Notes
103 -----
104 New records can always be added by calling this method multiple times,
105 as long as no existing records have changed (if existing records have
106 changed, ``update=True`` must be used). Old records can never be
107 removed by this method.
109 Implementations should guarantee that registration is atomic (the
110 registry should not be modified if any error occurs) and idempotent at
111 the level of individual dimension entries; new detectors and filters
112 should be added, but changes to any existing record should not be.
113 This can generally be achieved via a block like
115 .. code-block:: python
117 with registry.transaction():
118 registry.syncDimensionData("instrument", ...)
119 registry.syncDimensionData("detector", ...)
120 self.registerFilters(registry)
122 """
123 raise NotImplementedError()
125 @staticmethod
126 def fromName(name: str, registry: Registry, collection_prefix: Optional[str] = None) -> Instrument:
127 """Given an instrument name and a butler registry, retrieve a
128 corresponding instantiated instrument object.
130 Parameters
131 ----------
132 name : `str`
133 Name of the instrument (must match the return value of `getName`).
134 registry : `lsst.daf.butler.Registry`
135 Butler registry to query to find the information.
136 collection_prefix : `str`, optional
137 Prefix for collection names to use instead of the intrument's own
138 name. This is primarily for use in simulated-data repositories,
139 where the instrument name may not be necessary and/or sufficient to
140 distinguish between collections.
142 Returns
143 -------
144 instrument : `Instrument`
145 An instance of the relevant `Instrument`.
147 Notes
148 -----
149 The instrument must be registered in the corresponding butler.
151 Raises
152 ------
153 LookupError
154 Raised if the instrument is not known to the supplied registry.
155 ModuleNotFoundError
156 Raised if the class could not be imported. This could mean
157 that the relevant obs package has not been setup.
158 TypeError
159 Raised if the class name retrieved is not a string or the imported
160 symbol is not an `Instrument` subclass.
161 """
162 try:
163 records = list(registry.queryDimensionRecords("instrument", instrument=name))
164 except DataIdError:
165 records = None
166 if not records:
167 raise LookupError(f"No registered instrument with name '{name}'.")
168 cls_name = records[0].class_name
169 if not isinstance(cls_name, str):
170 raise TypeError(
171 f"Unexpected class name retrieved from {name} instrument dimension (got {cls_name})"
172 )
173 instrument_cls: type = doImportType(cls_name)
174 if not issubclass(instrument_cls, Instrument):
175 raise TypeError(
176 f"{instrument_cls!r}, obtained from importing {cls_name}, is not an Instrument subclass."
177 )
178 return instrument_cls(collection_prefix=collection_prefix)
180 @staticmethod
181 def from_string(
182 name: str, registry: Optional[Registry] = None, collection_prefix: Optional[str] = None
183 ) -> Instrument:
184 """Return an instance from the short name or class name.
186 If the instrument name is not qualified (does not contain a '.') and a
187 butler registry is provided, this will attempt to load the instrument
188 using `Instrument.fromName()`. Otherwise the instrument will be
189 imported and instantiated.
191 Parameters
192 ----------
193 name : `str`
194 The name or fully-qualified class name of an instrument.
195 registry : `lsst.daf.butler.Registry`, optional
196 Butler registry to query to find information about the instrument,
197 by default `None`.
198 collection_prefix : `str`, optional
199 Prefix for collection names to use instead of the intrument's own
200 name. This is primarily for use in simulated-data repositories,
201 where the instrument name may not be necessary and/or sufficient
202 to distinguish between collections.
204 Returns
205 -------
206 instrument : `Instrument`
207 The instantiated instrument.
209 Raises
210 ------
211 RuntimeError
212 Raised if the instrument can not be imported, instantiated, or
213 obtained from the registry.
214 TypeError
215 Raised if the instrument is not a subclass of
216 `~lsst.pipe.base.Instrument`.
218 See Also
219 --------
220 Instrument.fromName
221 """
222 if "." not in name and registry is not None:
223 try:
224 instr = Instrument.fromName(name, registry, collection_prefix=collection_prefix)
225 except Exception as err:
226 raise RuntimeError(
227 f"Could not get instrument from name: {name}. Failed with exception: {err}"
228 ) from err
229 else:
230 try:
231 instr_class = doImportType(name)
232 except Exception as err:
233 raise RuntimeError(
234 f"Could not import instrument: {name}. Failed with exception: {err}"
235 ) from err
236 instr = instr_class(collection_prefix=collection_prefix)
237 if not isinstance(instr, Instrument):
238 raise TypeError(f"{name} is not an Instrument subclass.")
239 return instr
241 @staticmethod
242 def importAll(registry: Registry) -> None:
243 """Import all the instruments known to this registry.
245 This will ensure that all metadata translators have been registered.
247 Parameters
248 ----------
249 registry : `lsst.daf.butler.Registry`
250 Butler registry to query to find the information.
252 Notes
253 -----
254 It is allowed for a particular instrument class to fail on import.
255 This might simply indicate that a particular obs package has
256 not been setup.
257 """
258 records = list(registry.queryDimensionRecords("instrument"))
259 for record in records:
260 cls = record.class_name
261 try:
262 doImportType(cls)
263 except Exception:
264 pass
266 @abstractmethod
267 def getRawFormatter(self, dataId: DataId) -> Type[Formatter]:
268 """Return the Formatter class that should be used to read a particular
269 raw file.
271 Parameters
272 ----------
273 dataId : `DataId`
274 Dimension-based ID for the raw file or files being ingested.
276 Returns
277 -------
278 formatter : `lsst.daf.butler.Formatter` class
279 Class to be used that reads the file into the correct
280 Python object for the raw data.
281 """
282 raise NotImplementedError()
284 def applyConfigOverrides(self, name: str, config: Config) -> None:
285 """Apply instrument-specific overrides for a task config.
287 Parameters
288 ----------
289 name : `str`
290 Name of the object being configured; typically the _DefaultName
291 of a Task.
292 config : `lsst.pex.config.Config`
293 Config instance to which overrides should be applied.
294 """
295 for root in self.configPaths:
296 path = os.path.join(root, f"{name}.py")
297 if os.path.exists(path):
298 config.load(path)
300 @staticmethod
301 def formatCollectionTimestamp(timestamp: Union[str, datetime.datetime]) -> str:
302 """Format a timestamp for use in a collection name.
304 Parameters
305 ----------
306 timestamp : `str` or `datetime.datetime`
307 Timestamp to format. May be a date or datetime string in extended
308 ISO format (assumed UTC), with or without a timezone specifier, a
309 datetime string in basic ISO format with a timezone specifier, a
310 naive `datetime.datetime` instance (assumed UTC) or a
311 timezone-aware `datetime.datetime` instance (converted to UTC).
312 This is intended to cover all forms that string ``CALIBDATE``
313 metadata values have taken in the past, as well as the format this
314 method itself writes out (to enable round-tripping).
316 Returns
317 -------
318 formatted : `str`
319 Standardized string form for the timestamp.
320 """
321 if isinstance(timestamp, str):
322 if "-" in timestamp:
323 # extended ISO format, with - and : delimiters
324 timestamp = datetime.datetime.fromisoformat(timestamp)
325 else:
326 # basic ISO format, with no delimiters (what this method
327 # returns)
328 timestamp = datetime.datetime.strptime(timestamp, "%Y%m%dT%H%M%S%z")
329 if not isinstance(timestamp, datetime.datetime):
330 raise TypeError(f"Unexpected date/time object: {timestamp!r}.")
331 if timestamp.tzinfo is not None:
332 timestamp = timestamp.astimezone(datetime.timezone.utc)
333 return f"{timestamp:%Y%m%dT%H%M%S}Z"
335 @staticmethod
336 def makeCollectionTimestamp() -> str:
337 """Create a timestamp string for use in a collection name from the
338 current time.
340 Returns
341 -------
342 formatted : `str`
343 Standardized string form of the current time.
344 """
345 return Instrument.formatCollectionTimestamp(datetime.datetime.now(tz=datetime.timezone.utc))
347 def makeDefaultRawIngestRunName(self) -> str:
348 """Make the default instrument-specific run collection string for raw
349 data ingest.
351 Returns
352 -------
353 coll : `str`
354 Run collection name to be used as the default for ingestion of
355 raws.
356 """
357 return self.makeCollectionName("raw", "all")
359 def makeUnboundedCalibrationRunName(self, *labels: str) -> str:
360 """Make a RUN collection name appropriate for inserting calibration
361 datasets whose validity ranges are unbounded.
363 Parameters
364 ----------
365 *labels : `str`
366 Extra strings to be included in the base name, using the default
367 delimiter for collection names. Usually this is the name of the
368 ticket on which the calibration collection is being created.
370 Returns
371 -------
372 name : `str`
373 Run collection name.
374 """
375 return self.makeCollectionName("calib", *labels, "unbounded")
377 def makeCuratedCalibrationRunName(self, calibDate: str, *labels: str) -> str:
378 """Make a RUN collection name appropriate for inserting curated
379 calibration datasets with the given ``CALIBDATE`` metadata value.
381 Parameters
382 ----------
383 calibDate : `str`
384 The ``CALIBDATE`` metadata value.
385 *labels : `str`
386 Strings to be included in the collection name (before
387 ``calibDate``, but after all other terms), using the default
388 delimiter for collection names. Usually this is the name of the
389 ticket on which the calibration collection is being created.
391 Returns
392 -------
393 name : `str`
394 Run collection name.
395 """
396 return self.makeCollectionName("calib", *labels, "curated", self.formatCollectionTimestamp(calibDate))
398 def makeCalibrationCollectionName(self, *labels: str) -> str:
399 """Make a CALIBRATION collection name appropriate for associating
400 calibration datasets with validity ranges.
402 Parameters
403 ----------
404 *labels : `str`
405 Strings to be appended to the base name, using the default
406 delimiter for collection names. Usually this is the name of the
407 ticket on which the calibration collection is being created.
409 Returns
410 -------
411 name : `str`
412 Calibration collection name.
413 """
414 return self.makeCollectionName("calib", *labels)
416 @staticmethod
417 def makeRefCatCollectionName(*labels: str) -> str:
418 """Return a global (not instrument-specific) name for a collection that
419 holds reference catalogs.
421 With no arguments, this returns the name of the collection that holds
422 all reference catalogs (usually a ``CHAINED`` collection, at least in
423 long-lived repos that may contain more than one reference catalog).
425 Parameters
426 ----------
427 *labels : `str`
428 Strings to be added to the global collection name, in order to
429 define a collection name for one or more reference catalogs being
430 ingested at the same time.
432 Returns
433 -------
434 name : `str`
435 Collection name.
437 Notes
438 -----
439 This is a ``staticmethod``, not a ``classmethod``, because it should
440 be the same for all instruments.
441 """
442 return "/".join(("refcats",) + labels)
444 def makeUmbrellaCollectionName(self) -> str:
445 """Return the name of the umbrella ``CHAINED`` collection for this
446 instrument that combines all standard recommended input collections.
448 This method should almost never be overridden by derived classes.
450 Returns
451 -------
452 name : `str`
453 Name for the umbrella collection.
454 """
455 return self.makeCollectionName("defaults")
457 def makeCollectionName(self, *labels: str) -> str:
458 """Get the instrument-specific collection string to use as derived
459 from the supplied labels.
461 Parameters
462 ----------
463 *labels : `str`
464 Strings to be combined with the instrument name to form a
465 collection name.
467 Returns
468 -------
469 name : `str`
470 Collection name to use that includes the instrument's recommended
471 prefix.
472 """
473 return "/".join((self.collection_prefix,) + labels)