Coverage for python/lsst/obs/base/defineVisits.py: 27%
399 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-11-30 12:18 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-11-30 12:18 +0000
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = [
25 "DefineVisitsConfig",
26 "DefineVisitsTask",
27 "GroupExposuresConfig",
28 "GroupExposuresTask",
29 "VisitDefinitionData",
30 "VisitSystem",
31]
33import cmath
34import dataclasses
35import enum
36import math
37import operator
38from abc import ABCMeta, abstractmethod
39from collections import defaultdict
40from collections.abc import Callable, Iterable
41from typing import Any, ClassVar, TypeVar, cast
43import lsst.geom
44from lsst.afw.cameraGeom import FOCAL_PLANE, PIXELS
45from lsst.daf.butler import Butler, DataCoordinate, DataId, DimensionRecord, Progress, Timespan
46from lsst.geom import Box2D
47from lsst.pex.config import Config, Field, makeRegistry, registerConfigurable
48from lsst.pipe.base import Instrument, Task
49from lsst.sphgeom import ConvexPolygon, Region, UnitVector3d
50from lsst.utils.introspection import get_full_type_name
52from ._instrument import loadCamera
55class VisitSystem(enum.Enum):
56 """Enumeration used to label different visit systems."""
58 ONE_TO_ONE = 0
59 """Each exposure is assigned to its own visit."""
61 BY_GROUP_METADATA = 1
62 """Visit membership is defined by the value of the exposure.group_id."""
64 BY_SEQ_START_END = 2
65 """Visit membership is defined by the values of the ``exposure.day_obs``,
66 ``exposure.seq_start``, and ``exposure.seq_end`` values.
67 """
69 @classmethod
70 def all(cls) -> frozenset[VisitSystem]:
71 """Return a `frozenset` containing all members."""
72 return frozenset(cls.__members__.values())
74 @classmethod
75 def from_name(cls, external_name: str) -> VisitSystem:
76 """Construct the enumeration from given name."""
77 name = external_name.upper()
78 name = name.replace("-", "_")
79 try:
80 return cls.__members__[name]
81 except KeyError:
82 raise KeyError(f"Visit system named '{external_name}' not known.") from None
84 @classmethod
85 def from_names(cls, names: Iterable[str] | None) -> frozenset[VisitSystem]:
86 """Return a `frozenset` of all the visit systems matching the supplied
87 names.
89 Parameters
90 ----------
91 names : iterable of `str`, or `None`
92 Names of visit systems. Case insensitive. If `None` or empty, all
93 the visit systems are returned.
95 Returns
96 -------
97 systems : `frozenset` of `VisitSystem`
98 The matching visit systems.
99 """
100 if not names:
101 return cls.all()
103 return frozenset({cls.from_name(name) for name in names})
105 def __str__(self) -> str:
106 name = self.name.lower()
107 name = name.replace("_", "-")
108 return name
111@dataclasses.dataclass
112class VisitDefinitionData:
113 """Struct representing a group of exposures that will be used to define a
114 visit.
115 """
117 instrument: str
118 """Name of the instrument this visit will be associated with.
119 """
121 id: int
122 """Integer ID of the visit.
124 This must be unique across all visit systems for the instrument.
125 """
127 name: str
128 """String name for the visit.
130 This must be unique across all visit systems for the instrument.
131 """
133 visit_systems: set[VisitSystem]
134 """All the visit systems associated with this visit."""
136 exposures: list[DimensionRecord] = dataclasses.field(default_factory=list)
137 """Dimension records for the exposures that are part of this visit.
138 """
141@dataclasses.dataclass
142class _VisitRecords:
143 """Struct containing the dimension records associated with a visit."""
145 visit: DimensionRecord
146 """Record for the 'visit' dimension itself.
147 """
149 visit_definition: list[DimensionRecord]
150 """Records for 'visit_definition', which relates 'visit' to 'exposure'.
151 """
153 visit_detector_region: list[DimensionRecord]
154 """Records for 'visit_detector_region', which associates the combination
155 of a 'visit' and a 'detector' with a region on the sky.
156 """
158 visit_system_membership: list[DimensionRecord]
159 """Records relating visits to an associated visit system."""
162class GroupExposuresConfig(Config):
163 """Configure exposure grouping."""
166class GroupExposuresTask(Task, metaclass=ABCMeta):
167 """Abstract base class for the subtask of `DefineVisitsTask` that is
168 responsible for grouping exposures into visits.
170 Subclasses should be registered with `GroupExposuresTask.registry` to
171 enable use by `DefineVisitsTask`, and should generally correspond to a
172 particular 'visit_system' dimension value. They are also responsible for
173 defining visit IDs and names that are unique across all visit systems in
174 use by an instrument.
176 Parameters
177 ----------
178 config : `GroupExposuresConfig`
179 Configuration information.
180 **kwargs
181 Additional keyword arguments forwarded to the `Task` constructor.
182 """
184 def __init__(self, config: GroupExposuresConfig, **kwargs: Any):
185 Task.__init__(self, config=config, **kwargs)
187 ConfigClass = GroupExposuresConfig
189 _DefaultName = "groupExposures"
191 registry = makeRegistry(
192 doc="Registry of algorithms for grouping exposures into visits.",
193 configBaseType=GroupExposuresConfig,
194 )
196 @abstractmethod
197 def find_missing(
198 self, exposures: list[DimensionRecord], registry: lsst.daf.butler.Registry
199 ) -> list[DimensionRecord]:
200 """Determine, if possible, which exposures might be missing.
202 Parameters
203 ----------
204 exposures : `list` of `lsst.daf.butler.DimensionRecord`
205 The exposure records to analyze.
206 registry : `lsst.daf.butler.Registry`
207 A butler registry that contains these exposure records.
209 Returns
210 -------
211 missing : `list` of `lsst.daf.butler.DimensionRecord`
212 Any exposure records present in registry that were related to
213 the given exposures but were missing from that list and deemed
214 to be relevant.
216 Notes
217 -----
218 Only some grouping schemes are able to find missing exposures. It
219 is acceptable to return an empty list.
220 """
221 raise NotImplementedError()
223 @abstractmethod
224 def group_exposures(self, exposures: list[DimensionRecord]) -> dict[Any, list[DimensionRecord]]:
225 """Group the exposures in a way most natural for this visit definition.
227 Parameters
228 ----------
229 exposures : `list` of `lsst.daf.butler.DimensionRecord`
230 The exposure records to group.
232 Returns
233 -------
234 groups : `dict` [Any, `list` of `DimensionRecord`]
235 Groupings of exposure records. The key type is relevant to the
236 specific visit definition and could be a string or a tuple.
237 """
238 raise NotImplementedError()
240 @abstractmethod
241 def group(self, exposures: list[DimensionRecord]) -> Iterable[VisitDefinitionData]:
242 """Group the given exposures into visits.
244 Parameters
245 ----------
246 exposures : `list` [ `DimensionRecord` ]
247 DimensionRecords (for the 'exposure' dimension) describing the
248 exposures to group.
250 Returns
251 -------
252 visits : `Iterable` [ `VisitDefinitionData` ]
253 Structs identifying the visits and the exposures associated with
254 them. This may be an iterator or a container.
255 """
256 raise NotImplementedError()
258 def getVisitSystems(self) -> set[VisitSystem]:
259 """Return identifiers for the 'visit_system' dimension this
260 algorithm implements.
262 Returns
263 -------
264 visit_systems : `Set` [`VisitSystem`]
265 The visit systems used by this algorithm.
266 """
267 raise NotImplementedError()
270class ComputeVisitRegionsConfig(Config):
271 """Configure visit region calculations."""
273 padding: Field[int] = Field(
274 dtype=int,
275 default=250,
276 doc=(
277 "Pad raw image bounding boxes with specified number of pixels "
278 "when calculating their (conservatively large) region on the "
279 "sky. Note that the config value for pixelMargin of the "
280 "reference object loaders in meas_algorithms should be <= "
281 "the value set here."
282 ),
283 )
286class ComputeVisitRegionsTask(Task, metaclass=ABCMeta):
287 """Abstract base class for the subtask of `DefineVisitsTask` that is
288 responsible for extracting spatial regions for visits and visit+detector
289 combinations.
291 Subclasses should be registered with `ComputeVisitRegionsTask.registry` to
292 enable use by `DefineVisitsTask`.
294 Parameters
295 ----------
296 config : `ComputeVisitRegionsConfig`
297 Configuration information.
298 butler : `lsst.daf.butler.Butler`
299 The butler to use.
300 **kwargs
301 Additional keyword arguments forwarded to the `Task` constructor.
302 """
304 def __init__(self, config: ComputeVisitRegionsConfig, *, butler: Butler, **kwargs: Any):
305 Task.__init__(self, config=config, **kwargs)
306 self.butler = butler
307 self.instrumentMap: dict[str, Instrument] = {}
309 ConfigClass = ComputeVisitRegionsConfig
311 _DefaultName = "computeVisitRegions"
313 registry = makeRegistry(
314 doc="Registry of algorithms for computing on-sky regions for visits and visit+detector combinations.",
315 configBaseType=ComputeVisitRegionsConfig,
316 )
318 def getInstrument(self, instrumentName: str) -> Instrument:
319 """Retrieve an `~lsst.obs.base.Instrument` associated with this
320 instrument name.
322 Parameters
323 ----------
324 instrumentName : `str`
325 The name of the instrument.
327 Returns
328 -------
329 instrument : `~lsst.obs.base.Instrument`
330 The associated instrument object.
332 Notes
333 -----
334 The result is cached.
335 """
336 instrument = self.instrumentMap.get(instrumentName)
337 if instrument is None:
338 instrument = Instrument.fromName(instrumentName, self.butler.registry)
339 self.instrumentMap[instrumentName] = instrument
340 return instrument
342 @abstractmethod
343 def compute(
344 self, visit: VisitDefinitionData, *, collections: Any = None
345 ) -> tuple[Region, dict[int, Region]]:
346 """Compute regions for the given visit and all detectors in that visit.
348 Parameters
349 ----------
350 visit : `VisitDefinitionData`
351 Struct describing the visit and the exposures associated with it.
352 collections : Any, optional
353 Collections to be searched for raws and camera geometry, overriding
354 ``self.butler.collections``.
355 Can be any of the types supported by the ``collections`` argument
356 to butler construction.
358 Returns
359 -------
360 visitRegion : `lsst.sphgeom.Region`
361 Region for the full visit.
362 visitDetectorRegions : `dict` [ `int`, `lsst.sphgeom.Region` ]
363 Dictionary mapping detector ID to the region for that detector.
364 Should include all detectors in the visit.
365 """
366 raise NotImplementedError()
369class DefineVisitsConfig(Config):
370 """Configure visit definition."""
372 groupExposures = GroupExposuresTask.registry.makeField(
373 doc="Algorithm for grouping exposures into visits.",
374 default="one-to-one-and-by-counter",
375 )
376 computeVisitRegions = ComputeVisitRegionsTask.registry.makeField(
377 doc="Algorithm from computing visit and visit+detector regions.",
378 default="single-raw-wcs",
379 )
380 ignoreNonScienceExposures: Field[bool] = Field(
381 doc=(
382 "If True, silently ignore input exposures that do not have "
383 "observation_type=SCIENCE. If False, raise an exception if one "
384 "encountered."
385 ),
386 dtype=bool,
387 optional=False,
388 default=True,
389 )
390 updateObsCoreTable: Field[bool] = Field(
391 doc=(
392 "If True, update exposure regions in obscore table after visits "
393 "are defined. If False, do not update obscore table."
394 ),
395 dtype=bool,
396 default=True,
397 )
400class DefineVisitsTask(Task):
401 """Driver Task for defining visits (and their spatial regions) in Gen3
402 Butler repositories.
404 Parameters
405 ----------
406 config : `DefineVisitsConfig`
407 Configuration for the task.
408 butler : `~lsst.daf.butler.Butler`
409 Writeable butler instance. Will be used to read `raw.wcs` and `camera`
410 datasets and insert/sync dimension data.
411 **kwargs
412 Additional keyword arguments are forwarded to the `lsst.pipe.base.Task`
413 constructor.
415 Notes
416 -----
417 Each instance of `DefineVisitsTask` reads from / writes to the same Butler.
418 Each invocation of `DefineVisitsTask.run` processes an independent group of
419 exposures into one or more new visits, all belonging to the same visit
420 system and instrument.
422 The actual work of grouping exposures and computing regions is delegated
423 to pluggable subtasks (`GroupExposuresTask` and `ComputeVisitRegionsTask`),
424 respectively. The defaults are to create one visit for every exposure,
425 and to use exactly one (arbitrary) detector-level raw dataset's WCS along
426 with camera geometry to compute regions for all detectors. Other
427 implementations can be created and configured for instruments for which
428 these choices are unsuitable (e.g. because visits and exposures are not
429 one-to-one, or because ``raw.wcs`` datasets for different detectors may not
430 be consistent with camera geometry).
432 It is not necessary in general to ingest all raws for an exposure before
433 defining a visit that includes the exposure; this depends entirely on the
434 `ComputeVisitRegionTask` subclass used. For the default configuration,
435 a single raw for each exposure is sufficient.
437 Defining the same visit the same way multiple times (e.g. via multiple
438 invocations of this task on the same exposures, with the same
439 configuration) is safe, but it may be inefficient, as most of the work must
440 be done before new visits can be compared to existing visits.
441 """
443 def __init__(self, config: DefineVisitsConfig, *, butler: Butler, **kwargs: Any):
444 config.validate() # Not a CmdlineTask nor PipelineTask, so have to validate the config here.
445 super().__init__(config, **kwargs)
446 self.butler = butler
447 self.universe = self.butler.dimensions
448 self.progress = Progress("obs.base.DefineVisitsTask")
449 self.makeSubtask("groupExposures")
450 self.makeSubtask("computeVisitRegions", butler=self.butler)
452 def _reduce_kwargs(self) -> dict:
453 # Add extra parameters to pickle
454 return dict(**super()._reduce_kwargs(), butler=self.butler)
456 ConfigClass: ClassVar[type[Config]] = DefineVisitsConfig
458 _DefaultName: ClassVar[str] = "defineVisits"
460 config: DefineVisitsConfig
461 groupExposures: GroupExposuresTask
462 computeVisitRegions: ComputeVisitRegionsTask
464 def _buildVisitRecords(
465 self, definition: VisitDefinitionData, *, collections: Any = None
466 ) -> _VisitRecords:
467 """Build the DimensionRecords associated with a visit.
469 Parameters
470 ----------
471 definition : `VisitDefinitionData`
472 Struct with identifiers for the visit and records for its
473 constituent exposures.
474 collections : Any, optional
475 Collections to be searched for raws and camera geometry, overriding
476 ``self.butler.collections``.
477 Can be any of the types supported by the ``collections`` argument
478 to butler construction.
480 Results
481 -------
482 records : `_VisitRecords`
483 Struct containing DimensionRecords for the visit, including
484 associated dimension elements.
485 """
486 dimension = self.universe["visit"]
488 # Some registries support additional items.
489 supported = {meta.name for meta in dimension.metadata}
491 # Compute all regions.
492 visitRegion, visitDetectorRegions = self.computeVisitRegions.compute(
493 definition, collections=collections
494 )
495 # Aggregate other exposure quantities.
496 timespan = Timespan(
497 begin=_reduceOrNone(min, (e.timespan.begin for e in definition.exposures)),
498 end=_reduceOrNone(max, (e.timespan.end for e in definition.exposures)),
499 )
500 exposure_time = _reduceOrNone(operator.add, (e.exposure_time for e in definition.exposures))
501 physical_filter = _reduceOrNone(_value_if_equal, (e.physical_filter for e in definition.exposures))
502 target_name = _reduceOrNone(_value_if_equal, (e.target_name for e in definition.exposures))
503 science_program = _reduceOrNone(_value_if_equal, (e.science_program for e in definition.exposures))
505 # observing day for a visit is defined by the earliest observation
506 # of the visit
507 observing_day = _reduceOrNone(min, (e.day_obs for e in definition.exposures))
508 observation_reason = _reduceOrNone(
509 _value_if_equal, (e.observation_reason for e in definition.exposures)
510 )
511 if observation_reason is None:
512 # Be explicit about there being multiple reasons
513 observation_reason = "various"
515 # Use the mean zenith angle as an approximation
516 zenith_angle = _reduceOrNone(operator.add, (e.zenith_angle for e in definition.exposures))
517 if zenith_angle is not None:
518 zenith_angle /= len(definition.exposures)
520 # New records that may not be supported.
521 extras: dict[str, Any] = {}
522 if "seq_num" in supported:
523 extras["seq_num"] = _reduceOrNone(min, (e.seq_num for e in definition.exposures))
524 if "azimuth" in supported:
525 # Must take into account 0/360 problem.
526 extras["azimuth"] = _calc_mean_angle([e.azimuth for e in definition.exposures])
528 # visit_system handling changed. This is the logic for visit/exposure
529 # that has support for seq_start/seq_end.
530 if "seq_num" in supported:
531 # Map visit to exposure.
532 visit_definition = [
533 self.universe["visit_definition"].RecordClass(
534 instrument=definition.instrument,
535 visit=definition.id,
536 exposure=exposure.id,
537 )
538 for exposure in definition.exposures
539 ]
541 # Map visit to visit system.
542 visit_system_membership = []
543 for visit_system in self.groupExposures.getVisitSystems():
544 if visit_system in definition.visit_systems:
545 record = self.universe["visit_system_membership"].RecordClass(
546 instrument=definition.instrument,
547 visit=definition.id,
548 visit_system=visit_system.value,
549 )
550 visit_system_membership.append(record)
552 else:
553 # The old approach can only handle one visit system at a time.
554 # If we have been configured with multiple options, prefer the
555 # one-to-one.
556 visit_systems = self.groupExposures.getVisitSystems()
557 if len(visit_systems) > 1:
558 one_to_one = VisitSystem.from_name("one-to-one")
559 if one_to_one not in visit_systems:
560 raise ValueError(
561 f"Multiple visit systems specified ({visit_systems}) for use with old"
562 " dimension universe but unable to find one-to-one."
563 )
564 visit_system = one_to_one
565 else:
566 visit_system = visit_systems.pop()
568 extras["visit_system"] = visit_system.value
570 # The old visit_definition included visit system.
571 visit_definition = [
572 self.universe["visit_definition"].RecordClass(
573 instrument=definition.instrument,
574 visit=definition.id,
575 exposure=exposure.id,
576 visit_system=visit_system.value,
577 )
578 for exposure in definition.exposures
579 ]
581 # This concept does not exist in old schema.
582 visit_system_membership = []
584 # Construct the actual DimensionRecords.
585 return _VisitRecords(
586 visit=dimension.RecordClass(
587 instrument=definition.instrument,
588 id=definition.id,
589 name=definition.name,
590 physical_filter=physical_filter,
591 target_name=target_name,
592 science_program=science_program,
593 observation_reason=observation_reason,
594 day_obs=observing_day,
595 zenith_angle=zenith_angle,
596 exposure_time=exposure_time,
597 timespan=timespan,
598 region=visitRegion,
599 # TODO: no seeing value in exposure dimension records, so we
600 # can't set that here. But there are many other columns that
601 # both dimensions should probably have as well.
602 **extras,
603 ),
604 visit_definition=visit_definition,
605 visit_system_membership=visit_system_membership,
606 visit_detector_region=[
607 self.universe["visit_detector_region"].RecordClass(
608 instrument=definition.instrument,
609 visit=definition.id,
610 detector=detectorId,
611 region=detectorRegion,
612 )
613 for detectorId, detectorRegion in visitDetectorRegions.items()
614 ],
615 )
617 def run(
618 self,
619 dataIds: Iterable[DataId],
620 *,
621 collections: str | None = None,
622 update_records: bool = False,
623 incremental: bool = False,
624 ) -> None:
625 """Add visit definitions to the registry for the given exposures.
627 Parameters
628 ----------
629 dataIds : `Iterable` [ `dict` or `~lsst.daf.butler.DataCoordinate` ]
630 Exposure-level data IDs. These must all correspond to the same
631 instrument, and are expected to be on-sky science exposures.
632 collections : Any, optional
633 Collections to be searched for raws and camera geometry, overriding
634 ``self.butler.collections``.
635 Can be any of the types supported by the ``collections`` argument
636 to butler construction.
637 update_records : `bool`, optional
638 If `True` (`False` is default), update existing visit records that
639 conflict with the new ones instead of rejecting them (and when this
640 occurs, update visit_detector_region as well). THIS IS AN ADVANCED
641 OPTION THAT SHOULD ONLY BE USED TO FIX REGIONS AND/OR METADATA THAT
642 ARE KNOWN TO BE BAD, AND IT CANNOT BE USED TO REMOVE EXPOSURES OR
643 DETECTORS FROM A VISIT.
644 incremental : `bool`, optional
645 If `True` indicate that exposures are being ingested incrementally
646 and visit definition will be run on partial visits. This will
647 force ``update_records`` to `True`. If there is any risk that
648 files are being ingested incrementally it is critical that this
649 parameter is set to `True` and not to rely on ``updated_records``.
651 Raises
652 ------
653 lsst.daf.butler.registry.ConflictingDefinitionError
654 Raised if a visit ID conflict is detected and the existing visit
655 differs from the new one.
656 """
657 # Normalize, expand, and deduplicate data IDs.
658 self.log.info("Preprocessing data IDs.")
659 dimensions = self.universe.conform(["exposure"])
660 data_id_set: set[DataCoordinate] = {
661 self.butler.registry.expandDataId(d, dimensions=dimensions) for d in dataIds
662 }
663 if not data_id_set:
664 raise RuntimeError("No exposures given.")
665 if incremental:
666 update_records = True
667 # Extract exposure DimensionRecords, check that there's only one
668 # instrument in play, and check for non-science exposures.
669 exposures = []
670 instruments = set()
671 for dataId in data_id_set:
672 record = dataId.records["exposure"]
673 assert record is not None, "Guaranteed by expandDataIds call earlier."
674 if record.tracking_ra is None or record.tracking_dec is None or record.sky_angle is None:
675 if self.config.ignoreNonScienceExposures:
676 continue
677 else:
678 raise RuntimeError(
679 f"Input exposure {dataId} has observation_type "
680 f"{record.observation_type}, but is not on sky."
681 )
682 instruments.add(dataId["instrument"])
683 exposures.append(record)
684 if not exposures:
685 self.log.info("No on-sky exposures found after filtering.")
686 return
687 if len(instruments) > 1:
688 raise RuntimeError(
689 "All data IDs passed to DefineVisitsTask.run must be "
690 f"from the same instrument; got {instruments}."
691 )
692 (instrument,) = instruments
693 # Ensure the visit_system our grouping algorithm uses is in the
694 # registry, if it wasn't already.
695 visitSystems = self.groupExposures.getVisitSystems()
696 for visitSystem in visitSystems:
697 self.log.info("Registering visit_system %d: %s.", visitSystem.value, visitSystem)
698 self.butler.registry.syncDimensionData(
699 "visit_system",
700 {"instrument": instrument, "id": visitSystem.value, "name": str(visitSystem)},
701 )
703 # In true incremental we will be given the second snap on its
704 # own on the assumption that the previous snap was already handled.
705 # For correct grouping we need access to the other exposures in the
706 # visit.
707 if incremental:
708 exposures.extend(self.groupExposures.find_missing(exposures, self.butler.registry))
710 # Group exposures into visits, delegating to subtask.
711 self.log.info("Grouping %d exposure(s) into visits.", len(exposures))
712 definitions = list(self.groupExposures.group(exposures))
713 # Iterate over visits, compute regions, and insert dimension data, one
714 # transaction per visit. If a visit already exists, we skip all other
715 # inserts.
716 self.log.info("Computing regions and other metadata for %d visit(s).", len(definitions))
717 for visitDefinition in self.progress.wrap(
718 definitions, total=len(definitions), desc="Computing regions and inserting visits"
719 ):
720 visitRecords = self._buildVisitRecords(visitDefinition, collections=collections)
721 with self.butler.registry.transaction():
722 inserted_or_updated = self.butler.registry.syncDimensionData(
723 "visit",
724 visitRecords.visit,
725 update=update_records,
726 )
727 if inserted_or_updated:
728 if inserted_or_updated is True:
729 # This is a new visit, not an update to an existing
730 # one, so insert visit definition.
731 # We don't allow visit definitions to change even when
732 # asked to update, because we'd have to delete the old
733 # visit_definitions first and also worry about what
734 # this does to datasets that already use the visit.
735 self.butler.registry.insertDimensionData(
736 "visit_definition", *visitRecords.visit_definition
737 )
738 if visitRecords.visit_system_membership:
739 self.butler.registry.insertDimensionData(
740 "visit_system_membership", *visitRecords.visit_system_membership
741 )
742 elif incremental and len(visitRecords.visit_definition) > 1:
743 # The visit record was modified. This could happen
744 # if a multi-snap visit was redefined with an
745 # additional snap so play it safe and allow for the
746 # visit definition to be updated. We use update=False
747 # here since there should not be any rows updated,
748 # just additional rows added. update=True does not work
749 # correctly with multiple records. In incremental mode
750 # we assume that the caller wants the visit definition
751 # to be updated and has no worries about provenance
752 # with the previous definition.
753 for definition in visitRecords.visit_definition:
754 self.butler.registry.syncDimensionData("visit_definition", definition)
756 # [Re]Insert visit_detector_region records for both inserts
757 # and updates, because we do allow updating to affect the
758 # region calculations.
759 self.butler.registry.insertDimensionData(
760 "visit_detector_region", *visitRecords.visit_detector_region, replace=update_records
761 )
763 # Update obscore exposure records with region information
764 # from corresponding visits.
765 if self.config.updateObsCoreTable:
766 if obscore_manager := self.butler.registry.obsCoreTableManager:
767 obscore_updates: list[tuple[int, int, Region]] = []
768 exposure_ids = [rec.exposure for rec in visitRecords.visit_definition]
769 for record in visitRecords.visit_detector_region:
770 obscore_updates += [
771 (exposure, record.detector, record.region) for exposure in exposure_ids
772 ]
773 if obscore_updates:
774 obscore_manager.update_exposure_regions(
775 cast(str, instrument), obscore_updates
776 )
779_T = TypeVar("_T")
782def _reduceOrNone(func: Callable[[_T, _T], _T | None], iterable: Iterable[_T | None]) -> _T | None:
783 """Apply a binary function to pairs of elements in an iterable until a
784 single value is returned, but return `None` if any element is `None` or
785 there are no elements.
786 """
787 r: _T | None = None
788 for v in iterable:
789 if v is None:
790 return None
791 if r is None:
792 r = v
793 else:
794 r = func(r, v)
795 return r
798def _value_if_equal(a: _T, b: _T) -> _T | None:
799 """Return either argument if they are equal, or `None` if they are not."""
800 return a if a == b else None
803def _calc_mean_angle(angles: list[float]) -> float:
804 """Calculate the mean angle, taking into account 0/360 wrapping.
806 Parameters
807 ----------
808 angles : `list` [`float`]
809 Angles to average together, in degrees.
811 Returns
812 -------
813 average : `float`
814 Average angle in degrees.
815 """
816 # Save on all the math if we only have one value.
817 if len(angles) == 1:
818 return angles[0]
820 # Convert polar coordinates of unit circle to complex values.
821 # Average the complex values.
822 # Convert back to a phase angle.
823 return math.degrees(cmath.phase(sum(cmath.rect(1.0, math.radians(d)) for d in angles) / len(angles)))
826class _GroupExposuresOneToOneConfig(GroupExposuresConfig):
827 visitSystemId: Field[int] = Field(
828 doc="Integer ID of the visit_system implemented by this grouping algorithm.",
829 dtype=int,
830 default=0,
831 deprecated="No longer used. Replaced by enum.",
832 )
833 visitSystemName: Field[str] = Field(
834 doc="String name of the visit_system implemented by this grouping algorithm.",
835 dtype=str,
836 default="one-to-one",
837 deprecated="No longer used. Replaced by enum.",
838 )
841@registerConfigurable("one-to-one", GroupExposuresTask.registry)
842class _GroupExposuresOneToOneTask(GroupExposuresTask, metaclass=ABCMeta):
843 """An exposure grouping algorithm that simply defines one visit for each
844 exposure, reusing the exposures identifiers for the visit.
845 """
847 ConfigClass = _GroupExposuresOneToOneConfig
849 def find_missing(
850 self, exposures: list[DimensionRecord], registry: lsst.daf.butler.Registry
851 ) -> list[DimensionRecord]:
852 # By definition no exposures can be missing.
853 return []
855 def group_exposures(self, exposures: list[DimensionRecord]) -> dict[Any, list[DimensionRecord]]:
856 # No grouping.
857 return {exposure.id: [exposure] for exposure in exposures}
859 def group(self, exposures: list[DimensionRecord]) -> Iterable[VisitDefinitionData]:
860 # Docstring inherited from GroupExposuresTask.
861 visit_systems = {VisitSystem.from_name("one-to-one")}
862 for exposure in exposures:
863 yield VisitDefinitionData(
864 instrument=exposure.instrument,
865 id=exposure.id,
866 name=exposure.obs_id,
867 exposures=[exposure],
868 visit_systems=visit_systems,
869 )
871 def getVisitSystems(self) -> set[VisitSystem]:
872 # Docstring inherited from GroupExposuresTask.
873 return set(VisitSystem.from_names(["one-to-one"]))
876class _GroupExposuresByGroupMetadataConfig(GroupExposuresConfig):
877 visitSystemId: Field[int] = Field(
878 doc="Integer ID of the visit_system implemented by this grouping algorithm.",
879 dtype=int,
880 default=1,
881 deprecated="No longer used. Replaced by enum.",
882 )
883 visitSystemName: Field[str] = Field(
884 doc="String name of the visit_system implemented by this grouping algorithm.",
885 dtype=str,
886 default="by-group-metadata",
887 deprecated="No longer used. Replaced by enum.",
888 )
891@registerConfigurable("by-group-metadata", GroupExposuresTask.registry)
892class _GroupExposuresByGroupMetadataTask(GroupExposuresTask, metaclass=ABCMeta):
893 """An exposure grouping algorithm that uses exposure.group_name and
894 exposure.group_id.
896 This algorithm _assumes_ exposure.group_id (generally populated from
897 `astro_metadata_translator.ObservationInfo.visit_id`) is not just unique,
898 but disjoint from all `ObservationInfo.exposure_id` values - if it isn't,
899 it will be impossible to ever use both this grouping algorithm and the
900 one-to-one algorithm for a particular camera in the same data repository.
901 """
903 ConfigClass = _GroupExposuresByGroupMetadataConfig
905 def find_missing(
906 self, exposures: list[DimensionRecord], registry: lsst.daf.butler.Registry
907 ) -> list[DimensionRecord]:
908 groups = self.group_exposures(exposures)
909 missing_exposures: list[DimensionRecord] = []
910 for exposures_in_group in groups.values():
911 # We can not tell how many exposures are expected to be in the
912 # visit so we have to query every time.
913 first = exposures_in_group[0]
914 records = set(
915 registry.queryDimensionRecords(
916 "exposure",
917 where="exposure.group_name = group",
918 bind={"group": first.group_name},
919 instrument=first.instrument,
920 )
921 )
922 records.difference_update(set(exposures_in_group))
923 missing_exposures.extend(list(records))
924 return missing_exposures
926 def group_exposures(self, exposures: list[DimensionRecord]) -> dict[Any, list[DimensionRecord]]:
927 groups = defaultdict(list)
928 for exposure in exposures:
929 groups[exposure.group_name].append(exposure)
930 return groups
932 def group(self, exposures: list[DimensionRecord]) -> Iterable[VisitDefinitionData]:
933 # Docstring inherited from GroupExposuresTask.
934 visit_systems = {VisitSystem.from_name("by-group-metadata")}
935 groups = self.group_exposures(exposures)
936 for visitName, exposuresInGroup in groups.items():
937 instrument = exposuresInGroup[0].instrument
938 visitId = exposuresInGroup[0].group_id
939 assert all(
940 e.group_id == visitId for e in exposuresInGroup
941 ), "Grouping by exposure.group_name does not yield consistent group IDs"
942 yield VisitDefinitionData(
943 instrument=instrument,
944 id=visitId,
945 name=visitName,
946 exposures=exposuresInGroup,
947 visit_systems=visit_systems,
948 )
950 def getVisitSystems(self) -> set[VisitSystem]:
951 # Docstring inherited from GroupExposuresTask.
952 return set(VisitSystem.from_names(["by-group-metadata"]))
955class _GroupExposuresByCounterAndExposuresConfig(GroupExposuresConfig):
956 visitSystemId: Field[int] = Field(
957 doc="Integer ID of the visit_system implemented by this grouping algorithm.",
958 dtype=int,
959 default=2,
960 deprecated="No longer used. Replaced by enum.",
961 )
962 visitSystemName: Field[str] = Field(
963 doc="String name of the visit_system implemented by this grouping algorithm.",
964 dtype=str,
965 default="by-counter-and-exposures",
966 deprecated="No longer used. Replaced by enum.",
967 )
970@registerConfigurable("one-to-one-and-by-counter", GroupExposuresTask.registry)
971class _GroupExposuresByCounterAndExposuresTask(GroupExposuresTask, metaclass=ABCMeta):
972 """An exposure grouping algorithm that uses the sequence start and
973 sequence end metadata to create multi-exposure visits, but also
974 creates one-to-one visits.
976 This algorithm uses the exposure.seq_start and
977 exposure.seq_end fields to collect related snaps.
978 It also groups single exposures.
979 """
981 ConfigClass = _GroupExposuresByCounterAndExposuresConfig
983 def find_missing(
984 self, exposures: list[DimensionRecord], registry: lsst.daf.butler.Registry
985 ) -> list[DimensionRecord]:
986 """Analyze the exposures and return relevant exposures known to
987 registry.
988 """
989 groups = self.group_exposures(exposures)
990 missing_exposures: list[DimensionRecord] = []
991 for exposures_in_group in groups.values():
992 sorted_exposures = sorted(exposures_in_group, key=lambda e: e.seq_num)
993 first = sorted_exposures[0]
995 # Only need to look for the seq_nums that we don't already have.
996 seq_nums = set(range(first.seq_start, first.seq_end + 1))
997 seq_nums.difference_update({exp.seq_num for exp in sorted_exposures})
999 if seq_nums:
1000 # Missing something. Check registry.
1001 records = list(
1002 registry.queryDimensionRecords(
1003 "exposure",
1004 where="exposure.seq_start = seq_start AND exposure.seq_end = seq_end AND "
1005 "exposure.seq_num IN (seq_nums)",
1006 bind={"seq_start": first.seq_start, "seq_end": first.seq_end, "seq_nums": seq_nums},
1007 instrument=first.instrument,
1008 )
1009 )
1010 missing_exposures.extend(records)
1012 return missing_exposures
1014 def group_exposures(self, exposures: list[DimensionRecord]) -> dict[Any, list[DimensionRecord]]:
1015 groups = defaultdict(list)
1016 for exposure in exposures:
1017 groups[exposure.day_obs, exposure.seq_start, exposure.seq_end].append(exposure)
1018 return groups
1020 def group(self, exposures: list[DimensionRecord]) -> Iterable[VisitDefinitionData]:
1021 # Docstring inherited from GroupExposuresTask.
1022 system_one_to_one = VisitSystem.from_name("one-to-one")
1023 system_seq_start_end = VisitSystem.from_name("by-seq-start-end")
1025 groups = self.group_exposures(exposures)
1026 for visit_key, exposures_in_group in groups.items():
1027 instrument = exposures_in_group[0].instrument
1029 # It is possible that the first exposure in a visit has not
1030 # been ingested. This can be determined and if that is the case
1031 # we can not reliably define the multi-exposure visit.
1032 skip_multi = False
1033 sorted_exposures = sorted(exposures_in_group, key=lambda e: e.seq_num)
1034 first = sorted_exposures.pop(0)
1035 if first.seq_num != first.seq_start:
1036 # Special case seq_num == 0 since that implies that the
1037 # instrument has no counters and therefore no multi-exposure
1038 # visits.
1039 if first.seq_num != 0:
1040 self.log.warning(
1041 "First exposure for visit %s is not present. Skipping the multi-snap definition.",
1042 visit_key,
1043 )
1044 skip_multi = True
1046 multi_exposure = False
1047 if first.seq_start != first.seq_end:
1048 # This is a multi-exposure visit regardless of the number
1049 # of exposures present.
1050 multi_exposure = True
1052 # Define the one-to-one visits.
1053 for exposure in exposures_in_group:
1054 # Default is to use the exposure ID and name unless
1055 # this is the first exposure in a multi-exposure visit.
1056 visit_name = exposure.obs_id
1057 visit_id = exposure.id
1058 visit_systems = {system_one_to_one}
1060 if not multi_exposure:
1061 # This is also a by-counter visit.
1062 # It will use the same visit_name and visit_id.
1063 visit_systems.add(system_seq_start_end)
1065 elif not skip_multi and exposure == first:
1066 # This is the first legitimate exposure in a multi-exposure
1067 # visit. It therefore needs a modified visit name and ID
1068 # so it does not clash with the multi-exposure visit
1069 # definition.
1070 visit_name = f"{visit_name}_first"
1071 visit_id = int(f"9{visit_id}")
1073 yield VisitDefinitionData(
1074 instrument=instrument,
1075 id=visit_id,
1076 name=visit_name,
1077 exposures=[exposure],
1078 visit_systems=visit_systems,
1079 )
1081 # Multi-exposure visit.
1082 if not skip_multi and multi_exposure:
1083 # Define the visit using the first exposure
1084 visit_name = first.obs_id
1085 visit_id = first.id
1087 yield VisitDefinitionData(
1088 instrument=instrument,
1089 id=visit_id,
1090 name=visit_name,
1091 exposures=exposures_in_group,
1092 visit_systems={system_seq_start_end},
1093 )
1095 def getVisitSystems(self) -> set[VisitSystem]:
1096 # Docstring inherited from GroupExposuresTask.
1097 # Using a Config for this is difficult because what this grouping
1098 # algorithm is doing is using two visit systems.
1099 # One is using metadata (but not by-group) and the other is the
1100 # one-to-one. For now hard-code in class.
1101 return set(VisitSystem.from_names(["one-to-one", "by-seq-start-end"]))
1104class _ComputeVisitRegionsFromSingleRawWcsConfig(ComputeVisitRegionsConfig):
1105 mergeExposures: Field[bool] = Field(
1106 doc=(
1107 "If True, merge per-detector regions over all exposures in a "
1108 "visit (via convex hull) instead of using the first exposure and "
1109 "assuming its regions are valid for all others."
1110 ),
1111 dtype=bool,
1112 default=False,
1113 )
1114 detectorId: Field[int | None] = Field(
1115 doc=(
1116 "Load the WCS for the detector with this ID. If None, use an "
1117 "arbitrary detector (the first found in a query of the data "
1118 "repository for each exposure (or all exposures, if "
1119 "mergeExposures is True)."
1120 ),
1121 dtype=int,
1122 optional=True,
1123 default=None,
1124 )
1125 requireVersionedCamera: Field[bool] = Field(
1126 doc=(
1127 "If True, raise LookupError if version camera geometry cannot be "
1128 "loaded for an exposure. If False, use the nominal camera from "
1129 "the Instrument class instead."
1130 ),
1131 dtype=bool,
1132 optional=False,
1133 default=False,
1134 )
1137@registerConfigurable("single-raw-wcs", ComputeVisitRegionsTask.registry)
1138class _ComputeVisitRegionsFromSingleRawWcsTask(ComputeVisitRegionsTask):
1139 """A visit region calculator that uses a single raw WCS and a camera to
1140 project the bounding boxes of all detectors onto the sky, relating
1141 different detectors by their positions in focal plane coordinates.
1143 Notes
1144 -----
1145 Most instruments should have their raw WCSs determined from a combination
1146 of boresight angle, rotator angle, and camera geometry, and hence this
1147 algorithm should produce stable results regardless of which detector the
1148 raw corresponds to. If this is not the case (e.g. because a per-file FITS
1149 WCS is used instead), either the ID of the detector should be fixed (see
1150 the ``detectorId`` config parameter) or a different algorithm used.
1151 """
1153 ConfigClass = _ComputeVisitRegionsFromSingleRawWcsConfig
1154 config: _ComputeVisitRegionsFromSingleRawWcsConfig
1156 def computeExposureBounds(
1157 self, exposure: DimensionRecord, *, collections: Any = None
1158 ) -> dict[int, list[UnitVector3d]]:
1159 """Compute the lists of unit vectors on the sphere that correspond to
1160 the sky positions of detector corners.
1162 Parameters
1163 ----------
1164 exposure : `DimensionRecord`
1165 Dimension record for the exposure.
1166 collections : Any, optional
1167 Collections to be searched for raws and camera geometry, overriding
1168 ``self.butler.collections``.
1169 Can be any of the types supported by the ``collections`` argument
1170 to butler construction.
1172 Returns
1173 -------
1174 bounds : `dict`
1175 Dictionary mapping detector ID to a list of unit vectors on the
1176 sphere representing that detector's corners projected onto the sky.
1177 """
1178 if collections is None:
1179 collections = self.butler.collections
1180 camera, versioned = loadCamera(self.butler, exposure.dataId, collections=collections)
1181 if not versioned and self.config.requireVersionedCamera:
1182 raise LookupError(f"No versioned camera found for exposure {exposure.dataId}.")
1184 # Derive WCS from boresight information -- if available in registry
1185 use_registry = True
1186 try:
1187 orientation = lsst.geom.Angle(exposure.sky_angle, lsst.geom.degrees)
1188 radec = lsst.geom.SpherePoint(
1189 lsst.geom.Angle(exposure.tracking_ra, lsst.geom.degrees),
1190 lsst.geom.Angle(exposure.tracking_dec, lsst.geom.degrees),
1191 )
1192 except AttributeError:
1193 use_registry = False
1195 if use_registry:
1196 if self.config.detectorId is None:
1197 detectorId = next(camera.getIdIter())
1198 else:
1199 detectorId = self.config.detectorId
1200 wcsDetector = camera[detectorId]
1202 # Ask the raw formatter to create the relevant WCS
1203 # This allows flips to be taken into account
1204 instrument = self.getInstrument(exposure.instrument)
1205 rawFormatter = instrument.getRawFormatter({"detector": detectorId})
1207 try:
1208 wcs = rawFormatter.makeRawSkyWcsFromBoresight(radec, orientation, wcsDetector) # type: ignore
1209 except AttributeError:
1210 raise TypeError(
1211 f"Raw formatter is {get_full_type_name(rawFormatter)} but visit"
1212 " definition requires it to support 'makeRawSkyWcsFromBoresight'"
1213 ) from None
1214 else:
1215 if self.config.detectorId is None:
1216 wcsRefsIter = self.butler.registry.queryDatasets(
1217 "raw.wcs", dataId=exposure.dataId, collections=collections
1218 )
1219 if not wcsRefsIter:
1220 raise LookupError(
1221 f"No raw.wcs datasets found for data ID {exposure.dataId} "
1222 f"in collections {collections}."
1223 )
1224 wcsRef = next(iter(wcsRefsIter))
1225 wcsDetector = camera[wcsRef.dataId["detector"]]
1226 wcs = self.butler.get(wcsRef)
1227 else:
1228 wcsDetector = camera[self.config.detectorId]
1229 wcs = self.butler.get(
1230 "raw.wcs",
1231 dataId=exposure.dataId,
1232 detector=self.config.detectorId,
1233 collections=collections,
1234 )
1235 fpToSky = wcsDetector.getTransform(FOCAL_PLANE, PIXELS).then(wcs.getTransform())
1236 bounds = {}
1237 for detector in camera:
1238 pixelsToSky = detector.getTransform(PIXELS, FOCAL_PLANE).then(fpToSky)
1239 pixCorners = Box2D(detector.getBBox().dilatedBy(self.config.padding)).getCorners()
1240 bounds[detector.getId()] = [
1241 skyCorner.getVector() for skyCorner in pixelsToSky.applyForward(pixCorners)
1242 ]
1243 return bounds
1245 def compute(
1246 self, visit: VisitDefinitionData, *, collections: Any = None
1247 ) -> tuple[Region, dict[int, Region]]:
1248 # Docstring inherited from ComputeVisitRegionsTask.
1249 if self.config.mergeExposures:
1250 detectorBounds: dict[int, list[UnitVector3d]] = defaultdict(list)
1251 for exposure in visit.exposures:
1252 exposureDetectorBounds = self.computeExposureBounds(exposure, collections=collections)
1253 for detectorId, bounds in exposureDetectorBounds.items():
1254 detectorBounds[detectorId].extend(bounds)
1255 else:
1256 detectorBounds = self.computeExposureBounds(visit.exposures[0], collections=collections)
1257 visitBounds = []
1258 detectorRegions = {}
1259 for detectorId, bounds in detectorBounds.items():
1260 detectorRegions[detectorId] = ConvexPolygon.convexHull(bounds)
1261 visitBounds.extend(bounds)
1262 return ConvexPolygon.convexHull(visitBounds), detectorRegions