Coverage for python/lsst/obs/base/defineVisits.py: 28%
344 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-11 02:55 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-11 02:55 -0800
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = [
25 "DefineVisitsConfig",
26 "DefineVisitsTask",
27 "GroupExposuresConfig",
28 "GroupExposuresTask",
29 "VisitDefinitionData",
30 "VisitSystem",
31]
33import cmath
34import dataclasses
35import enum
36import math
37import operator
38from abc import ABCMeta, abstractmethod
39from collections import defaultdict
40from typing import (
41 Any,
42 Callable,
43 ClassVar,
44 Dict,
45 FrozenSet,
46 Iterable,
47 List,
48 Optional,
49 Set,
50 Tuple,
51 Type,
52 TypeVar,
53)
55import lsst.geom
56from lsst.afw.cameraGeom import FOCAL_PLANE, PIXELS
57from lsst.daf.butler import (
58 Butler,
59 DataCoordinate,
60 DataId,
61 DimensionGraph,
62 DimensionRecord,
63 Progress,
64 Timespan,
65)
66from lsst.geom import Box2D
67from lsst.pex.config import Config, Field, makeRegistry, registerConfigurable
68from lsst.pipe.base import Instrument, Task
69from lsst.sphgeom import ConvexPolygon, Region, UnitVector3d
70from lsst.utils.introspection import get_full_type_name
72from ._instrument import loadCamera
75class VisitSystem(enum.Enum):
76 """Enumeration used to label different visit systems."""
78 ONE_TO_ONE = 0
79 """Each exposure is assigned to its own visit."""
81 BY_GROUP_METADATA = 1
82 """Visit membership is defined by the value of the exposure.group_id."""
84 BY_SEQ_START_END = 2
85 """Visit membership is defined by the values of the ``exposure.day_obs``,
86 ``exposure.seq_start``, and ``exposure.seq_end`` values.
87 """
89 @classmethod
90 def all(cls) -> FrozenSet[VisitSystem]:
91 """Return a `frozenset` containing all members."""
92 return frozenset(cls.__members__.values())
94 @classmethod
95 def from_name(cls, external_name: str) -> VisitSystem:
96 """Construct the enumeration from given name."""
97 name = external_name.upper()
98 name = name.replace("-", "_")
99 try:
100 return cls.__members__[name]
101 except KeyError:
102 raise KeyError(f"Visit system named '{external_name}' not known.") from None
104 @classmethod
105 def from_names(cls, names: Optional[Iterable[str]]) -> FrozenSet[VisitSystem]:
106 """Return a `frozenset` of all the visit systems matching the supplied
107 names.
109 Parameters
110 ----------
111 names : iterable of `str`, or `None`
112 Names of visit systems. Case insensitive. If `None` or empty, all
113 the visit systems are returned.
115 Returns
116 -------
117 systems : `frozenset` of `VisitSystem`
118 The matching visit systems.
119 """
120 if not names:
121 return cls.all()
123 return frozenset({cls.from_name(name) for name in names})
125 def __str__(self) -> str:
126 name = self.name.lower()
127 name = name.replace("_", "-")
128 return name
131@dataclasses.dataclass
132class VisitDefinitionData:
133 """Struct representing a group of exposures that will be used to define a
134 visit.
135 """
137 instrument: str
138 """Name of the instrument this visit will be associated with.
139 """
141 id: int
142 """Integer ID of the visit.
144 This must be unique across all visit systems for the instrument.
145 """
147 name: str
148 """String name for the visit.
150 This must be unique across all visit systems for the instrument.
151 """
153 visit_systems: Set[VisitSystem]
154 """All the visit systems associated with this visit."""
156 exposures: List[DimensionRecord] = dataclasses.field(default_factory=list)
157 """Dimension records for the exposures that are part of this visit.
158 """
161@dataclasses.dataclass
162class _VisitRecords:
163 """Struct containing the dimension records associated with a visit."""
165 visit: DimensionRecord
166 """Record for the 'visit' dimension itself.
167 """
169 visit_definition: List[DimensionRecord]
170 """Records for 'visit_definition', which relates 'visit' to 'exposure'.
171 """
173 visit_detector_region: List[DimensionRecord]
174 """Records for 'visit_detector_region', which associates the combination
175 of a 'visit' and a 'detector' with a region on the sky.
176 """
178 visit_system_membership: List[DimensionRecord]
179 """Records relating visits to an associated visit system."""
182class GroupExposuresConfig(Config):
183 pass
186class GroupExposuresTask(Task, metaclass=ABCMeta):
187 """Abstract base class for the subtask of `DefineVisitsTask` that is
188 responsible for grouping exposures into visits.
190 Subclasses should be registered with `GroupExposuresTask.registry` to
191 enable use by `DefineVisitsTask`, and should generally correspond to a
192 particular 'visit_system' dimension value. They are also responsible for
193 defining visit IDs and names that are unique across all visit systems in
194 use by an instrument.
196 Parameters
197 ----------
198 config : `GroupExposuresConfig`
199 Configuration information.
200 **kwargs
201 Additional keyword arguments forwarded to the `Task` constructor.
202 """
204 def __init__(self, config: GroupExposuresConfig, **kwargs: Any):
205 Task.__init__(self, config=config, **kwargs)
207 ConfigClass = GroupExposuresConfig
209 _DefaultName = "groupExposures"
211 registry = makeRegistry(
212 doc="Registry of algorithms for grouping exposures into visits.",
213 configBaseType=GroupExposuresConfig,
214 )
216 @abstractmethod
217 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]:
218 """Group the given exposures into visits.
220 Parameters
221 ----------
222 exposures : `list` [ `DimensionRecord` ]
223 DimensionRecords (for the 'exposure' dimension) describing the
224 exposures to group.
226 Returns
227 -------
228 visits : `Iterable` [ `VisitDefinitionData` ]
229 Structs identifying the visits and the exposures associated with
230 them. This may be an iterator or a container.
231 """
232 raise NotImplementedError()
234 def getVisitSystems(self) -> Set[VisitSystem]:
235 """Return identifiers for the 'visit_system' dimension this
236 algorithm implements.
238 Returns
239 -------
240 visit_systems : `Set` [`VisitSystem`]
241 The visit systems used by this algorithm.
242 """
243 raise NotImplementedError()
246class ComputeVisitRegionsConfig(Config):
247 padding: Field[int] = Field(
248 dtype=int,
249 default=250,
250 doc=(
251 "Pad raw image bounding boxes with specified number of pixels "
252 "when calculating their (conservatively large) region on the "
253 "sky. Note that the config value for pixelMargin of the "
254 "reference object loaders in meas_algorithms should be <= "
255 "the value set here."
256 ),
257 )
260class ComputeVisitRegionsTask(Task, metaclass=ABCMeta):
261 """Abstract base class for the subtask of `DefineVisitsTask` that is
262 responsible for extracting spatial regions for visits and visit+detector
263 combinations.
265 Subclasses should be registered with `ComputeVisitRegionsTask.registry` to
266 enable use by `DefineVisitsTask`.
268 Parameters
269 ----------
270 config : `ComputeVisitRegionsConfig`
271 Configuration information.
272 butler : `lsst.daf.butler.Butler`
273 The butler to use.
274 **kwargs
275 Additional keyword arguments forwarded to the `Task` constructor.
276 """
278 def __init__(self, config: ComputeVisitRegionsConfig, *, butler: Butler, **kwargs: Any):
279 Task.__init__(self, config=config, **kwargs)
280 self.butler = butler
281 self.instrumentMap: Dict[str, Instrument] = {}
283 ConfigClass = ComputeVisitRegionsConfig
285 _DefaultName = "computeVisitRegions"
287 registry = makeRegistry(
288 doc="Registry of algorithms for computing on-sky regions for visits and visit+detector combinations.",
289 configBaseType=ComputeVisitRegionsConfig,
290 )
292 def getInstrument(self, instrumentName: str) -> Instrument:
293 """Retrieve an `~lsst.obs.base.Instrument` associated with this
294 instrument name.
296 Parameters
297 ----------
298 instrumentName : `str`
299 The name of the instrument.
301 Returns
302 -------
303 instrument : `~lsst.obs.base.Instrument`
304 The associated instrument object.
306 Notes
307 -----
308 The result is cached.
309 """
310 instrument = self.instrumentMap.get(instrumentName)
311 if instrument is None:
312 instrument = Instrument.fromName(instrumentName, self.butler.registry)
313 self.instrumentMap[instrumentName] = instrument
314 return instrument
316 @abstractmethod
317 def compute(
318 self, visit: VisitDefinitionData, *, collections: Any = None
319 ) -> Tuple[Region, Dict[int, Region]]:
320 """Compute regions for the given visit and all detectors in that visit.
322 Parameters
323 ----------
324 visit : `VisitDefinitionData`
325 Struct describing the visit and the exposures associated with it.
326 collections : Any, optional
327 Collections to be searched for raws and camera geometry, overriding
328 ``self.butler.collections``.
329 Can be any of the types supported by the ``collections`` argument
330 to butler construction.
332 Returns
333 -------
334 visitRegion : `lsst.sphgeom.Region`
335 Region for the full visit.
336 visitDetectorRegions : `dict` [ `int`, `lsst.sphgeom.Region` ]
337 Dictionary mapping detector ID to the region for that detector.
338 Should include all detectors in the visit.
339 """
340 raise NotImplementedError()
343class DefineVisitsConfig(Config):
344 groupExposures = GroupExposuresTask.registry.makeField(
345 doc="Algorithm for grouping exposures into visits.",
346 default="one-to-one-and-by-counter",
347 )
348 computeVisitRegions = ComputeVisitRegionsTask.registry.makeField(
349 doc="Algorithm from computing visit and visit+detector regions.",
350 default="single-raw-wcs",
351 )
352 ignoreNonScienceExposures: Field[bool] = Field(
353 doc=(
354 "If True, silently ignore input exposures that do not have "
355 "observation_type=SCIENCE. If False, raise an exception if one "
356 "encountered."
357 ),
358 dtype=bool,
359 optional=False,
360 default=True,
361 )
364class DefineVisitsTask(Task):
365 """Driver Task for defining visits (and their spatial regions) in Gen3
366 Butler repositories.
368 Parameters
369 ----------
370 config : `DefineVisitsConfig`
371 Configuration for the task.
372 butler : `~lsst.daf.butler.Butler`
373 Writeable butler instance. Will be used to read `raw.wcs` and `camera`
374 datasets and insert/sync dimension data.
375 **kwargs
376 Additional keyword arguments are forwarded to the `lsst.pipe.base.Task`
377 constructor.
379 Notes
380 -----
381 Each instance of `DefineVisitsTask` reads from / writes to the same Butler.
382 Each invocation of `DefineVisitsTask.run` processes an independent group of
383 exposures into one or more new vists, all belonging to the same visit
384 system and instrument.
386 The actual work of grouping exposures and computing regions is delegated
387 to pluggable subtasks (`GroupExposuresTask` and `ComputeVisitRegionsTask`),
388 respectively. The defaults are to create one visit for every exposure,
389 and to use exactly one (arbitrary) detector-level raw dataset's WCS along
390 with camera geometry to compute regions for all detectors. Other
391 implementations can be created and configured for instruments for which
392 these choices are unsuitable (e.g. because visits and exposures are not
393 one-to-one, or because ``raw.wcs`` datasets for different detectors may not
394 be consistent with camera geomery).
396 It is not necessary in general to ingest all raws for an exposure before
397 defining a visit that includes the exposure; this depends entirely on the
398 `ComputeVisitRegionTask` subclass used. For the default configuration,
399 a single raw for each exposure is sufficient.
401 Defining the same visit the same way multiple times (e.g. via multiple
402 invocations of this task on the same exposures, with the same
403 configuration) is safe, but it may be inefficient, as most of the work must
404 be done before new visits can be compared to existing visits.
405 """
407 def __init__(self, config: DefineVisitsConfig, *, butler: Butler, **kwargs: Any):
408 config.validate() # Not a CmdlineTask nor PipelineTask, so have to validate the config here.
409 super().__init__(config, **kwargs)
410 self.butler = butler
411 self.universe = self.butler.registry.dimensions
412 self.progress = Progress("obs.base.DefineVisitsTask")
413 self.makeSubtask("groupExposures")
414 self.makeSubtask("computeVisitRegions", butler=self.butler)
416 def _reduce_kwargs(self) -> dict:
417 # Add extra parameters to pickle
418 return dict(**super()._reduce_kwargs(), butler=self.butler)
420 ConfigClass: ClassVar[Type[Config]] = DefineVisitsConfig
422 _DefaultName: ClassVar[str] = "defineVisits"
424 config: DefineVisitsConfig
425 groupExposures: GroupExposuresTask
426 computeVisitRegions: ComputeVisitRegionsTask
428 def _buildVisitRecords(
429 self, definition: VisitDefinitionData, *, collections: Any = None
430 ) -> _VisitRecords:
431 """Build the DimensionRecords associated with a visit.
433 Parameters
434 ----------
435 definition : `VisitDefinitionData`
436 Struct with identifiers for the visit and records for its
437 constituent exposures.
438 collections : Any, optional
439 Collections to be searched for raws and camera geometry, overriding
440 ``self.butler.collections``.
441 Can be any of the types supported by the ``collections`` argument
442 to butler construction.
444 Results
445 -------
446 records : `_VisitRecords`
447 Struct containing DimensionRecords for the visit, including
448 associated dimension elements.
449 """
450 dimension = self.universe["visit"]
452 # Some registries support additional items.
453 supported = {meta.name for meta in dimension.metadata}
455 # Compute all regions.
456 visitRegion, visitDetectorRegions = self.computeVisitRegions.compute(
457 definition, collections=collections
458 )
459 # Aggregate other exposure quantities.
460 timespan = Timespan(
461 begin=_reduceOrNone(min, (e.timespan.begin for e in definition.exposures)),
462 end=_reduceOrNone(max, (e.timespan.end for e in definition.exposures)),
463 )
464 exposure_time = _reduceOrNone(operator.add, (e.exposure_time for e in definition.exposures))
465 physical_filter = _reduceOrNone(_value_if_equal, (e.physical_filter for e in definition.exposures))
466 target_name = _reduceOrNone(_value_if_equal, (e.target_name for e in definition.exposures))
467 science_program = _reduceOrNone(_value_if_equal, (e.science_program for e in definition.exposures))
469 # observing day for a visit is defined by the earliest observation
470 # of the visit
471 observing_day = _reduceOrNone(min, (e.day_obs for e in definition.exposures))
472 observation_reason = _reduceOrNone(
473 _value_if_equal, (e.observation_reason for e in definition.exposures)
474 )
475 if observation_reason is None:
476 # Be explicit about there being multiple reasons
477 # MyPy can't really handle DimensionRecord fields as
478 # DimensionRecord classes are dynamically defined; easiest to just
479 # shush it when it complains.
480 observation_reason = "various" # type: ignore
482 # Use the mean zenith angle as an approximation
483 zenith_angle = _reduceOrNone(operator.add, (e.zenith_angle for e in definition.exposures))
484 if zenith_angle is not None:
485 zenith_angle /= len(definition.exposures)
487 # New records that may not be supported.
488 extras: Dict[str, Any] = {}
489 if "seq_num" in supported:
490 extras["seq_num"] = _reduceOrNone(min, (e.seq_num for e in definition.exposures))
491 if "azimuth" in supported:
492 # Must take into account 0/360 problem.
493 extras["azimuth"] = _calc_mean_angle([e.azimuth for e in definition.exposures])
495 # visit_system handling changed. This is the logic for visit/exposure
496 # that has support for seq_start/seq_end.
497 if "seq_num" in supported:
498 # Map visit to exposure.
499 visit_definition = [
500 self.universe["visit_definition"].RecordClass(
501 instrument=definition.instrument,
502 visit=definition.id,
503 exposure=exposure.id,
504 )
505 for exposure in definition.exposures
506 ]
508 # Map visit to visit system.
509 visit_system_membership = []
510 for visit_system in self.groupExposures.getVisitSystems():
511 if visit_system in definition.visit_systems:
512 record = self.universe["visit_system_membership"].RecordClass(
513 instrument=definition.instrument,
514 visit=definition.id,
515 visit_system=visit_system.value,
516 )
517 visit_system_membership.append(record)
519 else:
520 # The old approach can only handle one visit system at a time.
521 # If we have been configured with multiple options, prefer the
522 # one-to-one.
523 visit_systems = self.groupExposures.getVisitSystems()
524 if len(visit_systems) > 1:
525 one_to_one = VisitSystem.from_name("one-to-one")
526 if one_to_one not in visit_systems:
527 raise ValueError(
528 f"Multiple visit systems specified ({visit_systems}) for use with old"
529 " dimension universe but unable to find one-to-one."
530 )
531 visit_system = one_to_one
532 else:
533 visit_system = visit_systems.pop()
535 extras["visit_system"] = visit_system.value
537 # The old visit_definition included visit system.
538 visit_definition = [
539 self.universe["visit_definition"].RecordClass(
540 instrument=definition.instrument,
541 visit=definition.id,
542 exposure=exposure.id,
543 visit_system=visit_system.value,
544 )
545 for exposure in definition.exposures
546 ]
548 # This concept does not exist in old schema.
549 visit_system_membership = []
551 # Construct the actual DimensionRecords.
552 return _VisitRecords(
553 visit=dimension.RecordClass(
554 instrument=definition.instrument,
555 id=definition.id,
556 name=definition.name,
557 physical_filter=physical_filter,
558 target_name=target_name,
559 science_program=science_program,
560 observation_reason=observation_reason,
561 day_obs=observing_day,
562 zenith_angle=zenith_angle,
563 exposure_time=exposure_time,
564 timespan=timespan,
565 region=visitRegion,
566 # TODO: no seeing value in exposure dimension records, so we
567 # can't set that here. But there are many other columns that
568 # both dimensions should probably have as well.
569 **extras,
570 ),
571 visit_definition=visit_definition,
572 visit_system_membership=visit_system_membership,
573 visit_detector_region=[
574 self.universe["visit_detector_region"].RecordClass(
575 instrument=definition.instrument,
576 visit=definition.id,
577 detector=detectorId,
578 region=detectorRegion,
579 )
580 for detectorId, detectorRegion in visitDetectorRegions.items()
581 ],
582 )
584 def run(
585 self,
586 dataIds: Iterable[DataId],
587 *,
588 collections: Optional[str] = None,
589 update_records: bool = False,
590 ) -> None:
591 """Add visit definitions to the registry for the given exposures.
593 Parameters
594 ----------
595 dataIds : `Iterable` [ `dict` or `DataCoordinate` ]
596 Exposure-level data IDs. These must all correspond to the same
597 instrument, and are expected to be on-sky science exposures.
598 collections : Any, optional
599 Collections to be searched for raws and camera geometry, overriding
600 ``self.butler.collections``.
601 Can be any of the types supported by the ``collections`` argument
602 to butler construction.
603 update_records : `bool`, optional
604 If `True` (`False` is default), update existing visit records that
605 conflict with the new ones instead of rejecting them (and when this
606 occurs, update visit_detector_region as well). THIS IS AN ADVANCED
607 OPTION THAT SHOULD ONLY BE USED TO FIX REGIONS AND/OR METADATA THAT
608 ARE KNOWN TO BE BAD, AND IT CANNOT BE USED TO REMOVE EXPOSURES OR
609 DETECTORS FROM A VISIT.
611 Raises
612 ------
613 lsst.daf.butler.registry.ConflictingDefinitionError
614 Raised if a visit ID conflict is detected and the existing visit
615 differs from the new one.
616 """
617 # Normalize, expand, and deduplicate data IDs.
618 self.log.info("Preprocessing data IDs.")
619 dimensions = DimensionGraph(self.universe, names=["exposure"])
620 data_id_set: Set[DataCoordinate] = {
621 self.butler.registry.expandDataId(d, graph=dimensions) for d in dataIds
622 }
623 if not data_id_set:
624 raise RuntimeError("No exposures given.")
625 # Extract exposure DimensionRecords, check that there's only one
626 # instrument in play, and check for non-science exposures.
627 exposures = []
628 instruments = set()
629 for dataId in data_id_set:
630 record = dataId.records["exposure"]
631 assert record is not None, "Guaranteed by expandDataIds call earlier."
632 if record.tracking_ra is None or record.tracking_dec is None or record.sky_angle is None:
633 if self.config.ignoreNonScienceExposures:
634 continue
635 else:
636 raise RuntimeError(
637 f"Input exposure {dataId} has observation_type "
638 f"{record.observation_type}, but is not on sky."
639 )
640 instruments.add(dataId["instrument"])
641 exposures.append(record)
642 if not exposures:
643 self.log.info("No on-sky exposures found after filtering.")
644 return
645 if len(instruments) > 1:
646 raise RuntimeError(
647 "All data IDs passed to DefineVisitsTask.run must be "
648 f"from the same instrument; got {instruments}."
649 )
650 (instrument,) = instruments
651 # Ensure the visit_system our grouping algorithm uses is in the
652 # registry, if it wasn't already.
653 visitSystems = self.groupExposures.getVisitSystems()
654 for visitSystem in visitSystems:
655 self.log.info("Registering visit_system %d: %s.", visitSystem.value, visitSystem)
656 self.butler.registry.syncDimensionData(
657 "visit_system", {"instrument": instrument, "id": visitSystem.value, "name": str(visitSystem)}
658 )
659 # Group exposures into visits, delegating to subtask.
660 self.log.info("Grouping %d exposure(s) into visits.", len(exposures))
661 definitions = list(self.groupExposures.group(exposures))
662 # Iterate over visits, compute regions, and insert dimension data, one
663 # transaction per visit. If a visit already exists, we skip all other
664 # inserts.
665 self.log.info("Computing regions and other metadata for %d visit(s).", len(definitions))
666 for visitDefinition in self.progress.wrap(
667 definitions, total=len(definitions), desc="Computing regions and inserting visits"
668 ):
669 visitRecords = self._buildVisitRecords(visitDefinition, collections=collections)
670 with self.butler.registry.transaction():
671 inserted_or_updated = self.butler.registry.syncDimensionData(
672 "visit",
673 visitRecords.visit,
674 update=update_records,
675 )
676 if inserted_or_updated:
677 if inserted_or_updated is True:
678 # This is a new visit, not an update to an existing
679 # one, so insert visit definition.
680 # We don't allow visit definitions to change even when
681 # asked to update, because we'd have to delete the old
682 # visit_definitions first and also worry about what
683 # this does to datasets that already use the visit.
684 self.butler.registry.insertDimensionData(
685 "visit_definition", *visitRecords.visit_definition
686 )
687 if visitRecords.visit_system_membership:
688 self.butler.registry.insertDimensionData(
689 "visit_system_membership", *visitRecords.visit_system_membership
690 )
691 # [Re]Insert visit_detector_region records for both inserts
692 # and updates, because we do allow updating to affect the
693 # region calculations.
694 self.butler.registry.insertDimensionData(
695 "visit_detector_region", *visitRecords.visit_detector_region, replace=update_records
696 )
699_T = TypeVar("_T")
702def _reduceOrNone(func: Callable[[_T, _T], Optional[_T]], iterable: Iterable[Optional[_T]]) -> Optional[_T]:
703 """Apply a binary function to pairs of elements in an iterable until a
704 single value is returned, but return `None` if any element is `None` or
705 there are no elements.
706 """
707 r: Optional[_T] = None
708 for v in iterable:
709 if v is None:
710 return None
711 if r is None:
712 r = v
713 else:
714 r = func(r, v)
715 return r
718def _value_if_equal(a: _T, b: _T) -> Optional[_T]:
719 """Return either argument if they are equal, or `None` if they are not."""
720 return a if a == b else None
723def _calc_mean_angle(angles: List[float]) -> float:
724 """Calculate the mean angle, taking into account 0/360 wrapping.
726 Parameters
727 ----------
728 angles : `list` [`float`]
729 Angles to average together, in degrees.
731 Returns
732 -------
733 average : `float`
734 Average angle in degrees.
735 """
736 # Save on all the math if we only have one value.
737 if len(angles) == 1:
738 return angles[0]
740 # Convert polar coordinates of unit circle to complex values.
741 # Average the complex values.
742 # Convert back to a phase angle.
743 return math.degrees(cmath.phase(sum(cmath.rect(1.0, math.radians(d)) for d in angles) / len(angles)))
746class _GroupExposuresOneToOneConfig(GroupExposuresConfig):
747 visitSystemId: Field[int] = Field(
748 doc="Integer ID of the visit_system implemented by this grouping algorithm.",
749 dtype=int,
750 default=0,
751 deprecated="No longer used. Replaced by enum.",
752 )
753 visitSystemName: Field[str] = Field(
754 doc="String name of the visit_system implemented by this grouping algorithm.",
755 dtype=str,
756 default="one-to-one",
757 deprecated="No longer used. Replaced by enum.",
758 )
761@registerConfigurable("one-to-one", GroupExposuresTask.registry)
762class _GroupExposuresOneToOneTask(GroupExposuresTask, metaclass=ABCMeta):
763 """An exposure grouping algorithm that simply defines one visit for each
764 exposure, reusing the exposures identifiers for the visit.
765 """
767 ConfigClass = _GroupExposuresOneToOneConfig
769 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]:
770 # Docstring inherited from GroupExposuresTask.
771 visit_systems = {VisitSystem.from_name("one-to-one")}
772 for exposure in exposures:
773 yield VisitDefinitionData(
774 instrument=exposure.instrument,
775 id=exposure.id,
776 name=exposure.obs_id,
777 exposures=[exposure],
778 visit_systems=visit_systems,
779 )
781 def getVisitSystems(self) -> Set[VisitSystem]:
782 # Docstring inherited from GroupExposuresTask.
783 return set(VisitSystem.from_names(["one-to-one"]))
786class _GroupExposuresByGroupMetadataConfig(GroupExposuresConfig):
787 visitSystemId: Field[int] = Field(
788 doc="Integer ID of the visit_system implemented by this grouping algorithm.",
789 dtype=int,
790 default=1,
791 deprecated="No longer used. Replaced by enum.",
792 )
793 visitSystemName: Field[str] = Field(
794 doc="String name of the visit_system implemented by this grouping algorithm.",
795 dtype=str,
796 default="by-group-metadata",
797 deprecated="No longer used. Replaced by enum.",
798 )
801@registerConfigurable("by-group-metadata", GroupExposuresTask.registry)
802class _GroupExposuresByGroupMetadataTask(GroupExposuresTask, metaclass=ABCMeta):
803 """An exposure grouping algorithm that uses exposure.group_name and
804 exposure.group_id.
806 This algorithm _assumes_ exposure.group_id (generally populated from
807 `astro_metadata_translator.ObservationInfo.visit_id`) is not just unique,
808 but disjoint from all `ObservationInfo.exposure_id` values - if it isn't,
809 it will be impossible to ever use both this grouping algorithm and the
810 one-to-one algorithm for a particular camera in the same data repository.
811 """
813 ConfigClass = _GroupExposuresByGroupMetadataConfig
815 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]:
816 # Docstring inherited from GroupExposuresTask.
817 visit_systems = {VisitSystem.from_name("by-group-metadata")}
818 groups = defaultdict(list)
819 for exposure in exposures:
820 groups[exposure.group_name].append(exposure)
821 for visitName, exposuresInGroup in groups.items():
822 instrument = exposuresInGroup[0].instrument
823 visitId = exposuresInGroup[0].group_id
824 assert all(
825 e.group_id == visitId for e in exposuresInGroup
826 ), "Grouping by exposure.group_name does not yield consistent group IDs"
827 yield VisitDefinitionData(
828 instrument=instrument,
829 id=visitId,
830 name=visitName,
831 exposures=exposuresInGroup,
832 visit_systems=visit_systems,
833 )
835 def getVisitSystems(self) -> Set[VisitSystem]:
836 # Docstring inherited from GroupExposuresTask.
837 return set(VisitSystem.from_names(["by-group-metadata"]))
840class _GroupExposuresByCounterAndExposuresConfig(GroupExposuresConfig):
841 visitSystemId: Field[int] = Field(
842 doc="Integer ID of the visit_system implemented by this grouping algorithm.",
843 dtype=int,
844 default=2,
845 deprecated="No longer used. Replaced by enum.",
846 )
847 visitSystemName: Field[str] = Field(
848 doc="String name of the visit_system implemented by this grouping algorithm.",
849 dtype=str,
850 default="by-counter-and-exposures",
851 deprecated="No longer used. Replaced by enum.",
852 )
855@registerConfigurable("one-to-one-and-by-counter", GroupExposuresTask.registry)
856class _GroupExposuresByCounterAndExposuresTask(GroupExposuresTask, metaclass=ABCMeta):
857 """An exposure grouping algorithm that uses the sequence start and
858 sequence end metadata to create multi-exposure visits, but also
859 creates one-to-one visits.
861 This algorithm uses the exposure.seq_start and
862 exposure.seq_end fields to collect related snaps.
863 It also groups single exposures.
864 """
866 ConfigClass = _GroupExposuresByCounterAndExposuresConfig
868 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]:
869 # Docstring inherited from GroupExposuresTask.
870 system_one_to_one = VisitSystem.from_name("one-to-one")
871 system_seq_start_end = VisitSystem.from_name("by-seq-start-end")
873 groups = defaultdict(list)
874 for exposure in exposures:
875 groups[exposure.day_obs, exposure.seq_start, exposure.seq_end].append(exposure)
876 for visit_key, exposures_in_group in groups.items():
877 instrument = exposures_in_group[0].instrument
879 # It is possible that the first exposure in a visit has not
880 # been ingested. This can be determined and if that is the case
881 # we can not reliably define the multi-exposure visit.
882 skip_multi = False
883 sorted_exposures = sorted(exposures_in_group, key=lambda e: e.seq_num)
884 first = sorted_exposures.pop(0)
885 if first.seq_num != first.seq_start:
886 # Special case seq_num == 0 since that implies that the
887 # instrument has no counters and therefore no multi-exposure
888 # visits.
889 if first.seq_num != 0:
890 self.log.warning(
891 "First exposure for visit %s is not present. Skipping the multi-snap definition.",
892 visit_key,
893 )
894 skip_multi = True
896 # Define the one-to-one visits.
897 num_exposures = len(exposures_in_group)
898 for exposure in exposures_in_group:
899 # Default is to use the exposure ID and name unless
900 # this is the first exposure in a multi-exposure visit.
901 visit_name = exposure.obs_id
902 visit_id = exposure.id
903 visit_systems = {system_one_to_one}
905 if num_exposures == 1:
906 # This is also a by-counter visit.
907 # It will use the same visit_name and visit_id.
908 visit_systems.add(system_seq_start_end)
910 elif num_exposures > 1 and not skip_multi and exposure == first:
911 # This is the first legitimate exposure in a multi-exposure
912 # visit. It therefore needs a modified visit name and ID
913 # so it does not clash with the multi-exposure visit
914 # definition.
915 visit_name = f"{visit_name}_first"
916 visit_id = int(f"9{visit_id}")
918 yield VisitDefinitionData(
919 instrument=instrument,
920 id=visit_id,
921 name=visit_name,
922 exposures=[exposure],
923 visit_systems=visit_systems,
924 )
926 # Multi-exposure visit.
927 if not skip_multi and num_exposures > 1:
928 # Define the visit using the first exposure
929 visit_name = first.obs_id
930 visit_id = first.id
932 yield VisitDefinitionData(
933 instrument=instrument,
934 id=visit_id,
935 name=visit_name,
936 exposures=exposures_in_group,
937 visit_systems={system_seq_start_end},
938 )
940 def getVisitSystems(self) -> Set[VisitSystem]:
941 # Docstring inherited from GroupExposuresTask.
942 # Using a Config for this is difficult because what this grouping
943 # algorithm is doing is using two visit systems.
944 # One is using metadata (but not by-group) and the other is the
945 # one-to-one. For now hard-code in class.
946 return set(VisitSystem.from_names(["one-to-one", "by-seq-start-end"]))
949class _ComputeVisitRegionsFromSingleRawWcsConfig(ComputeVisitRegionsConfig):
950 mergeExposures: Field[bool] = Field(
951 doc=(
952 "If True, merge per-detector regions over all exposures in a "
953 "visit (via convex hull) instead of using the first exposure and "
954 "assuming its regions are valid for all others."
955 ),
956 dtype=bool,
957 default=False,
958 )
959 detectorId: Field[Optional[int]] = Field(
960 doc=(
961 "Load the WCS for the detector with this ID. If None, use an "
962 "arbitrary detector (the first found in a query of the data "
963 "repository for each exposure (or all exposures, if "
964 "mergeExposures is True)."
965 ),
966 dtype=int,
967 optional=True,
968 default=None,
969 )
970 requireVersionedCamera: Field[bool] = Field(
971 doc=(
972 "If True, raise LookupError if version camera geometry cannot be "
973 "loaded for an exposure. If False, use the nominal camera from "
974 "the Instrument class instead."
975 ),
976 dtype=bool,
977 optional=False,
978 default=False,
979 )
982@registerConfigurable("single-raw-wcs", ComputeVisitRegionsTask.registry)
983class _ComputeVisitRegionsFromSingleRawWcsTask(ComputeVisitRegionsTask):
984 """A visit region calculator that uses a single raw WCS and a camera to
985 project the bounding boxes of all detectors onto the sky, relating
986 different detectors by their positions in focal plane coordinates.
988 Notes
989 -----
990 Most instruments should have their raw WCSs determined from a combination
991 of boresight angle, rotator angle, and camera geometry, and hence this
992 algorithm should produce stable results regardless of which detector the
993 raw corresponds to. If this is not the case (e.g. because a per-file FITS
994 WCS is used instead), either the ID of the detector should be fixed (see
995 the ``detectorId`` config parameter) or a different algorithm used.
996 """
998 ConfigClass = _ComputeVisitRegionsFromSingleRawWcsConfig
999 config: _ComputeVisitRegionsFromSingleRawWcsConfig
1001 def computeExposureBounds(
1002 self, exposure: DimensionRecord, *, collections: Any = None
1003 ) -> Dict[int, List[UnitVector3d]]:
1004 """Compute the lists of unit vectors on the sphere that correspond to
1005 the sky positions of detector corners.
1007 Parameters
1008 ----------
1009 exposure : `DimensionRecord`
1010 Dimension record for the exposure.
1011 collections : Any, optional
1012 Collections to be searched for raws and camera geometry, overriding
1013 ``self.butler.collections``.
1014 Can be any of the types supported by the ``collections`` argument
1015 to butler construction.
1017 Returns
1018 -------
1019 bounds : `dict`
1020 Dictionary mapping detector ID to a list of unit vectors on the
1021 sphere representing that detector's corners projected onto the sky.
1022 """
1023 if collections is None:
1024 collections = self.butler.collections
1025 camera, versioned = loadCamera(self.butler, exposure.dataId, collections=collections)
1026 if not versioned and self.config.requireVersionedCamera:
1027 raise LookupError(f"No versioned camera found for exposure {exposure.dataId}.")
1029 # Derive WCS from boresight information -- if available in registry
1030 use_registry = True
1031 try:
1032 orientation = lsst.geom.Angle(exposure.sky_angle, lsst.geom.degrees)
1033 radec = lsst.geom.SpherePoint(
1034 lsst.geom.Angle(exposure.tracking_ra, lsst.geom.degrees),
1035 lsst.geom.Angle(exposure.tracking_dec, lsst.geom.degrees),
1036 )
1037 except AttributeError:
1038 use_registry = False
1040 if use_registry:
1041 if self.config.detectorId is None:
1042 detectorId = next(camera.getIdIter())
1043 else:
1044 detectorId = self.config.detectorId
1045 wcsDetector = camera[detectorId]
1047 # Ask the raw formatter to create the relevant WCS
1048 # This allows flips to be taken into account
1049 instrument = self.getInstrument(exposure.instrument)
1050 rawFormatter = instrument.getRawFormatter({"detector": detectorId})
1052 try:
1053 wcs = rawFormatter.makeRawSkyWcsFromBoresight(radec, orientation, wcsDetector) # type: ignore
1054 except AttributeError:
1055 raise TypeError(
1056 f"Raw formatter is {get_full_type_name(rawFormatter)} but visit"
1057 " definition requires it to support 'makeRawSkyWcsFromBoresight'"
1058 ) from None
1059 else:
1060 if self.config.detectorId is None:
1061 wcsRefsIter = self.butler.registry.queryDatasets(
1062 "raw.wcs", dataId=exposure.dataId, collections=collections
1063 )
1064 if not wcsRefsIter:
1065 raise LookupError(
1066 f"No raw.wcs datasets found for data ID {exposure.dataId} "
1067 f"in collections {collections}."
1068 )
1069 wcsRef = next(iter(wcsRefsIter))
1070 wcsDetector = camera[wcsRef.dataId["detector"]]
1071 wcs = self.butler.getDirect(wcsRef)
1072 else:
1073 wcsDetector = camera[self.config.detectorId]
1074 wcs = self.butler.get(
1075 "raw.wcs",
1076 dataId=exposure.dataId,
1077 detector=self.config.detectorId,
1078 collections=collections,
1079 )
1080 fpToSky = wcsDetector.getTransform(FOCAL_PLANE, PIXELS).then(wcs.getTransform())
1081 bounds = {}
1082 for detector in camera:
1083 pixelsToSky = detector.getTransform(PIXELS, FOCAL_PLANE).then(fpToSky)
1084 pixCorners = Box2D(detector.getBBox().dilatedBy(self.config.padding)).getCorners()
1085 bounds[detector.getId()] = [
1086 skyCorner.getVector() for skyCorner in pixelsToSky.applyForward(pixCorners)
1087 ]
1088 return bounds
1090 def compute(
1091 self, visit: VisitDefinitionData, *, collections: Any = None
1092 ) -> Tuple[Region, Dict[int, Region]]:
1093 # Docstring inherited from ComputeVisitRegionsTask.
1094 if self.config.mergeExposures:
1095 detectorBounds: Dict[int, List[UnitVector3d]] = defaultdict(list)
1096 for exposure in visit.exposures:
1097 exposureDetectorBounds = self.computeExposureBounds(exposure, collections=collections)
1098 for detectorId, bounds in exposureDetectorBounds.items():
1099 detectorBounds[detectorId].extend(bounds)
1100 else:
1101 detectorBounds = self.computeExposureBounds(visit.exposures[0], collections=collections)
1102 visitBounds = []
1103 detectorRegions = {}
1104 for detectorId, bounds in detectorBounds.items():
1105 detectorRegions[detectorId] = ConvexPolygon.convexHull(bounds)
1106 visitBounds.extend(bounds)
1107 return ConvexPolygon.convexHull(visitBounds), detectorRegions