Coverage for python/lsst/obs/base/defineVisits.py: 31%
344 statements
« prev ^ index » next coverage.py v6.4.4, created at 2022-08-31 04:25 -0700
« prev ^ index » next coverage.py v6.4.4, created at 2022-08-31 04:25 -0700
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = [
25 "DefineVisitsConfig",
26 "DefineVisitsTask",
27 "GroupExposuresConfig",
28 "GroupExposuresTask",
29 "VisitDefinitionData",
30 "VisitSystem",
31]
33import cmath
34import dataclasses
35import enum
36import math
37import operator
38from abc import ABCMeta, abstractmethod
39from collections import defaultdict
40from typing import (
41 Any,
42 Callable,
43 ClassVar,
44 Dict,
45 FrozenSet,
46 Iterable,
47 List,
48 Optional,
49 Set,
50 Tuple,
51 Type,
52 TypeVar,
53)
55import lsst.geom
56from lsst.afw.cameraGeom import FOCAL_PLANE, PIXELS
57from lsst.daf.butler import (
58 Butler,
59 DataCoordinate,
60 DataId,
61 DimensionGraph,
62 DimensionRecord,
63 Progress,
64 Timespan,
65)
66from lsst.geom import Box2D
67from lsst.pex.config import Config, Field, makeRegistry, registerConfigurable
68from lsst.pipe.base import Instrument, Task
69from lsst.sphgeom import ConvexPolygon, Region, UnitVector3d
70from lsst.utils.introspection import get_full_type_name
72from ._instrument import loadCamera
75class VisitSystem(enum.Enum):
76 """Enumeration used to label different visit systems."""
78 ONE_TO_ONE = 0
79 """Each exposure is assigned to its own visit."""
81 BY_GROUP_METADATA = 1
82 """Visit membership is defined by the value of the exposure.group_id."""
84 BY_SEQ_START_END = 2
85 """Visit membership is defined by the values of the ``exposure.day_obs``,
86 ``exposure.seq_start``, and ``exposure.seq_end`` values.
87 """
89 @classmethod
90 def all(cls) -> FrozenSet[VisitSystem]:
91 """Return a `frozenset` containing all members."""
92 return frozenset(cls.__members__.values())
94 @classmethod
95 def from_name(cls, external_name: str) -> VisitSystem:
96 """Construct the enumeration from given name."""
97 name = external_name.upper()
98 name = name.replace("-", "_")
99 try:
100 return cls.__members__[name]
101 except KeyError:
102 raise KeyError(f"Visit system named '{external_name}' not known.") from None
104 @classmethod
105 def from_names(cls, names: Optional[Iterable[str]]) -> FrozenSet[VisitSystem]:
106 """Return a `frozenset` of all the visit systems matching the supplied
107 names.
109 Parameters
110 ----------
111 names : iterable of `str`, or `None`
112 Names of visit systems. Case insensitive. If `None` or empty, all
113 the visit systems are returned.
115 Returns
116 -------
117 systems : `frozenset` of `VisitSystem`
118 The matching visit systems.
119 """
120 if not names:
121 return cls.all()
123 return frozenset({cls.from_name(name) for name in names})
125 def __str__(self) -> str:
126 name = self.name.lower()
127 name = name.replace("_", "-")
128 return name
131@dataclasses.dataclass
132class VisitDefinitionData:
133 """Struct representing a group of exposures that will be used to define a
134 visit.
135 """
137 instrument: str
138 """Name of the instrument this visit will be associated with.
139 """
141 id: int
142 """Integer ID of the visit.
144 This must be unique across all visit systems for the instrument.
145 """
147 name: str
148 """String name for the visit.
150 This must be unique across all visit systems for the instrument.
151 """
153 visit_systems: Set[VisitSystem]
154 """All the visit systems associated with this visit."""
156 exposures: List[DimensionRecord] = dataclasses.field(default_factory=list)
157 """Dimension records for the exposures that are part of this visit.
158 """
161@dataclasses.dataclass
162class _VisitRecords:
163 """Struct containing the dimension records associated with a visit."""
165 visit: DimensionRecord
166 """Record for the 'visit' dimension itself.
167 """
169 visit_definition: List[DimensionRecord]
170 """Records for 'visit_definition', which relates 'visit' to 'exposure'.
171 """
173 visit_detector_region: List[DimensionRecord]
174 """Records for 'visit_detector_region', which associates the combination
175 of a 'visit' and a 'detector' with a region on the sky.
176 """
178 visit_system_membership: List[DimensionRecord]
179 """Records relating visits to an associated visit system."""
182class GroupExposuresConfig(Config):
183 pass
186class GroupExposuresTask(Task, metaclass=ABCMeta):
187 """Abstract base class for the subtask of `DefineVisitsTask` that is
188 responsible for grouping exposures into visits.
190 Subclasses should be registered with `GroupExposuresTask.registry` to
191 enable use by `DefineVisitsTask`, and should generally correspond to a
192 particular 'visit_system' dimension value. They are also responsible for
193 defining visit IDs and names that are unique across all visit systems in
194 use by an instrument.
196 Parameters
197 ----------
198 config : `GroupExposuresConfig`
199 Configuration information.
200 **kwargs
201 Additional keyword arguments forwarded to the `Task` constructor.
202 """
204 def __init__(self, config: GroupExposuresConfig, **kwargs: Any):
205 Task.__init__(self, config=config, **kwargs)
207 ConfigClass = GroupExposuresConfig
209 _DefaultName = "groupExposures"
211 registry = makeRegistry(
212 doc="Registry of algorithms for grouping exposures into visits.",
213 configBaseType=GroupExposuresConfig,
214 )
216 @abstractmethod
217 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]:
218 """Group the given exposures into visits.
220 Parameters
221 ----------
222 exposures : `list` [ `DimensionRecord` ]
223 DimensionRecords (for the 'exposure' dimension) describing the
224 exposures to group.
226 Returns
227 -------
228 visits : `Iterable` [ `VisitDefinitionData` ]
229 Structs identifying the visits and the exposures associated with
230 them. This may be an iterator or a container.
231 """
232 raise NotImplementedError()
234 def getVisitSystems(self) -> Set[VisitSystem]:
235 """Return identifiers for the 'visit_system' dimension this
236 algorithm implements.
238 Returns
239 -------
240 visit_systems : `Set` [`VisitSystem`]
241 The visit systems used by this algorithm.
242 """
243 raise NotImplementedError()
246class ComputeVisitRegionsConfig(Config):
247 padding: Field[int] = Field(
248 dtype=int,
249 default=250,
250 doc=(
251 "Pad raw image bounding boxes with specified number of pixels "
252 "when calculating their (conservatively large) region on the "
253 "sky. Note that the config value for pixelMargin of the "
254 "reference object loaders in meas_algorithms should be <= "
255 "the value set here."
256 ),
257 )
260class ComputeVisitRegionsTask(Task, metaclass=ABCMeta):
261 """Abstract base class for the subtask of `DefineVisitsTask` that is
262 responsible for extracting spatial regions for visits and visit+detector
263 combinations.
265 Subclasses should be registered with `ComputeVisitRegionsTask.registry` to
266 enable use by `DefineVisitsTask`.
268 Parameters
269 ----------
270 config : `ComputeVisitRegionsConfig`
271 Configuration information.
272 butler : `lsst.daf.butler.Butler`
273 The butler to use.
274 **kwargs
275 Additional keyword arguments forwarded to the `Task` constructor.
276 """
278 def __init__(self, config: ComputeVisitRegionsConfig, *, butler: Butler, **kwargs: Any):
279 Task.__init__(self, config=config, **kwargs)
280 self.butler = butler
281 self.instrumentMap: Dict[str, Instrument] = {}
283 ConfigClass = ComputeVisitRegionsConfig
285 _DefaultName = "computeVisitRegions"
287 registry = makeRegistry(
288 doc=(
289 "Registry of algorithms for computing on-sky regions for visits "
290 "and visit+detector combinations."
291 ),
292 configBaseType=ComputeVisitRegionsConfig,
293 )
295 def getInstrument(self, instrumentName: str) -> Instrument:
296 """Retrieve an `~lsst.obs.base.Instrument` associated with this
297 instrument name.
299 Parameters
300 ----------
301 instrumentName : `str`
302 The name of the instrument.
304 Returns
305 -------
306 instrument : `~lsst.obs.base.Instrument`
307 The associated instrument object.
309 Notes
310 -----
311 The result is cached.
312 """
313 instrument = self.instrumentMap.get(instrumentName)
314 if instrument is None:
315 instrument = Instrument.fromName(instrumentName, self.butler.registry)
316 self.instrumentMap[instrumentName] = instrument
317 return instrument
319 @abstractmethod
320 def compute(
321 self, visit: VisitDefinitionData, *, collections: Any = None
322 ) -> Tuple[Region, Dict[int, Region]]:
323 """Compute regions for the given visit and all detectors in that visit.
325 Parameters
326 ----------
327 visit : `VisitDefinitionData`
328 Struct describing the visit and the exposures associated with it.
329 collections : Any, optional
330 Collections to be searched for raws and camera geometry, overriding
331 ``self.butler.collections``.
332 Can be any of the types supported by the ``collections`` argument
333 to butler construction.
335 Returns
336 -------
337 visitRegion : `lsst.sphgeom.Region`
338 Region for the full visit.
339 visitDetectorRegions : `dict` [ `int`, `lsst.sphgeom.Region` ]
340 Dictionary mapping detector ID to the region for that detector.
341 Should include all detectors in the visit.
342 """
343 raise NotImplementedError()
346class DefineVisitsConfig(Config):
347 groupExposures = GroupExposuresTask.registry.makeField(
348 doc="Algorithm for grouping exposures into visits.",
349 default="one-to-one-and-by-counter",
350 )
351 computeVisitRegions = ComputeVisitRegionsTask.registry.makeField(
352 doc="Algorithm from computing visit and visit+detector regions.",
353 default="single-raw-wcs",
354 )
355 ignoreNonScienceExposures: Field[bool] = Field(
356 doc=(
357 "If True, silently ignore input exposures that do not have "
358 "observation_type=SCIENCE. If False, raise an exception if one "
359 "encountered."
360 ),
361 dtype=bool,
362 optional=False,
363 default=True,
364 )
367class DefineVisitsTask(Task):
368 """Driver Task for defining visits (and their spatial regions) in Gen3
369 Butler repositories.
371 Parameters
372 ----------
373 config : `DefineVisitsConfig`
374 Configuration for the task.
375 butler : `~lsst.daf.butler.Butler`
376 Writeable butler instance. Will be used to read `raw.wcs` and `camera`
377 datasets and insert/sync dimension data.
378 **kwargs
379 Additional keyword arguments are forwarded to the `lsst.pipe.base.Task`
380 constructor.
382 Notes
383 -----
384 Each instance of `DefineVisitsTask` reads from / writes to the same Butler.
385 Each invocation of `DefineVisitsTask.run` processes an independent group of
386 exposures into one or more new vists, all belonging to the same visit
387 system and instrument.
389 The actual work of grouping exposures and computing regions is delegated
390 to pluggable subtasks (`GroupExposuresTask` and `ComputeVisitRegionsTask`),
391 respectively. The defaults are to create one visit for every exposure,
392 and to use exactly one (arbitrary) detector-level raw dataset's WCS along
393 with camera geometry to compute regions for all detectors. Other
394 implementations can be created and configured for instruments for which
395 these choices are unsuitable (e.g. because visits and exposures are not
396 one-to-one, or because ``raw.wcs`` datasets for different detectors may not
397 be consistent with camera geomery).
399 It is not necessary in general to ingest all raws for an exposure before
400 defining a visit that includes the exposure; this depends entirely on the
401 `ComputeVisitRegionTask` subclass used. For the default configuration,
402 a single raw for each exposure is sufficient.
404 Defining the same visit the same way multiple times (e.g. via multiple
405 invocations of this task on the same exposures, with the same
406 configuration) is safe, but it may be inefficient, as most of the work must
407 be done before new visits can be compared to existing visits.
408 """
410 def __init__(self, config: DefineVisitsConfig, *, butler: Butler, **kwargs: Any):
411 config.validate() # Not a CmdlineTask nor PipelineTask, so have to validate the config here.
412 super().__init__(config, **kwargs)
413 self.butler = butler
414 self.universe = self.butler.registry.dimensions
415 self.progress = Progress("obs.base.DefineVisitsTask")
416 self.makeSubtask("groupExposures")
417 self.makeSubtask("computeVisitRegions", butler=self.butler)
419 def _reduce_kwargs(self) -> dict:
420 # Add extra parameters to pickle
421 return dict(**super()._reduce_kwargs(), butler=self.butler)
423 ConfigClass: ClassVar[Type[Config]] = DefineVisitsConfig
425 _DefaultName: ClassVar[str] = "defineVisits"
427 config: DefineVisitsConfig
428 groupExposures: GroupExposuresTask
429 computeVisitRegions: ComputeVisitRegionsTask
431 def _buildVisitRecords(
432 self, definition: VisitDefinitionData, *, collections: Any = None
433 ) -> _VisitRecords:
434 """Build the DimensionRecords associated with a visit.
436 Parameters
437 ----------
438 definition : `VisitDefinitionData`
439 Struct with identifiers for the visit and records for its
440 constituent exposures.
441 collections : Any, optional
442 Collections to be searched for raws and camera geometry, overriding
443 ``self.butler.collections``.
444 Can be any of the types supported by the ``collections`` argument
445 to butler construction.
447 Results
448 -------
449 records : `_VisitRecords`
450 Struct containing DimensionRecords for the visit, including
451 associated dimension elements.
452 """
453 dimension = self.universe["visit"]
455 # Some registries support additional items.
456 supported = {meta.name for meta in dimension.metadata}
458 # Compute all regions.
459 visitRegion, visitDetectorRegions = self.computeVisitRegions.compute(
460 definition, collections=collections
461 )
462 # Aggregate other exposure quantities.
463 timespan = Timespan(
464 begin=_reduceOrNone(min, (e.timespan.begin for e in definition.exposures)),
465 end=_reduceOrNone(max, (e.timespan.end for e in definition.exposures)),
466 )
467 exposure_time = _reduceOrNone(operator.add, (e.exposure_time for e in definition.exposures))
468 physical_filter = _reduceOrNone(_value_if_equal, (e.physical_filter for e in definition.exposures))
469 target_name = _reduceOrNone(_value_if_equal, (e.target_name for e in definition.exposures))
470 science_program = _reduceOrNone(_value_if_equal, (e.science_program for e in definition.exposures))
472 # observing day for a visit is defined by the earliest observation
473 # of the visit
474 observing_day = _reduceOrNone(min, (e.day_obs for e in definition.exposures))
475 observation_reason = _reduceOrNone(
476 _value_if_equal, (e.observation_reason for e in definition.exposures)
477 )
478 if observation_reason is None:
479 # Be explicit about there being multiple reasons
480 # MyPy can't really handle DimensionRecord fields as
481 # DimensionRecord classes are dynamically defined; easiest to just
482 # shush it when it complains.
483 observation_reason = "various" # type: ignore
485 # Use the mean zenith angle as an approximation
486 zenith_angle = _reduceOrNone(operator.add, (e.zenith_angle for e in definition.exposures))
487 if zenith_angle is not None:
488 zenith_angle /= len(definition.exposures)
490 # New records that may not be supported.
491 extras: Dict[str, Any] = {}
492 if "seq_num" in supported:
493 extras["seq_num"] = _reduceOrNone(min, (e.seq_num for e in definition.exposures))
494 if "azimuth" in supported:
495 # Must take into account 0/360 problem.
496 extras["azimuth"] = _calc_mean_angle([e.azimuth for e in definition.exposures])
498 # visit_system handling changed. This is the logic for visit/exposure
499 # that has support for seq_start/seq_end.
500 if "seq_num" in supported:
501 # Map visit to exposure.
502 visit_definition = [
503 self.universe["visit_definition"].RecordClass(
504 instrument=definition.instrument,
505 visit=definition.id,
506 exposure=exposure.id,
507 )
508 for exposure in definition.exposures
509 ]
511 # Map visit to visit system.
512 visit_system_membership = []
513 for visit_system in self.groupExposures.getVisitSystems():
514 if visit_system in definition.visit_systems:
515 record = self.universe["visit_system_membership"].RecordClass(
516 instrument=definition.instrument,
517 visit=definition.id,
518 visit_system=visit_system.value,
519 )
520 visit_system_membership.append(record)
522 else:
523 # The old approach can only handle one visit system at a time.
524 # If we have been configured with multiple options, prefer the
525 # one-to-one.
526 visit_systems = self.groupExposures.getVisitSystems()
527 if len(visit_systems) > 1:
528 one_to_one = VisitSystem.from_name("one-to-one")
529 if one_to_one not in visit_systems:
530 raise ValueError(
531 f"Multiple visit systems specified ({visit_systems}) for use with old"
532 " dimension universe but unable to find one-to-one."
533 )
534 visit_system = one_to_one
535 else:
536 visit_system = visit_systems.pop()
538 extras["visit_system"] = visit_system.value
540 # The old visit_definition included visit system.
541 visit_definition = [
542 self.universe["visit_definition"].RecordClass(
543 instrument=definition.instrument,
544 visit=definition.id,
545 exposure=exposure.id,
546 visit_system=visit_system.value,
547 )
548 for exposure in definition.exposures
549 ]
551 # This concept does not exist in old schema.
552 visit_system_membership = []
554 # Construct the actual DimensionRecords.
555 return _VisitRecords(
556 visit=dimension.RecordClass(
557 instrument=definition.instrument,
558 id=definition.id,
559 name=definition.name,
560 physical_filter=physical_filter,
561 target_name=target_name,
562 science_program=science_program,
563 observation_reason=observation_reason,
564 day_obs=observing_day,
565 zenith_angle=zenith_angle,
566 exposure_time=exposure_time,
567 timespan=timespan,
568 region=visitRegion,
569 # TODO: no seeing value in exposure dimension records, so we
570 # can't set that here. But there are many other columns that
571 # both dimensions should probably have as well.
572 **extras,
573 ),
574 visit_definition=visit_definition,
575 visit_system_membership=visit_system_membership,
576 visit_detector_region=[
577 self.universe["visit_detector_region"].RecordClass(
578 instrument=definition.instrument,
579 visit=definition.id,
580 detector=detectorId,
581 region=detectorRegion,
582 )
583 for detectorId, detectorRegion in visitDetectorRegions.items()
584 ],
585 )
587 def run(
588 self,
589 dataIds: Iterable[DataId],
590 *,
591 collections: Optional[str] = None,
592 update_records: bool = False,
593 ) -> None:
594 """Add visit definitions to the registry for the given exposures.
596 Parameters
597 ----------
598 dataIds : `Iterable` [ `dict` or `DataCoordinate` ]
599 Exposure-level data IDs. These must all correspond to the same
600 instrument, and are expected to be on-sky science exposures.
601 collections : Any, optional
602 Collections to be searched for raws and camera geometry, overriding
603 ``self.butler.collections``.
604 Can be any of the types supported by the ``collections`` argument
605 to butler construction.
606 update_records : `bool`, optional
607 If `True` (`False` is default), update existing visit records that
608 conflict with the new ones instead of rejecting them (and when this
609 occurs, update visit_detector_region as well). THIS IS AN ADVANCED
610 OPTION THAT SHOULD ONLY BE USED TO FIX REGIONS AND/OR METADATA THAT
611 ARE KNOWN TO BE BAD, AND IT CANNOT BE USED TO REMOVE EXPOSURES OR
612 DETECTORS FROM A VISIT.
614 Raises
615 ------
616 lsst.daf.butler.registry.ConflictingDefinitionError
617 Raised if a visit ID conflict is detected and the existing visit
618 differs from the new one.
619 """
620 # Normalize, expand, and deduplicate data IDs.
621 self.log.info("Preprocessing data IDs.")
622 dimensions = DimensionGraph(self.universe, names=["exposure"])
623 data_id_set: Set[DataCoordinate] = {
624 self.butler.registry.expandDataId(d, graph=dimensions) for d in dataIds
625 }
626 if not data_id_set:
627 raise RuntimeError("No exposures given.")
628 # Extract exposure DimensionRecords, check that there's only one
629 # instrument in play, and check for non-science exposures.
630 exposures = []
631 instruments = set()
632 for dataId in data_id_set:
633 record = dataId.records["exposure"]
634 assert record is not None, "Guaranteed by expandDataIds call earlier."
635 if record.tracking_ra is None or record.tracking_dec is None or record.sky_angle is None:
636 if self.config.ignoreNonScienceExposures:
637 continue
638 else:
639 raise RuntimeError(
640 f"Input exposure {dataId} has observation_type "
641 f"{record.observation_type}, but is not on sky."
642 )
643 instruments.add(dataId["instrument"])
644 exposures.append(record)
645 if not exposures:
646 self.log.info("No on-sky exposures found after filtering.")
647 return
648 if len(instruments) > 1:
649 raise RuntimeError(
650 f"All data IDs passed to DefineVisitsTask.run must be "
651 f"from the same instrument; got {instruments}."
652 )
653 (instrument,) = instruments
654 # Ensure the visit_system our grouping algorithm uses is in the
655 # registry, if it wasn't already.
656 visitSystems = self.groupExposures.getVisitSystems()
657 for visitSystem in visitSystems:
658 self.log.info("Registering visit_system %d: %s.", visitSystem.value, visitSystem)
659 self.butler.registry.syncDimensionData(
660 "visit_system", {"instrument": instrument, "id": visitSystem.value, "name": str(visitSystem)}
661 )
662 # Group exposures into visits, delegating to subtask.
663 self.log.info("Grouping %d exposure(s) into visits.", len(exposures))
664 definitions = list(self.groupExposures.group(exposures))
665 # Iterate over visits, compute regions, and insert dimension data, one
666 # transaction per visit. If a visit already exists, we skip all other
667 # inserts.
668 self.log.info("Computing regions and other metadata for %d visit(s).", len(definitions))
669 for visitDefinition in self.progress.wrap(
670 definitions, total=len(definitions), desc="Computing regions and inserting visits"
671 ):
672 visitRecords = self._buildVisitRecords(visitDefinition, collections=collections)
673 with self.butler.registry.transaction():
674 inserted_or_updated = self.butler.registry.syncDimensionData(
675 "visit",
676 visitRecords.visit,
677 update=update_records,
678 )
679 if inserted_or_updated:
680 if inserted_or_updated is True:
681 # This is a new visit, not an update to an existing
682 # one, so insert visit definition.
683 # We don't allow visit definitions to change even when
684 # asked to update, because we'd have to delete the old
685 # visit_definitions first and also worry about what
686 # this does to datasets that already use the visit.
687 self.butler.registry.insertDimensionData(
688 "visit_definition", *visitRecords.visit_definition
689 )
690 if visitRecords.visit_system_membership:
691 self.butler.registry.insertDimensionData(
692 "visit_system_membership", *visitRecords.visit_system_membership
693 )
694 # [Re]Insert visit_detector_region records for both inserts
695 # and updates, because we do allow updating to affect the
696 # region calculations.
697 self.butler.registry.insertDimensionData(
698 "visit_detector_region", *visitRecords.visit_detector_region, replace=update_records
699 )
702_T = TypeVar("_T")
705def _reduceOrNone(func: Callable[[_T, _T], Optional[_T]], iterable: Iterable[Optional[_T]]) -> Optional[_T]:
706 """Apply a binary function to pairs of elements in an iterable until a
707 single value is returned, but return `None` if any element is `None` or
708 there are no elements.
709 """
710 r: Optional[_T] = None
711 for v in iterable:
712 if v is None:
713 return None
714 if r is None:
715 r = v
716 else:
717 r = func(r, v)
718 return r
721def _value_if_equal(a: _T, b: _T) -> Optional[_T]:
722 """Return either argument if they are equal, or `None` if they are not."""
723 return a if a == b else None
726def _calc_mean_angle(angles: List[float]) -> float:
727 """Calculate the mean angle, taking into account 0/360 wrapping.
729 Parameters
730 ----------
731 angles : `list` [`float`]
732 Angles to average together, in degrees.
734 Returns
735 -------
736 average : `float`
737 Average angle in degrees.
738 """
739 # Save on all the math if we only have one value.
740 if len(angles) == 1:
741 return angles[0]
743 # Convert polar coordinates of unit circle to complex values.
744 # Average the complex values.
745 # Convert back to a phase angle.
746 return math.degrees(cmath.phase(sum(cmath.rect(1.0, math.radians(d)) for d in angles) / len(angles)))
749class _GroupExposuresOneToOneConfig(GroupExposuresConfig):
750 visitSystemId: Field[int] = Field(
751 doc="Integer ID of the visit_system implemented by this grouping algorithm.",
752 dtype=int,
753 default=0,
754 deprecated="No longer used. Replaced by enum.",
755 )
756 visitSystemName: Field[str] = Field(
757 doc="String name of the visit_system implemented by this grouping algorithm.",
758 dtype=str,
759 default="one-to-one",
760 deprecated="No longer used. Replaced by enum.",
761 )
764@registerConfigurable("one-to-one", GroupExposuresTask.registry)
765class _GroupExposuresOneToOneTask(GroupExposuresTask, metaclass=ABCMeta):
766 """An exposure grouping algorithm that simply defines one visit for each
767 exposure, reusing the exposures identifiers for the visit.
768 """
770 ConfigClass = _GroupExposuresOneToOneConfig
772 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]:
773 # Docstring inherited from GroupExposuresTask.
774 visit_systems = {VisitSystem.from_name("one-to-one")}
775 for exposure in exposures:
776 yield VisitDefinitionData(
777 instrument=exposure.instrument,
778 id=exposure.id,
779 name=exposure.obs_id,
780 exposures=[exposure],
781 visit_systems=visit_systems,
782 )
784 def getVisitSystems(self) -> Set[VisitSystem]:
785 # Docstring inherited from GroupExposuresTask.
786 return set(VisitSystem.from_names(["one-to-one"]))
789class _GroupExposuresByGroupMetadataConfig(GroupExposuresConfig):
790 visitSystemId: Field[int] = Field(
791 doc="Integer ID of the visit_system implemented by this grouping algorithm.",
792 dtype=int,
793 default=1,
794 deprecated="No longer used. Replaced by enum.",
795 )
796 visitSystemName: Field[str] = Field(
797 doc="String name of the visit_system implemented by this grouping algorithm.",
798 dtype=str,
799 default="by-group-metadata",
800 deprecated="No longer used. Replaced by enum.",
801 )
804@registerConfigurable("by-group-metadata", GroupExposuresTask.registry)
805class _GroupExposuresByGroupMetadataTask(GroupExposuresTask, metaclass=ABCMeta):
806 """An exposure grouping algorithm that uses exposure.group_name and
807 exposure.group_id.
809 This algorithm _assumes_ exposure.group_id (generally populated from
810 `astro_metadata_translator.ObservationInfo.visit_id`) is not just unique,
811 but disjoint from all `ObservationInfo.exposure_id` values - if it isn't,
812 it will be impossible to ever use both this grouping algorithm and the
813 one-to-one algorithm for a particular camera in the same data repository.
814 """
816 ConfigClass = _GroupExposuresByGroupMetadataConfig
818 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]:
819 # Docstring inherited from GroupExposuresTask.
820 visit_systems = {VisitSystem.from_name("by-group-metadata")}
821 groups = defaultdict(list)
822 for exposure in exposures:
823 groups[exposure.group_name].append(exposure)
824 for visitName, exposuresInGroup in groups.items():
825 instrument = exposuresInGroup[0].instrument
826 visitId = exposuresInGroup[0].group_id
827 assert all(
828 e.group_id == visitId for e in exposuresInGroup
829 ), "Grouping by exposure.group_name does not yield consistent group IDs"
830 yield VisitDefinitionData(
831 instrument=instrument,
832 id=visitId,
833 name=visitName,
834 exposures=exposuresInGroup,
835 visit_systems=visit_systems,
836 )
838 def getVisitSystems(self) -> Set[VisitSystem]:
839 # Docstring inherited from GroupExposuresTask.
840 return set(VisitSystem.from_names(["by-group-metadata"]))
843class _GroupExposuresByCounterAndExposuresConfig(GroupExposuresConfig):
844 visitSystemId: Field[int] = Field(
845 doc="Integer ID of the visit_system implemented by this grouping algorithm.",
846 dtype=int,
847 default=2,
848 deprecated="No longer used. Replaced by enum.",
849 )
850 visitSystemName: Field[str] = Field(
851 doc="String name of the visit_system implemented by this grouping algorithm.",
852 dtype=str,
853 default="by-counter-and-exposures",
854 deprecated="No longer used. Replaced by enum.",
855 )
858@registerConfigurable("one-to-one-and-by-counter", GroupExposuresTask.registry)
859class _GroupExposuresByCounterAndExposuresTask(GroupExposuresTask, metaclass=ABCMeta):
860 """An exposure grouping algorithm that uses the sequence start and
861 sequence end metadata to create multi-exposure visits, but also
862 creates one-to-one visits.
864 This algorithm uses the exposure.seq_start and
865 exposure.seq_end fields to collect related snaps.
866 It also groups single exposures.
867 """
869 ConfigClass = _GroupExposuresByCounterAndExposuresConfig
871 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]:
872 # Docstring inherited from GroupExposuresTask.
873 system_one_to_one = VisitSystem.from_name("one-to-one")
874 system_seq_start_end = VisitSystem.from_name("by-seq-start-end")
876 groups = defaultdict(list)
877 for exposure in exposures:
878 groups[exposure.day_obs, exposure.seq_start, exposure.seq_end].append(exposure)
879 for visit_key, exposures_in_group in groups.items():
880 instrument = exposures_in_group[0].instrument
882 # It is possible that the first exposure in a visit has not
883 # been ingested. This can be determined and if that is the case
884 # we can not reliably define the multi-exposure visit.
885 skip_multi = False
886 sorted_exposures = sorted(exposures_in_group, key=lambda e: e.seq_num)
887 first = sorted_exposures.pop(0)
888 if first.seq_num != first.seq_start:
889 # Special case seq_num == 0 since that implies that the
890 # instrument has no counters and therefore no multi-exposure
891 # visits.
892 if first.seq_num == 0:
893 self.log.warning(
894 "First exposure for visit %s is not present. Skipping the multi-snap definition.",
895 visit_key,
896 )
897 skip_multi = True
899 # Define the one-to-one visits.
900 num_exposures = len(exposures_in_group)
901 for exposure in exposures_in_group:
902 # Default is to use the exposure ID and name unless
903 # this is the first exposure in a multi-exposure visit.
904 visit_name = exposure.obs_id
905 visit_id = exposure.id
906 visit_systems = {system_one_to_one}
908 if num_exposures == 1:
909 # This is also a by-counter visit.
910 # It will use the same visit_name and visit_id.
911 visit_systems.add(system_seq_start_end)
913 elif num_exposures > 1 and not skip_multi and exposure == first:
914 # This is the first legitimate exposure in a multi-exposure
915 # visit. It therefore needs a modified visit name and ID
916 # so it does not clash with the multi-exposure visit
917 # definition.
918 visit_name = f"{visit_name}_first"
919 visit_id = int(f"9{visit_id}")
921 yield VisitDefinitionData(
922 instrument=instrument,
923 id=visit_id,
924 name=visit_name,
925 exposures=[exposure],
926 visit_systems=visit_systems,
927 )
929 # Multi-exposure visit.
930 if not skip_multi and num_exposures > 1:
931 # Define the visit using the first exposure
932 visit_name = first.obs_id
933 visit_id = first.id
935 yield VisitDefinitionData(
936 instrument=instrument,
937 id=visit_id,
938 name=visit_name,
939 exposures=exposures_in_group,
940 visit_systems={system_seq_start_end},
941 )
943 def getVisitSystems(self) -> Set[VisitSystem]:
944 # Docstring inherited from GroupExposuresTask.
945 # Using a Config for this is difficult because what this grouping
946 # algorithm is doing is using two visit systems.
947 # One is using metadata (but not by-group) and the other is the
948 # one-to-one. For now hard-code in class.
949 return set(VisitSystem.from_names(["one-to-one", "by-seq-start-end"]))
952class _ComputeVisitRegionsFromSingleRawWcsConfig(ComputeVisitRegionsConfig):
953 mergeExposures: Field[bool] = Field(
954 doc=(
955 "If True, merge per-detector regions over all exposures in a "
956 "visit (via convex hull) instead of using the first exposure and "
957 "assuming its regions are valid for all others."
958 ),
959 dtype=bool,
960 default=False,
961 )
962 detectorId: Field[Optional[int]] = Field(
963 doc=(
964 "Load the WCS for the detector with this ID. If None, use an "
965 "arbitrary detector (the first found in a query of the data "
966 "repository for each exposure (or all exposures, if "
967 "mergeExposures is True)."
968 ),
969 dtype=int,
970 optional=True,
971 default=None,
972 )
973 requireVersionedCamera: Field[bool] = Field(
974 doc=(
975 "If True, raise LookupError if version camera geometry cannot be "
976 "loaded for an exposure. If False, use the nominal camera from "
977 "the Instrument class instead."
978 ),
979 dtype=bool,
980 optional=False,
981 default=False,
982 )
985@registerConfigurable("single-raw-wcs", ComputeVisitRegionsTask.registry)
986class _ComputeVisitRegionsFromSingleRawWcsTask(ComputeVisitRegionsTask):
987 """A visit region calculator that uses a single raw WCS and a camera to
988 project the bounding boxes of all detectors onto the sky, relating
989 different detectors by their positions in focal plane coordinates.
991 Notes
992 -----
993 Most instruments should have their raw WCSs determined from a combination
994 of boresight angle, rotator angle, and camera geometry, and hence this
995 algorithm should produce stable results regardless of which detector the
996 raw corresponds to. If this is not the case (e.g. because a per-file FITS
997 WCS is used instead), either the ID of the detector should be fixed (see
998 the ``detectorId`` config parameter) or a different algorithm used.
999 """
1001 ConfigClass = _ComputeVisitRegionsFromSingleRawWcsConfig
1002 config: _ComputeVisitRegionsFromSingleRawWcsConfig
1004 def computeExposureBounds(
1005 self, exposure: DimensionRecord, *, collections: Any = None
1006 ) -> Dict[int, List[UnitVector3d]]:
1007 """Compute the lists of unit vectors on the sphere that correspond to
1008 the sky positions of detector corners.
1010 Parameters
1011 ----------
1012 exposure : `DimensionRecord`
1013 Dimension record for the exposure.
1014 collections : Any, optional
1015 Collections to be searched for raws and camera geometry, overriding
1016 ``self.butler.collections``.
1017 Can be any of the types supported by the ``collections`` argument
1018 to butler construction.
1020 Returns
1021 -------
1022 bounds : `dict`
1023 Dictionary mapping detector ID to a list of unit vectors on the
1024 sphere representing that detector's corners projected onto the sky.
1025 """
1026 if collections is None:
1027 collections = self.butler.collections
1028 camera, versioned = loadCamera(self.butler, exposure.dataId, collections=collections)
1029 if not versioned and self.config.requireVersionedCamera:
1030 raise LookupError(f"No versioned camera found for exposure {exposure.dataId}.")
1032 # Derive WCS from boresight information -- if available in registry
1033 use_registry = True
1034 try:
1035 orientation = lsst.geom.Angle(exposure.sky_angle, lsst.geom.degrees)
1036 radec = lsst.geom.SpherePoint(
1037 lsst.geom.Angle(exposure.tracking_ra, lsst.geom.degrees),
1038 lsst.geom.Angle(exposure.tracking_dec, lsst.geom.degrees),
1039 )
1040 except AttributeError:
1041 use_registry = False
1043 if use_registry:
1044 if self.config.detectorId is None:
1045 detectorId = next(camera.getIdIter())
1046 else:
1047 detectorId = self.config.detectorId
1048 wcsDetector = camera[detectorId]
1050 # Ask the raw formatter to create the relevant WCS
1051 # This allows flips to be taken into account
1052 instrument = self.getInstrument(exposure.instrument)
1053 rawFormatter = instrument.getRawFormatter({"detector": detectorId})
1055 try:
1056 wcs = rawFormatter.makeRawSkyWcsFromBoresight(radec, orientation, wcsDetector) # type: ignore
1057 except AttributeError:
1058 raise TypeError(
1059 f"Raw formatter is {get_full_type_name(rawFormatter)} but visit"
1060 " definition requires it to support 'makeRawSkyWcsFromBoresight'"
1061 ) from None
1062 else:
1063 if self.config.detectorId is None:
1064 wcsRefsIter = self.butler.registry.queryDatasets(
1065 "raw.wcs", dataId=exposure.dataId, collections=collections
1066 )
1067 if not wcsRefsIter:
1068 raise LookupError(
1069 f"No raw.wcs datasets found for data ID {exposure.dataId} "
1070 f"in collections {collections}."
1071 )
1072 wcsRef = next(iter(wcsRefsIter))
1073 wcsDetector = camera[wcsRef.dataId["detector"]]
1074 wcs = self.butler.getDirect(wcsRef)
1075 else:
1076 wcsDetector = camera[self.config.detectorId]
1077 wcs = self.butler.get(
1078 "raw.wcs",
1079 dataId=exposure.dataId,
1080 detector=self.config.detectorId,
1081 collections=collections,
1082 )
1083 fpToSky = wcsDetector.getTransform(FOCAL_PLANE, PIXELS).then(wcs.getTransform())
1084 bounds = {}
1085 for detector in camera:
1086 pixelsToSky = detector.getTransform(PIXELS, FOCAL_PLANE).then(fpToSky)
1087 pixCorners = Box2D(detector.getBBox().dilatedBy(self.config.padding)).getCorners()
1088 bounds[detector.getId()] = [
1089 skyCorner.getVector() for skyCorner in pixelsToSky.applyForward(pixCorners)
1090 ]
1091 return bounds
1093 def compute(
1094 self, visit: VisitDefinitionData, *, collections: Any = None
1095 ) -> Tuple[Region, Dict[int, Region]]:
1096 # Docstring inherited from ComputeVisitRegionsTask.
1097 if self.config.mergeExposures:
1098 detectorBounds: Dict[int, List[UnitVector3d]] = defaultdict(list)
1099 for exposure in visit.exposures:
1100 exposureDetectorBounds = self.computeExposureBounds(exposure, collections=collections)
1101 for detectorId, bounds in exposureDetectorBounds.items():
1102 detectorBounds[detectorId].extend(bounds)
1103 else:
1104 detectorBounds = self.computeExposureBounds(visit.exposures[0], collections=collections)
1105 visitBounds = []
1106 detectorRegions = {}
1107 for detectorId, bounds in detectorBounds.items():
1108 detectorRegions[detectorId] = ConvexPolygon.convexHull(bounds)
1109 visitBounds.extend(bounds)
1110 return ConvexPolygon.convexHull(visitBounds), detectorRegions