Coverage for python/lsst/obs/base/defineVisits.py: 31%
342 statements
« prev ^ index » next coverage.py v6.4, created at 2022-06-02 03:54 -0700
« prev ^ index » next coverage.py v6.4, created at 2022-06-02 03:54 -0700
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = [
25 "DefineVisitsConfig",
26 "DefineVisitsTask",
27 "GroupExposuresConfig",
28 "GroupExposuresTask",
29 "VisitDefinitionData",
30 "VisitSystem",
31]
33import cmath
34import dataclasses
35import enum
36import math
37import operator
38from abc import ABCMeta, abstractmethod
39from collections import defaultdict
40from typing import Any, Callable, ClassVar, Dict, FrozenSet, Iterable, List, Optional, Set, Tuple, TypeVar
42import lsst.geom
43from lsst.afw.cameraGeom import FOCAL_PLANE, PIXELS
44from lsst.daf.butler import (
45 Butler,
46 DataCoordinate,
47 DataId,
48 DimensionGraph,
49 DimensionRecord,
50 Progress,
51 Timespan,
52)
53from lsst.geom import Box2D
54from lsst.pex.config import Config, Field, makeRegistry, registerConfigurable
55from lsst.pipe.base import Instrument, Task
56from lsst.sphgeom import ConvexPolygon, Region, UnitVector3d
57from lsst.utils.introspection import get_full_type_name
59from ._instrument import loadCamera
62class VisitSystem(enum.Enum):
63 """Enumeration used to label different visit systems."""
65 ONE_TO_ONE = 0
66 """Each exposure is assigned to its own visit."""
68 BY_GROUP_METADATA = 1
69 """Visit membership is defined by the value of the exposure.group_id."""
71 BY_SEQ_START_END = 2
72 """Visit membership is defined by the values of the ``exposure.day_obs``,
73 ``exposure.seq_start``, and ``exposure.seq_end`` values.
74 """
76 @classmethod
77 def all(cls) -> FrozenSet[VisitSystem]:
78 """Return a `frozenset` containing all members."""
79 return frozenset(cls.__members__.values())
81 @classmethod
82 def from_name(cls, external_name: str) -> VisitSystem:
83 """Construct the enumeration from given name."""
84 name = external_name.upper()
85 name = name.replace("-", "_")
86 try:
87 return cls.__members__[name]
88 except KeyError:
89 raise KeyError(f"Visit system named '{external_name}' not known.") from None
91 @classmethod
92 def from_names(cls, names: Optional[Iterable[str]]) -> FrozenSet[VisitSystem]:
93 """Return a `frozenset` of all the visit systems matching the supplied
94 names.
96 Parameters
97 ----------
98 names : iterable of `str`, or `None`
99 Names of visit systems. Case insensitive. If `None` or empty, all
100 the visit systems are returned.
102 Returns
103 -------
104 systems : `frozenset` of `VisitSystem`
105 The matching visit systems.
106 """
107 if not names:
108 return cls.all()
110 return frozenset({cls.from_name(name) for name in names})
112 def __str__(self) -> str:
113 name = self.name.lower()
114 name = name.replace("_", "-")
115 return name
118@dataclasses.dataclass
119class VisitDefinitionData:
120 """Struct representing a group of exposures that will be used to define a
121 visit.
122 """
124 instrument: str
125 """Name of the instrument this visit will be associated with.
126 """
128 id: int
129 """Integer ID of the visit.
131 This must be unique across all visit systems for the instrument.
132 """
134 name: str
135 """String name for the visit.
137 This must be unique across all visit systems for the instrument.
138 """
140 visit_systems: Set[VisitSystem]
141 """All the visit systems associated with this visit."""
143 exposures: List[DimensionRecord] = dataclasses.field(default_factory=list)
144 """Dimension records for the exposures that are part of this visit.
145 """
148@dataclasses.dataclass
149class _VisitRecords:
150 """Struct containing the dimension records associated with a visit."""
152 visit: DimensionRecord
153 """Record for the 'visit' dimension itself.
154 """
156 visit_definition: List[DimensionRecord]
157 """Records for 'visit_definition', which relates 'visit' to 'exposure'.
158 """
160 visit_detector_region: List[DimensionRecord]
161 """Records for 'visit_detector_region', which associates the combination
162 of a 'visit' and a 'detector' with a region on the sky.
163 """
165 visit_system_membership: List[DimensionRecord]
166 """Records relating visits to an associated visit system."""
169class GroupExposuresConfig(Config):
170 pass
173class GroupExposuresTask(Task, metaclass=ABCMeta):
174 """Abstract base class for the subtask of `DefineVisitsTask` that is
175 responsible for grouping exposures into visits.
177 Subclasses should be registered with `GroupExposuresTask.registry` to
178 enable use by `DefineVisitsTask`, and should generally correspond to a
179 particular 'visit_system' dimension value. They are also responsible for
180 defining visit IDs and names that are unique across all visit systems in
181 use by an instrument.
183 Parameters
184 ----------
185 config : `GroupExposuresConfig`
186 Configuration information.
187 **kwargs
188 Additional keyword arguments forwarded to the `Task` constructor.
189 """
191 def __init__(self, config: GroupExposuresConfig, **kwargs: Any):
192 Task.__init__(self, config=config, **kwargs)
194 ConfigClass = GroupExposuresConfig
196 _DefaultName = "groupExposures"
198 registry = makeRegistry(
199 doc="Registry of algorithms for grouping exposures into visits.",
200 configBaseType=GroupExposuresConfig,
201 )
203 @abstractmethod
204 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]:
205 """Group the given exposures into visits.
207 Parameters
208 ----------
209 exposures : `list` [ `DimensionRecord` ]
210 DimensionRecords (for the 'exposure' dimension) describing the
211 exposures to group.
213 Returns
214 -------
215 visits : `Iterable` [ `VisitDefinitionData` ]
216 Structs identifying the visits and the exposures associated with
217 them. This may be an iterator or a container.
218 """
219 raise NotImplementedError()
221 def getVisitSystems(self) -> Set[VisitSystem]:
222 """Return identifiers for the 'visit_system' dimension this
223 algorithm implements.
225 Returns
226 -------
227 visit_systems : `Set` [`VisitSystem`]
228 The visit systems used by this algorithm.
229 """
230 raise NotImplementedError()
233class ComputeVisitRegionsConfig(Config):
234 padding = Field(
235 dtype=int,
236 default=250,
237 doc=(
238 "Pad raw image bounding boxes with specified number of pixels "
239 "when calculating their (conservatively large) region on the "
240 "sky. Note that the config value for pixelMargin of the "
241 "reference object loaders in meas_algorithms should be <= "
242 "the value set here."
243 ),
244 )
247class ComputeVisitRegionsTask(Task, metaclass=ABCMeta):
248 """Abstract base class for the subtask of `DefineVisitsTask` that is
249 responsible for extracting spatial regions for visits and visit+detector
250 combinations.
252 Subclasses should be registered with `ComputeVisitRegionsTask.registry` to
253 enable use by `DefineVisitsTask`.
255 Parameters
256 ----------
257 config : `ComputeVisitRegionsConfig`
258 Configuration information.
259 butler : `lsst.daf.butler.Butler`
260 The butler to use.
261 **kwargs
262 Additional keyword arguments forwarded to the `Task` constructor.
263 """
265 def __init__(self, config: ComputeVisitRegionsConfig, *, butler: Butler, **kwargs: Any):
266 Task.__init__(self, config=config, **kwargs)
267 self.butler = butler
268 self.instrumentMap: Dict[str, Instrument] = {}
270 ConfigClass = ComputeVisitRegionsConfig
272 _DefaultName = "computeVisitRegions"
274 registry = makeRegistry(
275 doc=(
276 "Registry of algorithms for computing on-sky regions for visits "
277 "and visit+detector combinations."
278 ),
279 configBaseType=ComputeVisitRegionsConfig,
280 )
282 def getInstrument(self, instrumentName: str) -> Instrument:
283 """Retrieve an `~lsst.obs.base.Instrument` associated with this
284 instrument name.
286 Parameters
287 ----------
288 instrumentName : `str`
289 The name of the instrument.
291 Returns
292 -------
293 instrument : `~lsst.obs.base.Instrument`
294 The associated instrument object.
296 Notes
297 -----
298 The result is cached.
299 """
300 instrument = self.instrumentMap.get(instrumentName)
301 if instrument is None:
302 instrument = Instrument.fromName(instrumentName, self.butler.registry)
303 self.instrumentMap[instrumentName] = instrument
304 return instrument
306 @abstractmethod
307 def compute(
308 self, visit: VisitDefinitionData, *, collections: Any = None
309 ) -> Tuple[Region, Dict[int, Region]]:
310 """Compute regions for the given visit and all detectors in that visit.
312 Parameters
313 ----------
314 visit : `VisitDefinitionData`
315 Struct describing the visit and the exposures associated with it.
316 collections : Any, optional
317 Collections to be searched for raws and camera geometry, overriding
318 ``self.butler.collections``.
319 Can be any of the types supported by the ``collections`` argument
320 to butler construction.
322 Returns
323 -------
324 visitRegion : `lsst.sphgeom.Region`
325 Region for the full visit.
326 visitDetectorRegions : `dict` [ `int`, `lsst.sphgeom.Region` ]
327 Dictionary mapping detector ID to the region for that detector.
328 Should include all detectors in the visit.
329 """
330 raise NotImplementedError()
333class DefineVisitsConfig(Config):
334 groupExposures = GroupExposuresTask.registry.makeField(
335 doc="Algorithm for grouping exposures into visits.",
336 default="one-to-one-and-by-counter",
337 )
338 computeVisitRegions = ComputeVisitRegionsTask.registry.makeField(
339 doc="Algorithm from computing visit and visit+detector regions.",
340 default="single-raw-wcs",
341 )
342 ignoreNonScienceExposures = Field(
343 doc=(
344 "If True, silently ignore input exposures that do not have "
345 "observation_type=SCIENCE. If False, raise an exception if one "
346 "encountered."
347 ),
348 dtype=bool,
349 optional=False,
350 default=True,
351 )
354class DefineVisitsTask(Task):
355 """Driver Task for defining visits (and their spatial regions) in Gen3
356 Butler repositories.
358 Parameters
359 ----------
360 config : `DefineVisitsConfig`
361 Configuration for the task.
362 butler : `~lsst.daf.butler.Butler`
363 Writeable butler instance. Will be used to read `raw.wcs` and `camera`
364 datasets and insert/sync dimension data.
365 **kwargs
366 Additional keyword arguments are forwarded to the `lsst.pipe.base.Task`
367 constructor.
369 Notes
370 -----
371 Each instance of `DefineVisitsTask` reads from / writes to the same Butler.
372 Each invocation of `DefineVisitsTask.run` processes an independent group of
373 exposures into one or more new vists, all belonging to the same visit
374 system and instrument.
376 The actual work of grouping exposures and computing regions is delegated
377 to pluggable subtasks (`GroupExposuresTask` and `ComputeVisitRegionsTask`),
378 respectively. The defaults are to create one visit for every exposure,
379 and to use exactly one (arbitrary) detector-level raw dataset's WCS along
380 with camera geometry to compute regions for all detectors. Other
381 implementations can be created and configured for instruments for which
382 these choices are unsuitable (e.g. because visits and exposures are not
383 one-to-one, or because ``raw.wcs`` datasets for different detectors may not
384 be consistent with camera geomery).
386 It is not necessary in general to ingest all raws for an exposure before
387 defining a visit that includes the exposure; this depends entirely on the
388 `ComputeVisitRegionTask` subclass used. For the default configuration,
389 a single raw for each exposure is sufficient.
391 Defining the same visit the same way multiple times (e.g. via multiple
392 invocations of this task on the same exposures, with the same
393 configuration) is safe, but it may be inefficient, as most of the work must
394 be done before new visits can be compared to existing visits.
395 """
397 def __init__(self, config: DefineVisitsConfig, *, butler: Butler, **kwargs: Any):
398 config.validate() # Not a CmdlineTask nor PipelineTask, so have to validate the config here.
399 super().__init__(config, **kwargs)
400 self.butler = butler
401 self.universe = self.butler.registry.dimensions
402 self.progress = Progress("obs.base.DefineVisitsTask")
403 self.makeSubtask("groupExposures")
404 self.makeSubtask("computeVisitRegions", butler=self.butler)
406 def _reduce_kwargs(self) -> dict:
407 # Add extra parameters to pickle
408 return dict(**super()._reduce_kwargs(), butler=self.butler)
410 ConfigClass: ClassVar[Config] = DefineVisitsConfig
412 _DefaultName: ClassVar[str] = "defineVisits"
414 groupExposures: GroupExposuresTask
415 computeVisitRegions: ComputeVisitRegionsTask
417 def _buildVisitRecords(
418 self, definition: VisitDefinitionData, *, collections: Any = None
419 ) -> _VisitRecords:
420 """Build the DimensionRecords associated with a visit.
422 Parameters
423 ----------
424 definition : `VisitDefinitionData`
425 Struct with identifiers for the visit and records for its
426 constituent exposures.
427 collections : Any, optional
428 Collections to be searched for raws and camera geometry, overriding
429 ``self.butler.collections``.
430 Can be any of the types supported by the ``collections`` argument
431 to butler construction.
433 Results
434 -------
435 records : `_VisitRecords`
436 Struct containing DimensionRecords for the visit, including
437 associated dimension elements.
438 """
439 dimension = self.universe["visit"]
441 # Some registries support additional items.
442 supported = {meta.name for meta in dimension.metadata}
444 # Compute all regions.
445 visitRegion, visitDetectorRegions = self.computeVisitRegions.compute(
446 definition, collections=collections
447 )
448 # Aggregate other exposure quantities.
449 timespan = Timespan(
450 begin=_reduceOrNone(min, (e.timespan.begin for e in definition.exposures)),
451 end=_reduceOrNone(max, (e.timespan.end for e in definition.exposures)),
452 )
453 exposure_time = _reduceOrNone(operator.add, (e.exposure_time for e in definition.exposures))
454 physical_filter = _reduceOrNone(_value_if_equal, (e.physical_filter for e in definition.exposures))
455 target_name = _reduceOrNone(_value_if_equal, (e.target_name for e in definition.exposures))
456 science_program = _reduceOrNone(_value_if_equal, (e.science_program for e in definition.exposures))
458 # observing day for a visit is defined by the earliest observation
459 # of the visit
460 observing_day = _reduceOrNone(min, (e.day_obs for e in definition.exposures))
461 observation_reason = _reduceOrNone(
462 _value_if_equal, (e.observation_reason for e in definition.exposures)
463 )
464 if observation_reason is None:
465 # Be explicit about there being multiple reasons
466 # MyPy can't really handle DimensionRecord fields as
467 # DimensionRecord classes are dynamically defined; easiest to just
468 # shush it when it complains.
469 observation_reason = "various" # type: ignore
471 # Use the mean zenith angle as an approximation
472 zenith_angle = _reduceOrNone(operator.add, (e.zenith_angle for e in definition.exposures))
473 if zenith_angle is not None:
474 zenith_angle /= len(definition.exposures)
476 # New records that may not be supported.
477 extras: Dict[str, Any] = {}
478 if "seq_num" in supported:
479 extras["seq_num"] = _reduceOrNone(min, (e.seq_num for e in definition.exposures))
480 if "azimuth" in supported:
481 # Must take into account 0/360 problem.
482 extras["azimuth"] = _calc_mean_angle([e.azimuth for e in definition.exposures])
484 # visit_system handling changed. This is the logic for visit/exposure
485 # that has support for seq_start/seq_end.
486 if "seq_num" in supported:
487 # Map visit to exposure.
488 visit_definition = [
489 self.universe["visit_definition"].RecordClass(
490 instrument=definition.instrument,
491 visit=definition.id,
492 exposure=exposure.id,
493 )
494 for exposure in definition.exposures
495 ]
497 # Map visit to visit system.
498 visit_system_membership = []
499 for visit_system in self.groupExposures.getVisitSystems():
500 if visit_system in definition.visit_systems:
501 record = self.universe["visit_system_membership"].RecordClass(
502 instrument=definition.instrument,
503 visit=definition.id,
504 visit_system=visit_system.value,
505 )
506 visit_system_membership.append(record)
508 else:
509 # The old approach can only handle one visit system at a time.
510 # If we have been configured with multiple options, prefer the
511 # one-to-one.
512 visit_systems = self.groupExposures.getVisitSystems()
513 if len(visit_systems) > 1:
514 one_to_one = VisitSystem.from_name("one-to-one")
515 if one_to_one not in visit_systems:
516 raise ValueError(
517 f"Multiple visit systems specified ({visit_systems}) for use with old"
518 " dimension universe but unable to find one-to-one."
519 )
520 visit_system = one_to_one
521 else:
522 visit_system = visit_systems.pop()
524 extras["visit_system"] = visit_system.value
526 # The old visit_definition included visit system.
527 visit_definition = [
528 self.universe["visit_definition"].RecordClass(
529 instrument=definition.instrument,
530 visit=definition.id,
531 exposure=exposure.id,
532 visit_system=visit_system.value,
533 )
534 for exposure in definition.exposures
535 ]
537 # This concept does not exist in old schema.
538 visit_system_membership = []
540 # Construct the actual DimensionRecords.
541 return _VisitRecords(
542 visit=dimension.RecordClass(
543 instrument=definition.instrument,
544 id=definition.id,
545 name=definition.name,
546 physical_filter=physical_filter,
547 target_name=target_name,
548 science_program=science_program,
549 observation_reason=observation_reason,
550 day_obs=observing_day,
551 zenith_angle=zenith_angle,
552 exposure_time=exposure_time,
553 timespan=timespan,
554 region=visitRegion,
555 # TODO: no seeing value in exposure dimension records, so we
556 # can't set that here. But there are many other columns that
557 # both dimensions should probably have as well.
558 **extras,
559 ),
560 visit_definition=visit_definition,
561 visit_system_membership=visit_system_membership,
562 visit_detector_region=[
563 self.universe["visit_detector_region"].RecordClass(
564 instrument=definition.instrument,
565 visit=definition.id,
566 detector=detectorId,
567 region=detectorRegion,
568 )
569 for detectorId, detectorRegion in visitDetectorRegions.items()
570 ],
571 )
573 def run(
574 self,
575 dataIds: Iterable[DataId],
576 *,
577 collections: Optional[str] = None,
578 update_records: bool = False,
579 ) -> None:
580 """Add visit definitions to the registry for the given exposures.
582 Parameters
583 ----------
584 dataIds : `Iterable` [ `dict` or `DataCoordinate` ]
585 Exposure-level data IDs. These must all correspond to the same
586 instrument, and are expected to be on-sky science exposures.
587 collections : Any, optional
588 Collections to be searched for raws and camera geometry, overriding
589 ``self.butler.collections``.
590 Can be any of the types supported by the ``collections`` argument
591 to butler construction.
592 update_records : `bool`, optional
593 If `True` (`False` is default), update existing visit records that
594 conflict with the new ones instead of rejecting them (and when this
595 occurs, update visit_detector_region as well). THIS IS AN ADVANCED
596 OPTION THAT SHOULD ONLY BE USED TO FIX REGIONS AND/OR METADATA THAT
597 ARE KNOWN TO BE BAD, AND IT CANNOT BE USED TO REMOVE EXPOSURES OR
598 DETECTORS FROM A VISIT.
600 Raises
601 ------
602 lsst.daf.butler.registry.ConflictingDefinitionError
603 Raised if a visit ID conflict is detected and the existing visit
604 differs from the new one.
605 """
606 # Normalize, expand, and deduplicate data IDs.
607 self.log.info("Preprocessing data IDs.")
608 dimensions = DimensionGraph(self.universe, names=["exposure"])
609 data_id_set: Set[DataCoordinate] = {
610 self.butler.registry.expandDataId(d, graph=dimensions) for d in dataIds
611 }
612 if not data_id_set:
613 raise RuntimeError("No exposures given.")
614 # Extract exposure DimensionRecords, check that there's only one
615 # instrument in play, and check for non-science exposures.
616 exposures = []
617 instruments = set()
618 for dataId in data_id_set:
619 record = dataId.records["exposure"]
620 assert record is not None, "Guaranteed by expandDataIds call earlier."
621 if record.tracking_ra is None or record.tracking_dec is None or record.sky_angle is None:
622 if self.config.ignoreNonScienceExposures:
623 continue
624 else:
625 raise RuntimeError(
626 f"Input exposure {dataId} has observation_type "
627 f"{record.observation_type}, but is not on sky."
628 )
629 instruments.add(dataId["instrument"])
630 exposures.append(record)
631 if not exposures:
632 self.log.info("No on-sky exposures found after filtering.")
633 return
634 if len(instruments) > 1:
635 raise RuntimeError(
636 f"All data IDs passed to DefineVisitsTask.run must be "
637 f"from the same instrument; got {instruments}."
638 )
639 (instrument,) = instruments
640 # Ensure the visit_system our grouping algorithm uses is in the
641 # registry, if it wasn't already.
642 visitSystems = self.groupExposures.getVisitSystems()
643 for visitSystem in visitSystems:
644 self.log.info("Registering visit_system %d: %s.", visitSystem.value, visitSystem)
645 self.butler.registry.syncDimensionData(
646 "visit_system", {"instrument": instrument, "id": visitSystem.value, "name": str(visitSystem)}
647 )
648 # Group exposures into visits, delegating to subtask.
649 self.log.info("Grouping %d exposure(s) into visits.", len(exposures))
650 definitions = list(self.groupExposures.group(exposures))
651 # Iterate over visits, compute regions, and insert dimension data, one
652 # transaction per visit. If a visit already exists, we skip all other
653 # inserts.
654 self.log.info("Computing regions and other metadata for %d visit(s).", len(definitions))
655 for visitDefinition in self.progress.wrap(
656 definitions, total=len(definitions), desc="Computing regions and inserting visits"
657 ):
658 visitRecords = self._buildVisitRecords(visitDefinition, collections=collections)
659 with self.butler.registry.transaction():
660 inserted_or_updated = self.butler.registry.syncDimensionData(
661 "visit",
662 visitRecords.visit,
663 update=update_records,
664 )
665 if inserted_or_updated:
666 if inserted_or_updated is True:
667 # This is a new visit, not an update to an existing
668 # one, so insert visit definition.
669 # We don't allow visit definitions to change even when
670 # asked to update, because we'd have to delete the old
671 # visit_definitions first and also worry about what
672 # this does to datasets that already use the visit.
673 self.butler.registry.insertDimensionData(
674 "visit_definition", *visitRecords.visit_definition
675 )
676 if visitRecords.visit_system_membership:
677 self.butler.registry.insertDimensionData(
678 "visit_system_membership", *visitRecords.visit_system_membership
679 )
680 # [Re]Insert visit_detector_region records for both inserts
681 # and updates, because we do allow updating to affect the
682 # region calculations.
683 self.butler.registry.insertDimensionData(
684 "visit_detector_region", *visitRecords.visit_detector_region, replace=update_records
685 )
688_T = TypeVar("_T")
691def _reduceOrNone(func: Callable[[_T, _T], Optional[_T]], iterable: Iterable[Optional[_T]]) -> Optional[_T]:
692 """Apply a binary function to pairs of elements in an iterable until a
693 single value is returned, but return `None` if any element is `None` or
694 there are no elements.
695 """
696 r: Optional[_T] = None
697 for v in iterable:
698 if v is None:
699 return None
700 if r is None:
701 r = v
702 else:
703 r = func(r, v)
704 return r
707def _value_if_equal(a: _T, b: _T) -> Optional[_T]:
708 """Return either argument if they are equal, or `None` if they are not."""
709 return a if a == b else None
712def _calc_mean_angle(angles: List[float]) -> float:
713 """Calculate the mean angle, taking into account 0/360 wrapping.
715 Parameters
716 ----------
717 angles : `list` [`float`]
718 Angles to average together, in degrees.
720 Returns
721 -------
722 average : `float`
723 Average angle in degrees.
724 """
725 # Save on all the math if we only have one value.
726 if len(angles) == 1:
727 return angles[0]
729 # Convert polar coordinates of unit circle to complex values.
730 # Average the complex values.
731 # Convert back to a phase angle.
732 return math.degrees(cmath.phase(sum(cmath.rect(1.0, math.radians(d)) for d in angles) / len(angles)))
735class _GroupExposuresOneToOneConfig(GroupExposuresConfig):
736 visitSystemId = Field(
737 doc="Integer ID of the visit_system implemented by this grouping algorithm.",
738 dtype=int,
739 default=0,
740 deprecated="No longer used. Replaced by enum.",
741 )
742 visitSystemName = Field(
743 doc="String name of the visit_system implemented by this grouping algorithm.",
744 dtype=str,
745 default="one-to-one",
746 deprecated="No longer used. Replaced by enum.",
747 )
750@registerConfigurable("one-to-one", GroupExposuresTask.registry)
751class _GroupExposuresOneToOneTask(GroupExposuresTask, metaclass=ABCMeta):
752 """An exposure grouping algorithm that simply defines one visit for each
753 exposure, reusing the exposures identifiers for the visit.
754 """
756 ConfigClass = _GroupExposuresOneToOneConfig
758 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]:
759 # Docstring inherited from GroupExposuresTask.
760 visit_systems = {VisitSystem.from_name("one-to-one")}
761 for exposure in exposures:
762 yield VisitDefinitionData(
763 instrument=exposure.instrument,
764 id=exposure.id,
765 name=exposure.obs_id,
766 exposures=[exposure],
767 visit_systems=visit_systems,
768 )
770 def getVisitSystems(self) -> Set[VisitSystem]:
771 # Docstring inherited from GroupExposuresTask.
772 return set(VisitSystem.from_names(["one-to-one"]))
775class _GroupExposuresByGroupMetadataConfig(GroupExposuresConfig):
776 visitSystemId = Field(
777 doc="Integer ID of the visit_system implemented by this grouping algorithm.",
778 dtype=int,
779 default=1,
780 deprecated="No longer used. Replaced by enum.",
781 )
782 visitSystemName = Field(
783 doc="String name of the visit_system implemented by this grouping algorithm.",
784 dtype=str,
785 default="by-group-metadata",
786 deprecated="No longer used. Replaced by enum.",
787 )
790@registerConfigurable("by-group-metadata", GroupExposuresTask.registry)
791class _GroupExposuresByGroupMetadataTask(GroupExposuresTask, metaclass=ABCMeta):
792 """An exposure grouping algorithm that uses exposure.group_name and
793 exposure.group_id.
795 This algorithm _assumes_ exposure.group_id (generally populated from
796 `astro_metadata_translator.ObservationInfo.visit_id`) is not just unique,
797 but disjoint from all `ObservationInfo.exposure_id` values - if it isn't,
798 it will be impossible to ever use both this grouping algorithm and the
799 one-to-one algorithm for a particular camera in the same data repository.
800 """
802 ConfigClass = _GroupExposuresByGroupMetadataConfig
804 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]:
805 # Docstring inherited from GroupExposuresTask.
806 visit_systems = {VisitSystem.from_name("by-group-metadata")}
807 groups = defaultdict(list)
808 for exposure in exposures:
809 groups[exposure.group_name].append(exposure)
810 for visitName, exposuresInGroup in groups.items():
811 instrument = exposuresInGroup[0].instrument
812 visitId = exposuresInGroup[0].group_id
813 assert all(
814 e.group_id == visitId for e in exposuresInGroup
815 ), "Grouping by exposure.group_name does not yield consistent group IDs"
816 yield VisitDefinitionData(
817 instrument=instrument,
818 id=visitId,
819 name=visitName,
820 exposures=exposuresInGroup,
821 visit_systems=visit_systems,
822 )
824 def getVisitSystems(self) -> Set[VisitSystem]:
825 # Docstring inherited from GroupExposuresTask.
826 return set(VisitSystem.from_names(["by-group-metadata"]))
829class _GroupExposuresByCounterAndExposuresConfig(GroupExposuresConfig):
830 visitSystemId = Field(
831 doc="Integer ID of the visit_system implemented by this grouping algorithm.",
832 dtype=int,
833 default=2,
834 deprecated="No longer used. Replaced by enum.",
835 )
836 visitSystemName = Field(
837 doc="String name of the visit_system implemented by this grouping algorithm.",
838 dtype=str,
839 default="by-counter-and-exposures",
840 deprecated="No longer used. Replaced by enum.",
841 )
844@registerConfigurable("one-to-one-and-by-counter", GroupExposuresTask.registry)
845class _GroupExposuresByCounterAndExposuresTask(GroupExposuresTask, metaclass=ABCMeta):
846 """An exposure grouping algorithm that uses the sequence start and
847 sequence end metadata to create multi-exposure visits, but also
848 creates one-to-one visits.
850 This algorithm uses the exposure.seq_start and
851 exposure.seq_end fields to collect related snaps.
852 It also groups single exposures.
853 """
855 ConfigClass = _GroupExposuresByCounterAndExposuresConfig
857 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]:
858 # Docstring inherited from GroupExposuresTask.
859 system_one_to_one = VisitSystem.from_name("one-to-one")
860 system_seq_start_end = VisitSystem.from_name("by-seq-start-end")
862 groups = defaultdict(list)
863 for exposure in exposures:
864 groups[exposure.day_obs, exposure.seq_start, exposure.seq_end].append(exposure)
865 for visit_key, exposures_in_group in groups.items():
866 instrument = exposures_in_group[0].instrument
868 # It is possible that the first exposure in a visit has not
869 # been ingested. This can be determined and if that is the case
870 # we can not reliably define the multi-exposure visit.
871 skip_multi = False
872 sorted_exposures = sorted(exposures_in_group, key=lambda e: e.seq_num)
873 first = sorted_exposures.pop(0)
874 if first.seq_num != first.seq_start:
875 # Special case seq_num == 0 since that implies that the
876 # instrument has no counters and therefore no multi-exposure
877 # visits.
878 if first.seq_num == 0:
879 self.log.warning(
880 "First exposure for visit %s is not present. Skipping the multi-snap definition.",
881 visit_key,
882 )
883 skip_multi = True
885 # Define the one-to-one visits.
886 num_exposures = len(exposures_in_group)
887 for exposure in exposures_in_group:
888 # Default is to use the exposure ID and name unless
889 # this is the first exposure in a multi-exposure visit.
890 visit_name = exposure.obs_id
891 visit_id = exposure.id
892 visit_systems = {system_one_to_one}
894 if num_exposures == 1:
895 # This is also a by-counter visit.
896 # It will use the same visit_name and visit_id.
897 visit_systems.add(system_seq_start_end)
899 elif num_exposures > 1 and not skip_multi and exposure == first:
900 # This is the first legitimate exposure in a multi-exposure
901 # visit. It therefore needs a modified visit name and ID
902 # so it does not clash with the multi-exposure visit
903 # definition.
904 visit_name = f"{visit_name}_first"
905 visit_id = int(f"9{visit_id}")
907 yield VisitDefinitionData(
908 instrument=instrument,
909 id=visit_id,
910 name=visit_name,
911 exposures=[exposure],
912 visit_systems=visit_systems,
913 )
915 # Multi-exposure visit.
916 if not skip_multi and num_exposures > 1:
917 # Define the visit using the first exposure
918 visit_name = first.obs_id
919 visit_id = first.id
921 yield VisitDefinitionData(
922 instrument=instrument,
923 id=visit_id,
924 name=visit_name,
925 exposures=exposures_in_group,
926 visit_systems={system_seq_start_end},
927 )
929 def getVisitSystems(self) -> Set[VisitSystem]:
930 # Docstring inherited from GroupExposuresTask.
931 # Using a Config for this is difficult because what this grouping
932 # algorithm is doing is using two visit systems.
933 # One is using metadata (but not by-group) and the other is the
934 # one-to-one. For now hard-code in class.
935 return set(VisitSystem.from_names(["one-to-one", "by-seq-start-end"]))
938class _ComputeVisitRegionsFromSingleRawWcsConfig(ComputeVisitRegionsConfig):
939 mergeExposures = Field(
940 doc=(
941 "If True, merge per-detector regions over all exposures in a "
942 "visit (via convex hull) instead of using the first exposure and "
943 "assuming its regions are valid for all others."
944 ),
945 dtype=bool,
946 default=False,
947 )
948 detectorId = Field(
949 doc=(
950 "Load the WCS for the detector with this ID. If None, use an "
951 "arbitrary detector (the first found in a query of the data "
952 "repository for each exposure (or all exposures, if "
953 "mergeExposures is True)."
954 ),
955 dtype=int,
956 optional=True,
957 default=None,
958 )
959 requireVersionedCamera = Field(
960 doc=(
961 "If True, raise LookupError if version camera geometry cannot be "
962 "loaded for an exposure. If False, use the nominal camera from "
963 "the Instrument class instead."
964 ),
965 dtype=bool,
966 optional=False,
967 default=False,
968 )
971@registerConfigurable("single-raw-wcs", ComputeVisitRegionsTask.registry)
972class _ComputeVisitRegionsFromSingleRawWcsTask(ComputeVisitRegionsTask):
973 """A visit region calculator that uses a single raw WCS and a camera to
974 project the bounding boxes of all detectors onto the sky, relating
975 different detectors by their positions in focal plane coordinates.
977 Notes
978 -----
979 Most instruments should have their raw WCSs determined from a combination
980 of boresight angle, rotator angle, and camera geometry, and hence this
981 algorithm should produce stable results regardless of which detector the
982 raw corresponds to. If this is not the case (e.g. because a per-file FITS
983 WCS is used instead), either the ID of the detector should be fixed (see
984 the ``detectorId`` config parameter) or a different algorithm used.
985 """
987 ConfigClass = _ComputeVisitRegionsFromSingleRawWcsConfig
989 def computeExposureBounds(
990 self, exposure: DimensionRecord, *, collections: Any = None
991 ) -> Dict[int, List[UnitVector3d]]:
992 """Compute the lists of unit vectors on the sphere that correspond to
993 the sky positions of detector corners.
995 Parameters
996 ----------
997 exposure : `DimensionRecord`
998 Dimension record for the exposure.
999 collections : Any, optional
1000 Collections to be searched for raws and camera geometry, overriding
1001 ``self.butler.collections``.
1002 Can be any of the types supported by the ``collections`` argument
1003 to butler construction.
1005 Returns
1006 -------
1007 bounds : `dict`
1008 Dictionary mapping detector ID to a list of unit vectors on the
1009 sphere representing that detector's corners projected onto the sky.
1010 """
1011 if collections is None:
1012 collections = self.butler.collections
1013 camera, versioned = loadCamera(self.butler, exposure.dataId, collections=collections)
1014 if not versioned and self.config.requireVersionedCamera:
1015 raise LookupError(f"No versioned camera found for exposure {exposure.dataId}.")
1017 # Derive WCS from boresight information -- if available in registry
1018 use_registry = True
1019 try:
1020 orientation = lsst.geom.Angle(exposure.sky_angle, lsst.geom.degrees)
1021 radec = lsst.geom.SpherePoint(
1022 lsst.geom.Angle(exposure.tracking_ra, lsst.geom.degrees),
1023 lsst.geom.Angle(exposure.tracking_dec, lsst.geom.degrees),
1024 )
1025 except AttributeError:
1026 use_registry = False
1028 if use_registry:
1029 if self.config.detectorId is None:
1030 detectorId = next(camera.getIdIter())
1031 else:
1032 detectorId = self.config.detectorId
1033 wcsDetector = camera[detectorId]
1035 # Ask the raw formatter to create the relevant WCS
1036 # This allows flips to be taken into account
1037 instrument = self.getInstrument(exposure.instrument)
1038 rawFormatter = instrument.getRawFormatter({"detector": detectorId})
1040 try:
1041 wcs = rawFormatter.makeRawSkyWcsFromBoresight(radec, orientation, wcsDetector) # type: ignore
1042 except AttributeError:
1043 raise TypeError(
1044 f"Raw formatter is {get_full_type_name(rawFormatter)} but visit"
1045 " definition requires it to support 'makeRawSkyWcsFromBoresight'"
1046 ) from None
1047 else:
1048 if self.config.detectorId is None:
1049 wcsRefsIter = self.butler.registry.queryDatasets(
1050 "raw.wcs", dataId=exposure.dataId, collections=collections
1051 )
1052 if not wcsRefsIter:
1053 raise LookupError(
1054 f"No raw.wcs datasets found for data ID {exposure.dataId} "
1055 f"in collections {collections}."
1056 )
1057 wcsRef = next(iter(wcsRefsIter))
1058 wcsDetector = camera[wcsRef.dataId["detector"]]
1059 wcs = self.butler.getDirect(wcsRef)
1060 else:
1061 wcsDetector = camera[self.config.detectorId]
1062 wcs = self.butler.get(
1063 "raw.wcs",
1064 dataId=exposure.dataId,
1065 detector=self.config.detectorId,
1066 collections=collections,
1067 )
1068 fpToSky = wcsDetector.getTransform(FOCAL_PLANE, PIXELS).then(wcs.getTransform())
1069 bounds = {}
1070 for detector in camera:
1071 pixelsToSky = detector.getTransform(PIXELS, FOCAL_PLANE).then(fpToSky)
1072 pixCorners = Box2D(detector.getBBox().dilatedBy(self.config.padding)).getCorners()
1073 bounds[detector.getId()] = [
1074 skyCorner.getVector() for skyCorner in pixelsToSky.applyForward(pixCorners)
1075 ]
1076 return bounds
1078 def compute(
1079 self, visit: VisitDefinitionData, *, collections: Any = None
1080 ) -> Tuple[Region, Dict[int, Region]]:
1081 # Docstring inherited from ComputeVisitRegionsTask.
1082 if self.config.mergeExposures:
1083 detectorBounds: Dict[int, List[UnitVector3d]] = defaultdict(list)
1084 for exposure in visit.exposures:
1085 exposureDetectorBounds = self.computeExposureBounds(exposure, collections=collections)
1086 for detectorId, bounds in exposureDetectorBounds.items():
1087 detectorBounds[detectorId].extend(bounds)
1088 else:
1089 detectorBounds = self.computeExposureBounds(visit.exposures[0], collections=collections)
1090 visitBounds = []
1091 detectorRegions = {}
1092 for detectorId, bounds in detectorBounds.items():
1093 detectorRegions[detectorId] = ConvexPolygon.convexHull(bounds)
1094 visitBounds.extend(bounds)
1095 return ConvexPolygon.convexHull(visitBounds), detectorRegions