22 from __future__
import annotations
27 "GroupExposuresConfig",
29 "VisitDefinitionData",
32 from abc
import ABCMeta, abstractmethod
33 from collections
import defaultdict
36 from typing
import Any, Dict, Iterable, List, Optional, Tuple
37 from multiprocessing
import Pool
39 from lsst.daf.butler
import (
49 from lsst.geom
import Box2D
50 from lsst.pex.config
import Config, Field, makeRegistry, registerConfigurable
51 from lsst.afw.cameraGeom
import FOCAL_PLANE, PIXELS
52 from lsst.pipe.base
import Task
53 from lsst.sphgeom
import ConvexPolygon, Region, UnitVector3d
54 from ._instrument
import loadCamera, Instrument
57 @dataclasses.dataclass
59 """Struct representing a group of exposures that will be used to define a
64 """Name of the instrument this visit will be associated with.
68 """Integer ID of the visit.
70 This must be unique across all visit systems for the instrument.
74 """String name for the visit.
76 This must be unique across all visit systems for the instrument.
79 exposures: List[DimensionRecord] = dataclasses.field(default_factory=list)
80 """Dimension records for the exposures that are part of this visit.
84 @dataclasses.dataclass
86 """Struct containing the dimension records associated with a visit.
89 visit: DimensionRecord
90 """Record for the 'visit' dimension itself.
93 visit_definition: List[DimensionRecord]
94 """Records for 'visit_definition', which relates 'visit' to 'exposure'.
97 visit_detector_region: List[DimensionRecord]
98 """Records for 'visit_detector_region', which associates the combination
99 of a 'visit' and a 'detector' with a region on the sky.
107 class GroupExposuresTask(Task, metaclass=ABCMeta):
108 """Abstract base class for the subtask of `DefineVisitsTask` that is
109 responsible for grouping exposures into visits.
111 Subclasses should be registered with `GroupExposuresTask.registry` to
112 enable use by `DefineVisitsTask`, and should generally correspond to a
113 particular 'visit_system' dimension value. They are also responsible for
114 defining visit IDs and names that are unique across all visit systems in
115 use by an instrument.
119 config : `GroupExposuresConfig`
120 Configuration information.
122 Additional keyword arguments forwarded to the `Task` constructor.
124 def __init__(self, config: GroupExposuresConfig, **kwargs: Any):
125 Task.__init__(self, config=config, **kwargs)
127 ConfigClass = GroupExposuresConfig
129 _DefaultName =
"groupExposures"
131 registry = makeRegistry(
132 doc=
"Registry of algorithms for grouping exposures into visits.",
133 configBaseType=GroupExposuresConfig,
137 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]:
138 """Group the given exposures into visits.
142 exposures : `list` [ `DimensionRecord` ]
143 DimensionRecords (for the 'exposure' dimension) describing the
148 visits : `Iterable` [ `VisitDefinitionData` ]
149 Structs identifying the visits and the exposures associated with
150 them. This may be an iterator or a container.
152 raise NotImplementedError()
156 """Return identifiers for the 'visit_system' dimension this
157 algorithm implements.
162 Integer ID for the visit system (given an instrument).
164 Unique string identifier for the visit system (given an
167 raise NotImplementedError()
174 doc=(
"Pad raw image bounding boxes with specified number of pixels "
175 "when calculating their (conservatively large) region on the "
181 """Abstract base class for the subtask of `DefineVisitsTask` that is
182 responsible for extracting spatial regions for visits and visit+detector
185 Subclasses should be registered with `ComputeVisitRegionsTask.registry` to
186 enable use by `DefineVisitsTask`.
190 config : `ComputeVisitRegionsConfig`
191 Configuration information.
192 butler : `lsst.daf.butler.Butler`
195 Additional keyword arguments forwarded to the `Task` constructor.
197 def __init__(self, config: ComputeVisitRegionsConfig, *, butler: Butler, **kwargs: Any):
198 Task.__init__(self, config=config, **kwargs)
202 ConfigClass = ComputeVisitRegionsConfig
204 _DefaultName =
"computeVisitRegions"
206 registry = makeRegistry(
207 doc=(
"Registry of algorithms for computing on-sky regions for visits "
208 "and visit+detector combinations."),
209 configBaseType=ComputeVisitRegionsConfig,
213 """Retrieve an `~lsst.obs.base.Instrument` associated with this
218 instrumentName : `str`
219 The name of the instrument.
223 instrument : `~lsst.obs.base.Instrument`
224 The associated instrument object.
228 The result is cached.
231 if instrument
is None:
232 instrument = Instrument.fromName(instrumentName, self.
butler.registry)
237 def compute(self, visit: VisitDefinitionData, *, collections: Any =
None
238 ) -> Tuple[Region, Dict[int, Region]]:
239 """Compute regions for the given visit and all detectors in that visit.
243 visit : `VisitDefinitionData`
244 Struct describing the visit and the exposures associated with it.
245 collections : Any, optional
246 Collections to be searched for raws and camera geometry, overriding
247 ``self.butler.collections``.
248 Can be any of the types supported by the ``collections`` argument
249 to butler construction.
253 visitRegion : `lsst.sphgeom.Region`
254 Region for the full visit.
255 visitDetectorRegions : `dict` [ `int`, `lsst.sphgeom.Region` ]
256 Dictionary mapping detector ID to the region for that detector.
257 Should include all detectors in the visit.
259 raise NotImplementedError()
263 groupExposures = GroupExposuresTask.registry.makeField(
264 doc=
"Algorithm for grouping exposures into visits.",
265 default=
"one-to-one",
267 computeVisitRegions = ComputeVisitRegionsTask.registry.makeField(
268 doc=
"Algorithm from computing visit and visit+detector regions.",
269 default=
"single-raw-wcs",
271 ignoreNonScienceExposures = Field(
272 doc=(
"If True, silently ignore input exposures that do not have "
273 "observation_type=SCIENCE. If False, raise an exception if one "
282 """Driver Task for defining visits (and their spatial regions) in Gen3
287 config : `DefineVisitsConfig`
288 Configuration for the task.
289 butler : `~lsst.daf.butler.Butler`
290 Writeable butler instance. Will be used to read `raw.wcs` and `camera`
291 datasets and insert/sync dimension data.
293 Additional keyword arguments are forwarded to the `lsst.pipe.base.Task`
298 Each instance of `DefineVisitsTask` reads from / writes to the same Butler.
299 Each invocation of `DefineVisitsTask.run` processes an independent group of
300 exposures into one or more new vists, all belonging to the same visit
301 system and instrument.
303 The actual work of grouping exposures and computing regions is delegated
304 to pluggable subtasks (`GroupExposuresTask` and `ComputeVisitRegionsTask`),
305 respectively. The defaults are to create one visit for every exposure,
306 and to use exactly one (arbitrary) detector-level raw dataset's WCS along
307 with camera geometry to compute regions for all detectors. Other
308 implementations can be created and configured for instruments for which
309 these choices are unsuitable (e.g. because visits and exposures are not
310 one-to-one, or because ``raw.wcs`` datasets for different detectors may not
311 be consistent with camera geomery).
313 It is not necessary in general to ingest all raws for an exposure before
314 defining a visit that includes the exposure; this depends entirely on the
315 `ComputeVisitRegionTask` subclass used. For the default configuration,
316 a single raw for each exposure is sufficient.
318 Defining the same visit the same way multiple times (e.g. via multiple
319 invocations of this task on the same exposures, with the same
320 configuration) is safe, but it may be inefficient, as most of the work must
321 be done before new visits can be compared to existing visits.
323 def __init__(self, config: Optional[DefineVisitsConfig] =
None, *, butler: Butler, **kwargs: Any):
328 self.makeSubtask(
"groupExposures")
329 self.makeSubtask(
"computeVisitRegions", butler=self.
butler)
331 def _reduce_kwargs(self):
333 return dict(**super()._reduce_kwargs(), butler=self.
butler)
335 ConfigClass = DefineVisitsConfig
337 _DefaultName =
"defineVisits"
339 def _buildVisitRecords(self, definition: VisitDefinitionData, *,
340 collections: Any =
None) -> _VisitRecords:
341 """Build the DimensionRecords associated with a visit.
345 definition : `VisitDefinition`
346 Struct with identifiers for the visit and records for its
347 constituent exposures.
348 collections : Any, optional
349 Collections to be searched for raws and camera geometry, overriding
350 ``self.butler.collections``.
351 Can be any of the types supported by the ``collections`` argument
352 to butler construction.
356 records : `_VisitRecords`
357 Struct containing DimensionRecords for the visit, including
358 associated dimension elements.
361 visitRegion, visitDetectorRegions = self.computeVisitRegions.compute(definition,
362 collections=collections)
365 begin=_reduceOrNone(min, (e.timespan.begin
for e
in definition.exposures)),
366 end=_reduceOrNone(max, (e.timespan.end
for e
in definition.exposures)),
368 exposure_time = _reduceOrNone(sum, (e.exposure_time
for e
in definition.exposures))
369 physical_filter = _reduceOrNone(
lambda a, b: a
if a == b
else None,
370 (e.physical_filter
for e
in definition.exposures))
371 target_name = _reduceOrNone(
lambda a, b: a
if a == b
else None,
372 (e.target_name
for e
in definition.exposures))
373 science_program = _reduceOrNone(
lambda a, b: a
if a == b
else None,
374 (e.science_program
for e
in definition.exposures))
375 observation_reason = _reduceOrNone(
lambda a, b: a
if a == b
else None,
376 (e.observation_reason
for e
in definition.exposures))
377 if observation_reason
is None:
379 observation_reason =
"various"
382 zenith_angle = _reduceOrNone(sum, (e.zenith_angle
for e
in definition.exposures))
383 if zenith_angle
is not None:
384 zenith_angle /= len(definition.exposures)
388 visit=self.
universe[
"visit"].RecordClass(
389 instrument=definition.instrument,
391 name=definition.name,
392 physical_filter=physical_filter,
393 target_name=target_name,
394 science_program=science_program,
395 observation_reason=observation_reason,
396 zenith_angle=zenith_angle,
397 visit_system=self.groupExposures.getVisitSystem()[0],
398 exposure_time=exposure_time,
406 self.
universe[
"visit_definition"].RecordClass(
407 instrument=definition.instrument,
409 exposure=exposure.id,
410 visit_system=self.groupExposures.getVisitSystem()[0],
412 for exposure
in definition.exposures
414 visit_detector_region=[
415 self.
universe[
"visit_detector_region"].RecordClass(
416 instrument=definition.instrument,
419 region=detectorRegion,
421 for detectorId, detectorRegion
in visitDetectorRegions.items()
425 def _expandExposureId(self, dataId: DataId) -> DataCoordinate:
426 """Return the expanded version of an exposure ID.
428 A private method to allow ID expansion in a pool without resorting
433 dataId : `dict` or `DataCoordinate`
434 Exposure-level data ID.
438 expanded : `DataCoordinate`
439 A data ID that includes full metadata for all exposure dimensions.
441 dimensions = DimensionGraph(self.
universe, names=[
"exposure"])
442 return self.
butler.registry.expandDataId(dataId, graph=dimensions)
444 def _buildVisitRecordsSingle(self, args) -> _VisitRecords:
445 """Build the DimensionRecords associated with a visit and collection.
447 A wrapper for `_buildVisitRecords` to allow it to be run as part of
448 a pool without resorting to local callables.
452 args : `tuple` [`VisitDefinition`, any]
453 A tuple consisting of the ``definition`` and ``collections``
454 arguments to `_buildVisitRecords`, in that order.
458 records : `_VisitRecords`
459 Struct containing DimensionRecords for the visit, including
460 associated dimension elements.
464 def run(self, dataIds: Iterable[DataId], *,
465 pool: Optional[Pool] =
None,
467 collections: Optional[str] =
None):
468 """Add visit definitions to the registry for the given exposures.
472 dataIds : `Iterable` [ `dict` or `DataCoordinate` ]
473 Exposure-level data IDs. These must all correspond to the same
474 instrument, and are expected to be on-sky science exposures.
475 pool : `multiprocessing.Pool`, optional
476 If not `None`, a process pool with which to parallelize some
478 processes : `int`, optional
479 The number of processes to use. Ignored if ``pool`` is not `None`.
480 collections : Any, optional
481 Collections to be searched for raws and camera geometry, overriding
482 ``self.butler.collections``.
483 Can be any of the types supported by the ``collections`` argument
484 to butler construction.
488 lsst.daf.butler.registry.ConflictingDefinitionError
489 Raised if a visit ID conflict is detected and the existing visit
490 differs from the new one.
493 if pool
is None and processes > 1:
494 pool = Pool(processes)
495 mapFunc = map
if pool
is None else pool.imap_unordered
497 self.log.info(
"Preprocessing data IDs.")
500 raise RuntimeError(
"No exposures given.")
505 for dataId
in dataIds:
506 record = dataId.records[
"exposure"]
507 if record.observation_type !=
"science":
508 if self.config.ignoreNonScienceExposures:
511 raise RuntimeError(f
"Input exposure {dataId} has observation_type "
512 f
"{record.observation_type}, not 'science'.")
513 instruments.add(dataId[
"instrument"])
514 exposures.append(record)
516 self.log.info(
"No science exposures found after filtering.")
518 if len(instruments) > 1:
520 f
"All data IDs passed to DefineVisitsTask.run must be "
521 f
"from the same instrument; got {instruments}."
523 instrument, = instruments
526 visitSystemId, visitSystemName = self.groupExposures.getVisitSystem()
527 self.log.info(
"Registering visit_system %d: %s.", visitSystemId, visitSystemName)
528 self.
butler.registry.syncDimensionData(
530 {
"instrument": instrument,
"id": visitSystemId,
"name": visitSystemName}
533 self.log.info(
"Grouping %d exposure(s) into visits.", len(exposures))
534 definitions = list(self.groupExposures.group(exposures))
538 self.log.info(
"Computing regions and other metadata for %d visit(s).", len(definitions))
540 zip(definitions, itertools.repeat(collections)))
543 for visitRecords
in allRecords:
544 with self.
butler.registry.transaction():
545 if self.
butler.registry.syncDimensionData(
"visit", visitRecords.visit):
546 self.
butler.registry.insertDimensionData(
"visit_definition",
547 *visitRecords.visit_definition)
548 self.
butler.registry.insertDimensionData(
"visit_detector_region",
549 *visitRecords.visit_detector_region)
552 def _reduceOrNone(func, iterable):
553 """Apply a binary function to pairs of elements in an iterable until a
554 single value is returned, but return `None` if any element is `None` or
555 there are no elements.
569 visitSystemId = Field(
570 doc=(
"Integer ID of the visit_system implemented by this grouping "
575 visitSystemName = Field(
576 doc=(
"String name of the visit_system implemented by this grouping "
579 default=
"one-to-one",
583 @registerConfigurable(
"one-to-one", GroupExposuresTask.registry)
585 """An exposure grouping algorithm that simply defines one visit for each
586 exposure, reusing the exposures identifiers for the visit.
589 ConfigClass = _GroupExposuresOneToOneConfig
591 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]:
593 for exposure
in exposures:
595 instrument=exposure.instrument,
598 exposures=[exposure],
603 return (self.config.visitSystemId, self.config.visitSystemName)
607 visitSystemId = Field(
608 doc=(
"Integer ID of the visit_system implemented by this grouping "
613 visitSystemName = Field(
614 doc=(
"String name of the visit_system implemented by this grouping "
617 default=
"by-group-metadata",
621 @registerConfigurable(
"by-group-metadata", GroupExposuresTask.registry)
623 """An exposure grouping algorithm that uses exposure.group_name and
626 This algorithm _assumes_ exposure.group_id (generally populated from
627 `astro_metadata_translator.ObservationInfo.visit_id`) is not just unique,
628 but disjoint from all `ObservationInfo.exposure_id` values - if it isn't,
629 it will be impossible to ever use both this grouping algorithm and the
630 one-to-one algorithm for a particular camera in the same data repository.
633 ConfigClass = _GroupExposuresByGroupMetadataConfig
635 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]:
637 groups = defaultdict(list)
638 for exposure
in exposures:
639 groups[exposure.group_name].append(exposure)
640 for visitName, exposuresInGroup
in groups.items():
641 instrument = exposuresInGroup[0].instrument
642 visitId = exposuresInGroup[0].group_id
643 assert all(e.group_id == visitId
for e
in exposuresInGroup), \
644 "Grouping by exposure.group_name does not yield consistent group IDs"
646 exposures=exposuresInGroup)
650 return (self.config.visitSystemId, self.config.visitSystemName)
654 mergeExposures = Field(
655 doc=(
"If True, merge per-detector regions over all exposures in a "
656 "visit (via convex hull) instead of using the first exposure and "
657 "assuming its regions are valid for all others."),
662 doc=(
"Load the WCS for the detector with this ID. If None, use an "
663 "arbitrary detector (the first found in a query of the data "
664 "repository for each exposure (or all exposures, if "
665 "mergeExposures is True)."),
670 requireVersionedCamera = Field(
671 doc=(
"If True, raise LookupError if version camera geometry cannot be "
672 "loaded for an exposure. If False, use the nominal camera from "
673 "the Instrument class instead."),
680 @registerConfigurable(
"single-raw-wcs", ComputeVisitRegionsTask.registry)
682 """A visit region calculator that uses a single raw WCS and a camera to
683 project the bounding boxes of all detectors onto the sky, relating
684 different detectors by their positions in focal plane coordinates.
688 Most instruments should have their raw WCSs determined from a combination
689 of boresight angle, rotator angle, and camera geometry, and hence this
690 algorithm should produce stable results regardless of which detector the
691 raw corresponds to. If this is not the case (e.g. because a per-file FITS
692 WCS is used instead), either the ID of the detector should be fixed (see
693 the ``detectorId`` config parameter) or a different algorithm used.
696 ConfigClass = _ComputeVisitRegionsFromSingleRawWcsConfig
699 ) -> Dict[int, List[UnitVector3d]]:
700 """Compute the lists of unit vectors on the sphere that correspond to
701 the sky positions of detector corners.
705 exposure : `DimensionRecord`
706 Dimension record for the exposure.
707 collections : Any, optional
708 Collections to be searched for raws and camera geometry, overriding
709 ``self.butler.collections``.
710 Can be any of the types supported by the ``collections`` argument
711 to butler construction.
716 Dictionary mapping detector ID to a list of unit vectors on the
717 sphere representing that detector's corners projected onto the sky.
719 if collections
is None:
720 collections = self.
butler.collections
721 camera, versioned =
loadCamera(self.
butler, exposure.dataId, collections=collections)
722 if not versioned
and self.config.requireVersionedCamera:
723 raise LookupError(f
"No versioned camera found for exposure {exposure.dataId}.")
728 orientation = lsst.geom.Angle(exposure.sky_angle, lsst.geom.degrees)
729 radec = lsst.geom.SpherePoint(lsst.geom.Angle(exposure.tracking_ra, lsst.geom.degrees),
730 lsst.geom.Angle(exposure.tracking_dec, lsst.geom.degrees))
731 except AttributeError:
735 if self.config.detectorId
is None:
736 detectorId = next(camera.getIdIter())
738 detectorId = self.config.detectorId
739 wcsDetector = camera[detectorId]
744 rawFormatter = instrument.getRawFormatter({
"detector": detectorId})
745 wcs = rawFormatter.makeRawSkyWcsFromBoresight(radec, orientation, wcsDetector)
748 if self.config.detectorId
is None:
749 wcsRefsIter = self.
butler.registry.queryDatasets(
"raw.wcs", dataId=exposure.dataId,
750 collections=collections)
752 raise LookupError(f
"No raw.wcs datasets found for data ID {exposure.dataId} "
753 f
"in collections {collections}.")
754 wcsRef = next(iter(wcsRefsIter))
755 wcsDetector = camera[wcsRef.dataId[
"detector"]]
756 wcs = self.
butler.getDirect(wcsRef)
758 wcsDetector = camera[self.config.detectorId]
759 wcs = self.
butler.get(
"raw.wcs", dataId=exposure.dataId, detector=self.config.detectorId,
760 collections=collections)
761 fpToSky = wcsDetector.getTransform(FOCAL_PLANE, PIXELS).then(wcs.getTransform())
763 for detector
in camera:
764 pixelsToSky = detector.getTransform(PIXELS, FOCAL_PLANE).then(fpToSky)
765 pixCorners = Box2D(detector.getBBox().dilatedBy(self.config.padding)).getCorners()
766 bounds[detector.getId()] = [
767 skyCorner.getVector()
for skyCorner
in pixelsToSky.applyForward(pixCorners)
771 def compute(self, visit: VisitDefinitionData, *, collections: Any =
None
772 ) -> Tuple[Region, Dict[int, Region]]:
774 if self.config.mergeExposures:
775 detectorBounds = defaultdict(list)
776 for exposure
in visit.exposures:
778 for detectorId, bounds
in exposureDetectorBounds.items():
779 detectorBounds[detectorId].extend(bounds)
784 for detectorId, bounds
in detectorBounds.items():
785 detectorRegions[detectorId] = ConvexPolygon.convexHull(bounds)
786 visitBounds.extend(bounds)
787 return ConvexPolygon.convexHull(visitBounds), detectorRegions