22 from __future__
import annotations
27 "GroupExposuresConfig",
29 "VisitDefinitionData",
32 from abc
import ABCMeta, abstractmethod
33 from collections
import defaultdict
35 from typing
import Any, Dict, Iterable, List, Optional, Tuple
36 from multiprocessing
import Pool
38 from lsst.daf.butler
import (
47 from lsst.geom
import Box2D
48 from lsst.pex.config
import Config, Field, makeRegistry, registerConfigurable
49 from lsst.afw.cameraGeom
import FOCAL_PLANE, PIXELS
50 from lsst.pipe.base
import Task
51 from lsst.sphgeom
import ConvexPolygon, Region, UnitVector3d
52 from ._instrument
import loadCamera, Instrument
55 @dataclasses.dataclass
57 """Struct representing a group of exposures that will be used to define a
62 """Name of the instrument this visit will be associated with.
66 """Integer ID of the visit.
68 This must be unique across all visit systems for the instrument.
72 """String name for the visit.
74 This must be unique across all visit systems for the instrument.
77 exposures: List[DimensionRecord] = dataclasses.field(default_factory=list)
78 """Dimension records for the exposures that are part of this visit.
82 @dataclasses.dataclass
84 """Struct containing the dimension records associated with a visit.
87 visit: DimensionRecord
88 """Record for the 'visit' dimension itself.
91 visit_definition: List[DimensionRecord]
92 """Records for 'visit_definition', which relates 'visit' to 'exposure'.
95 visit_detector_region: List[DimensionRecord]
96 """Records for 'visit_detector_region', which associates the combination
97 of a 'visit' and a 'detector' with a region on the sky.
105 class GroupExposuresTask(Task, metaclass=ABCMeta):
106 """Abstract base class for the subtask of `DefineVisitsTask` that is
107 responsible for grouping exposures into visits.
109 Subclasses should be registered with `GroupExposuresTask.registry` to
110 enable use by `DefineVisitsTask`, and should generally correspond to a
111 particular 'visit_system' dimension value. They are also responsible for
112 defining visit IDs and names that are unique across all visit systems in
113 use by an instrument.
117 config : `GroupExposuresConfig`
118 Configuration information.
120 Additional keyword arguments forwarded to the `Task` constructor.
122 def __init__(self, config: GroupExposuresConfig, **kwargs: Any):
123 Task.__init__(self, config=config, **kwargs)
125 ConfigClass = GroupExposuresConfig
127 _DefaultName =
"groupExposures"
129 registry = makeRegistry(
130 doc=
"Registry of algorithms for grouping exposures into visits.",
131 configBaseType=GroupExposuresConfig,
135 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]:
136 """Group the given exposures into visits.
140 exposures : `list` [ `DimensionRecord` ]
141 DimensionRecords (for the 'exposure' dimension) describing the
146 visits : `Iterable` [ `VisitDefinitionData` ]
147 Structs identifying the visits and the exposures associated with
148 them. This may be an iterator or a container.
150 raise NotImplementedError()
154 """Return identifiers for the 'visit_system' dimension this
155 algorithm implements.
160 Integer ID for the visit system (given an instrument).
162 Unique string identifier for the visit system (given an
165 raise NotImplementedError()
172 doc=(
"Pad raw image bounding boxes with specified number of pixels "
173 "when calculating their (conservatively large) region on the "
179 """Abstract base class for the subtask of `DefineVisitsTask` that is
180 responsible for extracting spatial regions for visits and visit+detector
183 Subclasses should be registered with `ComputeVisitRegionsTask.registry` to
184 enable use by `DefineVisitsTask`.
188 config : `ComputeVisitRegionsConfig`
189 Configuration information.
190 butler : `lsst.daf.butler.Butler`
193 Additional keyword arguments forwarded to the `Task` constructor.
195 def __init__(self, config: ComputeVisitRegionsConfig, *, butler: Butler, **kwargs: Any):
196 Task.__init__(self, config=config, **kwargs)
200 ConfigClass = ComputeVisitRegionsConfig
202 _DefaultName =
"computeVisitRegions"
204 registry = makeRegistry(
205 doc=(
"Registry of algorithms for computing on-sky regions for visits "
206 "and visit+detector combinations."),
207 configBaseType=ComputeVisitRegionsConfig,
211 """Retrieve an `~lsst.obs.base.Instrument` associated with this
216 instrumentName : `str`
217 The name of the instrument.
221 instrument : `~lsst.obs.base.Instrument`
222 The associated instrument object.
226 The result is cached.
229 if instrument
is None:
230 instrument = Instrument.fromName(instrumentName, self.
butler.registry)
235 def compute(self, visit: VisitDefinitionData, *, collections: Any =
None
236 ) -> Tuple[Region, Dict[int, Region]]:
237 """Compute regions for the given visit and all detectors in that visit.
241 visit : `VisitDefinitionData`
242 Struct describing the visit and the exposures associated with it.
243 collections : Any, optional
244 Collections to be searched for raws and camera geometry, overriding
245 ``self.butler.collections``.
246 Can be any of the types supported by the ``collections`` argument
247 to butler construction.
251 visitRegion : `lsst.sphgeom.Region`
252 Region for the full visit.
253 visitDetectorRegions : `dict` [ `int`, `lsst.sphgeom.Region` ]
254 Dictionary mapping detector ID to the region for that detector.
255 Should include all detectors in the visit.
257 raise NotImplementedError()
261 groupExposures = GroupExposuresTask.registry.makeField(
262 doc=
"Algorithm for grouping exposures into visits.",
263 default=
"one-to-one",
265 computeVisitRegions = ComputeVisitRegionsTask.registry.makeField(
266 doc=
"Algorithm from computing visit and visit+detector regions.",
267 default=
"single-raw-wcs",
269 ignoreNonScienceExposures = Field(
270 doc=(
"If True, silently ignore input exposures that do not have "
271 "observation_type=SCIENCE. If False, raise an exception if one "
280 """Driver Task for defining visits (and their spatial regions) in Gen3
285 config : `DefineVisitsConfig`
286 Configuration for the task.
287 butler : `~lsst.daf.butler.Butler`
288 Writeable butler instance. Will be used to read `raw.wcs` and `camera`
289 datasets and insert/sync dimension data.
291 Additional keyword arguments are forwarded to the `lsst.pipe.base.Task`
296 Each instance of `DefineVisitsTask` reads from / writes to the same Butler.
297 Each invocation of `DefineVisitsTask.run` processes an independent group of
298 exposures into one or more new vists, all belonging to the same visit
299 system and instrument.
301 The actual work of grouping exposures and computing regions is delegated
302 to pluggable subtasks (`GroupExposuresTask` and `ComputeVisitRegionsTask`),
303 respectively. The defaults are to create one visit for every exposure,
304 and to use exactly one (arbitrary) detector-level raw dataset's WCS along
305 with camera geometry to compute regions for all detectors. Other
306 implementations can be created and configured for instruments for which
307 these choices are unsuitable (e.g. because visits and exposures are not
308 one-to-one, or because ``raw.wcs`` datasets for different detectors may not
309 be consistent with camera geomery).
311 It is not necessary in general to ingest all raws for an exposure before
312 defining a visit that includes the exposure; this depends entirely on the
313 `ComputeVisitRegionTask` subclass used. For the default configuration,
314 a single raw for each exposure is sufficient.
316 def __init__(self, config: Optional[DefineVisitsConfig] =
None, *, butler: Butler, **kwargs: Any):
321 self.makeSubtask(
"groupExposures")
322 self.makeSubtask(
"computeVisitRegions", butler=self.
butler)
324 ConfigClass = DefineVisitsConfig
326 _DefaultName =
"defineVisits"
328 def _buildVisitRecords(self, definition: VisitDefinitionData, *,
329 collections: Any =
None) -> _VisitRecords:
330 """Build the DimensionRecords associated with a visit.
334 definition : `VisitDefinition`
335 Struct with identifiers for the visit and records for its
336 constituent exposures.
337 collections : Any, optional
338 Collections to be searched for raws and camera geometry, overriding
339 ``self.butler.collections``.
340 Can be any of the types supported by the ``collections`` argument
341 to butler construction.
345 records : `_VisitRecords`
346 Struct containing DimensionRecords for the visit, including
347 associated dimension elements.
350 visitRegion, visitDetectorRegions = self.computeVisitRegions.compute(definition,
351 collections=collections)
354 begin=_reduceOrNone(min, (e.timespan.begin
for e
in definition.exposures)),
355 end=_reduceOrNone(max, (e.timespan.end
for e
in definition.exposures)),
357 exposure_time = _reduceOrNone(sum, (e.exposure_time
for e
in definition.exposures))
358 physical_filter = _reduceOrNone(
lambda a, b: a
if a == b
else None,
359 (e.physical_filter
for e
in definition.exposures))
360 target_name = _reduceOrNone(
lambda a, b: a
if a == b
else None,
361 (e.target_name
for e
in definition.exposures))
362 science_program = _reduceOrNone(
lambda a, b: a
if a == b
else None,
363 (e.science_program
for e
in definition.exposures))
366 zenith_angle = _reduceOrNone(sum, (e.zenith_angle
for e
in definition.exposures))
367 if zenith_angle
is not None:
368 zenith_angle /= len(definition.exposures)
372 visit=self.
universe[
"visit"].RecordClass(
373 instrument=definition.instrument,
375 name=definition.name,
376 physical_filter=physical_filter,
377 target_name=target_name,
378 science_program=science_program,
379 zenith_angle=zenith_angle,
380 visit_system=self.groupExposures.getVisitSystem()[0],
381 exposure_time=exposure_time,
389 self.
universe[
"visit_definition"].RecordClass(
390 instrument=definition.instrument,
392 exposure=exposure.id,
393 visit_system=self.groupExposures.getVisitSystem()[0],
395 for exposure
in definition.exposures
397 visit_detector_region=[
398 self.
universe[
"visit_detector_region"].RecordClass(
399 instrument=definition.instrument,
402 region=detectorRegion,
404 for detectorId, detectorRegion
in visitDetectorRegions.items()
408 def run(self, dataIds: Iterable[DataId], *,
409 pool: Optional[Pool] =
None,
411 collections: Optional[str] =
None):
412 """Add visit definitions to the registry for the given exposures.
416 dataIds : `Iterable` [ `dict` or `DataCoordinate` ]
417 Exposure-level data IDs. These must all correspond to the same
418 instrument, and are expected to be on-sky science exposures.
419 pool : `multiprocessing.Pool`, optional
420 If not `None`, a process pool with which to parallelize some
422 processes : `int`, optional
423 The number of processes to use. Ignored if ``pool`` is not `None`.
424 collections : Any, optional
425 Collections to be searched for raws and camera geometry, overriding
426 ``self.butler.collections``.
427 Can be any of the types supported by the ``collections`` argument
428 to butler construction.
431 if pool
is None and processes > 1:
432 pool = Pool(processes)
433 mapFunc = map
if pool
is None else pool.imap_unordered
435 self.log.info(
"Preprocessing data IDs.")
436 dimensions = DimensionGraph(self.
universe, names=[
"exposure"])
437 dataIds = set(mapFunc(
lambda d: self.
butler.registry.expandDataId(d, graph=dimensions), dataIds))
439 raise RuntimeError(
"No exposures given.")
444 for dataId
in dataIds:
445 record = dataId.records[
"exposure"]
446 if record.observation_type !=
"science":
447 if self.config.ignoreNonScienceExposures:
450 raise RuntimeError(f
"Input exposure {dataId} has observation_type "
451 f
"{record.observation_type}, not 'science'.")
452 instruments.add(dataId[
"instrument"])
453 exposures.append(record)
455 self.log.info(
"No science exposures found after filtering.")
457 if len(instruments) > 1:
459 f
"All data IDs passed to DefineVisitsTask.run must be "
460 f
"from the same instrument; got {instruments}."
462 instrument, = instruments
465 visitSystemId, visitSystemName = self.groupExposures.getVisitSystem()
466 self.log.info(
"Registering visit_system %d: %s.", visitSystemId, visitSystemName)
467 self.
butler.registry.syncDimensionData(
469 {
"instrument": instrument,
"id": visitSystemId,
"name": visitSystemName}
472 self.log.info(
"Grouping %d exposure(s) into visits.", len(exposures))
473 definitions = list(self.groupExposures.group(exposures))
477 self.log.info(
"Computing regions and other metadata for %d visit(s).", len(definitions))
478 allRecords = mapFunc(
lambda d: self.
_buildVisitRecords(d, collections=collections), definitions)
481 for visitRecords
in allRecords:
482 with self.
butler.registry.transaction():
483 self.
butler.registry.insertDimensionData(
"visit", visitRecords.visit)
484 self.
butler.registry.insertDimensionData(
"visit_definition",
485 *visitRecords.visit_definition)
486 self.
butler.registry.insertDimensionData(
"visit_detector_region",
487 *visitRecords.visit_detector_region)
490 def _reduceOrNone(func, iterable):
491 """Apply a binary function to pairs of elements in an iterable until a
492 single value is returned, but return `None` if any element is `None` or
493 there are no elements.
507 visitSystemId = Field(
508 doc=(
"Integer ID of the visit_system implemented by this grouping "
513 visitSystemName = Field(
514 doc=(
"String name of the visit_system implemented by this grouping "
517 default=
"one-to-one",
521 @registerConfigurable(
"one-to-one", GroupExposuresTask.registry)
523 """An exposure grouping algorithm that simply defines one visit for each
524 exposure, reusing the exposures identifiers for the visit.
527 ConfigClass = _GroupExposuresOneToOneConfig
529 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]:
531 for exposure
in exposures:
533 instrument=exposure.instrument,
536 exposures=[exposure],
541 return (self.config.visitSystemId, self.config.visitSystemName)
545 visitSystemId = Field(
546 doc=(
"Integer ID of the visit_system implemented by this grouping "
551 visitSystemName = Field(
552 doc=(
"String name of the visit_system implemented by this grouping "
555 default=
"by-group-metadata",
559 @registerConfigurable(
"by-group-metadata", GroupExposuresTask.registry)
561 """An exposure grouping algorithm that uses exposure.group_name and
564 This algorithm _assumes_ exposure.group_id (generally populated from
565 `astro_metadata_translator.ObservationInfo.visit_id`) is not just unique,
566 but disjoint from all `ObservationInfo.exposure_id` values - if it isn't,
567 it will be impossible to ever use both this grouping algorithm and the
568 one-to-one algorithm for a particular camera in the same data repository.
571 ConfigClass = _GroupExposuresByGroupMetadataConfig
573 def group(self, exposures: List[DimensionRecord]) -> Iterable[VisitDefinitionData]:
575 groups = defaultdict(list)
576 for exposure
in exposures:
577 groups[exposure.group_name].append(exposure)
578 for visitName, exposuresInGroup
in groups.items():
579 instrument = exposuresInGroup[0].instrument
580 visitId = exposuresInGroup[0].group_id
581 assert all(e.group_id == visitId
for e
in exposuresInGroup), \
582 "Grouping by exposure.group_name does not yield consistent group IDs"
584 exposures=exposuresInGroup)
588 return (self.config.visitSystemId, self.config.visitSystemName)
592 mergeExposures = Field(
593 doc=(
"If True, merge per-detector regions over all exposures in a "
594 "visit (via convex hull) instead of using the first exposure and "
595 "assuming its regions are valid for all others."),
600 doc=(
"Load the WCS for the detector with this ID. If None, use an "
601 "arbitrary detector (the first found in a query of the data "
602 "repository for each exposure (or all exposures, if "
603 "mergeExposures is True)."),
608 requireVersionedCamera = Field(
609 doc=(
"If True, raise LookupError if version camera geometry cannot be "
610 "loaded for an exposure. If False, use the nominal camera from "
611 "the Instrument class instead."),
618 @registerConfigurable(
"single-raw-wcs", ComputeVisitRegionsTask.registry)
620 """A visit region calculator that uses a single raw WCS and a camera to
621 project the bounding boxes of all detectors onto the sky, relating
622 different detectors by their positions in focal plane coordinates.
626 Most instruments should have their raw WCSs determined from a combination
627 of boresight angle, rotator angle, and camera geometry, and hence this
628 algorithm should produce stable results regardless of which detector the
629 raw corresponds to. If this is not the case (e.g. because a per-file FITS
630 WCS is used instead), either the ID of the detector should be fixed (see
631 the ``detectorId`` config parameter) or a different algorithm used.
634 ConfigClass = _ComputeVisitRegionsFromSingleRawWcsConfig
637 ) -> Dict[int, List[UnitVector3d]]:
638 """Compute the lists of unit vectors on the sphere that correspond to
639 the sky positions of detector corners.
643 exposure : `DimensionRecord`
644 Dimension record for the exposure.
645 collections : Any, optional
646 Collections to be searched for raws and camera geometry, overriding
647 ``self.butler.collections``.
648 Can be any of the types supported by the ``collections`` argument
649 to butler construction.
654 Dictionary mapping detector ID to a list of unit vectors on the
655 sphere representing that detector's corners projected onto the sky.
657 if collections
is None:
658 collections = self.
butler.collections
659 camera, versioned =
loadCamera(self.
butler, exposure.dataId, collections=collections)
660 if not versioned
and self.config.requireVersionedCamera:
661 raise LookupError(f
"No versioned camera found for exposure {exposure.dataId}.")
666 orientation = lsst.geom.Angle(exposure.sky_angle, lsst.geom.degrees)
667 radec = lsst.geom.SpherePoint(lsst.geom.Angle(exposure.tracking_ra, lsst.geom.degrees),
668 lsst.geom.Angle(exposure.tracking_dec, lsst.geom.degrees))
669 except AttributeError:
673 if self.config.detectorId
is None:
674 detectorId = next(camera.getIdIter())
676 detectorId = self.config.detectorId
677 wcsDetector = camera[detectorId]
682 rawFormatter = instrument.getRawFormatter({
"detector": detectorId})
683 wcs = rawFormatter.makeRawSkyWcsFromBoresight(radec, orientation, wcsDetector)
686 if self.config.detectorId
is None:
687 wcsRefsIter = self.
butler.registry.queryDatasets(
"raw.wcs", dataId=exposure.dataId,
688 collections=collections)
690 raise LookupError(f
"No raw.wcs datasets found for data ID {exposure.dataId} "
691 f
"in collections {collections}.")
692 wcsRef = next(iter(wcsRefsIter))
693 wcsDetector = camera[wcsRef.dataId[
"detector"]]
694 wcs = self.
butler.getDirect(wcsRef)
696 wcsDetector = camera[self.config.detectorId]
697 wcs = self.
butler.get(
"raw.wcs", dataId=exposure.dataId, detector=self.config.detectorId,
698 collections=collections)
699 fpToSky = wcsDetector.getTransform(FOCAL_PLANE, PIXELS).then(wcs.getTransform())
701 for detector
in camera:
702 pixelsToSky = detector.getTransform(PIXELS, FOCAL_PLANE).then(fpToSky)
703 pixCorners = Box2D(detector.getBBox().dilatedBy(self.config.padding)).getCorners()
704 bounds[detector.getId()] = [
705 skyCorner.getVector()
for skyCorner
in pixelsToSky.applyForward(pixCorners)
709 def compute(self, visit: VisitDefinitionData, *, collections: Any =
None
710 ) -> Tuple[Region, Dict[int, Region]]:
712 if self.config.mergeExposures:
713 detectorBounds = defaultdict(list)
714 for exposure
in visit.exposures:
716 for detectorId, bounds
in exposureDetectorBounds.items():
717 detectorBounds[detectorId].extend(bounds)
722 for detectorId, bounds
in detectorBounds.items():
723 detectorRegions[detectorId] = ConvexPolygon.convexHull(bounds)
724 visitBounds.extend(bounds)
725 return ConvexPolygon.convexHull(visitBounds), detectorRegions