22from __future__
import annotations
25 "UpdateVisitSummaryConnections",
26 "UpdateVisitSummaryConfig",
27 "UpdateVisitSummaryTask",
28 "PossiblyMultipleInput",
34from abc
import ABC, abstractmethod
35from collections.abc
import Iterable, Mapping
39import lsst.pipe.base.connectionTypes
as cT
43from lsst.afw.table import ExposureCatalog, ExposureRecord, SchemaMapper
44from lsst.daf.butler
import Butler, DatasetRef, DeferredDatasetHandle
45from lsst.geom import Angle, Box2I, SpherePoint, degrees
49 InputQuantizedConnection,
51 OutputQuantizedConnection,
54 PipelineTaskConnections,
59from .computeExposureSummaryStats
import ComputeExposureSummaryStatsTask
63 record: ExposureRecord, bbox: Box2I |
None =
None, wcs: SkyWcs |
None =
None
64) -> SpherePoint |
None:
65 """Compute the sky coordinate center for a detector to be used when
66 testing distance to tract center.
71 Exposure record to obtain WCS and bbox
from if not provided.
73 Bounding box
for the detector
in its own pixel coordinates.
75 WCS that maps the detector
's pixel coordinate system to celestial
81 Center of the detector
in sky coordinates,
or `
None`
if no WCS was
82 given
or present
in the given record.
85 bbox = record.getBBox()
90 region = makeSkyPolygonFromBBox(bbox, wcs)
95 """A helper ABC for handling input `~lsst.afw.table.ExposureCatalog`
96 datasets that may be multiple (one per tract/visit combination) or
97 unique/
global (one per visit).
104 center: SpherePoint |
None =
None,
105 bbox: Box2I |
None =
None,
106 ) -> tuple[int, ExposureRecord |
None]:
107 """Return the exposure record for this detector that is the best match
113 Detector ID; used to find the right row
in the catalog
or catalogs.
115 Center of the detector
in sky coordinates. If
not provided, one
116 will be computed via `compute_center_for_detector_record`.
118 Bounding box
for the detector
in its own pixel coordinates.
123 ID of the tract that supplied this record,
or `-1`
if ``record``
is
124 `
None`
or if the input was
not per-tract.
126 Best record
for this detector,
or `
None`
if there either were no
127 records
for this detector
or no WCS available to compute a center.
129 raise NotImplementedError()
132@dataclasses.dataclass
134 """Wrapper class for input `~lsst.afw.table.ExposureCatalog` datasets
137 This selects the best tract via the minimum average distance (on the sky)
138 from the detector
's corners to the tract center.
141 catalogs_by_tract: list[tuple[TractInfo, ExposureCatalog]]
142 """List of tuples of catalogs and the tracts they correspond to
143 (`list` [`tuple` [`lsst.skymap.TractInfo`,
150 butler: ButlerQuantumContext | Butler,
152 refs: Iterable[DatasetRef],
154 """Load and wrap input catalogs.
158 butler : `lsst.pipe.base.ButlerQuantumContext`
159 Butler proxy used in `~lsst.pipe.base.PipelineTask.runQuantum`.
160 sky_map : `lsst.skymap.BaseSkyMap`
161 Definition of tracts
and patches.
162 refs : `~collections.abc.Iterable` [`lsst.daf.butler.DatasetRef`]
163 References to the catalog datasets to load.
167 wrapper : `PerTractInput`
168 Wrapper object
for the loaded catalogs.
170 catalogs_by_tract = []
172 tract_id = ref.dataId[
"tract"]
173 tract_info = sky_map[tract_id]
174 catalogs_by_tract.append(
180 return cls(catalogs_by_tract)
185 center: SpherePoint |
None =
None,
186 bbox: Box2I |
None =
None,
187 ) -> tuple[int, ExposureRecord |
None]:
189 best_result: tuple[int, ExposureRecord |
None] = (-1,
None)
190 best_distance: Angle = float(
"inf") * degrees
191 for tract_info, catalog
in self.catalogs_by_tract:
192 record = catalog.find(detector_id)
199 if center_for_record
is None:
202 center_for_record = center
203 center_distance = tract_info.ctr_coord.separation(center_for_record)
204 if best_distance > center_distance:
205 best_result = (tract_info.tract_id, record)
206 best_distance = center_distance
210@dataclasses.dataclass
212 """Wrapper class for input `~lsst.afw.table.ExposureCatalog` datasets
213 that are not per-tract.
216 catalog: ExposureCatalog
217 """Loaded per-visit catalog dataset (`lsst.afw.table.ExposureCatalog`).
223 center: SpherePoint |
None =
None,
224 bbox: Box2I |
None =
None,
225 ) -> tuple[int, ExposureRecord |
None]:
227 return -1, self.catalog.find(detector_id)
231 PipelineTaskConnections,
232 dimensions=(
"instrument",
"visit"),
234 "skyWcsName":
"jointcal",
235 "photoCalibName":
"fgcm",
239 doc=
"Description of tract/patch geometry.",
240 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
241 dimensions=(
"skymap",),
242 storageClass=
"SkyMap",
244 input_summary_schema = cT.InitInput(
245 doc=
"Schema for input_summary_catalog.",
246 name=
"visitSummary_schema",
247 storageClass=
"ExposureCatalog",
249 input_summary_catalog = cT.Input(
250 doc=
"Visit summary table to load and modify.",
252 dimensions=(
"instrument",
"visit"),
253 storageClass=
"ExposureCatalog",
255 input_exposures = cT.Input(
257 "Per-detector images to obtain image, mask, and variance from "
258 "(embedded summary stats and other components are ignored)."
261 dimensions=(
"instrument",
"detector",
"visit"),
262 storageClass=
"ExposureF",
265 deferGraphConstraint=
True,
267 psf_overrides = cT.Input(
268 doc=
"Visit-level catalog of updated PSFs to use.",
269 name=
"finalized_psf_ap_corr_catalog",
270 dimensions=(
"instrument",
"visit"),
271 storageClass=
"ExposureCatalog",
272 deferGraphConstraint=
True,
274 psf_star_catalog = cT.Input(
275 doc=
"Per-visit table of PSF reserved- and used-star measurements.",
276 name=
"finalized_src_table",
277 dimensions=(
"instrument",
"visit"),
278 storageClass=
"DataFrame",
279 deferGraphConstraint=
True,
281 ap_corr_overrides = cT.Input(
282 doc=
"Visit-level catalog of updated aperture correction maps to use.",
283 name=
"finalized_psf_ap_corr_catalog",
284 dimensions=(
"instrument",
"visit"),
285 storageClass=
"ExposureCatalog",
286 deferGraphConstraint=
True,
288 photo_calib_overrides_tract = cT.Input(
289 doc=
"Per-Tract visit-level catalog of updated photometric calibration objects to use.",
290 name=
"{photoCalibName}PhotoCalibCatalog",
291 dimensions=(
"instrument",
"visit",
"tract"),
292 storageClass=
"ExposureCatalog",
294 deferGraphConstraint=
True,
296 photo_calib_overrides_global = cT.Input(
297 doc=
"Global visit-level catalog of updated photometric calibration objects to use.",
298 name=
"{photoCalibName}PhotoCalibCatalog",
299 dimensions=(
"instrument",
"visit"),
300 storageClass=
"ExposureCatalog",
301 deferGraphConstraint=
True,
303 wcs_overrides_tract = cT.Input(
304 doc=
"Per-tract visit-level catalog of updated astrometric calibration objects to use.",
305 name=
"{skyWcsName}SkyWcsCatalog",
306 dimensions=(
"instrument",
"visit",
"tract"),
307 storageClass=
"ExposureCatalog",
309 deferGraphConstraint=
True,
311 wcs_overrides_global = cT.Input(
312 doc=
"Global visit-level catalog of updated astrometric calibration objects to use.",
313 name=
"{skyWcsName}SkyWcsCatalog",
314 dimensions=(
"instrument",
"visit"),
315 storageClass=
"ExposureCatalog",
316 deferGraphConstraint=
True,
318 background_originals = cT.Input(
319 doc=
"Per-detector original background that has already been subtracted from 'input_exposures'.",
320 name=
"calexpBackground",
321 dimensions=(
"instrument",
"visit",
"detector"),
322 storageClass=
"Background",
325 deferGraphConstraint=
True,
327 background_overrides = cT.Input(
328 doc=
"Per-detector background that can be subtracted directly from 'input_exposures'.",
330 dimensions=(
"instrument",
"visit",
"detector"),
331 storageClass=
"Background",
334 deferGraphConstraint=
True,
336 output_summary_schema = cT.InitOutput(
337 doc=
"Schema of the output visit summary catalog.",
338 name=
"finalVisitSummary_schema",
339 storageClass=
"ExposureCatalog",
341 output_summary_catalog = cT.Output(
342 doc=
"Visit-level catalog summarizing all image characterizations and calibrations.",
343 name=
"finalVisitSummary",
344 dimensions=(
"instrument",
"visit"),
345 storageClass=
"ExposureCatalog",
348 def __init__(self, *, config: UpdateVisitSummaryConfig |
None =
None):
349 super().__init__(config=config)
350 match self.config.wcs_provider:
351 case
"input_summary":
352 self.inputs.remove(
"wcs_overrides_tract")
353 self.inputs.remove(
"wcs_overrides_global")
355 self.inputs.remove(
"wcs_overrides_global")
357 self.inputs.remove(
"wcs_overrides_tract")
360 f
"Invalid value wcs_provider={bad!r}; config was not validated."
362 match self.config.photo_calib_provider:
363 case
"input_summary":
364 self.inputs.remove(
"photo_calib_overrides_tract")
365 self.inputs.remove(
"photo_calib_overrides_global")
367 self.inputs.remove(
"photo_calib_overrides_global")
369 self.inputs.remove(
"photo_calib_overrides_tract")
372 f
"Invalid value photo_calib_provider={bad!r}; config was not validated."
374 match self.config.background_provider:
375 case
"input_summary":
376 self.inputs.remove(
"background_originals")
377 self.inputs.remove(
"background_overrides")
382 f
"Invalid value background_provider={bad!r}; config was not validated."
386class UpdateVisitSummaryConfig(
387 PipelineTaskConfig, pipelineConnections=UpdateVisitSummaryConnections
389 """Configuration for UpdateVisitSummaryTask.
393 The configuration defaults for this task reflect a simple
or "least common
394 denominator" pipeline, not the more complete, more sophisticated pipeline
395 we run on the instruments we support best. The expectation is that the
396 various full pipeline definitions will generally
import the simpler
397 definition, so making the defaults correspond to any full pipeline would
398 just lead to the simple pipeline setting them back to the simple-pipeline
399 values
and the full pipeline still having to then override them to the
400 full-pipeline values.
403 compute_summary_stats = ConfigurableField(
404 doc="Subtask that computes summary statistics from Exposure components.",
405 target=ComputeExposureSummaryStatsTask,
407 wcs_provider = ChoiceField(
408 doc=
"Which connection and behavior to use when applying WCS overrides.",
412 "Propagate the WCS from the input visit summary catalog "
413 "and do not recompute WCS-based summary statistics."
416 "Use the 'wcs_overrides_tract' connection to load an "
417 "`ExposureCatalog` with {visit, tract} dimensions and per-"
418 "detector rows, and recommpute WCS-based summary statistics."
421 "Use the 'wcs_overrides_global' connection to load an "
422 "`ExposureCatalog` with {visit} dimensions and per-"
423 "detector rows, and recommpute WCS-based summary statistics."
431 default=
"input_summary",
434 photo_calib_provider = ChoiceField(
435 doc=
"Which connection and behavior to use when applying photometric calibration overrides.",
439 "Propagate the PhotoCalib from the input visit summary catalog "
440 "and do not recompute photometric calibration summary "
444 "Use the 'photo_calib_overrides_tract' connection to load an "
445 "`ExposureCatalog` with {visit, tract} dimensions and per-"
446 "detector rows, and recommpute photometric calibration summary "
450 "Use the 'photo_calib_overrides_global' connection to load an "
451 "`ExposureCatalog` with {visit} dimensions and per-"
452 "detector rows, and recommpute photometric calibration summary "
461 default=
"input_summary",
464 background_provider = ChoiceField(
465 doc=
"Which connection(s) and behavior to use when applying background overrides.",
469 "The input visit summary catalog already includes summary "
470 "statistics for the final backgrounds that can be used as-is."
473 "The 'background_originals' connection refers to a background "
474 "model that has been superseded by the model referred to by "
475 "the 'background_overrides' connection."
480 default=
"input_summary",
488class UpdateVisitSummaryTask(PipelineTask):
489 """A pipeline task that creates a new visit-summary table after all
494 This task is designed to be run just prior to making warps
for coaddition,
495 as it aggregates all inputs other than the images
and backgrounds into a
496 single ``ExposureCatalog`` dataset
and recomputes summary statistics that
497 are useful
in selecting which images should go into a coadd. Its output
498 can also be used to reconstruct a final processed visit image when combined
499 with a post-ISR image, the background model,
and the final mask.
507 _DefaultName =
"updateVisitSummary"
508 ConfigClass = UpdateVisitSummaryConfig
510 compute_summary_stats: ComputeExposureSummaryStatsTask
512 def __init__(self, *, initInputs: dict[str, Any] |
None =
None, **kwargs: Any):
513 super().__init__(initInputs=initInputs, **kwargs)
514 self.makeSubtask(
"compute_summary_stats")
515 if initInputs
is None or "input_summary_schema" not in initInputs:
516 raise RuntimeError(
"Task requires 'input_summary_schema' in initInputs.")
517 input_summary_schema = initInputs[
"input_summary_schema"].schema
519 self.schema_mapper.addMinimalSchema(input_summary_schema)
520 self.schema = self.schema_mapper.getOutputSchema()
521 if self.config.wcs_provider ==
"tract":
522 self.schema.addField(
523 "wcsTractId", type=
"L", doc=
"ID of the tract that provided the WCS."
525 if self.config.photo_calib_provider ==
"tract":
526 self.schema.addField(
529 doc=
"ID of the tract that provided the PhotoCalib.",
531 self.output_summary_schema = ExposureCatalog(self.schema)
535 butlerQC: ButlerQuantumContext,
536 inputRefs: InputQuantizedConnection,
537 outputRefs: OutputQuantizedConnection,
540 sky_map = butlerQC.get(inputRefs.sky_map)
541 del inputRefs.sky_map
546 match self.config.wcs_provider:
548 inputs[
"wcs_overrides"] = PerTractInput.load(
549 butlerQC, sky_map, inputRefs.wcs_overrides_tract
551 del inputRefs.wcs_overrides_tract
554 butlerQC.get(inputRefs.wcs_overrides_global)
556 del inputRefs.wcs_overrides_global
557 case
"input_summary":
558 inputs[
"wcs_overrides"] =
None
559 match self.config.photo_calib_provider:
561 inputs[
"photo_calib_overrides"] = PerTractInput.load(
562 butlerQC, sky_map, inputRefs.photo_calib_overrides_tract
564 del inputRefs.photo_calib_overrides_tract
567 butlerQC.get(inputRefs.photo_calib_overrides_global)
569 del inputRefs.photo_calib_overrides_global
570 case
"input_summary":
571 inputs[
"photo_calib_overrides"] =
None
573 inputs.update(butlerQC.get(inputRefs))
574 deferred_dataset_types = [
"input_exposures"]
576 match self.config.background_provider:
578 deferred_dataset_types.append(
"background_originals")
579 deferred_dataset_types.append(
"background_overrides")
582 for name
in deferred_dataset_types:
583 handles_list = inputs[name]
585 handle.dataId[
"detector"]: handle
for handle
in handles_list
587 for record
in inputs[
"input_summary_catalog"]:
588 detector_id = record.getId()
589 if detector_id
not in inputs[name]:
590 raise InvalidQuantumError(
591 f
"No {name!r} with detector {detector_id} for visit "
592 f
"{butlerQC.quantum.dataId['visit']} even though this detector is present "
593 "in the input visit summary catalog. "
594 "This is most likely to occur when the QuantumGraph that includes this task "
595 "was incorrectly generated with an explicit or implicit (from datasets) tract "
605 inputs[
"psf_star_catalog"] = astropy.table.Table.from_pandas(inputs[
"psf_star_catalog"], index=
True)
607 outputs = self.run(**inputs)
608 butlerQC.put(outputs, outputRefs)
612 input_summary_catalog: ExposureCatalog,
613 input_exposures: Mapping[int, DeferredDatasetHandle],
614 psf_overrides: ExposureCatalog |
None =
None,
615 psf_star_catalog: astropy.table.Table |
None =
None,
616 ap_corr_overrides: ExposureCatalog |
None =
None,
617 photo_calib_overrides: PossiblyMultipleInput |
None =
None,
618 wcs_overrides: PossiblyMultipleInput |
None =
None,
619 background_originals: Mapping[int, DeferredDatasetHandle] |
None =
None,
620 background_overrides: Mapping[int, DeferredDatasetHandle] |
None =
None,
622 """Build an updated version of a visit summary catalog.
627 Input catalog. Each row in this catalog will be used to produce
628 a row
in the output catalog. Any override parameter that
is `
None`
629 will leave the corresponding values unchanged
from those
in this
631 input_exposures : `collections.abc.Mapping` [`int`,
632 `lsst.daf.butler.DeferredDatasetHandle`]
634 instances. Only the image, mask,
and variance are used; all other
635 components are assumed to be superceded by at least
636 ``input_summary_catalog``
and probably some ``_overrides``
637 arguments
as well. This usually corresponds to the ``calexp``
641 supersede the input catalog
's PSFs.
642 psf_star_catalog : `astropy.table.Table`, optional
643 Table containing PSF stars for use
in computing PSF summary
644 statistics. Must be provided
if ``psf_overrides``
is.
647 supersede the input catalog
's aperture corrections.
648 photo_calib_overrides : `PossiblyMultipleInput`, optional
650 objects that supersede the input catalog
's photometric
652 wcs_overrides : `PossiblyMultipleInput`, optional
654 that supersede the input catalog
's astrometric calibrations.
655 background_originals : `collections.abc.Mapping` [`int`,
656 `lsst.daf.butler.DeferredDatasetHandle`], optional
657 Deferred-load objects that fetch `lsst.afw.math.BackgroundList`
658 instances. These should correspond to the background already
659 subtracted from ``input_exposures``. If
not provided
and
660 ``background_overrides``
is, it
is assumed that the background
in
661 ``input_exposures`` has
not been subtracted. If provided, all keys
662 in ``background_overrides`` must also be present
in
663 ``background_originals``.
664 background_overrides : `collections.abc.Mapping` [`int`,
665 `lsst.daf.butler.DeferredDatasetHandle`], optional
666 Deferred-load objects that fetch `lsst.afw.math.BackgroundList`
667 instances. These should correspond to the background that should
668 now be subtracted
from``input_exposures`` to
yield the final
669 background-subtracted image.
674 Output visit summary catalog.
678 If any override parameter
is provided but does
not have a value
for a
679 particular detector, that component will be set to `
None`
in the
680 returned catalog
for that detector
and all summary statistics derived
681 from that component will be reset (usually to ``NaN``)
as well. Not
682 passing an override parameter at all will instead
pass through the
683 original component
and values
from the input catalog unchanged.
685 output_summary_catalog = ExposureCatalog(self.schema)
686 output_summary_catalog.setMetadata(input_summary_catalog.getMetadata())
687 for input_record
in input_summary_catalog:
688 detector_id = input_record.getId()
689 output_record = output_summary_catalog.addNew()
692 summary_stats = ExposureSummaryStats.from_record(input_record)
699 output_record.assign(input_record, self.schema_mapper)
701 exposure = input_exposures[detector_id].get()
702 bbox = exposure.getBBox()
705 wcs_tract, wcs_record = wcs_overrides.best_for_detector(
706 detector_id, bbox=bbox
708 if wcs_record
is not None:
709 wcs = wcs_record.getWcs()
712 if self.config.wcs_provider ==
"tract":
713 output_record[
"wcsTractId"] = wcs_tract
714 output_record.setWcs(wcs)
715 self.compute_summary_stats.update_wcs_stats(
716 summary_stats, wcs, bbox, output_record.getVisitInfo()
719 wcs = input_record.getWcs()
722 if (psf_record := psf_overrides.find(detector_id))
is not None:
723 psf = psf_record.getPsf()
726 output_record.setPsf(psf)
727 sources = psf_star_catalog[psf_star_catalog[
"detector"] == detector_id]
728 self.compute_summary_stats.update_psf_stats(
733 image_mask=exposure.mask,
734 sources_is_astropy=
True,
737 if ap_corr_overrides:
738 if (ap_corr_record := ap_corr_overrides.find(detector_id))
is not None:
739 ap_corr = ap_corr_record.getApCorrMap()
742 output_record.setApCorrMap(ap_corr)
744 if photo_calib_overrides:
749 ) = photo_calib_overrides.best_for_detector(detector_id, center=center)
750 if photo_calib_record
is not None:
751 photo_calib = photo_calib_record.getPhotoCalib()
754 if self.config.photo_calib_provider ==
"tract":
755 output_record[
"photoCalibTractId"] = photo_calib_tract
756 output_record.setPhotoCalib(photo_calib)
757 self.compute_summary_stats.update_photo_calib_stats(
758 summary_stats, photo_calib
761 if background_overrides
is not None:
762 if (handle := background_overrides.get(detector_id))
is not None:
763 new_bkg = handle.get()
764 if background_originals
is not None:
765 orig_bkg = background_originals[detector_id].get()
767 orig_bkg = BackgroundList()
769 full_bkg = orig_bkg.clone()
770 for layer
in new_bkg:
771 full_bkg.append(layer)
772 exposure.image -= new_bkg.getImage()
773 self.compute_summary_stats.update_background_stats(
774 summary_stats, full_bkg
776 self.compute_summary_stats.update_masked_image_stats(
777 summary_stats, exposure.getMaskedImage()
780 summary_stats.update_record(output_record)
783 return Struct(output_summary_catalog=output_summary_catalog)
SpherePoint|None compute_center_for_detector_record(ExposureRecord record, Box2I|None bbox=None, SkyWcs|None wcs=None)