Coverage for python/lsst/pipe/tasks/characterizeImage.py: 29%
189 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-12 10:09 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-12 10:09 +0000
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["CharacterizeImageConfig", "CharacterizeImageTask"]
24import numpy as np
25import warnings
27from lsstDebug import getDebugFrame
28import lsst.afw.table as afwTable
29import lsst.pex.config as pexConfig
30import lsst.pipe.base as pipeBase
31import lsst.daf.base as dafBase
32import lsst.pipe.base.connectionTypes as cT
33from lsst.afw.math import BackgroundList
34from lsst.afw.table import SourceTable
35from lsst.meas.algorithms import (
36 SubtractBackgroundTask,
37 SourceDetectionTask,
38 MeasureApCorrTask,
39 MeasureApCorrError,
40)
41from lsst.meas.algorithms.installGaussianPsf import InstallGaussianPsfTask
42from lsst.meas.astrom import RefMatchTask, displayAstrometry
43from lsst.meas.algorithms import LoadReferenceObjectsConfig
44from lsst.meas.base import (
45 SingleFrameMeasurementTask,
46 ApplyApCorrTask,
47 CatalogCalculationTask,
48 IdGenerator,
49 DetectorVisitIdGeneratorConfig,
50)
51from lsst.meas.deblender import SourceDeblendTask
52import lsst.meas.extensions.shapeHSM # noqa: F401 needed for default shape plugin
53from .measurePsf import MeasurePsfTask
54from .repair import RepairTask
55from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask
56from lsst.pex.exceptions import LengthError
57from lsst.utils.timer import timeMethod
60class CharacterizeImageConnections(pipeBase.PipelineTaskConnections,
61 dimensions=("instrument", "visit", "detector")):
62 exposure = cT.Input(
63 doc="Input exposure data",
64 name="postISRCCD",
65 storageClass="Exposure",
66 dimensions=["instrument", "exposure", "detector"],
67 )
68 characterized = cT.Output(
69 doc="Output characterized data.",
70 name="icExp",
71 storageClass="ExposureF",
72 dimensions=["instrument", "visit", "detector"],
73 )
74 sourceCat = cT.Output(
75 doc="Output source catalog.",
76 name="icSrc",
77 storageClass="SourceCatalog",
78 dimensions=["instrument", "visit", "detector"],
79 )
80 backgroundModel = cT.Output(
81 doc="Output background model.",
82 name="icExpBackground",
83 storageClass="Background",
84 dimensions=["instrument", "visit", "detector"],
85 )
86 outputSchema = cT.InitOutput(
87 doc="Schema of the catalog produced by CharacterizeImage",
88 name="icSrc_schema",
89 storageClass="SourceCatalog",
90 )
92 def adjustQuantum(self, inputs, outputs, label, dataId):
93 # Docstring inherited from PipelineTaskConnections
94 try:
95 return super().adjustQuantum(inputs, outputs, label, dataId)
96 except pipeBase.ScalarError as err:
97 raise pipeBase.ScalarError(
98 "CharacterizeImageTask can at present only be run on visits that are associated with "
99 "exactly one exposure. Either this is not a valid exposure for this pipeline, or the "
100 "snap-combination step you probably want hasn't been configured to run between ISR and "
101 "this task (as of this writing, that would be because it hasn't been implemented yet)."
102 ) from err
105class CharacterizeImageConfig(pipeBase.PipelineTaskConfig,
106 pipelineConnections=CharacterizeImageConnections):
107 """Config for CharacterizeImageTask."""
109 doMeasurePsf = pexConfig.Field(
110 dtype=bool,
111 default=True,
112 doc="Measure PSF? If False then for all subsequent operations use either existing PSF "
113 "model when present, or install simple PSF model when not (see installSimplePsf "
114 "config options)"
115 )
116 doWrite = pexConfig.Field(
117 dtype=bool,
118 default=True,
119 doc="Persist results?",
120 )
121 doWriteExposure = pexConfig.Field(
122 dtype=bool,
123 default=True,
124 doc="Write icExp and icExpBackground in addition to icSrc? Ignored if doWrite False.",
125 )
126 psfIterations = pexConfig.RangeField(
127 dtype=int,
128 default=2,
129 min=1,
130 doc="Number of iterations of detect sources, measure sources, "
131 "estimate PSF. If useSimplePsf is True then 2 should be plenty; "
132 "otherwise more may be wanted.",
133 )
134 background = pexConfig.ConfigurableField(
135 target=SubtractBackgroundTask,
136 doc="Configuration for initial background estimation",
137 )
138 detection = pexConfig.ConfigurableField(
139 target=SourceDetectionTask,
140 doc="Detect sources"
141 )
142 doDeblend = pexConfig.Field(
143 dtype=bool,
144 default=True,
145 doc="Run deblender input exposure"
146 )
147 deblend = pexConfig.ConfigurableField(
148 target=SourceDeblendTask,
149 doc="Split blended source into their components"
150 )
151 measurement = pexConfig.ConfigurableField(
152 target=SingleFrameMeasurementTask,
153 doc="Measure sources"
154 )
155 doApCorr = pexConfig.Field(
156 dtype=bool,
157 default=True,
158 doc="Run subtasks to measure and apply aperture corrections"
159 )
160 measureApCorr = pexConfig.ConfigurableField(
161 target=MeasureApCorrTask,
162 doc="Subtask to measure aperture corrections"
163 )
164 applyApCorr = pexConfig.ConfigurableField(
165 target=ApplyApCorrTask,
166 doc="Subtask to apply aperture corrections"
167 )
168 # If doApCorr is False, and the exposure does not have apcorrections already applied, the
169 # active plugins in catalogCalculation almost certainly should not contain the characterization plugin
170 catalogCalculation = pexConfig.ConfigurableField(
171 target=CatalogCalculationTask,
172 doc="Subtask to run catalogCalculation plugins on catalog"
173 )
174 doComputeSummaryStats = pexConfig.Field(
175 dtype=bool,
176 default=True,
177 doc="Run subtask to measure exposure summary statistics",
178 deprecated=("This subtask has been moved to CalibrateTask "
179 "with DM-30701.")
180 )
181 computeSummaryStats = pexConfig.ConfigurableField(
182 target=ComputeExposureSummaryStatsTask,
183 doc="Subtask to run computeSummaryStats on exposure",
184 deprecated=("This subtask has been moved to CalibrateTask "
185 "with DM-30701.")
186 )
187 useSimplePsf = pexConfig.Field(
188 dtype=bool,
189 default=True,
190 doc="Replace the existing PSF model with a simplified version that has the same sigma "
191 "at the start of each PSF determination iteration? Doing so makes PSF determination "
192 "converge more robustly and quickly.",
193 )
194 installSimplePsf = pexConfig.ConfigurableField(
195 target=InstallGaussianPsfTask,
196 doc="Install a simple PSF model",
197 )
198 refObjLoader = pexConfig.ConfigField(
199 dtype=LoadReferenceObjectsConfig,
200 deprecated="This field does nothing. Will be removed after v24 (see DM-34768).",
201 doc="reference object loader",
202 )
203 ref_match = pexConfig.ConfigurableField(
204 target=RefMatchTask,
205 deprecated="This field was never usable. Will be removed after v24 (see DM-34768).",
206 doc="Task to load and match reference objects. Only used if measurePsf can use matches. "
207 "Warning: matching will only work well if the initial WCS is accurate enough "
208 "to give good matches (roughly: good to 3 arcsec across the CCD).",
209 )
210 measurePsf = pexConfig.ConfigurableField(
211 target=MeasurePsfTask,
212 doc="Measure PSF",
213 )
214 repair = pexConfig.ConfigurableField(
215 target=RepairTask,
216 doc="Remove cosmic rays",
217 )
218 requireCrForPsf = pexConfig.Field(
219 dtype=bool,
220 default=True,
221 doc="Require cosmic ray detection and masking to run successfully before measuring the PSF."
222 )
223 checkUnitsParseStrict = pexConfig.Field(
224 doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
225 dtype=str,
226 default="raise",
227 )
228 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
230 def setDefaults(self):
231 super().setDefaults()
232 # just detect bright stars; includeThresholdMultipler=10 seems large,
233 # but these are the values we have been using
234 self.detection.thresholdValue = 5.0
235 self.detection.includeThresholdMultiplier = 10.0
236 self.detection.doTempLocalBackground = False
237 # do not deblend, as it makes a mess
238 self.doDeblend = False
239 # measure and apply aperture correction; note: measuring and applying aperture
240 # correction are disabled until the final measurement, after PSF is measured
241 self.doApCorr = True
242 # During characterization, we don't have full source measurement information,
243 # so must do the aperture correction with only psf stars, combined with the
244 # default signal-to-noise cuts in MeasureApCorrTask.
245 selector = self.measureApCorr.sourceSelector["science"]
246 selector.doUnresolved = False
247 selector.flags.good = ["calib_psf_used"]
248 selector.flags.bad = []
250 # minimal set of measurements needed to determine PSF
251 self.measurement.plugins.names = [
252 "base_PixelFlags",
253 "base_SdssCentroid",
254 "ext_shapeHSM_HsmSourceMoments",
255 "base_GaussianFlux",
256 "base_PsfFlux",
257 "base_CircularApertureFlux",
258 ]
259 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments"
261 def validate(self):
262 if self.doApCorr and not self.measurePsf:
263 raise RuntimeError("Must measure PSF to measure aperture correction, "
264 "because flags determined by PSF measurement are used to identify "
265 "sources used to measure aperture correction")
268class CharacterizeImageTask(pipeBase.PipelineTask):
269 """Measure bright sources and use this to estimate background and PSF of
270 an exposure.
272 Given an exposure with defects repaired (masked and interpolated over,
273 e.g. as output by `~lsst.ip.isr.IsrTask`):
274 - detect and measure bright sources
275 - repair cosmic rays
276 - measure and subtract background
277 - measure PSF
279 Parameters
280 ----------
281 butler : `None`
282 Compatibility parameter. Should always be `None`.
283 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
284 Reference object loader if using a catalog-based star-selector.
285 schema : `lsst.afw.table.Schema`, optional
286 Initial schema for icSrc catalog.
287 **kwargs
288 Additional keyword arguments.
290 Notes
291 -----
292 Debugging:
293 CharacterizeImageTask has a debug dictionary with the following keys:
295 frame
296 int: if specified, the frame of first debug image displayed (defaults to 1)
297 repair_iter
298 bool; if True display image after each repair in the measure PSF loop
299 background_iter
300 bool; if True display image after each background subtraction in the measure PSF loop
301 measure_iter
302 bool; if True display image and sources at the end of each iteration of the measure PSF loop
303 See `~lsst.meas.astrom.displayAstrometry` for the meaning of the various symbols.
304 psf
305 bool; if True display image and sources after PSF is measured;
306 this will be identical to the final image displayed by measure_iter if measure_iter is true
307 repair
308 bool; if True display image and sources after final repair
309 measure
310 bool; if True display image and sources after final measurement
311 """
313 ConfigClass = CharacterizeImageConfig
314 _DefaultName = "characterizeImage"
316 def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs):
317 super().__init__(**kwargs)
319 if butler is not None:
320 warnings.warn("The 'butler' parameter is no longer used and can be safely removed.",
321 category=FutureWarning, stacklevel=2)
322 butler = None
324 if schema is None:
325 schema = SourceTable.makeMinimalSchema()
326 self.schema = schema
327 self.makeSubtask("background")
328 self.makeSubtask("installSimplePsf")
329 self.makeSubtask("repair")
330 self.makeSubtask("measurePsf", schema=self.schema)
331 # TODO DM-34769: remove this `if` block
332 if self.config.doMeasurePsf and self.measurePsf.usesMatches:
333 self.makeSubtask("ref_match", refObjLoader=refObjLoader)
334 self.algMetadata = dafBase.PropertyList()
335 self.makeSubtask('detection', schema=self.schema)
336 if self.config.doDeblend:
337 self.makeSubtask("deblend", schema=self.schema)
338 self.makeSubtask('measurement', schema=self.schema, algMetadata=self.algMetadata)
339 if self.config.doApCorr:
340 self.makeSubtask('measureApCorr', schema=self.schema)
341 self.makeSubtask('applyApCorr', schema=self.schema)
342 self.makeSubtask('catalogCalculation', schema=self.schema)
343 self._initialFrame = getDebugFrame(self._display, "frame") or 1
344 self._frame = self._initialFrame
345 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
346 self.outputSchema = afwTable.SourceCatalog(self.schema)
348 def runQuantum(self, butlerQC, inputRefs, outputRefs):
349 inputs = butlerQC.get(inputRefs)
350 if 'idGenerator' not in inputs.keys():
351 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId)
352 outputs = self.run(**inputs)
353 butlerQC.put(outputs, outputRefs)
355 @timeMethod
356 def run(self, exposure, exposureIdInfo=None, background=None, idGenerator=None):
357 """Characterize a science image.
359 Peforms the following operations:
360 - Iterate the following config.psfIterations times, or once if config.doMeasurePsf false:
361 - detect and measure sources and estimate PSF (see detectMeasureAndEstimatePsf for details)
362 - interpolate over cosmic rays
363 - perform final measurement
365 Parameters
366 ----------
367 exposure : `lsst.afw.image.ExposureF`
368 Exposure to characterize.
369 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`, optional
370 Exposure ID info. Deprecated in favor of ``idGenerator``, and
371 ignored if that is provided.
372 background : `lsst.afw.math.BackgroundList`, optional
373 Initial model of background already subtracted from exposure.
374 idGenerator : `lsst.meas.base.IdGenerator`, optional
375 Object that generates source IDs and provides RNG seeds.
377 Returns
378 -------
379 result : `lsst.pipe.base.Struct`
380 Results as a struct with attributes:
382 ``exposure``
383 Characterized exposure (`lsst.afw.image.ExposureF`).
384 ``sourceCat``
385 Detected sources (`lsst.afw.table.SourceCatalog`).
386 ``background``
387 Model of subtracted background (`lsst.afw.math.BackgroundList`).
388 ``psfCellSet``
389 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`).
390 ``characterized``
391 Another reference to ``exposure`` for compatibility.
392 ``backgroundModel``
393 Another reference to ``background`` for compatibility.
395 Raises
396 ------
397 RuntimeError
398 Raised if PSF sigma is NaN.
399 """
400 self._frame = self._initialFrame # reset debug display frame
402 if not self.config.doMeasurePsf and not exposure.hasPsf():
403 self.log.info("CharacterizeImageTask initialized with 'simple' PSF.")
404 self.installSimplePsf.run(exposure=exposure)
406 if idGenerator is None:
407 if exposureIdInfo is not None:
408 idGenerator = IdGenerator._from_exposure_id_info(exposureIdInfo)
409 else:
410 idGenerator = IdGenerator()
412 del exposureIdInfo
414 # subtract an initial estimate of background level
415 background = self.background.run(exposure).background
417 psfIterations = self.config.psfIterations if self.config.doMeasurePsf else 1
418 for i in range(psfIterations):
419 dmeRes = self.detectMeasureAndEstimatePsf(
420 exposure=exposure,
421 idGenerator=idGenerator,
422 background=background,
423 )
425 psf = dmeRes.exposure.getPsf()
426 # Just need a rough estimate; average positions are fine
427 psfAvgPos = psf.getAveragePosition()
428 psfSigma = psf.computeShape(psfAvgPos).getDeterminantRadius()
429 psfDimensions = psf.computeImage(psfAvgPos).getDimensions()
430 medBackground = np.median(dmeRes.background.getImage().getArray())
431 self.log.info("iter %s; PSF sigma=%0.2f, dimensions=%s; median background=%0.2f",
432 i + 1, psfSigma, psfDimensions, medBackground)
433 if np.isnan(psfSigma):
434 raise RuntimeError("PSF sigma is NaN, cannot continue PSF determination.")
436 self.display("psf", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
438 # perform final repair with final PSF
439 self.repair.run(exposure=dmeRes.exposure)
440 self.display("repair", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
442 # perform final measurement with final PSF, including measuring and applying aperture correction,
443 # if wanted
444 self.measurement.run(measCat=dmeRes.sourceCat, exposure=dmeRes.exposure,
445 exposureId=idGenerator.catalog_id)
446 if self.config.doApCorr:
447 try:
448 apCorrMap = self.measureApCorr.run(
449 exposure=dmeRes.exposure,
450 catalog=dmeRes.sourceCat,
451 ).apCorrMap
452 except MeasureApCorrError:
453 # We have failed to get a valid aperture correction map.
454 # Proceed with processing, and image will be filtered
455 # downstream.
456 dmeRes.exposure.info.setApCorrMap(None)
457 else:
458 dmeRes.exposure.info.setApCorrMap(apCorrMap)
459 self.applyApCorr.run(catalog=dmeRes.sourceCat, apCorrMap=exposure.getInfo().getApCorrMap())
461 self.catalogCalculation.run(dmeRes.sourceCat)
463 self.display("measure", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
465 return pipeBase.Struct(
466 exposure=dmeRes.exposure,
467 sourceCat=dmeRes.sourceCat,
468 background=dmeRes.background,
469 psfCellSet=dmeRes.psfCellSet,
471 characterized=dmeRes.exposure,
472 backgroundModel=dmeRes.background
473 )
475 @timeMethod
476 def detectMeasureAndEstimatePsf(self, exposure, idGenerator, background):
477 """Perform one iteration of detect, measure, and estimate PSF.
479 Performs the following operations:
481 - if config.doMeasurePsf or not exposure.hasPsf():
483 - install a simple PSF model (replacing the existing one, if need be)
485 - interpolate over cosmic rays with keepCRs=True
486 - estimate background and subtract it from the exposure
487 - detect, deblend and measure sources, and subtract a refined background model;
488 - if config.doMeasurePsf:
489 - measure PSF
491 Parameters
492 ----------
493 exposure : `lsst.afw.image.ExposureF`
494 Exposure to characterize.
495 idGenerator : `lsst.meas.base.IdGenerator`
496 Object that generates source IDs and provides RNG seeds.
497 background : `lsst.afw.math.BackgroundList`, optional
498 Initial model of background already subtracted from exposure.
500 Returns
501 -------
502 result : `lsst.pipe.base.Struct`
503 Results as a struct with attributes:
505 ``exposure``
506 Characterized exposure (`lsst.afw.image.ExposureF`).
507 ``sourceCat``
508 Detected sources (`lsst.afw.table.SourceCatalog`).
509 ``background``
510 Model of subtracted background (`lsst.afw.math.BackgroundList`).
511 ``psfCellSet``
512 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`).
514 Raises
515 ------
516 LengthError
517 Raised if there are too many CR pixels.
518 """
519 # install a simple PSF model, if needed or wanted
520 if not exposure.hasPsf() or (self.config.doMeasurePsf and self.config.useSimplePsf):
521 self.log.info("PSF estimation initialized with 'simple' PSF")
522 self.installSimplePsf.run(exposure=exposure)
524 # run repair, but do not interpolate over cosmic rays (do that elsewhere, with the final PSF model)
525 if self.config.requireCrForPsf:
526 self.repair.run(exposure=exposure, keepCRs=True)
527 else:
528 try:
529 self.repair.run(exposure=exposure, keepCRs=True)
530 except LengthError:
531 self.log.warning("Skipping cosmic ray detection: Too many CR pixels (max %0.f)",
532 self.config.repair.cosmicray.nCrPixelMax)
534 self.display("repair_iter", exposure=exposure)
536 if background is None:
537 background = BackgroundList()
539 sourceIdFactory = idGenerator.make_table_id_factory()
540 table = SourceTable.make(self.schema, sourceIdFactory)
541 table.setMetadata(self.algMetadata)
543 detRes = self.detection.run(table=table, exposure=exposure, doSmooth=True)
544 sourceCat = detRes.sources
545 if detRes.background:
546 for bg in detRes.background:
547 background.append(bg)
549 if self.config.doDeblend:
550 self.deblend.run(exposure=exposure, sources=sourceCat)
551 # We need the output catalog to be contiguous for further processing.
552 if not sourceCat.isContiguous():
553 sourceCat = sourceCat.copy(deep=True)
555 self.measurement.run(measCat=sourceCat, exposure=exposure, exposureId=idGenerator.catalog_id)
557 measPsfRes = pipeBase.Struct(cellSet=None)
558 if self.config.doMeasurePsf:
559 # TODO DM-34769: remove this `if` block, and the `matches` kwarg from measurePsf.run below.
560 if self.measurePsf.usesMatches:
561 matches = self.ref_match.loadAndMatch(exposure=exposure, sourceCat=sourceCat).matches
562 else:
563 matches = None
564 measPsfRes = self.measurePsf.run(exposure=exposure, sources=sourceCat, matches=matches,
565 expId=idGenerator.catalog_id)
566 self.display("measure_iter", exposure=exposure, sourceCat=sourceCat)
568 return pipeBase.Struct(
569 exposure=exposure,
570 sourceCat=sourceCat,
571 background=background,
572 psfCellSet=measPsfRes.cellSet,
573 )
575 def display(self, itemName, exposure, sourceCat=None):
576 """Display exposure and sources on next frame (for debugging).
578 Parameters
579 ----------
580 itemName : `str`
581 Name of item in ``debugInfo``.
582 exposure : `lsst.afw.image.ExposureF`
583 Exposure to display.
584 sourceCat : `lsst.afw.table.SourceCatalog`, optional
585 Catalog of sources detected on the exposure.
586 """
587 val = getDebugFrame(self._display, itemName)
588 if not val:
589 return
591 displayAstrometry(exposure=exposure, sourceCat=sourceCat, frame=self._frame, pause=False)
592 self._frame += 1