Coverage for python/lsst/pipe/tasks/characterizeImage.py: 31%
179 statements
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-10 03:09 -0700
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-10 03:09 -0700
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["CharacterizeImageConfig", "CharacterizeImageTask"]
24import numpy as np
26from lsstDebug import getDebugFrame
27import lsst.afw.table as afwTable
28import lsst.pex.config as pexConfig
29import lsst.pipe.base as pipeBase
30import lsst.daf.base as dafBase
31import lsst.pipe.base.connectionTypes as cT
32from lsst.afw.math import BackgroundList
33from lsst.afw.table import SourceTable
34from lsst.meas.algorithms import (
35 SubtractBackgroundTask,
36 SourceDetectionTask,
37 MeasureApCorrTask,
38 MeasureApCorrError,
39 MaskStreaksTask,
40)
41from lsst.meas.algorithms.installGaussianPsf import InstallGaussianPsfTask
42from lsst.meas.astrom import displayAstrometry
43from lsst.meas.base import (
44 SingleFrameMeasurementTask,
45 ApplyApCorrTask,
46 CatalogCalculationTask,
47 IdGenerator,
48 DetectorVisitIdGeneratorConfig,
49)
50from lsst.meas.deblender import SourceDeblendTask
51import lsst.meas.extensions.shapeHSM # noqa: F401 needed for default shape plugin
52from .measurePsf import MeasurePsfTask
53from .repair import RepairTask
54from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask
55from lsst.pex.exceptions import LengthError
56from lsst.utils.timer import timeMethod
59class CharacterizeImageConnections(pipeBase.PipelineTaskConnections,
60 dimensions=("instrument", "visit", "detector")):
61 exposure = cT.Input(
62 doc="Input exposure data",
63 name="postISRCCD",
64 storageClass="Exposure",
65 dimensions=["instrument", "exposure", "detector"],
66 )
67 characterized = cT.Output(
68 doc="Output characterized data.",
69 name="icExp",
70 storageClass="ExposureF",
71 dimensions=["instrument", "visit", "detector"],
72 )
73 sourceCat = cT.Output(
74 doc="Output source catalog.",
75 name="icSrc",
76 storageClass="SourceCatalog",
77 dimensions=["instrument", "visit", "detector"],
78 )
79 backgroundModel = cT.Output(
80 doc="Output background model.",
81 name="icExpBackground",
82 storageClass="Background",
83 dimensions=["instrument", "visit", "detector"],
84 )
85 outputSchema = cT.InitOutput(
86 doc="Schema of the catalog produced by CharacterizeImage",
87 name="icSrc_schema",
88 storageClass="SourceCatalog",
89 )
91 def adjustQuantum(self, inputs, outputs, label, dataId):
92 # Docstring inherited from PipelineTaskConnections
93 try:
94 return super().adjustQuantum(inputs, outputs, label, dataId)
95 except pipeBase.ScalarError as err:
96 raise pipeBase.ScalarError(
97 "CharacterizeImageTask can at present only be run on visits that are associated with "
98 "exactly one exposure. Either this is not a valid exposure for this pipeline, or the "
99 "snap-combination step you probably want hasn't been configured to run between ISR and "
100 "this task (as of this writing, that would be because it hasn't been implemented yet)."
101 ) from err
104class CharacterizeImageConfig(pipeBase.PipelineTaskConfig,
105 pipelineConnections=CharacterizeImageConnections):
106 """Config for CharacterizeImageTask."""
108 doMeasurePsf = pexConfig.Field(
109 dtype=bool,
110 default=True,
111 doc="Measure PSF? If False then for all subsequent operations use either existing PSF "
112 "model when present, or install simple PSF model when not (see installSimplePsf "
113 "config options)"
114 )
115 doWrite = pexConfig.Field(
116 dtype=bool,
117 default=True,
118 doc="Persist results?",
119 )
120 doWriteExposure = pexConfig.Field(
121 dtype=bool,
122 default=True,
123 doc="Write icExp and icExpBackground in addition to icSrc? Ignored if doWrite False.",
124 )
125 psfIterations = pexConfig.RangeField(
126 dtype=int,
127 default=2,
128 min=1,
129 doc="Number of iterations of detect sources, measure sources, "
130 "estimate PSF. If useSimplePsf is True then 2 should be plenty; "
131 "otherwise more may be wanted.",
132 )
133 background = pexConfig.ConfigurableField(
134 target=SubtractBackgroundTask,
135 doc="Configuration for initial background estimation",
136 )
137 detection = pexConfig.ConfigurableField(
138 target=SourceDetectionTask,
139 doc="Detect sources"
140 )
141 doDeblend = pexConfig.Field(
142 dtype=bool,
143 default=True,
144 doc="Run deblender input exposure"
145 )
146 deblend = pexConfig.ConfigurableField(
147 target=SourceDeblendTask,
148 doc="Split blended source into their components"
149 )
150 measurement = pexConfig.ConfigurableField(
151 target=SingleFrameMeasurementTask,
152 doc="Measure sources"
153 )
154 doApCorr = pexConfig.Field(
155 dtype=bool,
156 default=True,
157 doc="Run subtasks to measure and apply aperture corrections"
158 )
159 measureApCorr = pexConfig.ConfigurableField(
160 target=MeasureApCorrTask,
161 doc="Subtask to measure aperture corrections"
162 )
163 applyApCorr = pexConfig.ConfigurableField(
164 target=ApplyApCorrTask,
165 doc="Subtask to apply aperture corrections"
166 )
167 # If doApCorr is False, and the exposure does not have apcorrections already applied, the
168 # active plugins in catalogCalculation almost certainly should not contain the characterization plugin
169 catalogCalculation = pexConfig.ConfigurableField(
170 target=CatalogCalculationTask,
171 doc="Subtask to run catalogCalculation plugins on catalog"
172 )
173 doComputeSummaryStats = pexConfig.Field(
174 dtype=bool,
175 default=True,
176 doc="Run subtask to measure exposure summary statistics",
177 deprecated=("This subtask has been moved to CalibrateTask "
178 "with DM-30701.")
179 )
180 computeSummaryStats = pexConfig.ConfigurableField(
181 target=ComputeExposureSummaryStatsTask,
182 doc="Subtask to run computeSummaryStats on exposure",
183 deprecated=("This subtask has been moved to CalibrateTask "
184 "with DM-30701.")
185 )
186 useSimplePsf = pexConfig.Field(
187 dtype=bool,
188 default=True,
189 doc="Replace the existing PSF model with a simplified version that has the same sigma "
190 "at the start of each PSF determination iteration? Doing so makes PSF determination "
191 "converge more robustly and quickly.",
192 )
193 installSimplePsf = pexConfig.ConfigurableField(
194 target=InstallGaussianPsfTask,
195 doc="Install a simple PSF model",
196 )
197 measurePsf = pexConfig.ConfigurableField(
198 target=MeasurePsfTask,
199 doc="Measure PSF",
200 )
201 repair = pexConfig.ConfigurableField(
202 target=RepairTask,
203 doc="Remove cosmic rays",
204 )
205 requireCrForPsf = pexConfig.Field(
206 dtype=bool,
207 default=True,
208 doc="Require cosmic ray detection and masking to run successfully before measuring the PSF."
209 )
210 checkUnitsParseStrict = pexConfig.Field(
211 doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
212 dtype=str,
213 default="raise",
214 )
215 doMaskStreaks = pexConfig.Field(
216 doc="Mask streaks",
217 default=True,
218 dtype=bool,
219 )
220 maskStreaks = pexConfig.ConfigurableField(
221 target=MaskStreaksTask,
222 doc="Subtask for masking streaks. Only used if doMaskStreaks is True. "
223 "Adds a mask plane to an exposure, with the mask plane name set by streakMaskName.",
224 )
225 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
227 def setDefaults(self):
228 super().setDefaults()
229 # Just detect bright stars.
230 # The thresholdValue sets the minimum flux in a pixel to be included in the
231 # footprint, while peaks are only detected when they are above
232 # thresholdValue * includeThresholdMultiplier. The low thresholdValue
233 # ensures that the footprints are large enough for the noise replacer
234 # to mask out faint undetected neighbors that are not to be measured.
235 self.detection.thresholdValue = 5.0
236 self.detection.includeThresholdMultiplier = 10.0
237 # do not deblend, as it makes a mess
238 self.doDeblend = False
239 # measure and apply aperture correction; note: measuring and applying aperture
240 # correction are disabled until the final measurement, after PSF is measured
241 self.doApCorr = True
242 # During characterization, we don't have full source measurement information,
243 # so must do the aperture correction with only psf stars, combined with the
244 # default signal-to-noise cuts in MeasureApCorrTask.
245 selector = self.measureApCorr.sourceSelector["science"]
246 selector.doUnresolved = False
247 selector.flags.good = ["calib_psf_used"]
248 selector.flags.bad = []
250 # minimal set of measurements needed to determine PSF
251 self.measurement.plugins.names = [
252 "base_PixelFlags",
253 "base_SdssCentroid",
254 "ext_shapeHSM_HsmSourceMoments",
255 "base_GaussianFlux",
256 "base_PsfFlux",
257 "base_CircularApertureFlux",
258 "base_ClassificationSizeExtendedness",
259 ]
260 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments"
262 def validate(self):
263 if self.doApCorr and not self.measurePsf:
264 raise RuntimeError("Must measure PSF to measure aperture correction, "
265 "because flags determined by PSF measurement are used to identify "
266 "sources used to measure aperture correction")
269class CharacterizeImageTask(pipeBase.PipelineTask):
270 """Measure bright sources and use this to estimate background and PSF of
271 an exposure.
273 Given an exposure with defects repaired (masked and interpolated over,
274 e.g. as output by `~lsst.ip.isr.IsrTask`):
275 - detect and measure bright sources
276 - repair cosmic rays
277 - detect and mask streaks
278 - measure and subtract background
279 - measure PSF
281 Parameters
282 ----------
283 schema : `lsst.afw.table.Schema`, optional
284 Initial schema for icSrc catalog.
285 **kwargs
286 Additional keyword arguments.
288 Notes
289 -----
290 Debugging:
291 CharacterizeImageTask has a debug dictionary with the following keys:
293 frame
294 int: if specified, the frame of first debug image displayed (defaults to 1)
295 repair_iter
296 bool; if True display image after each repair in the measure PSF loop
297 background_iter
298 bool; if True display image after each background subtraction in the measure PSF loop
299 measure_iter
300 bool; if True display image and sources at the end of each iteration of the measure PSF loop
301 See `~lsst.meas.astrom.displayAstrometry` for the meaning of the various symbols.
302 psf
303 bool; if True display image and sources after PSF is measured;
304 this will be identical to the final image displayed by measure_iter if measure_iter is true
305 repair
306 bool; if True display image and sources after final repair
307 measure
308 bool; if True display image and sources after final measurement
309 """
311 ConfigClass = CharacterizeImageConfig
312 _DefaultName = "characterizeImage"
314 def __init__(self, schema=None, **kwargs):
315 super().__init__(**kwargs)
317 if schema is None:
318 schema = SourceTable.makeMinimalSchema()
319 self.schema = schema
320 self.makeSubtask("background")
321 self.makeSubtask("installSimplePsf")
322 self.makeSubtask("repair")
323 if self.config.doMaskStreaks:
324 self.makeSubtask("maskStreaks")
325 self.makeSubtask("measurePsf", schema=self.schema)
326 self.algMetadata = dafBase.PropertyList()
327 self.makeSubtask('detection', schema=self.schema)
328 if self.config.doDeblend:
329 self.makeSubtask("deblend", schema=self.schema)
330 self.makeSubtask('measurement', schema=self.schema, algMetadata=self.algMetadata)
331 if self.config.doApCorr:
332 self.makeSubtask('measureApCorr', schema=self.schema)
333 self.makeSubtask('applyApCorr', schema=self.schema)
334 self.makeSubtask('catalogCalculation', schema=self.schema)
335 self._initialFrame = getDebugFrame(self._display, "frame") or 1
336 self._frame = self._initialFrame
337 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
338 self.outputSchema = afwTable.SourceCatalog(self.schema)
340 def runQuantum(self, butlerQC, inputRefs, outputRefs):
341 inputs = butlerQC.get(inputRefs)
342 if 'idGenerator' not in inputs.keys():
343 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId)
344 outputs = self.run(**inputs)
345 butlerQC.put(outputs, outputRefs)
347 @timeMethod
348 def run(self, exposure, background=None, idGenerator=None):
349 """Characterize a science image.
351 Peforms the following operations:
352 - Iterate the following config.psfIterations times, or once if config.doMeasurePsf false:
353 - detect and measure sources and estimate PSF (see detectMeasureAndEstimatePsf for details)
354 - interpolate over cosmic rays
355 - perform final measurement
357 Parameters
358 ----------
359 exposure : `lsst.afw.image.ExposureF`
360 Exposure to characterize.
361 background : `lsst.afw.math.BackgroundList`, optional
362 Initial model of background already subtracted from exposure.
363 idGenerator : `lsst.meas.base.IdGenerator`, optional
364 Object that generates source IDs and provides RNG seeds.
366 Returns
367 -------
368 result : `lsst.pipe.base.Struct`
369 Results as a struct with attributes:
371 ``exposure``
372 Characterized exposure (`lsst.afw.image.ExposureF`).
373 ``sourceCat``
374 Detected sources (`lsst.afw.table.SourceCatalog`).
375 ``background``
376 Model of subtracted background (`lsst.afw.math.BackgroundList`).
377 ``psfCellSet``
378 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`).
379 ``characterized``
380 Another reference to ``exposure`` for compatibility.
381 ``backgroundModel``
382 Another reference to ``background`` for compatibility.
384 Raises
385 ------
386 RuntimeError
387 Raised if PSF sigma is NaN.
388 """
389 self._frame = self._initialFrame # reset debug display frame
391 if not self.config.doMeasurePsf and not exposure.hasPsf():
392 self.log.info("CharacterizeImageTask initialized with 'simple' PSF.")
393 self.installSimplePsf.run(exposure=exposure)
395 if idGenerator is None:
396 idGenerator = IdGenerator()
398 # subtract an initial estimate of background level
399 background = self.background.run(exposure).background
401 psfIterations = self.config.psfIterations if self.config.doMeasurePsf else 1
402 for i in range(psfIterations):
403 dmeRes = self.detectMeasureAndEstimatePsf(
404 exposure=exposure,
405 idGenerator=idGenerator,
406 background=background,
407 )
409 psf = dmeRes.exposure.getPsf()
410 # Just need a rough estimate; average positions are fine
411 psfAvgPos = psf.getAveragePosition()
412 psfSigma = psf.computeShape(psfAvgPos).getDeterminantRadius()
413 psfDimensions = psf.computeImage(psfAvgPos).getDimensions()
414 medBackground = np.median(dmeRes.background.getImage().getArray())
415 self.log.info("iter %s; PSF sigma=%0.4f, dimensions=%s; median background=%0.2f",
416 i + 1, psfSigma, psfDimensions, medBackground)
417 if np.isnan(psfSigma):
418 raise RuntimeError("PSF sigma is NaN, cannot continue PSF determination.")
420 self.display("psf", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
422 # perform final repair with final PSF
423 self.repair.run(exposure=dmeRes.exposure)
424 self.display("repair", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
426 # mask streaks
427 if self.config.doMaskStreaks:
428 _ = self.maskStreaks.run(dmeRes.exposure)
430 # perform final measurement with final PSF, including measuring and applying aperture correction,
431 # if wanted
432 self.measurement.run(measCat=dmeRes.sourceCat, exposure=dmeRes.exposure,
433 exposureId=idGenerator.catalog_id)
434 if self.config.doApCorr:
435 try:
436 apCorrMap = self.measureApCorr.run(
437 exposure=dmeRes.exposure,
438 catalog=dmeRes.sourceCat,
439 ).apCorrMap
440 except MeasureApCorrError:
441 # We have failed to get a valid aperture correction map.
442 # Proceed with processing, and image will be filtered
443 # downstream.
444 dmeRes.exposure.info.setApCorrMap(None)
445 else:
446 dmeRes.exposure.info.setApCorrMap(apCorrMap)
447 self.applyApCorr.run(catalog=dmeRes.sourceCat, apCorrMap=exposure.getInfo().getApCorrMap())
449 self.catalogCalculation.run(dmeRes.sourceCat)
451 self.display("measure", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
453 return pipeBase.Struct(
454 exposure=dmeRes.exposure,
455 sourceCat=dmeRes.sourceCat,
456 background=dmeRes.background,
457 psfCellSet=dmeRes.psfCellSet,
459 characterized=dmeRes.exposure,
460 backgroundModel=dmeRes.background
461 )
463 @timeMethod
464 def detectMeasureAndEstimatePsf(self, exposure, idGenerator, background):
465 """Perform one iteration of detect, measure, and estimate PSF.
467 Performs the following operations:
469 - if config.doMeasurePsf or not exposure.hasPsf():
471 - install a simple PSF model (replacing the existing one, if need be)
473 - interpolate over cosmic rays with keepCRs=True
474 - estimate background and subtract it from the exposure
475 - detect, deblend and measure sources, and subtract a refined background model;
476 - if config.doMeasurePsf:
477 - measure PSF
479 Parameters
480 ----------
481 exposure : `lsst.afw.image.ExposureF`
482 Exposure to characterize.
483 idGenerator : `lsst.meas.base.IdGenerator`
484 Object that generates source IDs and provides RNG seeds.
485 background : `lsst.afw.math.BackgroundList`, optional
486 Initial model of background already subtracted from exposure.
488 Returns
489 -------
490 result : `lsst.pipe.base.Struct`
491 Results as a struct with attributes:
493 ``exposure``
494 Characterized exposure (`lsst.afw.image.ExposureF`).
495 ``sourceCat``
496 Detected sources (`lsst.afw.table.SourceCatalog`).
497 ``background``
498 Model of subtracted background (`lsst.afw.math.BackgroundList`).
499 ``psfCellSet``
500 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`).
502 Raises
503 ------
504 LengthError
505 Raised if there are too many CR pixels.
506 """
507 # install a simple PSF model, if needed or wanted
508 if not exposure.hasPsf() or (self.config.doMeasurePsf and self.config.useSimplePsf):
509 self.log.info("PSF estimation initialized with 'simple' PSF")
510 self.installSimplePsf.run(exposure=exposure)
512 # run repair, but do not interpolate over cosmic rays (do that elsewhere, with the final PSF model)
513 if self.config.requireCrForPsf:
514 self.repair.run(exposure=exposure, keepCRs=True)
515 else:
516 try:
517 self.repair.run(exposure=exposure, keepCRs=True)
518 except LengthError:
519 self.log.warning("Skipping cosmic ray detection: Too many CR pixels (max %0.f)",
520 self.config.repair.cosmicray.nCrPixelMax)
522 self.display("repair_iter", exposure=exposure)
524 if background is None:
525 background = BackgroundList()
527 sourceIdFactory = idGenerator.make_table_id_factory()
528 table = SourceTable.make(self.schema, sourceIdFactory)
529 table.setMetadata(self.algMetadata)
531 detRes = self.detection.run(table=table, exposure=exposure, doSmooth=True)
532 sourceCat = detRes.sources
533 if detRes.background:
534 for bg in detRes.background:
535 background.append(bg)
537 if self.config.doDeblend:
538 self.deblend.run(exposure=exposure, sources=sourceCat)
539 # We need the output catalog to be contiguous for further processing.
540 if not sourceCat.isContiguous():
541 sourceCat = sourceCat.copy(deep=True)
543 self.measurement.run(measCat=sourceCat, exposure=exposure, exposureId=idGenerator.catalog_id)
545 measPsfRes = pipeBase.Struct(cellSet=None)
546 if self.config.doMeasurePsf:
547 measPsfRes = self.measurePsf.run(exposure=exposure, sources=sourceCat,
548 expId=idGenerator.catalog_id)
549 self.display("measure_iter", exposure=exposure, sourceCat=sourceCat)
551 return pipeBase.Struct(
552 exposure=exposure,
553 sourceCat=sourceCat,
554 background=background,
555 psfCellSet=measPsfRes.cellSet,
556 )
558 def display(self, itemName, exposure, sourceCat=None):
559 """Display exposure and sources on next frame (for debugging).
561 Parameters
562 ----------
563 itemName : `str`
564 Name of item in ``debugInfo``.
565 exposure : `lsst.afw.image.ExposureF`
566 Exposure to display.
567 sourceCat : `lsst.afw.table.SourceCatalog`, optional
568 Catalog of sources detected on the exposure.
569 """
570 val = getDebugFrame(self._display, itemName)
571 if not val:
572 return
574 displayAstrometry(exposure=exposure, sourceCat=sourceCat, frame=self._frame, pause=False)
575 self._frame += 1