Coverage for python/lsst/pipe/tasks/characterizeImage.py: 30%
185 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-17 10:06 +0000
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-17 10:06 +0000
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["CharacterizeImageConfig", "CharacterizeImageTask"]
24import numpy as np
26from lsstDebug import getDebugFrame
27import lsst.afw.table as afwTable
28import lsst.pex.config as pexConfig
29import lsst.pipe.base as pipeBase
30import lsst.daf.base as dafBase
31import lsst.pipe.base.connectionTypes as cT
32from lsst.afw.math import BackgroundList
33from lsst.afw.table import SourceTable
34from lsst.meas.algorithms import (
35 SubtractBackgroundTask,
36 SourceDetectionTask,
37 MeasureApCorrTask,
38 MeasureApCorrError,
39)
40from lsst.meas.algorithms.installGaussianPsf import InstallGaussianPsfTask
41from lsst.meas.astrom import RefMatchTask, displayAstrometry
42from lsst.meas.algorithms import LoadReferenceObjectsConfig
43from lsst.meas.base import (
44 SingleFrameMeasurementTask,
45 ApplyApCorrTask,
46 CatalogCalculationTask,
47 IdGenerator,
48 DetectorVisitIdGeneratorConfig,
49)
50from lsst.meas.deblender import SourceDeblendTask
51import lsst.meas.extensions.shapeHSM # noqa: F401 needed for default shape plugin
52from .measurePsf import MeasurePsfTask
53from .repair import RepairTask
54from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask
55from lsst.pex.exceptions import LengthError
56from lsst.utils.timer import timeMethod
59class CharacterizeImageConnections(pipeBase.PipelineTaskConnections,
60 dimensions=("instrument", "visit", "detector")):
61 exposure = cT.Input(
62 doc="Input exposure data",
63 name="postISRCCD",
64 storageClass="Exposure",
65 dimensions=["instrument", "exposure", "detector"],
66 )
67 characterized = cT.Output(
68 doc="Output characterized data.",
69 name="icExp",
70 storageClass="ExposureF",
71 dimensions=["instrument", "visit", "detector"],
72 )
73 sourceCat = cT.Output(
74 doc="Output source catalog.",
75 name="icSrc",
76 storageClass="SourceCatalog",
77 dimensions=["instrument", "visit", "detector"],
78 )
79 backgroundModel = cT.Output(
80 doc="Output background model.",
81 name="icExpBackground",
82 storageClass="Background",
83 dimensions=["instrument", "visit", "detector"],
84 )
85 outputSchema = cT.InitOutput(
86 doc="Schema of the catalog produced by CharacterizeImage",
87 name="icSrc_schema",
88 storageClass="SourceCatalog",
89 )
91 def adjustQuantum(self, inputs, outputs, label, dataId):
92 # Docstring inherited from PipelineTaskConnections
93 try:
94 return super().adjustQuantum(inputs, outputs, label, dataId)
95 except pipeBase.ScalarError as err:
96 raise pipeBase.ScalarError(
97 "CharacterizeImageTask can at present only be run on visits that are associated with "
98 "exactly one exposure. Either this is not a valid exposure for this pipeline, or the "
99 "snap-combination step you probably want hasn't been configured to run between ISR and "
100 "this task (as of this writing, that would be because it hasn't been implemented yet)."
101 ) from err
104class CharacterizeImageConfig(pipeBase.PipelineTaskConfig,
105 pipelineConnections=CharacterizeImageConnections):
106 """Config for CharacterizeImageTask."""
108 doMeasurePsf = pexConfig.Field(
109 dtype=bool,
110 default=True,
111 doc="Measure PSF? If False then for all subsequent operations use either existing PSF "
112 "model when present, or install simple PSF model when not (see installSimplePsf "
113 "config options)"
114 )
115 doWrite = pexConfig.Field(
116 dtype=bool,
117 default=True,
118 doc="Persist results?",
119 )
120 doWriteExposure = pexConfig.Field(
121 dtype=bool,
122 default=True,
123 doc="Write icExp and icExpBackground in addition to icSrc? Ignored if doWrite False.",
124 )
125 psfIterations = pexConfig.RangeField(
126 dtype=int,
127 default=2,
128 min=1,
129 doc="Number of iterations of detect sources, measure sources, "
130 "estimate PSF. If useSimplePsf is True then 2 should be plenty; "
131 "otherwise more may be wanted.",
132 )
133 background = pexConfig.ConfigurableField(
134 target=SubtractBackgroundTask,
135 doc="Configuration for initial background estimation",
136 )
137 detection = pexConfig.ConfigurableField(
138 target=SourceDetectionTask,
139 doc="Detect sources"
140 )
141 doDeblend = pexConfig.Field(
142 dtype=bool,
143 default=True,
144 doc="Run deblender input exposure"
145 )
146 deblend = pexConfig.ConfigurableField(
147 target=SourceDeblendTask,
148 doc="Split blended source into their components"
149 )
150 measurement = pexConfig.ConfigurableField(
151 target=SingleFrameMeasurementTask,
152 doc="Measure sources"
153 )
154 doApCorr = pexConfig.Field(
155 dtype=bool,
156 default=True,
157 doc="Run subtasks to measure and apply aperture corrections"
158 )
159 measureApCorr = pexConfig.ConfigurableField(
160 target=MeasureApCorrTask,
161 doc="Subtask to measure aperture corrections"
162 )
163 applyApCorr = pexConfig.ConfigurableField(
164 target=ApplyApCorrTask,
165 doc="Subtask to apply aperture corrections"
166 )
167 # If doApCorr is False, and the exposure does not have apcorrections already applied, the
168 # active plugins in catalogCalculation almost certainly should not contain the characterization plugin
169 catalogCalculation = pexConfig.ConfigurableField(
170 target=CatalogCalculationTask,
171 doc="Subtask to run catalogCalculation plugins on catalog"
172 )
173 doComputeSummaryStats = pexConfig.Field(
174 dtype=bool,
175 default=True,
176 doc="Run subtask to measure exposure summary statistics",
177 deprecated=("This subtask has been moved to CalibrateTask "
178 "with DM-30701.")
179 )
180 computeSummaryStats = pexConfig.ConfigurableField(
181 target=ComputeExposureSummaryStatsTask,
182 doc="Subtask to run computeSummaryStats on exposure",
183 deprecated=("This subtask has been moved to CalibrateTask "
184 "with DM-30701.")
185 )
186 useSimplePsf = pexConfig.Field(
187 dtype=bool,
188 default=True,
189 doc="Replace the existing PSF model with a simplified version that has the same sigma "
190 "at the start of each PSF determination iteration? Doing so makes PSF determination "
191 "converge more robustly and quickly.",
192 )
193 installSimplePsf = pexConfig.ConfigurableField(
194 target=InstallGaussianPsfTask,
195 doc="Install a simple PSF model",
196 )
197 refObjLoader = pexConfig.ConfigField(
198 dtype=LoadReferenceObjectsConfig,
199 deprecated="This field does nothing. Will be removed after v24 (see DM-34768).",
200 doc="reference object loader",
201 )
202 ref_match = pexConfig.ConfigurableField(
203 target=RefMatchTask,
204 deprecated="This field was never usable. Will be removed after v24 (see DM-34768).",
205 doc="Task to load and match reference objects. Only used if measurePsf can use matches. "
206 "Warning: matching will only work well if the initial WCS is accurate enough "
207 "to give good matches (roughly: good to 3 arcsec across the CCD).",
208 )
209 measurePsf = pexConfig.ConfigurableField(
210 target=MeasurePsfTask,
211 doc="Measure PSF",
212 )
213 repair = pexConfig.ConfigurableField(
214 target=RepairTask,
215 doc="Remove cosmic rays",
216 )
217 requireCrForPsf = pexConfig.Field(
218 dtype=bool,
219 default=True,
220 doc="Require cosmic ray detection and masking to run successfully before measuring the PSF."
221 )
222 checkUnitsParseStrict = pexConfig.Field(
223 doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
224 dtype=str,
225 default="raise",
226 )
227 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
229 def setDefaults(self):
230 super().setDefaults()
231 # just detect bright stars; includeThresholdMultipler=10 seems large,
232 # but these are the values we have been using
233 self.detection.thresholdValue = 5.0
234 self.detection.includeThresholdMultiplier = 10.0
235 self.detection.doTempLocalBackground = False
236 # do not deblend, as it makes a mess
237 self.doDeblend = False
238 # measure and apply aperture correction; note: measuring and applying aperture
239 # correction are disabled until the final measurement, after PSF is measured
240 self.doApCorr = True
241 # During characterization, we don't have full source measurement information,
242 # so must do the aperture correction with only psf stars, combined with the
243 # default signal-to-noise cuts in MeasureApCorrTask.
244 selector = self.measureApCorr.sourceSelector["science"]
245 selector.doUnresolved = False
246 selector.flags.good = ["calib_psf_used"]
247 selector.flags.bad = []
249 # minimal set of measurements needed to determine PSF
250 self.measurement.plugins.names = [
251 "base_PixelFlags",
252 "base_SdssCentroid",
253 "ext_shapeHSM_HsmSourceMoments",
254 "base_GaussianFlux",
255 "base_PsfFlux",
256 "base_CircularApertureFlux",
257 ]
258 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments"
260 def validate(self):
261 if self.doApCorr and not self.measurePsf:
262 raise RuntimeError("Must measure PSF to measure aperture correction, "
263 "because flags determined by PSF measurement are used to identify "
264 "sources used to measure aperture correction")
267class CharacterizeImageTask(pipeBase.PipelineTask):
268 """Measure bright sources and use this to estimate background and PSF of
269 an exposure.
271 Given an exposure with defects repaired (masked and interpolated over,
272 e.g. as output by `~lsst.ip.isr.IsrTask`):
273 - detect and measure bright sources
274 - repair cosmic rays
275 - measure and subtract background
276 - measure PSF
278 Parameters
279 ----------
280 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
281 Reference object loader if using a catalog-based star-selector.
282 schema : `lsst.afw.table.Schema`, optional
283 Initial schema for icSrc catalog.
284 **kwargs
285 Additional keyword arguments.
287 Notes
288 -----
289 Debugging:
290 CharacterizeImageTask has a debug dictionary with the following keys:
292 frame
293 int: if specified, the frame of first debug image displayed (defaults to 1)
294 repair_iter
295 bool; if True display image after each repair in the measure PSF loop
296 background_iter
297 bool; if True display image after each background subtraction in the measure PSF loop
298 measure_iter
299 bool; if True display image and sources at the end of each iteration of the measure PSF loop
300 See `~lsst.meas.astrom.displayAstrometry` for the meaning of the various symbols.
301 psf
302 bool; if True display image and sources after PSF is measured;
303 this will be identical to the final image displayed by measure_iter if measure_iter is true
304 repair
305 bool; if True display image and sources after final repair
306 measure
307 bool; if True display image and sources after final measurement
308 """
310 ConfigClass = CharacterizeImageConfig
311 _DefaultName = "characterizeImage"
313 def __init__(self, refObjLoader=None, schema=None, **kwargs):
314 super().__init__(**kwargs)
316 if schema is None:
317 schema = SourceTable.makeMinimalSchema()
318 self.schema = schema
319 self.makeSubtask("background")
320 self.makeSubtask("installSimplePsf")
321 self.makeSubtask("repair")
322 self.makeSubtask("measurePsf", schema=self.schema)
323 # TODO DM-34769: remove this `if` block
324 if self.config.doMeasurePsf and self.measurePsf.usesMatches:
325 self.makeSubtask("ref_match", refObjLoader=refObjLoader)
326 self.algMetadata = dafBase.PropertyList()
327 self.makeSubtask('detection', schema=self.schema)
328 if self.config.doDeblend:
329 self.makeSubtask("deblend", schema=self.schema)
330 self.makeSubtask('measurement', schema=self.schema, algMetadata=self.algMetadata)
331 if self.config.doApCorr:
332 self.makeSubtask('measureApCorr', schema=self.schema)
333 self.makeSubtask('applyApCorr', schema=self.schema)
334 self.makeSubtask('catalogCalculation', schema=self.schema)
335 self._initialFrame = getDebugFrame(self._display, "frame") or 1
336 self._frame = self._initialFrame
337 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
338 self.outputSchema = afwTable.SourceCatalog(self.schema)
340 def runQuantum(self, butlerQC, inputRefs, outputRefs):
341 inputs = butlerQC.get(inputRefs)
342 if 'idGenerator' not in inputs.keys():
343 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId)
344 outputs = self.run(**inputs)
345 butlerQC.put(outputs, outputRefs)
347 @timeMethod
348 def run(self, exposure, exposureIdInfo=None, background=None, idGenerator=None):
349 """Characterize a science image.
351 Peforms the following operations:
352 - Iterate the following config.psfIterations times, or once if config.doMeasurePsf false:
353 - detect and measure sources and estimate PSF (see detectMeasureAndEstimatePsf for details)
354 - interpolate over cosmic rays
355 - perform final measurement
357 Parameters
358 ----------
359 exposure : `lsst.afw.image.ExposureF`
360 Exposure to characterize.
361 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`, optional
362 Exposure ID info. Deprecated in favor of ``idGenerator``, and
363 ignored if that is provided.
364 background : `lsst.afw.math.BackgroundList`, optional
365 Initial model of background already subtracted from exposure.
366 idGenerator : `lsst.meas.base.IdGenerator`, optional
367 Object that generates source IDs and provides RNG seeds.
369 Returns
370 -------
371 result : `lsst.pipe.base.Struct`
372 Results as a struct with attributes:
374 ``exposure``
375 Characterized exposure (`lsst.afw.image.ExposureF`).
376 ``sourceCat``
377 Detected sources (`lsst.afw.table.SourceCatalog`).
378 ``background``
379 Model of subtracted background (`lsst.afw.math.BackgroundList`).
380 ``psfCellSet``
381 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`).
382 ``characterized``
383 Another reference to ``exposure`` for compatibility.
384 ``backgroundModel``
385 Another reference to ``background`` for compatibility.
387 Raises
388 ------
389 RuntimeError
390 Raised if PSF sigma is NaN.
391 """
392 self._frame = self._initialFrame # reset debug display frame
394 if not self.config.doMeasurePsf and not exposure.hasPsf():
395 self.log.info("CharacterizeImageTask initialized with 'simple' PSF.")
396 self.installSimplePsf.run(exposure=exposure)
398 if idGenerator is None:
399 if exposureIdInfo is not None:
400 idGenerator = IdGenerator._from_exposure_id_info(exposureIdInfo)
401 else:
402 idGenerator = IdGenerator()
404 del exposureIdInfo
406 # subtract an initial estimate of background level
407 background = self.background.run(exposure).background
409 psfIterations = self.config.psfIterations if self.config.doMeasurePsf else 1
410 for i in range(psfIterations):
411 dmeRes = self.detectMeasureAndEstimatePsf(
412 exposure=exposure,
413 idGenerator=idGenerator,
414 background=background,
415 )
417 psf = dmeRes.exposure.getPsf()
418 # Just need a rough estimate; average positions are fine
419 psfAvgPos = psf.getAveragePosition()
420 psfSigma = psf.computeShape(psfAvgPos).getDeterminantRadius()
421 psfDimensions = psf.computeImage(psfAvgPos).getDimensions()
422 medBackground = np.median(dmeRes.background.getImage().getArray())
423 self.log.info("iter %s; PSF sigma=%0.2f, dimensions=%s; median background=%0.2f",
424 i + 1, psfSigma, psfDimensions, medBackground)
425 if np.isnan(psfSigma):
426 raise RuntimeError("PSF sigma is NaN, cannot continue PSF determination.")
428 self.display("psf", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
430 # perform final repair with final PSF
431 self.repair.run(exposure=dmeRes.exposure)
432 self.display("repair", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
434 # perform final measurement with final PSF, including measuring and applying aperture correction,
435 # if wanted
436 self.measurement.run(measCat=dmeRes.sourceCat, exposure=dmeRes.exposure,
437 exposureId=idGenerator.catalog_id)
438 if self.config.doApCorr:
439 try:
440 apCorrMap = self.measureApCorr.run(
441 exposure=dmeRes.exposure,
442 catalog=dmeRes.sourceCat,
443 ).apCorrMap
444 except MeasureApCorrError:
445 # We have failed to get a valid aperture correction map.
446 # Proceed with processing, and image will be filtered
447 # downstream.
448 dmeRes.exposure.info.setApCorrMap(None)
449 else:
450 dmeRes.exposure.info.setApCorrMap(apCorrMap)
451 self.applyApCorr.run(catalog=dmeRes.sourceCat, apCorrMap=exposure.getInfo().getApCorrMap())
453 self.catalogCalculation.run(dmeRes.sourceCat)
455 self.display("measure", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
457 return pipeBase.Struct(
458 exposure=dmeRes.exposure,
459 sourceCat=dmeRes.sourceCat,
460 background=dmeRes.background,
461 psfCellSet=dmeRes.psfCellSet,
463 characterized=dmeRes.exposure,
464 backgroundModel=dmeRes.background
465 )
467 @timeMethod
468 def detectMeasureAndEstimatePsf(self, exposure, idGenerator, background):
469 """Perform one iteration of detect, measure, and estimate PSF.
471 Performs the following operations:
473 - if config.doMeasurePsf or not exposure.hasPsf():
475 - install a simple PSF model (replacing the existing one, if need be)
477 - interpolate over cosmic rays with keepCRs=True
478 - estimate background and subtract it from the exposure
479 - detect, deblend and measure sources, and subtract a refined background model;
480 - if config.doMeasurePsf:
481 - measure PSF
483 Parameters
484 ----------
485 exposure : `lsst.afw.image.ExposureF`
486 Exposure to characterize.
487 idGenerator : `lsst.meas.base.IdGenerator`
488 Object that generates source IDs and provides RNG seeds.
489 background : `lsst.afw.math.BackgroundList`, optional
490 Initial model of background already subtracted from exposure.
492 Returns
493 -------
494 result : `lsst.pipe.base.Struct`
495 Results as a struct with attributes:
497 ``exposure``
498 Characterized exposure (`lsst.afw.image.ExposureF`).
499 ``sourceCat``
500 Detected sources (`lsst.afw.table.SourceCatalog`).
501 ``background``
502 Model of subtracted background (`lsst.afw.math.BackgroundList`).
503 ``psfCellSet``
504 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`).
506 Raises
507 ------
508 LengthError
509 Raised if there are too many CR pixels.
510 """
511 # install a simple PSF model, if needed or wanted
512 if not exposure.hasPsf() or (self.config.doMeasurePsf and self.config.useSimplePsf):
513 self.log.info("PSF estimation initialized with 'simple' PSF")
514 self.installSimplePsf.run(exposure=exposure)
516 # run repair, but do not interpolate over cosmic rays (do that elsewhere, with the final PSF model)
517 if self.config.requireCrForPsf:
518 self.repair.run(exposure=exposure, keepCRs=True)
519 else:
520 try:
521 self.repair.run(exposure=exposure, keepCRs=True)
522 except LengthError:
523 self.log.warning("Skipping cosmic ray detection: Too many CR pixels (max %0.f)",
524 self.config.repair.cosmicray.nCrPixelMax)
526 self.display("repair_iter", exposure=exposure)
528 if background is None:
529 background = BackgroundList()
531 sourceIdFactory = idGenerator.make_table_id_factory()
532 table = SourceTable.make(self.schema, sourceIdFactory)
533 table.setMetadata(self.algMetadata)
535 detRes = self.detection.run(table=table, exposure=exposure, doSmooth=True)
536 sourceCat = detRes.sources
537 if detRes.background:
538 for bg in detRes.background:
539 background.append(bg)
541 if self.config.doDeblend:
542 self.deblend.run(exposure=exposure, sources=sourceCat)
543 # We need the output catalog to be contiguous for further processing.
544 if not sourceCat.isContiguous():
545 sourceCat = sourceCat.copy(deep=True)
547 self.measurement.run(measCat=sourceCat, exposure=exposure, exposureId=idGenerator.catalog_id)
549 measPsfRes = pipeBase.Struct(cellSet=None)
550 if self.config.doMeasurePsf:
551 # TODO DM-34769: remove this `if` block, and the `matches` kwarg from measurePsf.run below.
552 if self.measurePsf.usesMatches:
553 matches = self.ref_match.loadAndMatch(exposure=exposure, sourceCat=sourceCat).matches
554 else:
555 matches = None
556 measPsfRes = self.measurePsf.run(exposure=exposure, sources=sourceCat, matches=matches,
557 expId=idGenerator.catalog_id)
558 self.display("measure_iter", exposure=exposure, sourceCat=sourceCat)
560 return pipeBase.Struct(
561 exposure=exposure,
562 sourceCat=sourceCat,
563 background=background,
564 psfCellSet=measPsfRes.cellSet,
565 )
567 def display(self, itemName, exposure, sourceCat=None):
568 """Display exposure and sources on next frame (for debugging).
570 Parameters
571 ----------
572 itemName : `str`
573 Name of item in ``debugInfo``.
574 exposure : `lsst.afw.image.ExposureF`
575 Exposure to display.
576 sourceCat : `lsst.afw.table.SourceCatalog`, optional
577 Catalog of sources detected on the exposure.
578 """
579 val = getDebugFrame(self._display, itemName)
580 if not val:
581 return
583 displayAstrometry(exposure=exposure, sourceCat=sourceCat, frame=self._frame, pause=False)
584 self._frame += 1