Coverage for python/lsst/pipe/tasks/characterizeImage.py: 30%
181 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-09 04:16 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-09 04:16 -0800
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["CharacterizeImageConfig", "CharacterizeImageTask"]
24import numpy as np
25import warnings
27from lsstDebug import getDebugFrame
28import lsst.afw.table as afwTable
29import lsst.pex.config as pexConfig
30import lsst.pipe.base as pipeBase
31import lsst.daf.base as dafBase
32import lsst.pipe.base.connectionTypes as cT
33from lsst.afw.math import BackgroundList
34from lsst.afw.table import SourceTable
35from lsst.meas.algorithms import SubtractBackgroundTask, SourceDetectionTask, MeasureApCorrTask
36from lsst.meas.algorithms.installGaussianPsf import InstallGaussianPsfTask
37from lsst.meas.astrom import RefMatchTask, displayAstrometry
38from lsst.meas.algorithms import LoadReferenceObjectsConfig
39from lsst.obs.base import ExposureIdInfo
40from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
41from lsst.meas.deblender import SourceDeblendTask
42import lsst.meas.extensions.shapeHSM # noqa: F401 needed for default shape plugin
43from .measurePsf import MeasurePsfTask
44from .repair import RepairTask
45from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask
46from lsst.pex.exceptions import LengthError
47from lsst.utils.timer import timeMethod
50class CharacterizeImageConnections(pipeBase.PipelineTaskConnections,
51 dimensions=("instrument", "visit", "detector")):
52 exposure = cT.Input(
53 doc="Input exposure data",
54 name="postISRCCD",
55 storageClass="Exposure",
56 dimensions=["instrument", "exposure", "detector"],
57 )
58 characterized = cT.Output(
59 doc="Output characterized data.",
60 name="icExp",
61 storageClass="ExposureF",
62 dimensions=["instrument", "visit", "detector"],
63 )
64 sourceCat = cT.Output(
65 doc="Output source catalog.",
66 name="icSrc",
67 storageClass="SourceCatalog",
68 dimensions=["instrument", "visit", "detector"],
69 )
70 backgroundModel = cT.Output(
71 doc="Output background model.",
72 name="icExpBackground",
73 storageClass="Background",
74 dimensions=["instrument", "visit", "detector"],
75 )
76 outputSchema = cT.InitOutput(
77 doc="Schema of the catalog produced by CharacterizeImage",
78 name="icSrc_schema",
79 storageClass="SourceCatalog",
80 )
82 def adjustQuantum(self, inputs, outputs, label, dataId):
83 # Docstring inherited from PipelineTaskConnections
84 try:
85 return super().adjustQuantum(inputs, outputs, label, dataId)
86 except pipeBase.ScalarError as err:
87 raise pipeBase.ScalarError(
88 "CharacterizeImageTask can at present only be run on visits that are associated with "
89 "exactly one exposure. Either this is not a valid exposure for this pipeline, or the "
90 "snap-combination step you probably want hasn't been configured to run between ISR and "
91 "this task (as of this writing, that would be because it hasn't been implemented yet)."
92 ) from err
95class CharacterizeImageConfig(pipeBase.PipelineTaskConfig,
96 pipelineConnections=CharacterizeImageConnections):
97 """Config for CharacterizeImageTask."""
99 doMeasurePsf = pexConfig.Field(
100 dtype=bool,
101 default=True,
102 doc="Measure PSF? If False then for all subsequent operations use either existing PSF "
103 "model when present, or install simple PSF model when not (see installSimplePsf "
104 "config options)"
105 )
106 doWrite = pexConfig.Field(
107 dtype=bool,
108 default=True,
109 doc="Persist results?",
110 )
111 doWriteExposure = pexConfig.Field(
112 dtype=bool,
113 default=True,
114 doc="Write icExp and icExpBackground in addition to icSrc? Ignored if doWrite False.",
115 )
116 psfIterations = pexConfig.RangeField(
117 dtype=int,
118 default=2,
119 min=1,
120 doc="Number of iterations of detect sources, measure sources, "
121 "estimate PSF. If useSimplePsf is True then 2 should be plenty; "
122 "otherwise more may be wanted.",
123 )
124 background = pexConfig.ConfigurableField(
125 target=SubtractBackgroundTask,
126 doc="Configuration for initial background estimation",
127 )
128 detection = pexConfig.ConfigurableField(
129 target=SourceDetectionTask,
130 doc="Detect sources"
131 )
132 doDeblend = pexConfig.Field(
133 dtype=bool,
134 default=True,
135 doc="Run deblender input exposure"
136 )
137 deblend = pexConfig.ConfigurableField(
138 target=SourceDeblendTask,
139 doc="Split blended source into their components"
140 )
141 measurement = pexConfig.ConfigurableField(
142 target=SingleFrameMeasurementTask,
143 doc="Measure sources"
144 )
145 doApCorr = pexConfig.Field(
146 dtype=bool,
147 default=True,
148 doc="Run subtasks to measure and apply aperture corrections"
149 )
150 measureApCorr = pexConfig.ConfigurableField(
151 target=MeasureApCorrTask,
152 doc="Subtask to measure aperture corrections"
153 )
154 applyApCorr = pexConfig.ConfigurableField(
155 target=ApplyApCorrTask,
156 doc="Subtask to apply aperture corrections"
157 )
158 # If doApCorr is False, and the exposure does not have apcorrections already applied, the
159 # active plugins in catalogCalculation almost certainly should not contain the characterization plugin
160 catalogCalculation = pexConfig.ConfigurableField(
161 target=CatalogCalculationTask,
162 doc="Subtask to run catalogCalculation plugins on catalog"
163 )
164 doComputeSummaryStats = pexConfig.Field(
165 dtype=bool,
166 default=True,
167 doc="Run subtask to measure exposure summary statistics",
168 deprecated=("This subtask has been moved to CalibrateTask "
169 "with DM-30701.")
170 )
171 computeSummaryStats = pexConfig.ConfigurableField(
172 target=ComputeExposureSummaryStatsTask,
173 doc="Subtask to run computeSummaryStats on exposure",
174 deprecated=("This subtask has been moved to CalibrateTask "
175 "with DM-30701.")
176 )
177 useSimplePsf = pexConfig.Field(
178 dtype=bool,
179 default=True,
180 doc="Replace the existing PSF model with a simplified version that has the same sigma "
181 "at the start of each PSF determination iteration? Doing so makes PSF determination "
182 "converge more robustly and quickly.",
183 )
184 installSimplePsf = pexConfig.ConfigurableField(
185 target=InstallGaussianPsfTask,
186 doc="Install a simple PSF model",
187 )
188 refObjLoader = pexConfig.ConfigField(
189 dtype=LoadReferenceObjectsConfig,
190 deprecated="This field does nothing. Will be removed after v24 (see DM-34768).",
191 doc="reference object loader",
192 )
193 ref_match = pexConfig.ConfigurableField(
194 target=RefMatchTask,
195 deprecated="This field was never usable. Will be removed after v24 (see DM-34768).",
196 doc="Task to load and match reference objects. Only used if measurePsf can use matches. "
197 "Warning: matching will only work well if the initial WCS is accurate enough "
198 "to give good matches (roughly: good to 3 arcsec across the CCD).",
199 )
200 measurePsf = pexConfig.ConfigurableField(
201 target=MeasurePsfTask,
202 doc="Measure PSF",
203 )
204 repair = pexConfig.ConfigurableField(
205 target=RepairTask,
206 doc="Remove cosmic rays",
207 )
208 requireCrForPsf = pexConfig.Field(
209 dtype=bool,
210 default=True,
211 doc="Require cosmic ray detection and masking to run successfully before measuring the PSF."
212 )
213 checkUnitsParseStrict = pexConfig.Field(
214 doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
215 dtype=str,
216 default="raise",
217 )
219 def setDefaults(self):
220 super().setDefaults()
221 # just detect bright stars; includeThresholdMultipler=10 seems large,
222 # but these are the values we have been using
223 self.detection.thresholdValue = 5.0
224 self.detection.includeThresholdMultiplier = 10.0
225 self.detection.doTempLocalBackground = False
226 # do not deblend, as it makes a mess
227 self.doDeblend = False
228 # measure and apply aperture correction; note: measuring and applying aperture
229 # correction are disabled until the final measurement, after PSF is measured
230 self.doApCorr = True
231 # During characterization, we don't have full source measurement information,
232 # so must do the aperture correction with only psf stars, combined with the
233 # default signal-to-noise cuts in MeasureApCorrTask.
234 selector = self.measureApCorr.sourceSelector["science"]
235 selector.doUnresolved = False
236 selector.flags.good = ["calib_psf_used"]
237 selector.flags.bad = []
239 # minimal set of measurements needed to determine PSF
240 self.measurement.plugins.names = [
241 "base_PixelFlags",
242 "base_SdssCentroid",
243 "ext_shapeHSM_HsmSourceMoments",
244 "base_GaussianFlux",
245 "base_PsfFlux",
246 "base_CircularApertureFlux",
247 ]
248 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments"
250 def validate(self):
251 if self.doApCorr and not self.measurePsf:
252 raise RuntimeError("Must measure PSF to measure aperture correction, "
253 "because flags determined by PSF measurement are used to identify "
254 "sources used to measure aperture correction")
257class CharacterizeImageTask(pipeBase.PipelineTask):
258 """Measure bright sources and use this to estimate background and PSF of
259 an exposure.
261 Given an exposure with defects repaired (masked and interpolated over,
262 e.g. as output by `~lsst.ip.isr.IsrTask`):
263 - detect and measure bright sources
264 - repair cosmic rays
265 - measure and subtract background
266 - measure PSF
268 Parameters
269 ----------
270 butler : `None`
271 Compatibility parameter. Should always be `None`.
272 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
273 Reference object loader if using a catalog-based star-selector.
274 schema : `lsst.afw.table.Schema`, optional
275 Initial schema for icSrc catalog.
276 **kwargs
277 Additional keyword arguments.
279 Notes
280 -----
281 Debugging:
282 CharacterizeImageTask has a debug dictionary with the following keys:
284 frame
285 int: if specified, the frame of first debug image displayed (defaults to 1)
286 repair_iter
287 bool; if True display image after each repair in the measure PSF loop
288 background_iter
289 bool; if True display image after each background subtraction in the measure PSF loop
290 measure_iter
291 bool; if True display image and sources at the end of each iteration of the measure PSF loop
292 See `~lsst.meas.astrom.displayAstrometry` for the meaning of the various symbols.
293 psf
294 bool; if True display image and sources after PSF is measured;
295 this will be identical to the final image displayed by measure_iter if measure_iter is true
296 repair
297 bool; if True display image and sources after final repair
298 measure
299 bool; if True display image and sources after final measurement
300 """
302 ConfigClass = CharacterizeImageConfig
303 _DefaultName = "characterizeImage"
305 def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs):
306 super().__init__(**kwargs)
308 if butler is not None:
309 warnings.warn("The 'butler' parameter is no longer used and can be safely removed.",
310 category=FutureWarning, stacklevel=2)
311 butler = None
313 if schema is None:
314 schema = SourceTable.makeMinimalSchema()
315 self.schema = schema
316 self.makeSubtask("background")
317 self.makeSubtask("installSimplePsf")
318 self.makeSubtask("repair")
319 self.makeSubtask("measurePsf", schema=self.schema)
320 # TODO DM-34769: remove this `if` block
321 if self.config.doMeasurePsf and self.measurePsf.usesMatches:
322 self.makeSubtask("ref_match", refObjLoader=refObjLoader)
323 self.algMetadata = dafBase.PropertyList()
324 self.makeSubtask('detection', schema=self.schema)
325 if self.config.doDeblend:
326 self.makeSubtask("deblend", schema=self.schema)
327 self.makeSubtask('measurement', schema=self.schema, algMetadata=self.algMetadata)
328 if self.config.doApCorr:
329 self.makeSubtask('measureApCorr', schema=self.schema)
330 self.makeSubtask('applyApCorr', schema=self.schema)
331 self.makeSubtask('catalogCalculation', schema=self.schema)
332 self._initialFrame = getDebugFrame(self._display, "frame") or 1
333 self._frame = self._initialFrame
334 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
335 self.outputSchema = afwTable.SourceCatalog(self.schema)
337 def runQuantum(self, butlerQC, inputRefs, outputRefs):
338 inputs = butlerQC.get(inputRefs)
339 if 'exposureIdInfo' not in inputs.keys():
340 inputs['exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "visit_detector")
341 outputs = self.run(**inputs)
342 butlerQC.put(outputs, outputRefs)
344 @timeMethod
345 def run(self, exposure, exposureIdInfo=None, background=None):
346 """Characterize a science image.
348 Peforms the following operations:
349 - Iterate the following config.psfIterations times, or once if config.doMeasurePsf false:
350 - detect and measure sources and estimate PSF (see detectMeasureAndEstimatePsf for details)
351 - interpolate over cosmic rays
352 - perform final measurement
354 Parameters
355 ----------
356 exposure : `lsst.afw.image.ExposureF`
357 Exposure to characterize.
358 exposureIdInfo : `lsst.obs.baseExposureIdInfo`, optional
359 Exposure ID info. If not provided, returned SourceCatalog IDs will not
360 be globally unique.
361 background : `lsst.afw.math.BackgroundList`, optional
362 Initial model of background already subtracted from exposure.
364 Returns
365 -------
366 result : `lsst.pipe.base.Struct`
367 Results as a struct with attributes:
369 ``exposure``
370 Characterized exposure (`lsst.afw.image.ExposureF`).
371 ``sourceCat``
372 Detected sources (`lsst.afw.table.SourceCatalog`).
373 ``background``
374 Model of subtracted background (`lsst.afw.math.BackgroundList`).
375 ``psfCellSet``
376 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`).
377 ``characterized``
378 Another reference to ``exposure`` for compatibility.
379 ``backgroundModel``
380 Another reference to ``background`` for compatibility.
382 Raises
383 ------
384 RuntimeError
385 Raised if PSF sigma is NaN.
386 """
387 self._frame = self._initialFrame # reset debug display frame
389 if not self.config.doMeasurePsf and not exposure.hasPsf():
390 self.log.info("CharacterizeImageTask initialized with 'simple' PSF.")
391 self.installSimplePsf.run(exposure=exposure)
393 if exposureIdInfo is None:
394 exposureIdInfo = ExposureIdInfo()
396 # subtract an initial estimate of background level
397 background = self.background.run(exposure).background
399 psfIterations = self.config.psfIterations if self.config.doMeasurePsf else 1
400 for i in range(psfIterations):
401 dmeRes = self.detectMeasureAndEstimatePsf(
402 exposure=exposure,
403 exposureIdInfo=exposureIdInfo,
404 background=background,
405 )
407 psf = dmeRes.exposure.getPsf()
408 # Just need a rough estimate; average positions are fine
409 psfAvgPos = psf.getAveragePosition()
410 psfSigma = psf.computeShape(psfAvgPos).getDeterminantRadius()
411 psfDimensions = psf.computeImage(psfAvgPos).getDimensions()
412 medBackground = np.median(dmeRes.background.getImage().getArray())
413 self.log.info("iter %s; PSF sigma=%0.2f, dimensions=%s; median background=%0.2f",
414 i + 1, psfSigma, psfDimensions, medBackground)
415 if np.isnan(psfSigma):
416 raise RuntimeError("PSF sigma is NaN, cannot continue PSF determination.")
418 self.display("psf", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
420 # perform final repair with final PSF
421 self.repair.run(exposure=dmeRes.exposure)
422 self.display("repair", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
424 # perform final measurement with final PSF, including measuring and applying aperture correction,
425 # if wanted
426 self.measurement.run(measCat=dmeRes.sourceCat, exposure=dmeRes.exposure,
427 exposureId=exposureIdInfo.expId)
428 if self.config.doApCorr:
429 apCorrMap = self.measureApCorr.run(exposure=dmeRes.exposure, catalog=dmeRes.sourceCat).apCorrMap
430 dmeRes.exposure.getInfo().setApCorrMap(apCorrMap)
431 self.applyApCorr.run(catalog=dmeRes.sourceCat, apCorrMap=exposure.getInfo().getApCorrMap())
432 self.catalogCalculation.run(dmeRes.sourceCat)
434 self.display("measure", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
436 return pipeBase.Struct(
437 exposure=dmeRes.exposure,
438 sourceCat=dmeRes.sourceCat,
439 background=dmeRes.background,
440 psfCellSet=dmeRes.psfCellSet,
442 characterized=dmeRes.exposure,
443 backgroundModel=dmeRes.background
444 )
446 @timeMethod
447 def detectMeasureAndEstimatePsf(self, exposure, exposureIdInfo, background):
448 """Perform one iteration of detect, measure, and estimate PSF.
450 Performs the following operations:
452 - if config.doMeasurePsf or not exposure.hasPsf():
454 - install a simple PSF model (replacing the existing one, if need be)
456 - interpolate over cosmic rays with keepCRs=True
457 - estimate background and subtract it from the exposure
458 - detect, deblend and measure sources, and subtract a refined background model;
459 - if config.doMeasurePsf:
460 - measure PSF
462 Parameters
463 ----------
464 exposure : `lsst.afw.image.ExposureF`
465 Exposure to characterize.
466 exposureIdInfo : `lsst.obs.baseExposureIdInfo`
467 Exposure ID info.
468 background : `lsst.afw.math.BackgroundList`, optional
469 Initial model of background already subtracted from exposure.
471 Returns
472 -------
473 result : `lsst.pipe.base.Struct`
474 Results as a struct with attributes:
476 ``exposure``
477 Characterized exposure (`lsst.afw.image.ExposureF`).
478 ``sourceCat``
479 Detected sources (`lsst.afw.table.SourceCatalog`).
480 ``background``
481 Model of subtracted background (`lsst.afw.math.BackgroundList`).
482 ``psfCellSet``
483 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`).
485 Raises
486 ------
487 LengthError
488 Raised if there are too many CR pixels.
489 """
490 # install a simple PSF model, if needed or wanted
491 if not exposure.hasPsf() or (self.config.doMeasurePsf and self.config.useSimplePsf):
492 self.log.info("PSF estimation initialized with 'simple' PSF")
493 self.installSimplePsf.run(exposure=exposure)
495 # run repair, but do not interpolate over cosmic rays (do that elsewhere, with the final PSF model)
496 if self.config.requireCrForPsf:
497 self.repair.run(exposure=exposure, keepCRs=True)
498 else:
499 try:
500 self.repair.run(exposure=exposure, keepCRs=True)
501 except LengthError:
502 self.log.warning("Skipping cosmic ray detection: Too many CR pixels (max %0.f)",
503 self.config.repair.cosmicray.nCrPixelMax)
505 self.display("repair_iter", exposure=exposure)
507 if background is None:
508 background = BackgroundList()
510 sourceIdFactory = exposureIdInfo.makeSourceIdFactory()
511 table = SourceTable.make(self.schema, sourceIdFactory)
512 table.setMetadata(self.algMetadata)
514 detRes = self.detection.run(table=table, exposure=exposure, doSmooth=True)
515 sourceCat = detRes.sources
516 if detRes.background:
517 for bg in detRes.background:
518 background.append(bg)
520 if self.config.doDeblend:
521 self.deblend.run(exposure=exposure, sources=sourceCat)
523 self.measurement.run(measCat=sourceCat, exposure=exposure, exposureId=exposureIdInfo.expId)
525 measPsfRes = pipeBase.Struct(cellSet=None)
526 if self.config.doMeasurePsf:
527 # TODO DM-34769: remove this `if` block, and the `matches` kwarg from measurePsf.run below.
528 if self.measurePsf.usesMatches:
529 matches = self.ref_match.loadAndMatch(exposure=exposure, sourceCat=sourceCat).matches
530 else:
531 matches = None
532 measPsfRes = self.measurePsf.run(exposure=exposure, sources=sourceCat, matches=matches,
533 expId=exposureIdInfo.expId)
534 self.display("measure_iter", exposure=exposure, sourceCat=sourceCat)
536 return pipeBase.Struct(
537 exposure=exposure,
538 sourceCat=sourceCat,
539 background=background,
540 psfCellSet=measPsfRes.cellSet,
541 )
543 def display(self, itemName, exposure, sourceCat=None):
544 """Display exposure and sources on next frame (for debugging).
546 Parameters
547 ----------
548 itemName : `str`
549 Name of item in ``debugInfo``.
550 exposure : `lsst.afw.image.ExposureF`
551 Exposure to display.
552 sourceCat : `lsst.afw.table.SourceCatalog`, optional
553 Catalog of sources detected on the exposure.
554 """
555 val = getDebugFrame(self._display, itemName)
556 if not val:
557 return
559 displayAstrometry(exposure=exposure, sourceCat=sourceCat, frame=self._frame, pause=False)
560 self._frame += 1