lsst.pipe.tasks gcd254cd47f+c028490dc5
Loading...
Searching...
No Matches
characterizeImage.py
Go to the documentation of this file.
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22__all__ = ["CharacterizeImageConfig", "CharacterizeImageTask"]
23
24import numpy as np
25
26from lsstDebug import getDebugFrame
27import lsst.afw.table as afwTable
28import lsst.pex.config as pexConfig
29import lsst.pipe.base as pipeBase
30import lsst.daf.base as dafBase
31import lsst.pipe.base.connectionTypes as cT
32from lsst.afw.math import BackgroundList
33from lsst.afw.table import SourceTable
34from lsst.meas.algorithms import (
35 SubtractBackgroundTask,
36 SourceDetectionTask,
37 MeasureApCorrTask,
38 MeasureApCorrError,
39)
40from lsst.meas.algorithms.installGaussianPsf import InstallGaussianPsfTask
41from lsst.meas.astrom import RefMatchTask, displayAstrometry
42from lsst.meas.algorithms import LoadReferenceObjectsConfig
43from lsst.meas.base import (
44 SingleFrameMeasurementTask,
45 ApplyApCorrTask,
46 CatalogCalculationTask,
47 IdGenerator,
48 DetectorVisitIdGeneratorConfig,
49)
50from lsst.meas.deblender import SourceDeblendTask
51import lsst.meas.extensions.shapeHSM # noqa: F401 needed for default shape plugin
52from .measurePsf import MeasurePsfTask
53from .repair import RepairTask
54from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask
55from lsst.pex.exceptions import LengthError
56from lsst.utils.timer import timeMethod
57
58
59class CharacterizeImageConnections(pipeBase.PipelineTaskConnections,
60 dimensions=("instrument", "visit", "detector")):
61 exposure = cT.Input(
62 doc="Input exposure data",
63 name="postISRCCD",
64 storageClass="Exposure",
65 dimensions=["instrument", "exposure", "detector"],
66 )
67 characterized = cT.Output(
68 doc="Output characterized data.",
69 name="icExp",
70 storageClass="ExposureF",
71 dimensions=["instrument", "visit", "detector"],
72 )
73 sourceCat = cT.Output(
74 doc="Output source catalog.",
75 name="icSrc",
76 storageClass="SourceCatalog",
77 dimensions=["instrument", "visit", "detector"],
78 )
79 backgroundModel = cT.Output(
80 doc="Output background model.",
81 name="icExpBackground",
82 storageClass="Background",
83 dimensions=["instrument", "visit", "detector"],
84 )
85 outputSchema = cT.InitOutput(
86 doc="Schema of the catalog produced by CharacterizeImage",
87 name="icSrc_schema",
88 storageClass="SourceCatalog",
89 )
90
91 def adjustQuantum(self, inputs, outputs, label, dataId):
92 # Docstring inherited from PipelineTaskConnections
93 try:
94 return super().adjustQuantum(inputs, outputs, label, dataId)
95 except pipeBase.ScalarError as err:
96 raise pipeBase.ScalarError(
97 "CharacterizeImageTask can at present only be run on visits that are associated with "
98 "exactly one exposure. Either this is not a valid exposure for this pipeline, or the "
99 "snap-combination step you probably want hasn't been configured to run between ISR and "
100 "this task (as of this writing, that would be because it hasn't been implemented yet)."
101 ) from err
102
103
104class CharacterizeImageConfig(pipeBase.PipelineTaskConfig,
105 pipelineConnections=CharacterizeImageConnections):
106 """Config for CharacterizeImageTask."""
107
108 doMeasurePsf = pexConfig.Field(
109 dtype=bool,
110 default=True,
111 doc="Measure PSF? If False then for all subsequent operations use either existing PSF "
112 "model when present, or install simple PSF model when not (see installSimplePsf "
113 "config options)"
114 )
115 doWrite = pexConfig.Field(
116 dtype=bool,
117 default=True,
118 doc="Persist results?",
119 )
120 doWriteExposure = pexConfig.Field(
121 dtype=bool,
122 default=True,
123 doc="Write icExp and icExpBackground in addition to icSrc? Ignored if doWrite False.",
124 )
125 psfIterations = pexConfig.RangeField(
126 dtype=int,
127 default=2,
128 min=1,
129 doc="Number of iterations of detect sources, measure sources, "
130 "estimate PSF. If useSimplePsf is True then 2 should be plenty; "
131 "otherwise more may be wanted.",
132 )
133 background = pexConfig.ConfigurableField(
134 target=SubtractBackgroundTask,
135 doc="Configuration for initial background estimation",
136 )
137 detection = pexConfig.ConfigurableField(
138 target=SourceDetectionTask,
139 doc="Detect sources"
140 )
141 doDeblend = pexConfig.Field(
142 dtype=bool,
143 default=True,
144 doc="Run deblender input exposure"
145 )
146 deblend = pexConfig.ConfigurableField(
147 target=SourceDeblendTask,
148 doc="Split blended source into their components"
149 )
150 measurement = pexConfig.ConfigurableField(
151 target=SingleFrameMeasurementTask,
152 doc="Measure sources"
153 )
154 doApCorr = pexConfig.Field(
155 dtype=bool,
156 default=True,
157 doc="Run subtasks to measure and apply aperture corrections"
158 )
159 measureApCorr = pexConfig.ConfigurableField(
160 target=MeasureApCorrTask,
161 doc="Subtask to measure aperture corrections"
162 )
163 applyApCorr = pexConfig.ConfigurableField(
164 target=ApplyApCorrTask,
165 doc="Subtask to apply aperture corrections"
166 )
167 # If doApCorr is False, and the exposure does not have apcorrections already applied, the
168 # active plugins in catalogCalculation almost certainly should not contain the characterization plugin
169 catalogCalculation = pexConfig.ConfigurableField(
170 target=CatalogCalculationTask,
171 doc="Subtask to run catalogCalculation plugins on catalog"
172 )
173 doComputeSummaryStats = pexConfig.Field(
174 dtype=bool,
175 default=True,
176 doc="Run subtask to measure exposure summary statistics",
177 deprecated=("This subtask has been moved to CalibrateTask "
178 "with DM-30701.")
179 )
180 computeSummaryStats = pexConfig.ConfigurableField(
181 target=ComputeExposureSummaryStatsTask,
182 doc="Subtask to run computeSummaryStats on exposure",
183 deprecated=("This subtask has been moved to CalibrateTask "
184 "with DM-30701.")
185 )
186 useSimplePsf = pexConfig.Field(
187 dtype=bool,
188 default=True,
189 doc="Replace the existing PSF model with a simplified version that has the same sigma "
190 "at the start of each PSF determination iteration? Doing so makes PSF determination "
191 "converge more robustly and quickly.",
192 )
193 installSimplePsf = pexConfig.ConfigurableField(
194 target=InstallGaussianPsfTask,
195 doc="Install a simple PSF model",
196 )
197 refObjLoader = pexConfig.ConfigField(
198 dtype=LoadReferenceObjectsConfig,
199 deprecated="This field does nothing. Will be removed after v24 (see DM-34768).",
200 doc="reference object loader",
201 )
202 ref_match = pexConfig.ConfigurableField(
203 target=RefMatchTask,
204 deprecated="This field was never usable. Will be removed after v24 (see DM-34768).",
205 doc="Task to load and match reference objects. Only used if measurePsf can use matches. "
206 "Warning: matching will only work well if the initial WCS is accurate enough "
207 "to give good matches (roughly: good to 3 arcsec across the CCD).",
208 )
209 measurePsf = pexConfig.ConfigurableField(
210 target=MeasurePsfTask,
211 doc="Measure PSF",
212 )
213 repair = pexConfig.ConfigurableField(
214 target=RepairTask,
215 doc="Remove cosmic rays",
216 )
217 requireCrForPsf = pexConfig.Field(
218 dtype=bool,
219 default=True,
220 doc="Require cosmic ray detection and masking to run successfully before measuring the PSF."
221 )
222 checkUnitsParseStrict = pexConfig.Field(
223 doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
224 dtype=str,
225 default="raise",
226 )
227 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
228
229 def setDefaults(self):
230 super().setDefaults()
231 # just detect bright stars; includeThresholdMultipler=10 seems large,
232 # but these are the values we have been using
233 self.detection.thresholdValue = 5.0
234 self.detection.includeThresholdMultiplier = 10.0
235 # do not deblend, as it makes a mess
236 self.doDeblend = False
237 # measure and apply aperture correction; note: measuring and applying aperture
238 # correction are disabled until the final measurement, after PSF is measured
239 self.doApCorr = True
240 # During characterization, we don't have full source measurement information,
241 # so must do the aperture correction with only psf stars, combined with the
242 # default signal-to-noise cuts in MeasureApCorrTask.
243 selector = self.measureApCorr.sourceSelector["science"]
244 selector.doUnresolved = False
245 selector.flags.good = ["calib_psf_used"]
246 selector.flags.bad = []
247
248 # minimal set of measurements needed to determine PSF
249 self.measurement.plugins.names = [
250 "base_PixelFlags",
251 "base_SdssCentroid",
252 "ext_shapeHSM_HsmSourceMoments",
253 "base_GaussianFlux",
254 "base_PsfFlux",
255 "base_CircularApertureFlux",
256 ]
257 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments"
258
259 def validate(self):
260 if self.doApCorr and not self.measurePsf:
261 raise RuntimeError("Must measure PSF to measure aperture correction, "
262 "because flags determined by PSF measurement are used to identify "
263 "sources used to measure aperture correction")
264
265
266class CharacterizeImageTask(pipeBase.PipelineTask):
267 """Measure bright sources and use this to estimate background and PSF of
268 an exposure.
269
270 Given an exposure with defects repaired (masked and interpolated over,
271 e.g. as output by `~lsst.ip.isr.IsrTask`):
272 - detect and measure bright sources
273 - repair cosmic rays
274 - measure and subtract background
275 - measure PSF
276
277 Parameters
278 ----------
279 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
280 Reference object loader if using a catalog-based star-selector.
281 schema : `lsst.afw.table.Schema`, optional
282 Initial schema for icSrc catalog.
283 **kwargs
284 Additional keyword arguments.
285
286 Notes
287 -----
288 Debugging:
289 CharacterizeImageTask has a debug dictionary with the following keys:
290
291 frame
292 int: if specified, the frame of first debug image displayed (defaults to 1)
293 repair_iter
294 bool; if True display image after each repair in the measure PSF loop
295 background_iter
296 bool; if True display image after each background subtraction in the measure PSF loop
297 measure_iter
298 bool; if True display image and sources at the end of each iteration of the measure PSF loop
299 See `~lsst.meas.astrom.displayAstrometry` for the meaning of the various symbols.
300 psf
301 bool; if True display image and sources after PSF is measured;
302 this will be identical to the final image displayed by measure_iter if measure_iter is true
303 repair
304 bool; if True display image and sources after final repair
305 measure
306 bool; if True display image and sources after final measurement
307 """
308
309 ConfigClass = CharacterizeImageConfig
310 _DefaultName = "characterizeImage"
311
312 def __init__(self, refObjLoader=None, schema=None, **kwargs):
313 super().__init__(**kwargs)
314
315 if schema is None:
316 schema = SourceTable.makeMinimalSchema()
317 self.schema = schema
318 self.makeSubtask("background")
319 self.makeSubtask("installSimplePsf")
320 self.makeSubtask("repair")
321 self.makeSubtask("measurePsf", schema=self.schema)
322 # TODO DM-34769: remove this `if` block
323 if self.config.doMeasurePsf and self.measurePsf.usesMatches:
324 self.makeSubtask("ref_match", refObjLoader=refObjLoader)
325 self.algMetadata = dafBase.PropertyList()
326 self.makeSubtask('detection', schema=self.schema)
327 if self.config.doDeblend:
328 self.makeSubtask("deblend", schema=self.schema)
329 self.makeSubtask('measurement', schema=self.schema, algMetadata=self.algMetadata)
330 if self.config.doApCorr:
331 self.makeSubtask('measureApCorr', schema=self.schema)
332 self.makeSubtask('applyApCorr', schema=self.schema)
333 self.makeSubtask('catalogCalculation', schema=self.schema)
334 self._initialFrame = getDebugFrame(self._display, "frame") or 1
336 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
337 self.outputSchema = afwTable.SourceCatalog(self.schema)
338
339 def runQuantum(self, butlerQC, inputRefs, outputRefs):
340 inputs = butlerQC.get(inputRefs)
341 if 'idGenerator' not in inputs.keys():
342 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId)
343 outputs = self.run(**inputs)
344 butlerQC.put(outputs, outputRefs)
345
346 @timeMethod
347 def run(self, exposure, exposureIdInfo=None, background=None, idGenerator=None):
348 """Characterize a science image.
349
350 Peforms the following operations:
351 - Iterate the following config.psfIterations times, or once if config.doMeasurePsf false:
352 - detect and measure sources and estimate PSF (see detectMeasureAndEstimatePsf for details)
353 - interpolate over cosmic rays
354 - perform final measurement
355
356 Parameters
357 ----------
358 exposure : `lsst.afw.image.ExposureF`
359 Exposure to characterize.
360 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`, optional
361 Exposure ID info. Deprecated in favor of ``idGenerator``, and
362 ignored if that is provided.
363 background : `lsst.afw.math.BackgroundList`, optional
364 Initial model of background already subtracted from exposure.
365 idGenerator : `lsst.meas.base.IdGenerator`, optional
366 Object that generates source IDs and provides RNG seeds.
367
368 Returns
369 -------
370 result : `lsst.pipe.base.Struct`
371 Results as a struct with attributes:
372
373 ``exposure``
374 Characterized exposure (`lsst.afw.image.ExposureF`).
375 ``sourceCat``
376 Detected sources (`lsst.afw.table.SourceCatalog`).
377 ``background``
378 Model of subtracted background (`lsst.afw.math.BackgroundList`).
379 ``psfCellSet``
380 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`).
381 ``characterized``
382 Another reference to ``exposure`` for compatibility.
383 ``backgroundModel``
384 Another reference to ``background`` for compatibility.
385
386 Raises
387 ------
388 RuntimeError
389 Raised if PSF sigma is NaN.
390 """
391 self._frame = self._initialFrame # reset debug display frame
392
393 if not self.config.doMeasurePsf and not exposure.hasPsf():
394 self.log.info("CharacterizeImageTask initialized with 'simple' PSF.")
395 self.installSimplePsf.run(exposure=exposure)
396
397 if idGenerator is None:
398 if exposureIdInfo is not None:
399 idGenerator = IdGenerator._from_exposure_id_info(exposureIdInfo)
400 else:
401 idGenerator = IdGenerator()
402
403 del exposureIdInfo
404
405 # subtract an initial estimate of background level
406 background = self.background.run(exposure).background
407
408 psfIterations = self.config.psfIterations if self.config.doMeasurePsf else 1
409 for i in range(psfIterations):
410 dmeRes = self.detectMeasureAndEstimatePsf(
411 exposure=exposure,
412 idGenerator=idGenerator,
413 background=background,
414 )
415
416 psf = dmeRes.exposure.getPsf()
417 # Just need a rough estimate; average positions are fine
418 psfAvgPos = psf.getAveragePosition()
419 psfSigma = psf.computeShape(psfAvgPos).getDeterminantRadius()
420 psfDimensions = psf.computeImage(psfAvgPos).getDimensions()
421 medBackground = np.median(dmeRes.background.getImage().getArray())
422 self.log.info("iter %s; PSF sigma=%0.2f, dimensions=%s; median background=%0.2f",
423 i + 1, psfSigma, psfDimensions, medBackground)
424 if np.isnan(psfSigma):
425 raise RuntimeError("PSF sigma is NaN, cannot continue PSF determination.")
426
427 self.display("psf", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
428
429 # perform final repair with final PSF
430 self.repair.run(exposure=dmeRes.exposure)
431 self.display("repair", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
432
433 # perform final measurement with final PSF, including measuring and applying aperture correction,
434 # if wanted
435 self.measurement.run(measCat=dmeRes.sourceCat, exposure=dmeRes.exposure,
436 exposureId=idGenerator.catalog_id)
437 if self.config.doApCorr:
438 try:
439 apCorrMap = self.measureApCorr.run(
440 exposure=dmeRes.exposure,
441 catalog=dmeRes.sourceCat,
442 ).apCorrMap
443 except MeasureApCorrError:
444 # We have failed to get a valid aperture correction map.
445 # Proceed with processing, and image will be filtered
446 # downstream.
447 dmeRes.exposure.info.setApCorrMap(None)
448 else:
449 dmeRes.exposure.info.setApCorrMap(apCorrMap)
450 self.applyApCorr.run(catalog=dmeRes.sourceCat, apCorrMap=exposure.getInfo().getApCorrMap())
451
452 self.catalogCalculation.run(dmeRes.sourceCat)
453
454 self.display("measure", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
455
456 return pipeBase.Struct(
457 exposure=dmeRes.exposure,
458 sourceCat=dmeRes.sourceCat,
459 background=dmeRes.background,
460 psfCellSet=dmeRes.psfCellSet,
461
462 characterized=dmeRes.exposure,
463 backgroundModel=dmeRes.background
464 )
465
466 @timeMethod
467 def detectMeasureAndEstimatePsf(self, exposure, idGenerator, background):
468 """Perform one iteration of detect, measure, and estimate PSF.
469
470 Performs the following operations:
471
472 - if config.doMeasurePsf or not exposure.hasPsf():
473
474 - install a simple PSF model (replacing the existing one, if need be)
475
476 - interpolate over cosmic rays with keepCRs=True
477 - estimate background and subtract it from the exposure
478 - detect, deblend and measure sources, and subtract a refined background model;
479 - if config.doMeasurePsf:
480 - measure PSF
481
482 Parameters
483 ----------
484 exposure : `lsst.afw.image.ExposureF`
485 Exposure to characterize.
486 idGenerator : `lsst.meas.base.IdGenerator`
487 Object that generates source IDs and provides RNG seeds.
488 background : `lsst.afw.math.BackgroundList`, optional
489 Initial model of background already subtracted from exposure.
490
491 Returns
492 -------
493 result : `lsst.pipe.base.Struct`
494 Results as a struct with attributes:
495
496 ``exposure``
497 Characterized exposure (`lsst.afw.image.ExposureF`).
498 ``sourceCat``
499 Detected sources (`lsst.afw.table.SourceCatalog`).
500 ``background``
501 Model of subtracted background (`lsst.afw.math.BackgroundList`).
502 ``psfCellSet``
503 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`).
504
505 Raises
506 ------
507 LengthError
508 Raised if there are too many CR pixels.
509 """
510 # install a simple PSF model, if needed or wanted
511 if not exposure.hasPsf() or (self.config.doMeasurePsf and self.config.useSimplePsf):
512 self.log.info("PSF estimation initialized with 'simple' PSF")
513 self.installSimplePsf.run(exposure=exposure)
514
515 # run repair, but do not interpolate over cosmic rays (do that elsewhere, with the final PSF model)
516 if self.config.requireCrForPsf:
517 self.repair.run(exposure=exposure, keepCRs=True)
518 else:
519 try:
520 self.repair.run(exposure=exposure, keepCRs=True)
521 except LengthError:
522 self.log.warning("Skipping cosmic ray detection: Too many CR pixels (max %0.f)",
523 self.config.repair.cosmicray.nCrPixelMax)
524
525 self.display("repair_iter", exposure=exposure)
526
527 if background is None:
528 background = BackgroundList()
529
530 sourceIdFactory = idGenerator.make_table_id_factory()
531 table = SourceTable.make(self.schema, sourceIdFactory)
532 table.setMetadata(self.algMetadata)
533
534 detRes = self.detection.run(table=table, exposure=exposure, doSmooth=True)
535 sourceCat = detRes.sources
536 if detRes.background:
537 for bg in detRes.background:
538 background.append(bg)
539
540 if self.config.doDeblend:
541 self.deblend.run(exposure=exposure, sources=sourceCat)
542 # We need the output catalog to be contiguous for further processing.
543 if not sourceCat.isContiguous():
544 sourceCat = sourceCat.copy(deep=True)
545
546 self.measurement.run(measCat=sourceCat, exposure=exposure, exposureId=idGenerator.catalog_id)
547
548 measPsfRes = pipeBase.Struct(cellSet=None)
549 if self.config.doMeasurePsf:
550 # TODO DM-34769: remove this `if` block, and the `matches` kwarg from measurePsf.run below.
551 if self.measurePsf.usesMatches:
552 matches = self.ref_match.loadAndMatch(exposure=exposure, sourceCat=sourceCat).matches
553 else:
554 matches = None
555 measPsfRes = self.measurePsf.run(exposure=exposure, sources=sourceCat, matches=matches,
556 expId=idGenerator.catalog_id)
557 self.display("measure_iter", exposure=exposure, sourceCat=sourceCat)
558
559 return pipeBase.Struct(
560 exposure=exposure,
561 sourceCat=sourceCat,
562 background=background,
563 psfCellSet=measPsfRes.cellSet,
564 )
565
566 def display(self, itemName, exposure, sourceCat=None):
567 """Display exposure and sources on next frame (for debugging).
568
569 Parameters
570 ----------
571 itemName : `str`
572 Name of item in ``debugInfo``.
573 exposure : `lsst.afw.image.ExposureF`
574 Exposure to display.
575 sourceCat : `lsst.afw.table.SourceCatalog`, optional
576 Catalog of sources detected on the exposure.
577 """
578 val = getDebugFrame(self._display, itemName)
579 if not val:
580 return
581
582 displayAstrometry(exposure=exposure, sourceCat=sourceCat, frame=self._frame, pause=False)
583 self._frame += 1
detectMeasureAndEstimatePsf(self, exposure, idGenerator, background)
display(self, itemName, exposure, sourceCat=None)
runQuantum(self, butlerQC, inputRefs, outputRefs)
__init__(self, refObjLoader=None, schema=None, **kwargs)
run(self, exposure, exposureIdInfo=None, background=None, idGenerator=None)