lsst.pipe.tasks g281251e7f9+9d3b0d3b84
Loading...
Searching...
No Matches
characterizeImage.py
Go to the documentation of this file.
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22__all__ = ["CharacterizeImageConfig", "CharacterizeImageTask"]
23
24import numpy as np
25
26from lsstDebug import getDebugFrame
27import lsst.afw.table as afwTable
28import lsst.pex.config as pexConfig
29import lsst.pipe.base as pipeBase
30import lsst.daf.base as dafBase
31import lsst.pipe.base.connectionTypes as cT
32from lsst.afw.math import BackgroundList
33from lsst.afw.table import SourceTable
34from lsst.meas.algorithms import (
35 SubtractBackgroundTask,
36 SourceDetectionTask,
37 MeasureApCorrTask,
38 MeasureApCorrError,
39)
40from lsst.meas.algorithms.installGaussianPsf import InstallGaussianPsfTask
41from lsst.meas.astrom import RefMatchTask, displayAstrometry
42from lsst.meas.algorithms import LoadReferenceObjectsConfig
43from lsst.meas.base import (
44 SingleFrameMeasurementTask,
45 ApplyApCorrTask,
46 CatalogCalculationTask,
47 IdGenerator,
48 DetectorVisitIdGeneratorConfig,
49)
50from lsst.meas.deblender import SourceDeblendTask
51import lsst.meas.extensions.shapeHSM # noqa: F401 needed for default shape plugin
52from .measurePsf import MeasurePsfTask
53from .repair import RepairTask
54from .maskStreaks import MaskStreaksTask
55from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask
56from lsst.pex.exceptions import LengthError
57from lsst.utils.timer import timeMethod
58
59
60class CharacterizeImageConnections(pipeBase.PipelineTaskConnections,
61 dimensions=("instrument", "visit", "detector")):
62 exposure = cT.Input(
63 doc="Input exposure data",
64 name="postISRCCD",
65 storageClass="Exposure",
66 dimensions=["instrument", "exposure", "detector"],
67 )
68 characterized = cT.Output(
69 doc="Output characterized data.",
70 name="icExp",
71 storageClass="ExposureF",
72 dimensions=["instrument", "visit", "detector"],
73 )
74 sourceCat = cT.Output(
75 doc="Output source catalog.",
76 name="icSrc",
77 storageClass="SourceCatalog",
78 dimensions=["instrument", "visit", "detector"],
79 )
80 backgroundModel = cT.Output(
81 doc="Output background model.",
82 name="icExpBackground",
83 storageClass="Background",
84 dimensions=["instrument", "visit", "detector"],
85 )
86 outputSchema = cT.InitOutput(
87 doc="Schema of the catalog produced by CharacterizeImage",
88 name="icSrc_schema",
89 storageClass="SourceCatalog",
90 )
91
92 def adjustQuantum(self, inputs, outputs, label, dataId):
93 # Docstring inherited from PipelineTaskConnections
94 try:
95 return super().adjustQuantum(inputs, outputs, label, dataId)
96 except pipeBase.ScalarError as err:
97 raise pipeBase.ScalarError(
98 "CharacterizeImageTask can at present only be run on visits that are associated with "
99 "exactly one exposure. Either this is not a valid exposure for this pipeline, or the "
100 "snap-combination step you probably want hasn't been configured to run between ISR and "
101 "this task (as of this writing, that would be because it hasn't been implemented yet)."
102 ) from err
103
104
105class CharacterizeImageConfig(pipeBase.PipelineTaskConfig,
106 pipelineConnections=CharacterizeImageConnections):
107 """Config for CharacterizeImageTask."""
108
109 doMeasurePsf = pexConfig.Field(
110 dtype=bool,
111 default=True,
112 doc="Measure PSF? If False then for all subsequent operations use either existing PSF "
113 "model when present, or install simple PSF model when not (see installSimplePsf "
114 "config options)"
115 )
116 doWrite = pexConfig.Field(
117 dtype=bool,
118 default=True,
119 doc="Persist results?",
120 )
121 doWriteExposure = pexConfig.Field(
122 dtype=bool,
123 default=True,
124 doc="Write icExp and icExpBackground in addition to icSrc? Ignored if doWrite False.",
125 )
126 psfIterations = pexConfig.RangeField(
127 dtype=int,
128 default=2,
129 min=1,
130 doc="Number of iterations of detect sources, measure sources, "
131 "estimate PSF. If useSimplePsf is True then 2 should be plenty; "
132 "otherwise more may be wanted.",
133 )
134 background = pexConfig.ConfigurableField(
135 target=SubtractBackgroundTask,
136 doc="Configuration for initial background estimation",
137 )
138 detection = pexConfig.ConfigurableField(
139 target=SourceDetectionTask,
140 doc="Detect sources"
141 )
142 doDeblend = pexConfig.Field(
143 dtype=bool,
144 default=True,
145 doc="Run deblender input exposure"
146 )
147 deblend = pexConfig.ConfigurableField(
148 target=SourceDeblendTask,
149 doc="Split blended source into their components"
150 )
151 measurement = pexConfig.ConfigurableField(
152 target=SingleFrameMeasurementTask,
153 doc="Measure sources"
154 )
155 doApCorr = pexConfig.Field(
156 dtype=bool,
157 default=True,
158 doc="Run subtasks to measure and apply aperture corrections"
159 )
160 measureApCorr = pexConfig.ConfigurableField(
161 target=MeasureApCorrTask,
162 doc="Subtask to measure aperture corrections"
163 )
164 applyApCorr = pexConfig.ConfigurableField(
165 target=ApplyApCorrTask,
166 doc="Subtask to apply aperture corrections"
167 )
168 # If doApCorr is False, and the exposure does not have apcorrections already applied, the
169 # active plugins in catalogCalculation almost certainly should not contain the characterization plugin
170 catalogCalculation = pexConfig.ConfigurableField(
171 target=CatalogCalculationTask,
172 doc="Subtask to run catalogCalculation plugins on catalog"
173 )
174 doComputeSummaryStats = pexConfig.Field(
175 dtype=bool,
176 default=True,
177 doc="Run subtask to measure exposure summary statistics",
178 deprecated=("This subtask has been moved to CalibrateTask "
179 "with DM-30701.")
180 )
181 computeSummaryStats = pexConfig.ConfigurableField(
182 target=ComputeExposureSummaryStatsTask,
183 doc="Subtask to run computeSummaryStats on exposure",
184 deprecated=("This subtask has been moved to CalibrateTask "
185 "with DM-30701.")
186 )
187 useSimplePsf = pexConfig.Field(
188 dtype=bool,
189 default=True,
190 doc="Replace the existing PSF model with a simplified version that has the same sigma "
191 "at the start of each PSF determination iteration? Doing so makes PSF determination "
192 "converge more robustly and quickly.",
193 )
194 installSimplePsf = pexConfig.ConfigurableField(
195 target=InstallGaussianPsfTask,
196 doc="Install a simple PSF model",
197 )
198 refObjLoader = pexConfig.ConfigField(
199 dtype=LoadReferenceObjectsConfig,
200 deprecated="This field does nothing. Will be removed after v24 (see DM-34768).",
201 doc="reference object loader",
202 )
203 ref_match = pexConfig.ConfigurableField(
204 target=RefMatchTask,
205 deprecated="This field was never usable. Will be removed after v24 (see DM-34768).",
206 doc="Task to load and match reference objects. Only used if measurePsf can use matches. "
207 "Warning: matching will only work well if the initial WCS is accurate enough "
208 "to give good matches (roughly: good to 3 arcsec across the CCD).",
209 )
210 measurePsf = pexConfig.ConfigurableField(
211 target=MeasurePsfTask,
212 doc="Measure PSF",
213 )
214 repair = pexConfig.ConfigurableField(
215 target=RepairTask,
216 doc="Remove cosmic rays",
217 )
218 requireCrForPsf = pexConfig.Field(
219 dtype=bool,
220 default=True,
221 doc="Require cosmic ray detection and masking to run successfully before measuring the PSF."
222 )
223 checkUnitsParseStrict = pexConfig.Field(
224 doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
225 dtype=str,
226 default="raise",
227 )
228 doMaskStreaks = pexConfig.Field(
229 doc="Mask streaks",
230 default=True,
231 dtype=bool,
232 )
233 maskStreaks = pexConfig.ConfigurableField(
234 target=MaskStreaksTask,
235 doc="Subtask for masking streaks. Only used if doMaskStreaks is True. "
236 "Adds a mask plane to an exposure, with the mask plane name set by streakMaskName.",
237 )
238 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
239
240 def setDefaults(self):
241 super().setDefaults()
242 # just detect bright stars; includeThresholdMultipler=10 seems large,
243 # but these are the values we have been using
244 self.detection.thresholdValue = 5.0
245 self.detection.includeThresholdMultiplier = 10.0
246 # do not deblend, as it makes a mess
247 self.doDeblend = False
248 # measure and apply aperture correction; note: measuring and applying aperture
249 # correction are disabled until the final measurement, after PSF is measured
250 self.doApCorr = True
251 # During characterization, we don't have full source measurement information,
252 # so must do the aperture correction with only psf stars, combined with the
253 # default signal-to-noise cuts in MeasureApCorrTask.
254 selector = self.measureApCorr.sourceSelector["science"]
255 selector.doUnresolved = False
256 selector.flags.good = ["calib_psf_used"]
257 selector.flags.bad = []
258
259 # minimal set of measurements needed to determine PSF
260 self.measurement.plugins.names = [
261 "base_PixelFlags",
262 "base_SdssCentroid",
263 "ext_shapeHSM_HsmSourceMoments",
264 "base_GaussianFlux",
265 "base_PsfFlux",
266 "base_CircularApertureFlux",
267 ]
268 self.measurement.slots.shape = "ext_shapeHSM_HsmSourceMoments"
269
270 def validate(self):
271 if self.doApCorr and not self.measurePsf:
272 raise RuntimeError("Must measure PSF to measure aperture correction, "
273 "because flags determined by PSF measurement are used to identify "
274 "sources used to measure aperture correction")
275
276
277class CharacterizeImageTask(pipeBase.PipelineTask):
278 """Measure bright sources and use this to estimate background and PSF of
279 an exposure.
280
281 Given an exposure with defects repaired (masked and interpolated over,
282 e.g. as output by `~lsst.ip.isr.IsrTask`):
283 - detect and measure bright sources
284 - repair cosmic rays
285 - detect and mask streaks
286 - measure and subtract background
287 - measure PSF
288
289 Parameters
290 ----------
291 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
292 Reference object loader if using a catalog-based star-selector.
293 schema : `lsst.afw.table.Schema`, optional
294 Initial schema for icSrc catalog.
295 **kwargs
296 Additional keyword arguments.
297
298 Notes
299 -----
300 Debugging:
301 CharacterizeImageTask has a debug dictionary with the following keys:
302
303 frame
304 int: if specified, the frame of first debug image displayed (defaults to 1)
305 repair_iter
306 bool; if True display image after each repair in the measure PSF loop
307 background_iter
308 bool; if True display image after each background subtraction in the measure PSF loop
309 measure_iter
310 bool; if True display image and sources at the end of each iteration of the measure PSF loop
311 See `~lsst.meas.astrom.displayAstrometry` for the meaning of the various symbols.
312 psf
313 bool; if True display image and sources after PSF is measured;
314 this will be identical to the final image displayed by measure_iter if measure_iter is true
315 repair
316 bool; if True display image and sources after final repair
317 measure
318 bool; if True display image and sources after final measurement
319 """
320
321 ConfigClass = CharacterizeImageConfig
322 _DefaultName = "characterizeImage"
323
324 def __init__(self, refObjLoader=None, schema=None, **kwargs):
325 super().__init__(**kwargs)
326
327 if schema is None:
328 schema = SourceTable.makeMinimalSchema()
329 self.schema = schema
330 self.makeSubtask("background")
331 self.makeSubtask("installSimplePsf")
332 self.makeSubtask("repair")
333 if self.config.doMaskStreaks:
334 self.makeSubtask("maskStreaks")
335 self.makeSubtask("measurePsf", schema=self.schema)
336 # TODO DM-34769: remove this `if` block
337 if self.config.doMeasurePsf and self.measurePsf.usesMatches:
338 self.makeSubtask("ref_match", refObjLoader=refObjLoader)
339 self.algMetadata = dafBase.PropertyList()
340 self.makeSubtask('detection', schema=self.schema)
341 if self.config.doDeblend:
342 self.makeSubtask("deblend", schema=self.schema)
343 self.makeSubtask('measurement', schema=self.schema, algMetadata=self.algMetadata)
344 if self.config.doApCorr:
345 self.makeSubtask('measureApCorr', schema=self.schema)
346 self.makeSubtask('applyApCorr', schema=self.schema)
347 self.makeSubtask('catalogCalculation', schema=self.schema)
348 self._initialFrame = getDebugFrame(self._display, "frame") or 1
350 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
351 self.outputSchema = afwTable.SourceCatalog(self.schema)
352
353 def runQuantum(self, butlerQC, inputRefs, outputRefs):
354 inputs = butlerQC.get(inputRefs)
355 if 'idGenerator' not in inputs.keys():
356 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId)
357 outputs = self.run(**inputs)
358 butlerQC.put(outputs, outputRefs)
359
360 @timeMethod
361 def run(self, exposure, exposureIdInfo=None, background=None, idGenerator=None):
362 """Characterize a science image.
363
364 Peforms the following operations:
365 - Iterate the following config.psfIterations times, or once if config.doMeasurePsf false:
366 - detect and measure sources and estimate PSF (see detectMeasureAndEstimatePsf for details)
367 - interpolate over cosmic rays
368 - perform final measurement
369
370 Parameters
371 ----------
372 exposure : `lsst.afw.image.ExposureF`
373 Exposure to characterize.
374 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`, optional
375 Exposure ID info. Deprecated in favor of ``idGenerator``, and
376 ignored if that is provided.
377 background : `lsst.afw.math.BackgroundList`, optional
378 Initial model of background already subtracted from exposure.
379 idGenerator : `lsst.meas.base.IdGenerator`, optional
380 Object that generates source IDs and provides RNG seeds.
381
382 Returns
383 -------
384 result : `lsst.pipe.base.Struct`
385 Results as a struct with attributes:
386
387 ``exposure``
388 Characterized exposure (`lsst.afw.image.ExposureF`).
389 ``sourceCat``
390 Detected sources (`lsst.afw.table.SourceCatalog`).
391 ``background``
392 Model of subtracted background (`lsst.afw.math.BackgroundList`).
393 ``psfCellSet``
394 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`).
395 ``characterized``
396 Another reference to ``exposure`` for compatibility.
397 ``backgroundModel``
398 Another reference to ``background`` for compatibility.
399
400 Raises
401 ------
402 RuntimeError
403 Raised if PSF sigma is NaN.
404 """
405 self._frame = self._initialFrame # reset debug display frame
406
407 if not self.config.doMeasurePsf and not exposure.hasPsf():
408 self.log.info("CharacterizeImageTask initialized with 'simple' PSF.")
409 self.installSimplePsf.run(exposure=exposure)
410
411 if idGenerator is None:
412 if exposureIdInfo is not None:
413 idGenerator = IdGenerator._from_exposure_id_info(exposureIdInfo)
414 else:
415 idGenerator = IdGenerator()
416
417 del exposureIdInfo
418
419 # subtract an initial estimate of background level
420 background = self.background.run(exposure).background
421
422 psfIterations = self.config.psfIterations if self.config.doMeasurePsf else 1
423 for i in range(psfIterations):
424 dmeRes = self.detectMeasureAndEstimatePsf(
425 exposure=exposure,
426 idGenerator=idGenerator,
427 background=background,
428 )
429
430 psf = dmeRes.exposure.getPsf()
431 # Just need a rough estimate; average positions are fine
432 psfAvgPos = psf.getAveragePosition()
433 psfSigma = psf.computeShape(psfAvgPos).getDeterminantRadius()
434 psfDimensions = psf.computeImage(psfAvgPos).getDimensions()
435 medBackground = np.median(dmeRes.background.getImage().getArray())
436 self.log.info("iter %s; PSF sigma=%0.4f, dimensions=%s; median background=%0.2f",
437 i + 1, psfSigma, psfDimensions, medBackground)
438 if np.isnan(psfSigma):
439 raise RuntimeError("PSF sigma is NaN, cannot continue PSF determination.")
440
441 self.display("psf", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
442
443 # perform final repair with final PSF
444 self.repair.run(exposure=dmeRes.exposure)
445 self.display("repair", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
446
447 # mask streaks
448 if self.config.doMaskStreaks:
449 _ = self.maskStreaks.run(dmeRes.exposure)
450
451 # perform final measurement with final PSF, including measuring and applying aperture correction,
452 # if wanted
453 self.measurement.run(measCat=dmeRes.sourceCat, exposure=dmeRes.exposure,
454 exposureId=idGenerator.catalog_id)
455 if self.config.doApCorr:
456 try:
457 apCorrMap = self.measureApCorr.run(
458 exposure=dmeRes.exposure,
459 catalog=dmeRes.sourceCat,
460 ).apCorrMap
461 except MeasureApCorrError:
462 # We have failed to get a valid aperture correction map.
463 # Proceed with processing, and image will be filtered
464 # downstream.
465 dmeRes.exposure.info.setApCorrMap(None)
466 else:
467 dmeRes.exposure.info.setApCorrMap(apCorrMap)
468 self.applyApCorr.run(catalog=dmeRes.sourceCat, apCorrMap=exposure.getInfo().getApCorrMap())
469
470 self.catalogCalculation.run(dmeRes.sourceCat)
471
472 self.display("measure", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
473
474 return pipeBase.Struct(
475 exposure=dmeRes.exposure,
476 sourceCat=dmeRes.sourceCat,
477 background=dmeRes.background,
478 psfCellSet=dmeRes.psfCellSet,
479
480 characterized=dmeRes.exposure,
481 backgroundModel=dmeRes.background
482 )
483
484 @timeMethod
485 def detectMeasureAndEstimatePsf(self, exposure, idGenerator, background):
486 """Perform one iteration of detect, measure, and estimate PSF.
487
488 Performs the following operations:
489
490 - if config.doMeasurePsf or not exposure.hasPsf():
491
492 - install a simple PSF model (replacing the existing one, if need be)
493
494 - interpolate over cosmic rays with keepCRs=True
495 - estimate background and subtract it from the exposure
496 - detect, deblend and measure sources, and subtract a refined background model;
497 - if config.doMeasurePsf:
498 - measure PSF
499
500 Parameters
501 ----------
502 exposure : `lsst.afw.image.ExposureF`
503 Exposure to characterize.
504 idGenerator : `lsst.meas.base.IdGenerator`
505 Object that generates source IDs and provides RNG seeds.
506 background : `lsst.afw.math.BackgroundList`, optional
507 Initial model of background already subtracted from exposure.
508
509 Returns
510 -------
511 result : `lsst.pipe.base.Struct`
512 Results as a struct with attributes:
513
514 ``exposure``
515 Characterized exposure (`lsst.afw.image.ExposureF`).
516 ``sourceCat``
517 Detected sources (`lsst.afw.table.SourceCatalog`).
518 ``background``
519 Model of subtracted background (`lsst.afw.math.BackgroundList`).
520 ``psfCellSet``
521 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`).
522
523 Raises
524 ------
525 LengthError
526 Raised if there are too many CR pixels.
527 """
528 # install a simple PSF model, if needed or wanted
529 if not exposure.hasPsf() or (self.config.doMeasurePsf and self.config.useSimplePsf):
530 self.log.info("PSF estimation initialized with 'simple' PSF")
531 self.installSimplePsf.run(exposure=exposure)
532
533 # run repair, but do not interpolate over cosmic rays (do that elsewhere, with the final PSF model)
534 if self.config.requireCrForPsf:
535 self.repair.run(exposure=exposure, keepCRs=True)
536 else:
537 try:
538 self.repair.run(exposure=exposure, keepCRs=True)
539 except LengthError:
540 self.log.warning("Skipping cosmic ray detection: Too many CR pixels (max %0.f)",
541 self.config.repair.cosmicray.nCrPixelMax)
542
543 self.display("repair_iter", exposure=exposure)
544
545 if background is None:
546 background = BackgroundList()
547
548 sourceIdFactory = idGenerator.make_table_id_factory()
549 table = SourceTable.make(self.schema, sourceIdFactory)
550 table.setMetadata(self.algMetadata)
551
552 detRes = self.detection.run(table=table, exposure=exposure, doSmooth=True)
553 sourceCat = detRes.sources
554 if detRes.background:
555 for bg in detRes.background:
556 background.append(bg)
557
558 if self.config.doDeblend:
559 self.deblend.run(exposure=exposure, sources=sourceCat)
560 # We need the output catalog to be contiguous for further processing.
561 if not sourceCat.isContiguous():
562 sourceCat = sourceCat.copy(deep=True)
563
564 self.measurement.run(measCat=sourceCat, exposure=exposure, exposureId=idGenerator.catalog_id)
565
566 measPsfRes = pipeBase.Struct(cellSet=None)
567 if self.config.doMeasurePsf:
568 # TODO DM-34769: remove this `if` block, and the `matches` kwarg from measurePsf.run below.
569 if self.measurePsf.usesMatches:
570 matches = self.ref_match.loadAndMatch(exposure=exposure, sourceCat=sourceCat).matches
571 else:
572 matches = None
573 measPsfRes = self.measurePsf.run(exposure=exposure, sources=sourceCat, matches=matches,
574 expId=idGenerator.catalog_id)
575 self.display("measure_iter", exposure=exposure, sourceCat=sourceCat)
576
577 return pipeBase.Struct(
578 exposure=exposure,
579 sourceCat=sourceCat,
580 background=background,
581 psfCellSet=measPsfRes.cellSet,
582 )
583
584 def display(self, itemName, exposure, sourceCat=None):
585 """Display exposure and sources on next frame (for debugging).
586
587 Parameters
588 ----------
589 itemName : `str`
590 Name of item in ``debugInfo``.
591 exposure : `lsst.afw.image.ExposureF`
592 Exposure to display.
593 sourceCat : `lsst.afw.table.SourceCatalog`, optional
594 Catalog of sources detected on the exposure.
595 """
596 val = getDebugFrame(self._display, itemName)
597 if not val:
598 return
599
600 displayAstrometry(exposure=exposure, sourceCat=sourceCat, frame=self._frame, pause=False)
601 self._frame += 1
detectMeasureAndEstimatePsf(self, exposure, idGenerator, background)
display(self, itemName, exposure, sourceCat=None)
runQuantum(self, butlerQC, inputRefs, outputRefs)
__init__(self, refObjLoader=None, schema=None, **kwargs)
run(self, exposure, exposureIdInfo=None, background=None, idGenerator=None)