24from lsstDebug
import getDebugFrame
29import lsst.pipe.base.connectionTypes
as cT
36from lsst.obs.base
import ExposureIdInfo
37from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
39import lsst.meas.extensions.shapeHSM
40from .measurePsf
import MeasurePsfTask
41from .repair
import RepairTask
42from .computeExposureSummaryStats
import ComputeExposureSummaryStatsTask
44from lsst.utils.timer
import timeMethod
46__all__ = [
"CharacterizeImageConfig",
"CharacterizeImageTask"]
50 dimensions=(
"instrument",
"visit",
"detector")):
52 doc=
"Input exposure data",
54 storageClass=
"Exposure",
55 dimensions=[
"instrument",
"exposure",
"detector"],
57 characterized = cT.Output(
58 doc=
"Output characterized data.",
60 storageClass=
"ExposureF",
61 dimensions=[
"instrument",
"visit",
"detector"],
63 sourceCat = cT.Output(
64 doc=
"Output source catalog.",
66 storageClass=
"SourceCatalog",
67 dimensions=[
"instrument",
"visit",
"detector"],
69 backgroundModel = cT.Output(
70 doc=
"Output background model.",
71 name=
"icExpBackground",
72 storageClass=
"Background",
73 dimensions=[
"instrument",
"visit",
"detector"],
75 outputSchema = cT.InitOutput(
76 doc=
"Schema of the catalog produced by CharacterizeImage",
78 storageClass=
"SourceCatalog",
85 except pipeBase.ScalarError
as err:
86 raise pipeBase.ScalarError(
87 "CharacterizeImageTask can at present only be run on visits that are associated with "
88 "exactly one exposure. Either this is not a valid exposure for this pipeline, or the "
89 "snap-combination step you probably want hasn't been configured to run between ISR and "
90 "this task (as of this writing, that would be because it hasn't been implemented yet)."
95 pipelineConnections=CharacterizeImageConnections):
97 """!Config for CharacterizeImageTask"""
98 doMeasurePsf = pexConfig.Field(
101 doc=
"Measure PSF? If False then for all subsequent operations use either existing PSF "
102 "model when present, or install simple PSF model when not (see installSimplePsf "
105 doWrite = pexConfig.Field(
108 doc=
"Persist results?",
110 doWriteExposure = pexConfig.Field(
113 doc=
"Write icExp and icExpBackground in addition to icSrc? Ignored if doWrite False.",
115 psfIterations = pexConfig.RangeField(
119 doc=
"Number of iterations of detect sources, measure sources, "
120 "estimate PSF. If useSimplePsf is True then 2 should be plenty; "
121 "otherwise more may be wanted.",
123 background = pexConfig.ConfigurableField(
124 target=SubtractBackgroundTask,
125 doc=
"Configuration for initial background estimation",
127 detection = pexConfig.ConfigurableField(
128 target=SourceDetectionTask,
131 doDeblend = pexConfig.Field(
134 doc=
"Run deblender input exposure"
136 deblend = pexConfig.ConfigurableField(
137 target=SourceDeblendTask,
138 doc=
"Split blended source into their components"
140 measurement = pexConfig.ConfigurableField(
141 target=SingleFrameMeasurementTask,
142 doc=
"Measure sources"
144 doApCorr = pexConfig.Field(
147 doc=
"Run subtasks to measure and apply aperture corrections"
149 measureApCorr = pexConfig.ConfigurableField(
150 target=MeasureApCorrTask,
151 doc=
"Subtask to measure aperture corrections"
153 applyApCorr = pexConfig.ConfigurableField(
154 target=ApplyApCorrTask,
155 doc=
"Subtask to apply aperture corrections"
159 catalogCalculation = pexConfig.ConfigurableField(
160 target=CatalogCalculationTask,
161 doc=
"Subtask to run catalogCalculation plugins on catalog"
163 doComputeSummaryStats = pexConfig.Field(
166 doc=
"Run subtask to measure exposure summary statistics",
167 deprecated=(
"This subtask has been moved to CalibrateTask "
170 computeSummaryStats = pexConfig.ConfigurableField(
171 target=ComputeExposureSummaryStatsTask,
172 doc=
"Subtask to run computeSummaryStats on exposure",
173 deprecated=(
"This subtask has been moved to CalibrateTask "
176 useSimplePsf = pexConfig.Field(
179 doc=
"Replace the existing PSF model with a simplified version that has the same sigma "
180 "at the start of each PSF determination iteration? Doing so makes PSF determination "
181 "converge more robustly and quickly.",
183 installSimplePsf = pexConfig.ConfigurableField(
184 target=InstallGaussianPsfTask,
185 doc=
"Install a simple PSF model",
187 refObjLoader = pexConfig.ConfigurableField(
188 target=LoadIndexedReferenceObjectsTask,
189 deprecated=
"This field does nothing. Will be removed after v24 (see DM-34768).",
190 doc=
"reference object loader",
192 ref_match = pexConfig.ConfigurableField(
194 deprecated=
"This field was never usable. Will be removed after v24 (see DM-34768).",
195 doc=
"Task to load and match reference objects. Only used if measurePsf can use matches. "
196 "Warning: matching will only work well if the initial WCS is accurate enough "
197 "to give good matches (roughly: good to 3 arcsec across the CCD).",
199 measurePsf = pexConfig.ConfigurableField(
200 target=MeasurePsfTask,
203 repair = pexConfig.ConfigurableField(
205 doc=
"Remove cosmic rays",
207 requireCrForPsf = pexConfig.Field(
210 doc=
"Require cosmic ray detection and masking to run successfully before measuring the PSF."
212 checkUnitsParseStrict = pexConfig.Field(
213 doc=
"Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
223 self.
detection.includeThresholdMultiplier = 10.0
224 self.
detection.doTempLocalBackground =
False
234 "ext_shapeHSM_HsmSourceMoments",
237 "base_CircularApertureFlux",
239 self.
measurement.slots.shape =
"ext_shapeHSM_HsmSourceMoments"
243 raise RuntimeError(
"Must measure PSF to measure aperture correction, "
244 "because flags determined by PSF measurement are used to identify "
245 "sources used to measure aperture correction")
257 Measure bright sources and use this to estimate background
and PSF of an exposure
259 @anchor CharacterizeImageTask_
261 @section pipe_tasks_characterizeImage_Contents Contents
263 -
@ref pipe_tasks_characterizeImage_Purpose
264 -
@ref pipe_tasks_characterizeImage_Initialize
265 -
@ref pipe_tasks_characterizeImage_IO
266 -
@ref pipe_tasks_characterizeImage_Config
267 -
@ref pipe_tasks_characterizeImage_Debug
269 @section pipe_tasks_characterizeImage_Purpose Description
271 Given an exposure
with defects repaired (masked
and interpolated over, e.g.
as output by IsrTask):
272 - detect
and measure bright sources
274 - measure
and subtract background
277 @section pipe_tasks_characterizeImage_Initialize Task initialisation
279 @copydoc \_\_init\_\_
281 @section pipe_tasks_characterizeImage_IO Invoking the Task
283 If you want this task to unpersist inputs
or persist outputs, then call
284 the `runDataRef` method (a thin wrapper around the `run` method).
286 If you already have the inputs unpersisted
and do
not want to persist the output
287 then it
is more direct to call the `run` method:
289 @section pipe_tasks_characterizeImage_Config Configuration parameters
291 See
@ref CharacterizeImageConfig
293 @section pipe_tasks_characterizeImage_Debug Debug variables
295 The command line task interface supports a flag
296 `--debug` to
import `debug.py`
from your `$PYTHONPATH`; see
297 <a href=
"https://pipelines.lsst.io/modules/lsstDebug/">the lsstDebug documentation</a>
298 for more about `debug.py`.
300 CharacterizeImageTask has a debug dictionary
with the following keys:
303 <dd>int:
if specified, the frame of first debug image displayed (defaults to 1)
305 <dd>bool;
if True display image after each repair
in the measure PSF loop
307 <dd>bool;
if True display image after each background subtraction
in the measure PSF loop
309 <dd>bool;
if True display image
and sources at the end of each iteration of the measure PSF loop
310 See
@ref lsst.meas.astrom.displayAstrometry
for the meaning of the various symbols.
312 <dd>bool;
if True display image
and sources after PSF
is measured;
313 this will be identical to the final image displayed by measure_iter
if measure_iter
is true
315 <dd>bool;
if True display image
and sources after final repair
317 <dd>bool;
if True display image
and sources after final measurement
320 For example, put something like:
325 if name ==
"lsst.pipe.tasks.characterizeImage":
334 into your `debug.py` file
and run `calibrateTask.py`
with the `--debug` flag.
336 Some subtasks may have their own debug variables; see individual Task documentation.
341 ConfigClass = CharacterizeImageConfig
342 _DefaultName =
"characterizeImage"
343 RunnerClass = pipeBase.ButlerInitializedTaskRunner
346 inputs = butlerQC.get(inputRefs)
347 if 'exposureIdInfo' not in inputs.keys():
348 inputs[
'exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId,
"visit_detector")
349 outputs = self.
run(**inputs)
350 butlerQC.put(outputs, outputRefs)
352 def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs):
353 """!Construct a CharacterizeImageTask
355 @param[
in] butler A butler object
is passed to the refObjLoader constructor
in case
356 it
is needed to load catalogs. May be
None if a catalog-based star selector
is
357 not used,
if the reference object loader constructor does
not require a butler,
358 or if a reference object loader
is passed directly via the refObjLoader argument.
360 @param[
in] refObjLoader An instance of LoadReferenceObjectsTasks that supplies an
361 external reference catalog to a catalog-based star selector. May be
None if a
362 catalog star selector
is not used
or the loader can be constructed
from the
365 @param[
in,out] kwargs other keyword arguments
for lsst.pipe.base.CmdLineTask
370 schema = SourceTable.makeMinimalSchema()
372 self.makeSubtask(
"background")
373 self.makeSubtask(
"installSimplePsf")
374 self.makeSubtask(
"repair")
375 self.makeSubtask(
"measurePsf", schema=self.
schema)
377 if self.config.doMeasurePsf
and self.measurePsf.usesMatches:
379 self.makeSubtask(
'refObjLoader', butler=butler)
380 refObjLoader = self.refObjLoader
381 self.makeSubtask(
"ref_match", refObjLoader=refObjLoader)
383 self.makeSubtask(
'detection', schema=self.
schema)
384 if self.config.doDeblend:
385 self.makeSubtask(
"deblend", schema=self.
schema)
386 self.makeSubtask(
'measurement', schema=self.
schema, algMetadata=self.
algMetadata)
387 if self.config.doApCorr:
388 self.makeSubtask(
'measureApCorr', schema=self.
schema)
389 self.makeSubtask(
'applyApCorr', schema=self.
schema)
390 self.makeSubtask(
'catalogCalculation', schema=self.
schema)
391 self.
_initialFrame = getDebugFrame(self._display,
"frame")
or 1
393 self.
schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
397 outputCatSchema = afwTable.SourceCatalog(self.
schema)
398 outputCatSchema.getTable().setMetadata(self.
algMetadata)
399 return {
'outputSchema': outputCatSchema}
402 def runDataRef(self, dataRef, exposure=None, background=None, doUnpersist=True):
403 """!Characterize a science image and, if wanted, persist the results
405 This simply unpacks the exposure and passes it to the characterize method to do the work.
407 @param[
in] dataRef: butler data reference
for science exposure
408 @param[
in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF
or similar).
409 If
None then unpersist
from "postISRCCD".
410 The following changes are made, depending on the config:
411 - set psf to the measured PSF
412 - set apCorrMap to the measured aperture correction
413 - subtract background
414 - interpolate over cosmic rays
415 - update detection
and cosmic ray mask planes
416 @param[
in,out] background initial model of background already subtracted
from exposure
417 (an lsst.afw.math.BackgroundList). May be
None if no background has been subtracted,
418 which
is typical
for image characterization.
419 A refined background model
is output.
420 @param[
in] doUnpersist
if True the exposure
is read
from the repository
421 and the exposure
and background arguments must be
None;
422 if False the exposure must be provided.
423 True is intended
for running
as a command-line task,
False for running
as a subtask
425 @return same data
as the characterize method
428 self.log.info(
"Processing %s", dataRef.dataId)
431 if exposure
is not None or background
is not None:
432 raise RuntimeError(
"doUnpersist true; exposure and background must be None")
433 exposure = dataRef.get(
"postISRCCD", immediate=
True)
434 elif exposure
is None:
435 raise RuntimeError(
"doUnpersist false; exposure must be provided")
437 exposureIdInfo = dataRef.get(
"expIdInfo")
441 exposureIdInfo=exposureIdInfo,
442 background=background,
445 if self.config.doWrite:
446 dataRef.put(charRes.sourceCat,
"icSrc")
447 if self.config.doWriteExposure:
448 dataRef.put(charRes.exposure,
"icExp")
449 dataRef.put(charRes.background,
"icExpBackground")
454 def run(self, exposure, exposureIdInfo=None, background=None):
455 """!Characterize a science image
457 Peforms the following operations:
458 - Iterate the following config.psfIterations times, or once
if config.doMeasurePsf false:
459 - detect
and measure sources
and estimate PSF (see detectMeasureAndEstimatePsf
for details)
460 - interpolate over cosmic rays
461 - perform final measurement
463 @param[
in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF
or similar).
464 The following changes are made:
467 - update detection
and cosmic ray mask planes
468 - subtract background
and interpolate over cosmic rays
469 @param[
in] exposureIdInfo ID info
for exposure (an lsst.obs.base.ExposureIdInfo).
470 If
not provided, returned SourceCatalog IDs will
not be globally unique.
471 @param[
in,out] background initial model of background already subtracted
from exposure
472 (an lsst.afw.math.BackgroundList). May be
None if no background has been subtracted,
473 which
is typical
for image characterization.
475 @return pipe_base Struct containing these fields, all
from the final iteration
476 of detectMeasureAndEstimatePsf:
477 - exposure: characterized exposure; image
is repaired by interpolating over cosmic rays,
478 mask
is updated accordingly,
and the PSF model
is set
480 - background: model of background subtracted
from exposure (an lsst.afw.math.BackgroundList)
485 if not self.config.doMeasurePsf
and not exposure.hasPsf():
486 self.log.info(
"CharacterizeImageTask initialized with 'simple' PSF.")
487 self.installSimplePsf.
run(exposure=exposure)
489 if exposureIdInfo
is None:
490 exposureIdInfo = ExposureIdInfo()
493 background = self.background.
run(exposure).background
495 psfIterations = self.config.psfIterations
if self.config.doMeasurePsf
else 1
496 for i
in range(psfIterations):
499 exposureIdInfo=exposureIdInfo,
500 background=background,
503 psf = dmeRes.exposure.getPsf()
505 psfAvgPos = psf.getAveragePosition()
506 psfSigma = psf.computeShape(psfAvgPos).getDeterminantRadius()
507 psfDimensions = psf.computeImage(psfAvgPos).getDimensions()
508 medBackground = np.median(dmeRes.background.getImage().getArray())
509 self.log.info(
"iter %s; PSF sigma=%0.2f, dimensions=%s; median background=%0.2f",
510 i + 1, psfSigma, psfDimensions, medBackground)
511 if np.isnan(psfSigma):
512 raise RuntimeError(
"PSF sigma is NaN, cannot continue PSF determination.")
514 self.
display(
"psf", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
517 self.repair.
run(exposure=dmeRes.exposure)
518 self.
display(
"repair", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
522 self.measurement.
run(measCat=dmeRes.sourceCat, exposure=dmeRes.exposure,
523 exposureId=exposureIdInfo.expId)
524 if self.config.doApCorr:
525 apCorrMap = self.measureApCorr.
run(exposure=dmeRes.exposure, catalog=dmeRes.sourceCat).apCorrMap
526 dmeRes.exposure.getInfo().setApCorrMap(apCorrMap)
527 self.applyApCorr.
run(catalog=dmeRes.sourceCat, apCorrMap=exposure.getInfo().getApCorrMap())
528 self.catalogCalculation.
run(dmeRes.sourceCat)
530 self.
display(
"measure", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
532 return pipeBase.Struct(
533 exposure=dmeRes.exposure,
534 sourceCat=dmeRes.sourceCat,
535 background=dmeRes.background,
536 psfCellSet=dmeRes.psfCellSet,
538 characterized=dmeRes.exposure,
539 backgroundModel=dmeRes.background
544 """!Perform one iteration of detect, measure and estimate PSF
546 Performs the following operations:
547 - if config.doMeasurePsf
or not exposure.hasPsf():
548 - install a simple PSF model (replacing the existing one,
if need be)
549 - interpolate over cosmic rays
with keepCRs=
True
550 - estimate background
and subtract it
from the exposure
551 - detect, deblend
and measure sources,
and subtract a refined background model;
552 -
if config.doMeasurePsf:
555 @param[
in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF
or similar)
556 The following changes are made:
558 - update detection
and cosmic ray mask planes
559 - subtract background
560 @param[
in] exposureIdInfo ID info
for exposure (an lsst.obs_base.ExposureIdInfo)
561 @param[
in,out] background initial model of background already subtracted
from exposure
562 (an lsst.afw.math.BackgroundList).
564 @return pipe_base Struct containing these fields, all
from the final iteration
565 of detect sources, measure sources
and estimate PSF:
566 - exposure characterized exposure; image
is repaired by interpolating over cosmic rays,
567 mask
is updated accordingly,
and the PSF model
is set
569 - background model of background subtracted
from exposure (an lsst.afw.math.BackgroundList)
573 if not exposure.hasPsf()
or (self.config.doMeasurePsf
and self.config.useSimplePsf):
574 self.log.info(
"PSF estimation initialized with 'simple' PSF")
575 self.installSimplePsf.
run(exposure=exposure)
578 if self.config.requireCrForPsf:
579 self.repair.
run(exposure=exposure, keepCRs=
True)
582 self.repair.
run(exposure=exposure, keepCRs=
True)
584 self.log.warning(
"Skipping cosmic ray detection: Too many CR pixels (max %0.f)",
585 self.config.repair.cosmicray.nCrPixelMax)
587 self.
display(
"repair_iter", exposure=exposure)
589 if background
is None:
590 background = BackgroundList()
592 sourceIdFactory = exposureIdInfo.makeSourceIdFactory()
593 table = SourceTable.make(self.
schema, sourceIdFactory)
596 detRes = self.detection.
run(table=table, exposure=exposure, doSmooth=
True)
597 sourceCat = detRes.sources
598 if detRes.fpSets.background:
599 for bg
in detRes.fpSets.background:
600 background.append(bg)
602 if self.config.doDeblend:
603 self.deblend.
run(exposure=exposure, sources=sourceCat)
605 self.measurement.
run(measCat=sourceCat, exposure=exposure, exposureId=exposureIdInfo.expId)
607 measPsfRes = pipeBase.Struct(cellSet=
None)
608 if self.config.doMeasurePsf:
610 if self.measurePsf.usesMatches:
611 matches = self.ref_match.loadAndMatch(exposure=exposure, sourceCat=sourceCat).matches
614 measPsfRes = self.measurePsf.
run(exposure=exposure, sources=sourceCat, matches=matches,
615 expId=exposureIdInfo.expId)
616 self.
display(
"measure_iter", exposure=exposure, sourceCat=sourceCat)
618 return pipeBase.Struct(
621 background=background,
622 psfCellSet=measPsfRes.cellSet,
626 """Return a dict of empty catalogs for each catalog dataset produced by this task.
628 sourceCat = SourceCatalog(self.schema)
630 return {
"icSrc": sourceCat}
632 def display(self, itemName, exposure, sourceCat=None):
633 """Display exposure and sources on next frame, if display of itemName has been requested
635 @param[
in] itemName name of item
in debugInfo
636 @param[
in] exposure exposure to display
637 @param[
in] sourceCat source catalog to display
639 val = getDebugFrame(self._display, itemName)
643 displayAstrometry(exposure=exposure, sourceCat=sourceCat, frame=self.
_frame, pause=
False)
Config for CharacterizeImageTask.
def adjustQuantum(self, inputs, outputs, label, dataId)
Measure bright sources and use this to estimate background and PSF of an exposure.
def getSchemaCatalogs(self)
def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs)
Construct a CharacterizeImageTask.
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def getInitOutputDatasets(self)
def runDataRef(self, dataRef, exposure=None, background=None, doUnpersist=True)
Characterize a science image and, if wanted, persist the results.
def run(self, exposure, exposureIdInfo=None, background=None)
Characterize a science image.
def detectMeasureAndEstimatePsf(self, exposure, exposureIdInfo, background)
Perform one iteration of detect, measure and estimate PSF.
def display(self, itemName, exposure, sourceCat=None)