24from lsstDebug
import getDebugFrame
29import lsst.pipe.base.connectionTypes
as cT
36from lsst.obs.base
import ExposureIdInfo
37from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
39from .measurePsf
import MeasurePsfTask
40from .repair
import RepairTask
41from .computeExposureSummaryStats
import ComputeExposureSummaryStatsTask
43from lsst.utils.timer
import timeMethod
45__all__ = [
"CharacterizeImageConfig",
"CharacterizeImageTask"]
49 dimensions=(
"instrument",
"visit",
"detector")):
51 doc=
"Input exposure data",
53 storageClass=
"Exposure",
54 dimensions=[
"instrument",
"exposure",
"detector"],
56 characterized = cT.Output(
57 doc=
"Output characterized data.",
59 storageClass=
"ExposureF",
60 dimensions=[
"instrument",
"visit",
"detector"],
62 sourceCat = cT.Output(
63 doc=
"Output source catalog.",
65 storageClass=
"SourceCatalog",
66 dimensions=[
"instrument",
"visit",
"detector"],
68 backgroundModel = cT.Output(
69 doc=
"Output background model.",
70 name=
"icExpBackground",
71 storageClass=
"Background",
72 dimensions=[
"instrument",
"visit",
"detector"],
74 outputSchema = cT.InitOutput(
75 doc=
"Schema of the catalog produced by CharacterizeImage",
77 storageClass=
"SourceCatalog",
84 except pipeBase.ScalarError
as err:
85 raise pipeBase.ScalarError(
86 "CharacterizeImageTask can at present only be run on visits that are associated with "
87 "exactly one exposure. Either this is not a valid exposure for this pipeline, or the "
88 "snap-combination step you probably want hasn't been configured to run between ISR and "
89 "this task (as of this writing, that would be because it hasn't been implemented yet)."
94 pipelineConnections=CharacterizeImageConnections):
96 """!Config for CharacterizeImageTask"""
97 doMeasurePsf = pexConfig.Field(
100 doc=
"Measure PSF? If False then for all subsequent operations use either existing PSF "
101 "model when present, or install simple PSF model when not (see installSimplePsf "
104 doWrite = pexConfig.Field(
107 doc=
"Persist results?",
109 doWriteExposure = pexConfig.Field(
112 doc=
"Write icExp and icExpBackground in addition to icSrc? Ignored if doWrite False.",
114 psfIterations = pexConfig.RangeField(
118 doc=
"Number of iterations of detect sources, measure sources, "
119 "estimate PSF. If useSimplePsf is True then 2 should be plenty; "
120 "otherwise more may be wanted.",
122 background = pexConfig.ConfigurableField(
123 target=SubtractBackgroundTask,
124 doc=
"Configuration for initial background estimation",
126 detection = pexConfig.ConfigurableField(
127 target=SourceDetectionTask,
130 doDeblend = pexConfig.Field(
133 doc=
"Run deblender input exposure"
135 deblend = pexConfig.ConfigurableField(
136 target=SourceDeblendTask,
137 doc=
"Split blended source into their components"
139 measurement = pexConfig.ConfigurableField(
140 target=SingleFrameMeasurementTask,
141 doc=
"Measure sources"
143 doApCorr = pexConfig.Field(
146 doc=
"Run subtasks to measure and apply aperture corrections"
148 measureApCorr = pexConfig.ConfigurableField(
149 target=MeasureApCorrTask,
150 doc=
"Subtask to measure aperture corrections"
152 applyApCorr = pexConfig.ConfigurableField(
153 target=ApplyApCorrTask,
154 doc=
"Subtask to apply aperture corrections"
158 catalogCalculation = pexConfig.ConfigurableField(
159 target=CatalogCalculationTask,
160 doc=
"Subtask to run catalogCalculation plugins on catalog"
162 doComputeSummaryStats = pexConfig.Field(
165 doc=
"Run subtask to measure exposure summary statistics",
166 deprecated=(
"This subtask has been moved to CalibrateTask "
169 computeSummaryStats = pexConfig.ConfigurableField(
170 target=ComputeExposureSummaryStatsTask,
171 doc=
"Subtask to run computeSummaryStats on exposure",
172 deprecated=(
"This subtask has been moved to CalibrateTask "
175 useSimplePsf = pexConfig.Field(
178 doc=
"Replace the existing PSF model with a simplified version that has the same sigma "
179 "at the start of each PSF determination iteration? Doing so makes PSF determination "
180 "converge more robustly and quickly.",
182 installSimplePsf = pexConfig.ConfigurableField(
183 target=InstallGaussianPsfTask,
184 doc=
"Install a simple PSF model",
186 refObjLoader = pexConfig.ConfigurableField(
187 target=LoadIndexedReferenceObjectsTask,
188 doc=
"reference object loader",
190 ref_match = pexConfig.ConfigurableField(
192 doc=
"Task to load and match reference objects. Only used if measurePsf can use matches. "
193 "Warning: matching will only work well if the initial WCS is accurate enough "
194 "to give good matches (roughly: good to 3 arcsec across the CCD).",
196 measurePsf = pexConfig.ConfigurableField(
197 target=MeasurePsfTask,
200 repair = pexConfig.ConfigurableField(
202 doc=
"Remove cosmic rays",
204 requireCrForPsf = pexConfig.Field(
207 doc=
"Require cosmic ray detection and masking to run successfully before measuring the PSF."
209 checkUnitsParseStrict = pexConfig.Field(
210 doc=
"Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
219 self.
detectiondetection.thresholdValue = 5.0
220 self.
detectiondetection.includeThresholdMultiplier = 10.0
221 self.
detectiondetection.doTempLocalBackground =
False
234 "base_CircularApertureFlux",
239 raise RuntimeError(
"Must measure PSF to measure aperture correction, "
240 "because flags determined by PSF measurement are used to identify "
241 "sources used to measure aperture correction")
253 Measure bright sources and use this to estimate background
and PSF of an exposure
255 @anchor CharacterizeImageTask_
257 @section pipe_tasks_characterizeImage_Contents Contents
259 -
@ref pipe_tasks_characterizeImage_Purpose
260 -
@ref pipe_tasks_characterizeImage_Initialize
261 -
@ref pipe_tasks_characterizeImage_IO
262 -
@ref pipe_tasks_characterizeImage_Config
263 -
@ref pipe_tasks_characterizeImage_Debug
265 @section pipe_tasks_characterizeImage_Purpose Description
267 Given an exposure
with defects repaired (masked
and interpolated over, e.g.
as output by IsrTask):
268 - detect
and measure bright sources
270 - measure
and subtract background
273 @section pipe_tasks_characterizeImage_Initialize Task initialisation
275 @copydoc \_\_init\_\_
277 @section pipe_tasks_characterizeImage_IO Invoking the Task
279 If you want this task to unpersist inputs
or persist outputs, then call
280 the `runDataRef` method (a thin wrapper around the `run` method).
282 If you already have the inputs unpersisted
and do
not want to persist the output
283 then it
is more direct to call the `run` method:
285 @section pipe_tasks_characterizeImage_Config Configuration parameters
287 See
@ref CharacterizeImageConfig
289 @section pipe_tasks_characterizeImage_Debug Debug variables
291 The command line task interface supports a flag
292 `--debug` to
import `debug.py`
from your `$PYTHONPATH`; see
293 <a href=
"https://pipelines.lsst.io/modules/lsstDebug/">the lsstDebug documentation</a>
294 for more about `debug.py`.
296 CharacterizeImageTask has a debug dictionary
with the following keys:
299 <dd>int:
if specified, the frame of first debug image displayed (defaults to 1)
301 <dd>bool;
if True display image after each repair
in the measure PSF loop
303 <dd>bool;
if True display image after each background subtraction
in the measure PSF loop
305 <dd>bool;
if True display image
and sources at the end of each iteration of the measure PSF loop
306 See
@ref lsst.meas.astrom.displayAstrometry
for the meaning of the various symbols.
308 <dd>bool;
if True display image
and sources after PSF
is measured;
309 this will be identical to the final image displayed by measure_iter
if measure_iter
is true
311 <dd>bool;
if True display image
and sources after final repair
313 <dd>bool;
if True display image
and sources after final measurement
316 For example, put something like:
321 if name ==
"lsst.pipe.tasks.characterizeImage":
330 into your `debug.py` file
and run `calibrateTask.py`
with the `--debug` flag.
332 Some subtasks may have their own debug variables; see individual Task documentation.
337 ConfigClass = CharacterizeImageConfig
338 _DefaultName =
"characterizeImage"
339 RunnerClass = pipeBase.ButlerInitializedTaskRunner
342 inputs = butlerQC.get(inputRefs)
343 if 'exposureIdInfo' not in inputs.keys():
344 inputs[
'exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId,
"visit_detector")
345 outputs = self.
runrun(**inputs)
346 butlerQC.put(outputs, outputRefs)
348 def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs):
349 """!Construct a CharacterizeImageTask
351 @param[
in] butler A butler object
is passed to the refObjLoader constructor
in case
352 it
is needed to load catalogs. May be
None if a catalog-based star selector
is
353 not used,
if the reference object loader constructor does
not require a butler,
354 or if a reference object loader
is passed directly via the refObjLoader argument.
355 @param[
in] refObjLoader An instance of LoadReferenceObjectsTasks that supplies an
356 external reference catalog to a catalog-based star selector. May be
None if a
357 catalog star selector
is not used
or the loader can be constructed
from the
360 @param[
in,out] kwargs other keyword arguments
for lsst.pipe.base.CmdLineTask
365 schema = SourceTable.makeMinimalSchema()
367 self.makeSubtask(
"background")
368 self.makeSubtask(
"installSimplePsf")
369 self.makeSubtask(
"repair")
370 self.makeSubtask(
"measurePsf", schema=self.
schemaschema)
371 if self.config.doMeasurePsf
and self.measurePsf.usesMatches:
373 self.makeSubtask(
'refObjLoader', butler=butler)
374 refObjLoader = self.refObjLoader
375 self.makeSubtask(
"ref_match", refObjLoader=refObjLoader)
377 self.makeSubtask(
'detection', schema=self.
schemaschema)
378 if self.config.doDeblend:
379 self.makeSubtask(
"deblend", schema=self.
schemaschema)
380 self.makeSubtask(
'measurement', schema=self.
schemaschema, algMetadata=self.
algMetadataalgMetadata)
381 if self.config.doApCorr:
382 self.makeSubtask(
'measureApCorr', schema=self.
schemaschema)
383 self.makeSubtask(
'applyApCorr', schema=self.
schemaschema)
384 self.makeSubtask(
'catalogCalculation', schema=self.
schemaschema)
385 self.
_initialFrame_initialFrame = getDebugFrame(self._display,
"frame")
or 1
387 self.
schemaschema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
391 outputCatSchema = afwTable.SourceCatalog(self.
schemaschema)
392 outputCatSchema.getTable().setMetadata(self.
algMetadataalgMetadata)
393 return {
'outputSchema': outputCatSchema}
396 def runDataRef(self, dataRef, exposure=None, background=None, doUnpersist=True):
397 """!Characterize a science image and, if wanted, persist the results
399 This simply unpacks the exposure and passes it to the characterize method to do the work.
401 @param[
in] dataRef: butler data reference
for science exposure
402 @param[
in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF
or similar).
403 If
None then unpersist
from "postISRCCD".
404 The following changes are made, depending on the config:
405 - set psf to the measured PSF
406 - set apCorrMap to the measured aperture correction
407 - subtract background
408 - interpolate over cosmic rays
409 - update detection
and cosmic ray mask planes
410 @param[
in,out] background initial model of background already subtracted
from exposure
411 (an lsst.afw.math.BackgroundList). May be
None if no background has been subtracted,
412 which
is typical
for image characterization.
413 A refined background model
is output.
414 @param[
in] doUnpersist
if True the exposure
is read
from the repository
415 and the exposure
and background arguments must be
None;
416 if False the exposure must be provided.
417 True is intended
for running
as a command-line task,
False for running
as a subtask
419 @return same data
as the characterize method
422 self.log.info(
"Processing %s", dataRef.dataId)
425 if exposure
is not None or background
is not None:
426 raise RuntimeError(
"doUnpersist true; exposure and background must be None")
427 exposure = dataRef.get(
"postISRCCD", immediate=
True)
428 elif exposure
is None:
429 raise RuntimeError(
"doUnpersist false; exposure must be provided")
431 exposureIdInfo = dataRef.get(
"expIdInfo")
433 charRes = self.
runrun(
435 exposureIdInfo=exposureIdInfo,
436 background=background,
439 if self.config.doWrite:
440 dataRef.put(charRes.sourceCat,
"icSrc")
441 if self.config.doWriteExposure:
442 dataRef.put(charRes.exposure,
"icExp")
443 dataRef.put(charRes.background,
"icExpBackground")
448 def run(self, exposure, exposureIdInfo=None, background=None):
449 """!Characterize a science image
451 Peforms the following operations:
452 - Iterate the following config.psfIterations times, or once
if config.doMeasurePsf false:
453 - detect
and measure sources
and estimate PSF (see detectMeasureAndEstimatePsf
for details)
454 - interpolate over cosmic rays
455 - perform final measurement
457 @param[
in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF
or similar).
458 The following changes are made:
461 - update detection
and cosmic ray mask planes
462 - subtract background
and interpolate over cosmic rays
463 @param[
in] exposureIdInfo ID info
for exposure (an lsst.obs.base.ExposureIdInfo).
464 If
not provided, returned SourceCatalog IDs will
not be globally unique.
465 @param[
in,out] background initial model of background already subtracted
from exposure
466 (an lsst.afw.math.BackgroundList). May be
None if no background has been subtracted,
467 which
is typical
for image characterization.
469 @return pipe_base Struct containing these fields, all
from the final iteration
470 of detectMeasureAndEstimatePsf:
471 - exposure: characterized exposure; image
is repaired by interpolating over cosmic rays,
472 mask
is updated accordingly,
and the PSF model
is set
474 - background: model of background subtracted
from exposure (an lsst.afw.math.BackgroundList)
479 if not self.config.doMeasurePsf
and not exposure.hasPsf():
480 self.log.info(
"CharacterizeImageTask initialized with 'simple' PSF.")
481 self.installSimplePsf.
run(exposure=exposure)
483 if exposureIdInfo
is None:
484 exposureIdInfo = ExposureIdInfo()
487 background = self.background.
run(exposure).background
489 psfIterations = self.config.psfIterations
if self.config.doMeasurePsf
else 1
490 for i
in range(psfIterations):
493 exposureIdInfo=exposureIdInfo,
494 background=background,
497 psf = dmeRes.exposure.getPsf()
499 psfAvgPos = psf.getAveragePosition()
500 psfSigma = psf.computeShape(psfAvgPos).getDeterminantRadius()
501 psfDimensions = psf.computeImage(psfAvgPos).getDimensions()
502 medBackground = np.median(dmeRes.background.getImage().getArray())
503 self.log.info(
"iter %s; PSF sigma=%0.2f, dimensions=%s; median background=%0.2f",
504 i + 1, psfSigma, psfDimensions, medBackground)
506 self.
displaydisplay(
"psf", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
509 self.repair.
run(exposure=dmeRes.exposure)
510 self.
displaydisplay(
"repair", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
514 self.measurement.
run(measCat=dmeRes.sourceCat, exposure=dmeRes.exposure,
515 exposureId=exposureIdInfo.expId)
516 if self.config.doApCorr:
517 apCorrMap = self.measureApCorr.
run(exposure=dmeRes.exposure, catalog=dmeRes.sourceCat).apCorrMap
518 dmeRes.exposure.getInfo().setApCorrMap(apCorrMap)
519 self.applyApCorr.
run(catalog=dmeRes.sourceCat, apCorrMap=exposure.getInfo().getApCorrMap())
520 self.catalogCalculation.
run(dmeRes.sourceCat)
522 self.
displaydisplay(
"measure", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
524 return pipeBase.Struct(
525 exposure=dmeRes.exposure,
526 sourceCat=dmeRes.sourceCat,
527 background=dmeRes.background,
528 psfCellSet=dmeRes.psfCellSet,
530 characterized=dmeRes.exposure,
531 backgroundModel=dmeRes.background
536 """!Perform one iteration of detect, measure and estimate PSF
538 Performs the following operations:
539 - if config.doMeasurePsf
or not exposure.hasPsf():
540 - install a simple PSF model (replacing the existing one,
if need be)
541 - interpolate over cosmic rays
with keepCRs=
True
542 - estimate background
and subtract it
from the exposure
543 - detect, deblend
and measure sources,
and subtract a refined background model;
544 -
if config.doMeasurePsf:
547 @param[
in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF
or similar)
548 The following changes are made:
550 - update detection
and cosmic ray mask planes
551 - subtract background
552 @param[
in] exposureIdInfo ID info
for exposure (an lsst.obs_base.ExposureIdInfo)
553 @param[
in,out] background initial model of background already subtracted
from exposure
554 (an lsst.afw.math.BackgroundList).
556 @return pipe_base Struct containing these fields, all
from the final iteration
557 of detect sources, measure sources
and estimate PSF:
558 - exposure characterized exposure; image
is repaired by interpolating over cosmic rays,
559 mask
is updated accordingly,
and the PSF model
is set
561 - background model of background subtracted
from exposure (an lsst.afw.math.BackgroundList)
565 if not exposure.hasPsf()
or (self.config.doMeasurePsf
and self.config.useSimplePsf):
566 self.log.info(
"PSF estimation initialized with 'simple' PSF")
567 self.installSimplePsf.
run(exposure=exposure)
570 if self.config.requireCrForPsf:
571 self.repair.
run(exposure=exposure, keepCRs=
True)
574 self.repair.
run(exposure=exposure, keepCRs=
True)
576 self.log.warning(
"Skipping cosmic ray detection: Too many CR pixels (max %0.f)",
577 self.config.repair.cosmicray.nCrPixelMax)
579 self.
displaydisplay(
"repair_iter", exposure=exposure)
581 if background
is None:
582 background = BackgroundList()
584 sourceIdFactory = exposureIdInfo.makeSourceIdFactory()
585 table = SourceTable.make(self.
schemaschema, sourceIdFactory)
588 detRes = self.detection.
run(table=table, exposure=exposure, doSmooth=
True)
589 sourceCat = detRes.sources
590 if detRes.fpSets.background:
591 for bg
in detRes.fpSets.background:
592 background.append(bg)
594 if self.config.doDeblend:
595 self.deblend.
run(exposure=exposure, sources=sourceCat)
597 self.measurement.
run(measCat=sourceCat, exposure=exposure, exposureId=exposureIdInfo.expId)
599 measPsfRes = pipeBase.Struct(cellSet=
None)
600 if self.config.doMeasurePsf:
601 if self.measurePsf.usesMatches:
602 matches = self.ref_match.loadAndMatch(exposure=exposure, sourceCat=sourceCat).matches
605 measPsfRes = self.measurePsf.
run(exposure=exposure, sources=sourceCat, matches=matches,
606 expId=exposureIdInfo.expId)
607 self.
displaydisplay(
"measure_iter", exposure=exposure, sourceCat=sourceCat)
609 return pipeBase.Struct(
612 background=background,
613 psfCellSet=measPsfRes.cellSet,
617 """Return a dict of empty catalogs for each catalog dataset produced by this task.
619 sourceCat = SourceCatalog(self.schemaschema)
620 sourceCat.getTable().setMetadata(self.algMetadataalgMetadata)
621 return {
"icSrc": sourceCat}
623 def display(self, itemName, exposure, sourceCat=None):
624 """Display exposure and sources on next frame, if display of itemName has been requested
626 @param[
in] itemName name of item
in debugInfo
627 @param[
in] exposure exposure to display
628 @param[
in] sourceCat source catalog to display
630 val = getDebugFrame(self._display, itemName)
634 displayAstrometry(exposure=exposure, sourceCat=sourceCat, frame=self.
_frame_frame, pause=
False)
Config for CharacterizeImageTask.
def adjustQuantum(self, inputs, outputs, label, dataId)
Measure bright sources and use this to estimate background and PSF of an exposure.
def getSchemaCatalogs(self)
def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs)
Construct a CharacterizeImageTask.
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def getInitOutputDatasets(self)
def runDataRef(self, dataRef, exposure=None, background=None, doUnpersist=True)
Characterize a science image and, if wanted, persist the results.
def run(self, exposure, exposureIdInfo=None, background=None)
Characterize a science image.
def detectMeasureAndEstimatePsf(self, exposure, exposureIdInfo, background)
Perform one iteration of detect, measure and estimate PSF.
def display(self, itemName, exposure, sourceCat=None)