22 from __future__
import absolute_import, division, print_function
25 from lsstDebug
import getDebugFrame
26 import lsst.pex.config
as pexConfig
27 import lsst.pipe.base
as pipeBase
28 import lsst.afw.table
as afwTable
29 from lsst.meas.astrom
import (AstrometryTask, displayAstrometry,
30 createMatchMetadata, denormalizeMatches)
31 from lsst.meas.extensions.astrometryNet
import LoadAstrometryNetObjectsTask
32 from lsst.obs.base
import ExposureIdInfo
33 import lsst.daf.base
as dafBase
34 from lsst.afw.math
import BackgroundList
35 from lsst.afw.table
import IdFactory, SourceTable
36 from lsst.meas.algorithms
import SourceDetectionTask
37 from lsst.meas.base
import (SingleFrameMeasurementTask, ApplyApCorrTask,
38 CatalogCalculationTask)
39 from lsst.meas.deblender
import SourceDeblendTask
40 from .fakes
import BaseFakeSourcesTask
41 from .photoCal
import PhotoCalTask
43 __all__ = [
"CalibrateConfig",
"CalibrateTask"]
47 """Config for CalibrateTask"""
48 doWrite = pexConfig.Field(
51 doc=
"Save calibration results?",
53 doWriteHeavyFootprintsInSources = pexConfig.Field(
56 doc=
"Include HeavyFootprint data in source table? If false then heavy "
57 "footprints are saved as normal footprints, which saves some space"
59 doWriteMatches = pexConfig.Field(
62 doc=
"Write reference matches (ignored if doWrite false)?",
64 doWriteMatchesDenormalized = pexConfig.Field(
67 doc=(
"Write reference matches in denormalized format? "
68 "This format uses more disk space, but is more convenient to "
69 "read. Ignored if doWriteMatches=False or doWrite=False."),
71 doAstrometry = pexConfig.Field(
74 doc=
"Perform astrometric calibration?",
76 astromRefObjLoader = pexConfig.ConfigurableField(
77 target=LoadAstrometryNetObjectsTask,
78 doc=
"reference object loader for astrometric calibration",
80 photoRefObjLoader = pexConfig.ConfigurableField(
81 target=LoadAstrometryNetObjectsTask,
82 doc=
"reference object loader for photometric calibration",
84 astrometry = pexConfig.ConfigurableField(
85 target=AstrometryTask,
86 doc=
"Perform astrometric calibration to refine the WCS",
88 requireAstrometry = pexConfig.Field(
91 doc=(
"Raise an exception if astrometry fails? Ignored if doAstrometry "
94 doPhotoCal = pexConfig.Field(
97 doc=
"Perform phometric calibration?",
99 requirePhotoCal = pexConfig.Field(
102 doc=(
"Raise an exception if photoCal fails? Ignored if doPhotoCal "
105 photoCal = pexConfig.ConfigurableField(
107 doc=
"Perform photometric calibration",
109 icSourceFieldsToCopy = pexConfig.ListField(
111 default=(
"calib_psfCandidate",
"calib_psfUsed",
"calib_psfReserved"),
112 doc=(
"Fields to copy from the icSource catalog to the output catalog "
113 "for matching sources Any missing fields will trigger a "
114 "RuntimeError exception. Ignored if icSourceCat is not provided.")
116 matchRadiusPix = pexConfig.Field(
119 doc=(
"Match radius for matching icSourceCat objects to sourceCat "
122 checkUnitsParseStrict = pexConfig.Field(
123 doc=(
"Strictness of Astropy unit compatibility check, can be 'raise', "
124 "'warn' or 'silent'"),
128 detection = pexConfig.ConfigurableField(
129 target=SourceDetectionTask,
132 doDeblend = pexConfig.Field(
135 doc=
"Run deblender input exposure"
137 deblend = pexConfig.ConfigurableField(
138 target=SourceDeblendTask,
139 doc=
"Split blended sources into their components"
141 measurement = pexConfig.ConfigurableField(
142 target=SingleFrameMeasurementTask,
143 doc=
"Measure sources"
145 doApCorr = pexConfig.Field(
148 doc=
"Run subtask to apply aperture correction"
150 applyApCorr = pexConfig.ConfigurableField(
151 target=ApplyApCorrTask,
152 doc=
"Subtask to apply aperture corrections"
157 catalogCalculation = pexConfig.ConfigurableField(
158 target=CatalogCalculationTask,
159 doc=
"Subtask to run catalogCalculation plugins on catalog"
161 doInsertFakes = pexConfig.Field(
164 doc=
"Run fake sources injection task"
166 insertFakes = pexConfig.ConfigurableField(
167 target=BaseFakeSourcesTask,
168 doc=
"Injection of fake sources for testing purposes (must be "
173 pexConfig.Config.setDefaults(self)
185 """!Calibrate an exposure: measure sources and perform astrometric and
186 photometric calibration
188 @anchor CalibrateTask_
190 @section pipe_tasks_calibrate_Contents Contents
192 - @ref pipe_tasks_calibrate_Purpose
193 - @ref pipe_tasks_calibrate_Initialize
194 - @ref pipe_tasks_calibrate_IO
195 - @ref pipe_tasks_calibrate_Config
196 - @ref pipe_tasks_calibrate_Metadata
197 - @ref pipe_tasks_calibrate_Debug
200 @section pipe_tasks_calibrate_Purpose Description
202 Given an exposure with a good PSF model and aperture correction map
203 (e.g. as provided by @ref CharacterizeImageTask), perform the following
205 - Run detection and measurement
206 - Run astrometry subtask to fit an improved WCS
207 - Run photoCal subtask to fit the exposure's photometric zero-point
209 @section pipe_tasks_calibrate_Initialize Task initialisation
211 @copydoc \_\_init\_\_
213 @section pipe_tasks_calibrate_IO Invoking the Task
215 If you want this task to unpersist inputs or persist outputs, then call
216 the `run` method (a wrapper around the `calibrate` method).
218 If you already have the inputs unpersisted and do not want to persist the
219 output then it is more direct to call the `calibrate` method:
221 @section pipe_tasks_calibrate_Config Configuration parameters
223 See @ref CalibrateConfig
225 @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata
229 <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task
230 <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal
232 <dt>COLORTERM1 <dd>?? (always 0.0)
233 <dt>COLORTERM2 <dd>?? (always 0.0)
234 <dt>COLORTERM3 <dd>?? (always 0.0)
237 @section pipe_tasks_calibrate_Debug Debug variables
239 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink
240 interface supports a flag
241 `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug
242 for more about `debug.py`.
244 CalibrateTask has a debug dictionary containing one key:
247 <dd>frame (an int; <= 0 to not display) in which to display the exposure,
248 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for
249 the meaning of the various symbols.
252 For example, put something like:
256 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would
257 # call us recursively
258 if name == "lsst.pipe.tasks.calibrate":
265 lsstDebug.Info = DebugInfo
267 into your `debug.py` file and run `calibrateTask.py` with the `--debug`
270 Some subtasks may have their own debug variables; see individual Task
277 ConfigClass = CalibrateConfig
278 _DefaultName =
"calibrate"
279 RunnerClass = pipeBase.ButlerInitializedTaskRunner
281 def __init__(self, butler=None, astromRefObjLoader=None,
282 photoRefObjLoader=
None, icSourceSchema=
None, **kwargs):
283 """!Construct a CalibrateTask
285 @param[in] butler The butler is passed to the refObjLoader constructor
286 in case it is needed. Ignored if the refObjLoader argument
287 provides a loader directly.
288 @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks
289 that supplies an external reference catalog for astrometric
290 calibration. May be None if the desired loader can be constructed
291 from the butler argument or all steps requiring a reference catalog
293 @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks
294 that supplies an external reference catalog for photometric
295 calibration. May be None if the desired loader can be constructed
296 from the butler argument or all steps requiring a reference catalog
298 @param[in] icSourceSchema schema for icSource catalog, or None.
299 Schema values specified in config.icSourceFieldsToCopy will be
300 taken from this schema. If set to None, no values will be
301 propagated from the icSourceCatalog
302 @param[in,out] kwargs other keyword arguments for
303 lsst.pipe.base.CmdLineTask
305 pipeBase.CmdLineTask.__init__(self, **kwargs)
307 if icSourceSchema
is None and butler
is not None:
309 icSourceSchema = butler.get(
"icSrc_schema", immediate=
True).schema
311 if icSourceSchema
is not None:
314 minimumSchema = afwTable.SourceTable.makeMinimalSchema()
315 self.schemaMapper.addMinimalSchema(minimumSchema,
False)
323 afwTable.Field[
"Flag"](
"calib_detected",
324 "Source was detected as an icSource"))
325 missingFieldNames = []
326 for fieldName
in self.config.icSourceFieldsToCopy:
328 schemaItem = icSourceSchema.find(fieldName)
330 missingFieldNames.append(fieldName)
333 self.schemaMapper.addMapping(schemaItem.getKey())
335 if missingFieldNames:
336 raise RuntimeError(
"isSourceCat is missing fields {} "
337 "specified in icSourceFieldsToCopy"
338 .format(missingFieldNames))
342 self.
schema = self.schemaMapper.editOutputSchema()
345 self.
schema = afwTable.SourceTable.makeMinimalSchema()
346 self.makeSubtask(
'detection', schema=self.
schema)
353 if self.config.doInsertFakes:
354 self.makeSubtask(
"insertFakes")
356 if self.config.doDeblend:
357 self.makeSubtask(
"deblend", schema=self.
schema)
358 self.makeSubtask(
'measurement', schema=self.
schema,
360 if self.config.doApCorr:
361 self.makeSubtask(
'applyApCorr', schema=self.
schema)
362 self.makeSubtask(
'catalogCalculation', schema=self.
schema)
364 if self.config.doAstrometry:
365 if astromRefObjLoader
is None:
366 self.makeSubtask(
'astromRefObjLoader', butler=butler)
367 astromRefObjLoader = self.astromRefObjLoader
369 self.makeSubtask(
"astrometry", refObjLoader=astromRefObjLoader,
371 if self.config.doPhotoCal:
372 if photoRefObjLoader
is None:
373 self.makeSubtask(
'photoRefObjLoader', butler=butler)
374 photoRefObjLoader = self.photoRefObjLoader
375 self.
pixelMargin = photoRefObjLoader.config.pixelMargin
376 self.makeSubtask(
"photoCal", refObjLoader=photoRefObjLoader,
381 self.
schema = self.schemaMapper.getOutputSchema()
382 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
385 def run(self, dataRef, exposure=None, background=None, icSourceCat=None,
387 """!Calibrate an exposure, optionally unpersisting inputs and
390 This is a wrapper around the `calibrate` method that unpersists inputs
391 (if `doUnpersist` true) and persists outputs (if `config.doWrite` true)
393 @param[in] dataRef butler data reference corresponding to a science
395 @param[in,out] exposure characterized exposure (an
396 lsst.afw.image.ExposureF or similar), or None to unpersist existing
397 icExp and icBackground. See calibrate method for details of what is
399 @param[in,out] background initial model of background already
400 subtracted from exposure (an lsst.afw.math.BackgroundList). May be
401 None if no background has been subtracted, though that is unusual
402 for calibration. A refined background model is output. Ignored if
404 @param[in] icSourceCat catalog from which to copy the fields specified
405 by icSourceKeys, or None;
406 @param[in] doUnpersist unpersist data:
407 - if True, exposure, background and icSourceCat are read from
408 dataRef and those three arguments must all be None;
409 - if False the exposure must be provided; background and
410 icSourceCat are optional. True is intended for running as a
411 command-line task, False for running as a subtask
413 @return same data as the calibrate method
415 self.log.info(
"Processing %s" % (dataRef.dataId))
418 if any(item
is not None for item
in (exposure, background,
420 raise RuntimeError(
"doUnpersist true; exposure, background "
421 "and icSourceCat must all be None")
422 exposure = dataRef.get(
"icExp", immediate=
True)
423 background = dataRef.get(
"icExpBackground", immediate=
True)
424 icSourceCat = dataRef.get(
"icSrc", immediate=
True)
425 elif exposure
is None:
426 raise RuntimeError(
"doUnpersist false; exposure must be provided")
428 if self.config.doWrite
and self.config.doAstrometry:
429 matchMeta = createMatchMetadata(exposure, border=self.
pixelMargin)
433 exposureIdInfo = dataRef.get(
"expIdInfo")
437 exposureIdInfo=exposureIdInfo,
438 background=background,
439 icSourceCat=icSourceCat,
442 if self.config.doWrite:
445 exposure=calRes.exposure,
446 background=calRes.background,
447 sourceCat=calRes.sourceCat,
448 astromMatches=calRes.astromMatches,
454 def calibrate(self, exposure, exposureIdInfo=None, background=None,
456 """!Calibrate an exposure (science image or coadd)
458 @param[in,out] exposure exposure to calibrate (an
459 lsst.afw.image.ExposureF or similar);
464 - MaskedImage has background subtracted
466 - Calib zero-point is set
467 @param[in] exposureIdInfo ID info for exposure (an
468 lsst.obs.base.ExposureIdInfo) If not provided, returned
469 SourceCatalog IDs will not be globally unique.
470 @param[in,out] background background model already subtracted from
471 exposure (an lsst.afw.math.BackgroundList). May be None if no
472 background has been subtracted, though that is unusual for
473 calibration. A refined background model is output.
474 @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
475 from which we can copy some fields.
477 @return pipe_base Struct containing these fields:
478 - exposure calibrate science exposure with refined WCS and Calib
479 - background model of background subtracted from exposure (an
480 lsst.afw.math.BackgroundList)
481 - sourceCat catalog of measured sources
482 - astromMatches list of source/refObj matches from the astrometry
486 if exposureIdInfo
is None:
487 exposureIdInfo = ExposureIdInfo()
489 if background
is None:
490 background = BackgroundList()
491 sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
492 exposureIdInfo.unusedBits)
493 table = SourceTable.make(self.
schema, sourceIdFactory)
496 if self.config.doInsertFakes:
497 self.insertFakes.run(exposure, background=background)
499 detRes = self.detection.run(table=table, exposure=exposure,
501 sourceCat = detRes.sources
502 if detRes.fpSets.background:
503 background.append(detRes.fpSets.background)
504 if self.config.doDeblend:
505 self.deblend.run(exposure=exposure, sources=sourceCat)
506 self.measurement.run(
509 exposureId=exposureIdInfo.expId
511 if self.config.doApCorr:
512 self.applyApCorr.run(
514 apCorrMap=exposure.getInfo().getApCorrMap()
516 self.catalogCalculation.run(sourceCat)
518 if icSourceCat
is not None and \
519 len(self.config.icSourceFieldsToCopy) > 0:
526 if self.config.doAstrometry:
528 astromRes = self.astrometry.run(
532 astromMatches = astromRes.matches
533 except Exception
as e:
534 if self.config.requireAstrometry:
536 self.log.warn(
"Unable to perform astrometric calibration "
537 "(%s): attempting to proceed" % e)
540 if self.config.doPhotoCal:
542 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat)
543 exposure.getCalib().setFluxMag0(photoRes.calib.getFluxMag0())
544 self.log.info(
"Photometric zero-point: %f" %
545 photoRes.calib.getMagnitude(1.0))
546 self.
setMetadata(exposure=exposure, photoRes=photoRes)
547 except Exception
as e:
548 if self.config.requirePhotoCal:
550 self.log.warn(
"Unable to perform photometric calibration "
551 "(%s): attempting to proceed" % e)
554 frame = getDebugFrame(self._display,
"calibrate")
559 matches=astromMatches,
564 return pipeBase.Struct(
566 background=background,
568 astromMatches=astromMatches,
571 def writeOutputs(self, dataRef, exposure, background, sourceCat,
572 astromMatches, matchMeta):
573 """Write output data to the output repository
575 @param[in] dataRef butler data reference corresponding to a science
577 @param[in] exposure exposure to write
578 @param[in] background background model for exposure
579 @param[in] sourceCat catalog of measured sources
580 @param[in] astromMatches list of source/refObj matches from the
583 sourceWriteFlags = 0
if self.config.doWriteHeavyFootprintsInSources \
584 else afwTable.SOURCE_IO_NO_HEAVY_FOOTPRINTS
585 dataRef.put(sourceCat,
"src")
586 if self.config.doWriteMatches
and astromMatches
is not None:
587 normalizedMatches = afwTable.packMatches(astromMatches)
588 normalizedMatches.table.setMetadata(matchMeta)
589 dataRef.put(normalizedMatches,
"srcMatch")
590 if self.config.doWriteMatchesDenormalized:
591 denormMatches = denormalizeMatches(astromMatches, matchMeta)
592 dataRef.put(denormMatches,
"srcMatchFull")
593 dataRef.put(exposure,
"calexp")
594 dataRef.put(background,
"calexpBackground")
597 """Return a dict of empty catalogs for each catalog dataset produced
600 sourceCat = afwTable.SourceCatalog(self.
schema)
602 return {
"src": sourceCat}
605 """!Set task and exposure metadata
607 Logs a warning and continues if needed data is missing.
609 @param[in,out] exposure exposure whose metadata is to be set
610 @param[in] photoRes results of running photoCal; if None then it was
618 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
619 magZero = photoRes.zp - 2.5*math.log10(exposureTime)
620 self.metadata.set(
'MAGZERO', magZero)
622 self.log.warn(
"Could not set normalized MAGZERO in header: no "
626 metadata = exposure.getMetadata()
627 metadata.set(
'MAGZERO_RMS', photoRes.sigma)
628 metadata.set(
'MAGZERO_NOBJ', photoRes.ngood)
629 metadata.set(
'COLORTERM1', 0.0)
630 metadata.set(
'COLORTERM2', 0.0)
631 metadata.set(
'COLORTERM3', 0.0)
632 except Exception
as e:
633 self.log.warn(
"Could not set exposure metadata: %s" % (e,))
636 """!Match sources in icSourceCat and sourceCat and copy the specified fields
638 @param[in] icSourceCat catalog from which to copy fields
639 @param[in,out] sourceCat catalog to which to copy fields
641 The fields copied are those specified by `config.icSourceFieldsToCopy`
642 that actually exist in the schema. This was set up by the constructor
643 using self.schemaMapper.
646 raise RuntimeError(
"To copy icSource fields you must specify "
647 "icSourceSchema nd icSourceKeys when "
648 "constructing this task")
649 if icSourceCat
is None or sourceCat
is None:
650 raise RuntimeError(
"icSourceCat and sourceCat must both be "
652 if len(self.config.icSourceFieldsToCopy) == 0:
653 self.log.warn(
"copyIcSourceFields doing nothing because "
654 "icSourceFieldsToCopy is empty")
657 mc = afwTable.MatchControl()
658 mc.findOnlyClosest =
False
659 matches = afwTable.matchXy(icSourceCat, sourceCat,
660 self.config.matchRadiusPix, mc)
661 if self.config.doDeblend:
662 deblendKey = sourceCat.schema[
"deblend_nChild"].asKey()
664 matches = [m
for m
in matches
if m[1].get(deblendKey) == 0]
671 for m0, m1, d
in matches:
673 match = bestMatches.get(id0)
674 if match
is None or d <= match[2]:
675 bestMatches[id0] = (m0, m1, d)
676 matches = list(bestMatches.values())
681 numMatches = len(matches)
682 numUniqueSources = len(set(m[1].getId()
for m
in matches))
683 if numUniqueSources != numMatches:
684 self.log.warn(
"{} icSourceCat sources matched only {} sourceCat "
685 "sources".format(numMatches, numUniqueSources))
687 self.log.info(
"Copying flags from icSourceCat to sourceCat for "
688 "%s sources" % (numMatches,))
692 for icSrc, src, d
in matches:
698 icSrcFootprint = icSrc.getFootprint()
700 icSrc.setFootprint(src.getFootprint())
703 icSrc.setFootprint(icSrcFootprint)
def __init__
Construct a CalibrateTask.
def setMetadata
Set task and exposure metadata.
def calibrate
Calibrate an exposure (science image or coadd)
def copyIcSourceFields
Match sources in icSourceCat and sourceCat and copy the specified fields.
def run
Calibrate an exposure, optionally unpersisting inputs and persisting outputs.
Calibrate an exposure: measure sources and perform astrometric and photometric calibration.