22 from __future__
import absolute_import, division, print_function
25 from lsstDebug
import getDebugFrame
26 import lsst.pex.config
as pexConfig
27 import lsst.pipe.base
as pipeBase
28 import lsst.afw.table
as afwTable
29 from lsst.meas.astrom
import AstrometryTask, displayAstrometry, denormalizeMatches
30 from lsst.meas.extensions.astrometryNet
import LoadAstrometryNetObjectsTask
31 from lsst.obs.base
import ExposureIdInfo
32 import lsst.daf.base
as dafBase
33 from lsst.afw.math
import BackgroundList
34 from lsst.afw.table
import IdFactory, SourceTable
35 from lsst.meas.algorithms
import SourceDetectionTask
36 from lsst.meas.base
import (SingleFrameMeasurementTask, ApplyApCorrTask,
37 CatalogCalculationTask)
38 from lsst.meas.deblender
import SourceDeblendTask
39 from .fakes
import BaseFakeSourcesTask
40 from .photoCal
import PhotoCalTask
42 __all__ = [
"CalibrateConfig",
"CalibrateTask"]
46 """Config for CalibrateTask"""
47 doWrite = pexConfig.Field(
50 doc=
"Save calibration results?",
52 doWriteHeavyFootprintsInSources = pexConfig.Field(
55 doc=
"Include HeavyFootprint data in source table? If false then heavy "
56 "footprints are saved as normal footprints, which saves some space"
58 doWriteMatches = pexConfig.Field(
61 doc=
"Write reference matches (ignored if doWrite false)?",
63 doWriteMatchesDenormalized = pexConfig.Field(
66 doc=(
"Write reference matches in denormalized format? "
67 "This format uses more disk space, but is more convenient to "
68 "read. Ignored if doWriteMatches=False or doWrite=False."),
70 doAstrometry = pexConfig.Field(
73 doc=
"Perform astrometric calibration?",
75 astromRefObjLoader = pexConfig.ConfigurableField(
76 target=LoadAstrometryNetObjectsTask,
77 doc=
"reference object loader for astrometric calibration",
79 photoRefObjLoader = pexConfig.ConfigurableField(
80 target=LoadAstrometryNetObjectsTask,
81 doc=
"reference object loader for photometric calibration",
83 astrometry = pexConfig.ConfigurableField(
84 target=AstrometryTask,
85 doc=
"Perform astrometric calibration to refine the WCS",
87 requireAstrometry = pexConfig.Field(
90 doc=(
"Raise an exception if astrometry fails? Ignored if doAstrometry "
93 doPhotoCal = pexConfig.Field(
96 doc=
"Perform phometric calibration?",
98 requirePhotoCal = pexConfig.Field(
101 doc=(
"Raise an exception if photoCal fails? Ignored if doPhotoCal "
104 photoCal = pexConfig.ConfigurableField(
106 doc=
"Perform photometric calibration",
108 icSourceFieldsToCopy = pexConfig.ListField(
110 default=(
"calib_psfCandidate",
"calib_psfUsed",
"calib_psfReserved"),
111 doc=(
"Fields to copy from the icSource catalog to the output catalog "
112 "for matching sources Any missing fields will trigger a "
113 "RuntimeError exception. Ignored if icSourceCat is not provided.")
115 matchRadiusPix = pexConfig.Field(
118 doc=(
"Match radius for matching icSourceCat objects to sourceCat "
121 checkUnitsParseStrict = pexConfig.Field(
122 doc=(
"Strictness of Astropy unit compatibility check, can be 'raise', "
123 "'warn' or 'silent'"),
127 detection = pexConfig.ConfigurableField(
128 target=SourceDetectionTask,
131 doDeblend = pexConfig.Field(
134 doc=
"Run deblender input exposure"
136 deblend = pexConfig.ConfigurableField(
137 target=SourceDeblendTask,
138 doc=
"Split blended sources into their components"
140 measurement = pexConfig.ConfigurableField(
141 target=SingleFrameMeasurementTask,
142 doc=
"Measure sources"
144 doApCorr = pexConfig.Field(
147 doc=
"Run subtask to apply aperture correction"
149 applyApCorr = pexConfig.ConfigurableField(
150 target=ApplyApCorrTask,
151 doc=
"Subtask to apply aperture corrections"
156 catalogCalculation = pexConfig.ConfigurableField(
157 target=CatalogCalculationTask,
158 doc=
"Subtask to run catalogCalculation plugins on catalog"
160 doInsertFakes = pexConfig.Field(
163 doc=
"Run fake sources injection task"
165 insertFakes = pexConfig.ConfigurableField(
166 target=BaseFakeSourcesTask,
167 doc=
"Injection of fake sources for testing purposes (must be "
172 pexConfig.Config.setDefaults(self)
184 """!Calibrate an exposure: measure sources and perform astrometric and
185 photometric calibration
187 @anchor CalibrateTask_
189 @section pipe_tasks_calibrate_Contents Contents
191 - @ref pipe_tasks_calibrate_Purpose
192 - @ref pipe_tasks_calibrate_Initialize
193 - @ref pipe_tasks_calibrate_IO
194 - @ref pipe_tasks_calibrate_Config
195 - @ref pipe_tasks_calibrate_Metadata
196 - @ref pipe_tasks_calibrate_Debug
199 @section pipe_tasks_calibrate_Purpose Description
201 Given an exposure with a good PSF model and aperture correction map
202 (e.g. as provided by @ref CharacterizeImageTask), perform the following
204 - Run detection and measurement
205 - Run astrometry subtask to fit an improved WCS
206 - Run photoCal subtask to fit the exposure's photometric zero-point
208 @section pipe_tasks_calibrate_Initialize Task initialisation
210 @copydoc \_\_init\_\_
212 @section pipe_tasks_calibrate_IO Invoking the Task
214 If you want this task to unpersist inputs or persist outputs, then call
215 the `run` method (a wrapper around the `calibrate` method).
217 If you already have the inputs unpersisted and do not want to persist the
218 output then it is more direct to call the `calibrate` method:
220 @section pipe_tasks_calibrate_Config Configuration parameters
222 See @ref CalibrateConfig
224 @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata
228 <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task
229 <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal
231 <dt>COLORTERM1 <dd>?? (always 0.0)
232 <dt>COLORTERM2 <dd>?? (always 0.0)
233 <dt>COLORTERM3 <dd>?? (always 0.0)
236 @section pipe_tasks_calibrate_Debug Debug variables
238 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink
239 interface supports a flag
240 `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug
241 for more about `debug.py`.
243 CalibrateTask has a debug dictionary containing one key:
246 <dd>frame (an int; <= 0 to not display) in which to display the exposure,
247 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for
248 the meaning of the various symbols.
251 For example, put something like:
255 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would
256 # call us recursively
257 if name == "lsst.pipe.tasks.calibrate":
264 lsstDebug.Info = DebugInfo
266 into your `debug.py` file and run `calibrateTask.py` with the `--debug`
269 Some subtasks may have their own debug variables; see individual Task
276 ConfigClass = CalibrateConfig
277 _DefaultName =
"calibrate"
278 RunnerClass = pipeBase.ButlerInitializedTaskRunner
280 def __init__(self, butler=None, astromRefObjLoader=None,
281 photoRefObjLoader=
None, icSourceSchema=
None, **kwargs):
282 """!Construct a CalibrateTask
284 @param[in] butler The butler is passed to the refObjLoader constructor
285 in case it is needed. Ignored if the refObjLoader argument
286 provides a loader directly.
287 @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks
288 that supplies an external reference catalog for astrometric
289 calibration. May be None if the desired loader can be constructed
290 from the butler argument or all steps requiring a reference catalog
292 @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks
293 that supplies an external reference catalog for photometric
294 calibration. May be None if the desired loader can be constructed
295 from the butler argument or all steps requiring a reference catalog
297 @param[in] icSourceSchema schema for icSource catalog, or None.
298 Schema values specified in config.icSourceFieldsToCopy will be
299 taken from this schema. If set to None, no values will be
300 propagated from the icSourceCatalog
301 @param[in,out] kwargs other keyword arguments for
302 lsst.pipe.base.CmdLineTask
304 pipeBase.CmdLineTask.__init__(self, **kwargs)
306 if icSourceSchema
is None and butler
is not None:
308 icSourceSchema = butler.get(
"icSrc_schema", immediate=
True).schema
310 if icSourceSchema
is not None:
313 minimumSchema = afwTable.SourceTable.makeMinimalSchema()
314 self.schemaMapper.addMinimalSchema(minimumSchema,
False)
322 afwTable.Field[
"Flag"](
"calib_detected",
323 "Source was detected as an icSource"))
324 missingFieldNames = []
325 for fieldName
in self.config.icSourceFieldsToCopy:
327 schemaItem = icSourceSchema.find(fieldName)
329 missingFieldNames.append(fieldName)
332 self.schemaMapper.addMapping(schemaItem.getKey())
334 if missingFieldNames:
335 raise RuntimeError(
"isSourceCat is missing fields {} "
336 "specified in icSourceFieldsToCopy"
337 .format(missingFieldNames))
341 self.
schema = self.schemaMapper.editOutputSchema()
344 self.
schema = afwTable.SourceTable.makeMinimalSchema()
345 self.makeSubtask(
'detection', schema=self.
schema)
352 if self.config.doInsertFakes:
353 self.makeSubtask(
"insertFakes")
355 if self.config.doDeblend:
356 self.makeSubtask(
"deblend", schema=self.
schema)
357 self.makeSubtask(
'measurement', schema=self.
schema,
359 if self.config.doApCorr:
360 self.makeSubtask(
'applyApCorr', schema=self.
schema)
361 self.makeSubtask(
'catalogCalculation', schema=self.
schema)
363 if self.config.doAstrometry:
364 if astromRefObjLoader
is None:
365 self.makeSubtask(
'astromRefObjLoader', butler=butler)
366 astromRefObjLoader = self.astromRefObjLoader
368 self.makeSubtask(
"astrometry", refObjLoader=astromRefObjLoader,
370 if self.config.doPhotoCal:
371 if photoRefObjLoader
is None:
372 self.makeSubtask(
'photoRefObjLoader', butler=butler)
373 photoRefObjLoader = self.photoRefObjLoader
374 self.
pixelMargin = photoRefObjLoader.config.pixelMargin
375 self.makeSubtask(
"photoCal", refObjLoader=photoRefObjLoader,
380 self.
schema = self.schemaMapper.getOutputSchema()
381 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
384 def run(self, dataRef, exposure=None, background=None, icSourceCat=None,
386 """!Calibrate an exposure, optionally unpersisting inputs and
389 This is a wrapper around the `calibrate` method that unpersists inputs
390 (if `doUnpersist` true) and persists outputs (if `config.doWrite` true)
392 @param[in] dataRef butler data reference corresponding to a science
394 @param[in,out] exposure characterized exposure (an
395 lsst.afw.image.ExposureF or similar), or None to unpersist existing
396 icExp and icBackground. See calibrate method for details of what is
398 @param[in,out] background initial model of background already
399 subtracted from exposure (an lsst.afw.math.BackgroundList). May be
400 None if no background has been subtracted, though that is unusual
401 for calibration. A refined background model is output. Ignored if
403 @param[in] icSourceCat catalog from which to copy the fields specified
404 by icSourceKeys, or None;
405 @param[in] doUnpersist unpersist data:
406 - if True, exposure, background and icSourceCat are read from
407 dataRef and those three arguments must all be None;
408 - if False the exposure must be provided; background and
409 icSourceCat are optional. True is intended for running as a
410 command-line task, False for running as a subtask
412 @return same data as the calibrate method
414 self.log.info(
"Processing %s" % (dataRef.dataId))
417 if any(item
is not None for item
in (exposure, background,
419 raise RuntimeError(
"doUnpersist true; exposure, background "
420 "and icSourceCat must all be None")
421 exposure = dataRef.get(
"icExp", immediate=
True)
422 background = dataRef.get(
"icExpBackground", immediate=
True)
423 icSourceCat = dataRef.get(
"icSrc", immediate=
True)
424 elif exposure
is None:
425 raise RuntimeError(
"doUnpersist false; exposure must be provided")
427 exposureIdInfo = dataRef.get(
"expIdInfo")
431 exposureIdInfo=exposureIdInfo,
432 background=background,
433 icSourceCat=icSourceCat,
436 if self.config.doWrite:
439 exposure=calRes.exposure,
440 background=calRes.background,
441 sourceCat=calRes.sourceCat,
442 astromMatches=calRes.astromMatches,
443 matchMeta=calRes.matchMeta,
448 def calibrate(self, exposure, exposureIdInfo=None, background=None,
450 """!Calibrate an exposure (science image or coadd)
452 @param[in,out] exposure exposure to calibrate (an
453 lsst.afw.image.ExposureF or similar);
458 - MaskedImage has background subtracted
460 - Calib zero-point is set
461 @param[in] exposureIdInfo ID info for exposure (an
462 lsst.obs.base.ExposureIdInfo) If not provided, returned
463 SourceCatalog IDs will not be globally unique.
464 @param[in,out] background background model already subtracted from
465 exposure (an lsst.afw.math.BackgroundList). May be None if no
466 background has been subtracted, though that is unusual for
467 calibration. A refined background model is output.
468 @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
469 from which we can copy some fields.
471 @return pipe_base Struct containing these fields:
472 - exposure calibrate science exposure with refined WCS and Calib
473 - background model of background subtracted from exposure (an
474 lsst.afw.math.BackgroundList)
475 - sourceCat catalog of measured sources
476 - astromMatches list of source/refObj matches from the astrometry
480 if exposureIdInfo
is None:
481 exposureIdInfo = ExposureIdInfo()
483 if background
is None:
484 background = BackgroundList()
485 sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
486 exposureIdInfo.unusedBits)
487 table = SourceTable.make(self.
schema, sourceIdFactory)
490 if self.config.doInsertFakes:
491 self.insertFakes.run(exposure, background=background)
493 detRes = self.detection.run(table=table, exposure=exposure,
495 sourceCat = detRes.sources
496 if detRes.fpSets.background:
497 background.append(detRes.fpSets.background)
498 if self.config.doDeblend:
499 self.deblend.run(exposure=exposure, sources=sourceCat)
500 self.measurement.run(
503 exposureId=exposureIdInfo.expId
505 if self.config.doApCorr:
506 self.applyApCorr.run(
508 apCorrMap=exposure.getInfo().getApCorrMap()
510 self.catalogCalculation.run(sourceCat)
512 if icSourceCat
is not None and \
513 len(self.config.icSourceFieldsToCopy) > 0:
521 if self.config.doAstrometry:
523 astromRes = self.astrometry.run(
527 astromMatches = astromRes.matches
528 matchMeta = astromRes.matchMeta
529 except Exception
as e:
530 if self.config.requireAstrometry:
532 self.log.warn(
"Unable to perform astrometric calibration "
533 "(%s): attempting to proceed" % e)
536 if self.config.doPhotoCal:
538 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat)
539 exposure.getCalib().setFluxMag0(photoRes.calib.getFluxMag0())
540 self.log.info(
"Photometric zero-point: %f" %
541 photoRes.calib.getMagnitude(1.0))
542 self.
setMetadata(exposure=exposure, photoRes=photoRes)
543 except Exception
as e:
544 if self.config.requirePhotoCal:
546 self.log.warn(
"Unable to perform photometric calibration "
547 "(%s): attempting to proceed" % e)
550 frame = getDebugFrame(self._display,
"calibrate")
555 matches=astromMatches,
560 return pipeBase.Struct(
562 background=background,
564 astromMatches=astromMatches,
568 def writeOutputs(self, dataRef, exposure, background, sourceCat,
569 astromMatches, matchMeta):
570 """Write output data to the output repository
572 @param[in] dataRef butler data reference corresponding to a science
574 @param[in] exposure exposure to write
575 @param[in] background background model for exposure
576 @param[in] sourceCat catalog of measured sources
577 @param[in] astromMatches list of source/refObj matches from the
580 sourceWriteFlags = 0
if self.config.doWriteHeavyFootprintsInSources \
581 else afwTable.SOURCE_IO_NO_HEAVY_FOOTPRINTS
582 dataRef.put(sourceCat,
"src")
583 if self.config.doWriteMatches
and astromMatches
is not None:
584 normalizedMatches = afwTable.packMatches(astromMatches)
585 normalizedMatches.table.setMetadata(matchMeta)
586 dataRef.put(normalizedMatches,
"srcMatch")
587 if self.config.doWriteMatchesDenormalized:
588 denormMatches = denormalizeMatches(astromMatches, matchMeta)
589 dataRef.put(denormMatches,
"srcMatchFull")
590 dataRef.put(exposure,
"calexp")
591 dataRef.put(background,
"calexpBackground")
594 """Return a dict of empty catalogs for each catalog dataset produced
597 sourceCat = afwTable.SourceCatalog(self.
schema)
599 return {
"src": sourceCat}
602 """!Set task and exposure metadata
604 Logs a warning and continues if needed data is missing.
606 @param[in,out] exposure exposure whose metadata is to be set
607 @param[in] photoRes results of running photoCal; if None then it was
615 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
616 magZero = photoRes.zp - 2.5*math.log10(exposureTime)
617 self.metadata.set(
'MAGZERO', magZero)
619 self.log.warn(
"Could not set normalized MAGZERO in header: no "
623 metadata = exposure.getMetadata()
624 metadata.set(
'MAGZERO_RMS', photoRes.sigma)
625 metadata.set(
'MAGZERO_NOBJ', photoRes.ngood)
626 metadata.set(
'COLORTERM1', 0.0)
627 metadata.set(
'COLORTERM2', 0.0)
628 metadata.set(
'COLORTERM3', 0.0)
629 except Exception
as e:
630 self.log.warn(
"Could not set exposure metadata: %s" % (e,))
633 """!Match sources in icSourceCat and sourceCat and copy the specified fields
635 @param[in] icSourceCat catalog from which to copy fields
636 @param[in,out] sourceCat catalog to which to copy fields
638 The fields copied are those specified by `config.icSourceFieldsToCopy`
639 that actually exist in the schema. This was set up by the constructor
640 using self.schemaMapper.
643 raise RuntimeError(
"To copy icSource fields you must specify "
644 "icSourceSchema nd icSourceKeys when "
645 "constructing this task")
646 if icSourceCat
is None or sourceCat
is None:
647 raise RuntimeError(
"icSourceCat and sourceCat must both be "
649 if len(self.config.icSourceFieldsToCopy) == 0:
650 self.log.warn(
"copyIcSourceFields doing nothing because "
651 "icSourceFieldsToCopy is empty")
654 mc = afwTable.MatchControl()
655 mc.findOnlyClosest =
False
656 matches = afwTable.matchXy(icSourceCat, sourceCat,
657 self.config.matchRadiusPix, mc)
658 if self.config.doDeblend:
659 deblendKey = sourceCat.schema[
"deblend_nChild"].asKey()
661 matches = [m
for m
in matches
if m[1].get(deblendKey) == 0]
668 for m0, m1, d
in matches:
670 match = bestMatches.get(id0)
671 if match
is None or d <= match[2]:
672 bestMatches[id0] = (m0, m1, d)
673 matches = list(bestMatches.values())
678 numMatches = len(matches)
679 numUniqueSources = len(set(m[1].getId()
for m
in matches))
680 if numUniqueSources != numMatches:
681 self.log.warn(
"{} icSourceCat sources matched only {} sourceCat "
682 "sources".format(numMatches, numUniqueSources))
684 self.log.info(
"Copying flags from icSourceCat to sourceCat for "
685 "%s sources" % (numMatches,))
689 for icSrc, src, d
in matches:
695 icSrcFootprint = icSrc.getFootprint()
697 icSrc.setFootprint(src.getFootprint())
700 icSrc.setFootprint(icSrcFootprint)
def __init__
Construct a CalibrateTask.
def setMetadata
Set task and exposure metadata.
def calibrate
Calibrate an exposure (science image or coadd)
def copyIcSourceFields
Match sources in icSourceCat and sourceCat and copy the specified fields.
def run
Calibrate an exposure, optionally unpersisting inputs and persisting outputs.
Calibrate an exposure: measure sources and perform astrometric and photometric calibration.