22 from __future__
import absolute_import, division, print_function
25 from lsstDebug
import getDebugFrame
26 import lsst.pex.config
as pexConfig
27 import lsst.pipe.base
as pipeBase
28 import lsst.afw.table
as afwTable
29 from lsst.meas.astrom
import (AstrometryTask, displayAstrometry,
30 createMatchMetadata, denormalizeMatches)
31 from lsst.meas.extensions.astrometryNet
import LoadAstrometryNetObjectsTask
32 from lsst.obs.base
import ExposureIdInfo
33 import lsst.daf.base
as dafBase
34 from lsst.afw.math
import BackgroundList
35 from lsst.afw.table
import IdFactory, SourceTable
36 from lsst.meas.algorithms
import SourceDetectionTask
37 from lsst.meas.base
import (SingleFrameMeasurementTask, ApplyApCorrTask,
38 CatalogCalculationTask)
39 from lsst.meas.deblender
import SourceDeblendTask
40 from .fakes
import BaseFakeSourcesTask
41 from .photoCal
import PhotoCalTask
43 __all__ = [
"CalibrateConfig",
"CalibrateTask"]
47 """Config for CalibrateTask""" 48 doWrite = pexConfig.Field(
51 doc=
"Save calibration results?",
53 doWriteHeavyFootprintsInSources = pexConfig.Field(
56 doc=
"Include HeavyFootprint data in source table? If false then heavy " 57 "footprints are saved as normal footprints, which saves some space" 59 doWriteMatches = pexConfig.Field(
62 doc=
"Write reference matches (ignored if doWrite false)?",
64 doWriteMatchesDenormalized = pexConfig.Field(
67 doc=(
"Write reference matches in denormalized format? " 68 "This format uses more disk space, but is more convenient to " 69 "read. Ignored if doWriteMatches=False or doWrite=False."),
71 doAstrometry = pexConfig.Field(
74 doc=
"Perform astrometric calibration?",
76 astromRefObjLoader = pexConfig.ConfigurableField(
77 target=LoadAstrometryNetObjectsTask,
78 doc=
"reference object loader for astrometric calibration",
80 photoRefObjLoader = pexConfig.ConfigurableField(
81 target=LoadAstrometryNetObjectsTask,
82 doc=
"reference object loader for photometric calibration",
84 astrometry = pexConfig.ConfigurableField(
85 target=AstrometryTask,
86 doc=
"Perform astrometric calibration to refine the WCS",
88 requireAstrometry = pexConfig.Field(
91 doc=(
"Raise an exception if astrometry fails? Ignored if doAstrometry " 94 doPhotoCal = pexConfig.Field(
97 doc=
"Perform phometric calibration?",
99 requirePhotoCal = pexConfig.Field(
102 doc=(
"Raise an exception if photoCal fails? Ignored if doPhotoCal " 105 photoCal = pexConfig.ConfigurableField(
107 doc=
"Perform photometric calibration",
109 icSourceFieldsToCopy = pexConfig.ListField(
111 default=(
"calib_psfCandidate",
"calib_psfUsed",
"calib_psfReserved"),
112 doc=(
"Fields to copy from the icSource catalog to the output catalog " 113 "for matching sources Any missing fields will trigger a " 114 "RuntimeError exception. Ignored if icSourceCat is not provided.")
116 matchRadiusPix = pexConfig.Field(
119 doc=(
"Match radius for matching icSourceCat objects to sourceCat " 122 checkUnitsParseStrict = pexConfig.Field(
123 doc=(
"Strictness of Astropy unit compatibility check, can be 'raise', " 124 "'warn' or 'silent'"),
128 detection = pexConfig.ConfigurableField(
129 target=SourceDetectionTask,
132 doDeblend = pexConfig.Field(
135 doc=
"Run deblender input exposure" 137 deblend = pexConfig.ConfigurableField(
138 target=SourceDeblendTask,
139 doc=
"Split blended sources into their components" 141 measurement = pexConfig.ConfigurableField(
142 target=SingleFrameMeasurementTask,
143 doc=
"Measure sources" 145 doApCorr = pexConfig.Field(
148 doc=
"Run subtask to apply aperture correction" 150 applyApCorr = pexConfig.ConfigurableField(
151 target=ApplyApCorrTask,
152 doc=
"Subtask to apply aperture corrections" 157 catalogCalculation = pexConfig.ConfigurableField(
158 target=CatalogCalculationTask,
159 doc=
"Subtask to run catalogCalculation plugins on catalog" 161 doInsertFakes = pexConfig.Field(
164 doc=
"Run fake sources injection task" 166 insertFakes = pexConfig.ConfigurableField(
167 target=BaseFakeSourcesTask,
168 doc=
"Injection of fake sources for testing purposes (must be " 173 pexConfig.Config.setDefaults(self)
185 """!Calibrate an exposure: measure sources and perform astrometric and 186 photometric calibration 188 @anchor CalibrateTask_ 190 @section pipe_tasks_calibrate_Contents Contents 192 - @ref pipe_tasks_calibrate_Purpose 193 - @ref pipe_tasks_calibrate_Initialize 194 - @ref pipe_tasks_calibrate_IO 195 - @ref pipe_tasks_calibrate_Config 196 - @ref pipe_tasks_calibrate_Metadata 197 - @ref pipe_tasks_calibrate_Debug 200 @section pipe_tasks_calibrate_Purpose Description 202 Given an exposure with a good PSF model and aperture correction map 203 (e.g. as provided by @ref CharacterizeImageTask), perform the following 205 - Run detection and measurement 206 - Run astrometry subtask to fit an improved WCS 207 - Run photoCal subtask to fit the exposure's photometric zero-point 209 @section pipe_tasks_calibrate_Initialize Task initialisation 211 @copydoc \_\_init\_\_ 213 @section pipe_tasks_calibrate_IO Invoking the Task 215 If you want this task to unpersist inputs or persist outputs, then call 216 the `run` method (a wrapper around the `calibrate` method). 218 If you already have the inputs unpersisted and do not want to persist the 219 output then it is more direct to call the `calibrate` method: 221 @section pipe_tasks_calibrate_Config Configuration parameters 223 See @ref CalibrateConfig 225 @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata 229 <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task 230 <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal 232 <dt>COLORTERM1 <dd>?? (always 0.0) 233 <dt>COLORTERM2 <dd>?? (always 0.0) 234 <dt>COLORTERM3 <dd>?? (always 0.0) 237 @section pipe_tasks_calibrate_Debug Debug variables 239 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink 240 interface supports a flag 241 `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug 242 for more about `debug.py`. 244 CalibrateTask has a debug dictionary containing one key: 247 <dd>frame (an int; <= 0 to not display) in which to display the exposure, 248 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for 249 the meaning of the various symbols. 252 For example, put something like: 256 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would 257 # call us recursively 258 if name == "lsst.pipe.tasks.calibrate": 265 lsstDebug.Info = DebugInfo 267 into your `debug.py` file and run `calibrateTask.py` with the `--debug` 270 Some subtasks may have their own debug variables; see individual Task 277 ConfigClass = CalibrateConfig
278 _DefaultName =
"calibrate" 279 RunnerClass = pipeBase.ButlerInitializedTaskRunner
281 def __init__(self, butler=None, astromRefObjLoader=None,
282 photoRefObjLoader=None, icSourceSchema=None, **kwargs):
283 """!Construct a CalibrateTask 285 @param[in] butler The butler is passed to the refObjLoader constructor 286 in case it is needed. Ignored if the refObjLoader argument 287 provides a loader directly. 288 @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks 289 that supplies an external reference catalog for astrometric 290 calibration. May be None if the desired loader can be constructed 291 from the butler argument or all steps requiring a reference catalog 293 @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks 294 that supplies an external reference catalog for photometric 295 calibration. May be None if the desired loader can be constructed 296 from the butler argument or all steps requiring a reference catalog 298 @param[in] icSourceSchema schema for icSource catalog, or None. 299 Schema values specified in config.icSourceFieldsToCopy will be 300 taken from this schema. If set to None, no values will be 301 propagated from the icSourceCatalog 302 @param[in,out] kwargs other keyword arguments for 303 lsst.pipe.base.CmdLineTask 305 pipeBase.CmdLineTask.__init__(self, **kwargs)
307 if icSourceSchema
is None and butler
is not None:
309 icSourceSchema = butler.get(
"icSrc_schema", immediate=
True).schema
311 if icSourceSchema
is not None:
314 minimumSchema = afwTable.SourceTable.makeMinimalSchema()
315 self.
schemaMapper.addMinimalSchema(minimumSchema,
False)
323 afwTable.Field[
"Flag"](
"calib_detected",
324 "Source was detected as an icSource"))
325 missingFieldNames = []
326 for fieldName
in self.config.icSourceFieldsToCopy:
328 schemaItem = icSourceSchema.find(fieldName)
330 missingFieldNames.append(fieldName)
335 if missingFieldNames:
336 raise RuntimeError(
"isSourceCat is missing fields {} " 337 "specified in icSourceFieldsToCopy" 338 .format(missingFieldNames))
345 self.
schema = afwTable.SourceTable.makeMinimalSchema()
346 self.makeSubtask(
'detection', schema=self.
schema)
353 if self.config.doInsertFakes:
354 self.makeSubtask(
"insertFakes")
356 if self.config.doDeblend:
357 self.makeSubtask(
"deblend", schema=self.
schema)
358 self.makeSubtask(
'measurement', schema=self.
schema,
360 if self.config.doApCorr:
361 self.makeSubtask(
'applyApCorr', schema=self.
schema)
362 self.makeSubtask(
'catalogCalculation', schema=self.
schema)
364 if self.config.doAstrometry:
365 if astromRefObjLoader
is None:
366 self.makeSubtask(
'astromRefObjLoader', butler=butler)
367 astromRefObjLoader = self.astromRefObjLoader
369 self.makeSubtask(
"astrometry", refObjLoader=astromRefObjLoader,
371 if self.config.doPhotoCal:
372 if photoRefObjLoader
is None:
373 self.makeSubtask(
'photoRefObjLoader', butler=butler)
374 photoRefObjLoader = self.photoRefObjLoader
375 self.
pixelMargin = photoRefObjLoader.config.pixelMargin
376 self.makeSubtask(
"photoCal", refObjLoader=photoRefObjLoader,
382 self.
schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
385 def run(self, dataRef, exposure=None, background=None, icSourceCat=None,
387 """!Calibrate an exposure, optionally unpersisting inputs and 390 This is a wrapper around the `calibrate` method that unpersists inputs 391 (if `doUnpersist` true) and persists outputs (if `config.doWrite` true) 393 @param[in] dataRef butler data reference corresponding to a science 395 @param[in,out] exposure characterized exposure (an 396 lsst.afw.image.ExposureF or similar), or None to unpersist existing 397 icExp and icBackground. See calibrate method for details of what is 399 @param[in,out] background initial model of background already 400 subtracted from exposure (an lsst.afw.math.BackgroundList). May be 401 None if no background has been subtracted, though that is unusual 402 for calibration. A refined background model is output. Ignored if 404 @param[in] icSourceCat catalog from which to copy the fields specified 405 by icSourceKeys, or None; 406 @param[in] doUnpersist unpersist data: 407 - if True, exposure, background and icSourceCat are read from 408 dataRef and those three arguments must all be None; 409 - if False the exposure must be provided; background and 410 icSourceCat are optional. True is intended for running as a 411 command-line task, False for running as a subtask 413 @return same data as the calibrate method 415 self.log.info(
"Processing %s" % (dataRef.dataId))
418 if any(item
is not None for item
in (exposure, background,
420 raise RuntimeError(
"doUnpersist true; exposure, background " 421 "and icSourceCat must all be None")
422 exposure = dataRef.get(
"icExp", immediate=
True)
423 background = dataRef.get(
"icExpBackground", immediate=
True)
424 icSourceCat = dataRef.get(
"icSrc", immediate=
True)
425 elif exposure
is None:
426 raise RuntimeError(
"doUnpersist false; exposure must be provided")
428 if self.config.doWrite
and self.config.doAstrometry:
429 matchMeta = createMatchMetadata(exposure, border=self.
pixelMargin)
433 exposureIdInfo = dataRef.get(
"expIdInfo")
437 exposureIdInfo=exposureIdInfo,
438 background=background,
439 icSourceCat=icSourceCat,
442 if self.config.doWrite:
445 exposure=calRes.exposure,
446 background=calRes.background,
447 sourceCat=calRes.sourceCat,
448 astromMatches=calRes.astromMatches,
454 def calibrate(self, exposure, exposureIdInfo=None, background=None,
456 """!Calibrate an exposure (science image or coadd) 458 @param[in,out] exposure exposure to calibrate (an 459 lsst.afw.image.ExposureF or similar); 464 - MaskedImage has background subtracted 466 - Calib zero-point is set 467 @param[in] exposureIdInfo ID info for exposure (an 468 lsst.obs.base.ExposureIdInfo) If not provided, returned 469 SourceCatalog IDs will not be globally unique. 470 @param[in,out] background background model already subtracted from 471 exposure (an lsst.afw.math.BackgroundList). May be None if no 472 background has been subtracted, though that is unusual for 473 calibration. A refined background model is output. 474 @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask 475 from which we can copy some fields. 477 @return pipe_base Struct containing these fields: 478 - exposure calibrate science exposure with refined WCS and Calib 479 - background model of background subtracted from exposure (an 480 lsst.afw.math.BackgroundList) 481 - sourceCat catalog of measured sources 482 - astromMatches list of source/refObj matches from the astrometry 486 if exposureIdInfo
is None:
487 exposureIdInfo = ExposureIdInfo()
489 if background
is None:
490 background = BackgroundList()
491 sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
492 exposureIdInfo.unusedBits)
493 table = SourceTable.make(self.
schema, sourceIdFactory)
496 if self.config.doInsertFakes:
497 self.insertFakes.
run(exposure, background=background)
499 detRes = self.detection.
run(table=table, exposure=exposure,
501 sourceCat = detRes.sources
502 if detRes.fpSets.background:
503 background.append(detRes.fpSets.background)
504 if self.config.doDeblend:
505 self.deblend.
run(exposure=exposure, sources=sourceCat)
506 self.measurement.
run(
509 exposureId=exposureIdInfo.expId
511 if self.config.doApCorr:
512 self.applyApCorr.
run(
514 apCorrMap=exposure.getInfo().getApCorrMap()
516 self.catalogCalculation.
run(sourceCat)
518 if icSourceCat
is not None and \
519 len(self.config.icSourceFieldsToCopy) > 0:
526 if self.config.doAstrometry:
528 astromRes = self.astrometry.
run(
532 astromMatches = astromRes.matches
533 except Exception
as e:
534 if self.config.requireAstrometry:
536 self.log.warn(
"Unable to perform astrometric calibration " 537 "(%s): attempting to proceed" % e)
540 if self.config.doPhotoCal:
542 photoRes = self.photoCal.
run(exposure, sourceCat=sourceCat)
543 exposure.getCalib().setFluxMag0(photoRes.calib.getFluxMag0())
544 self.log.info(
"Photometric zero-point: %f" %
545 photoRes.calib.getMagnitude(1.0))
546 self.
setMetadata(exposure=exposure, photoRes=photoRes)
547 except Exception
as e:
548 if self.config.requirePhotoCal:
550 self.log.warn(
"Unable to perform photometric calibration " 551 "(%s): attempting to proceed" % e)
554 frame = getDebugFrame(self._display,
"calibrate")
559 matches=astromMatches,
564 return pipeBase.Struct(
566 background=background,
568 astromMatches=astromMatches,
571 def writeOutputs(self, dataRef, exposure, background, sourceCat,
572 astromMatches, matchMeta):
573 """Write output data to the output repository 575 @param[in] dataRef butler data reference corresponding to a science 577 @param[in] exposure exposure to write 578 @param[in] background background model for exposure 579 @param[in] sourceCat catalog of measured sources 580 @param[in] astromMatches list of source/refObj matches from the 583 sourceWriteFlags = 0
if self.config.doWriteHeavyFootprintsInSources \
584 else afwTable.SOURCE_IO_NO_HEAVY_FOOTPRINTS
585 dataRef.put(sourceCat,
"src")
586 if self.config.doWriteMatches
and astromMatches
is not None:
587 normalizedMatches = afwTable.packMatches(astromMatches)
588 normalizedMatches.table.setMetadata(matchMeta)
589 dataRef.put(normalizedMatches,
"srcMatch")
590 if self.config.doWriteMatchesDenormalized:
591 denormMatches = denormalizeMatches(astromMatches, matchMeta)
592 dataRef.put(denormMatches,
"srcMatchFull")
593 dataRef.put(exposure,
"calexp")
594 dataRef.put(background,
"calexpBackground")
597 """Return a dict of empty catalogs for each catalog dataset produced 600 sourceCat = afwTable.SourceCatalog(self.
schema)
602 return {
"src": sourceCat}
605 """!Set task and exposure metadata 607 Logs a warning and continues if needed data is missing. 609 @param[in,out] exposure exposure whose metadata is to be set 610 @param[in] photoRes results of running photoCal; if None then it was 618 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
619 magZero = photoRes.zp - 2.5*math.log10(exposureTime)
620 self.metadata.set(
'MAGZERO', magZero)
622 self.log.warn(
"Could not set normalized MAGZERO in header: no " 626 metadata = exposure.getMetadata()
627 metadata.set(
'MAGZERO_RMS', photoRes.sigma)
628 metadata.set(
'MAGZERO_NOBJ', photoRes.ngood)
629 metadata.set(
'COLORTERM1', 0.0)
630 metadata.set(
'COLORTERM2', 0.0)
631 metadata.set(
'COLORTERM3', 0.0)
632 except Exception
as e:
633 self.log.warn(
"Could not set exposure metadata: %s" % (e,))
636 """!Match sources in icSourceCat and sourceCat and copy the specified fields 638 @param[in] icSourceCat catalog from which to copy fields 639 @param[in,out] sourceCat catalog to which to copy fields 641 The fields copied are those specified by `config.icSourceFieldsToCopy` 642 that actually exist in the schema. This was set up by the constructor 643 using self.schemaMapper. 646 raise RuntimeError(
"To copy icSource fields you must specify " 647 "icSourceSchema nd icSourceKeys when " 648 "constructing this task")
649 if icSourceCat
is None or sourceCat
is None:
650 raise RuntimeError(
"icSourceCat and sourceCat must both be " 652 if len(self.config.icSourceFieldsToCopy) == 0:
653 self.log.warn(
"copyIcSourceFields doing nothing because " 654 "icSourceFieldsToCopy is empty")
657 mc = afwTable.MatchControl()
658 mc.findOnlyClosest =
False 659 matches = afwTable.matchXy(icSourceCat, sourceCat,
660 self.config.matchRadiusPix, mc)
661 if self.config.doDeblend:
662 deblendKey = sourceCat.schema[
"deblend_nChild"].asKey()
664 matches = [m
for m
in matches
if m[1].get(deblendKey) == 0]
671 for m0, m1, d
in matches:
673 match = bestMatches.get(id0)
674 if match
is None or d <= match[2]:
675 bestMatches[id0] = (m0, m1, d)
676 matches = list(bestMatches.values())
681 numMatches = len(matches)
682 numUniqueSources = len(set(m[1].getId()
for m
in matches))
683 if numUniqueSources != numMatches:
684 self.log.warn(
"{} icSourceCat sources matched only {} sourceCat " 685 "sources".format(numMatches, numUniqueSources))
687 self.log.info(
"Copying flags from icSourceCat to sourceCat for " 688 "%s sources" % (numMatches,))
692 for icSrc, src, d
in matches:
698 icSrcFootprint = icSrc.getFootprint()
700 icSrc.setFootprint(src.getFootprint())
703 icSrc.setFootprint(icSrcFootprint)
def copyIcSourceFields(self, icSourceCat, sourceCat)
Match sources in icSourceCat and sourceCat and copy the specified fields.
def calibrate(self, exposure, exposureIdInfo=None, background=None, icSourceCat=None)
Calibrate an exposure (science image or coadd)
def writeOutputs(self, dataRef, exposure, background, sourceCat, astromMatches, matchMeta)
def run(self, dataRef, exposure=None, background=None, icSourceCat=None, doUnpersist=True)
Calibrate an exposure, optionally unpersisting inputs and persisting outputs.
def __init__(self, butler=None, astromRefObjLoader=None, photoRefObjLoader=None, icSourceSchema=None, kwargs)
Construct a CalibrateTask.
def setMetadata(self, exposure, photoRes=None)
Set task and exposure metadata.
def getSchemaCatalogs(self)
Calibrate an exposure: measure sources and perform astrometric and photometric calibration.