22 from __future__
import absolute_import, division, print_function
25 from lsstDebug
import getDebugFrame
26 import lsst.pex.config
as pexConfig
27 import lsst.pipe.base
as pipeBase
28 import lsst.afw.table
as afwTable
29 from lsst.meas.astrom
import AstrometryTask, displayAstrometry, denormalizeMatches
30 from lsst.meas.extensions.astrometryNet
import LoadAstrometryNetObjectsTask
31 from lsst.obs.base
import ExposureIdInfo
32 import lsst.daf.base
as dafBase
33 from lsst.afw.math
import BackgroundList
34 from lsst.afw.table
import IdFactory, SourceTable
35 from lsst.meas.algorithms
import SourceDetectionTask
36 from lsst.meas.base
import (SingleFrameMeasurementTask, ApplyApCorrTask,
37 CatalogCalculationTask)
38 from lsst.meas.deblender
import SourceDeblendTask
39 from .fakes
import BaseFakeSourcesTask
40 from .photoCal
import PhotoCalTask
42 __all__ = [
"CalibrateConfig",
"CalibrateTask"]
46 """Config for CalibrateTask""" 47 doWrite = pexConfig.Field(
50 doc=
"Save calibration results?",
52 doWriteHeavyFootprintsInSources = pexConfig.Field(
55 doc=
"Include HeavyFootprint data in source table? If false then heavy " 56 "footprints are saved as normal footprints, which saves some space" 58 doWriteMatches = pexConfig.Field(
61 doc=
"Write reference matches (ignored if doWrite false)?",
63 doWriteMatchesDenormalized = pexConfig.Field(
66 doc=(
"Write reference matches in denormalized format? " 67 "This format uses more disk space, but is more convenient to " 68 "read. Ignored if doWriteMatches=False or doWrite=False."),
70 doAstrometry = pexConfig.Field(
73 doc=
"Perform astrometric calibration?",
75 astromRefObjLoader = pexConfig.ConfigurableField(
76 target=LoadAstrometryNetObjectsTask,
77 doc=
"reference object loader for astrometric calibration",
79 photoRefObjLoader = pexConfig.ConfigurableField(
80 target=LoadAstrometryNetObjectsTask,
81 doc=
"reference object loader for photometric calibration",
83 astrometry = pexConfig.ConfigurableField(
84 target=AstrometryTask,
85 doc=
"Perform astrometric calibration to refine the WCS",
87 requireAstrometry = pexConfig.Field(
90 doc=(
"Raise an exception if astrometry fails? Ignored if doAstrometry " 93 doPhotoCal = pexConfig.Field(
96 doc=
"Perform phometric calibration?",
98 requirePhotoCal = pexConfig.Field(
101 doc=(
"Raise an exception if photoCal fails? Ignored if doPhotoCal " 104 photoCal = pexConfig.ConfigurableField(
106 doc=
"Perform photometric calibration",
108 icSourceFieldsToCopy = pexConfig.ListField(
110 default=(
"calib_psfCandidate",
"calib_psfUsed",
"calib_psfReserved"),
111 doc=(
"Fields to copy from the icSource catalog to the output catalog " 112 "for matching sources Any missing fields will trigger a " 113 "RuntimeError exception. Ignored if icSourceCat is not provided.")
115 matchRadiusPix = pexConfig.Field(
118 doc=(
"Match radius for matching icSourceCat objects to sourceCat " 121 checkUnitsParseStrict = pexConfig.Field(
122 doc=(
"Strictness of Astropy unit compatibility check, can be 'raise', " 123 "'warn' or 'silent'"),
127 detection = pexConfig.ConfigurableField(
128 target=SourceDetectionTask,
131 doDeblend = pexConfig.Field(
134 doc=
"Run deblender input exposure" 136 deblend = pexConfig.ConfigurableField(
137 target=SourceDeblendTask,
138 doc=
"Split blended sources into their components" 140 measurement = pexConfig.ConfigurableField(
141 target=SingleFrameMeasurementTask,
142 doc=
"Measure sources" 144 doApCorr = pexConfig.Field(
147 doc=
"Run subtask to apply aperture correction" 149 applyApCorr = pexConfig.ConfigurableField(
150 target=ApplyApCorrTask,
151 doc=
"Subtask to apply aperture corrections" 156 catalogCalculation = pexConfig.ConfigurableField(
157 target=CatalogCalculationTask,
158 doc=
"Subtask to run catalogCalculation plugins on catalog" 160 doInsertFakes = pexConfig.Field(
163 doc=
"Run fake sources injection task" 165 insertFakes = pexConfig.ConfigurableField(
166 target=BaseFakeSourcesTask,
167 doc=
"Injection of fake sources for testing purposes (must be " 172 pexConfig.Config.setDefaults(self)
184 """!Calibrate an exposure: measure sources and perform astrometric and 185 photometric calibration 187 @anchor CalibrateTask_ 189 @section pipe_tasks_calibrate_Contents Contents 191 - @ref pipe_tasks_calibrate_Purpose 192 - @ref pipe_tasks_calibrate_Initialize 193 - @ref pipe_tasks_calibrate_IO 194 - @ref pipe_tasks_calibrate_Config 195 - @ref pipe_tasks_calibrate_Metadata 196 - @ref pipe_tasks_calibrate_Debug 199 @section pipe_tasks_calibrate_Purpose Description 201 Given an exposure with a good PSF model and aperture correction map 202 (e.g. as provided by @ref CharacterizeImageTask), perform the following 204 - Run detection and measurement 205 - Run astrometry subtask to fit an improved WCS 206 - Run photoCal subtask to fit the exposure's photometric zero-point 208 @section pipe_tasks_calibrate_Initialize Task initialisation 210 @copydoc \_\_init\_\_ 212 @section pipe_tasks_calibrate_IO Invoking the Task 214 If you want this task to unpersist inputs or persist outputs, then call 215 the `run` method (a wrapper around the `calibrate` method). 217 If you already have the inputs unpersisted and do not want to persist the 218 output then it is more direct to call the `calibrate` method: 220 @section pipe_tasks_calibrate_Config Configuration parameters 222 See @ref CalibrateConfig 224 @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata 228 <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task 229 <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal 231 <dt>COLORTERM1 <dd>?? (always 0.0) 232 <dt>COLORTERM2 <dd>?? (always 0.0) 233 <dt>COLORTERM3 <dd>?? (always 0.0) 236 @section pipe_tasks_calibrate_Debug Debug variables 238 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink 239 interface supports a flag 240 `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug 241 for more about `debug.py`. 243 CalibrateTask has a debug dictionary containing one key: 246 <dd>frame (an int; <= 0 to not display) in which to display the exposure, 247 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for 248 the meaning of the various symbols. 251 For example, put something like: 255 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would 256 # call us recursively 257 if name == "lsst.pipe.tasks.calibrate": 264 lsstDebug.Info = DebugInfo 266 into your `debug.py` file and run `calibrateTask.py` with the `--debug` 269 Some subtasks may have their own debug variables; see individual Task 276 ConfigClass = CalibrateConfig
277 _DefaultName =
"calibrate" 278 RunnerClass = pipeBase.ButlerInitializedTaskRunner
280 def __init__(self, butler=None, astromRefObjLoader=None,
281 photoRefObjLoader=None, icSourceSchema=None, **kwargs):
282 """!Construct a CalibrateTask 284 @param[in] butler The butler is passed to the refObjLoader constructor 285 in case it is needed. Ignored if the refObjLoader argument 286 provides a loader directly. 287 @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks 288 that supplies an external reference catalog for astrometric 289 calibration. May be None if the desired loader can be constructed 290 from the butler argument or all steps requiring a reference catalog 292 @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks 293 that supplies an external reference catalog for photometric 294 calibration. May be None if the desired loader can be constructed 295 from the butler argument or all steps requiring a reference catalog 297 @param[in] icSourceSchema schema for icSource catalog, or None. 298 Schema values specified in config.icSourceFieldsToCopy will be 299 taken from this schema. If set to None, no values will be 300 propagated from the icSourceCatalog 301 @param[in,out] kwargs other keyword arguments for 302 lsst.pipe.base.CmdLineTask 304 pipeBase.CmdLineTask.__init__(self, **kwargs)
306 if icSourceSchema
is None and butler
is not None:
308 icSourceSchema = butler.get(
"icSrc_schema", immediate=
True).schema
310 if icSourceSchema
is not None:
313 minimumSchema = afwTable.SourceTable.makeMinimalSchema()
314 self.
schemaMapper.addMinimalSchema(minimumSchema,
False)
322 afwTable.Field[
"Flag"](
"calib_detected",
323 "Source was detected as an icSource"))
324 missingFieldNames = []
325 for fieldName
in self.config.icSourceFieldsToCopy:
327 schemaItem = icSourceSchema.find(fieldName)
329 missingFieldNames.append(fieldName)
334 if missingFieldNames:
335 raise RuntimeError(
"isSourceCat is missing fields {} " 336 "specified in icSourceFieldsToCopy" 337 .format(missingFieldNames))
344 self.
schema = afwTable.SourceTable.makeMinimalSchema()
345 self.makeSubtask(
'detection', schema=self.
schema)
352 if self.config.doInsertFakes:
353 self.makeSubtask(
"insertFakes")
355 if self.config.doDeblend:
356 self.makeSubtask(
"deblend", schema=self.
schema)
357 self.makeSubtask(
'measurement', schema=self.
schema,
359 if self.config.doApCorr:
360 self.makeSubtask(
'applyApCorr', schema=self.
schema)
361 self.makeSubtask(
'catalogCalculation', schema=self.
schema)
363 if self.config.doAstrometry:
364 if astromRefObjLoader
is None:
365 self.makeSubtask(
'astromRefObjLoader', butler=butler)
366 astromRefObjLoader = self.astromRefObjLoader
368 self.makeSubtask(
"astrometry", refObjLoader=astromRefObjLoader,
370 if self.config.doPhotoCal:
371 if photoRefObjLoader
is None:
372 self.makeSubtask(
'photoRefObjLoader', butler=butler)
373 photoRefObjLoader = self.photoRefObjLoader
374 self.
pixelMargin = photoRefObjLoader.config.pixelMargin
375 self.makeSubtask(
"photoCal", refObjLoader=photoRefObjLoader,
381 self.
schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
384 def run(self, dataRef, exposure=None, background=None, icSourceCat=None,
386 """!Calibrate an exposure, optionally unpersisting inputs and 389 This is a wrapper around the `calibrate` method that unpersists inputs 390 (if `doUnpersist` true) and persists outputs (if `config.doWrite` true) 392 @param[in] dataRef butler data reference corresponding to a science 394 @param[in,out] exposure characterized exposure (an 395 lsst.afw.image.ExposureF or similar), or None to unpersist existing 396 icExp and icBackground. See calibrate method for details of what is 398 @param[in,out] background initial model of background already 399 subtracted from exposure (an lsst.afw.math.BackgroundList). May be 400 None if no background has been subtracted, though that is unusual 401 for calibration. A refined background model is output. Ignored if 403 @param[in] icSourceCat catalog from which to copy the fields specified 404 by icSourceKeys, or None; 405 @param[in] doUnpersist unpersist data: 406 - if True, exposure, background and icSourceCat are read from 407 dataRef and those three arguments must all be None; 408 - if False the exposure must be provided; background and 409 icSourceCat are optional. True is intended for running as a 410 command-line task, False for running as a subtask 412 @return same data as the calibrate method 414 self.log.info(
"Processing %s" % (dataRef.dataId))
417 if any(item
is not None for item
in (exposure, background,
419 raise RuntimeError(
"doUnpersist true; exposure, background " 420 "and icSourceCat must all be None")
421 exposure = dataRef.get(
"icExp", immediate=
True)
422 background = dataRef.get(
"icExpBackground", immediate=
True)
423 icSourceCat = dataRef.get(
"icSrc", immediate=
True)
424 elif exposure
is None:
425 raise RuntimeError(
"doUnpersist false; exposure must be provided")
427 exposureIdInfo = dataRef.get(
"expIdInfo")
431 exposureIdInfo=exposureIdInfo,
432 background=background,
433 icSourceCat=icSourceCat,
436 if self.config.doWrite:
439 exposure=calRes.exposure,
440 background=calRes.background,
441 sourceCat=calRes.sourceCat,
442 astromMatches=calRes.astromMatches,
443 matchMeta=calRes.matchMeta,
448 def calibrate(self, exposure, exposureIdInfo=None, background=None,
450 """!Calibrate an exposure (science image or coadd) 452 @param[in,out] exposure exposure to calibrate (an 453 lsst.afw.image.ExposureF or similar); 458 - MaskedImage has background subtracted 460 - Calib zero-point is set 461 @param[in] exposureIdInfo ID info for exposure (an 462 lsst.obs.base.ExposureIdInfo) If not provided, returned 463 SourceCatalog IDs will not be globally unique. 464 @param[in,out] background background model already subtracted from 465 exposure (an lsst.afw.math.BackgroundList). May be None if no 466 background has been subtracted, though that is unusual for 467 calibration. A refined background model is output. 468 @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask 469 from which we can copy some fields. 471 @return pipe_base Struct containing these fields: 472 - exposure calibrate science exposure with refined WCS and Calib 473 - background model of background subtracted from exposure (an 474 lsst.afw.math.BackgroundList) 475 - sourceCat catalog of measured sources 476 - astromMatches list of source/refObj matches from the astrometry 480 if exposureIdInfo
is None:
481 exposureIdInfo = ExposureIdInfo()
483 if background
is None:
484 background = BackgroundList()
485 sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
486 exposureIdInfo.unusedBits)
487 table = SourceTable.make(self.
schema, sourceIdFactory)
490 if self.config.doInsertFakes:
491 self.insertFakes.
run(exposure, background=background)
493 detRes = self.detection.
run(table=table, exposure=exposure,
495 sourceCat = detRes.sources
496 if detRes.fpSets.background:
497 background.append(detRes.fpSets.background)
498 if self.config.doDeblend:
499 self.deblend.
run(exposure=exposure, sources=sourceCat)
500 self.measurement.
run(
503 exposureId=exposureIdInfo.expId
505 if self.config.doApCorr:
506 self.applyApCorr.
run(
508 apCorrMap=exposure.getInfo().getApCorrMap()
510 self.catalogCalculation.
run(sourceCat)
512 if icSourceCat
is not None and \
513 len(self.config.icSourceFieldsToCopy) > 0:
521 if self.config.doAstrometry:
523 astromRes = self.astrometry.
run(
527 astromMatches = astromRes.matches
528 matchMeta = astromRes.matchMeta
529 except Exception
as e:
530 if self.config.requireAstrometry:
532 self.log.warn(
"Unable to perform astrometric calibration " 533 "(%s): attempting to proceed" % e)
536 if self.config.doPhotoCal:
538 photoRes = self.photoCal.
run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
539 exposure.getCalib().setFluxMag0(photoRes.calib.getFluxMag0())
540 self.log.info(
"Photometric zero-point: %f" %
541 photoRes.calib.getMagnitude(1.0))
542 self.
setMetadata(exposure=exposure, photoRes=photoRes)
543 except Exception
as e:
544 if self.config.requirePhotoCal:
546 self.log.warn(
"Unable to perform photometric calibration " 547 "(%s): attempting to proceed" % e)
550 frame = getDebugFrame(self._display,
"calibrate")
555 matches=astromMatches,
560 return pipeBase.Struct(
562 background=background,
564 astromMatches=astromMatches,
568 def writeOutputs(self, dataRef, exposure, background, sourceCat,
569 astromMatches, matchMeta):
570 """Write output data to the output repository 572 @param[in] dataRef butler data reference corresponding to a science 574 @param[in] exposure exposure to write 575 @param[in] background background model for exposure 576 @param[in] sourceCat catalog of measured sources 577 @param[in] astromMatches list of source/refObj matches from the 580 sourceWriteFlags = 0
if self.config.doWriteHeavyFootprintsInSources \
581 else afwTable.SOURCE_IO_NO_HEAVY_FOOTPRINTS
582 dataRef.put(sourceCat,
"src")
583 if self.config.doWriteMatches
and astromMatches
is not None:
584 normalizedMatches = afwTable.packMatches(astromMatches)
585 normalizedMatches.table.setMetadata(matchMeta)
586 dataRef.put(normalizedMatches,
"srcMatch")
587 if self.config.doWriteMatchesDenormalized:
588 denormMatches = denormalizeMatches(astromMatches, matchMeta)
589 dataRef.put(denormMatches,
"srcMatchFull")
590 dataRef.put(exposure,
"calexp")
591 dataRef.put(background,
"calexpBackground")
594 """Return a dict of empty catalogs for each catalog dataset produced 597 sourceCat = afwTable.SourceCatalog(self.
schema)
599 return {
"src": sourceCat}
602 """!Set task and exposure metadata 604 Logs a warning and continues if needed data is missing. 606 @param[in,out] exposure exposure whose metadata is to be set 607 @param[in] photoRes results of running photoCal; if None then it was 615 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
616 magZero = photoRes.zp - 2.5*math.log10(exposureTime)
617 self.metadata.set(
'MAGZERO', magZero)
619 self.log.warn(
"Could not set normalized MAGZERO in header: no " 623 metadata = exposure.getMetadata()
624 metadata.set(
'MAGZERO_RMS', photoRes.sigma)
625 metadata.set(
'MAGZERO_NOBJ', photoRes.ngood)
626 metadata.set(
'COLORTERM1', 0.0)
627 metadata.set(
'COLORTERM2', 0.0)
628 metadata.set(
'COLORTERM3', 0.0)
629 except Exception
as e:
630 self.log.warn(
"Could not set exposure metadata: %s" % (e,))
633 """!Match sources in icSourceCat and sourceCat and copy the specified fields 635 @param[in] icSourceCat catalog from which to copy fields 636 @param[in,out] sourceCat catalog to which to copy fields 638 The fields copied are those specified by `config.icSourceFieldsToCopy` 639 that actually exist in the schema. This was set up by the constructor 640 using self.schemaMapper. 643 raise RuntimeError(
"To copy icSource fields you must specify " 644 "icSourceSchema nd icSourceKeys when " 645 "constructing this task")
646 if icSourceCat
is None or sourceCat
is None:
647 raise RuntimeError(
"icSourceCat and sourceCat must both be " 649 if len(self.config.icSourceFieldsToCopy) == 0:
650 self.log.warn(
"copyIcSourceFields doing nothing because " 651 "icSourceFieldsToCopy is empty")
654 mc = afwTable.MatchControl()
655 mc.findOnlyClosest =
False 656 matches = afwTable.matchXy(icSourceCat, sourceCat,
657 self.config.matchRadiusPix, mc)
658 if self.config.doDeblend:
659 deblendKey = sourceCat.schema[
"deblend_nChild"].asKey()
661 matches = [m
for m
in matches
if m[1].get(deblendKey) == 0]
668 for m0, m1, d
in matches:
670 match = bestMatches.get(id0)
671 if match
is None or d <= match[2]:
672 bestMatches[id0] = (m0, m1, d)
673 matches = list(bestMatches.values())
678 numMatches = len(matches)
679 numUniqueSources = len(set(m[1].getId()
for m
in matches))
680 if numUniqueSources != numMatches:
681 self.log.warn(
"{} icSourceCat sources matched only {} sourceCat " 682 "sources".format(numMatches, numUniqueSources))
684 self.log.info(
"Copying flags from icSourceCat to sourceCat for " 685 "%s sources" % (numMatches,))
689 for icSrc, src, d
in matches:
695 icSrcFootprint = icSrc.getFootprint()
697 icSrc.setFootprint(src.getFootprint())
700 icSrc.setFootprint(icSrcFootprint)
def copyIcSourceFields(self, icSourceCat, sourceCat)
Match sources in icSourceCat and sourceCat and copy the specified fields.
def calibrate(self, exposure, exposureIdInfo=None, background=None, icSourceCat=None)
Calibrate an exposure (science image or coadd)
def writeOutputs(self, dataRef, exposure, background, sourceCat, astromMatches, matchMeta)
def run(self, dataRef, exposure=None, background=None, icSourceCat=None, doUnpersist=True)
Calibrate an exposure, optionally unpersisting inputs and persisting outputs.
def __init__(self, butler=None, astromRefObjLoader=None, photoRefObjLoader=None, icSourceSchema=None, kwargs)
Construct a CalibrateTask.
def setMetadata(self, exposure, photoRes=None)
Set task and exposure metadata.
def getSchemaCatalogs(self)
Calibrate an exposure: measure sources and perform astrometric and photometric calibration.