22 from __future__
import absolute_import, division, print_function
25 from lsstDebug
import getDebugFrame
29 from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches
30 from lsst.meas.extensions.astrometryNet
import LoadAstrometryNetObjectsTask
36 from lsst.meas.base import (SingleFrameMeasurementTask, ApplyApCorrTask,
37 CatalogCalculationTask)
39 from .fakes
import BaseFakeSourcesTask
40 from .photoCal
import PhotoCalTask
42 __all__ = [
"CalibrateConfig",
"CalibrateTask"]
46 """Config for CalibrateTask""" 47 doWrite = pexConfig.Field(
50 doc=
"Save calibration results?",
52 doWriteHeavyFootprintsInSources = pexConfig.Field(
55 doc=
"Include HeavyFootprint data in source table? If false then heavy " 56 "footprints are saved as normal footprints, which saves some space" 58 doWriteMatches = pexConfig.Field(
61 doc=
"Write reference matches (ignored if doWrite false)?",
63 doWriteMatchesDenormalized = pexConfig.Field(
66 doc=(
"Write reference matches in denormalized format? " 67 "This format uses more disk space, but is more convenient to " 68 "read. Ignored if doWriteMatches=False or doWrite=False."),
70 doAstrometry = pexConfig.Field(
73 doc=
"Perform astrometric calibration?",
75 astromRefObjLoader = pexConfig.ConfigurableField(
76 target=LoadAstrometryNetObjectsTask,
77 doc=
"reference object loader for astrometric calibration",
79 photoRefObjLoader = pexConfig.ConfigurableField(
80 target=LoadAstrometryNetObjectsTask,
81 doc=
"reference object loader for photometric calibration",
83 astrometry = pexConfig.ConfigurableField(
84 target=AstrometryTask,
85 doc=
"Perform astrometric calibration to refine the WCS",
87 requireAstrometry = pexConfig.Field(
90 doc=(
"Raise an exception if astrometry fails? Ignored if doAstrometry " 93 doPhotoCal = pexConfig.Field(
96 doc=
"Perform phometric calibration?",
98 requirePhotoCal = pexConfig.Field(
101 doc=(
"Raise an exception if photoCal fails? Ignored if doPhotoCal " 104 photoCal = pexConfig.ConfigurableField(
106 doc=
"Perform photometric calibration",
108 icSourceFieldsToCopy = pexConfig.ListField(
110 default=(
"calib_psfCandidate",
"calib_psfUsed",
"calib_psfReserved"),
111 doc=(
"Fields to copy from the icSource catalog to the output catalog " 112 "for matching sources Any missing fields will trigger a " 113 "RuntimeError exception. Ignored if icSourceCat is not provided.")
115 matchRadiusPix = pexConfig.Field(
118 doc=(
"Match radius for matching icSourceCat objects to sourceCat " 121 checkUnitsParseStrict = pexConfig.Field(
122 doc=(
"Strictness of Astropy unit compatibility check, can be 'raise', " 123 "'warn' or 'silent'"),
127 detection = pexConfig.ConfigurableField(
128 target=SourceDetectionTask,
131 doDeblend = pexConfig.Field(
134 doc=
"Run deblender input exposure" 136 deblend = pexConfig.ConfigurableField(
137 target=SourceDeblendTask,
138 doc=
"Split blended sources into their components" 140 measurement = pexConfig.ConfigurableField(
141 target=SingleFrameMeasurementTask,
142 doc=
"Measure sources" 144 doApCorr = pexConfig.Field(
147 doc=
"Run subtask to apply aperture correction" 149 applyApCorr = pexConfig.ConfigurableField(
150 target=ApplyApCorrTask,
151 doc=
"Subtask to apply aperture corrections" 156 catalogCalculation = pexConfig.ConfigurableField(
157 target=CatalogCalculationTask,
158 doc=
"Subtask to run catalogCalculation plugins on catalog" 160 doInsertFakes = pexConfig.Field(
163 doc=
"Run fake sources injection task" 165 insertFakes = pexConfig.ConfigurableField(
166 target=BaseFakeSourcesTask,
167 doc=
"Injection of fake sources for testing purposes (must be " 172 pexConfig.Config.setDefaults(self)
184 """!Calibrate an exposure: measure sources and perform astrometric and 185 photometric calibration 187 @anchor CalibrateTask_ 189 @section pipe_tasks_calibrate_Contents Contents 191 - @ref pipe_tasks_calibrate_Purpose 192 - @ref pipe_tasks_calibrate_Initialize 193 - @ref pipe_tasks_calibrate_IO 194 - @ref pipe_tasks_calibrate_Config 195 - @ref pipe_tasks_calibrate_Metadata 196 - @ref pipe_tasks_calibrate_Debug 199 @section pipe_tasks_calibrate_Purpose Description 201 Given an exposure with a good PSF model and aperture correction map 202 (e.g. as provided by @ref CharacterizeImageTask), perform the following 204 - Run detection and measurement 205 - Run astrometry subtask to fit an improved WCS 206 - Run photoCal subtask to fit the exposure's photometric zero-point 208 @section pipe_tasks_calibrate_Initialize Task initialisation 210 @copydoc \_\_init\_\_ 212 @section pipe_tasks_calibrate_IO Invoking the Task 214 If you want this task to unpersist inputs or persist outputs, then call 215 the `run` method (a wrapper around the `calibrate` method). 217 If you already have the inputs unpersisted and do not want to persist the 218 output then it is more direct to call the `calibrate` method: 220 @section pipe_tasks_calibrate_Config Configuration parameters 222 See @ref CalibrateConfig 224 @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata 228 <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task 229 <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal 231 <dt>COLORTERM1 <dd>?? (always 0.0) 232 <dt>COLORTERM2 <dd>?? (always 0.0) 233 <dt>COLORTERM3 <dd>?? (always 0.0) 236 @section pipe_tasks_calibrate_Debug Debug variables 238 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink 239 interface supports a flag 240 `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug 241 for more about `debug.py`. 243 CalibrateTask has a debug dictionary containing one key: 246 <dd>frame (an int; <= 0 to not display) in which to display the exposure, 247 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for 248 the meaning of the various symbols. 251 For example, put something like: 255 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would 256 # call us recursively 257 if name == "lsst.pipe.tasks.calibrate": 264 lsstDebug.Info = DebugInfo 266 into your `debug.py` file and run `calibrateTask.py` with the `--debug` 269 Some subtasks may have their own debug variables; see individual Task 276 ConfigClass = CalibrateConfig
277 _DefaultName =
"calibrate" 278 RunnerClass = pipeBase.ButlerInitializedTaskRunner
280 def __init__(self, butler=None, astromRefObjLoader=None,
281 photoRefObjLoader=None, icSourceSchema=None, **kwargs):
282 """!Construct a CalibrateTask 284 @param[in] butler The butler is passed to the refObjLoader constructor 285 in case it is needed. Ignored if the refObjLoader argument 286 provides a loader directly. 287 @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks 288 that supplies an external reference catalog for astrometric 289 calibration. May be None if the desired loader can be constructed 290 from the butler argument or all steps requiring a reference catalog 292 @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks 293 that supplies an external reference catalog for photometric 294 calibration. May be None if the desired loader can be constructed 295 from the butler argument or all steps requiring a reference catalog 297 @param[in] icSourceSchema schema for icSource catalog, or None. 298 Schema values specified in config.icSourceFieldsToCopy will be 299 taken from this schema. If set to None, no values will be 300 propagated from the icSourceCatalog 301 @param[in,out] kwargs other keyword arguments for 302 lsst.pipe.base.CmdLineTask 304 pipeBase.CmdLineTask.__init__(self, **kwargs)
306 if icSourceSchema
is None and butler
is not None:
308 icSourceSchema = butler.get(
"icSrc_schema", immediate=
True).schema
310 if icSourceSchema
is not None:
313 minimumSchema = afwTable.SourceTable.makeMinimalSchema()
314 self.
schemaMapper.addMinimalSchema(minimumSchema,
False)
322 afwTable.Field[
"Flag"](
"calib_detected",
323 "Source was detected as an icSource"))
324 missingFieldNames = []
325 for fieldName
in self.config.icSourceFieldsToCopy:
327 schemaItem = icSourceSchema.find(fieldName)
329 missingFieldNames.append(fieldName)
334 if missingFieldNames:
335 raise RuntimeError(
"isSourceCat is missing fields {} " 336 "specified in icSourceFieldsToCopy" 337 .format(missingFieldNames))
344 self.
schema = afwTable.SourceTable.makeMinimalSchema()
345 self.makeSubtask(
'detection', schema=self.
schema)
352 if self.config.doInsertFakes:
353 self.makeSubtask(
"insertFakes")
355 if self.config.doDeblend:
356 self.makeSubtask(
"deblend", schema=self.
schema)
357 self.makeSubtask(
'measurement', schema=self.
schema,
359 if self.config.doApCorr:
360 self.makeSubtask(
'applyApCorr', schema=self.
schema)
361 self.makeSubtask(
'catalogCalculation', schema=self.
schema)
363 if self.config.doAstrometry:
364 if astromRefObjLoader
is None:
365 self.makeSubtask(
'astromRefObjLoader', butler=butler)
366 astromRefObjLoader = self.astromRefObjLoader
368 self.makeSubtask(
"astrometry", refObjLoader=astromRefObjLoader,
370 if self.config.doPhotoCal:
371 if photoRefObjLoader
is None:
372 self.makeSubtask(
'photoRefObjLoader', butler=butler)
373 photoRefObjLoader = self.photoRefObjLoader
374 self.
pixelMargin = photoRefObjLoader.config.pixelMargin
375 self.makeSubtask(
"photoCal", refObjLoader=photoRefObjLoader,
381 self.
schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
384 def run(self, dataRef, exposure=None, background=None, icSourceCat=None,
386 """!Calibrate an exposure, optionally unpersisting inputs and 389 This is a wrapper around the `calibrate` method that unpersists inputs 390 (if `doUnpersist` true) and persists outputs (if `config.doWrite` true) 392 @param[in] dataRef butler data reference corresponding to a science 394 @param[in,out] exposure characterized exposure (an 395 lsst.afw.image.ExposureF or similar), or None to unpersist existing 396 icExp and icBackground. See calibrate method for details of what is 398 @param[in,out] background initial model of background already 399 subtracted from exposure (an lsst.afw.math.BackgroundList). May be 400 None if no background has been subtracted, though that is unusual 401 for calibration. A refined background model is output. Ignored if 403 @param[in] icSourceCat catalog from which to copy the fields specified 404 by icSourceKeys, or None; 405 @param[in] doUnpersist unpersist data: 406 - if True, exposure, background and icSourceCat are read from 407 dataRef and those three arguments must all be None; 408 - if False the exposure must be provided; background and 409 icSourceCat are optional. True is intended for running as a 410 command-line task, False for running as a subtask 412 @return same data as the calibrate method 414 self.log.info(
"Processing %s" % (dataRef.dataId))
417 if any(item
is not None for item
in (exposure, background,
419 raise RuntimeError(
"doUnpersist true; exposure, background " 420 "and icSourceCat must all be None")
421 exposure = dataRef.get(
"icExp", immediate=
True)
422 background = dataRef.get(
"icExpBackground", immediate=
True)
423 icSourceCat = dataRef.get(
"icSrc", immediate=
True)
424 elif exposure
is None:
425 raise RuntimeError(
"doUnpersist false; exposure must be provided")
427 exposureIdInfo = dataRef.get(
"expIdInfo")
431 exposureIdInfo=exposureIdInfo,
432 background=background,
433 icSourceCat=icSourceCat,
436 if self.config.doWrite:
439 exposure=calRes.exposure,
440 background=calRes.background,
441 sourceCat=calRes.sourceCat,
442 astromMatches=calRes.astromMatches,
443 matchMeta=calRes.matchMeta,
448 def calibrate(self, exposure, exposureIdInfo=None, background=None,
450 """!Calibrate an exposure (science image or coadd) 452 @param[in,out] exposure exposure to calibrate (an 453 lsst.afw.image.ExposureF or similar); 458 - MaskedImage has background subtracted 460 - Calib zero-point is set 461 @param[in] exposureIdInfo ID info for exposure (an 462 lsst.obs.base.ExposureIdInfo) If not provided, returned 463 SourceCatalog IDs will not be globally unique. 464 @param[in,out] background background model already subtracted from 465 exposure (an lsst.afw.math.BackgroundList). May be None if no 466 background has been subtracted, though that is unusual for 467 calibration. A refined background model is output. 468 @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask 469 from which we can copy some fields. 471 @return pipe_base Struct containing these fields: 472 - exposure calibrate science exposure with refined WCS and Calib 473 - background model of background subtracted from exposure (an 474 lsst.afw.math.BackgroundList) 475 - sourceCat catalog of measured sources 476 - astromMatches list of source/refObj matches from the astrometry 480 if exposureIdInfo
is None:
481 exposureIdInfo = ExposureIdInfo()
483 if background
is None:
484 background = BackgroundList()
485 sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
486 exposureIdInfo.unusedBits)
487 table = SourceTable.make(self.
schema, sourceIdFactory)
490 detRes = self.detection.
run(table=table, exposure=exposure,
492 sourceCat = detRes.sources
493 if detRes.fpSets.background:
494 background.append(detRes.fpSets.background)
495 if self.config.doDeblend:
496 self.deblend.
run(exposure=exposure, sources=sourceCat)
497 self.measurement.
run(
500 exposureId=exposureIdInfo.expId
502 if self.config.doApCorr:
503 self.applyApCorr.
run(
505 apCorrMap=exposure.getInfo().getApCorrMap()
507 self.catalogCalculation.
run(sourceCat)
509 if icSourceCat
is not None and \
510 len(self.config.icSourceFieldsToCopy) > 0:
518 if self.config.doAstrometry:
520 astromRes = self.astrometry.
run(
524 astromMatches = astromRes.matches
525 matchMeta = astromRes.matchMeta
526 except Exception
as e:
527 if self.config.requireAstrometry:
529 self.log.warn(
"Unable to perform astrometric calibration " 530 "(%s): attempting to proceed" % e)
533 if self.config.doPhotoCal:
535 photoRes = self.photoCal.
run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
536 exposure.getCalib().setFluxMag0(photoRes.calib.getFluxMag0())
537 self.log.info(
"Photometric zero-point: %f" %
538 photoRes.calib.getMagnitude(1.0))
539 self.
setMetadata(exposure=exposure, photoRes=photoRes)
540 except Exception
as e:
541 if self.config.requirePhotoCal:
543 self.log.warn(
"Unable to perform photometric calibration " 544 "(%s): attempting to proceed" % e)
547 if self.config.doInsertFakes:
548 self.insertFakes.
run(exposure, background=background)
550 table = SourceTable.make(self.
schema, sourceIdFactory)
553 detRes = self.detection.
run(table=table, exposure=exposure,
555 sourceCat = detRes.sources
556 if detRes.fpSets.background:
557 background.append(detRes.fpSets.background)
558 if self.config.doDeblend:
559 self.deblend.
run(exposure=exposure, sources=sourceCat)
560 self.measurement.
run(
563 exposureId=exposureIdInfo.expId
565 if self.config.doApCorr:
566 self.applyApCorr.
run(
568 apCorrMap=exposure.getInfo().getApCorrMap()
570 self.catalogCalculation.
run(sourceCat)
572 if icSourceCat
is not None and \
573 len(self.config.icSourceFieldsToCopy) > 0:
577 frame = getDebugFrame(self._display,
"calibrate")
582 matches=astromMatches,
587 return pipeBase.Struct(
589 background=background,
591 astromMatches=astromMatches,
595 def writeOutputs(self, dataRef, exposure, background, sourceCat,
596 astromMatches, matchMeta):
597 """Write output data to the output repository 599 @param[in] dataRef butler data reference corresponding to a science 601 @param[in] exposure exposure to write 602 @param[in] background background model for exposure 603 @param[in] sourceCat catalog of measured sources 604 @param[in] astromMatches list of source/refObj matches from the 607 sourceWriteFlags = 0
if self.config.doWriteHeavyFootprintsInSources \
608 else afwTable.SOURCE_IO_NO_HEAVY_FOOTPRINTS
609 dataRef.put(sourceCat,
"src")
610 if self.config.doWriteMatches
and astromMatches
is not None:
611 normalizedMatches = afwTable.packMatches(astromMatches)
612 normalizedMatches.table.setMetadata(matchMeta)
613 dataRef.put(normalizedMatches,
"srcMatch")
614 if self.config.doWriteMatchesDenormalized:
615 denormMatches = denormalizeMatches(astromMatches, matchMeta)
616 dataRef.put(denormMatches,
"srcMatchFull")
617 dataRef.put(exposure,
"calexp")
618 dataRef.put(background,
"calexpBackground")
621 """Return a dict of empty catalogs for each catalog dataset produced 624 sourceCat = afwTable.SourceCatalog(self.
schema)
626 return {
"src": sourceCat}
629 """!Set task and exposure metadata 631 Logs a warning and continues if needed data is missing. 633 @param[in,out] exposure exposure whose metadata is to be set 634 @param[in] photoRes results of running photoCal; if None then it was 642 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
643 magZero = photoRes.zp - 2.5*math.log10(exposureTime)
644 self.metadata.set(
'MAGZERO', magZero)
646 self.log.warn(
"Could not set normalized MAGZERO in header: no " 650 metadata = exposure.getMetadata()
651 metadata.set(
'MAGZERO_RMS', photoRes.sigma)
652 metadata.set(
'MAGZERO_NOBJ', photoRes.ngood)
653 metadata.set(
'COLORTERM1', 0.0)
654 metadata.set(
'COLORTERM2', 0.0)
655 metadata.set(
'COLORTERM3', 0.0)
656 except Exception
as e:
657 self.log.warn(
"Could not set exposure metadata: %s" % (e,))
660 """!Match sources in icSourceCat and sourceCat and copy the specified fields 662 @param[in] icSourceCat catalog from which to copy fields 663 @param[in,out] sourceCat catalog to which to copy fields 665 The fields copied are those specified by `config.icSourceFieldsToCopy` 666 that actually exist in the schema. This was set up by the constructor 667 using self.schemaMapper. 670 raise RuntimeError(
"To copy icSource fields you must specify " 671 "icSourceSchema nd icSourceKeys when " 672 "constructing this task")
673 if icSourceCat
is None or sourceCat
is None:
674 raise RuntimeError(
"icSourceCat and sourceCat must both be " 676 if len(self.config.icSourceFieldsToCopy) == 0:
677 self.log.warn(
"copyIcSourceFields doing nothing because " 678 "icSourceFieldsToCopy is empty")
681 mc = afwTable.MatchControl()
682 mc.findOnlyClosest =
False 683 matches = afwTable.matchXy(icSourceCat, sourceCat,
684 self.config.matchRadiusPix, mc)
685 if self.config.doDeblend:
686 deblendKey = sourceCat.schema[
"deblend_nChild"].asKey()
688 matches = [m
for m
in matches
if m[1].get(deblendKey) == 0]
695 for m0, m1, d
in matches:
697 match = bestMatches.get(id0)
698 if match
is None or d <= match[2]:
699 bestMatches[id0] = (m0, m1, d)
700 matches = list(bestMatches.values())
705 numMatches = len(matches)
706 numUniqueSources = len(set(m[1].getId()
for m
in matches))
707 if numUniqueSources != numMatches:
708 self.log.warn(
"{} icSourceCat sources matched only {} sourceCat " 709 "sources".format(numMatches, numUniqueSources))
711 self.log.info(
"Copying flags from icSourceCat to sourceCat for " 712 "%s sources" % (numMatches,))
716 for icSrc, src, d
in matches:
722 icSrcFootprint = icSrc.getFootprint()
724 icSrc.setFootprint(src.getFootprint())
727 icSrc.setFootprint(icSrcFootprint)
def copyIcSourceFields(self, icSourceCat, sourceCat)
Match sources in icSourceCat and sourceCat and copy the specified fields.
def calibrate(self, exposure, exposureIdInfo=None, background=None, icSourceCat=None)
Calibrate an exposure (science image or coadd)
def writeOutputs(self, dataRef, exposure, background, sourceCat, astromMatches, matchMeta)
def run(self, dataRef, exposure=None, background=None, icSourceCat=None, doUnpersist=True)
Calibrate an exposure, optionally unpersisting inputs and persisting outputs.
def __init__(self, butler=None, astromRefObjLoader=None, photoRefObjLoader=None, icSourceSchema=None, kwargs)
Construct a CalibrateTask.
def setMetadata(self, exposure, photoRes=None)
Set task and exposure metadata.
def getSchemaCatalogs(self)
Calibrate an exposure: measure sources and perform astrometric and photometric calibration.