24 from lsstDebug
import getDebugFrame
28 from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches
29 from lsst.meas.extensions.astrometryNet
import LoadAstrometryNetObjectsTask
35 from lsst.meas.base import (SingleFrameMeasurementTask, ApplyApCorrTask,
36 CatalogCalculationTask)
38 from .fakes
import BaseFakeSourcesTask
39 from .photoCal
import PhotoCalTask
41 __all__ = [
"CalibrateConfig",
"CalibrateTask"]
45 """Config for CalibrateTask""" 46 doWrite = pexConfig.Field(
49 doc=
"Save calibration results?",
51 doWriteHeavyFootprintsInSources = pexConfig.Field(
54 doc=
"Include HeavyFootprint data in source table? If false then heavy " 55 "footprints are saved as normal footprints, which saves some space" 57 doWriteMatches = pexConfig.Field(
60 doc=
"Write reference matches (ignored if doWrite false)?",
62 doWriteMatchesDenormalized = pexConfig.Field(
65 doc=(
"Write reference matches in denormalized format? " 66 "This format uses more disk space, but is more convenient to " 67 "read. Ignored if doWriteMatches=False or doWrite=False."),
69 doAstrometry = pexConfig.Field(
72 doc=
"Perform astrometric calibration?",
74 astromRefObjLoader = pexConfig.ConfigurableField(
75 target=LoadAstrometryNetObjectsTask,
76 doc=
"reference object loader for astrometric calibration",
78 photoRefObjLoader = pexConfig.ConfigurableField(
79 target=LoadAstrometryNetObjectsTask,
80 doc=
"reference object loader for photometric calibration",
82 astrometry = pexConfig.ConfigurableField(
83 target=AstrometryTask,
84 doc=
"Perform astrometric calibration to refine the WCS",
86 requireAstrometry = pexConfig.Field(
89 doc=(
"Raise an exception if astrometry fails? Ignored if doAstrometry " 92 doPhotoCal = pexConfig.Field(
95 doc=
"Perform phometric calibration?",
97 requirePhotoCal = pexConfig.Field(
100 doc=(
"Raise an exception if photoCal fails? Ignored if doPhotoCal " 103 photoCal = pexConfig.ConfigurableField(
105 doc=
"Perform photometric calibration",
107 icSourceFieldsToCopy = pexConfig.ListField(
109 default=(
"calib_psfCandidate",
"calib_psfUsed",
"calib_psf_reserved"),
110 doc=(
"Fields to copy from the icSource catalog to the output catalog " 111 "for matching sources Any missing fields will trigger a " 112 "RuntimeError exception. Ignored if icSourceCat is not provided.")
114 matchRadiusPix = pexConfig.Field(
117 doc=(
"Match radius for matching icSourceCat objects to sourceCat " 120 checkUnitsParseStrict = pexConfig.Field(
121 doc=(
"Strictness of Astropy unit compatibility check, can be 'raise', " 122 "'warn' or 'silent'"),
126 detection = pexConfig.ConfigurableField(
127 target=SourceDetectionTask,
130 doDeblend = pexConfig.Field(
133 doc=
"Run deblender input exposure" 135 deblend = pexConfig.ConfigurableField(
136 target=SourceDeblendTask,
137 doc=
"Split blended sources into their components" 139 measurement = pexConfig.ConfigurableField(
140 target=SingleFrameMeasurementTask,
141 doc=
"Measure sources" 143 doApCorr = pexConfig.Field(
146 doc=
"Run subtask to apply aperture correction" 148 applyApCorr = pexConfig.ConfigurableField(
149 target=ApplyApCorrTask,
150 doc=
"Subtask to apply aperture corrections" 155 catalogCalculation = pexConfig.ConfigurableField(
156 target=CatalogCalculationTask,
157 doc=
"Subtask to run catalogCalculation plugins on catalog" 159 doInsertFakes = pexConfig.Field(
162 doc=
"Run fake sources injection task" 164 insertFakes = pexConfig.ConfigurableField(
165 target=BaseFakeSourcesTask,
166 doc=
"Injection of fake sources for testing purposes (must be " 171 pexConfig.Config.setDefaults(self)
183 """!Calibrate an exposure: measure sources and perform astrometric and 184 photometric calibration 186 @anchor CalibrateTask_ 188 @section pipe_tasks_calibrate_Contents Contents 190 - @ref pipe_tasks_calibrate_Purpose 191 - @ref pipe_tasks_calibrate_Initialize 192 - @ref pipe_tasks_calibrate_IO 193 - @ref pipe_tasks_calibrate_Config 194 - @ref pipe_tasks_calibrate_Metadata 195 - @ref pipe_tasks_calibrate_Debug 198 @section pipe_tasks_calibrate_Purpose Description 200 Given an exposure with a good PSF model and aperture correction map 201 (e.g. as provided by @ref CharacterizeImageTask), perform the following 203 - Run detection and measurement 204 - Run astrometry subtask to fit an improved WCS 205 - Run photoCal subtask to fit the exposure's photometric zero-point 207 @section pipe_tasks_calibrate_Initialize Task initialisation 209 @copydoc \_\_init\_\_ 211 @section pipe_tasks_calibrate_IO Invoking the Task 213 If you want this task to unpersist inputs or persist outputs, then call 214 the `run` method (a wrapper around the `calibrate` method). 216 If you already have the inputs unpersisted and do not want to persist the 217 output then it is more direct to call the `calibrate` method: 219 @section pipe_tasks_calibrate_Config Configuration parameters 221 See @ref CalibrateConfig 223 @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata 227 <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task 228 <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal 230 <dt>COLORTERM1 <dd>?? (always 0.0) 231 <dt>COLORTERM2 <dd>?? (always 0.0) 232 <dt>COLORTERM3 <dd>?? (always 0.0) 235 @section pipe_tasks_calibrate_Debug Debug variables 237 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink 238 interface supports a flag 239 `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug 240 for more about `debug.py`. 242 CalibrateTask has a debug dictionary containing one key: 245 <dd>frame (an int; <= 0 to not display) in which to display the exposure, 246 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for 247 the meaning of the various symbols. 250 For example, put something like: 254 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would 255 # call us recursively 256 if name == "lsst.pipe.tasks.calibrate": 263 lsstDebug.Info = DebugInfo 265 into your `debug.py` file and run `calibrateTask.py` with the `--debug` 268 Some subtasks may have their own debug variables; see individual Task 275 ConfigClass = CalibrateConfig
276 _DefaultName =
"calibrate" 277 RunnerClass = pipeBase.ButlerInitializedTaskRunner
279 def __init__(self, butler=None, astromRefObjLoader=None,
280 photoRefObjLoader=None, icSourceSchema=None, **kwargs):
281 """!Construct a CalibrateTask 283 @param[in] butler The butler is passed to the refObjLoader constructor 284 in case it is needed. Ignored if the refObjLoader argument 285 provides a loader directly. 286 @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks 287 that supplies an external reference catalog for astrometric 288 calibration. May be None if the desired loader can be constructed 289 from the butler argument or all steps requiring a reference catalog 291 @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks 292 that supplies an external reference catalog for photometric 293 calibration. May be None if the desired loader can be constructed 294 from the butler argument or all steps requiring a reference catalog 296 @param[in] icSourceSchema schema for icSource catalog, or None. 297 Schema values specified in config.icSourceFieldsToCopy will be 298 taken from this schema. If set to None, no values will be 299 propagated from the icSourceCatalog 300 @param[in,out] kwargs other keyword arguments for 301 lsst.pipe.base.CmdLineTask 303 pipeBase.CmdLineTask.__init__(self, **kwargs)
305 if icSourceSchema
is None and butler
is not None:
307 icSourceSchema = butler.get(
"icSrc_schema", immediate=
True).schema
309 if icSourceSchema
is not None:
312 minimumSchema = afwTable.SourceTable.makeMinimalSchema()
313 self.
schemaMapper.addMinimalSchema(minimumSchema,
False)
321 afwTable.Field[
"Flag"](
"calib_detected",
322 "Source was detected as an icSource"))
323 missingFieldNames = []
324 for fieldName
in self.config.icSourceFieldsToCopy:
326 schemaItem = icSourceSchema.find(fieldName)
328 missingFieldNames.append(fieldName)
333 if missingFieldNames:
334 raise RuntimeError(
"isSourceCat is missing fields {} " 335 "specified in icSourceFieldsToCopy" 336 .format(missingFieldNames))
343 self.
schema = afwTable.SourceTable.makeMinimalSchema()
344 self.makeSubtask(
'detection', schema=self.
schema)
351 if self.config.doInsertFakes:
352 self.makeSubtask(
"insertFakes")
354 if self.config.doDeblend:
355 self.makeSubtask(
"deblend", schema=self.
schema)
356 self.makeSubtask(
'measurement', schema=self.
schema,
358 if self.config.doApCorr:
359 self.makeSubtask(
'applyApCorr', schema=self.
schema)
360 self.makeSubtask(
'catalogCalculation', schema=self.
schema)
362 if self.config.doAstrometry:
363 if astromRefObjLoader
is None:
364 self.makeSubtask(
'astromRefObjLoader', butler=butler)
365 astromRefObjLoader = self.astromRefObjLoader
367 self.makeSubtask(
"astrometry", refObjLoader=astromRefObjLoader,
369 if self.config.doPhotoCal:
370 if photoRefObjLoader
is None:
371 self.makeSubtask(
'photoRefObjLoader', butler=butler)
372 photoRefObjLoader = self.photoRefObjLoader
373 self.
pixelMargin = photoRefObjLoader.config.pixelMargin
374 self.makeSubtask(
"photoCal", refObjLoader=photoRefObjLoader,
380 self.
schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
383 def run(self, dataRef, exposure=None, background=None, icSourceCat=None,
385 """!Calibrate an exposure, optionally unpersisting inputs and 388 This is a wrapper around the `calibrate` method that unpersists inputs 389 (if `doUnpersist` true) and persists outputs (if `config.doWrite` true) 391 @param[in] dataRef butler data reference corresponding to a science 393 @param[in,out] exposure characterized exposure (an 394 lsst.afw.image.ExposureF or similar), or None to unpersist existing 395 icExp and icBackground. See calibrate method for details of what is 397 @param[in,out] background initial model of background already 398 subtracted from exposure (an lsst.afw.math.BackgroundList). May be 399 None if no background has been subtracted, though that is unusual 400 for calibration. A refined background model is output. Ignored if 402 @param[in] icSourceCat catalog from which to copy the fields specified 403 by icSourceKeys, or None; 404 @param[in] doUnpersist unpersist data: 405 - if True, exposure, background and icSourceCat are read from 406 dataRef and those three arguments must all be None; 407 - if False the exposure must be provided; background and 408 icSourceCat are optional. True is intended for running as a 409 command-line task, False for running as a subtask 411 @return same data as the calibrate method 413 self.log.info(
"Processing %s" % (dataRef.dataId))
416 if any(item
is not None for item
in (exposure, background,
418 raise RuntimeError(
"doUnpersist true; exposure, background " 419 "and icSourceCat must all be None")
420 exposure = dataRef.get(
"icExp", immediate=
True)
421 background = dataRef.get(
"icExpBackground", immediate=
True)
422 icSourceCat = dataRef.get(
"icSrc", immediate=
True)
423 elif exposure
is None:
424 raise RuntimeError(
"doUnpersist false; exposure must be provided")
426 exposureIdInfo = dataRef.get(
"expIdInfo")
430 exposureIdInfo=exposureIdInfo,
431 background=background,
432 icSourceCat=icSourceCat,
435 if self.config.doWrite:
438 exposure=calRes.exposure,
439 background=calRes.background,
440 sourceCat=calRes.sourceCat,
441 astromMatches=calRes.astromMatches,
442 matchMeta=calRes.matchMeta,
447 def calibrate(self, exposure, exposureIdInfo=None, background=None,
449 """!Calibrate an exposure (science image or coadd) 451 @param[in,out] exposure exposure to calibrate (an 452 lsst.afw.image.ExposureF or similar); 457 - MaskedImage has background subtracted 459 - Calib zero-point is set 460 @param[in] exposureIdInfo ID info for exposure (an 461 lsst.obs.base.ExposureIdInfo) If not provided, returned 462 SourceCatalog IDs will not be globally unique. 463 @param[in,out] background background model already subtracted from 464 exposure (an lsst.afw.math.BackgroundList). May be None if no 465 background has been subtracted, though that is unusual for 466 calibration. A refined background model is output. 467 @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask 468 from which we can copy some fields. 470 @return pipe_base Struct containing these fields: 471 - exposure calibrate science exposure with refined WCS and Calib 472 - background model of background subtracted from exposure (an 473 lsst.afw.math.BackgroundList) 474 - sourceCat catalog of measured sources 475 - astromMatches list of source/refObj matches from the astrometry 479 if exposureIdInfo
is None:
480 exposureIdInfo = ExposureIdInfo()
482 if background
is None:
483 background = BackgroundList()
484 sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
485 exposureIdInfo.unusedBits)
486 table = SourceTable.make(self.
schema, sourceIdFactory)
489 detRes = self.detection.
run(table=table, exposure=exposure,
491 sourceCat = detRes.sources
492 if detRes.fpSets.background:
493 for bg
in detRes.fpSets.background:
494 background.append(bg)
495 if self.config.doDeblend:
496 self.deblend.
run(exposure=exposure, sources=sourceCat)
497 self.measurement.
run(
500 exposureId=exposureIdInfo.expId
502 if self.config.doApCorr:
503 self.applyApCorr.
run(
505 apCorrMap=exposure.getInfo().getApCorrMap()
507 self.catalogCalculation.
run(sourceCat)
509 if icSourceCat
is not None and \
510 len(self.config.icSourceFieldsToCopy) > 0:
518 if not sourceCat.isContiguous():
519 sourceCat = sourceCat.copy(deep=
True)
525 if self.config.doAstrometry:
527 astromRes = self.astrometry.
run(
531 astromMatches = astromRes.matches
532 matchMeta = astromRes.matchMeta
533 except Exception
as e:
534 if self.config.requireAstrometry:
536 self.log.warn(
"Unable to perform astrometric calibration " 537 "(%s): attempting to proceed" % e)
540 if self.config.doPhotoCal:
542 photoRes = self.photoCal.
run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
543 exposure.getCalib().setFluxMag0(photoRes.calib.getFluxMag0())
544 self.log.info(
"Photometric zero-point: %f" %
545 photoRes.calib.getMagnitude(1.0))
546 self.
setMetadata(exposure=exposure, photoRes=photoRes)
547 except Exception
as e:
548 if self.config.requirePhotoCal:
550 self.log.warn(
"Unable to perform photometric calibration " 551 "(%s): attempting to proceed" % e)
554 if self.config.doInsertFakes:
555 self.insertFakes.
run(exposure, background=background)
557 table = SourceTable.make(self.
schema, sourceIdFactory)
560 detRes = self.detection.
run(table=table, exposure=exposure,
562 sourceCat = detRes.sources
563 if detRes.fpSets.background:
564 for bg
in detRes.fpSets.background:
565 background.append(bg)
566 if self.config.doDeblend:
567 self.deblend.
run(exposure=exposure, sources=sourceCat)
568 self.measurement.
run(
571 exposureId=exposureIdInfo.expId
573 if self.config.doApCorr:
574 self.applyApCorr.
run(
576 apCorrMap=exposure.getInfo().getApCorrMap()
578 self.catalogCalculation.
run(sourceCat)
580 if icSourceCat
is not None and \
581 len(self.config.icSourceFieldsToCopy) > 0:
585 frame = getDebugFrame(self._display,
"calibrate")
590 matches=astromMatches,
595 return pipeBase.Struct(
597 background=background,
599 astromMatches=astromMatches,
603 def writeOutputs(self, dataRef, exposure, background, sourceCat,
604 astromMatches, matchMeta):
605 """Write output data to the output repository 607 @param[in] dataRef butler data reference corresponding to a science 609 @param[in] exposure exposure to write 610 @param[in] background background model for exposure 611 @param[in] sourceCat catalog of measured sources 612 @param[in] astromMatches list of source/refObj matches from the 615 dataRef.put(sourceCat,
"src")
616 if self.config.doWriteMatches
and astromMatches
is not None:
617 normalizedMatches = afwTable.packMatches(astromMatches)
618 normalizedMatches.table.setMetadata(matchMeta)
619 dataRef.put(normalizedMatches,
"srcMatch")
620 if self.config.doWriteMatchesDenormalized:
621 denormMatches = denormalizeMatches(astromMatches, matchMeta)
622 dataRef.put(denormMatches,
"srcMatchFull")
623 dataRef.put(exposure,
"calexp")
624 dataRef.put(background,
"calexpBackground")
627 """Return a dict of empty catalogs for each catalog dataset produced 630 sourceCat = afwTable.SourceCatalog(self.
schema)
632 return {
"src": sourceCat}
635 """!Set task and exposure metadata 637 Logs a warning and continues if needed data is missing. 639 @param[in,out] exposure exposure whose metadata is to be set 640 @param[in] photoRes results of running photoCal; if None then it was 648 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
649 magZero = photoRes.zp - 2.5*math.log10(exposureTime)
650 self.metadata.set(
'MAGZERO', magZero)
652 self.log.warn(
"Could not set normalized MAGZERO in header: no " 656 metadata = exposure.getMetadata()
657 metadata.set(
'MAGZERO_RMS', photoRes.sigma)
658 metadata.set(
'MAGZERO_NOBJ', photoRes.ngood)
659 metadata.set(
'COLORTERM1', 0.0)
660 metadata.set(
'COLORTERM2', 0.0)
661 metadata.set(
'COLORTERM3', 0.0)
662 except Exception
as e:
663 self.log.warn(
"Could not set exposure metadata: %s" % (e,))
666 """!Match sources in icSourceCat and sourceCat and copy the specified fields 668 @param[in] icSourceCat catalog from which to copy fields 669 @param[in,out] sourceCat catalog to which to copy fields 671 The fields copied are those specified by `config.icSourceFieldsToCopy` 672 that actually exist in the schema. This was set up by the constructor 673 using self.schemaMapper. 676 raise RuntimeError(
"To copy icSource fields you must specify " 677 "icSourceSchema nd icSourceKeys when " 678 "constructing this task")
679 if icSourceCat
is None or sourceCat
is None:
680 raise RuntimeError(
"icSourceCat and sourceCat must both be " 682 if len(self.config.icSourceFieldsToCopy) == 0:
683 self.log.warn(
"copyIcSourceFields doing nothing because " 684 "icSourceFieldsToCopy is empty")
687 mc = afwTable.MatchControl()
688 mc.findOnlyClosest =
False 689 matches = afwTable.matchXy(icSourceCat, sourceCat,
690 self.config.matchRadiusPix, mc)
691 if self.config.doDeblend:
692 deblendKey = sourceCat.schema[
"deblend_nChild"].asKey()
694 matches = [m
for m
in matches
if m[1].get(deblendKey) == 0]
701 for m0, m1, d
in matches:
703 match = bestMatches.get(id0)
704 if match
is None or d <= match[2]:
705 bestMatches[id0] = (m0, m1, d)
706 matches = list(bestMatches.values())
711 numMatches = len(matches)
712 numUniqueSources = len(set(m[1].getId()
for m
in matches))
713 if numUniqueSources != numMatches:
714 self.log.warn(
"{} icSourceCat sources matched only {} sourceCat " 715 "sources".format(numMatches, numUniqueSources))
717 self.log.info(
"Copying flags from icSourceCat to sourceCat for " 718 "%s sources" % (numMatches,))
722 for icSrc, src, d
in matches:
728 icSrcFootprint = icSrc.getFootprint()
730 icSrc.setFootprint(src.getFootprint())
733 icSrc.setFootprint(icSrcFootprint)
def copyIcSourceFields(self, icSourceCat, sourceCat)
Match sources in icSourceCat and sourceCat and copy the specified fields.
def calibrate(self, exposure, exposureIdInfo=None, background=None, icSourceCat=None)
Calibrate an exposure (science image or coadd)
def writeOutputs(self, dataRef, exposure, background, sourceCat, astromMatches, matchMeta)
def run(self, dataRef, exposure=None, background=None, icSourceCat=None, doUnpersist=True)
Calibrate an exposure, optionally unpersisting inputs and persisting outputs.
def __init__(self, butler=None, astromRefObjLoader=None, photoRefObjLoader=None, icSourceSchema=None, kwargs)
Construct a CalibrateTask.
def setMetadata(self, exposure, photoRes=None)
Set task and exposure metadata.
def getSchemaCatalogs(self)
Calibrate an exposure: measure sources and perform astrometric and photometric calibration.