25from lsstDebug
import getDebugFrame
28import lsst.pipe.base.connectionTypes
as cT
30from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches
32from lsst.obs.base
import ExposureIdInfo
39 CatalogCalculationTask)
41from lsst.utils.timer
import timeMethod
43from .fakes
import BaseFakeSourcesTask
44from .photoCal
import PhotoCalTask
45from .computeExposureSummaryStats
import ComputeExposureSummaryStatsTask
48__all__ = [
"CalibrateConfig",
"CalibrateTask"]
51class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=(
"instrument",
"visit",
"detector"),
54 icSourceSchema = cT.InitInput(
55 doc=
"Schema produced by characterize image task, used to initialize this task",
57 storageClass=
"SourceCatalog",
60 outputSchema = cT.InitOutput(
61 doc=
"Schema after CalibrateTask has been initialized",
63 storageClass=
"SourceCatalog",
67 doc=
"Input image to calibrate",
69 storageClass=
"ExposureF",
70 dimensions=(
"instrument",
"visit",
"detector"),
73 background = cT.Input(
74 doc=
"Backgrounds determined by characterize task",
75 name=
"icExpBackground",
76 storageClass=
"Background",
77 dimensions=(
"instrument",
"visit",
"detector"),
80 icSourceCat = cT.Input(
81 doc=
"Source catalog created by characterize task",
83 storageClass=
"SourceCatalog",
84 dimensions=(
"instrument",
"visit",
"detector"),
87 astromRefCat = cT.PrerequisiteInput(
88 doc=
"Reference catalog to use for astrometry",
89 name=
"gaia_dr2_20200414",
90 storageClass=
"SimpleCatalog",
91 dimensions=(
"skypix",),
96 photoRefCat = cT.PrerequisiteInput(
97 doc=
"Reference catalog to use for photometric calibration",
98 name=
"ps1_pv3_3pi_20170110",
99 storageClass=
"SimpleCatalog",
100 dimensions=(
"skypix",),
105 outputExposure = cT.Output(
106 doc=
"Exposure after running calibration task",
108 storageClass=
"ExposureF",
109 dimensions=(
"instrument",
"visit",
"detector"),
112 outputCat = cT.Output(
113 doc=
"Source catalog produced in calibrate task",
115 storageClass=
"SourceCatalog",
116 dimensions=(
"instrument",
"visit",
"detector"),
119 outputBackground = cT.Output(
120 doc=
"Background models estimated in calibration task",
121 name=
"calexpBackground",
122 storageClass=
"Background",
123 dimensions=(
"instrument",
"visit",
"detector"),
127 doc=
"Source/refObj matches from the astrometry solver",
129 storageClass=
"Catalog",
130 dimensions=(
"instrument",
"visit",
"detector"),
133 matchesDenormalized = cT.Output(
134 doc=
"Denormalized matches from astrometry solver",
136 storageClass=
"Catalog",
137 dimensions=(
"instrument",
"visit",
"detector"),
143 if config.doAstrometry
is False:
144 self.prerequisiteInputs.remove(
"astromRefCat")
145 if config.doPhotoCal
is False:
146 self.prerequisiteInputs.remove(
"photoRefCat")
148 if config.doWriteMatches
is False or config.doAstrometry
is False:
149 self.outputs.remove(
"matches")
150 if config.doWriteMatchesDenormalized
is False or config.doAstrometry
is False:
151 self.outputs.remove(
"matchesDenormalized")
154class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections):
155 """Config for CalibrateTask"""
156 doWrite = pexConfig.Field(
159 doc=
"Save calibration results?",
161 doWriteHeavyFootprintsInSources = pexConfig.Field(
164 doc=
"Include HeavyFootprint data in source table? If false then heavy "
165 "footprints are saved as normal footprints, which saves some space"
167 doWriteMatches = pexConfig.Field(
170 doc=
"Write reference matches (ignored if doWrite or doAstrometry false)?",
172 doWriteMatchesDenormalized = pexConfig.Field(
175 doc=(
"Write reference matches in denormalized format? "
176 "This format uses more disk space, but is more convenient to "
177 "read. Ignored if doWriteMatches=False or doWrite=False."),
179 doAstrometry = pexConfig.Field(
182 doc=
"Perform astrometric calibration?",
184 astromRefObjLoader = pexConfig.ConfigField(
185 dtype=LoadReferenceObjectsConfig,
186 doc=
"reference object loader for astrometric calibration",
188 photoRefObjLoader = pexConfig.ConfigField(
189 dtype=LoadReferenceObjectsConfig,
190 doc=
"reference object loader for photometric calibration",
192 astrometry = pexConfig.ConfigurableField(
193 target=AstrometryTask,
194 doc=
"Perform astrometric calibration to refine the WCS",
196 requireAstrometry = pexConfig.Field(
199 doc=(
"Raise an exception if astrometry fails? Ignored if doAstrometry "
202 doPhotoCal = pexConfig.Field(
205 doc=
"Perform phometric calibration?",
207 requirePhotoCal = pexConfig.Field(
210 doc=(
"Raise an exception if photoCal fails? Ignored if doPhotoCal "
213 photoCal = pexConfig.ConfigurableField(
215 doc=
"Perform photometric calibration",
217 icSourceFieldsToCopy = pexConfig.ListField(
219 default=(
"calib_psf_candidate",
"calib_psf_used",
"calib_psf_reserved"),
220 doc=(
"Fields to copy from the icSource catalog to the output catalog "
221 "for matching sources Any missing fields will trigger a "
222 "RuntimeError exception. Ignored if icSourceCat is not provided.")
224 matchRadiusPix = pexConfig.Field(
227 doc=(
"Match radius for matching icSourceCat objects to sourceCat "
230 checkUnitsParseStrict = pexConfig.Field(
231 doc=(
"Strictness of Astropy unit compatibility check, can be 'raise', "
232 "'warn' or 'silent'"),
236 detection = pexConfig.ConfigurableField(
237 target=SourceDetectionTask,
240 doDeblend = pexConfig.Field(
243 doc=
"Run deblender input exposure"
245 deblend = pexConfig.ConfigurableField(
246 target=SourceDeblendTask,
247 doc=
"Split blended sources into their components"
249 doSkySources = pexConfig.Field(
252 doc=
"Generate sky sources?",
254 skySources = pexConfig.ConfigurableField(
255 target=SkyObjectsTask,
256 doc=
"Generate sky sources",
258 measurement = pexConfig.ConfigurableField(
259 target=SingleFrameMeasurementTask,
260 doc=
"Measure sources"
262 postCalibrationMeasurement = pexConfig.ConfigurableField(
263 target=SingleFrameMeasurementTask,
264 doc=
"Second round of measurement for plugins that need to be run after photocal"
266 setPrimaryFlags = pexConfig.ConfigurableField(
267 target=SetPrimaryFlagsTask,
268 doc=(
"Set flags for primary source classification in single frame "
269 "processing. True if sources are not sky sources and not a parent.")
271 doApCorr = pexConfig.Field(
274 doc=
"Run subtask to apply aperture correction"
276 applyApCorr = pexConfig.ConfigurableField(
277 target=ApplyApCorrTask,
278 doc=
"Subtask to apply aperture corrections"
283 catalogCalculation = pexConfig.ConfigurableField(
284 target=CatalogCalculationTask,
285 doc=
"Subtask to run catalogCalculation plugins on catalog"
287 doInsertFakes = pexConfig.Field(
290 doc=
"Run fake sources injection task",
291 deprecated=(
"doInsertFakes is no longer supported. This config will be removed after v24. "
292 "Please use ProcessCcdWithFakesTask instead.")
294 insertFakes = pexConfig.ConfigurableField(
295 target=BaseFakeSourcesTask,
296 doc=
"Injection of fake sources for testing purposes (must be "
298 deprecated=(
"insertFakes is no longer supported. This config will be removed after v24. "
299 "Please use ProcessCcdWithFakesTask instead.")
301 doComputeSummaryStats = pexConfig.Field(
304 doc=
"Run subtask to measure exposure summary statistics?"
306 computeSummaryStats = pexConfig.ConfigurableField(
307 target=ComputeExposureSummaryStatsTask,
308 doc=
"Subtask to run computeSummaryStats on exposure"
310 doWriteExposure = pexConfig.Field(
313 doc=
"Write the calexp? If fakes have been added then we do not want to write out the calexp as a "
314 "normal calexp but as a fakes_calexp."
319 self.
detection.doTempLocalBackground =
False
320 self.
deblend.maxFootprintSize = 2000
327 self.
photoCal.photoCatName = self.connections.photoRefCat
331 """Task to calibrate an exposure.
336 Compatibility parameter. Should always be `
None`.
337 astromRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
338 Reference object loader
for astrometry task. Must be
None if
339 run
as part of PipelineTask.
340 photoRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
341 Reference object loader
for photometry task. Must be
None if
342 run
as part of PipelineTask.
344 Schema
for the icSource catalog.
345 initInputs : `dict`, optional
346 Dictionary that can contain a key ``icSourceSchema`` containing the
347 input schema. If present will override the value of ``icSourceSchema``.
349 ConfigClass = CalibrateConfig
350 _DefaultName = "calibrate"
352 def __init__(self, butler=None, astromRefObjLoader=None,
353 photoRefObjLoader=None, icSourceSchema=None,
354 initInputs=None, **kwargs):
357 if butler
is not None:
358 warnings.warn(
"The 'butler' parameter is no longer used and can be safely removed.",
359 category=FutureWarning, stacklevel=2)
362 if initInputs
is not None:
363 icSourceSchema = initInputs[
'icSourceSchema'].schema
365 if icSourceSchema
is not None:
368 minimumSchema = afwTable.SourceTable.makeMinimalSchema()
369 self.
schemaMapper.addMinimalSchema(minimumSchema,
False)
377 afwTable.Field[
"Flag"](
"calib_detected",
378 "Source was detected as an icSource"))
379 missingFieldNames = []
380 for fieldName
in self.config.icSourceFieldsToCopy:
382 schemaItem = icSourceSchema.find(fieldName)
384 missingFieldNames.append(fieldName)
389 if missingFieldNames:
390 raise RuntimeError(
"isSourceCat is missing fields {} "
391 "specified in icSourceFieldsToCopy"
392 .format(missingFieldNames))
399 self.
schema = afwTable.SourceTable.makeMinimalSchema()
400 self.makeSubtask(
'detection', schema=self.
schema)
404 if self.config.doDeblend:
405 self.makeSubtask(
"deblend", schema=self.
schema)
406 if self.config.doSkySources:
407 self.makeSubtask(
"skySources")
409 self.makeSubtask(
'measurement', schema=self.
schema,
411 self.makeSubtask(
'postCalibrationMeasurement', schema=self.
schema,
413 self.makeSubtask(
"setPrimaryFlags", schema=self.
schema, isSingleFrame=
True)
414 if self.config.doApCorr:
415 self.makeSubtask(
'applyApCorr', schema=self.
schema)
416 self.makeSubtask(
'catalogCalculation', schema=self.
schema)
418 if self.config.doAstrometry:
419 self.makeSubtask(
"astrometry", refObjLoader=astromRefObjLoader,
421 if self.config.doPhotoCal:
422 self.makeSubtask(
"photoCal", refObjLoader=photoRefObjLoader,
424 if self.config.doComputeSummaryStats:
425 self.makeSubtask(
'computeSummaryStats')
427 if initInputs
is not None and (astromRefObjLoader
is not None or photoRefObjLoader
is not None):
428 raise RuntimeError(
"PipelineTask form of this task should not be initialized with "
429 "reference object loaders.")
434 self.
schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
436 sourceCatSchema = afwTable.SourceCatalog(self.
schema)
441 inputs = butlerQC.get(inputRefs)
442 inputs[
'exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId,
"visit_detector")
444 if self.config.doAstrometry:
445 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
446 for ref
in inputRefs.astromRefCat],
447 refCats=inputs.pop(
'astromRefCat'),
448 name=self.config.connections.astromRefCat,
449 config=self.config.astromRefObjLoader, log=self.log)
450 self.astrometry.setRefObjLoader(refObjLoader)
452 if self.config.doPhotoCal:
453 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
454 for ref
in inputRefs.photoRefCat],
455 refCats=inputs.pop(
'photoRefCat'),
456 name=self.config.connections.photoRefCat,
457 config=self.config.photoRefObjLoader,
459 self.photoCal.match.setRefObjLoader(photoRefObjLoader)
461 outputs = self.
run(**inputs)
463 if self.config.doWriteMatches
and self.config.doAstrometry:
464 normalizedMatches = afwTable.packMatches(outputs.astromMatches)
465 normalizedMatches.table.setMetadata(outputs.matchMeta)
466 if self.config.doWriteMatchesDenormalized:
467 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
468 outputs.matchesDenormalized = denormMatches
469 outputs.matches = normalizedMatches
470 butlerQC.put(outputs, outputRefs)
473 def run(self, exposure, exposureIdInfo=None, background=None,
475 """Calibrate an exposure.
479 exposure : `lsst.afw.image.ExposureF`
480 Exposure to calibrate.
481 exposureIdInfo : `lsst.obs.baseExposureIdInfo`, optional
482 Exposure ID info. If not provided, returned SourceCatalog IDs will
not
484 background : `lsst.afw.math.BackgroundList`, optional
485 Initial model of background already subtracted
from exposure.
486 icSourceCat : `lsst.afw.image.SourceCatalog`, optional
487 SourceCatalog
from CharacterizeImageTask
from which we can copy some fields.
491 result : `lsst.pipe.base.Struct`
492 Result structure
with the following attributes:
495 Characterized exposure (`lsst.afw.image.ExposureF`).
499 Model of subtracted background (`lsst.afw.math.BackgroundList`).
501 List of source/ref matches
from astrometry solver.
503 Metadata
from astrometry matches.
505 Another reference to ``exposure``
for compatibility.
507 Another reference to ``sourceCat``
for compatibility.
510 if exposureIdInfo
is None:
511 exposureIdInfo = ExposureIdInfo()
513 if background
is None:
514 background = BackgroundList()
515 sourceIdFactory = exposureIdInfo.makeSourceIdFactory()
516 table = SourceTable.make(self.
schema, sourceIdFactory)
519 detRes = self.detection.run(table=table, exposure=exposure,
521 sourceCat = detRes.sources
522 if detRes.fpSets.background:
523 for bg
in detRes.fpSets.background:
524 background.append(bg)
525 if self.config.doSkySources:
526 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId)
527 if skySourceFootprints:
528 for foot
in skySourceFootprints:
529 s = sourceCat.addNew()
532 if self.config.doDeblend:
533 self.deblend.run(exposure=exposure, sources=sourceCat)
534 self.measurement.run(
537 exposureId=exposureIdInfo.expId
539 if self.config.doApCorr:
540 self.applyApCorr.run(
542 apCorrMap=exposure.getInfo().getApCorrMap()
544 self.catalogCalculation.run(sourceCat)
546 self.setPrimaryFlags.run(sourceCat)
548 if icSourceCat
is not None and \
549 len(self.config.icSourceFieldsToCopy) > 0:
557 if not sourceCat.isContiguous():
558 sourceCat = sourceCat.copy(deep=
True)
564 if self.config.doAstrometry:
566 astromRes = self.astrometry.run(
570 astromMatches = astromRes.matches
571 matchMeta = astromRes.matchMeta
572 except Exception
as e:
573 if self.config.requireAstrometry:
575 self.log.warning(
"Unable to perform astrometric calibration "
576 "(%s): attempting to proceed", e)
579 if self.config.doPhotoCal:
581 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
582 exposure.setPhotoCalib(photoRes.photoCalib)
584 self.log.info(
"Photometric zero-point: %f",
585 photoRes.photoCalib.instFluxToMagnitude(1.0))
586 self.
setMetadata(exposure=exposure, photoRes=photoRes)
587 except Exception
as e:
588 if self.config.requirePhotoCal:
590 self.log.warning(
"Unable to perform photometric calibration "
591 "(%s): attempting to proceed", e)
594 self.postCalibrationMeasurement.run(
597 exposureId=exposureIdInfo.expId
600 if self.config.doComputeSummaryStats:
601 summary = self.computeSummaryStats.run(exposure=exposure,
603 background=background)
604 exposure.getInfo().setSummaryStats(summary)
606 frame = getDebugFrame(self._display,
"calibrate")
611 matches=astromMatches,
616 return pipeBase.Struct(
618 astromMatches=astromMatches,
620 outputExposure=exposure,
622 outputBackground=background,
626 """Return a dict of empty catalogs for each catalog dataset produced
629 sourceCat = afwTable.SourceCatalog(self.schema)
631 return {
"src": sourceCat}
634 """Set task and exposure metadata.
636 Logs a warning continues if needed data
is missing.
640 exposure : `lsst.afw.image.ExposureF`
641 Exposure to set metadata on.
642 photoRes : `lsst.pipe.base.Struct`, optional
643 Result of running photoCal task.
648 metadata = exposure.getMetadata()
652 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
653 magZero = photoRes.zp - 2.5*math.log10(exposureTime)
655 self.log.warning(
"Could not set normalized MAGZERO in header: no "
660 metadata.set(
'MAGZERO', magZero)
661 metadata.set(
'MAGZERO_RMS', photoRes.sigma)
662 metadata.set(
'MAGZERO_NOBJ', photoRes.ngood)
663 metadata.set(
'COLORTERM1', 0.0)
664 metadata.set(
'COLORTERM2', 0.0)
665 metadata.set(
'COLORTERM3', 0.0)
666 except Exception
as e:
667 self.log.warning(
"Could not set exposure metadata: %s", e)
670 """Match sources in an icSourceCat and a sourceCat and copy fields.
672 The fields copied are those specified by ``config.icSourceFieldsToCopy``.
677 Catalog from which to copy fields.
679 Catalog to which to copy fields.
682 raise RuntimeError(
"To copy icSource fields you must specify "
683 "icSourceSchema nd icSourceKeys when "
684 "constructing this task")
685 if icSourceCat
is None or sourceCat
is None:
686 raise RuntimeError(
"icSourceCat and sourceCat must both be "
688 if len(self.config.icSourceFieldsToCopy) == 0:
689 self.log.warning(
"copyIcSourceFields doing nothing because "
690 "icSourceFieldsToCopy is empty")
693 mc = afwTable.MatchControl()
694 mc.findOnlyClosest =
False
695 matches = afwTable.matchXy(icSourceCat, sourceCat,
696 self.config.matchRadiusPix, mc)
697 if self.config.doDeblend:
698 deblendKey = sourceCat.schema[
"deblend_nChild"].asKey()
700 matches = [m
for m
in matches
if m[1].get(deblendKey) == 0]
707 for m0, m1, d
in matches:
709 match = bestMatches.get(id0)
710 if match
is None or d <= match[2]:
711 bestMatches[id0] = (m0, m1, d)
712 matches = list(bestMatches.values())
717 numMatches = len(matches)
718 numUniqueSources = len(set(m[1].getId()
for m
in matches))
719 if numUniqueSources != numMatches:
720 self.log.warning(
"%d icSourceCat sources matched only %d sourceCat "
721 "sources", numMatches, numUniqueSources)
723 self.log.info(
"Copying flags from icSourceCat to sourceCat for "
724 "%d sources", numMatches)
728 for icSrc, src, d
in matches:
734 icSrcFootprint = icSrc.getFootprint()
736 icSrc.setFootprint(src.getFootprint())
739 icSrc.setFootprint(icSrcFootprint)
postCalibrationMeasurement
def __init__(self, *config=None)
def __init__(self, butler=None, astromRefObjLoader=None, photoRefObjLoader=None, icSourceSchema=None, initInputs=None, **kwargs)
def setMetadata(self, exposure, photoRes=None)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def getSchemaCatalogs(self)
def copyIcSourceFields(self, icSourceCat, sourceCat)
def run(self, exposure, exposureIdInfo=None, background=None, icSourceCat=None)