25from lsstDebug
import getDebugFrame
28import lsst.pipe.base.connectionTypes
as cT
30from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches
32from lsst.obs.base
import ExposureIdInfo
39 CatalogCalculationTask)
41from lsst.utils.timer
import timeMethod
43from .fakes
import BaseFakeSourcesTask
44from .photoCal
import PhotoCalTask
45from .computeExposureSummaryStats
import ComputeExposureSummaryStatsTask
48__all__ = [
"CalibrateConfig",
"CalibrateTask"]
51class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=(
"instrument",
"visit",
"detector"),
54 icSourceSchema = cT.InitInput(
55 doc=
"Schema produced by characterize image task, used to initialize this task",
57 storageClass=
"SourceCatalog",
60 outputSchema = cT.InitOutput(
61 doc=
"Schema after CalibrateTask has been initialized",
63 storageClass=
"SourceCatalog",
67 doc=
"Input image to calibrate",
69 storageClass=
"ExposureF",
70 dimensions=(
"instrument",
"visit",
"detector"),
73 background = cT.Input(
74 doc=
"Backgrounds determined by characterize task",
75 name=
"icExpBackground",
76 storageClass=
"Background",
77 dimensions=(
"instrument",
"visit",
"detector"),
80 icSourceCat = cT.Input(
81 doc=
"Source catalog created by characterize task",
83 storageClass=
"SourceCatalog",
84 dimensions=(
"instrument",
"visit",
"detector"),
87 astromRefCat = cT.PrerequisiteInput(
88 doc=
"Reference catalog to use for astrometry",
89 name=
"gaia_dr2_20200414",
90 storageClass=
"SimpleCatalog",
91 dimensions=(
"skypix",),
96 photoRefCat = cT.PrerequisiteInput(
97 doc=
"Reference catalog to use for photometric calibration",
98 name=
"ps1_pv3_3pi_20170110",
99 storageClass=
"SimpleCatalog",
100 dimensions=(
"skypix",),
105 outputExposure = cT.Output(
106 doc=
"Exposure after running calibration task",
108 storageClass=
"ExposureF",
109 dimensions=(
"instrument",
"visit",
"detector"),
112 outputCat = cT.Output(
113 doc=
"Source catalog produced in calibrate task",
115 storageClass=
"SourceCatalog",
116 dimensions=(
"instrument",
"visit",
"detector"),
119 outputBackground = cT.Output(
120 doc=
"Background models estimated in calibration task",
121 name=
"calexpBackground",
122 storageClass=
"Background",
123 dimensions=(
"instrument",
"visit",
"detector"),
127 doc=
"Source/refObj matches from the astrometry solver",
129 storageClass=
"Catalog",
130 dimensions=(
"instrument",
"visit",
"detector"),
133 matchesDenormalized = cT.Output(
134 doc=
"Denormalized matches from astrometry solver",
136 storageClass=
"Catalog",
137 dimensions=(
"instrument",
"visit",
"detector"),
143 if config.doAstrometry
is False:
144 self.prerequisiteInputs.remove(
"astromRefCat")
145 if config.doPhotoCal
is False:
146 self.prerequisiteInputs.remove(
"photoRefCat")
148 if config.doWriteMatches
is False or config.doAstrometry
is False:
149 self.outputs.remove(
"matches")
150 if config.doWriteMatchesDenormalized
is False or config.doAstrometry
is False:
151 self.outputs.remove(
"matchesDenormalized")
154class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections):
155 """Config for CalibrateTask"""
156 doWrite = pexConfig.Field(
159 doc=
"Save calibration results?",
161 doWriteHeavyFootprintsInSources = pexConfig.Field(
164 doc=
"Include HeavyFootprint data in source table? If false then heavy "
165 "footprints are saved as normal footprints, which saves some space"
167 doWriteMatches = pexConfig.Field(
170 doc=
"Write reference matches (ignored if doWrite or doAstrometry false)?",
172 doWriteMatchesDenormalized = pexConfig.Field(
175 doc=(
"Write reference matches in denormalized format? "
176 "This format uses more disk space, but is more convenient to "
177 "read. Ignored if doWriteMatches=False or doWrite=False."),
179 doAstrometry = pexConfig.Field(
182 doc=
"Perform astrometric calibration?",
184 astromRefObjLoader = pexConfig.ConfigField(
185 dtype=LoadReferenceObjectsConfig,
186 doc=
"reference object loader for astrometric calibration",
188 photoRefObjLoader = pexConfig.ConfigField(
189 dtype=LoadReferenceObjectsConfig,
190 doc=
"reference object loader for photometric calibration",
192 astrometry = pexConfig.ConfigurableField(
193 target=AstrometryTask,
194 doc=
"Perform astrometric calibration to refine the WCS",
196 requireAstrometry = pexConfig.Field(
199 doc=(
"Raise an exception if astrometry fails? Ignored if doAstrometry "
202 doPhotoCal = pexConfig.Field(
205 doc=
"Perform phometric calibration?",
207 requirePhotoCal = pexConfig.Field(
210 doc=(
"Raise an exception if photoCal fails? Ignored if doPhotoCal "
213 photoCal = pexConfig.ConfigurableField(
215 doc=
"Perform photometric calibration",
217 icSourceFieldsToCopy = pexConfig.ListField(
219 default=(
"calib_psf_candidate",
"calib_psf_used",
"calib_psf_reserved"),
220 doc=(
"Fields to copy from the icSource catalog to the output catalog "
221 "for matching sources Any missing fields will trigger a "
222 "RuntimeError exception. Ignored if icSourceCat is not provided.")
224 matchRadiusPix = pexConfig.Field(
227 doc=(
"Match radius for matching icSourceCat objects to sourceCat "
230 checkUnitsParseStrict = pexConfig.Field(
231 doc=(
"Strictness of Astropy unit compatibility check, can be 'raise', "
232 "'warn' or 'silent'"),
236 detection = pexConfig.ConfigurableField(
237 target=SourceDetectionTask,
240 doDeblend = pexConfig.Field(
243 doc=
"Run deblender input exposure"
245 deblend = pexConfig.ConfigurableField(
246 target=SourceDeblendTask,
247 doc=
"Split blended sources into their components"
249 doSkySources = pexConfig.Field(
252 doc=
"Generate sky sources?",
254 skySources = pexConfig.ConfigurableField(
255 target=SkyObjectsTask,
256 doc=
"Generate sky sources",
258 measurement = pexConfig.ConfigurableField(
259 target=SingleFrameMeasurementTask,
260 doc=
"Measure sources"
262 postCalibrationMeasurement = pexConfig.ConfigurableField(
263 target=SingleFrameMeasurementTask,
264 doc=
"Second round of measurement for plugins that need to be run after photocal"
266 setPrimaryFlags = pexConfig.ConfigurableField(
267 target=SetPrimaryFlagsTask,
268 doc=(
"Set flags for primary source classification in single frame "
269 "processing. True if sources are not sky sources and not a parent.")
271 doApCorr = pexConfig.Field(
274 doc=
"Run subtask to apply aperture correction"
276 applyApCorr = pexConfig.ConfigurableField(
277 target=ApplyApCorrTask,
278 doc=
"Subtask to apply aperture corrections"
283 catalogCalculation = pexConfig.ConfigurableField(
284 target=CatalogCalculationTask,
285 doc=
"Subtask to run catalogCalculation plugins on catalog"
287 doInsertFakes = pexConfig.Field(
290 doc=
"Run fake sources injection task",
291 deprecated=(
"doInsertFakes is no longer supported. This config will be removed after v24. "
292 "Please use ProcessCcdWithFakesTask instead.")
294 insertFakes = pexConfig.ConfigurableField(
295 target=BaseFakeSourcesTask,
296 doc=
"Injection of fake sources for testing purposes (must be "
298 deprecated=(
"insertFakes is no longer supported. This config will be removed after v24. "
299 "Please use ProcessCcdWithFakesTask instead.")
301 doComputeSummaryStats = pexConfig.Field(
304 doc=
"Run subtask to measure exposure summary statistics?"
306 computeSummaryStats = pexConfig.ConfigurableField(
307 target=ComputeExposureSummaryStatsTask,
308 doc=
"Subtask to run computeSummaryStats on exposure"
310 doWriteExposure = pexConfig.Field(
313 doc=
"Write the calexp? If fakes have been added then we do not want to write out the calexp as a "
314 "normal calexp but as a fakes_calexp."
319 self.
detection.doTempLocalBackground =
False
320 self.
deblend.maxFootprintSize = 2000
327 self.
photoCal.photoCatName = self.connections.photoRefCat
336 if astromRefCatGen2
is not None and astromRefCatGen2 != self.connections.astromRefCat:
338 f
"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference "
339 f
"catalogs are different. These options must be kept in sync until Gen2 is retired."
342 if photoRefCatGen2
is not None and photoRefCatGen2 != self.connections.photoRefCat:
344 f
"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference "
345 f
"catalogs are different. These options must be kept in sync until Gen2 is retired."
350 """Task to calibrate an exposure.
355 Compatibility parameter. Should always be `
None`.
356 astromRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
357 Reference object loader
for astrometry task. Must be
None if
358 run
as part of PipelineTask.
359 photoRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
360 Reference object loader
for photometry task. Must be
None if
361 run
as part of PipelineTask.
363 Schema
for the icSource catalog.
364 initInputs : `dict`, optional
365 Dictionary that can contain a key ``icSourceSchema`` containing the
366 input schema. If present will override the value of ``icSourceSchema``.
368 ConfigClass = CalibrateConfig
369 _DefaultName = "calibrate"
371 def __init__(self, butler=None, astromRefObjLoader=None,
372 photoRefObjLoader=None, icSourceSchema=None,
373 initInputs=None, **kwargs):
376 if butler
is not None:
377 warnings.warn(
"The 'butler' parameter is no longer used and can be safely removed.",
378 category=FutureWarning, stacklevel=2)
381 if initInputs
is not None:
382 icSourceSchema = initInputs[
'icSourceSchema'].schema
384 if icSourceSchema
is not None:
387 minimumSchema = afwTable.SourceTable.makeMinimalSchema()
388 self.
schemaMapper.addMinimalSchema(minimumSchema,
False)
396 afwTable.Field[
"Flag"](
"calib_detected",
397 "Source was detected as an icSource"))
398 missingFieldNames = []
399 for fieldName
in self.config.icSourceFieldsToCopy:
401 schemaItem = icSourceSchema.find(fieldName)
403 missingFieldNames.append(fieldName)
408 if missingFieldNames:
409 raise RuntimeError(
"isSourceCat is missing fields {} "
410 "specified in icSourceFieldsToCopy"
411 .format(missingFieldNames))
418 self.
schema = afwTable.SourceTable.makeMinimalSchema()
419 self.makeSubtask(
'detection', schema=self.
schema)
423 if self.config.doDeblend:
424 self.makeSubtask(
"deblend", schema=self.
schema)
425 if self.config.doSkySources:
426 self.makeSubtask(
"skySources")
428 self.makeSubtask(
'measurement', schema=self.
schema,
430 self.makeSubtask(
'postCalibrationMeasurement', schema=self.
schema,
432 self.makeSubtask(
"setPrimaryFlags", schema=self.
schema, isSingleFrame=
True)
433 if self.config.doApCorr:
434 self.makeSubtask(
'applyApCorr', schema=self.
schema)
435 self.makeSubtask(
'catalogCalculation', schema=self.
schema)
437 if self.config.doAstrometry:
438 self.makeSubtask(
"astrometry", refObjLoader=astromRefObjLoader,
440 if self.config.doPhotoCal:
441 self.makeSubtask(
"photoCal", refObjLoader=photoRefObjLoader,
443 if self.config.doComputeSummaryStats:
444 self.makeSubtask(
'computeSummaryStats')
446 if initInputs
is not None and (astromRefObjLoader
is not None or photoRefObjLoader
is not None):
447 raise RuntimeError(
"PipelineTask form of this task should not be initialized with "
448 "reference object loaders.")
453 self.
schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
455 sourceCatSchema = afwTable.SourceCatalog(self.
schema)
460 inputs = butlerQC.get(inputRefs)
461 inputs[
'exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId,
"visit_detector")
463 if self.config.doAstrometry:
464 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
465 for ref
in inputRefs.astromRefCat],
466 refCats=inputs.pop(
'astromRefCat'),
467 config=self.config.astromRefObjLoader, log=self.log)
468 self.astrometry.setRefObjLoader(refObjLoader)
470 if self.config.doPhotoCal:
471 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
472 for ref
in inputRefs.photoRefCat],
473 refCats=inputs.pop(
'photoRefCat'),
474 config=self.config.photoRefObjLoader,
476 self.photoCal.match.setRefObjLoader(photoRefObjLoader)
478 outputs = self.
run(**inputs)
480 if self.config.doWriteMatches
and self.config.doAstrometry:
481 normalizedMatches = afwTable.packMatches(outputs.astromMatches)
482 normalizedMatches.table.setMetadata(outputs.matchMeta)
483 if self.config.doWriteMatchesDenormalized:
484 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
485 outputs.matchesDenormalized = denormMatches
486 outputs.matches = normalizedMatches
487 butlerQC.put(outputs, outputRefs)
490 def run(self, exposure, exposureIdInfo=None, background=None,
492 """Calibrate an exposure.
496 exposure : `lsst.afw.image.ExposureF`
497 Exposure to calibrate.
498 exposureIdInfo : `lsst.obs.baseExposureIdInfo`, optional
499 Exposure ID info. If not provided, returned SourceCatalog IDs will
not
501 background : `lsst.afw.math.BackgroundList`, optional
502 Initial model of background already subtracted
from exposure.
503 icSourceCat : `lsst.afw.image.SourceCatalog`, optional
504 SourceCatalog
from CharacterizeImageTask
from which we can copy some fields.
508 result : `lsst.pipe.base.Struct`
509 Result structure
with the following attributes:
512 Characterized exposure (`lsst.afw.image.ExposureF`).
516 Model of subtracted background (`lsst.afw.math.BackgroundList`).
518 List of source/ref matches
from astrometry solver.
520 Metadata
from astrometry matches.
522 Another reference to ``exposure``
for compatibility.
524 Another reference to ``sourceCat``
for compatibility.
527 if exposureIdInfo
is None:
528 exposureIdInfo = ExposureIdInfo()
530 if background
is None:
531 background = BackgroundList()
532 sourceIdFactory = exposureIdInfo.makeSourceIdFactory()
533 table = SourceTable.make(self.
schema, sourceIdFactory)
536 detRes = self.detection.
run(table=table, exposure=exposure,
538 sourceCat = detRes.sources
539 if detRes.fpSets.background:
540 for bg
in detRes.fpSets.background:
541 background.append(bg)
542 if self.config.doSkySources:
543 skySourceFootprints = self.skySources.
run(mask=exposure.mask, seed=exposureIdInfo.expId)
544 if skySourceFootprints:
545 for foot
in skySourceFootprints:
546 s = sourceCat.addNew()
549 if self.config.doDeblend:
550 self.deblend.
run(exposure=exposure, sources=sourceCat)
551 self.measurement.
run(
554 exposureId=exposureIdInfo.expId
556 if self.config.doApCorr:
557 self.applyApCorr.
run(
559 apCorrMap=exposure.getInfo().getApCorrMap()
561 self.catalogCalculation.
run(sourceCat)
563 self.setPrimaryFlags.
run(sourceCat)
565 if icSourceCat
is not None and \
566 len(self.config.icSourceFieldsToCopy) > 0:
574 if not sourceCat.isContiguous():
575 sourceCat = sourceCat.copy(deep=
True)
581 if self.config.doAstrometry:
583 astromRes = self.astrometry.
run(
587 astromMatches = astromRes.matches
588 matchMeta = astromRes.matchMeta
589 except Exception
as e:
590 if self.config.requireAstrometry:
592 self.log.warning(
"Unable to perform astrometric calibration "
593 "(%s): attempting to proceed", e)
596 if self.config.doPhotoCal:
598 photoRes = self.photoCal.
run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
599 exposure.setPhotoCalib(photoRes.photoCalib)
601 self.log.info(
"Photometric zero-point: %f",
602 photoRes.photoCalib.instFluxToMagnitude(1.0))
603 self.
setMetadata(exposure=exposure, photoRes=photoRes)
604 except Exception
as e:
605 if self.config.requirePhotoCal:
607 self.log.warning(
"Unable to perform photometric calibration "
608 "(%s): attempting to proceed", e)
611 self.postCalibrationMeasurement.
run(
614 exposureId=exposureIdInfo.expId
617 if self.config.doComputeSummaryStats:
618 summary = self.computeSummaryStats.
run(exposure=exposure,
620 background=background)
621 exposure.getInfo().setSummaryStats(summary)
623 frame = getDebugFrame(self._display,
"calibrate")
628 matches=astromMatches,
633 return pipeBase.Struct(
635 astromMatches=astromMatches,
637 outputExposure=exposure,
639 outputBackground=background,
643 """Return a dict of empty catalogs for each catalog dataset produced
646 sourceCat = afwTable.SourceCatalog(self.schema)
648 return {
"src": sourceCat}
651 """Set task and exposure metadata.
653 Logs a warning continues if needed data
is missing.
657 exposure : `lsst.afw.image.ExposureF`
658 Exposure to set metadata on.
659 photoRes : `lsst.pipe.base.Struct`, optional
660 Result of running photoCal task.
665 metadata = exposure.getMetadata()
669 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
670 magZero = photoRes.zp - 2.5*math.log10(exposureTime)
672 self.log.warning(
"Could not set normalized MAGZERO in header: no "
677 metadata.set(
'MAGZERO', magZero)
678 metadata.set(
'MAGZERO_RMS', photoRes.sigma)
679 metadata.set(
'MAGZERO_NOBJ', photoRes.ngood)
680 metadata.set(
'COLORTERM1', 0.0)
681 metadata.set(
'COLORTERM2', 0.0)
682 metadata.set(
'COLORTERM3', 0.0)
683 except Exception
as e:
684 self.log.warning(
"Could not set exposure metadata: %s", e)
687 """Match sources in an icSourceCat and a sourceCat and copy fields.
689 The fields copied are those specified by ``config.icSourceFieldsToCopy``.
694 Catalog from which to copy fields.
696 Catalog to which to copy fields.
699 raise RuntimeError(
"To copy icSource fields you must specify "
700 "icSourceSchema nd icSourceKeys when "
701 "constructing this task")
702 if icSourceCat
is None or sourceCat
is None:
703 raise RuntimeError(
"icSourceCat and sourceCat must both be "
705 if len(self.config.icSourceFieldsToCopy) == 0:
706 self.log.warning(
"copyIcSourceFields doing nothing because "
707 "icSourceFieldsToCopy is empty")
710 mc = afwTable.MatchControl()
711 mc.findOnlyClosest =
False
712 matches = afwTable.matchXy(icSourceCat, sourceCat,
713 self.config.matchRadiusPix, mc)
714 if self.config.doDeblend:
715 deblendKey = sourceCat.schema[
"deblend_nChild"].asKey()
717 matches = [m
for m
in matches
if m[1].get(deblendKey) == 0]
724 for m0, m1, d
in matches:
726 match = bestMatches.get(id0)
727 if match
is None or d <= match[2]:
728 bestMatches[id0] = (m0, m1, d)
729 matches = list(bestMatches.values())
734 numMatches = len(matches)
735 numUniqueSources = len(set(m[1].getId()
for m
in matches))
736 if numUniqueSources != numMatches:
737 self.log.warning(
"%d icSourceCat sources matched only %d sourceCat "
738 "sources", numMatches, numUniqueSources)
740 self.log.info(
"Copying flags from icSourceCat to sourceCat for "
741 "%d sources", numMatches)
745 for icSrc, src, d
in matches:
751 icSrcFootprint = icSrc.getFootprint()
753 icSrc.setFootprint(src.getFootprint())
756 icSrc.setFootprint(icSrcFootprint)
postCalibrationMeasurement
def __init__(self, *config=None)
def __init__(self, butler=None, astromRefObjLoader=None, photoRefObjLoader=None, icSourceSchema=None, initInputs=None, **kwargs)
def setMetadata(self, exposure, photoRes=None)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def getSchemaCatalogs(self)
def copyIcSourceFields(self, icSourceCat, sourceCat)
def run(self, exposure, exposureIdInfo=None, background=None, icSourceCat=None)