22__all__ = [
"CalibrateConfig",
"CalibrateTask"]
27from lsstDebug
import getDebugFrame
30import lsst.pipe.base.connectionTypes
as cT
32from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches
34from lsst.obs.base
import ExposureIdInfo
41 CatalogCalculationTask)
43from lsst.utils.timer
import timeMethod
45from .fakes
import BaseFakeSourcesTask
46from .photoCal
import PhotoCalTask
47from .computeExposureSummaryStats
import ComputeExposureSummaryStatsTask
50class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=(
"instrument",
"visit",
"detector"),
53 icSourceSchema = cT.InitInput(
54 doc=
"Schema produced by characterize image task, used to initialize this task",
56 storageClass=
"SourceCatalog",
59 outputSchema = cT.InitOutput(
60 doc=
"Schema after CalibrateTask has been initialized",
62 storageClass=
"SourceCatalog",
66 doc=
"Input image to calibrate",
68 storageClass=
"ExposureF",
69 dimensions=(
"instrument",
"visit",
"detector"),
72 background = cT.Input(
73 doc=
"Backgrounds determined by characterize task",
74 name=
"icExpBackground",
75 storageClass=
"Background",
76 dimensions=(
"instrument",
"visit",
"detector"),
79 icSourceCat = cT.Input(
80 doc=
"Source catalog created by characterize task",
82 storageClass=
"SourceCatalog",
83 dimensions=(
"instrument",
"visit",
"detector"),
86 astromRefCat = cT.PrerequisiteInput(
87 doc=
"Reference catalog to use for astrometry",
88 name=
"gaia_dr2_20200414",
89 storageClass=
"SimpleCatalog",
90 dimensions=(
"skypix",),
95 photoRefCat = cT.PrerequisiteInput(
96 doc=
"Reference catalog to use for photometric calibration",
97 name=
"ps1_pv3_3pi_20170110",
98 storageClass=
"SimpleCatalog",
99 dimensions=(
"skypix",),
104 outputExposure = cT.Output(
105 doc=
"Exposure after running calibration task",
107 storageClass=
"ExposureF",
108 dimensions=(
"instrument",
"visit",
"detector"),
111 outputCat = cT.Output(
112 doc=
"Source catalog produced in calibrate task",
114 storageClass=
"SourceCatalog",
115 dimensions=(
"instrument",
"visit",
"detector"),
118 outputBackground = cT.Output(
119 doc=
"Background models estimated in calibration task",
120 name=
"calexpBackground",
121 storageClass=
"Background",
122 dimensions=(
"instrument",
"visit",
"detector"),
126 doc=
"Source/refObj matches from the astrometry solver",
128 storageClass=
"Catalog",
129 dimensions=(
"instrument",
"visit",
"detector"),
132 matchesDenormalized = cT.Output(
133 doc=
"Denormalized matches from astrometry solver",
135 storageClass=
"Catalog",
136 dimensions=(
"instrument",
"visit",
"detector"),
142 if config.doAstrometry
is False:
143 self.prerequisiteInputs.remove(
"astromRefCat")
144 if config.doPhotoCal
is False:
145 self.prerequisiteInputs.remove(
"photoRefCat")
147 if config.doWriteMatches
is False or config.doAstrometry
is False:
148 self.outputs.remove(
"matches")
149 if config.doWriteMatchesDenormalized
is False or config.doAstrometry
is False:
150 self.outputs.remove(
"matchesDenormalized")
153class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections):
154 """Config for CalibrateTask."""
156 doWrite = pexConfig.Field(
159 doc=
"Save calibration results?",
161 doWriteHeavyFootprintsInSources = pexConfig.Field(
164 doc=
"Include HeavyFootprint data in source table? If false then heavy "
165 "footprints are saved as normal footprints, which saves some space"
167 doWriteMatches = pexConfig.Field(
170 doc=
"Write reference matches (ignored if doWrite or doAstrometry false)?",
172 doWriteMatchesDenormalized = pexConfig.Field(
175 doc=(
"Write reference matches in denormalized format? "
176 "This format uses more disk space, but is more convenient to "
177 "read. Ignored if doWriteMatches=False or doWrite=False."),
179 doAstrometry = pexConfig.Field(
182 doc=
"Perform astrometric calibration?",
184 astromRefObjLoader = pexConfig.ConfigField(
185 dtype=LoadReferenceObjectsConfig,
186 doc=
"reference object loader for astrometric calibration",
188 photoRefObjLoader = pexConfig.ConfigField(
189 dtype=LoadReferenceObjectsConfig,
190 doc=
"reference object loader for photometric calibration",
192 astrometry = pexConfig.ConfigurableField(
193 target=AstrometryTask,
194 doc=
"Perform astrometric calibration to refine the WCS",
196 requireAstrometry = pexConfig.Field(
199 doc=(
"Raise an exception if astrometry fails? Ignored if doAstrometry "
202 doPhotoCal = pexConfig.Field(
205 doc=
"Perform phometric calibration?",
207 requirePhotoCal = pexConfig.Field(
210 doc=(
"Raise an exception if photoCal fails? Ignored if doPhotoCal "
213 photoCal = pexConfig.ConfigurableField(
215 doc=
"Perform photometric calibration",
217 icSourceFieldsToCopy = pexConfig.ListField(
219 default=(
"calib_psf_candidate",
"calib_psf_used",
"calib_psf_reserved"),
220 doc=(
"Fields to copy from the icSource catalog to the output catalog "
221 "for matching sources Any missing fields will trigger a "
222 "RuntimeError exception. Ignored if icSourceCat is not provided.")
224 matchRadiusPix = pexConfig.Field(
227 doc=(
"Match radius for matching icSourceCat objects to sourceCat "
230 checkUnitsParseStrict = pexConfig.Field(
231 doc=(
"Strictness of Astropy unit compatibility check, can be 'raise', "
232 "'warn' or 'silent'"),
236 detection = pexConfig.ConfigurableField(
237 target=SourceDetectionTask,
240 doDeblend = pexConfig.Field(
243 doc=
"Run deblender input exposure"
245 deblend = pexConfig.ConfigurableField(
246 target=SourceDeblendTask,
247 doc=
"Split blended sources into their components"
249 doSkySources = pexConfig.Field(
252 doc=
"Generate sky sources?",
254 skySources = pexConfig.ConfigurableField(
255 target=SkyObjectsTask,
256 doc=
"Generate sky sources",
258 measurement = pexConfig.ConfigurableField(
259 target=SingleFrameMeasurementTask,
260 doc=
"Measure sources"
262 postCalibrationMeasurement = pexConfig.ConfigurableField(
263 target=SingleFrameMeasurementTask,
264 doc=
"Second round of measurement for plugins that need to be run after photocal"
266 setPrimaryFlags = pexConfig.ConfigurableField(
267 target=SetPrimaryFlagsTask,
268 doc=(
"Set flags for primary source classification in single frame "
269 "processing. True if sources are not sky sources and not a parent.")
271 doApCorr = pexConfig.Field(
274 doc=
"Run subtask to apply aperture correction"
276 applyApCorr = pexConfig.ConfigurableField(
277 target=ApplyApCorrTask,
278 doc=
"Subtask to apply aperture corrections"
283 catalogCalculation = pexConfig.ConfigurableField(
284 target=CatalogCalculationTask,
285 doc=
"Subtask to run catalogCalculation plugins on catalog"
287 doInsertFakes = pexConfig.Field(
290 doc=
"Run fake sources injection task",
291 deprecated=(
"doInsertFakes is no longer supported. This config will be removed after v24. "
292 "Please use ProcessCcdWithFakesTask instead.")
294 insertFakes = pexConfig.ConfigurableField(
295 target=BaseFakeSourcesTask,
296 doc=
"Injection of fake sources for testing purposes (must be "
298 deprecated=(
"insertFakes is no longer supported. This config will be removed after v24. "
299 "Please use ProcessCcdWithFakesTask instead.")
301 doComputeSummaryStats = pexConfig.Field(
304 doc=
"Run subtask to measure exposure summary statistics?"
306 computeSummaryStats = pexConfig.ConfigurableField(
307 target=ComputeExposureSummaryStatsTask,
308 doc=
"Subtask to run computeSummaryStats on exposure"
310 doWriteExposure = pexConfig.Field(
313 doc=
"Write the calexp? If fakes have been added then we do not want to write out the calexp as a "
314 "normal calexp but as a fakes_calexp."
319 self.
detection.doTempLocalBackground =
False
320 self.
deblend.maxFootprintSize = 2000
327 self.
photoCal.photoCatName = self.connections.photoRefCat
331 """Calibrate an exposure: measure sources and perform astrometric and
332 photometric calibration.
334 Given an exposure with a good PSF model
and aperture correction map(e.g.
as
336 perform the following operations:
337 - Run detection
and measurement
338 - Run astrometry subtask to fit an improved WCS
339 - Run photoCal subtask to fit the exposure
's photometric zero-point
344 Compatibility parameter. Should always be `
None`.
345 astromRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
346 Unused
in gen3: must be `
None`.
347 photoRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
348 Unused
in gen3: must be `
None`.
350 Schema
for the icSource catalog.
351 initInputs : `dict`, optional
352 Dictionary that can contain a key ``icSourceSchema`` containing the
353 input schema. If present will override the value of ``icSourceSchema``.
358 Raised
if any of the following occur:
359 - isSourceCat
is missing fields specified
in icSourceFieldsToCopy.
360 - PipelineTask form of this task
is initialized
with reference object loaders.
364 Quantities set
in exposure Metadata:
367 MAGZERO
's RMS == sigma reported by photoCal task
369 Number of stars used == ngood reported by photoCal task
378 CalibrateTask has a debug dictionary containing one key:
381 frame (an int; <= 0 to not display)
in which to display the exposure,
382 sources
and matches. See
@ref lsst.meas.astrom.displayAstrometry
for
383 the meaning of the various symbols.
386 ConfigClass = CalibrateConfig
387 _DefaultName = "calibrate"
389 def __init__(self, butler=None, astromRefObjLoader=None,
390 photoRefObjLoader=None, icSourceSchema=None,
391 initInputs=None, **kwargs):
394 if butler
is not None:
395 warnings.warn(
"The 'butler' parameter is no longer used and can be safely removed.",
396 category=FutureWarning, stacklevel=2)
399 if initInputs
is not None:
400 icSourceSchema = initInputs[
'icSourceSchema'].schema
402 if icSourceSchema
is not None:
405 minimumSchema = afwTable.SourceTable.makeMinimalSchema()
406 self.
schemaMapper.addMinimalSchema(minimumSchema,
False)
414 afwTable.Field[
"Flag"](
"calib_detected",
415 "Source was detected as an icSource"))
416 missingFieldNames = []
417 for fieldName
in self.config.icSourceFieldsToCopy:
419 schemaItem = icSourceSchema.find(fieldName)
421 missingFieldNames.append(fieldName)
426 if missingFieldNames:
427 raise RuntimeError(
"isSourceCat is missing fields {} "
428 "specified in icSourceFieldsToCopy"
429 .format(missingFieldNames))
436 self.
schema = afwTable.SourceTable.makeMinimalSchema()
437 self.makeSubtask(
'detection', schema=self.
schema)
441 if self.config.doDeblend:
442 self.makeSubtask(
"deblend", schema=self.
schema)
443 if self.config.doSkySources:
444 self.makeSubtask(
"skySources")
446 self.makeSubtask(
'measurement', schema=self.
schema,
448 self.makeSubtask(
'postCalibrationMeasurement', schema=self.
schema,
450 self.makeSubtask(
"setPrimaryFlags", schema=self.
schema, isSingleFrame=
True)
451 if self.config.doApCorr:
452 self.makeSubtask(
'applyApCorr', schema=self.
schema)
453 self.makeSubtask(
'catalogCalculation', schema=self.
schema)
455 if self.config.doAstrometry:
456 self.makeSubtask(
"astrometry", refObjLoader=astromRefObjLoader,
458 if self.config.doPhotoCal:
459 self.makeSubtask(
"photoCal", refObjLoader=photoRefObjLoader,
461 if self.config.doComputeSummaryStats:
462 self.makeSubtask(
'computeSummaryStats')
464 if initInputs
is not None and (astromRefObjLoader
is not None or photoRefObjLoader
is not None):
465 raise RuntimeError(
"PipelineTask form of this task should not be initialized with "
466 "reference object loaders.")
471 self.
schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
473 sourceCatSchema = afwTable.SourceCatalog(self.
schema)
478 inputs = butlerQC.get(inputRefs)
479 inputs[
'exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId,
"visit_detector")
481 if self.config.doAstrometry:
482 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
483 for ref
in inputRefs.astromRefCat],
484 refCats=inputs.pop(
'astromRefCat'),
485 name=self.config.connections.astromRefCat,
486 config=self.config.astromRefObjLoader, log=self.log)
487 self.astrometry.setRefObjLoader(refObjLoader)
489 if self.config.doPhotoCal:
490 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
491 for ref
in inputRefs.photoRefCat],
492 refCats=inputs.pop(
'photoRefCat'),
493 name=self.config.connections.photoRefCat,
494 config=self.config.photoRefObjLoader,
496 self.photoCal.match.setRefObjLoader(photoRefObjLoader)
498 outputs = self.
run(**inputs)
500 if self.config.doWriteMatches
and self.config.doAstrometry:
501 normalizedMatches = afwTable.packMatches(outputs.astromMatches)
502 normalizedMatches.table.setMetadata(outputs.matchMeta)
503 if self.config.doWriteMatchesDenormalized:
504 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
505 outputs.matchesDenormalized = denormMatches
506 outputs.matches = normalizedMatches
507 butlerQC.put(outputs, outputRefs)
510 def run(self, exposure, exposureIdInfo=None, background=None,
512 """Calibrate an exposure.
516 exposure : `lsst.afw.image.ExposureF`
517 Exposure to calibrate.
518 exposureIdInfo : `lsst.obs.baseExposureIdInfo`, optional
519 Exposure ID info. If not provided, returned SourceCatalog IDs will
not
521 background : `lsst.afw.math.BackgroundList`, optional
522 Initial model of background already subtracted
from exposure.
523 icSourceCat : `lsst.afw.image.SourceCatalog`, optional
524 SourceCatalog
from CharacterizeImageTask
from which we can copy some fields.
528 result : `lsst.pipe.base.Struct`
529 Results
as a struct
with attributes:
532 Characterized exposure (`lsst.afw.image.ExposureF`).
536 Model of subtracted background (`lsst.afw.math.BackgroundList`).
538 List of source/ref matches
from astrometry solver.
540 Metadata
from astrometry matches.
542 Another reference to ``exposure``
for compatibility.
544 Another reference to ``sourceCat``
for compatibility.
547 if exposureIdInfo
is None:
548 exposureIdInfo = ExposureIdInfo()
550 if background
is None:
551 background = BackgroundList()
552 sourceIdFactory = exposureIdInfo.makeSourceIdFactory()
553 table = SourceTable.make(self.
schema, sourceIdFactory)
556 detRes = self.detection.run(table=table, exposure=exposure,
558 sourceCat = detRes.sources
559 if detRes.fpSets.background:
560 for bg
in detRes.fpSets.background:
561 background.append(bg)
562 if self.config.doSkySources:
563 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId)
564 if skySourceFootprints:
565 for foot
in skySourceFootprints:
566 s = sourceCat.addNew()
569 if self.config.doDeblend:
570 self.deblend.run(exposure=exposure, sources=sourceCat)
571 self.measurement.run(
574 exposureId=exposureIdInfo.expId
576 if self.config.doApCorr:
577 self.applyApCorr.run(
579 apCorrMap=exposure.getInfo().getApCorrMap()
581 self.catalogCalculation.run(sourceCat)
583 self.setPrimaryFlags.run(sourceCat)
585 if icSourceCat
is not None and \
586 len(self.config.icSourceFieldsToCopy) > 0:
594 if not sourceCat.isContiguous():
595 sourceCat = sourceCat.copy(deep=
True)
601 if self.config.doAstrometry:
603 astromRes = self.astrometry.run(
607 astromMatches = astromRes.matches
608 matchMeta = astromRes.matchMeta
609 except Exception
as e:
610 if self.config.requireAstrometry:
612 self.log.warning(
"Unable to perform astrometric calibration "
613 "(%s): attempting to proceed", e)
616 if self.config.doPhotoCal:
618 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
619 exposure.setPhotoCalib(photoRes.photoCalib)
621 self.log.info(
"Photometric zero-point: %f",
622 photoRes.photoCalib.instFluxToMagnitude(1.0))
623 self.
setMetadata(exposure=exposure, photoRes=photoRes)
624 except Exception
as e:
625 if self.config.requirePhotoCal:
627 self.log.warning(
"Unable to perform photometric calibration "
628 "(%s): attempting to proceed", e)
631 self.postCalibrationMeasurement.run(
634 exposureId=exposureIdInfo.expId
637 if self.config.doComputeSummaryStats:
638 summary = self.computeSummaryStats.run(exposure=exposure,
640 background=background)
641 exposure.getInfo().setSummaryStats(summary)
643 frame = getDebugFrame(self._display,
"calibrate")
648 matches=astromMatches,
653 return pipeBase.Struct(
655 astromMatches=astromMatches,
657 outputExposure=exposure,
659 outputBackground=background,
663 """Return a dict of empty catalogs for each catalog dataset produced
666 sourceCat = afwTable.SourceCatalog(self.schema)
668 return {
"src": sourceCat}
671 """Set task and exposure metadata.
673 Logs a warning continues if needed data
is missing.
677 exposure : `lsst.afw.image.ExposureF`
678 Exposure to set metadata on.
679 photoRes : `lsst.pipe.base.Struct`, optional
680 Result of running photoCal task.
685 metadata = exposure.getMetadata()
689 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
690 magZero = photoRes.zp - 2.5*math.log10(exposureTime)
692 self.log.warning(
"Could not set normalized MAGZERO in header: no "
697 metadata.set(
'MAGZERO', magZero)
698 metadata.set(
'MAGZERO_RMS', photoRes.sigma)
699 metadata.set(
'MAGZERO_NOBJ', photoRes.ngood)
700 metadata.set(
'COLORTERM1', 0.0)
701 metadata.set(
'COLORTERM2', 0.0)
702 metadata.set(
'COLORTERM3', 0.0)
703 except Exception
as e:
704 self.log.warning(
"Could not set exposure metadata: %s", e)
707 """Match sources in an icSourceCat and a sourceCat and copy fields.
709 The fields copied are those specified by ``config.icSourceFieldsToCopy``.
714 Catalog from which to copy fields.
716 Catalog to which to copy fields.
721 Raised
if any of the following occur:
722 - icSourceSchema
and icSourceKeys are
not specified.
723 - icSourceCat
and sourceCat are
not specified.
724 - icSourceFieldsToCopy
is empty.
727 raise RuntimeError(
"To copy icSource fields you must specify "
728 "icSourceSchema and icSourceKeys when "
729 "constructing this task")
730 if icSourceCat
is None or sourceCat
is None:
731 raise RuntimeError(
"icSourceCat and sourceCat must both be "
733 if len(self.config.icSourceFieldsToCopy) == 0:
734 self.log.warning(
"copyIcSourceFields doing nothing because "
735 "icSourceFieldsToCopy is empty")
738 mc = afwTable.MatchControl()
739 mc.findOnlyClosest =
False
740 matches = afwTable.matchXy(icSourceCat, sourceCat,
741 self.config.matchRadiusPix, mc)
742 if self.config.doDeblend:
743 deblendKey = sourceCat.schema[
"deblend_nChild"].asKey()
745 matches = [m
for m
in matches
if m[1].get(deblendKey) == 0]
752 for m0, m1, d
in matches:
754 match = bestMatches.get(id0)
755 if match
is None or d <= match[2]:
756 bestMatches[id0] = (m0, m1, d)
757 matches = list(bestMatches.values())
762 numMatches = len(matches)
763 numUniqueSources = len(set(m[1].getId()
for m
in matches))
764 if numUniqueSources != numMatches:
765 self.log.warning(
"%d icSourceCat sources matched only %d sourceCat "
766 "sources", numMatches, numUniqueSources)
768 self.log.info(
"Copying flags from icSourceCat to sourceCat for "
769 "%d sources", numMatches)
773 for icSrc, src, d
in matches:
779 icSrcFootprint = icSrc.getFootprint()
781 icSrc.setFootprint(src.getFootprint())
784 icSrc.setFootprint(icSrcFootprint)
postCalibrationMeasurement
def __init__(self, *config=None)
def __init__(self, butler=None, astromRefObjLoader=None, photoRefObjLoader=None, icSourceSchema=None, initInputs=None, **kwargs)
def setMetadata(self, exposure, photoRes=None)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def getSchemaCatalogs(self)
def copyIcSourceFields(self, icSourceCat, sourceCat)
def run(self, exposure, exposureIdInfo=None, background=None, icSourceCat=None)