22__all__ = [
"CalibrateConfig",
"CalibrateTask"]
28from lsstDebug
import getDebugFrame
31import lsst.pipe.base.connectionTypes
as cT
33from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches
35from lsst.obs.base
import ExposureIdInfo
42 CatalogCalculationTask)
44from lsst.utils.timer
import timeMethod
46from .fakes
import BaseFakeSourcesTask
47from .photoCal
import PhotoCalTask
48from .computeExposureSummaryStats
import ComputeExposureSummaryStatsTask
51class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=(
"instrument",
"visit",
"detector"),
54 icSourceSchema = cT.InitInput(
55 doc=
"Schema produced by characterize image task, used to initialize this task",
57 storageClass=
"SourceCatalog",
60 outputSchema = cT.InitOutput(
61 doc=
"Schema after CalibrateTask has been initialized",
63 storageClass=
"SourceCatalog",
67 doc=
"Input image to calibrate",
69 storageClass=
"ExposureF",
70 dimensions=(
"instrument",
"visit",
"detector"),
73 background = cT.Input(
74 doc=
"Backgrounds determined by characterize task",
75 name=
"icExpBackground",
76 storageClass=
"Background",
77 dimensions=(
"instrument",
"visit",
"detector"),
80 icSourceCat = cT.Input(
81 doc=
"Source catalog created by characterize task",
83 storageClass=
"SourceCatalog",
84 dimensions=(
"instrument",
"visit",
"detector"),
87 astromRefCat = cT.PrerequisiteInput(
88 doc=
"Reference catalog to use for astrometry",
89 name=
"gaia_dr2_20200414",
90 storageClass=
"SimpleCatalog",
91 dimensions=(
"skypix",),
96 photoRefCat = cT.PrerequisiteInput(
97 doc=
"Reference catalog to use for photometric calibration",
98 name=
"ps1_pv3_3pi_20170110",
99 storageClass=
"SimpleCatalog",
100 dimensions=(
"skypix",),
105 outputExposure = cT.Output(
106 doc=
"Exposure after running calibration task",
108 storageClass=
"ExposureF",
109 dimensions=(
"instrument",
"visit",
"detector"),
112 outputCat = cT.Output(
113 doc=
"Source catalog produced in calibrate task",
115 storageClass=
"SourceCatalog",
116 dimensions=(
"instrument",
"visit",
"detector"),
119 outputBackground = cT.Output(
120 doc=
"Background models estimated in calibration task",
121 name=
"calexpBackground",
122 storageClass=
"Background",
123 dimensions=(
"instrument",
"visit",
"detector"),
127 doc=
"Source/refObj matches from the astrometry solver",
129 storageClass=
"Catalog",
130 dimensions=(
"instrument",
"visit",
"detector"),
133 matchesDenormalized = cT.Output(
134 doc=
"Denormalized matches from astrometry solver",
136 storageClass=
"Catalog",
137 dimensions=(
"instrument",
"visit",
"detector"),
143 if config.doAstrometry
is False:
144 self.prerequisiteInputs.remove(
"astromRefCat")
145 if config.doPhotoCal
is False:
146 self.prerequisiteInputs.remove(
"photoRefCat")
148 if config.doWriteMatches
is False or config.doAstrometry
is False:
149 self.outputs.remove(
"matches")
150 if config.doWriteMatchesDenormalized
is False or config.doAstrometry
is False:
151 self.outputs.remove(
"matchesDenormalized")
154class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections):
155 """Config for CalibrateTask."""
157 doWrite = pexConfig.Field(
160 doc=
"Save calibration results?",
162 doWriteHeavyFootprintsInSources = pexConfig.Field(
165 doc=
"Include HeavyFootprint data in source table? If false then heavy "
166 "footprints are saved as normal footprints, which saves some space"
168 doWriteMatches = pexConfig.Field(
171 doc=
"Write reference matches (ignored if doWrite or doAstrometry false)?",
173 doWriteMatchesDenormalized = pexConfig.Field(
176 doc=(
"Write reference matches in denormalized format? "
177 "This format uses more disk space, but is more convenient to "
178 "read. Ignored if doWriteMatches=False or doWrite=False."),
180 doAstrometry = pexConfig.Field(
183 doc=
"Perform astrometric calibration?",
185 astromRefObjLoader = pexConfig.ConfigField(
186 dtype=LoadReferenceObjectsConfig,
187 doc=
"reference object loader for astrometric calibration",
189 photoRefObjLoader = pexConfig.ConfigField(
190 dtype=LoadReferenceObjectsConfig,
191 doc=
"reference object loader for photometric calibration",
193 astrometry = pexConfig.ConfigurableField(
194 target=AstrometryTask,
195 doc=
"Perform astrometric calibration to refine the WCS",
197 requireAstrometry = pexConfig.Field(
200 doc=(
"Raise an exception if astrometry fails? Ignored if doAstrometry "
203 doPhotoCal = pexConfig.Field(
206 doc=
"Perform phometric calibration?",
208 requirePhotoCal = pexConfig.Field(
211 doc=(
"Raise an exception if photoCal fails? Ignored if doPhotoCal "
214 photoCal = pexConfig.ConfigurableField(
216 doc=
"Perform photometric calibration",
218 icSourceFieldsToCopy = pexConfig.ListField(
220 default=(
"calib_psf_candidate",
"calib_psf_used",
"calib_psf_reserved"),
221 doc=(
"Fields to copy from the icSource catalog to the output catalog "
222 "for matching sources Any missing fields will trigger a "
223 "RuntimeError exception. Ignored if icSourceCat is not provided.")
225 matchRadiusPix = pexConfig.Field(
228 doc=(
"Match radius for matching icSourceCat objects to sourceCat "
231 checkUnitsParseStrict = pexConfig.Field(
232 doc=(
"Strictness of Astropy unit compatibility check, can be 'raise', "
233 "'warn' or 'silent'"),
237 detection = pexConfig.ConfigurableField(
238 target=SourceDetectionTask,
241 doDeblend = pexConfig.Field(
244 doc=
"Run deblender input exposure"
246 deblend = pexConfig.ConfigurableField(
247 target=SourceDeblendTask,
248 doc=
"Split blended sources into their components"
250 doSkySources = pexConfig.Field(
253 doc=
"Generate sky sources?",
255 skySources = pexConfig.ConfigurableField(
256 target=SkyObjectsTask,
257 doc=
"Generate sky sources",
259 measurement = pexConfig.ConfigurableField(
260 target=SingleFrameMeasurementTask,
261 doc=
"Measure sources"
263 postCalibrationMeasurement = pexConfig.ConfigurableField(
264 target=SingleFrameMeasurementTask,
265 doc=
"Second round of measurement for plugins that need to be run after photocal"
267 setPrimaryFlags = pexConfig.ConfigurableField(
268 target=SetPrimaryFlagsTask,
269 doc=(
"Set flags for primary source classification in single frame "
270 "processing. True if sources are not sky sources and not a parent.")
272 doApCorr = pexConfig.Field(
275 doc=
"Run subtask to apply aperture correction"
277 applyApCorr = pexConfig.ConfigurableField(
278 target=ApplyApCorrTask,
279 doc=
"Subtask to apply aperture corrections"
284 catalogCalculation = pexConfig.ConfigurableField(
285 target=CatalogCalculationTask,
286 doc=
"Subtask to run catalogCalculation plugins on catalog"
288 doInsertFakes = pexConfig.Field(
291 doc=
"Run fake sources injection task",
292 deprecated=(
"doInsertFakes is no longer supported. This config will be removed after v24. "
293 "Please use ProcessCcdWithFakesTask instead.")
295 insertFakes = pexConfig.ConfigurableField(
296 target=BaseFakeSourcesTask,
297 doc=
"Injection of fake sources for testing purposes (must be "
299 deprecated=(
"insertFakes is no longer supported. This config will be removed after v24. "
300 "Please use ProcessCcdWithFakesTask instead.")
302 doComputeSummaryStats = pexConfig.Field(
305 doc=
"Run subtask to measure exposure summary statistics?"
307 computeSummaryStats = pexConfig.ConfigurableField(
308 target=ComputeExposureSummaryStatsTask,
309 doc=
"Subtask to run computeSummaryStats on exposure"
311 doWriteExposure = pexConfig.Field(
314 doc=
"Write the calexp? If fakes have been added then we do not want to write out the calexp as a "
315 "normal calexp but as a fakes_calexp."
320 self.
detection.doTempLocalBackground =
False
321 self.
deblend.maxFootprintSize = 2000
328 self.
photoCal.photoCatName = self.connections.photoRefCat
332 """Calibrate an exposure: measure sources and perform astrometric and
333 photometric calibration.
335 Given an exposure with a good PSF model
and aperture correction map(e.g.
as
337 perform the following operations:
338 - Run detection
and measurement
339 - Run astrometry subtask to fit an improved WCS
340 - Run photoCal subtask to fit the exposure
's photometric zero-point
345 Compatibility parameter. Should always be `
None`.
346 astromRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
347 Unused
in gen3: must be `
None`.
348 photoRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
349 Unused
in gen3: must be `
None`.
351 Schema
for the icSource catalog.
352 initInputs : `dict`, optional
353 Dictionary that can contain a key ``icSourceSchema`` containing the
354 input schema. If present will override the value of ``icSourceSchema``.
359 Raised
if any of the following occur:
360 - isSourceCat
is missing fields specified
in icSourceFieldsToCopy.
361 - PipelineTask form of this task
is initialized
with reference object
366 Quantities set
in exposure Metadata:
369 MAGZERO
's RMS == sigma reported by photoCal task
371 Number of stars used == ngood reported by photoCal task
380 CalibrateTask has a debug dictionary containing one key:
383 frame (an int; <= 0 to not display)
in which to display the exposure,
384 sources
and matches. See
@ref lsst.meas.astrom.displayAstrometry
for
385 the meaning of the various symbols.
388 ConfigClass = CalibrateConfig
389 _DefaultName = "calibrate"
391 def __init__(self, butler=None, astromRefObjLoader=None,
392 photoRefObjLoader=None, icSourceSchema=None,
393 initInputs=None, **kwargs):
396 if butler
is not None:
397 warnings.warn(
"The 'butler' parameter is no longer used and can be safely removed.",
398 category=FutureWarning, stacklevel=2)
401 if initInputs
is not None:
402 icSourceSchema = initInputs[
'icSourceSchema'].schema
404 if icSourceSchema
is not None:
407 minimumSchema = afwTable.SourceTable.makeMinimalSchema()
408 self.
schemaMapper.addMinimalSchema(minimumSchema,
False)
416 afwTable.Field[
"Flag"](
"calib_detected",
417 "Source was detected as an icSource"))
418 missingFieldNames = []
419 for fieldName
in self.config.icSourceFieldsToCopy:
421 schemaItem = icSourceSchema.find(fieldName)
423 missingFieldNames.append(fieldName)
428 if missingFieldNames:
429 raise RuntimeError(
"isSourceCat is missing fields {} "
430 "specified in icSourceFieldsToCopy"
431 .format(missingFieldNames))
438 self.
schema = afwTable.SourceTable.makeMinimalSchema()
439 self.makeSubtask(
'detection', schema=self.
schema)
443 if self.config.doDeblend:
444 self.makeSubtask(
"deblend", schema=self.
schema)
445 if self.config.doSkySources:
446 self.makeSubtask(
"skySources")
448 self.makeSubtask(
'measurement', schema=self.
schema,
450 self.makeSubtask(
'postCalibrationMeasurement', schema=self.
schema,
452 self.makeSubtask(
"setPrimaryFlags", schema=self.
schema, isSingleFrame=
True)
453 if self.config.doApCorr:
454 self.makeSubtask(
'applyApCorr', schema=self.
schema)
455 self.makeSubtask(
'catalogCalculation', schema=self.
schema)
457 if self.config.doAstrometry:
458 self.makeSubtask(
"astrometry", refObjLoader=astromRefObjLoader,
460 if self.config.doPhotoCal:
461 self.makeSubtask(
"photoCal", refObjLoader=photoRefObjLoader,
463 if self.config.doComputeSummaryStats:
464 self.makeSubtask(
'computeSummaryStats')
466 if initInputs
is not None and (astromRefObjLoader
is not None or photoRefObjLoader
is not None):
467 raise RuntimeError(
"PipelineTask form of this task should not be initialized with "
468 "reference object loaders.")
473 self.
schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
475 sourceCatSchema = afwTable.SourceCatalog(self.
schema)
480 inputs = butlerQC.get(inputRefs)
481 inputs[
'exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId,
"visit_detector")
483 if self.config.doAstrometry:
484 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
485 for ref
in inputRefs.astromRefCat],
486 refCats=inputs.pop(
'astromRefCat'),
487 name=self.config.connections.astromRefCat,
488 config=self.config.astromRefObjLoader, log=self.log)
489 self.astrometry.setRefObjLoader(refObjLoader)
491 if self.config.doPhotoCal:
492 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
493 for ref
in inputRefs.photoRefCat],
494 refCats=inputs.pop(
'photoRefCat'),
495 name=self.config.connections.photoRefCat,
496 config=self.config.photoRefObjLoader,
498 self.photoCal.match.setRefObjLoader(photoRefObjLoader)
500 outputs = self.
run(**inputs)
502 if self.config.doWriteMatches
and self.config.doAstrometry:
503 if outputs.astromMatches
is not None:
504 normalizedMatches = afwTable.packMatches(outputs.astromMatches)
505 normalizedMatches.table.setMetadata(outputs.matchMeta)
506 if self.config.doWriteMatchesDenormalized:
507 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
508 outputs.matchesDenormalized = denormMatches
509 outputs.matches = normalizedMatches
511 del outputRefs.matches
512 if self.config.doWriteMatchesDenormalized:
513 del outputRefs.matchesDenormalized
514 butlerQC.put(outputs, outputRefs)
517 def run(self, exposure, exposureIdInfo=None, background=None,
519 """Calibrate an exposure.
523 exposure : `lsst.afw.image.ExposureF`
524 Exposure to calibrate.
525 exposureIdInfo : `lsst.obs.baseExposureIdInfo`, optional
526 Exposure ID info. If not provided, returned SourceCatalog IDs will
527 not be globally unique.
528 background : `lsst.afw.math.BackgroundList`, optional
529 Initial model of background already subtracted
from exposure.
530 icSourceCat : `lsst.afw.image.SourceCatalog`, optional
531 SourceCatalog
from CharacterizeImageTask
from which we can copy
536 result : `lsst.pipe.base.Struct`
537 Results
as a struct
with attributes:
540 Characterized exposure (`lsst.afw.image.ExposureF`).
544 Model of subtracted background (`lsst.afw.math.BackgroundList`).
546 List of source/ref matches
from astrometry solver.
548 Metadata
from astrometry matches.
550 Another reference to ``exposure``
for compatibility.
552 Another reference to ``sourceCat``
for compatibility.
555 if exposureIdInfo
is None:
556 exposureIdInfo = ExposureIdInfo()
558 if background
is None:
559 background = BackgroundList()
560 sourceIdFactory = exposureIdInfo.makeSourceIdFactory()
561 table = SourceTable.make(self.
schema, sourceIdFactory)
564 detRes = self.detection.run(table=table, exposure=exposure,
566 sourceCat = detRes.sources
567 if detRes.fpSets.background:
568 for bg
in detRes.fpSets.background:
569 background.append(bg)
570 if self.config.doSkySources:
571 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId)
572 if skySourceFootprints:
573 for foot
in skySourceFootprints:
574 s = sourceCat.addNew()
577 if self.config.doDeblend:
578 self.deblend.run(exposure=exposure, sources=sourceCat)
579 self.measurement.run(
582 exposureId=exposureIdInfo.expId
584 if self.config.doApCorr:
585 self.applyApCorr.run(
587 apCorrMap=exposure.getInfo().getApCorrMap()
589 self.catalogCalculation.run(sourceCat)
591 self.setPrimaryFlags.run(sourceCat)
593 if icSourceCat
is not None and \
594 len(self.config.icSourceFieldsToCopy) > 0:
602 if not sourceCat.isContiguous():
603 sourceCat = sourceCat.copy(deep=
True)
609 if self.config.doAstrometry:
610 astromRes = self.astrometry.run(
614 astromMatches = astromRes.matches
615 matchMeta = astromRes.matchMeta
616 if exposure.getWcs()
is None:
617 if self.config.requireAstrometry:
618 raise RuntimeError(f
"WCS fit failed for {exposureIdInfo.expId} and requireAstrometry "
621 self.log.warning(
"Unable to perform astrometric calibration for %r but "
622 "requireAstrometry is False: attempting to proceed...",
623 exposureIdInfo.expId)
626 if self.config.doPhotoCal:
627 if np.all(np.isnan(sourceCat[
"coord_ra"]))
or np.all(np.isnan(sourceCat[
"coord_dec"])):
628 if self.config.requirePhotoCal:
629 raise RuntimeError(f
"Astrometry failed for {exposureIdInfo.expId}, so cannot do "
630 "photoCal, but requirePhotoCal is True.")
631 self.log.warning(
"Astrometry failed for %r, so cannot do photoCal. requirePhotoCal "
632 "is False, so skipping photometric calibration and setting photoCalib "
633 "to None. Attempting to proceed...", exposureIdInfo.expId)
634 exposure.setPhotoCalib(
None)
638 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
639 exposure.setPhotoCalib(photoRes.photoCalib)
642 self.log.info(
"Photometric zero-point: %f",
643 photoRes.photoCalib.instFluxToMagnitude(1.0))
644 self.
setMetadata(exposure=exposure, photoRes=photoRes)
645 except Exception
as e:
646 if self.config.requirePhotoCal:
648 self.log.warning(
"Unable to perform photometric calibration "
649 "(%s): attempting to proceed", e)
652 self.postCalibrationMeasurement.run(
655 exposureId=exposureIdInfo.expId
658 if self.config.doComputeSummaryStats:
659 summary = self.computeSummaryStats.run(exposure=exposure,
661 background=background)
662 exposure.getInfo().setSummaryStats(summary)
664 frame = getDebugFrame(self._display,
"calibrate")
669 matches=astromMatches,
674 return pipeBase.Struct(
676 astromMatches=astromMatches,
678 outputExposure=exposure,
680 outputBackground=background,
684 """Return a dict of empty catalogs for each catalog dataset produced
687 sourceCat = afwTable.SourceCatalog(self.schema)
689 return {
"src": sourceCat}
692 """Set task and exposure metadata.
694 Logs a warning continues if needed data
is missing.
698 exposure : `lsst.afw.image.ExposureF`
699 Exposure to set metadata on.
700 photoRes : `lsst.pipe.base.Struct`, optional
701 Result of running photoCal task.
706 metadata = exposure.getMetadata()
710 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
711 magZero = photoRes.zp - 2.5*math.log10(exposureTime)
713 self.log.warning(
"Could not set normalized MAGZERO in header: no "
718 metadata.set(
'MAGZERO', magZero)
719 metadata.set(
'MAGZERO_RMS', photoRes.sigma)
720 metadata.set(
'MAGZERO_NOBJ', photoRes.ngood)
721 metadata.set(
'COLORTERM1', 0.0)
722 metadata.set(
'COLORTERM2', 0.0)
723 metadata.set(
'COLORTERM3', 0.0)
724 except Exception
as e:
725 self.log.warning(
"Could not set exposure metadata: %s", e)
728 """Match sources in an icSourceCat and a sourceCat and copy fields.
730 The fields copied are those specified by
731 ``config.icSourceFieldsToCopy``.
736 Catalog from which to copy fields.
738 Catalog to which to copy fields.
743 Raised
if any of the following occur:
744 - icSourceSchema
and icSourceKeys are
not specified.
745 - icSourceCat
and sourceCat are
not specified.
746 - icSourceFieldsToCopy
is empty.
749 raise RuntimeError(
"To copy icSource fields you must specify "
750 "icSourceSchema and icSourceKeys when "
751 "constructing this task")
752 if icSourceCat
is None or sourceCat
is None:
753 raise RuntimeError(
"icSourceCat and sourceCat must both be "
755 if len(self.config.icSourceFieldsToCopy) == 0:
756 self.log.warning(
"copyIcSourceFields doing nothing because "
757 "icSourceFieldsToCopy is empty")
760 mc = afwTable.MatchControl()
761 mc.findOnlyClosest =
False
762 matches = afwTable.matchXy(icSourceCat, sourceCat,
763 self.config.matchRadiusPix, mc)
764 if self.config.doDeblend:
765 deblendKey = sourceCat.schema[
"deblend_nChild"].asKey()
767 matches = [m
for m
in matches
if m[1].get(deblendKey) == 0]
774 for m0, m1, d
in matches:
776 match = bestMatches.get(id0)
777 if match
is None or d <= match[2]:
778 bestMatches[id0] = (m0, m1, d)
779 matches = list(bestMatches.values())
784 numMatches = len(matches)
785 numUniqueSources = len(set(m[1].getId()
for m
in matches))
786 if numUniqueSources != numMatches:
787 self.log.warning(
"%d icSourceCat sources matched only %d sourceCat "
788 "sources", numMatches, numUniqueSources)
790 self.log.info(
"Copying flags from icSourceCat to sourceCat for "
791 "%d sources", numMatches)
795 for icSrc, src, d
in matches:
801 icSrcFootprint = icSrc.getFootprint()
803 icSrc.setFootprint(src.getFootprint())
806 icSrc.setFootprint(icSrcFootprint)
postCalibrationMeasurement
def __init__(self, *config=None)
def __init__(self, butler=None, astromRefObjLoader=None, photoRefObjLoader=None, icSourceSchema=None, initInputs=None, **kwargs)
def setMetadata(self, exposure, photoRes=None)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def getSchemaCatalogs(self)
def copyIcSourceFields(self, icSourceCat, sourceCat)
def run(self, exposure, exposureIdInfo=None, background=None, icSourceCat=None)