22__all__ = [
"CalibrateConfig",
"CalibrateTask"]
28from lsstDebug
import getDebugFrame
31import lsst.pipe.base.connectionTypes
as cT
33from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches
41 CatalogCalculationTask,
43 DetectorVisitIdGeneratorConfig)
45from lsst.utils.timer
import timeMethod
47from .fakes
import BaseFakeSourcesTask
48from .photoCal
import PhotoCalTask
49from .computeExposureSummaryStats
import ComputeExposureSummaryStatsTask
52class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=(
"instrument",
"visit",
"detector"),
55 icSourceSchema = cT.InitInput(
56 doc=
"Schema produced by characterize image task, used to initialize this task",
58 storageClass=
"SourceCatalog",
61 outputSchema = cT.InitOutput(
62 doc=
"Schema after CalibrateTask has been initialized",
64 storageClass=
"SourceCatalog",
68 doc=
"Input image to calibrate",
70 storageClass=
"ExposureF",
71 dimensions=(
"instrument",
"visit",
"detector"),
74 background = cT.Input(
75 doc=
"Backgrounds determined by characterize task",
76 name=
"icExpBackground",
77 storageClass=
"Background",
78 dimensions=(
"instrument",
"visit",
"detector"),
81 icSourceCat = cT.Input(
82 doc=
"Source catalog created by characterize task",
84 storageClass=
"SourceCatalog",
85 dimensions=(
"instrument",
"visit",
"detector"),
88 astromRefCat = cT.PrerequisiteInput(
89 doc=
"Reference catalog to use for astrometry",
90 name=
"gaia_dr2_20200414",
91 storageClass=
"SimpleCatalog",
92 dimensions=(
"skypix",),
97 photoRefCat = cT.PrerequisiteInput(
98 doc=
"Reference catalog to use for photometric calibration",
99 name=
"ps1_pv3_3pi_20170110",
100 storageClass=
"SimpleCatalog",
101 dimensions=(
"skypix",),
106 outputExposure = cT.Output(
107 doc=
"Exposure after running calibration task",
109 storageClass=
"ExposureF",
110 dimensions=(
"instrument",
"visit",
"detector"),
113 outputCat = cT.Output(
114 doc=
"Source catalog produced in calibrate task",
116 storageClass=
"SourceCatalog",
117 dimensions=(
"instrument",
"visit",
"detector"),
120 outputBackground = cT.Output(
121 doc=
"Background models estimated in calibration task",
122 name=
"calexpBackground",
123 storageClass=
"Background",
124 dimensions=(
"instrument",
"visit",
"detector"),
128 doc=
"Source/refObj matches from the astrometry solver",
130 storageClass=
"Catalog",
131 dimensions=(
"instrument",
"visit",
"detector"),
134 matchesDenormalized = cT.Output(
135 doc=
"Denormalized matches from astrometry solver",
137 storageClass=
"Catalog",
138 dimensions=(
"instrument",
"visit",
"detector"),
144 if config.doAstrometry
is False:
145 self.prerequisiteInputs.remove(
"astromRefCat")
146 if config.doPhotoCal
is False:
147 self.prerequisiteInputs.remove(
"photoRefCat")
149 if config.doWriteMatches
is False or config.doAstrometry
is False:
150 self.outputs.remove(
"matches")
151 if config.doWriteMatchesDenormalized
is False or config.doAstrometry
is False:
152 self.outputs.remove(
"matchesDenormalized")
155class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections):
156 """Config for CalibrateTask."""
158 doWrite = pexConfig.Field(
161 doc=
"Save calibration results?",
163 doWriteHeavyFootprintsInSources = pexConfig.Field(
166 doc=
"Include HeavyFootprint data in source table? If false then heavy "
167 "footprints are saved as normal footprints, which saves some space"
169 doWriteMatches = pexConfig.Field(
172 doc=
"Write reference matches (ignored if doWrite or doAstrometry false)?",
174 doWriteMatchesDenormalized = pexConfig.Field(
177 doc=(
"Write reference matches in denormalized format? "
178 "This format uses more disk space, but is more convenient to "
179 "read. Ignored if doWriteMatches=False or doWrite=False."),
181 doAstrometry = pexConfig.Field(
184 doc=
"Perform astrometric calibration?",
186 astromRefObjLoader = pexConfig.ConfigField(
187 dtype=LoadReferenceObjectsConfig,
188 doc=
"reference object loader for astrometric calibration",
190 photoRefObjLoader = pexConfig.ConfigField(
191 dtype=LoadReferenceObjectsConfig,
192 doc=
"reference object loader for photometric calibration",
194 astrometry = pexConfig.ConfigurableField(
195 target=AstrometryTask,
196 doc=
"Perform astrometric calibration to refine the WCS",
198 requireAstrometry = pexConfig.Field(
201 doc=(
"Raise an exception if astrometry fails? Ignored if doAstrometry "
204 doPhotoCal = pexConfig.Field(
207 doc=
"Perform phometric calibration?",
209 requirePhotoCal = pexConfig.Field(
212 doc=(
"Raise an exception if photoCal fails? Ignored if doPhotoCal "
215 photoCal = pexConfig.ConfigurableField(
217 doc=
"Perform photometric calibration",
219 icSourceFieldsToCopy = pexConfig.ListField(
221 default=(
"calib_psf_candidate",
"calib_psf_used",
"calib_psf_reserved"),
222 doc=(
"Fields to copy from the icSource catalog to the output catalog "
223 "for matching sources Any missing fields will trigger a "
224 "RuntimeError exception. Ignored if icSourceCat is not provided.")
226 matchRadiusPix = pexConfig.Field(
229 doc=(
"Match radius for matching icSourceCat objects to sourceCat "
232 checkUnitsParseStrict = pexConfig.Field(
233 doc=(
"Strictness of Astropy unit compatibility check, can be 'raise', "
234 "'warn' or 'silent'"),
238 detection = pexConfig.ConfigurableField(
239 target=SourceDetectionTask,
242 doDeblend = pexConfig.Field(
245 doc=
"Run deblender input exposure"
247 deblend = pexConfig.ConfigurableField(
248 target=SourceDeblendTask,
249 doc=
"Split blended sources into their components"
251 doSkySources = pexConfig.Field(
254 doc=
"Generate sky sources?",
256 skySources = pexConfig.ConfigurableField(
257 target=SkyObjectsTask,
258 doc=
"Generate sky sources",
260 measurement = pexConfig.ConfigurableField(
261 target=SingleFrameMeasurementTask,
262 doc=
"Measure sources"
264 postCalibrationMeasurement = pexConfig.ConfigurableField(
265 target=SingleFrameMeasurementTask,
266 doc=
"Second round of measurement for plugins that need to be run after photocal"
268 setPrimaryFlags = pexConfig.ConfigurableField(
269 target=SetPrimaryFlagsTask,
270 doc=(
"Set flags for primary source classification in single frame "
271 "processing. True if sources are not sky sources and not a parent.")
273 doApCorr = pexConfig.Field(
276 doc=
"Run subtask to apply aperture correction"
278 applyApCorr = pexConfig.ConfigurableField(
279 target=ApplyApCorrTask,
280 doc=
"Subtask to apply aperture corrections"
285 catalogCalculation = pexConfig.ConfigurableField(
286 target=CatalogCalculationTask,
287 doc=
"Subtask to run catalogCalculation plugins on catalog"
289 doInsertFakes = pexConfig.Field(
292 doc=
"Run fake sources injection task",
293 deprecated=(
"doInsertFakes is no longer supported. This config will be removed after v24. "
294 "Please use ProcessCcdWithFakesTask instead.")
296 insertFakes = pexConfig.ConfigurableField(
297 target=BaseFakeSourcesTask,
298 doc=
"Injection of fake sources for testing purposes (must be "
300 deprecated=(
"insertFakes is no longer supported. This config will be removed after v24. "
301 "Please use ProcessCcdWithFakesTask instead.")
303 doComputeSummaryStats = pexConfig.Field(
306 doc=
"Run subtask to measure exposure summary statistics?"
308 computeSummaryStats = pexConfig.ConfigurableField(
309 target=ComputeExposureSummaryStatsTask,
310 doc=
"Subtask to run computeSummaryStats on exposure"
312 doWriteExposure = pexConfig.Field(
315 doc=
"Write the calexp? If fakes have been added then we do not want to write out the calexp as a "
316 "normal calexp but as a fakes_calexp."
318 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
322 self.
detection.doTempLocalBackground =
False
323 self.
deblend.maxFootprintSize = 2000
330 self.
photoCal.photoCatName = self.connections.photoRefCat
334 """Calibrate an exposure: measure sources and perform astrometric and
335 photometric calibration.
337 Given an exposure with a good PSF model
and aperture correction map(e.g.
as
339 perform the following operations:
340 - Run detection
and measurement
341 - Run astrometry subtask to fit an improved WCS
342 - Run photoCal subtask to fit the exposure
's photometric zero-point
347 Compatibility parameter. Should always be `
None`.
348 astromRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
349 Unused
in gen3: must be `
None`.
350 photoRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
351 Unused
in gen3: must be `
None`.
353 Schema
for the icSource catalog.
354 initInputs : `dict`, optional
355 Dictionary that can contain a key ``icSourceSchema`` containing the
356 input schema. If present will override the value of ``icSourceSchema``.
361 Raised
if any of the following occur:
362 - isSourceCat
is missing fields specified
in icSourceFieldsToCopy.
363 - PipelineTask form of this task
is initialized
with reference object
368 Quantities set
in exposure Metadata:
371 MAGZERO
's RMS == sigma reported by photoCal task
373 Number of stars used == ngood reported by photoCal task
382 CalibrateTask has a debug dictionary containing one key:
385 frame (an int; <= 0 to not display)
in which to display the exposure,
386 sources
and matches. See
@ref lsst.meas.astrom.displayAstrometry
for
387 the meaning of the various symbols.
390 ConfigClass = CalibrateConfig
391 _DefaultName = "calibrate"
393 def __init__(self, butler=None, astromRefObjLoader=None,
394 photoRefObjLoader=None, icSourceSchema=None,
395 initInputs=None, **kwargs):
398 if butler
is not None:
399 warnings.warn(
"The 'butler' parameter is no longer used and can be safely removed.",
400 category=FutureWarning, stacklevel=2)
403 if initInputs
is not None:
404 icSourceSchema = initInputs[
'icSourceSchema'].schema
406 if icSourceSchema
is not None:
409 minimumSchema = afwTable.SourceTable.makeMinimalSchema()
410 self.
schemaMapper.addMinimalSchema(minimumSchema,
False)
418 afwTable.Field[
"Flag"](
"calib_detected",
419 "Source was detected as an icSource"))
420 missingFieldNames = []
421 for fieldName
in self.config.icSourceFieldsToCopy:
423 schemaItem = icSourceSchema.find(fieldName)
425 missingFieldNames.append(fieldName)
430 if missingFieldNames:
431 raise RuntimeError(
"isSourceCat is missing fields {} "
432 "specified in icSourceFieldsToCopy"
433 .format(missingFieldNames))
440 self.
schema = afwTable.SourceTable.makeMinimalSchema()
441 self.makeSubtask(
'detection', schema=self.
schema)
445 if self.config.doDeblend:
446 self.makeSubtask(
"deblend", schema=self.
schema)
447 if self.config.doSkySources:
448 self.makeSubtask(
"skySources")
450 self.makeSubtask(
'measurement', schema=self.
schema,
452 self.makeSubtask(
'postCalibrationMeasurement', schema=self.
schema,
454 self.makeSubtask(
"setPrimaryFlags", schema=self.
schema, isSingleFrame=
True)
455 if self.config.doApCorr:
456 self.makeSubtask(
'applyApCorr', schema=self.
schema)
457 self.makeSubtask(
'catalogCalculation', schema=self.
schema)
459 if self.config.doAstrometry:
460 self.makeSubtask(
"astrometry", refObjLoader=astromRefObjLoader,
462 if self.config.doPhotoCal:
463 self.makeSubtask(
"photoCal", refObjLoader=photoRefObjLoader,
465 if self.config.doComputeSummaryStats:
466 self.makeSubtask(
'computeSummaryStats')
468 if initInputs
is not None and (astromRefObjLoader
is not None or photoRefObjLoader
is not None):
469 raise RuntimeError(
"PipelineTask form of this task should not be initialized with "
470 "reference object loaders.")
475 self.
schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
477 sourceCatSchema = afwTable.SourceCatalog(self.
schema)
482 inputs = butlerQC.get(inputRefs)
483 inputs[
'idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId)
485 if self.config.doAstrometry:
486 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
487 for ref
in inputRefs.astromRefCat],
488 refCats=inputs.pop(
'astromRefCat'),
489 name=self.config.connections.astromRefCat,
490 config=self.config.astromRefObjLoader, log=self.log)
491 self.astrometry.setRefObjLoader(refObjLoader)
493 if self.config.doPhotoCal:
494 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
495 for ref
in inputRefs.photoRefCat],
496 refCats=inputs.pop(
'photoRefCat'),
497 name=self.config.connections.photoRefCat,
498 config=self.config.photoRefObjLoader,
500 self.photoCal.match.setRefObjLoader(photoRefObjLoader)
502 outputs = self.
run(**inputs)
504 if self.config.doWriteMatches
and self.config.doAstrometry:
505 if outputs.astromMatches
is not None:
506 normalizedMatches = afwTable.packMatches(outputs.astromMatches)
507 normalizedMatches.table.setMetadata(outputs.matchMeta)
508 if self.config.doWriteMatchesDenormalized:
509 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
510 outputs.matchesDenormalized = denormMatches
511 outputs.matches = normalizedMatches
513 del outputRefs.matches
514 if self.config.doWriteMatchesDenormalized:
515 del outputRefs.matchesDenormalized
516 butlerQC.put(outputs, outputRefs)
519 def run(self, exposure, exposureIdInfo=None, background=None,
520 icSourceCat=None, idGenerator=None):
521 """Calibrate an exposure.
525 exposure : `lsst.afw.image.ExposureF`
526 Exposure to calibrate.
527 exposureIdInfo : `lsst.obs.baseExposureIdInfo`, optional
528 Exposure ID info. Deprecated in favor of ``idGenerator``,
and
529 ignored
if that
is provided.
530 background : `lsst.afw.math.BackgroundList`, optional
531 Initial model of background already subtracted
from exposure.
532 icSourceCat : `lsst.afw.image.SourceCatalog`, optional
533 SourceCatalog
from CharacterizeImageTask
from which we can copy
535 idGenerator : `lsst.meas.base.IdGenerator`, optional
536 Object that generates source IDs
and provides RNG seeds.
540 result : `lsst.pipe.base.Struct`
541 Results
as a struct
with attributes:
544 Characterized exposure (`lsst.afw.image.ExposureF`).
548 Model of subtracted background (`lsst.afw.math.BackgroundList`).
550 List of source/ref matches
from astrometry solver.
552 Metadata
from astrometry matches.
554 Another reference to ``exposure``
for compatibility.
556 Another reference to ``sourceCat``
for compatibility.
559 if idGenerator
is None:
560 if exposureIdInfo
is not None:
561 idGenerator = IdGenerator._from_exposure_id_info(exposureIdInfo)
563 idGenerator = IdGenerator()
565 if background
is None:
566 background = BackgroundList()
567 table = SourceTable.make(self.
schema, idGenerator.make_table_id_factory())
570 detRes = self.detection.run(table=table, exposure=exposure,
572 sourceCat = detRes.sources
573 if detRes.background:
574 for bg
in detRes.background:
575 background.append(bg)
576 if self.config.doSkySources:
577 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=idGenerator.catalog_id)
578 if skySourceFootprints:
579 for foot
in skySourceFootprints:
580 s = sourceCat.addNew()
583 if self.config.doDeblend:
584 self.deblend.run(exposure=exposure, sources=sourceCat)
585 self.measurement.run(
588 exposureId=idGenerator.catalog_id,
590 if self.config.doApCorr:
591 apCorrMap = exposure.getInfo().getApCorrMap()
592 if apCorrMap
is None:
593 self.log.warning(
"Image does not have valid aperture correction map for %r; "
594 "skipping aperture correction", idGenerator)
596 self.applyApCorr.run(
600 self.catalogCalculation.run(sourceCat)
602 self.setPrimaryFlags.run(sourceCat)
604 if icSourceCat
is not None and \
605 len(self.config.icSourceFieldsToCopy) > 0:
613 if not sourceCat.isContiguous():
614 sourceCat = sourceCat.copy(deep=
True)
620 if self.config.doAstrometry:
621 astromRes = self.astrometry.run(
625 astromMatches = astromRes.matches
626 matchMeta = astromRes.matchMeta
627 if exposure.getWcs()
is None:
628 if self.config.requireAstrometry:
629 raise RuntimeError(f
"WCS fit failed for {idGenerator} and requireAstrometry "
632 self.log.warning(
"Unable to perform astrometric calibration for %r but "
633 "requireAstrometry is False: attempting to proceed...",
637 if self.config.doPhotoCal:
638 if np.all(np.isnan(sourceCat[
"coord_ra"]))
or np.all(np.isnan(sourceCat[
"coord_dec"])):
639 if self.config.requirePhotoCal:
640 raise RuntimeError(f
"Astrometry failed for {idGenerator}, so cannot do "
641 "photoCal, but requirePhotoCal is True.")
642 self.log.warning(
"Astrometry failed for %r, so cannot do photoCal. requirePhotoCal "
643 "is False, so skipping photometric calibration and setting photoCalib "
644 "to None. Attempting to proceed...", idGenerator)
645 exposure.setPhotoCalib(
None)
649 photoRes = self.photoCal.run(
650 exposure, sourceCat=sourceCat, expId=idGenerator.catalog_id
652 exposure.setPhotoCalib(photoRes.photoCalib)
655 self.log.info(
"Photometric zero-point: %f",
656 photoRes.photoCalib.instFluxToMagnitude(1.0))
657 self.
setMetadata(exposure=exposure, photoRes=photoRes)
658 except Exception
as e:
659 if self.config.requirePhotoCal:
661 self.log.warning(
"Unable to perform photometric calibration "
662 "(%s): attempting to proceed", e)
665 self.postCalibrationMeasurement.run(
668 exposureId=idGenerator.catalog_id,
671 if self.config.doComputeSummaryStats:
672 summary = self.computeSummaryStats.run(exposure=exposure,
674 background=background)
675 exposure.getInfo().setSummaryStats(summary)
677 frame = getDebugFrame(self._display,
"calibrate")
682 matches=astromMatches,
687 return pipeBase.Struct(
689 astromMatches=astromMatches,
691 outputExposure=exposure,
693 outputBackground=background,
697 """Set task and exposure metadata.
699 Logs a warning continues if needed data
is missing.
703 exposure : `lsst.afw.image.ExposureF`
704 Exposure to set metadata on.
705 photoRes : `lsst.pipe.base.Struct`, optional
706 Result of running photoCal task.
711 metadata = exposure.getMetadata()
715 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
716 magZero = photoRes.zp - 2.5*math.log10(exposureTime)
718 self.log.warning(
"Could not set normalized MAGZERO in header: no "
723 metadata.set(
'MAGZERO', magZero)
724 metadata.set(
'MAGZERO_RMS', photoRes.sigma)
725 metadata.set(
'MAGZERO_NOBJ', photoRes.ngood)
726 metadata.set(
'COLORTERM1', 0.0)
727 metadata.set(
'COLORTERM2', 0.0)
728 metadata.set(
'COLORTERM3', 0.0)
729 except Exception
as e:
730 self.log.warning(
"Could not set exposure metadata: %s", e)
733 """Match sources in an icSourceCat and a sourceCat and copy fields.
735 The fields copied are those specified by
736 ``config.icSourceFieldsToCopy``.
741 Catalog from which to copy fields.
743 Catalog to which to copy fields.
748 Raised
if any of the following occur:
749 - icSourceSchema
and icSourceKeys are
not specified.
750 - icSourceCat
and sourceCat are
not specified.
751 - icSourceFieldsToCopy
is empty.
754 raise RuntimeError(
"To copy icSource fields you must specify "
755 "icSourceSchema and icSourceKeys when "
756 "constructing this task")
757 if icSourceCat
is None or sourceCat
is None:
758 raise RuntimeError(
"icSourceCat and sourceCat must both be "
760 if len(self.config.icSourceFieldsToCopy) == 0:
761 self.log.warning(
"copyIcSourceFields doing nothing because "
762 "icSourceFieldsToCopy is empty")
765 mc = afwTable.MatchControl()
766 mc.findOnlyClosest =
False
767 matches = afwTable.matchXy(icSourceCat, sourceCat,
768 self.config.matchRadiusPix, mc)
769 if self.config.doDeblend:
770 deblendKey = sourceCat.schema[
"deblend_nChild"].asKey()
772 matches = [m
for m
in matches
if m[1].get(deblendKey) == 0]
779 for m0, m1, d
in matches:
781 match = bestMatches.get(id0)
782 if match
is None or d <= match[2]:
783 bestMatches[id0] = (m0, m1, d)
784 matches = list(bestMatches.values())
789 numMatches = len(matches)
790 numUniqueSources = len(set(m[1].getId()
for m
in matches))
791 if numUniqueSources != numMatches:
792 self.log.warning(
"%d icSourceCat sources matched only %d sourceCat "
793 "sources", numMatches, numUniqueSources)
795 self.log.info(
"Copying flags from icSourceCat to sourceCat for "
796 "%d sources", numMatches)
800 for icSrc, src, d
in matches:
806 icSrcFootprint = icSrc.getFootprint()
808 icSrc.setFootprint(src.getFootprint())
811 icSrc.setFootprint(icSrcFootprint)
postCalibrationMeasurement
def __init__(self, *config=None)
def __init__(self, butler=None, astromRefObjLoader=None, photoRefObjLoader=None, icSourceSchema=None, initInputs=None, **kwargs)
def setMetadata(self, exposure, photoRes=None)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def copyIcSourceFields(self, icSourceCat, sourceCat)
def run(self, exposure, exposureIdInfo=None, background=None, icSourceCat=None, idGenerator=None)