22__all__ = [
"CalibrateConfig",
"CalibrateTask"]
28from lsstDebug
import getDebugFrame
30import lsst.pipe.base
as pipeBase
31import lsst.pipe.base.connectionTypes
as cT
33from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches
41 CatalogCalculationTask,
43 DetectorVisitIdGeneratorConfig)
45from lsst.utils.timer
import timeMethod
47from .photoCal
import PhotoCalTask
48from .computeExposureSummaryStats
import ComputeExposureSummaryStatsTask
51class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=(
"instrument",
"visit",
"detector"),
54 icSourceSchema = cT.InitInput(
55 doc=
"Schema produced by characterize image task, used to initialize this task",
57 storageClass=
"SourceCatalog",
60 outputSchema = cT.InitOutput(
61 doc=
"Schema after CalibrateTask has been initialized",
63 storageClass=
"SourceCatalog",
67 doc=
"Input image to calibrate",
69 storageClass=
"ExposureF",
70 dimensions=(
"instrument",
"visit",
"detector"),
73 background = cT.Input(
74 doc=
"Backgrounds determined by characterize task",
75 name=
"icExpBackground",
76 storageClass=
"Background",
77 dimensions=(
"instrument",
"visit",
"detector"),
80 icSourceCat = cT.Input(
81 doc=
"Source catalog created by characterize task",
83 storageClass=
"SourceCatalog",
84 dimensions=(
"instrument",
"visit",
"detector"),
87 astromRefCat = cT.PrerequisiteInput(
88 doc=
"Reference catalog to use for astrometry",
89 name=
"gaia_dr2_20200414",
90 storageClass=
"SimpleCatalog",
91 dimensions=(
"skypix",),
96 photoRefCat = cT.PrerequisiteInput(
97 doc=
"Reference catalog to use for photometric calibration",
98 name=
"ps1_pv3_3pi_20170110",
99 storageClass=
"SimpleCatalog",
100 dimensions=(
"skypix",),
105 outputExposure = cT.Output(
106 doc=
"Exposure after running calibration task",
108 storageClass=
"ExposureF",
109 dimensions=(
"instrument",
"visit",
"detector"),
112 outputCat = cT.Output(
113 doc=
"Source catalog produced in calibrate task",
115 storageClass=
"SourceCatalog",
116 dimensions=(
"instrument",
"visit",
"detector"),
119 outputBackground = cT.Output(
120 doc=
"Background models estimated in calibration task",
121 name=
"calexpBackground",
122 storageClass=
"Background",
123 dimensions=(
"instrument",
"visit",
"detector"),
127 doc=
"Source/refObj matches from the astrometry solver",
129 storageClass=
"Catalog",
130 dimensions=(
"instrument",
"visit",
"detector"),
133 matchesDenormalized = cT.Output(
134 doc=
"Denormalized matches from astrometry solver",
136 storageClass=
"Catalog",
137 dimensions=(
"instrument",
"visit",
"detector"),
143 if config.doAstrometry
is False:
144 self.prerequisiteInputs.remove(
"astromRefCat")
145 if config.doPhotoCal
is False:
146 self.prerequisiteInputs.remove(
"photoRefCat")
148 if config.doWriteMatches
is False or config.doAstrometry
is False:
149 self.outputs.remove(
"matches")
150 if config.doWriteMatchesDenormalized
is False or config.doAstrometry
is False:
151 self.outputs.remove(
"matchesDenormalized")
154class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections):
155 """Config for CalibrateTask."""
157 doWrite = pexConfig.Field(
160 doc=
"Save calibration results?",
162 doWriteHeavyFootprintsInSources = pexConfig.Field(
165 doc=
"Include HeavyFootprint data in source table? If false then heavy "
166 "footprints are saved as normal footprints, which saves some space"
168 doWriteMatches = pexConfig.Field(
171 doc=
"Write reference matches (ignored if doWrite or doAstrometry false)?",
173 doWriteMatchesDenormalized = pexConfig.Field(
176 doc=(
"Write reference matches in denormalized format? "
177 "This format uses more disk space, but is more convenient to "
178 "read. Ignored if doWriteMatches=False or doWrite=False."),
180 doAstrometry = pexConfig.Field(
183 doc=
"Perform astrometric calibration?",
185 astromRefObjLoader = pexConfig.ConfigField(
186 dtype=LoadReferenceObjectsConfig,
187 doc=
"reference object loader for astrometric calibration",
189 photoRefObjLoader = pexConfig.ConfigField(
190 dtype=LoadReferenceObjectsConfig,
191 doc=
"reference object loader for photometric calibration",
193 astrometry = pexConfig.ConfigurableField(
194 target=AstrometryTask,
195 doc=
"Perform astrometric calibration to refine the WCS",
197 requireAstrometry = pexConfig.Field(
200 doc=(
"Raise an exception if astrometry fails? Ignored if doAstrometry "
203 doPhotoCal = pexConfig.Field(
206 doc=
"Perform phometric calibration?",
208 requirePhotoCal = pexConfig.Field(
211 doc=(
"Raise an exception if photoCal fails? Ignored if doPhotoCal "
214 photoCal = pexConfig.ConfigurableField(
216 doc=
"Perform photometric calibration",
218 icSourceFieldsToCopy = pexConfig.ListField(
220 default=(
"calib_psf_candidate",
"calib_psf_used",
"calib_psf_reserved"),
221 doc=(
"Fields to copy from the icSource catalog to the output catalog "
222 "for matching sources Any missing fields will trigger a "
223 "RuntimeError exception. Ignored if icSourceCat is not provided.")
225 matchRadiusPix = pexConfig.Field(
228 doc=(
"Match radius for matching icSourceCat objects to sourceCat "
231 checkUnitsParseStrict = pexConfig.Field(
232 doc=(
"Strictness of Astropy unit compatibility check, can be 'raise', "
233 "'warn' or 'silent'"),
237 detection = pexConfig.ConfigurableField(
238 target=SourceDetectionTask,
241 doDeblend = pexConfig.Field(
244 doc=
"Run deblender input exposure"
246 deblend = pexConfig.ConfigurableField(
247 target=SourceDeblendTask,
248 doc=
"Split blended sources into their components"
250 doSkySources = pexConfig.Field(
253 doc=
"Generate sky sources?",
255 skySources = pexConfig.ConfigurableField(
256 target=SkyObjectsTask,
257 doc=
"Generate sky sources",
259 measurement = pexConfig.ConfigurableField(
260 target=SingleFrameMeasurementTask,
261 doc=
"Measure sources"
263 postCalibrationMeasurement = pexConfig.ConfigurableField(
264 target=SingleFrameMeasurementTask,
265 doc=
"Second round of measurement for plugins that need to be run after photocal"
267 setPrimaryFlags = pexConfig.ConfigurableField(
268 target=SetPrimaryFlagsTask,
269 doc=(
"Set flags for primary source classification in single frame "
270 "processing. True if sources are not sky sources and not a parent.")
272 doApCorr = pexConfig.Field(
275 doc=
"Run subtask to apply aperture correction"
277 applyApCorr = pexConfig.ConfigurableField(
278 target=ApplyApCorrTask,
279 doc=
"Subtask to apply aperture corrections"
284 catalogCalculation = pexConfig.ConfigurableField(
285 target=CatalogCalculationTask,
286 doc=
"Subtask to run catalogCalculation plugins on catalog"
288 doComputeSummaryStats = pexConfig.Field(
291 doc=
"Run subtask to measure exposure summary statistics?"
293 computeSummaryStats = pexConfig.ConfigurableField(
294 target=ComputeExposureSummaryStatsTask,
295 doc=
"Subtask to run computeSummaryStats on exposure"
297 doWriteExposure = pexConfig.Field(
300 doc=
"Write the calexp? If fakes have been added then we do not want to write out the calexp as a "
301 "normal calexp but as a fakes_calexp."
303 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
307 self.
detection.doTempLocalBackground =
False
308 self.
deblend.maxFootprintSize = 2000
315 self.
photoCal.photoCatName = self.connections.photoRefCat
319 """Calibrate an exposure: measure sources and perform astrometric and
320 photometric calibration.
322 Given an exposure with a good PSF model
and aperture correction map(e.g.
as
324 perform the following operations:
325 - Run detection
and measurement
326 - Run astrometry subtask to fit an improved WCS
327 - Run photoCal subtask to fit the exposure
's photometric zero-point
332 Compatibility parameter. Should always be `
None`.
333 astromRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
334 Unused
in gen3: must be `
None`.
335 photoRefObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
336 Unused
in gen3: must be `
None`.
338 Schema
for the icSource catalog.
339 initInputs : `dict`, optional
340 Dictionary that can contain a key ``icSourceSchema`` containing the
341 input schema. If present will override the value of ``icSourceSchema``.
346 Raised
if any of the following occur:
347 - isSourceCat
is missing fields specified
in icSourceFieldsToCopy.
348 - PipelineTask form of this task
is initialized
with reference object
353 Quantities set
in exposure Metadata:
356 MAGZERO
's RMS == sigma reported by photoCal task
358 Number of stars used == ngood reported by photoCal task
367 CalibrateTask has a debug dictionary containing one key:
370 frame (an int; <= 0 to not display)
in which to display the exposure,
371 sources
and matches. See
@ref lsst.meas.astrom.displayAstrometry
for
372 the meaning of the various symbols.
375 ConfigClass = CalibrateConfig
376 _DefaultName = "calibrate"
378 def __init__(self, butler=None, astromRefObjLoader=None,
379 photoRefObjLoader=None, icSourceSchema=None,
380 initInputs=None, **kwargs):
383 if butler
is not None:
384 warnings.warn(
"The 'butler' parameter is no longer used and can be safely removed.",
385 category=FutureWarning, stacklevel=2)
388 if initInputs
is not None:
389 icSourceSchema = initInputs[
'icSourceSchema'].schema
391 if icSourceSchema
is not None:
394 minimumSchema = afwTable.SourceTable.makeMinimalSchema()
395 self.
schemaMapper.addMinimalSchema(minimumSchema,
False)
403 afwTable.Field[
"Flag"](
"calib_detected",
404 "Source was detected as an icSource"))
405 missingFieldNames = []
406 for fieldName
in self.config.icSourceFieldsToCopy:
408 schemaItem = icSourceSchema.find(fieldName)
410 missingFieldNames.append(fieldName)
415 if missingFieldNames:
416 raise RuntimeError(
"isSourceCat is missing fields {} "
417 "specified in icSourceFieldsToCopy"
418 .format(missingFieldNames))
425 self.
schema = afwTable.SourceTable.makeMinimalSchema()
426 self.makeSubtask(
'detection', schema=self.
schema)
430 if self.config.doDeblend:
431 self.makeSubtask(
"deblend", schema=self.
schema)
432 if self.config.doSkySources:
433 self.makeSubtask(
"skySources")
435 self.makeSubtask(
'measurement', schema=self.
schema,
437 self.makeSubtask(
'postCalibrationMeasurement', schema=self.
schema,
439 self.makeSubtask(
"setPrimaryFlags", schema=self.
schema, isSingleFrame=
True)
440 if self.config.doApCorr:
441 self.makeSubtask(
'applyApCorr', schema=self.
schema)
442 self.makeSubtask(
'catalogCalculation', schema=self.
schema)
444 if self.config.doAstrometry:
445 self.makeSubtask(
"astrometry", refObjLoader=astromRefObjLoader,
447 if self.config.doPhotoCal:
448 self.makeSubtask(
"photoCal", refObjLoader=photoRefObjLoader,
450 if self.config.doComputeSummaryStats:
451 self.makeSubtask(
'computeSummaryStats')
453 if initInputs
is not None and (astromRefObjLoader
is not None or photoRefObjLoader
is not None):
454 raise RuntimeError(
"PipelineTask form of this task should not be initialized with "
455 "reference object loaders.")
460 self.
schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
462 sourceCatSchema = afwTable.SourceCatalog(self.
schema)
467 inputs = butlerQC.get(inputRefs)
468 inputs[
'idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId)
470 if self.config.doAstrometry:
471 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
472 for ref
in inputRefs.astromRefCat],
473 refCats=inputs.pop(
'astromRefCat'),
474 name=self.config.connections.astromRefCat,
475 config=self.config.astromRefObjLoader, log=self.log)
476 self.astrometry.setRefObjLoader(refObjLoader)
478 if self.config.doPhotoCal:
479 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
480 for ref
in inputRefs.photoRefCat],
481 refCats=inputs.pop(
'photoRefCat'),
482 name=self.config.connections.photoRefCat,
483 config=self.config.photoRefObjLoader,
485 self.photoCal.match.setRefObjLoader(photoRefObjLoader)
487 outputs = self.
run(**inputs)
489 if self.config.doWriteMatches
and self.config.doAstrometry:
490 if outputs.astromMatches
is not None:
491 normalizedMatches = afwTable.packMatches(outputs.astromMatches)
492 normalizedMatches.table.setMetadata(outputs.matchMeta)
493 if self.config.doWriteMatchesDenormalized:
494 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
495 outputs.matchesDenormalized = denormMatches
496 outputs.matches = normalizedMatches
498 del outputRefs.matches
499 if self.config.doWriteMatchesDenormalized:
500 del outputRefs.matchesDenormalized
501 butlerQC.put(outputs, outputRefs)
504 def run(self, exposure, exposureIdInfo=None, background=None,
505 icSourceCat=None, idGenerator=None):
506 """Calibrate an exposure.
510 exposure : `lsst.afw.image.ExposureF`
511 Exposure to calibrate.
512 exposureIdInfo : `lsst.obs.baseExposureIdInfo`, optional
513 Exposure ID info. Deprecated in favor of ``idGenerator``,
and
514 ignored
if that
is provided.
515 background : `lsst.afw.math.BackgroundList`, optional
516 Initial model of background already subtracted
from exposure.
517 icSourceCat : `lsst.afw.image.SourceCatalog`, optional
518 SourceCatalog
from CharacterizeImageTask
from which we can copy
520 idGenerator : `lsst.meas.base.IdGenerator`, optional
521 Object that generates source IDs
and provides RNG seeds.
525 result : `lsst.pipe.base.Struct`
526 Results
as a struct
with attributes:
529 Characterized exposure (`lsst.afw.image.ExposureF`).
533 Model of subtracted background (`lsst.afw.math.BackgroundList`).
535 List of source/ref matches
from astrometry solver.
537 Metadata
from astrometry matches.
539 Another reference to ``exposure``
for compatibility.
541 Another reference to ``sourceCat``
for compatibility.
544 if idGenerator
is None:
545 if exposureIdInfo
is not None:
546 idGenerator = IdGenerator._from_exposure_id_info(exposureIdInfo)
548 idGenerator = IdGenerator()
550 if background
is None:
551 background = BackgroundList()
552 table = SourceTable.make(self.
schema, idGenerator.make_table_id_factory())
555 detRes = self.detection.run(table=table, exposure=exposure,
557 sourceCat = detRes.sources
558 if detRes.background:
559 for bg
in detRes.background:
560 background.append(bg)
561 if self.config.doSkySources:
562 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=idGenerator.catalog_id)
563 if skySourceFootprints:
564 for foot
in skySourceFootprints:
565 s = sourceCat.addNew()
568 if self.config.doDeblend:
569 self.deblend.run(exposure=exposure, sources=sourceCat)
570 self.measurement.run(
573 exposureId=idGenerator.catalog_id,
575 if self.config.doApCorr:
576 apCorrMap = exposure.getInfo().getApCorrMap()
577 if apCorrMap
is None:
578 self.log.warning(
"Image does not have valid aperture correction map for %r; "
579 "skipping aperture correction", idGenerator)
581 self.applyApCorr.run(
585 self.catalogCalculation.run(sourceCat)
587 self.setPrimaryFlags.run(sourceCat)
589 if icSourceCat
is not None and \
590 len(self.config.icSourceFieldsToCopy) > 0:
598 if not sourceCat.isContiguous():
599 sourceCat = sourceCat.copy(deep=
True)
605 if self.config.doAstrometry:
606 astromRes = self.astrometry.run(
610 astromMatches = astromRes.matches
611 matchMeta = astromRes.matchMeta
612 if exposure.getWcs()
is None:
613 if self.config.requireAstrometry:
614 raise RuntimeError(f
"WCS fit failed for {idGenerator} and requireAstrometry "
617 self.log.warning(
"Unable to perform astrometric calibration for %r but "
618 "requireAstrometry is False: attempting to proceed...",
622 if self.config.doPhotoCal:
623 if np.all(np.isnan(sourceCat[
"coord_ra"]))
or np.all(np.isnan(sourceCat[
"coord_dec"])):
624 if self.config.requirePhotoCal:
625 raise RuntimeError(f
"Astrometry failed for {idGenerator}, so cannot do "
626 "photoCal, but requirePhotoCal is True.")
627 self.log.warning(
"Astrometry failed for %r, so cannot do photoCal. requirePhotoCal "
628 "is False, so skipping photometric calibration and setting photoCalib "
629 "to None. Attempting to proceed...", idGenerator)
630 exposure.setPhotoCalib(
None)
634 photoRes = self.photoCal.run(
635 exposure, sourceCat=sourceCat, expId=idGenerator.catalog_id
637 exposure.setPhotoCalib(photoRes.photoCalib)
640 self.log.info(
"Photometric zero-point: %f",
641 photoRes.photoCalib.instFluxToMagnitude(1.0))
642 self.
setMetadata(exposure=exposure, photoRes=photoRes)
643 except Exception
as e:
644 if self.config.requirePhotoCal:
646 self.log.warning(
"Unable to perform photometric calibration "
647 "(%s): attempting to proceed", e)
650 self.postCalibrationMeasurement.run(
653 exposureId=idGenerator.catalog_id,
656 if self.config.doComputeSummaryStats:
657 summary = self.computeSummaryStats.run(exposure=exposure,
659 background=background)
660 exposure.getInfo().setSummaryStats(summary)
662 frame = getDebugFrame(self._display,
"calibrate")
667 matches=astromMatches,
672 return pipeBase.Struct(
674 astromMatches=astromMatches,
676 outputExposure=exposure,
678 outputBackground=background,
682 """Set task and exposure metadata.
684 Logs a warning continues if needed data
is missing.
688 exposure : `lsst.afw.image.ExposureF`
689 Exposure to set metadata on.
690 photoRes : `lsst.pipe.base.Struct`, optional
691 Result of running photoCal task.
696 metadata = exposure.getMetadata()
700 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
701 magZero = photoRes.zp - 2.5*math.log10(exposureTime)
703 self.log.warning(
"Could not set normalized MAGZERO in header: no "
708 metadata.set(
'MAGZERO', magZero)
709 metadata.set(
'MAGZERO_RMS', photoRes.sigma)
710 metadata.set(
'MAGZERO_NOBJ', photoRes.ngood)
711 metadata.set(
'COLORTERM1', 0.0)
712 metadata.set(
'COLORTERM2', 0.0)
713 metadata.set(
'COLORTERM3', 0.0)
714 except Exception
as e:
715 self.log.warning(
"Could not set exposure metadata: %s", e)
718 """Match sources in an icSourceCat and a sourceCat and copy fields.
720 The fields copied are those specified by
721 ``config.icSourceFieldsToCopy``.
726 Catalog from which to copy fields.
728 Catalog to which to copy fields.
733 Raised
if any of the following occur:
734 - icSourceSchema
and icSourceKeys are
not specified.
735 - icSourceCat
and sourceCat are
not specified.
736 - icSourceFieldsToCopy
is empty.
739 raise RuntimeError(
"To copy icSource fields you must specify "
740 "icSourceSchema and icSourceKeys when "
741 "constructing this task")
742 if icSourceCat
is None or sourceCat
is None:
743 raise RuntimeError(
"icSourceCat and sourceCat must both be "
745 if len(self.config.icSourceFieldsToCopy) == 0:
746 self.log.warning(
"copyIcSourceFields doing nothing because "
747 "icSourceFieldsToCopy is empty")
750 mc = afwTable.MatchControl()
751 mc.findOnlyClosest =
False
752 matches = afwTable.matchXy(icSourceCat, sourceCat,
753 self.config.matchRadiusPix, mc)
754 if self.config.doDeblend:
755 deblendKey = sourceCat.schema[
"deblend_nChild"].asKey()
757 matches = [m
for m
in matches
if m[1].get(deblendKey) == 0]
764 for m0, m1, d
in matches:
766 match = bestMatches.get(id0)
767 if match
is None or d <= match[2]:
768 bestMatches[id0] = (m0, m1, d)
769 matches = list(bestMatches.values())
774 numMatches = len(matches)
775 numUniqueSources = len(set(m[1].getId()
for m
in matches))
776 if numUniqueSources != numMatches:
777 self.log.warning(
"%d icSourceCat sources matched only %d sourceCat "
778 "sources", numMatches, numUniqueSources)
780 self.log.info(
"Copying flags from icSourceCat to sourceCat for "
781 "%d sources", numMatches)
785 for icSrc, src, d
in matches:
791 icSrcFootprint = icSrc.getFootprint()
793 icSrc.setFootprint(src.getFootprint())
796 icSrc.setFootprint(icSrcFootprint)
postCalibrationMeasurement
def __init__(self, *config=None)
def __init__(self, butler=None, astromRefObjLoader=None, photoRefObjLoader=None, icSourceSchema=None, initInputs=None, **kwargs)
def setMetadata(self, exposure, photoRes=None)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def copyIcSourceFields(self, icSourceCat, sourceCat)
def run(self, exposure, exposureIdInfo=None, background=None, icSourceCat=None, idGenerator=None)