Coverage for python/lsst/pipe/tasks/calibrate.py: 20%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008-2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22import math
24from lsstDebug import getDebugFrame
25import lsst.pex.config as pexConfig
26import lsst.pipe.base as pipeBase
27import lsst.pipe.base.connectionTypes as cT
28import lsst.afw.table as afwTable
29from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches
30from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, SkyObjectsTask
31from lsst.obs.base import ExposureIdInfo
32import lsst.daf.base as dafBase
33from lsst.afw.math import BackgroundList
34from lsst.afw.table import SourceTable
35from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader
36from lsst.meas.base import (SingleFrameMeasurementTask,
37 ApplyApCorrTask,
38 CatalogCalculationTask)
39from lsst.meas.deblender import SourceDeblendTask
40from lsst.utils.timer import timeMethod
41from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask
42from .fakes import BaseFakeSourcesTask
43from .photoCal import PhotoCalTask
44from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask
47__all__ = ["CalibrateConfig", "CalibrateTask"]
50class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"),
51 defaultTemplates={}):
53 icSourceSchema = cT.InitInput(
54 doc="Schema produced by characterize image task, used to initialize this task",
55 name="icSrc_schema",
56 storageClass="SourceCatalog",
57 )
59 outputSchema = cT.InitOutput(
60 doc="Schema after CalibrateTask has been initialized",
61 name="src_schema",
62 storageClass="SourceCatalog",
63 )
65 exposure = cT.Input(
66 doc="Input image to calibrate",
67 name="icExp",
68 storageClass="ExposureF",
69 dimensions=("instrument", "visit", "detector"),
70 )
72 background = cT.Input(
73 doc="Backgrounds determined by characterize task",
74 name="icExpBackground",
75 storageClass="Background",
76 dimensions=("instrument", "visit", "detector"),
77 )
79 icSourceCat = cT.Input(
80 doc="Source catalog created by characterize task",
81 name="icSrc",
82 storageClass="SourceCatalog",
83 dimensions=("instrument", "visit", "detector"),
84 )
86 astromRefCat = cT.PrerequisiteInput(
87 doc="Reference catalog to use for astrometry",
88 name="cal_ref_cat",
89 storageClass="SimpleCatalog",
90 dimensions=("skypix",),
91 deferLoad=True,
92 multiple=True,
93 )
95 photoRefCat = cT.PrerequisiteInput(
96 doc="Reference catalog to use for photometric calibration",
97 name="cal_ref_cat",
98 storageClass="SimpleCatalog",
99 dimensions=("skypix",),
100 deferLoad=True,
101 multiple=True
102 )
104 outputExposure = cT.Output(
105 doc="Exposure after running calibration task",
106 name="calexp",
107 storageClass="ExposureF",
108 dimensions=("instrument", "visit", "detector"),
109 )
111 outputCat = cT.Output(
112 doc="Source catalog produced in calibrate task",
113 name="src",
114 storageClass="SourceCatalog",
115 dimensions=("instrument", "visit", "detector"),
116 )
118 outputBackground = cT.Output(
119 doc="Background models estimated in calibration task",
120 name="calexpBackground",
121 storageClass="Background",
122 dimensions=("instrument", "visit", "detector"),
123 )
125 matches = cT.Output(
126 doc="Source/refObj matches from the astrometry solver",
127 name="srcMatch",
128 storageClass="Catalog",
129 dimensions=("instrument", "visit", "detector"),
130 )
132 matchesDenormalized = cT.Output(
133 doc="Denormalized matches from astrometry solver",
134 name="srcMatchFull",
135 storageClass="Catalog",
136 dimensions=("instrument", "visit", "detector"),
137 )
139 def __init__(self, *, config=None):
140 super().__init__(config=config)
142 if config.doAstrometry is False:
143 self.prerequisiteInputs.remove("astromRefCat")
144 if config.doPhotoCal is False:
145 self.prerequisiteInputs.remove("photoRefCat")
147 if config.doWriteMatches is False or config.doAstrometry is False:
148 self.outputs.remove("matches")
149 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False:
150 self.outputs.remove("matchesDenormalized")
153class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections):
154 """Config for CalibrateTask"""
155 doWrite = pexConfig.Field(
156 dtype=bool,
157 default=True,
158 doc="Save calibration results?",
159 )
160 doWriteHeavyFootprintsInSources = pexConfig.Field(
161 dtype=bool,
162 default=True,
163 doc="Include HeavyFootprint data in source table? If false then heavy "
164 "footprints are saved as normal footprints, which saves some space"
165 )
166 doWriteMatches = pexConfig.Field(
167 dtype=bool,
168 default=True,
169 doc="Write reference matches (ignored if doWrite or doAstrometry false)?",
170 )
171 doWriteMatchesDenormalized = pexConfig.Field(
172 dtype=bool,
173 default=False,
174 doc=("Write reference matches in denormalized format? "
175 "This format uses more disk space, but is more convenient to "
176 "read. Ignored if doWriteMatches=False or doWrite=False."),
177 )
178 doAstrometry = pexConfig.Field(
179 dtype=bool,
180 default=True,
181 doc="Perform astrometric calibration?",
182 )
183 astromRefObjLoader = pexConfig.ConfigurableField(
184 target=LoadIndexedReferenceObjectsTask,
185 doc="reference object loader for astrometric calibration",
186 )
187 photoRefObjLoader = pexConfig.ConfigurableField(
188 target=LoadIndexedReferenceObjectsTask,
189 doc="reference object loader for photometric calibration",
190 )
191 astrometry = pexConfig.ConfigurableField(
192 target=AstrometryTask,
193 doc="Perform astrometric calibration to refine the WCS",
194 )
195 requireAstrometry = pexConfig.Field(
196 dtype=bool,
197 default=True,
198 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry "
199 "false."),
200 )
201 doPhotoCal = pexConfig.Field(
202 dtype=bool,
203 default=True,
204 doc="Perform phometric calibration?",
205 )
206 requirePhotoCal = pexConfig.Field(
207 dtype=bool,
208 default=True,
209 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal "
210 "false."),
211 )
212 photoCal = pexConfig.ConfigurableField(
213 target=PhotoCalTask,
214 doc="Perform photometric calibration",
215 )
216 icSourceFieldsToCopy = pexConfig.ListField(
217 dtype=str,
218 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
219 doc=("Fields to copy from the icSource catalog to the output catalog "
220 "for matching sources Any missing fields will trigger a "
221 "RuntimeError exception. Ignored if icSourceCat is not provided.")
222 )
223 matchRadiusPix = pexConfig.Field(
224 dtype=float,
225 default=3,
226 doc=("Match radius for matching icSourceCat objects to sourceCat "
227 "objects (pixels)"),
228 )
229 checkUnitsParseStrict = pexConfig.Field(
230 doc=("Strictness of Astropy unit compatibility check, can be 'raise', "
231 "'warn' or 'silent'"),
232 dtype=str,
233 default="raise",
234 )
235 detection = pexConfig.ConfigurableField(
236 target=SourceDetectionTask,
237 doc="Detect sources"
238 )
239 doDeblend = pexConfig.Field(
240 dtype=bool,
241 default=True,
242 doc="Run deblender input exposure"
243 )
244 deblend = pexConfig.ConfigurableField(
245 target=SourceDeblendTask,
246 doc="Split blended sources into their components"
247 )
248 doSkySources = pexConfig.Field(
249 dtype=bool,
250 default=True,
251 doc="Generate sky sources?",
252 )
253 skySources = pexConfig.ConfigurableField(
254 target=SkyObjectsTask,
255 doc="Generate sky sources",
256 )
257 measurement = pexConfig.ConfigurableField(
258 target=SingleFrameMeasurementTask,
259 doc="Measure sources"
260 )
261 postCalibrationMeasurement = pexConfig.ConfigurableField(
262 target=SingleFrameMeasurementTask,
263 doc="Second round of measurement for plugins that need to be run after photocal"
264 )
265 setPrimaryFlags = pexConfig.ConfigurableField(
266 target=SetPrimaryFlagsTask,
267 doc=("Set flags for primary source classification in single frame "
268 "processing. True if sources are not sky sources and not a parent.")
269 )
270 doApCorr = pexConfig.Field(
271 dtype=bool,
272 default=True,
273 doc="Run subtask to apply aperture correction"
274 )
275 applyApCorr = pexConfig.ConfigurableField(
276 target=ApplyApCorrTask,
277 doc="Subtask to apply aperture corrections"
278 )
279 # If doApCorr is False, and the exposure does not have apcorrections
280 # already applied, the active plugins in catalogCalculation almost
281 # certainly should not contain the characterization plugin
282 catalogCalculation = pexConfig.ConfigurableField(
283 target=CatalogCalculationTask,
284 doc="Subtask to run catalogCalculation plugins on catalog"
285 )
286 doInsertFakes = pexConfig.Field(
287 dtype=bool,
288 default=False,
289 doc="Run fake sources injection task",
290 deprecated=("doInsertFakes is no longer supported. This config will be removed after v24. "
291 "Please use ProcessCcdWithFakesTask instead.")
292 )
293 insertFakes = pexConfig.ConfigurableField(
294 target=BaseFakeSourcesTask,
295 doc="Injection of fake sources for testing purposes (must be "
296 "retargeted)",
297 deprecated=("insertFakes is no longer supported. This config will be removed after v24. "
298 "Please use ProcessCcdWithFakesTask instead.")
299 )
300 doComputeSummaryStats = pexConfig.Field(
301 dtype=bool,
302 default=True,
303 doc="Run subtask to measure exposure summary statistics?"
304 )
305 computeSummaryStats = pexConfig.ConfigurableField(
306 target=ComputeExposureSummaryStatsTask,
307 doc="Subtask to run computeSummaryStats on exposure"
308 )
309 doWriteExposure = pexConfig.Field(
310 dtype=bool,
311 default=True,
312 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a "
313 "normal calexp but as a fakes_calexp."
314 )
316 def setDefaults(self):
317 super().setDefaults()
318 self.detection.doTempLocalBackground = False
319 self.deblend.maxFootprintSize = 2000
320 self.postCalibrationMeasurement.plugins.names = ["base_LocalPhotoCalib", "base_LocalWcs"]
321 self.postCalibrationMeasurement.doReplaceWithNoise = False
322 for key in self.postCalibrationMeasurement.slots:
323 setattr(self.postCalibrationMeasurement.slots, key, None)
325 def validate(self):
326 super().validate()
327 astromRefCatGen2 = getattr(self.astromRefObjLoader, "ref_dataset_name", None)
328 if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat:
329 raise ValueError(
330 f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference "
331 f"catalogs are different. These options must be kept in sync until Gen2 is retired."
332 )
333 photoRefCatGen2 = getattr(self.photoRefObjLoader, "ref_dataset_name", None)
334 if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat:
335 raise ValueError(
336 f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference "
337 f"catalogs are different. These options must be kept in sync until Gen2 is retired."
338 )
341## \addtogroup LSST_task_documentation
342## \{
343## \page page_CalibrateTask CalibrateTask
344## \ref CalibrateTask_ "CalibrateTask"
345## \copybrief CalibrateTask
346## \}
348class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
349 r"""!Calibrate an exposure: measure sources and perform astrometric and
350 photometric calibration
352 @anchor CalibrateTask_
354 @section pipe_tasks_calibrate_Contents Contents
356 - @ref pipe_tasks_calibrate_Purpose
357 - @ref pipe_tasks_calibrate_Initialize
358 - @ref pipe_tasks_calibrate_IO
359 - @ref pipe_tasks_calibrate_Config
360 - @ref pipe_tasks_calibrate_Metadata
361 - @ref pipe_tasks_calibrate_Debug
363 @section pipe_tasks_calibrate_Purpose Description
365 Given an exposure with a good PSF model and aperture correction map
366 (e.g. as provided by @ref characterizeImage::CharacterizeImageTask "CharacterizeImageTask"),
367 perform the following operations:
368 - Run detection and measurement
369 - Run astrometry subtask to fit an improved WCS
370 - Run photoCal subtask to fit the exposure's photometric zero-point
372 @section pipe_tasks_calibrate_Initialize Task initialisation
374 @copydoc \_\_init\_\_
376 @section pipe_tasks_calibrate_IO Invoking the Task
378 If you want this task to unpersist inputs or persist outputs, then call
379 the `runDataRef` method (a wrapper around the `run` method).
381 If you already have the inputs unpersisted and do not want to persist the
382 output then it is more direct to call the `run` method:
384 @section pipe_tasks_calibrate_Config Configuration parameters
386 See @ref CalibrateConfig
388 @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata
390 Exposure metadata
391 <dl>
392 <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task
393 <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal
394 task
395 <dt>COLORTERM1 <dd>?? (always 0.0)
396 <dt>COLORTERM2 <dd>?? (always 0.0)
397 <dt>COLORTERM3 <dd>?? (always 0.0)
398 </dl>
400 @section pipe_tasks_calibrate_Debug Debug variables
402 The command line task
403 interface supports a flag
404 `--debug` to import `debug.py` from your `$PYTHONPATH`; see
405 <a href="https://pipelines.lsst.io/modules/lsstDebug/">the lsstDebug documentation</a>
406 for more about `debug.py`.
408 CalibrateTask has a debug dictionary containing one key:
409 <dl>
410 <dt>calibrate
411 <dd>frame (an int; <= 0 to not display) in which to display the exposure,
412 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for
413 the meaning of the various symbols.
414 </dl>
416 For example, put something like:
417 @code{.py}
418 import lsstDebug
419 def DebugInfo(name):
420 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would
421 # call us recursively
422 if name == "lsst.pipe.tasks.calibrate":
423 di.display = dict(
424 calibrate = 1,
425 )
427 return di
429 lsstDebug.Info = DebugInfo
430 @endcode
431 into your `debug.py` file and run `calibrateTask.py` with the `--debug`
432 flag.
434 Some subtasks may have their own debug variables; see individual Task
435 documentation.
436 """
438 # Example description used to live here, removed 2-20-2017 as per
439 # https://jira.lsstcorp.org/browse/DM-9520
441 ConfigClass = CalibrateConfig
442 _DefaultName = "calibrate"
443 RunnerClass = pipeBase.ButlerInitializedTaskRunner
445 def __init__(self, butler=None, astromRefObjLoader=None,
446 photoRefObjLoader=None, icSourceSchema=None,
447 initInputs=None, **kwargs):
448 """!Construct a CalibrateTask
450 @param[in] butler The butler is passed to the refObjLoader constructor
451 in case it is needed. Ignored if the refObjLoader argument
452 provides a loader directly.
453 @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks
454 that supplies an external reference catalog for astrometric
455 calibration. May be None if the desired loader can be constructed
456 from the butler argument or all steps requiring a reference catalog
457 are disabled.
458 @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks
459 that supplies an external reference catalog for photometric
460 calibration. May be None if the desired loader can be constructed
461 from the butler argument or all steps requiring a reference catalog
462 are disabled.
463 @param[in] icSourceSchema schema for icSource catalog, or None.
464 Schema values specified in config.icSourceFieldsToCopy will be
465 taken from this schema. If set to None, no values will be
466 propagated from the icSourceCatalog
467 @param[in,out] kwargs other keyword arguments for
468 lsst.pipe.base.CmdLineTask
469 """
470 super().__init__(**kwargs)
472 if icSourceSchema is None and butler is not None:
473 # Use butler to read icSourceSchema from disk.
474 icSourceSchema = butler.get("icSrc_schema", immediate=True).schema
476 if icSourceSchema is None and butler is None and initInputs is not None:
477 icSourceSchema = initInputs['icSourceSchema'].schema
479 if icSourceSchema is not None:
480 # use a schema mapper to avoid copying each field separately
481 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema)
482 minimumSchema = afwTable.SourceTable.makeMinimalSchema()
483 self.schemaMapper.addMinimalSchema(minimumSchema, False)
485 # Add fields to copy from an icSource catalog
486 # and a field to indicate that the source matched a source in that
487 # catalog. If any fields are missing then raise an exception, but
488 # first find all missing fields in order to make the error message
489 # more useful.
490 self.calibSourceKey = self.schemaMapper.addOutputField(
491 afwTable.Field["Flag"]("calib_detected",
492 "Source was detected as an icSource"))
493 missingFieldNames = []
494 for fieldName in self.config.icSourceFieldsToCopy:
495 try:
496 schemaItem = icSourceSchema.find(fieldName)
497 except Exception:
498 missingFieldNames.append(fieldName)
499 else:
500 # field found; if addMapping fails then raise an exception
501 self.schemaMapper.addMapping(schemaItem.getKey())
503 if missingFieldNames:
504 raise RuntimeError("isSourceCat is missing fields {} "
505 "specified in icSourceFieldsToCopy"
506 .format(missingFieldNames))
508 # produce a temporary schema to pass to the subtasks; finalize it
509 # later
510 self.schema = self.schemaMapper.editOutputSchema()
511 else:
512 self.schemaMapper = None
513 self.schema = afwTable.SourceTable.makeMinimalSchema()
514 self.makeSubtask('detection', schema=self.schema)
516 self.algMetadata = dafBase.PropertyList()
518 if self.config.doDeblend:
519 self.makeSubtask("deblend", schema=self.schema)
520 if self.config.doSkySources:
521 self.makeSubtask("skySources")
522 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
523 self.makeSubtask('measurement', schema=self.schema,
524 algMetadata=self.algMetadata)
525 self.makeSubtask('postCalibrationMeasurement', schema=self.schema,
526 algMetadata=self.algMetadata)
527 self.makeSubtask("setPrimaryFlags", schema=self.schema, isSingleFrame=True)
528 if self.config.doApCorr:
529 self.makeSubtask('applyApCorr', schema=self.schema)
530 self.makeSubtask('catalogCalculation', schema=self.schema)
532 if self.config.doAstrometry:
533 if astromRefObjLoader is None and butler is not None:
534 self.makeSubtask('astromRefObjLoader', butler=butler)
535 astromRefObjLoader = self.astromRefObjLoader
536 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader,
537 schema=self.schema)
538 if self.config.doPhotoCal:
539 if photoRefObjLoader is None and butler is not None:
540 self.makeSubtask('photoRefObjLoader', butler=butler)
541 photoRefObjLoader = self.photoRefObjLoader
542 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader,
543 schema=self.schema)
544 if self.config.doComputeSummaryStats:
545 self.makeSubtask('computeSummaryStats')
547 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None):
548 raise RuntimeError("PipelineTask form of this task should not be initialized with "
549 "reference object loaders.")
551 if self.schemaMapper is not None:
552 # finalize the schema
553 self.schema = self.schemaMapper.getOutputSchema()
554 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
556 sourceCatSchema = afwTable.SourceCatalog(self.schema)
557 sourceCatSchema.getTable().setMetadata(self.algMetadata)
558 self.outputSchema = sourceCatSchema
560 @timeMethod
561 def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None,
562 doUnpersist=True):
563 """!Calibrate an exposure, optionally unpersisting inputs and
564 persisting outputs.
566 This is a wrapper around the `run` method that unpersists inputs
567 (if `doUnpersist` true) and persists outputs (if `config.doWrite` true)
569 @param[in] dataRef butler data reference corresponding to a science
570 image
571 @param[in,out] exposure characterized exposure (an
572 lsst.afw.image.ExposureF or similar), or None to unpersist existing
573 icExp and icBackground. See `run` method for details of what is
574 read and written.
575 @param[in,out] background initial model of background already
576 subtracted from exposure (an lsst.afw.math.BackgroundList). May be
577 None if no background has been subtracted, though that is unusual
578 for calibration. A refined background model is output. Ignored if
579 exposure is None.
580 @param[in] icSourceCat catalog from which to copy the fields specified
581 by icSourceKeys, or None;
582 @param[in] doUnpersist unpersist data:
583 - if True, exposure, background and icSourceCat are read from
584 dataRef and those three arguments must all be None;
585 - if False the exposure must be provided; background and
586 icSourceCat are optional. True is intended for running as a
587 command-line task, False for running as a subtask
588 @return same data as the calibrate method
589 """
590 self.log.info("Processing %s", dataRef.dataId)
592 if doUnpersist:
593 if any(item is not None for item in (exposure, background,
594 icSourceCat)):
595 raise RuntimeError("doUnpersist true; exposure, background "
596 "and icSourceCat must all be None")
597 exposure = dataRef.get("icExp", immediate=True)
598 background = dataRef.get("icExpBackground", immediate=True)
599 icSourceCat = dataRef.get("icSrc", immediate=True)
600 elif exposure is None:
601 raise RuntimeError("doUnpersist false; exposure must be provided")
603 exposureIdInfo = dataRef.get("expIdInfo")
605 calRes = self.run(
606 exposure=exposure,
607 exposureIdInfo=exposureIdInfo,
608 background=background,
609 icSourceCat=icSourceCat,
610 )
612 if self.config.doWrite:
613 self.writeOutputs(
614 dataRef=dataRef,
615 exposure=calRes.exposure,
616 background=calRes.background,
617 sourceCat=calRes.sourceCat,
618 astromMatches=calRes.astromMatches,
619 matchMeta=calRes.matchMeta,
620 )
622 return calRes
624 def runQuantum(self, butlerQC, inputRefs, outputRefs):
625 inputs = butlerQC.get(inputRefs)
626 inputs['exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "visit_detector")
628 if self.config.doAstrometry:
629 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
630 for ref in inputRefs.astromRefCat],
631 refCats=inputs.pop('astromRefCat'),
632 config=self.config.astromRefObjLoader, log=self.log)
633 self.astrometry.setRefObjLoader(refObjLoader)
635 if self.config.doPhotoCal:
636 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
637 for ref in inputRefs.photoRefCat],
638 refCats=inputs.pop('photoRefCat'),
639 config=self.config.photoRefObjLoader,
640 log=self.log)
641 self.photoCal.match.setRefObjLoader(photoRefObjLoader)
643 outputs = self.run(**inputs)
645 if self.config.doWriteMatches and self.config.doAstrometry:
646 normalizedMatches = afwTable.packMatches(outputs.astromMatches)
647 normalizedMatches.table.setMetadata(outputs.matchMeta)
648 if self.config.doWriteMatchesDenormalized:
649 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
650 outputs.matchesDenormalized = denormMatches
651 outputs.matches = normalizedMatches
652 butlerQC.put(outputs, outputRefs)
654 @timeMethod
655 def run(self, exposure, exposureIdInfo=None, background=None,
656 icSourceCat=None):
657 """!Calibrate an exposure (science image or coadd)
659 @param[in,out] exposure exposure to calibrate (an
660 lsst.afw.image.ExposureF or similar);
661 in:
662 - MaskedImage
663 - Psf
664 out:
665 - MaskedImage has background subtracted
666 - Wcs is replaced
667 - PhotoCalib is replaced
668 @param[in] exposureIdInfo ID info for exposure (an
669 lsst.obs.base.ExposureIdInfo) If not provided, returned
670 SourceCatalog IDs will not be globally unique.
671 @param[in,out] background background model already subtracted from
672 exposure (an lsst.afw.math.BackgroundList). May be None if no
673 background has been subtracted, though that is unusual for
674 calibration. A refined background model is output.
675 @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
676 from which we can copy some fields.
678 @return pipe_base Struct containing these fields:
679 - exposure calibrate science exposure with refined WCS and PhotoCalib
680 - background model of background subtracted from exposure (an
681 lsst.afw.math.BackgroundList)
682 - sourceCat catalog of measured sources
683 - astromMatches list of source/refObj matches from the astrometry
684 solver
685 """
686 # detect, deblend and measure sources
687 if exposureIdInfo is None:
688 exposureIdInfo = ExposureIdInfo()
690 if background is None:
691 background = BackgroundList()
692 sourceIdFactory = exposureIdInfo.makeSourceIdFactory()
693 table = SourceTable.make(self.schema, sourceIdFactory)
694 table.setMetadata(self.algMetadata)
696 detRes = self.detection.run(table=table, exposure=exposure,
697 doSmooth=True)
698 sourceCat = detRes.sources
699 if detRes.fpSets.background:
700 for bg in detRes.fpSets.background:
701 background.append(bg)
702 if self.config.doSkySources:
703 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId)
704 if skySourceFootprints:
705 for foot in skySourceFootprints:
706 s = sourceCat.addNew()
707 s.setFootprint(foot)
708 s.set(self.skySourceKey, True)
709 if self.config.doDeblend:
710 self.deblend.run(exposure=exposure, sources=sourceCat)
711 self.measurement.run(
712 measCat=sourceCat,
713 exposure=exposure,
714 exposureId=exposureIdInfo.expId
715 )
716 if self.config.doApCorr:
717 self.applyApCorr.run(
718 catalog=sourceCat,
719 apCorrMap=exposure.getInfo().getApCorrMap()
720 )
721 self.catalogCalculation.run(sourceCat)
723 self.setPrimaryFlags.run(sourceCat)
725 if icSourceCat is not None and \
726 len(self.config.icSourceFieldsToCopy) > 0:
727 self.copyIcSourceFields(icSourceCat=icSourceCat,
728 sourceCat=sourceCat)
730 # TODO DM-11568: this contiguous check-and-copy could go away if we
731 # reserve enough space during SourceDetection and/or SourceDeblend.
732 # NOTE: sourceSelectors require contiguous catalogs, so ensure
733 # contiguity now, so views are preserved from here on.
734 if not sourceCat.isContiguous():
735 sourceCat = sourceCat.copy(deep=True)
737 # perform astrometry calibration:
738 # fit an improved WCS and update the exposure's WCS in place
739 astromMatches = None
740 matchMeta = None
741 if self.config.doAstrometry:
742 try:
743 astromRes = self.astrometry.run(
744 exposure=exposure,
745 sourceCat=sourceCat,
746 )
747 astromMatches = astromRes.matches
748 matchMeta = astromRes.matchMeta
749 except Exception as e:
750 if self.config.requireAstrometry:
751 raise
752 self.log.warning("Unable to perform astrometric calibration "
753 "(%s): attempting to proceed", e)
755 # compute photometric calibration
756 if self.config.doPhotoCal:
757 try:
758 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
759 exposure.setPhotoCalib(photoRes.photoCalib)
760 # TODO: reword this to phrase it in terms of the calibration factor?
761 self.log.info("Photometric zero-point: %f",
762 photoRes.photoCalib.instFluxToMagnitude(1.0))
763 self.setMetadata(exposure=exposure, photoRes=photoRes)
764 except Exception as e:
765 if self.config.requirePhotoCal:
766 raise
767 self.log.warning("Unable to perform photometric calibration "
768 "(%s): attempting to proceed", e)
769 self.setMetadata(exposure=exposure, photoRes=None)
771 self.postCalibrationMeasurement.run(
772 measCat=sourceCat,
773 exposure=exposure,
774 exposureId=exposureIdInfo.expId
775 )
777 if self.config.doComputeSummaryStats:
778 summary = self.computeSummaryStats.run(exposure=exposure,
779 sources=sourceCat,
780 background=background)
781 exposure.getInfo().setSummaryStats(summary)
783 frame = getDebugFrame(self._display, "calibrate")
784 if frame:
785 displayAstrometry(
786 sourceCat=sourceCat,
787 exposure=exposure,
788 matches=astromMatches,
789 frame=frame,
790 pause=False,
791 )
793 return pipeBase.Struct(
794 exposure=exposure,
795 background=background,
796 sourceCat=sourceCat,
797 astromMatches=astromMatches,
798 matchMeta=matchMeta,
799 # These are duplicate entries with different names for use with
800 # gen3 middleware
801 outputExposure=exposure,
802 outputCat=sourceCat,
803 outputBackground=background,
804 )
806 def writeOutputs(self, dataRef, exposure, background, sourceCat,
807 astromMatches, matchMeta):
808 """Write output data to the output repository
810 @param[in] dataRef butler data reference corresponding to a science
811 image
812 @param[in] exposure exposure to write
813 @param[in] background background model for exposure
814 @param[in] sourceCat catalog of measured sources
815 @param[in] astromMatches list of source/refObj matches from the
816 astrometry solver
817 """
818 dataRef.put(sourceCat, "src")
819 if self.config.doWriteMatches and astromMatches is not None:
820 normalizedMatches = afwTable.packMatches(astromMatches)
821 normalizedMatches.table.setMetadata(matchMeta)
822 dataRef.put(normalizedMatches, "srcMatch")
823 if self.config.doWriteMatchesDenormalized:
824 denormMatches = denormalizeMatches(astromMatches, matchMeta)
825 dataRef.put(denormMatches, "srcMatchFull")
826 if self.config.doWriteExposure:
827 dataRef.put(exposure, "calexp")
828 dataRef.put(background, "calexpBackground")
830 def getSchemaCatalogs(self):
831 """Return a dict of empty catalogs for each catalog dataset produced
832 by this task.
833 """
834 sourceCat = afwTable.SourceCatalog(self.schema)
835 sourceCat.getTable().setMetadata(self.algMetadata)
836 return {"src": sourceCat}
838 def setMetadata(self, exposure, photoRes=None):
839 """!Set task and exposure metadata
841 Logs a warning and continues if needed data is missing.
843 @param[in,out] exposure exposure whose metadata is to be set
844 @param[in] photoRes results of running photoCal; if None then it was
845 not run
846 """
847 if photoRes is None:
848 return
850 metadata = exposure.getMetadata()
852 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata
853 try:
854 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
855 magZero = photoRes.zp - 2.5*math.log10(exposureTime)
856 except Exception:
857 self.log.warning("Could not set normalized MAGZERO in header: no "
858 "exposure time")
859 magZero = math.nan
861 try:
862 metadata.set('MAGZERO', magZero)
863 metadata.set('MAGZERO_RMS', photoRes.sigma)
864 metadata.set('MAGZERO_NOBJ', photoRes.ngood)
865 metadata.set('COLORTERM1', 0.0)
866 metadata.set('COLORTERM2', 0.0)
867 metadata.set('COLORTERM3', 0.0)
868 except Exception as e:
869 self.log.warning("Could not set exposure metadata: %s", e)
871 def copyIcSourceFields(self, icSourceCat, sourceCat):
872 """!Match sources in icSourceCat and sourceCat and copy the specified fields
874 @param[in] icSourceCat catalog from which to copy fields
875 @param[in,out] sourceCat catalog to which to copy fields
877 The fields copied are those specified by `config.icSourceFieldsToCopy`
878 that actually exist in the schema. This was set up by the constructor
879 using self.schemaMapper.
880 """
881 if self.schemaMapper is None:
882 raise RuntimeError("To copy icSource fields you must specify "
883 "icSourceSchema nd icSourceKeys when "
884 "constructing this task")
885 if icSourceCat is None or sourceCat is None:
886 raise RuntimeError("icSourceCat and sourceCat must both be "
887 "specified")
888 if len(self.config.icSourceFieldsToCopy) == 0:
889 self.log.warning("copyIcSourceFields doing nothing because "
890 "icSourceFieldsToCopy is empty")
891 return
893 mc = afwTable.MatchControl()
894 mc.findOnlyClosest = False # return all matched objects
895 matches = afwTable.matchXy(icSourceCat, sourceCat,
896 self.config.matchRadiusPix, mc)
897 if self.config.doDeblend:
898 deblendKey = sourceCat.schema["deblend_nChild"].asKey()
899 # if deblended, keep children
900 matches = [m for m in matches if m[1].get(deblendKey) == 0]
902 # Because we had to allow multiple matches to handle parents, we now
903 # need to prune to the best matches
904 # closest matches as a dict of icSourceCat source ID:
905 # (icSourceCat source, sourceCat source, distance in pixels)
906 bestMatches = {}
907 for m0, m1, d in matches:
908 id0 = m0.getId()
909 match = bestMatches.get(id0)
910 if match is None or d <= match[2]:
911 bestMatches[id0] = (m0, m1, d)
912 matches = list(bestMatches.values())
914 # Check that no sourceCat sources are listed twice (we already know
915 # that each match has a unique icSourceCat source ID, due to using
916 # that ID as the key in bestMatches)
917 numMatches = len(matches)
918 numUniqueSources = len(set(m[1].getId() for m in matches))
919 if numUniqueSources != numMatches:
920 self.log.warning("%d icSourceCat sources matched only %d sourceCat "
921 "sources", numMatches, numUniqueSources)
923 self.log.info("Copying flags from icSourceCat to sourceCat for "
924 "%d sources", numMatches)
926 # For each match: set the calibSourceKey flag and copy the desired
927 # fields
928 for icSrc, src, d in matches:
929 src.setFlag(self.calibSourceKey, True)
930 # src.assign copies the footprint from icSrc, which we don't want
931 # (DM-407)
932 # so set icSrc's footprint to src's footprint before src.assign,
933 # then restore it
934 icSrcFootprint = icSrc.getFootprint()
935 try:
936 icSrc.setFootprint(src.getFootprint())
937 src.assign(icSrc, self.schemaMapper)
938 finally:
939 icSrc.setFootprint(icSrcFootprint)