Coverage for python/lsst/pipe/tasks/calibrate.py: 17%
328 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-01 21:12 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-01 21:12 +0000
1#
2# LSST Data Management System
3# Copyright 2008-2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22import math
24from lsstDebug import getDebugFrame
25import lsst.pex.config as pexConfig
26import lsst.pipe.base as pipeBase
27import lsst.pipe.base.connectionTypes as cT
28import lsst.afw.table as afwTable
29from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches
30from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, SkyObjectsTask
31from lsst.obs.base import ExposureIdInfo
32import lsst.daf.base as dafBase
33from lsst.afw.math import BackgroundList
34from lsst.afw.table import SourceTable
35from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader
36from lsst.meas.base import (SingleFrameMeasurementTask,
37 ApplyApCorrTask,
38 CatalogCalculationTask)
39from lsst.meas.deblender import SourceDeblendTask
40from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask
41from .fakes import BaseFakeSourcesTask
42from .photoCal import PhotoCalTask
43from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask
46__all__ = ["CalibrateConfig", "CalibrateTask"]
49class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"),
50 defaultTemplates={}):
52 icSourceSchema = cT.InitInput(
53 doc="Schema produced by characterize image task, used to initialize this task",
54 name="icSrc_schema",
55 storageClass="SourceCatalog",
56 )
58 outputSchema = cT.InitOutput(
59 doc="Schema after CalibrateTask has been initialized",
60 name="src_schema",
61 storageClass="SourceCatalog",
62 )
64 exposure = cT.Input(
65 doc="Input image to calibrate",
66 name="icExp",
67 storageClass="ExposureF",
68 dimensions=("instrument", "visit", "detector"),
69 )
71 background = cT.Input(
72 doc="Backgrounds determined by characterize task",
73 name="icExpBackground",
74 storageClass="Background",
75 dimensions=("instrument", "visit", "detector"),
76 )
78 icSourceCat = cT.Input(
79 doc="Source catalog created by characterize task",
80 name="icSrc",
81 storageClass="SourceCatalog",
82 dimensions=("instrument", "visit", "detector"),
83 )
85 astromRefCat = cT.PrerequisiteInput(
86 doc="Reference catalog to use for astrometry",
87 name="cal_ref_cat",
88 storageClass="SimpleCatalog",
89 dimensions=("skypix",),
90 deferLoad=True,
91 multiple=True,
92 )
94 photoRefCat = cT.PrerequisiteInput(
95 doc="Reference catalog to use for photometric calibration",
96 name="cal_ref_cat",
97 storageClass="SimpleCatalog",
98 dimensions=("skypix",),
99 deferLoad=True,
100 multiple=True
101 )
103 outputExposure = cT.Output(
104 doc="Exposure after running calibration task",
105 name="calexp",
106 storageClass="ExposureF",
107 dimensions=("instrument", "visit", "detector"),
108 )
110 outputCat = cT.Output(
111 doc="Source catalog produced in calibrate task",
112 name="src",
113 storageClass="SourceCatalog",
114 dimensions=("instrument", "visit", "detector"),
115 )
117 outputBackground = cT.Output(
118 doc="Background models estimated in calibration task",
119 name="calexpBackground",
120 storageClass="Background",
121 dimensions=("instrument", "visit", "detector"),
122 )
124 matches = cT.Output(
125 doc="Source/refObj matches from the astrometry solver",
126 name="srcMatch",
127 storageClass="Catalog",
128 dimensions=("instrument", "visit", "detector"),
129 )
131 matchesDenormalized = cT.Output(
132 doc="Denormalized matches from astrometry solver",
133 name="srcMatchFull",
134 storageClass="Catalog",
135 dimensions=("instrument", "visit", "detector"),
136 )
138 def __init__(self, *, config=None):
139 super().__init__(config=config)
141 if config.doAstrometry is False:
142 self.prerequisiteInputs.remove("astromRefCat")
143 if config.doPhotoCal is False:
144 self.prerequisiteInputs.remove("photoRefCat")
146 if config.doWriteMatches is False or config.doAstrometry is False:
147 self.outputs.remove("matches")
148 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False:
149 self.outputs.remove("matchesDenormalized")
152class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections):
153 """Config for CalibrateTask"""
154 doWrite = pexConfig.Field(
155 dtype=bool,
156 default=True,
157 doc="Save calibration results?",
158 )
159 doWriteHeavyFootprintsInSources = pexConfig.Field(
160 dtype=bool,
161 default=True,
162 doc="Include HeavyFootprint data in source table? If false then heavy "
163 "footprints are saved as normal footprints, which saves some space"
164 )
165 doWriteMatches = pexConfig.Field(
166 dtype=bool,
167 default=True,
168 doc="Write reference matches (ignored if doWrite or doAstrometry false)?",
169 )
170 doWriteMatchesDenormalized = pexConfig.Field(
171 dtype=bool,
172 default=False,
173 doc=("Write reference matches in denormalized format? "
174 "This format uses more disk space, but is more convenient to "
175 "read. Ignored if doWriteMatches=False or doWrite=False."),
176 )
177 doAstrometry = pexConfig.Field(
178 dtype=bool,
179 default=True,
180 doc="Perform astrometric calibration?",
181 )
182 astromRefObjLoader = pexConfig.ConfigurableField(
183 target=LoadIndexedReferenceObjectsTask,
184 doc="reference object loader for astrometric calibration",
185 )
186 photoRefObjLoader = pexConfig.ConfigurableField(
187 target=LoadIndexedReferenceObjectsTask,
188 doc="reference object loader for photometric calibration",
189 )
190 astrometry = pexConfig.ConfigurableField(
191 target=AstrometryTask,
192 doc="Perform astrometric calibration to refine the WCS",
193 )
194 requireAstrometry = pexConfig.Field(
195 dtype=bool,
196 default=True,
197 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry "
198 "false."),
199 )
200 doPhotoCal = pexConfig.Field(
201 dtype=bool,
202 default=True,
203 doc="Perform phometric calibration?",
204 )
205 requirePhotoCal = pexConfig.Field(
206 dtype=bool,
207 default=True,
208 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal "
209 "false."),
210 )
211 photoCal = pexConfig.ConfigurableField(
212 target=PhotoCalTask,
213 doc="Perform photometric calibration",
214 )
215 icSourceFieldsToCopy = pexConfig.ListField(
216 dtype=str,
217 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
218 doc=("Fields to copy from the icSource catalog to the output catalog "
219 "for matching sources Any missing fields will trigger a "
220 "RuntimeError exception. Ignored if icSourceCat is not provided.")
221 )
222 matchRadiusPix = pexConfig.Field(
223 dtype=float,
224 default=3,
225 doc=("Match radius for matching icSourceCat objects to sourceCat "
226 "objects (pixels)"),
227 )
228 checkUnitsParseStrict = pexConfig.Field(
229 doc=("Strictness of Astropy unit compatibility check, can be 'raise', "
230 "'warn' or 'silent'"),
231 dtype=str,
232 default="raise",
233 )
234 detection = pexConfig.ConfigurableField(
235 target=SourceDetectionTask,
236 doc="Detect sources"
237 )
238 doDeblend = pexConfig.Field(
239 dtype=bool,
240 default=True,
241 doc="Run deblender input exposure"
242 )
243 deblend = pexConfig.ConfigurableField(
244 target=SourceDeblendTask,
245 doc="Split blended sources into their components"
246 )
247 doSkySources = pexConfig.Field(
248 dtype=bool,
249 default=True,
250 doc="Generate sky sources?",
251 )
252 skySources = pexConfig.ConfigurableField(
253 target=SkyObjectsTask,
254 doc="Generate sky sources",
255 )
256 measurement = pexConfig.ConfigurableField(
257 target=SingleFrameMeasurementTask,
258 doc="Measure sources"
259 )
260 setPrimaryFlags = pexConfig.ConfigurableField(
261 target=SetPrimaryFlagsTask,
262 doc=("Set flags for primary source classification in single frame "
263 "processing. True if sources are not sky sources and not a parent.")
264 )
265 doApCorr = pexConfig.Field(
266 dtype=bool,
267 default=True,
268 doc="Run subtask to apply aperture correction"
269 )
270 applyApCorr = pexConfig.ConfigurableField(
271 target=ApplyApCorrTask,
272 doc="Subtask to apply aperture corrections"
273 )
274 # If doApCorr is False, and the exposure does not have apcorrections
275 # already applied, the active plugins in catalogCalculation almost
276 # certainly should not contain the characterization plugin
277 catalogCalculation = pexConfig.ConfigurableField(
278 target=CatalogCalculationTask,
279 doc="Subtask to run catalogCalculation plugins on catalog"
280 )
281 doInsertFakes = pexConfig.Field(
282 dtype=bool,
283 default=False,
284 doc="Run fake sources injection task"
285 )
286 insertFakes = pexConfig.ConfigurableField(
287 target=BaseFakeSourcesTask,
288 doc="Injection of fake sources for testing purposes (must be "
289 "retargeted)"
290 )
291 doComputeSummaryStats = pexConfig.Field(
292 dtype=bool,
293 default=True,
294 doc="Run subtask to measure exposure summary statistics?"
295 )
296 computeSummaryStats = pexConfig.ConfigurableField(
297 target=ComputeExposureSummaryStatsTask,
298 doc="Subtask to run computeSummaryStats on exposure"
299 )
300 doWriteExposure = pexConfig.Field(
301 dtype=bool,
302 default=True,
303 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a "
304 "normal calexp but as a fakes_calexp."
305 )
307 def setDefaults(self):
308 super().setDefaults()
309 self.detection.doTempLocalBackground = False
310 self.deblend.maxFootprintSize = 2000
311 self.measurement.plugins.names |= ["base_LocalPhotoCalib", "base_LocalWcs"]
313 def validate(self):
314 super().validate()
315 astromRefCatGen2 = getattr(self.astromRefObjLoader, "ref_dataset_name", None)
316 if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat:
317 raise ValueError(
318 f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference "
319 f"catalogs are different. These options must be kept in sync until Gen2 is retired."
320 )
321 photoRefCatGen2 = getattr(self.photoRefObjLoader, "ref_dataset_name", None)
322 if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat:
323 raise ValueError(
324 f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference "
325 f"catalogs are different. These options must be kept in sync until Gen2 is retired."
326 )
329## \addtogroup LSST_task_documentation
330## \{
331## \page CalibrateTask
332## \ref CalibrateTask_ "CalibrateTask"
333## \copybrief CalibrateTask
334## \}
336class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
337 r"""!Calibrate an exposure: measure sources and perform astrometric and
338 photometric calibration
340 @anchor CalibrateTask_
342 @section pipe_tasks_calibrate_Contents Contents
344 - @ref pipe_tasks_calibrate_Purpose
345 - @ref pipe_tasks_calibrate_Initialize
346 - @ref pipe_tasks_calibrate_IO
347 - @ref pipe_tasks_calibrate_Config
348 - @ref pipe_tasks_calibrate_Metadata
349 - @ref pipe_tasks_calibrate_Debug
352 @section pipe_tasks_calibrate_Purpose Description
354 Given an exposure with a good PSF model and aperture correction map
355 (e.g. as provided by @ref CharacterizeImageTask), perform the following
356 operations:
357 - Run detection and measurement
358 - Run astrometry subtask to fit an improved WCS
359 - Run photoCal subtask to fit the exposure's photometric zero-point
361 @section pipe_tasks_calibrate_Initialize Task initialisation
363 @copydoc \_\_init\_\_
365 @section pipe_tasks_calibrate_IO Invoking the Task
367 If you want this task to unpersist inputs or persist outputs, then call
368 the `runDataRef` method (a wrapper around the `run` method).
370 If you already have the inputs unpersisted and do not want to persist the
371 output then it is more direct to call the `run` method:
373 @section pipe_tasks_calibrate_Config Configuration parameters
375 See @ref CalibrateConfig
377 @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata
379 Exposure metadata
380 <dl>
381 <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task
382 <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal
383 task
384 <dt>COLORTERM1 <dd>?? (always 0.0)
385 <dt>COLORTERM2 <dd>?? (always 0.0)
386 <dt>COLORTERM3 <dd>?? (always 0.0)
387 </dl>
389 @section pipe_tasks_calibrate_Debug Debug variables
391 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink
392 interface supports a flag
393 `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug
394 for more about `debug.py`.
396 CalibrateTask has a debug dictionary containing one key:
397 <dl>
398 <dt>calibrate
399 <dd>frame (an int; <= 0 to not display) in which to display the exposure,
400 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for
401 the meaning of the various symbols.
402 </dl>
404 For example, put something like:
405 @code{.py}
406 import lsstDebug
407 def DebugInfo(name):
408 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would
409 # call us recursively
410 if name == "lsst.pipe.tasks.calibrate":
411 di.display = dict(
412 calibrate = 1,
413 )
415 return di
417 lsstDebug.Info = DebugInfo
418 @endcode
419 into your `debug.py` file and run `calibrateTask.py` with the `--debug`
420 flag.
422 Some subtasks may have their own debug variables; see individual Task
423 documentation.
424 """
426 # Example description used to live here, removed 2-20-2017 as per
427 # https://jira.lsstcorp.org/browse/DM-9520
429 ConfigClass = CalibrateConfig
430 _DefaultName = "calibrate"
431 RunnerClass = pipeBase.ButlerInitializedTaskRunner
433 def __init__(self, butler=None, astromRefObjLoader=None,
434 photoRefObjLoader=None, icSourceSchema=None,
435 initInputs=None, **kwargs):
436 """!Construct a CalibrateTask
438 @param[in] butler The butler is passed to the refObjLoader constructor
439 in case it is needed. Ignored if the refObjLoader argument
440 provides a loader directly.
441 @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks
442 that supplies an external reference catalog for astrometric
443 calibration. May be None if the desired loader can be constructed
444 from the butler argument or all steps requiring a reference catalog
445 are disabled.
446 @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks
447 that supplies an external reference catalog for photometric
448 calibration. May be None if the desired loader can be constructed
449 from the butler argument or all steps requiring a reference catalog
450 are disabled.
451 @param[in] icSourceSchema schema for icSource catalog, or None.
452 Schema values specified in config.icSourceFieldsToCopy will be
453 taken from this schema. If set to None, no values will be
454 propagated from the icSourceCatalog
455 @param[in,out] kwargs other keyword arguments for
456 lsst.pipe.base.CmdLineTask
457 """
458 super().__init__(**kwargs)
460 if icSourceSchema is None and butler is not None:
461 # Use butler to read icSourceSchema from disk.
462 icSourceSchema = butler.get("icSrc_schema", immediate=True).schema
464 if icSourceSchema is None and butler is None and initInputs is not None:
465 icSourceSchema = initInputs['icSourceSchema'].schema
467 if icSourceSchema is not None:
468 # use a schema mapper to avoid copying each field separately
469 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema)
470 minimumSchema = afwTable.SourceTable.makeMinimalSchema()
471 self.schemaMapper.addMinimalSchema(minimumSchema, False)
473 # Add fields to copy from an icSource catalog
474 # and a field to indicate that the source matched a source in that
475 # catalog. If any fields are missing then raise an exception, but
476 # first find all missing fields in order to make the error message
477 # more useful.
478 self.calibSourceKey = self.schemaMapper.addOutputField(
479 afwTable.Field["Flag"]("calib_detected",
480 "Source was detected as an icSource"))
481 missingFieldNames = []
482 for fieldName in self.config.icSourceFieldsToCopy:
483 try:
484 schemaItem = icSourceSchema.find(fieldName)
485 except Exception:
486 missingFieldNames.append(fieldName)
487 else:
488 # field found; if addMapping fails then raise an exception
489 self.schemaMapper.addMapping(schemaItem.getKey())
491 if missingFieldNames:
492 raise RuntimeError("isSourceCat is missing fields {} "
493 "specified in icSourceFieldsToCopy"
494 .format(missingFieldNames))
496 # produce a temporary schema to pass to the subtasks; finalize it
497 # later
498 self.schema = self.schemaMapper.editOutputSchema()
499 else:
500 self.schemaMapper = None
501 self.schema = afwTable.SourceTable.makeMinimalSchema()
502 self.makeSubtask('detection', schema=self.schema)
504 self.algMetadata = dafBase.PropertyList()
506 # Only create a subtask for fakes if configuration option is set
507 # N.B. the config for fake object task must be retargeted to a child
508 # of BaseFakeSourcesTask
509 if self.config.doInsertFakes:
510 self.makeSubtask("insertFakes")
512 if self.config.doDeblend:
513 self.makeSubtask("deblend", schema=self.schema)
514 if self.config.doSkySources:
515 self.makeSubtask("skySources")
516 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
517 self.makeSubtask('measurement', schema=self.schema,
518 algMetadata=self.algMetadata)
519 self.makeSubtask("setPrimaryFlags", schema=self.schema, isSingleFrame=True)
520 if self.config.doApCorr:
521 self.makeSubtask('applyApCorr', schema=self.schema)
522 self.makeSubtask('catalogCalculation', schema=self.schema)
524 if self.config.doAstrometry:
525 if astromRefObjLoader is None and butler is not None:
526 self.makeSubtask('astromRefObjLoader', butler=butler)
527 astromRefObjLoader = self.astromRefObjLoader
528 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader,
529 schema=self.schema)
530 if self.config.doPhotoCal:
531 if photoRefObjLoader is None and butler is not None:
532 self.makeSubtask('photoRefObjLoader', butler=butler)
533 photoRefObjLoader = self.photoRefObjLoader
534 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader,
535 schema=self.schema)
536 if self.config.doComputeSummaryStats:
537 self.makeSubtask('computeSummaryStats')
539 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None):
540 raise RuntimeError("PipelineTask form of this task should not be initialized with "
541 "reference object loaders.")
543 if self.schemaMapper is not None:
544 # finalize the schema
545 self.schema = self.schemaMapper.getOutputSchema()
546 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
548 sourceCatSchema = afwTable.SourceCatalog(self.schema)
549 sourceCatSchema.getTable().setMetadata(self.algMetadata)
550 self.outputSchema = sourceCatSchema
552 @pipeBase.timeMethod
553 def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None,
554 doUnpersist=True):
555 """!Calibrate an exposure, optionally unpersisting inputs and
556 persisting outputs.
558 This is a wrapper around the `run` method that unpersists inputs
559 (if `doUnpersist` true) and persists outputs (if `config.doWrite` true)
561 @param[in] dataRef butler data reference corresponding to a science
562 image
563 @param[in,out] exposure characterized exposure (an
564 lsst.afw.image.ExposureF or similar), or None to unpersist existing
565 icExp and icBackground. See `run` method for details of what is
566 read and written.
567 @param[in,out] background initial model of background already
568 subtracted from exposure (an lsst.afw.math.BackgroundList). May be
569 None if no background has been subtracted, though that is unusual
570 for calibration. A refined background model is output. Ignored if
571 exposure is None.
572 @param[in] icSourceCat catalog from which to copy the fields specified
573 by icSourceKeys, or None;
574 @param[in] doUnpersist unpersist data:
575 - if True, exposure, background and icSourceCat are read from
576 dataRef and those three arguments must all be None;
577 - if False the exposure must be provided; background and
578 icSourceCat are optional. True is intended for running as a
579 command-line task, False for running as a subtask
580 @return same data as the calibrate method
581 """
582 self.log.info("Processing %s", dataRef.dataId)
584 if doUnpersist:
585 if any(item is not None for item in (exposure, background,
586 icSourceCat)):
587 raise RuntimeError("doUnpersist true; exposure, background "
588 "and icSourceCat must all be None")
589 exposure = dataRef.get("icExp", immediate=True)
590 background = dataRef.get("icExpBackground", immediate=True)
591 icSourceCat = dataRef.get("icSrc", immediate=True)
592 elif exposure is None:
593 raise RuntimeError("doUnpersist false; exposure must be provided")
595 exposureIdInfo = dataRef.get("expIdInfo")
597 calRes = self.run(
598 exposure=exposure,
599 exposureIdInfo=exposureIdInfo,
600 background=background,
601 icSourceCat=icSourceCat,
602 )
604 if self.config.doWrite:
605 self.writeOutputs(
606 dataRef=dataRef,
607 exposure=calRes.exposure,
608 background=calRes.background,
609 sourceCat=calRes.sourceCat,
610 astromMatches=calRes.astromMatches,
611 matchMeta=calRes.matchMeta,
612 )
614 return calRes
616 def runQuantum(self, butlerQC, inputRefs, outputRefs):
617 inputs = butlerQC.get(inputRefs)
618 inputs['exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "visit_detector")
620 if self.config.doAstrometry:
621 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
622 for ref in inputRefs.astromRefCat],
623 refCats=inputs.pop('astromRefCat'),
624 config=self.config.astromRefObjLoader, log=self.log)
625 self.astrometry.setRefObjLoader(refObjLoader)
627 if self.config.doPhotoCal:
628 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
629 for ref in inputRefs.photoRefCat],
630 refCats=inputs.pop('photoRefCat'),
631 config=self.config.photoRefObjLoader,
632 log=self.log)
633 self.photoCal.match.setRefObjLoader(photoRefObjLoader)
635 outputs = self.run(**inputs)
637 if self.config.doWriteMatches and self.config.doAstrometry:
638 normalizedMatches = afwTable.packMatches(outputs.astromMatches)
639 normalizedMatches.table.setMetadata(outputs.matchMeta)
640 if self.config.doWriteMatchesDenormalized:
641 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
642 outputs.matchesDenormalized = denormMatches
643 outputs.matches = normalizedMatches
644 butlerQC.put(outputs, outputRefs)
646 @pipeBase.timeMethod
647 def run(self, exposure, exposureIdInfo=None, background=None,
648 icSourceCat=None):
649 """!Calibrate an exposure (science image or coadd)
651 @param[in,out] exposure exposure to calibrate (an
652 lsst.afw.image.ExposureF or similar);
653 in:
654 - MaskedImage
655 - Psf
656 out:
657 - MaskedImage has background subtracted
658 - Wcs is replaced
659 - PhotoCalib is replaced
660 @param[in] exposureIdInfo ID info for exposure (an
661 lsst.obs.base.ExposureIdInfo) If not provided, returned
662 SourceCatalog IDs will not be globally unique.
663 @param[in,out] background background model already subtracted from
664 exposure (an lsst.afw.math.BackgroundList). May be None if no
665 background has been subtracted, though that is unusual for
666 calibration. A refined background model is output.
667 @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
668 from which we can copy some fields.
670 @return pipe_base Struct containing these fields:
671 - exposure calibrate science exposure with refined WCS and PhotoCalib
672 - background model of background subtracted from exposure (an
673 lsst.afw.math.BackgroundList)
674 - sourceCat catalog of measured sources
675 - astromMatches list of source/refObj matches from the astrometry
676 solver
677 """
678 # detect, deblend and measure sources
679 if exposureIdInfo is None:
680 exposureIdInfo = ExposureIdInfo()
682 if background is None:
683 background = BackgroundList()
684 sourceIdFactory = exposureIdInfo.makeSourceIdFactory()
685 table = SourceTable.make(self.schema, sourceIdFactory)
686 table.setMetadata(self.algMetadata)
688 detRes = self.detection.run(table=table, exposure=exposure,
689 doSmooth=True)
690 sourceCat = detRes.sources
691 if detRes.fpSets.background:
692 for bg in detRes.fpSets.background:
693 background.append(bg)
694 if self.config.doSkySources:
695 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId)
696 if skySourceFootprints:
697 for foot in skySourceFootprints:
698 s = sourceCat.addNew()
699 s.setFootprint(foot)
700 s.set(self.skySourceKey, True)
701 if self.config.doDeblend:
702 self.deblend.run(exposure=exposure, sources=sourceCat)
703 self.measurement.run(
704 measCat=sourceCat,
705 exposure=exposure,
706 exposureId=exposureIdInfo.expId
707 )
708 if self.config.doApCorr:
709 self.applyApCorr.run(
710 catalog=sourceCat,
711 apCorrMap=exposure.getInfo().getApCorrMap()
712 )
713 self.catalogCalculation.run(sourceCat)
715 self.setPrimaryFlags.run(sourceCat)
717 if icSourceCat is not None and \
718 len(self.config.icSourceFieldsToCopy) > 0:
719 self.copyIcSourceFields(icSourceCat=icSourceCat,
720 sourceCat=sourceCat)
722 # TODO DM-11568: this contiguous check-and-copy could go away if we
723 # reserve enough space during SourceDetection and/or SourceDeblend.
724 # NOTE: sourceSelectors require contiguous catalogs, so ensure
725 # contiguity now, so views are preserved from here on.
726 if not sourceCat.isContiguous():
727 sourceCat = sourceCat.copy(deep=True)
729 # perform astrometry calibration:
730 # fit an improved WCS and update the exposure's WCS in place
731 astromMatches = None
732 matchMeta = None
733 if self.config.doAstrometry:
734 try:
735 astromRes = self.astrometry.run(
736 exposure=exposure,
737 sourceCat=sourceCat,
738 )
739 astromMatches = astromRes.matches
740 matchMeta = astromRes.matchMeta
741 except Exception as e:
742 if self.config.requireAstrometry:
743 raise
744 self.log.warning("Unable to perform astrometric calibration "
745 "(%s): attempting to proceed", e)
747 # compute photometric calibration
748 if self.config.doPhotoCal:
749 try:
750 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
751 exposure.setPhotoCalib(photoRes.photoCalib)
752 # TODO: reword this to phrase it in terms of the calibration factor?
753 self.log.info("Photometric zero-point: %f",
754 photoRes.photoCalib.instFluxToMagnitude(1.0))
755 self.setMetadata(exposure=exposure, photoRes=photoRes)
756 except Exception as e:
757 if self.config.requirePhotoCal:
758 raise
759 self.log.warning("Unable to perform photometric calibration "
760 "(%s): attempting to proceed", e)
761 self.setMetadata(exposure=exposure, photoRes=None)
763 if self.config.doInsertFakes:
764 self.insertFakes.run(exposure, background=background)
766 table = SourceTable.make(self.schema, sourceIdFactory)
767 table.setMetadata(self.algMetadata)
769 detRes = self.detection.run(table=table, exposure=exposure,
770 doSmooth=True)
771 sourceCat = detRes.sources
772 if detRes.fpSets.background:
773 for bg in detRes.fpSets.background:
774 background.append(bg)
775 if self.config.doDeblend:
776 self.deblend.run(exposure=exposure, sources=sourceCat)
777 self.measurement.run(
778 measCat=sourceCat,
779 exposure=exposure,
780 exposureId=exposureIdInfo.expId
781 )
782 if self.config.doApCorr:
783 self.applyApCorr.run(
784 catalog=sourceCat,
785 apCorrMap=exposure.getInfo().getApCorrMap()
786 )
787 self.catalogCalculation.run(sourceCat)
789 if icSourceCat is not None and len(self.config.icSourceFieldsToCopy) > 0:
790 self.copyIcSourceFields(icSourceCat=icSourceCat,
791 sourceCat=sourceCat)
793 if self.config.doComputeSummaryStats:
794 summary = self.computeSummaryStats.run(exposure=exposure,
795 sources=sourceCat,
796 background=background)
797 exposure.getInfo().setSummaryStats(summary)
799 frame = getDebugFrame(self._display, "calibrate")
800 if frame:
801 displayAstrometry(
802 sourceCat=sourceCat,
803 exposure=exposure,
804 matches=astromMatches,
805 frame=frame,
806 pause=False,
807 )
809 return pipeBase.Struct(
810 exposure=exposure,
811 background=background,
812 sourceCat=sourceCat,
813 astromMatches=astromMatches,
814 matchMeta=matchMeta,
815 # These are duplicate entries with different names for use with
816 # gen3 middleware
817 outputExposure=exposure,
818 outputCat=sourceCat,
819 outputBackground=background,
820 )
822 def writeOutputs(self, dataRef, exposure, background, sourceCat,
823 astromMatches, matchMeta):
824 """Write output data to the output repository
826 @param[in] dataRef butler data reference corresponding to a science
827 image
828 @param[in] exposure exposure to write
829 @param[in] background background model for exposure
830 @param[in] sourceCat catalog of measured sources
831 @param[in] astromMatches list of source/refObj matches from the
832 astrometry solver
833 """
834 dataRef.put(sourceCat, "src")
835 if self.config.doWriteMatches and astromMatches is not None:
836 normalizedMatches = afwTable.packMatches(astromMatches)
837 normalizedMatches.table.setMetadata(matchMeta)
838 dataRef.put(normalizedMatches, "srcMatch")
839 if self.config.doWriteMatchesDenormalized:
840 denormMatches = denormalizeMatches(astromMatches, matchMeta)
841 dataRef.put(denormMatches, "srcMatchFull")
842 if self.config.doWriteExposure:
843 dataRef.put(exposure, "calexp")
844 dataRef.put(background, "calexpBackground")
846 def getSchemaCatalogs(self):
847 """Return a dict of empty catalogs for each catalog dataset produced
848 by this task.
849 """
850 sourceCat = afwTable.SourceCatalog(self.schema)
851 sourceCat.getTable().setMetadata(self.algMetadata)
852 return {"src": sourceCat}
854 def setMetadata(self, exposure, photoRes=None):
855 """!Set task and exposure metadata
857 Logs a warning and continues if needed data is missing.
859 @param[in,out] exposure exposure whose metadata is to be set
860 @param[in] photoRes results of running photoCal; if None then it was
861 not run
862 """
863 if photoRes is None:
864 return
866 metadata = exposure.getMetadata()
868 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata
869 try:
870 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
871 magZero = photoRes.zp - 2.5*math.log10(exposureTime)
872 except Exception:
873 self.log.warning("Could not set normalized MAGZERO in header: no "
874 "exposure time")
875 magZero = math.nan
877 try:
878 metadata.set('MAGZERO', magZero)
879 metadata.set('MAGZERO_RMS', photoRes.sigma)
880 metadata.set('MAGZERO_NOBJ', photoRes.ngood)
881 metadata.set('COLORTERM1', 0.0)
882 metadata.set('COLORTERM2', 0.0)
883 metadata.set('COLORTERM3', 0.0)
884 except Exception as e:
885 self.log.warning("Could not set exposure metadata: %s", e)
887 def copyIcSourceFields(self, icSourceCat, sourceCat):
888 """!Match sources in icSourceCat and sourceCat and copy the specified fields
890 @param[in] icSourceCat catalog from which to copy fields
891 @param[in,out] sourceCat catalog to which to copy fields
893 The fields copied are those specified by `config.icSourceFieldsToCopy`
894 that actually exist in the schema. This was set up by the constructor
895 using self.schemaMapper.
896 """
897 if self.schemaMapper is None:
898 raise RuntimeError("To copy icSource fields you must specify "
899 "icSourceSchema nd icSourceKeys when "
900 "constructing this task")
901 if icSourceCat is None or sourceCat is None:
902 raise RuntimeError("icSourceCat and sourceCat must both be "
903 "specified")
904 if len(self.config.icSourceFieldsToCopy) == 0:
905 self.log.warning("copyIcSourceFields doing nothing because "
906 "icSourceFieldsToCopy is empty")
907 return
909 mc = afwTable.MatchControl()
910 mc.findOnlyClosest = False # return all matched objects
911 matches = afwTable.matchXy(icSourceCat, sourceCat,
912 self.config.matchRadiusPix, mc)
913 if self.config.doDeblend:
914 deblendKey = sourceCat.schema["deblend_nChild"].asKey()
915 # if deblended, keep children
916 matches = [m for m in matches if m[1].get(deblendKey) == 0]
918 # Because we had to allow multiple matches to handle parents, we now
919 # need to prune to the best matches
920 # closest matches as a dict of icSourceCat source ID:
921 # (icSourceCat source, sourceCat source, distance in pixels)
922 bestMatches = {}
923 for m0, m1, d in matches:
924 id0 = m0.getId()
925 match = bestMatches.get(id0)
926 if match is None or d <= match[2]:
927 bestMatches[id0] = (m0, m1, d)
928 matches = list(bestMatches.values())
930 # Check that no sourceCat sources are listed twice (we already know
931 # that each match has a unique icSourceCat source ID, due to using
932 # that ID as the key in bestMatches)
933 numMatches = len(matches)
934 numUniqueSources = len(set(m[1].getId() for m in matches))
935 if numUniqueSources != numMatches:
936 self.log.warning("%d icSourceCat sources matched only %d sourceCat "
937 "sources", numMatches, numUniqueSources)
939 self.log.info("Copying flags from icSourceCat to sourceCat for "
940 "%d sources", numMatches)
942 # For each match: set the calibSourceKey flag and copy the desired
943 # fields
944 for icSrc, src, d in matches:
945 src.setFlag(self.calibSourceKey, True)
946 # src.assign copies the footprint from icSrc, which we don't want
947 # (DM-407)
948 # so set icSrc's footprint to src's footprint before src.assign,
949 # then restore it
950 icSrcFootprint = icSrc.getFootprint()
951 try:
952 icSrc.setFootprint(src.getFootprint())
953 src.assign(icSrc, self.schemaMapper)
954 finally:
955 icSrc.setFootprint(icSrcFootprint)