Coverage for python/lsst/pipe/tasks/calibrate.py : 17%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008-2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22import math
24from lsstDebug import getDebugFrame
25import lsst.pex.config as pexConfig
26import lsst.pipe.base as pipeBase
27import lsst.pipe.base.connectionTypes as cT
28import lsst.afw.table as afwTable
29from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches
30from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
31from lsst.obs.base import ExposureIdInfo
32import lsst.daf.base as dafBase
33from lsst.afw.math import BackgroundList
34from lsst.afw.table import IdFactory, SourceTable
35from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader
36from lsst.meas.base import (SingleFrameMeasurementTask,
37 ApplyApCorrTask,
38 CatalogCalculationTask)
39from lsst.meas.deblender import SourceDeblendTask
40from .fakes import BaseFakeSourcesTask
41from .photoCal import PhotoCalTask
43__all__ = ["CalibrateConfig", "CalibrateTask"]
46class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"),
47 defaultTemplates={}):
49 icSourceSchema = cT.InitInput(
50 doc="Schema produced by characterize image task, used to initialize this task",
51 name="icSrc_schema",
52 storageClass="SourceCatalog",
53 )
55 outputSchema = cT.InitOutput(
56 doc="Schema after CalibrateTask has been initialized",
57 name="src_schema",
58 storageClass="SourceCatalog",
59 )
61 exposure = cT.Input(
62 doc="Input image to calibrate",
63 name="icExp",
64 storageClass="ExposureF",
65 dimensions=("instrument", "visit", "detector"),
66 )
68 background = cT.Input(
69 doc="Backgrounds determined by characterize task",
70 name="icExpBackground",
71 storageClass="Background",
72 dimensions=("instrument", "visit", "detector"),
73 )
75 icSourceCat = cT.Input(
76 doc="Source catalog created by characterize task",
77 name="icSrc",
78 storageClass="SourceCatalog",
79 dimensions=("instrument", "visit", "detector"),
80 )
82 astromRefCat = cT.PrerequisiteInput(
83 doc="Reference catalog to use for astrometry",
84 name="cal_ref_cat",
85 storageClass="SimpleCatalog",
86 dimensions=("skypix",),
87 deferLoad=True,
88 multiple=True,
89 )
91 photoRefCat = cT.PrerequisiteInput(
92 doc="Reference catalog to use for photometric calibration",
93 name="cal_ref_cat",
94 storageClass="SimpleCatalog",
95 dimensions=("skypix",),
96 deferLoad=True,
97 multiple=True
98 )
100 outputExposure = cT.Output(
101 doc="Exposure after running calibration task",
102 name="calexp",
103 storageClass="ExposureF",
104 dimensions=("instrument", "visit", "detector"),
105 )
107 outputCat = cT.Output(
108 doc="Source catalog produced in calibrate task",
109 name="src",
110 storageClass="SourceCatalog",
111 dimensions=("instrument", "visit", "detector"),
112 )
114 outputBackground = cT.Output(
115 doc="Background models estimated in calibration task",
116 name="calexpBackground",
117 storageClass="Background",
118 dimensions=("instrument", "visit", "detector"),
119 )
121 matches = cT.Output(
122 doc="Source/refObj matches from the astrometry solver",
123 name="srcMatch",
124 storageClass="Catalog",
125 dimensions=("instrument", "visit", "detector"),
126 )
128 matchesDenormalized = cT.Output(
129 doc="Denormalized matches from astrometry solver",
130 name="srcMatchFull",
131 storageClass="Catalog",
132 dimensions=("instrument", "visit", "detector"),
133 )
135 def __init__(self, *, config=None):
136 super().__init__(config=config)
138 if config.doAstrometry is False:
139 self.prerequisiteInputs.remove("astromRefCat")
140 if config.doPhotoCal is False:
141 self.prerequisiteInputs.remove("photoRefCat")
143 if config.doWriteMatches is False or config.doAstrometry is False:
144 self.outputs.remove("matches")
145 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False:
146 self.outputs.remove("matchesDenormalized")
149class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections):
150 """Config for CalibrateTask"""
151 doWrite = pexConfig.Field(
152 dtype=bool,
153 default=True,
154 doc="Save calibration results?",
155 )
156 doWriteHeavyFootprintsInSources = pexConfig.Field(
157 dtype=bool,
158 default=True,
159 doc="Include HeavyFootprint data in source table? If false then heavy "
160 "footprints are saved as normal footprints, which saves some space"
161 )
162 doWriteMatches = pexConfig.Field(
163 dtype=bool,
164 default=True,
165 doc="Write reference matches (ignored if doWrite or doAstrometry false)?",
166 )
167 doWriteMatchesDenormalized = pexConfig.Field(
168 dtype=bool,
169 default=False,
170 doc=("Write reference matches in denormalized format? "
171 "This format uses more disk space, but is more convenient to "
172 "read. Ignored if doWriteMatches=False or doWrite=False."),
173 )
174 doAstrometry = pexConfig.Field(
175 dtype=bool,
176 default=True,
177 doc="Perform astrometric calibration?",
178 )
179 astromRefObjLoader = pexConfig.ConfigurableField(
180 target=LoadIndexedReferenceObjectsTask,
181 doc="reference object loader for astrometric calibration",
182 )
183 photoRefObjLoader = pexConfig.ConfigurableField(
184 target=LoadIndexedReferenceObjectsTask,
185 doc="reference object loader for photometric calibration",
186 )
187 astrometry = pexConfig.ConfigurableField(
188 target=AstrometryTask,
189 doc="Perform astrometric calibration to refine the WCS",
190 )
191 requireAstrometry = pexConfig.Field(
192 dtype=bool,
193 default=True,
194 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry "
195 "false."),
196 )
197 doPhotoCal = pexConfig.Field(
198 dtype=bool,
199 default=True,
200 doc="Perform phometric calibration?",
201 )
202 requirePhotoCal = pexConfig.Field(
203 dtype=bool,
204 default=True,
205 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal "
206 "false."),
207 )
208 photoCal = pexConfig.ConfigurableField(
209 target=PhotoCalTask,
210 doc="Perform photometric calibration",
211 )
212 icSourceFieldsToCopy = pexConfig.ListField(
213 dtype=str,
214 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
215 doc=("Fields to copy from the icSource catalog to the output catalog "
216 "for matching sources Any missing fields will trigger a "
217 "RuntimeError exception. Ignored if icSourceCat is not provided.")
218 )
219 matchRadiusPix = pexConfig.Field(
220 dtype=float,
221 default=3,
222 doc=("Match radius for matching icSourceCat objects to sourceCat "
223 "objects (pixels)"),
224 )
225 checkUnitsParseStrict = pexConfig.Field(
226 doc=("Strictness of Astropy unit compatibility check, can be 'raise', "
227 "'warn' or 'silent'"),
228 dtype=str,
229 default="raise",
230 )
231 detection = pexConfig.ConfigurableField(
232 target=SourceDetectionTask,
233 doc="Detect sources"
234 )
235 doDeblend = pexConfig.Field(
236 dtype=bool,
237 default=True,
238 doc="Run deblender input exposure"
239 )
240 deblend = pexConfig.ConfigurableField(
241 target=SourceDeblendTask,
242 doc="Split blended sources into their components"
243 )
244 measurement = pexConfig.ConfigurableField(
245 target=SingleFrameMeasurementTask,
246 doc="Measure sources"
247 )
248 doApCorr = pexConfig.Field(
249 dtype=bool,
250 default=True,
251 doc="Run subtask to apply aperture correction"
252 )
253 applyApCorr = pexConfig.ConfigurableField(
254 target=ApplyApCorrTask,
255 doc="Subtask to apply aperture corrections"
256 )
257 # If doApCorr is False, and the exposure does not have apcorrections
258 # already applied, the active plugins in catalogCalculation almost
259 # certainly should not contain the characterization plugin
260 catalogCalculation = pexConfig.ConfigurableField(
261 target=CatalogCalculationTask,
262 doc="Subtask to run catalogCalculation plugins on catalog"
263 )
264 doInsertFakes = pexConfig.Field(
265 dtype=bool,
266 default=False,
267 doc="Run fake sources injection task"
268 )
269 insertFakes = pexConfig.ConfigurableField(
270 target=BaseFakeSourcesTask,
271 doc="Injection of fake sources for testing purposes (must be "
272 "retargeted)"
273 )
274 doWriteExposure = pexConfig.Field(
275 dtype=bool,
276 default=True,
277 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a "
278 "normal calexp but as a fakes_calexp."
279 )
281 def setDefaults(self):
282 super().setDefaults()
283 self.detection.doTempLocalBackground = False
284 self.deblend.maxFootprintSize = 2000
285 self.measurement.plugins.names |= ["base_LocalPhotoCalib", "base_LocalWcs"]
287 def validate(self):
288 super().validate()
289 astromRefCatGen2 = getattr(self.astromRefObjLoader, "ref_dataset_name", None)
290 if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat:
291 raise ValueError(
292 f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference "
293 f"catalogs are different. These options must be kept in sync until Gen2 is retired."
294 )
295 photoRefCatGen2 = getattr(self.photoRefObjLoader, "ref_dataset_name", None)
296 if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat:
297 raise ValueError(
298 f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference "
299 f"catalogs are different. These options must be kept in sync until Gen2 is retired."
300 )
303## \addtogroup LSST_task_documentation
304## \{
305## \page CalibrateTask
306## \ref CalibrateTask_ "CalibrateTask"
307## \copybrief CalibrateTask
308## \}
310class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
311 r"""!Calibrate an exposure: measure sources and perform astrometric and
312 photometric calibration
314 @anchor CalibrateTask_
316 @section pipe_tasks_calibrate_Contents Contents
318 - @ref pipe_tasks_calibrate_Purpose
319 - @ref pipe_tasks_calibrate_Initialize
320 - @ref pipe_tasks_calibrate_IO
321 - @ref pipe_tasks_calibrate_Config
322 - @ref pipe_tasks_calibrate_Metadata
323 - @ref pipe_tasks_calibrate_Debug
326 @section pipe_tasks_calibrate_Purpose Description
328 Given an exposure with a good PSF model and aperture correction map
329 (e.g. as provided by @ref CharacterizeImageTask), perform the following
330 operations:
331 - Run detection and measurement
332 - Run astrometry subtask to fit an improved WCS
333 - Run photoCal subtask to fit the exposure's photometric zero-point
335 @section pipe_tasks_calibrate_Initialize Task initialisation
337 @copydoc \_\_init\_\_
339 @section pipe_tasks_calibrate_IO Invoking the Task
341 If you want this task to unpersist inputs or persist outputs, then call
342 the `runDataRef` method (a wrapper around the `run` method).
344 If you already have the inputs unpersisted and do not want to persist the
345 output then it is more direct to call the `run` method:
347 @section pipe_tasks_calibrate_Config Configuration parameters
349 See @ref CalibrateConfig
351 @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata
353 Exposure metadata
354 <dl>
355 <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task
356 <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal
357 task
358 <dt>COLORTERM1 <dd>?? (always 0.0)
359 <dt>COLORTERM2 <dd>?? (always 0.0)
360 <dt>COLORTERM3 <dd>?? (always 0.0)
361 </dl>
363 @section pipe_tasks_calibrate_Debug Debug variables
365 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink
366 interface supports a flag
367 `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug
368 for more about `debug.py`.
370 CalibrateTask has a debug dictionary containing one key:
371 <dl>
372 <dt>calibrate
373 <dd>frame (an int; <= 0 to not display) in which to display the exposure,
374 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for
375 the meaning of the various symbols.
376 </dl>
378 For example, put something like:
379 @code{.py}
380 import lsstDebug
381 def DebugInfo(name):
382 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would
383 # call us recursively
384 if name == "lsst.pipe.tasks.calibrate":
385 di.display = dict(
386 calibrate = 1,
387 )
389 return di
391 lsstDebug.Info = DebugInfo
392 @endcode
393 into your `debug.py` file and run `calibrateTask.py` with the `--debug`
394 flag.
396 Some subtasks may have their own debug variables; see individual Task
397 documentation.
398 """
400 # Example description used to live here, removed 2-20-2017 as per
401 # https://jira.lsstcorp.org/browse/DM-9520
403 ConfigClass = CalibrateConfig
404 _DefaultName = "calibrate"
405 RunnerClass = pipeBase.ButlerInitializedTaskRunner
407 def __init__(self, butler=None, astromRefObjLoader=None,
408 photoRefObjLoader=None, icSourceSchema=None,
409 initInputs=None, **kwargs):
410 """!Construct a CalibrateTask
412 @param[in] butler The butler is passed to the refObjLoader constructor
413 in case it is needed. Ignored if the refObjLoader argument
414 provides a loader directly.
415 @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks
416 that supplies an external reference catalog for astrometric
417 calibration. May be None if the desired loader can be constructed
418 from the butler argument or all steps requiring a reference catalog
419 are disabled.
420 @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks
421 that supplies an external reference catalog for photometric
422 calibration. May be None if the desired loader can be constructed
423 from the butler argument or all steps requiring a reference catalog
424 are disabled.
425 @param[in] icSourceSchema schema for icSource catalog, or None.
426 Schema values specified in config.icSourceFieldsToCopy will be
427 taken from this schema. If set to None, no values will be
428 propagated from the icSourceCatalog
429 @param[in,out] kwargs other keyword arguments for
430 lsst.pipe.base.CmdLineTask
431 """
432 super().__init__(**kwargs)
434 if icSourceSchema is None and butler is not None:
435 # Use butler to read icSourceSchema from disk.
436 icSourceSchema = butler.get("icSrc_schema", immediate=True).schema
438 if icSourceSchema is None and butler is None and initInputs is not None:
439 icSourceSchema = initInputs['icSourceSchema'].schema
441 if icSourceSchema is not None:
442 # use a schema mapper to avoid copying each field separately
443 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema)
444 minimumSchema = afwTable.SourceTable.makeMinimalSchema()
445 self.schemaMapper.addMinimalSchema(minimumSchema, False)
447 # Add fields to copy from an icSource catalog
448 # and a field to indicate that the source matched a source in that
449 # catalog. If any fields are missing then raise an exception, but
450 # first find all missing fields in order to make the error message
451 # more useful.
452 self.calibSourceKey = self.schemaMapper.addOutputField(
453 afwTable.Field["Flag"]("calib_detected",
454 "Source was detected as an icSource"))
455 missingFieldNames = []
456 for fieldName in self.config.icSourceFieldsToCopy:
457 try:
458 schemaItem = icSourceSchema.find(fieldName)
459 except Exception:
460 missingFieldNames.append(fieldName)
461 else:
462 # field found; if addMapping fails then raise an exception
463 self.schemaMapper.addMapping(schemaItem.getKey())
465 if missingFieldNames:
466 raise RuntimeError("isSourceCat is missing fields {} "
467 "specified in icSourceFieldsToCopy"
468 .format(missingFieldNames))
470 # produce a temporary schema to pass to the subtasks; finalize it
471 # later
472 self.schema = self.schemaMapper.editOutputSchema()
473 else:
474 self.schemaMapper = None
475 self.schema = afwTable.SourceTable.makeMinimalSchema()
476 self.makeSubtask('detection', schema=self.schema)
478 self.algMetadata = dafBase.PropertyList()
480 # Only create a subtask for fakes if configuration option is set
481 # N.B. the config for fake object task must be retargeted to a child
482 # of BaseFakeSourcesTask
483 if self.config.doInsertFakes:
484 self.makeSubtask("insertFakes")
486 if self.config.doDeblend:
487 self.makeSubtask("deblend", schema=self.schema)
488 self.makeSubtask('measurement', schema=self.schema,
489 algMetadata=self.algMetadata)
490 if self.config.doApCorr:
491 self.makeSubtask('applyApCorr', schema=self.schema)
492 self.makeSubtask('catalogCalculation', schema=self.schema)
494 if self.config.doAstrometry:
495 if astromRefObjLoader is None and butler is not None:
496 self.makeSubtask('astromRefObjLoader', butler=butler)
497 astromRefObjLoader = self.astromRefObjLoader
498 self.pixelMargin = astromRefObjLoader.config.pixelMargin
499 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader,
500 schema=self.schema)
501 if self.config.doPhotoCal:
502 if photoRefObjLoader is None and butler is not None:
503 self.makeSubtask('photoRefObjLoader', butler=butler)
504 photoRefObjLoader = self.photoRefObjLoader
505 self.pixelMargin = photoRefObjLoader.config.pixelMargin
506 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader,
507 schema=self.schema)
509 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None):
510 raise RuntimeError("PipelineTask form of this task should not be initialized with "
511 "reference object loaders.")
513 if self.schemaMapper is not None:
514 # finalize the schema
515 self.schema = self.schemaMapper.getOutputSchema()
516 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
518 sourceCatSchema = afwTable.SourceCatalog(self.schema)
519 sourceCatSchema.getTable().setMetadata(self.algMetadata)
520 self.outputSchema = sourceCatSchema
522 @pipeBase.timeMethod
523 def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None,
524 doUnpersist=True):
525 """!Calibrate an exposure, optionally unpersisting inputs and
526 persisting outputs.
528 This is a wrapper around the `run` method that unpersists inputs
529 (if `doUnpersist` true) and persists outputs (if `config.doWrite` true)
531 @param[in] dataRef butler data reference corresponding to a science
532 image
533 @param[in,out] exposure characterized exposure (an
534 lsst.afw.image.ExposureF or similar), or None to unpersist existing
535 icExp and icBackground. See `run` method for details of what is
536 read and written.
537 @param[in,out] background initial model of background already
538 subtracted from exposure (an lsst.afw.math.BackgroundList). May be
539 None if no background has been subtracted, though that is unusual
540 for calibration. A refined background model is output. Ignored if
541 exposure is None.
542 @param[in] icSourceCat catalog from which to copy the fields specified
543 by icSourceKeys, or None;
544 @param[in] doUnpersist unpersist data:
545 - if True, exposure, background and icSourceCat are read from
546 dataRef and those three arguments must all be None;
547 - if False the exposure must be provided; background and
548 icSourceCat are optional. True is intended for running as a
549 command-line task, False for running as a subtask
550 @return same data as the calibrate method
551 """
552 self.log.info("Processing %s" % (dataRef.dataId))
554 if doUnpersist:
555 if any(item is not None for item in (exposure, background,
556 icSourceCat)):
557 raise RuntimeError("doUnpersist true; exposure, background "
558 "and icSourceCat must all be None")
559 exposure = dataRef.get("icExp", immediate=True)
560 background = dataRef.get("icExpBackground", immediate=True)
561 icSourceCat = dataRef.get("icSrc", immediate=True)
562 elif exposure is None:
563 raise RuntimeError("doUnpersist false; exposure must be provided")
565 exposureIdInfo = dataRef.get("expIdInfo")
567 calRes = self.run(
568 exposure=exposure,
569 exposureIdInfo=exposureIdInfo,
570 background=background,
571 icSourceCat=icSourceCat,
572 )
574 if self.config.doWrite:
575 self.writeOutputs(
576 dataRef=dataRef,
577 exposure=calRes.exposure,
578 background=calRes.background,
579 sourceCat=calRes.sourceCat,
580 astromMatches=calRes.astromMatches,
581 matchMeta=calRes.matchMeta,
582 )
584 return calRes
586 def runQuantum(self, butlerQC, inputRefs, outputRefs):
587 inputs = butlerQC.get(inputRefs)
588 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
589 returnMaxBits=True)
590 inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
592 if self.config.doAstrometry:
593 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
594 for ref in inputRefs.astromRefCat],
595 refCats=inputs.pop('astromRefCat'),
596 config=self.config.astromRefObjLoader, log=self.log)
597 self.pixelMargin = refObjLoader.config.pixelMargin
598 self.astrometry.setRefObjLoader(refObjLoader)
600 if self.config.doPhotoCal:
601 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
602 for ref in inputRefs.photoRefCat],
603 refCats=inputs.pop('photoRefCat'),
604 config=self.config.photoRefObjLoader,
605 log=self.log)
606 self.pixelMargin = photoRefObjLoader.config.pixelMargin
607 self.photoCal.match.setRefObjLoader(photoRefObjLoader)
609 outputs = self.run(**inputs)
611 if self.config.doWriteMatches and self.config.doAstrometry:
612 normalizedMatches = afwTable.packMatches(outputs.astromMatches)
613 normalizedMatches.table.setMetadata(outputs.matchMeta)
614 if self.config.doWriteMatchesDenormalized:
615 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
616 outputs.matchesDenormalized = denormMatches
617 outputs.matches = normalizedMatches
618 butlerQC.put(outputs, outputRefs)
620 def run(self, exposure, exposureIdInfo=None, background=None,
621 icSourceCat=None):
622 """!Calibrate an exposure (science image or coadd)
624 @param[in,out] exposure exposure to calibrate (an
625 lsst.afw.image.ExposureF or similar);
626 in:
627 - MaskedImage
628 - Psf
629 out:
630 - MaskedImage has background subtracted
631 - Wcs is replaced
632 - PhotoCalib is replaced
633 @param[in] exposureIdInfo ID info for exposure (an
634 lsst.obs.base.ExposureIdInfo) If not provided, returned
635 SourceCatalog IDs will not be globally unique.
636 @param[in,out] background background model already subtracted from
637 exposure (an lsst.afw.math.BackgroundList). May be None if no
638 background has been subtracted, though that is unusual for
639 calibration. A refined background model is output.
640 @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
641 from which we can copy some fields.
643 @return pipe_base Struct containing these fields:
644 - exposure calibrate science exposure with refined WCS and PhotoCalib
645 - background model of background subtracted from exposure (an
646 lsst.afw.math.BackgroundList)
647 - sourceCat catalog of measured sources
648 - astromMatches list of source/refObj matches from the astrometry
649 solver
650 """
651 # detect, deblend and measure sources
652 if exposureIdInfo is None:
653 exposureIdInfo = ExposureIdInfo()
655 if background is None:
656 background = BackgroundList()
657 sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
658 exposureIdInfo.unusedBits)
659 table = SourceTable.make(self.schema, sourceIdFactory)
660 table.setMetadata(self.algMetadata)
662 detRes = self.detection.run(table=table, exposure=exposure,
663 doSmooth=True)
664 sourceCat = detRes.sources
665 if detRes.fpSets.background:
666 for bg in detRes.fpSets.background:
667 background.append(bg)
668 if self.config.doDeblend:
669 self.deblend.run(exposure=exposure, sources=sourceCat)
670 self.measurement.run(
671 measCat=sourceCat,
672 exposure=exposure,
673 exposureId=exposureIdInfo.expId
674 )
675 if self.config.doApCorr:
676 self.applyApCorr.run(
677 catalog=sourceCat,
678 apCorrMap=exposure.getInfo().getApCorrMap()
679 )
680 self.catalogCalculation.run(sourceCat)
682 if icSourceCat is not None and \
683 len(self.config.icSourceFieldsToCopy) > 0:
684 self.copyIcSourceFields(icSourceCat=icSourceCat,
685 sourceCat=sourceCat)
687 # TODO DM-11568: this contiguous check-and-copy could go away if we
688 # reserve enough space during SourceDetection and/or SourceDeblend.
689 # NOTE: sourceSelectors require contiguous catalogs, so ensure
690 # contiguity now, so views are preserved from here on.
691 if not sourceCat.isContiguous():
692 sourceCat = sourceCat.copy(deep=True)
694 # perform astrometry calibration:
695 # fit an improved WCS and update the exposure's WCS in place
696 astromMatches = None
697 matchMeta = None
698 if self.config.doAstrometry:
699 try:
700 astromRes = self.astrometry.run(
701 exposure=exposure,
702 sourceCat=sourceCat,
703 )
704 astromMatches = astromRes.matches
705 matchMeta = astromRes.matchMeta
706 except Exception as e:
707 if self.config.requireAstrometry:
708 raise
709 self.log.warn("Unable to perform astrometric calibration "
710 "(%s): attempting to proceed" % e)
712 # compute photometric calibration
713 if self.config.doPhotoCal:
714 try:
715 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
716 exposure.setPhotoCalib(photoRes.photoCalib)
717 # TODO: reword this to phrase it in terms of the calibration factor?
718 self.log.info("Photometric zero-point: %f" %
719 photoRes.photoCalib.instFluxToMagnitude(1.0))
720 self.setMetadata(exposure=exposure, photoRes=photoRes)
721 except Exception as e:
722 if self.config.requirePhotoCal:
723 raise
724 self.log.warn("Unable to perform photometric calibration "
725 "(%s): attempting to proceed" % e)
726 self.setMetadata(exposure=exposure, photoRes=None)
728 if self.config.doInsertFakes:
729 self.insertFakes.run(exposure, background=background)
731 table = SourceTable.make(self.schema, sourceIdFactory)
732 table.setMetadata(self.algMetadata)
734 detRes = self.detection.run(table=table, exposure=exposure,
735 doSmooth=True)
736 sourceCat = detRes.sources
737 if detRes.fpSets.background:
738 for bg in detRes.fpSets.background:
739 background.append(bg)
740 if self.config.doDeblend:
741 self.deblend.run(exposure=exposure, sources=sourceCat)
742 self.measurement.run(
743 measCat=sourceCat,
744 exposure=exposure,
745 exposureId=exposureIdInfo.expId
746 )
747 if self.config.doApCorr:
748 self.applyApCorr.run(
749 catalog=sourceCat,
750 apCorrMap=exposure.getInfo().getApCorrMap()
751 )
752 self.catalogCalculation.run(sourceCat)
754 if icSourceCat is not None and len(self.config.icSourceFieldsToCopy) > 0:
755 self.copyIcSourceFields(icSourceCat=icSourceCat,
756 sourceCat=sourceCat)
758 frame = getDebugFrame(self._display, "calibrate")
759 if frame:
760 displayAstrometry(
761 sourceCat=sourceCat,
762 exposure=exposure,
763 matches=astromMatches,
764 frame=frame,
765 pause=False,
766 )
768 return pipeBase.Struct(
769 exposure=exposure,
770 background=background,
771 sourceCat=sourceCat,
772 astromMatches=astromMatches,
773 matchMeta=matchMeta,
774 # These are duplicate entries with different names for use with
775 # gen3 middleware
776 outputExposure=exposure,
777 outputCat=sourceCat,
778 outputBackground=background,
779 )
781 def writeOutputs(self, dataRef, exposure, background, sourceCat,
782 astromMatches, matchMeta):
783 """Write output data to the output repository
785 @param[in] dataRef butler data reference corresponding to a science
786 image
787 @param[in] exposure exposure to write
788 @param[in] background background model for exposure
789 @param[in] sourceCat catalog of measured sources
790 @param[in] astromMatches list of source/refObj matches from the
791 astrometry solver
792 """
793 dataRef.put(sourceCat, "src")
794 if self.config.doWriteMatches and astromMatches is not None:
795 normalizedMatches = afwTable.packMatches(astromMatches)
796 normalizedMatches.table.setMetadata(matchMeta)
797 dataRef.put(normalizedMatches, "srcMatch")
798 if self.config.doWriteMatchesDenormalized:
799 denormMatches = denormalizeMatches(astromMatches, matchMeta)
800 dataRef.put(denormMatches, "srcMatchFull")
801 if self.config.doWriteExposure:
802 dataRef.put(exposure, "calexp")
803 dataRef.put(background, "calexpBackground")
805 def getSchemaCatalogs(self):
806 """Return a dict of empty catalogs for each catalog dataset produced
807 by this task.
808 """
809 sourceCat = afwTable.SourceCatalog(self.schema)
810 sourceCat.getTable().setMetadata(self.algMetadata)
811 return {"src": sourceCat}
813 def setMetadata(self, exposure, photoRes=None):
814 """!Set task and exposure metadata
816 Logs a warning and continues if needed data is missing.
818 @param[in,out] exposure exposure whose metadata is to be set
819 @param[in] photoRes results of running photoCal; if None then it was
820 not run
821 """
822 if photoRes is None:
823 return
825 metadata = exposure.getMetadata()
827 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata
828 try:
829 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
830 magZero = photoRes.zp - 2.5*math.log10(exposureTime)
831 except Exception:
832 self.log.warn("Could not set normalized MAGZERO in header: no "
833 "exposure time")
834 magZero = math.nan
836 try:
837 metadata.set('MAGZERO', magZero)
838 metadata.set('MAGZERO_RMS', photoRes.sigma)
839 metadata.set('MAGZERO_NOBJ', photoRes.ngood)
840 metadata.set('COLORTERM1', 0.0)
841 metadata.set('COLORTERM2', 0.0)
842 metadata.set('COLORTERM3', 0.0)
843 except Exception as e:
844 self.log.warn("Could not set exposure metadata: %s" % (e,))
846 def copyIcSourceFields(self, icSourceCat, sourceCat):
847 """!Match sources in icSourceCat and sourceCat and copy the specified fields
849 @param[in] icSourceCat catalog from which to copy fields
850 @param[in,out] sourceCat catalog to which to copy fields
852 The fields copied are those specified by `config.icSourceFieldsToCopy`
853 that actually exist in the schema. This was set up by the constructor
854 using self.schemaMapper.
855 """
856 if self.schemaMapper is None:
857 raise RuntimeError("To copy icSource fields you must specify "
858 "icSourceSchema nd icSourceKeys when "
859 "constructing this task")
860 if icSourceCat is None or sourceCat is None:
861 raise RuntimeError("icSourceCat and sourceCat must both be "
862 "specified")
863 if len(self.config.icSourceFieldsToCopy) == 0:
864 self.log.warn("copyIcSourceFields doing nothing because "
865 "icSourceFieldsToCopy is empty")
866 return
868 mc = afwTable.MatchControl()
869 mc.findOnlyClosest = False # return all matched objects
870 matches = afwTable.matchXy(icSourceCat, sourceCat,
871 self.config.matchRadiusPix, mc)
872 if self.config.doDeblend:
873 deblendKey = sourceCat.schema["deblend_nChild"].asKey()
874 # if deblended, keep children
875 matches = [m for m in matches if m[1].get(deblendKey) == 0]
877 # Because we had to allow multiple matches to handle parents, we now
878 # need to prune to the best matches
879 # closest matches as a dict of icSourceCat source ID:
880 # (icSourceCat source, sourceCat source, distance in pixels)
881 bestMatches = {}
882 for m0, m1, d in matches:
883 id0 = m0.getId()
884 match = bestMatches.get(id0)
885 if match is None or d <= match[2]:
886 bestMatches[id0] = (m0, m1, d)
887 matches = list(bestMatches.values())
889 # Check that no sourceCat sources are listed twice (we already know
890 # that each match has a unique icSourceCat source ID, due to using
891 # that ID as the key in bestMatches)
892 numMatches = len(matches)
893 numUniqueSources = len(set(m[1].getId() for m in matches))
894 if numUniqueSources != numMatches:
895 self.log.warn("{} icSourceCat sources matched only {} sourceCat "
896 "sources".format(numMatches, numUniqueSources))
898 self.log.info("Copying flags from icSourceCat to sourceCat for "
899 "%s sources" % (numMatches,))
901 # For each match: set the calibSourceKey flag and copy the desired
902 # fields
903 for icSrc, src, d in matches:
904 src.setFlag(self.calibSourceKey, True)
905 # src.assign copies the footprint from icSrc, which we don't want
906 # (DM-407)
907 # so set icSrc's footprint to src's footprint before src.assign,
908 # then restore it
909 icSrcFootprint = icSrc.getFootprint()
910 try:
911 icSrc.setFootprint(src.getFootprint())
912 src.assign(icSrc, self.schemaMapper)
913 finally:
914 icSrc.setFootprint(icSrcFootprint)