Coverage for python/lsst/pipe/tasks/calibrate.py : 17%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008-2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22import math
24from lsstDebug import getDebugFrame
25import lsst.pex.config as pexConfig
26import lsst.pipe.base as pipeBase
27import lsst.pipe.base.connectionTypes as cT
28import lsst.afw.table as afwTable
29from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches
30from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, SkyObjectsTask
31from lsst.obs.base import ExposureIdInfo
32import lsst.daf.base as dafBase
33from lsst.afw.math import BackgroundList
34from lsst.afw.table import IdFactory, SourceTable
35from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader
36from lsst.meas.base import (SingleFrameMeasurementTask,
37 ApplyApCorrTask,
38 CatalogCalculationTask)
39from lsst.meas.deblender import SourceDeblendTask
40from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask
41from .fakes import BaseFakeSourcesTask
42from .photoCal import PhotoCalTask
44__all__ = ["CalibrateConfig", "CalibrateTask"]
47class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"),
48 defaultTemplates={}):
50 icSourceSchema = cT.InitInput(
51 doc="Schema produced by characterize image task, used to initialize this task",
52 name="icSrc_schema",
53 storageClass="SourceCatalog",
54 )
56 outputSchema = cT.InitOutput(
57 doc="Schema after CalibrateTask has been initialized",
58 name="src_schema",
59 storageClass="SourceCatalog",
60 )
62 exposure = cT.Input(
63 doc="Input image to calibrate",
64 name="icExp",
65 storageClass="ExposureF",
66 dimensions=("instrument", "visit", "detector"),
67 )
69 background = cT.Input(
70 doc="Backgrounds determined by characterize task",
71 name="icExpBackground",
72 storageClass="Background",
73 dimensions=("instrument", "visit", "detector"),
74 )
76 icSourceCat = cT.Input(
77 doc="Source catalog created by characterize task",
78 name="icSrc",
79 storageClass="SourceCatalog",
80 dimensions=("instrument", "visit", "detector"),
81 )
83 astromRefCat = cT.PrerequisiteInput(
84 doc="Reference catalog to use for astrometry",
85 name="cal_ref_cat",
86 storageClass="SimpleCatalog",
87 dimensions=("skypix",),
88 deferLoad=True,
89 multiple=True,
90 )
92 photoRefCat = cT.PrerequisiteInput(
93 doc="Reference catalog to use for photometric calibration",
94 name="cal_ref_cat",
95 storageClass="SimpleCatalog",
96 dimensions=("skypix",),
97 deferLoad=True,
98 multiple=True
99 )
101 outputExposure = cT.Output(
102 doc="Exposure after running calibration task",
103 name="calexp",
104 storageClass="ExposureF",
105 dimensions=("instrument", "visit", "detector"),
106 )
108 outputCat = cT.Output(
109 doc="Source catalog produced in calibrate task",
110 name="src",
111 storageClass="SourceCatalog",
112 dimensions=("instrument", "visit", "detector"),
113 )
115 outputBackground = cT.Output(
116 doc="Background models estimated in calibration task",
117 name="calexpBackground",
118 storageClass="Background",
119 dimensions=("instrument", "visit", "detector"),
120 )
122 matches = cT.Output(
123 doc="Source/refObj matches from the astrometry solver",
124 name="srcMatch",
125 storageClass="Catalog",
126 dimensions=("instrument", "visit", "detector"),
127 )
129 matchesDenormalized = cT.Output(
130 doc="Denormalized matches from astrometry solver",
131 name="srcMatchFull",
132 storageClass="Catalog",
133 dimensions=("instrument", "visit", "detector"),
134 )
136 def __init__(self, *, config=None):
137 super().__init__(config=config)
139 if config.doAstrometry is False:
140 self.prerequisiteInputs.remove("astromRefCat")
141 if config.doPhotoCal is False:
142 self.prerequisiteInputs.remove("photoRefCat")
144 if config.doWriteMatches is False or config.doAstrometry is False:
145 self.outputs.remove("matches")
146 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False:
147 self.outputs.remove("matchesDenormalized")
150class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections):
151 """Config for CalibrateTask"""
152 doWrite = pexConfig.Field(
153 dtype=bool,
154 default=True,
155 doc="Save calibration results?",
156 )
157 doWriteHeavyFootprintsInSources = pexConfig.Field(
158 dtype=bool,
159 default=True,
160 doc="Include HeavyFootprint data in source table? If false then heavy "
161 "footprints are saved as normal footprints, which saves some space"
162 )
163 doWriteMatches = pexConfig.Field(
164 dtype=bool,
165 default=True,
166 doc="Write reference matches (ignored if doWrite or doAstrometry false)?",
167 )
168 doWriteMatchesDenormalized = pexConfig.Field(
169 dtype=bool,
170 default=False,
171 doc=("Write reference matches in denormalized format? "
172 "This format uses more disk space, but is more convenient to "
173 "read. Ignored if doWriteMatches=False or doWrite=False."),
174 )
175 doAstrometry = pexConfig.Field(
176 dtype=bool,
177 default=True,
178 doc="Perform astrometric calibration?",
179 )
180 astromRefObjLoader = pexConfig.ConfigurableField(
181 target=LoadIndexedReferenceObjectsTask,
182 doc="reference object loader for astrometric calibration",
183 )
184 photoRefObjLoader = pexConfig.ConfigurableField(
185 target=LoadIndexedReferenceObjectsTask,
186 doc="reference object loader for photometric calibration",
187 )
188 astrometry = pexConfig.ConfigurableField(
189 target=AstrometryTask,
190 doc="Perform astrometric calibration to refine the WCS",
191 )
192 requireAstrometry = pexConfig.Field(
193 dtype=bool,
194 default=True,
195 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry "
196 "false."),
197 )
198 doPhotoCal = pexConfig.Field(
199 dtype=bool,
200 default=True,
201 doc="Perform phometric calibration?",
202 )
203 requirePhotoCal = pexConfig.Field(
204 dtype=bool,
205 default=True,
206 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal "
207 "false."),
208 )
209 photoCal = pexConfig.ConfigurableField(
210 target=PhotoCalTask,
211 doc="Perform photometric calibration",
212 )
213 icSourceFieldsToCopy = pexConfig.ListField(
214 dtype=str,
215 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
216 doc=("Fields to copy from the icSource catalog to the output catalog "
217 "for matching sources Any missing fields will trigger a "
218 "RuntimeError exception. Ignored if icSourceCat is not provided.")
219 )
220 matchRadiusPix = pexConfig.Field(
221 dtype=float,
222 default=3,
223 doc=("Match radius for matching icSourceCat objects to sourceCat "
224 "objects (pixels)"),
225 )
226 checkUnitsParseStrict = pexConfig.Field(
227 doc=("Strictness of Astropy unit compatibility check, can be 'raise', "
228 "'warn' or 'silent'"),
229 dtype=str,
230 default="raise",
231 )
232 detection = pexConfig.ConfigurableField(
233 target=SourceDetectionTask,
234 doc="Detect sources"
235 )
236 doDeblend = pexConfig.Field(
237 dtype=bool,
238 default=True,
239 doc="Run deblender input exposure"
240 )
241 deblend = pexConfig.ConfigurableField(
242 target=SourceDeblendTask,
243 doc="Split blended sources into their components"
244 )
245 doSkySources = pexConfig.Field(
246 dtype=bool,
247 default=True,
248 doc="Generate sky sources?",
249 )
250 skySources = pexConfig.ConfigurableField(
251 target=SkyObjectsTask,
252 doc="Generate sky sources",
253 )
254 measurement = pexConfig.ConfigurableField(
255 target=SingleFrameMeasurementTask,
256 doc="Measure sources"
257 )
258 setPrimaryFlags = pexConfig.ConfigurableField(
259 target=SetPrimaryFlagsTask,
260 doc=("Set flags for primary source classification in single frame "
261 "processing. True if sources are not sky sources and not a parent.")
262 )
263 doApCorr = pexConfig.Field(
264 dtype=bool,
265 default=True,
266 doc="Run subtask to apply aperture correction"
267 )
268 applyApCorr = pexConfig.ConfigurableField(
269 target=ApplyApCorrTask,
270 doc="Subtask to apply aperture corrections"
271 )
272 # If doApCorr is False, and the exposure does not have apcorrections
273 # already applied, the active plugins in catalogCalculation almost
274 # certainly should not contain the characterization plugin
275 catalogCalculation = pexConfig.ConfigurableField(
276 target=CatalogCalculationTask,
277 doc="Subtask to run catalogCalculation plugins on catalog"
278 )
279 doInsertFakes = pexConfig.Field(
280 dtype=bool,
281 default=False,
282 doc="Run fake sources injection task"
283 )
284 insertFakes = pexConfig.ConfigurableField(
285 target=BaseFakeSourcesTask,
286 doc="Injection of fake sources for testing purposes (must be "
287 "retargeted)"
288 )
289 doWriteExposure = pexConfig.Field(
290 dtype=bool,
291 default=True,
292 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a "
293 "normal calexp but as a fakes_calexp."
294 )
296 def setDefaults(self):
297 super().setDefaults()
298 self.detection.doTempLocalBackground = False
299 self.deblend.maxFootprintSize = 2000
300 self.measurement.plugins.names |= ["base_LocalPhotoCalib", "base_LocalWcs"]
302 def validate(self):
303 super().validate()
304 astromRefCatGen2 = getattr(self.astromRefObjLoader, "ref_dataset_name", None)
305 if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat:
306 raise ValueError(
307 f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference "
308 f"catalogs are different. These options must be kept in sync until Gen2 is retired."
309 )
310 photoRefCatGen2 = getattr(self.photoRefObjLoader, "ref_dataset_name", None)
311 if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat:
312 raise ValueError(
313 f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference "
314 f"catalogs are different. These options must be kept in sync until Gen2 is retired."
315 )
318## \addtogroup LSST_task_documentation
319## \{
320## \page CalibrateTask
321## \ref CalibrateTask_ "CalibrateTask"
322## \copybrief CalibrateTask
323## \}
325class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
326 r"""!Calibrate an exposure: measure sources and perform astrometric and
327 photometric calibration
329 @anchor CalibrateTask_
331 @section pipe_tasks_calibrate_Contents Contents
333 - @ref pipe_tasks_calibrate_Purpose
334 - @ref pipe_tasks_calibrate_Initialize
335 - @ref pipe_tasks_calibrate_IO
336 - @ref pipe_tasks_calibrate_Config
337 - @ref pipe_tasks_calibrate_Metadata
338 - @ref pipe_tasks_calibrate_Debug
341 @section pipe_tasks_calibrate_Purpose Description
343 Given an exposure with a good PSF model and aperture correction map
344 (e.g. as provided by @ref CharacterizeImageTask), perform the following
345 operations:
346 - Run detection and measurement
347 - Run astrometry subtask to fit an improved WCS
348 - Run photoCal subtask to fit the exposure's photometric zero-point
350 @section pipe_tasks_calibrate_Initialize Task initialisation
352 @copydoc \_\_init\_\_
354 @section pipe_tasks_calibrate_IO Invoking the Task
356 If you want this task to unpersist inputs or persist outputs, then call
357 the `runDataRef` method (a wrapper around the `run` method).
359 If you already have the inputs unpersisted and do not want to persist the
360 output then it is more direct to call the `run` method:
362 @section pipe_tasks_calibrate_Config Configuration parameters
364 See @ref CalibrateConfig
366 @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata
368 Exposure metadata
369 <dl>
370 <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task
371 <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal
372 task
373 <dt>COLORTERM1 <dd>?? (always 0.0)
374 <dt>COLORTERM2 <dd>?? (always 0.0)
375 <dt>COLORTERM3 <dd>?? (always 0.0)
376 </dl>
378 @section pipe_tasks_calibrate_Debug Debug variables
380 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink
381 interface supports a flag
382 `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug
383 for more about `debug.py`.
385 CalibrateTask has a debug dictionary containing one key:
386 <dl>
387 <dt>calibrate
388 <dd>frame (an int; <= 0 to not display) in which to display the exposure,
389 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for
390 the meaning of the various symbols.
391 </dl>
393 For example, put something like:
394 @code{.py}
395 import lsstDebug
396 def DebugInfo(name):
397 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would
398 # call us recursively
399 if name == "lsst.pipe.tasks.calibrate":
400 di.display = dict(
401 calibrate = 1,
402 )
404 return di
406 lsstDebug.Info = DebugInfo
407 @endcode
408 into your `debug.py` file and run `calibrateTask.py` with the `--debug`
409 flag.
411 Some subtasks may have their own debug variables; see individual Task
412 documentation.
413 """
415 # Example description used to live here, removed 2-20-2017 as per
416 # https://jira.lsstcorp.org/browse/DM-9520
418 ConfigClass = CalibrateConfig
419 _DefaultName = "calibrate"
420 RunnerClass = pipeBase.ButlerInitializedTaskRunner
422 def __init__(self, butler=None, astromRefObjLoader=None,
423 photoRefObjLoader=None, icSourceSchema=None,
424 initInputs=None, **kwargs):
425 """!Construct a CalibrateTask
427 @param[in] butler The butler is passed to the refObjLoader constructor
428 in case it is needed. Ignored if the refObjLoader argument
429 provides a loader directly.
430 @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks
431 that supplies an external reference catalog for astrometric
432 calibration. May be None if the desired loader can be constructed
433 from the butler argument or all steps requiring a reference catalog
434 are disabled.
435 @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks
436 that supplies an external reference catalog for photometric
437 calibration. May be None if the desired loader can be constructed
438 from the butler argument or all steps requiring a reference catalog
439 are disabled.
440 @param[in] icSourceSchema schema for icSource catalog, or None.
441 Schema values specified in config.icSourceFieldsToCopy will be
442 taken from this schema. If set to None, no values will be
443 propagated from the icSourceCatalog
444 @param[in,out] kwargs other keyword arguments for
445 lsst.pipe.base.CmdLineTask
446 """
447 super().__init__(**kwargs)
449 if icSourceSchema is None and butler is not None:
450 # Use butler to read icSourceSchema from disk.
451 icSourceSchema = butler.get("icSrc_schema", immediate=True).schema
453 if icSourceSchema is None and butler is None and initInputs is not None:
454 icSourceSchema = initInputs['icSourceSchema'].schema
456 if icSourceSchema is not None:
457 # use a schema mapper to avoid copying each field separately
458 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema)
459 minimumSchema = afwTable.SourceTable.makeMinimalSchema()
460 self.schemaMapper.addMinimalSchema(minimumSchema, False)
462 # Add fields to copy from an icSource catalog
463 # and a field to indicate that the source matched a source in that
464 # catalog. If any fields are missing then raise an exception, but
465 # first find all missing fields in order to make the error message
466 # more useful.
467 self.calibSourceKey = self.schemaMapper.addOutputField(
468 afwTable.Field["Flag"]("calib_detected",
469 "Source was detected as an icSource"))
470 missingFieldNames = []
471 for fieldName in self.config.icSourceFieldsToCopy:
472 try:
473 schemaItem = icSourceSchema.find(fieldName)
474 except Exception:
475 missingFieldNames.append(fieldName)
476 else:
477 # field found; if addMapping fails then raise an exception
478 self.schemaMapper.addMapping(schemaItem.getKey())
480 if missingFieldNames:
481 raise RuntimeError("isSourceCat is missing fields {} "
482 "specified in icSourceFieldsToCopy"
483 .format(missingFieldNames))
485 # produce a temporary schema to pass to the subtasks; finalize it
486 # later
487 self.schema = self.schemaMapper.editOutputSchema()
488 else:
489 self.schemaMapper = None
490 self.schema = afwTable.SourceTable.makeMinimalSchema()
491 self.makeSubtask('detection', schema=self.schema)
493 self.algMetadata = dafBase.PropertyList()
495 # Only create a subtask for fakes if configuration option is set
496 # N.B. the config for fake object task must be retargeted to a child
497 # of BaseFakeSourcesTask
498 if self.config.doInsertFakes:
499 self.makeSubtask("insertFakes")
501 if self.config.doDeblend:
502 self.makeSubtask("deblend", schema=self.schema)
503 if self.config.doSkySources:
504 self.makeSubtask("skySources")
505 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
506 self.makeSubtask('measurement', schema=self.schema,
507 algMetadata=self.algMetadata)
508 self.makeSubtask("setPrimaryFlags", schema=self.schema, isSingleFrame=True)
509 if self.config.doApCorr:
510 self.makeSubtask('applyApCorr', schema=self.schema)
511 self.makeSubtask('catalogCalculation', schema=self.schema)
513 if self.config.doAstrometry:
514 if astromRefObjLoader is None and butler is not None:
515 self.makeSubtask('astromRefObjLoader', butler=butler)
516 astromRefObjLoader = self.astromRefObjLoader
517 self.pixelMargin = astromRefObjLoader.config.pixelMargin
518 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader,
519 schema=self.schema)
520 if self.config.doPhotoCal:
521 if photoRefObjLoader is None and butler is not None:
522 self.makeSubtask('photoRefObjLoader', butler=butler)
523 photoRefObjLoader = self.photoRefObjLoader
524 self.pixelMargin = photoRefObjLoader.config.pixelMargin
525 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader,
526 schema=self.schema)
528 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None):
529 raise RuntimeError("PipelineTask form of this task should not be initialized with "
530 "reference object loaders.")
532 if self.schemaMapper is not None:
533 # finalize the schema
534 self.schema = self.schemaMapper.getOutputSchema()
535 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
537 sourceCatSchema = afwTable.SourceCatalog(self.schema)
538 sourceCatSchema.getTable().setMetadata(self.algMetadata)
539 self.outputSchema = sourceCatSchema
541 @pipeBase.timeMethod
542 def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None,
543 doUnpersist=True):
544 """!Calibrate an exposure, optionally unpersisting inputs and
545 persisting outputs.
547 This is a wrapper around the `run` method that unpersists inputs
548 (if `doUnpersist` true) and persists outputs (if `config.doWrite` true)
550 @param[in] dataRef butler data reference corresponding to a science
551 image
552 @param[in,out] exposure characterized exposure (an
553 lsst.afw.image.ExposureF or similar), or None to unpersist existing
554 icExp and icBackground. See `run` method for details of what is
555 read and written.
556 @param[in,out] background initial model of background already
557 subtracted from exposure (an lsst.afw.math.BackgroundList). May be
558 None if no background has been subtracted, though that is unusual
559 for calibration. A refined background model is output. Ignored if
560 exposure is None.
561 @param[in] icSourceCat catalog from which to copy the fields specified
562 by icSourceKeys, or None;
563 @param[in] doUnpersist unpersist data:
564 - if True, exposure, background and icSourceCat are read from
565 dataRef and those three arguments must all be None;
566 - if False the exposure must be provided; background and
567 icSourceCat are optional. True is intended for running as a
568 command-line task, False for running as a subtask
569 @return same data as the calibrate method
570 """
571 self.log.info("Processing %s" % (dataRef.dataId))
573 if doUnpersist:
574 if any(item is not None for item in (exposure, background,
575 icSourceCat)):
576 raise RuntimeError("doUnpersist true; exposure, background "
577 "and icSourceCat must all be None")
578 exposure = dataRef.get("icExp", immediate=True)
579 background = dataRef.get("icExpBackground", immediate=True)
580 icSourceCat = dataRef.get("icSrc", immediate=True)
581 elif exposure is None:
582 raise RuntimeError("doUnpersist false; exposure must be provided")
584 exposureIdInfo = dataRef.get("expIdInfo")
586 calRes = self.run(
587 exposure=exposure,
588 exposureIdInfo=exposureIdInfo,
589 background=background,
590 icSourceCat=icSourceCat,
591 )
593 if self.config.doWrite:
594 self.writeOutputs(
595 dataRef=dataRef,
596 exposure=calRes.exposure,
597 background=calRes.background,
598 sourceCat=calRes.sourceCat,
599 astromMatches=calRes.astromMatches,
600 matchMeta=calRes.matchMeta,
601 )
603 return calRes
605 def runQuantum(self, butlerQC, inputRefs, outputRefs):
606 inputs = butlerQC.get(inputRefs)
607 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
608 returnMaxBits=True)
609 inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
611 if self.config.doAstrometry:
612 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
613 for ref in inputRefs.astromRefCat],
614 refCats=inputs.pop('astromRefCat'),
615 config=self.config.astromRefObjLoader, log=self.log)
616 self.pixelMargin = refObjLoader.config.pixelMargin
617 self.astrometry.setRefObjLoader(refObjLoader)
619 if self.config.doPhotoCal:
620 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
621 for ref in inputRefs.photoRefCat],
622 refCats=inputs.pop('photoRefCat'),
623 config=self.config.photoRefObjLoader,
624 log=self.log)
625 self.pixelMargin = photoRefObjLoader.config.pixelMargin
626 self.photoCal.match.setRefObjLoader(photoRefObjLoader)
628 outputs = self.run(**inputs)
630 if self.config.doWriteMatches and self.config.doAstrometry:
631 normalizedMatches = afwTable.packMatches(outputs.astromMatches)
632 normalizedMatches.table.setMetadata(outputs.matchMeta)
633 if self.config.doWriteMatchesDenormalized:
634 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
635 outputs.matchesDenormalized = denormMatches
636 outputs.matches = normalizedMatches
637 butlerQC.put(outputs, outputRefs)
639 @pipeBase.timeMethod
640 def run(self, exposure, exposureIdInfo=None, background=None,
641 icSourceCat=None):
642 """!Calibrate an exposure (science image or coadd)
644 @param[in,out] exposure exposure to calibrate (an
645 lsst.afw.image.ExposureF or similar);
646 in:
647 - MaskedImage
648 - Psf
649 out:
650 - MaskedImage has background subtracted
651 - Wcs is replaced
652 - PhotoCalib is replaced
653 @param[in] exposureIdInfo ID info for exposure (an
654 lsst.obs.base.ExposureIdInfo) If not provided, returned
655 SourceCatalog IDs will not be globally unique.
656 @param[in,out] background background model already subtracted from
657 exposure (an lsst.afw.math.BackgroundList). May be None if no
658 background has been subtracted, though that is unusual for
659 calibration. A refined background model is output.
660 @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
661 from which we can copy some fields.
663 @return pipe_base Struct containing these fields:
664 - exposure calibrate science exposure with refined WCS and PhotoCalib
665 - background model of background subtracted from exposure (an
666 lsst.afw.math.BackgroundList)
667 - sourceCat catalog of measured sources
668 - astromMatches list of source/refObj matches from the astrometry
669 solver
670 """
671 # detect, deblend and measure sources
672 if exposureIdInfo is None:
673 exposureIdInfo = ExposureIdInfo()
675 if background is None:
676 background = BackgroundList()
677 sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
678 exposureIdInfo.unusedBits)
679 table = SourceTable.make(self.schema, sourceIdFactory)
680 table.setMetadata(self.algMetadata)
682 detRes = self.detection.run(table=table, exposure=exposure,
683 doSmooth=True)
684 sourceCat = detRes.sources
685 if detRes.fpSets.background:
686 for bg in detRes.fpSets.background:
687 background.append(bg)
688 if self.config.doSkySources:
689 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId)
690 if skySourceFootprints:
691 for foot in skySourceFootprints:
692 s = sourceCat.addNew()
693 s.setFootprint(foot)
694 s.set(self.skySourceKey, True)
695 if self.config.doDeblend:
696 self.deblend.run(exposure=exposure, sources=sourceCat)
697 self.measurement.run(
698 measCat=sourceCat,
699 exposure=exposure,
700 exposureId=exposureIdInfo.expId
701 )
702 if self.config.doApCorr:
703 self.applyApCorr.run(
704 catalog=sourceCat,
705 apCorrMap=exposure.getInfo().getApCorrMap()
706 )
707 self.catalogCalculation.run(sourceCat)
709 self.setPrimaryFlags.run(sourceCat, includeDeblend=self.config.doDeblend)
711 if icSourceCat is not None and \
712 len(self.config.icSourceFieldsToCopy) > 0:
713 self.copyIcSourceFields(icSourceCat=icSourceCat,
714 sourceCat=sourceCat)
716 # TODO DM-11568: this contiguous check-and-copy could go away if we
717 # reserve enough space during SourceDetection and/or SourceDeblend.
718 # NOTE: sourceSelectors require contiguous catalogs, so ensure
719 # contiguity now, so views are preserved from here on.
720 if not sourceCat.isContiguous():
721 sourceCat = sourceCat.copy(deep=True)
723 # perform astrometry calibration:
724 # fit an improved WCS and update the exposure's WCS in place
725 astromMatches = None
726 matchMeta = None
727 if self.config.doAstrometry:
728 try:
729 astromRes = self.astrometry.run(
730 exposure=exposure,
731 sourceCat=sourceCat,
732 )
733 astromMatches = astromRes.matches
734 matchMeta = astromRes.matchMeta
735 except Exception as e:
736 if self.config.requireAstrometry:
737 raise
738 self.log.warn("Unable to perform astrometric calibration "
739 "(%s): attempting to proceed" % e)
741 # compute photometric calibration
742 if self.config.doPhotoCal:
743 try:
744 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
745 exposure.setPhotoCalib(photoRes.photoCalib)
746 # TODO: reword this to phrase it in terms of the calibration factor?
747 self.log.info("Photometric zero-point: %f" %
748 photoRes.photoCalib.instFluxToMagnitude(1.0))
749 self.setMetadata(exposure=exposure, photoRes=photoRes)
750 except Exception as e:
751 if self.config.requirePhotoCal:
752 raise
753 self.log.warn("Unable to perform photometric calibration "
754 "(%s): attempting to proceed" % e)
755 self.setMetadata(exposure=exposure, photoRes=None)
757 if self.config.doInsertFakes:
758 self.insertFakes.run(exposure, background=background)
760 table = SourceTable.make(self.schema, sourceIdFactory)
761 table.setMetadata(self.algMetadata)
763 detRes = self.detection.run(table=table, exposure=exposure,
764 doSmooth=True)
765 sourceCat = detRes.sources
766 if detRes.fpSets.background:
767 for bg in detRes.fpSets.background:
768 background.append(bg)
769 if self.config.doDeblend:
770 self.deblend.run(exposure=exposure, sources=sourceCat)
771 self.measurement.run(
772 measCat=sourceCat,
773 exposure=exposure,
774 exposureId=exposureIdInfo.expId
775 )
776 if self.config.doApCorr:
777 self.applyApCorr.run(
778 catalog=sourceCat,
779 apCorrMap=exposure.getInfo().getApCorrMap()
780 )
781 self.catalogCalculation.run(sourceCat)
783 if icSourceCat is not None and len(self.config.icSourceFieldsToCopy) > 0:
784 self.copyIcSourceFields(icSourceCat=icSourceCat,
785 sourceCat=sourceCat)
787 frame = getDebugFrame(self._display, "calibrate")
788 if frame:
789 displayAstrometry(
790 sourceCat=sourceCat,
791 exposure=exposure,
792 matches=astromMatches,
793 frame=frame,
794 pause=False,
795 )
797 return pipeBase.Struct(
798 exposure=exposure,
799 background=background,
800 sourceCat=sourceCat,
801 astromMatches=astromMatches,
802 matchMeta=matchMeta,
803 # These are duplicate entries with different names for use with
804 # gen3 middleware
805 outputExposure=exposure,
806 outputCat=sourceCat,
807 outputBackground=background,
808 )
810 def writeOutputs(self, dataRef, exposure, background, sourceCat,
811 astromMatches, matchMeta):
812 """Write output data to the output repository
814 @param[in] dataRef butler data reference corresponding to a science
815 image
816 @param[in] exposure exposure to write
817 @param[in] background background model for exposure
818 @param[in] sourceCat catalog of measured sources
819 @param[in] astromMatches list of source/refObj matches from the
820 astrometry solver
821 """
822 dataRef.put(sourceCat, "src")
823 if self.config.doWriteMatches and astromMatches is not None:
824 normalizedMatches = afwTable.packMatches(astromMatches)
825 normalizedMatches.table.setMetadata(matchMeta)
826 dataRef.put(normalizedMatches, "srcMatch")
827 if self.config.doWriteMatchesDenormalized:
828 denormMatches = denormalizeMatches(astromMatches, matchMeta)
829 dataRef.put(denormMatches, "srcMatchFull")
830 if self.config.doWriteExposure:
831 dataRef.put(exposure, "calexp")
832 dataRef.put(background, "calexpBackground")
834 def getSchemaCatalogs(self):
835 """Return a dict of empty catalogs for each catalog dataset produced
836 by this task.
837 """
838 sourceCat = afwTable.SourceCatalog(self.schema)
839 sourceCat.getTable().setMetadata(self.algMetadata)
840 return {"src": sourceCat}
842 def setMetadata(self, exposure, photoRes=None):
843 """!Set task and exposure metadata
845 Logs a warning and continues if needed data is missing.
847 @param[in,out] exposure exposure whose metadata is to be set
848 @param[in] photoRes results of running photoCal; if None then it was
849 not run
850 """
851 if photoRes is None:
852 return
854 metadata = exposure.getMetadata()
856 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata
857 try:
858 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
859 magZero = photoRes.zp - 2.5*math.log10(exposureTime)
860 except Exception:
861 self.log.warn("Could not set normalized MAGZERO in header: no "
862 "exposure time")
863 magZero = math.nan
865 try:
866 metadata.set('MAGZERO', magZero)
867 metadata.set('MAGZERO_RMS', photoRes.sigma)
868 metadata.set('MAGZERO_NOBJ', photoRes.ngood)
869 metadata.set('COLORTERM1', 0.0)
870 metadata.set('COLORTERM2', 0.0)
871 metadata.set('COLORTERM3', 0.0)
872 except Exception as e:
873 self.log.warn("Could not set exposure metadata: %s" % (e,))
875 def copyIcSourceFields(self, icSourceCat, sourceCat):
876 """!Match sources in icSourceCat and sourceCat and copy the specified fields
878 @param[in] icSourceCat catalog from which to copy fields
879 @param[in,out] sourceCat catalog to which to copy fields
881 The fields copied are those specified by `config.icSourceFieldsToCopy`
882 that actually exist in the schema. This was set up by the constructor
883 using self.schemaMapper.
884 """
885 if self.schemaMapper is None:
886 raise RuntimeError("To copy icSource fields you must specify "
887 "icSourceSchema nd icSourceKeys when "
888 "constructing this task")
889 if icSourceCat is None or sourceCat is None:
890 raise RuntimeError("icSourceCat and sourceCat must both be "
891 "specified")
892 if len(self.config.icSourceFieldsToCopy) == 0:
893 self.log.warn("copyIcSourceFields doing nothing because "
894 "icSourceFieldsToCopy is empty")
895 return
897 mc = afwTable.MatchControl()
898 mc.findOnlyClosest = False # return all matched objects
899 matches = afwTable.matchXy(icSourceCat, sourceCat,
900 self.config.matchRadiusPix, mc)
901 if self.config.doDeblend:
902 deblendKey = sourceCat.schema["deblend_nChild"].asKey()
903 # if deblended, keep children
904 matches = [m for m in matches if m[1].get(deblendKey) == 0]
906 # Because we had to allow multiple matches to handle parents, we now
907 # need to prune to the best matches
908 # closest matches as a dict of icSourceCat source ID:
909 # (icSourceCat source, sourceCat source, distance in pixels)
910 bestMatches = {}
911 for m0, m1, d in matches:
912 id0 = m0.getId()
913 match = bestMatches.get(id0)
914 if match is None or d <= match[2]:
915 bestMatches[id0] = (m0, m1, d)
916 matches = list(bestMatches.values())
918 # Check that no sourceCat sources are listed twice (we already know
919 # that each match has a unique icSourceCat source ID, due to using
920 # that ID as the key in bestMatches)
921 numMatches = len(matches)
922 numUniqueSources = len(set(m[1].getId() for m in matches))
923 if numUniqueSources != numMatches:
924 self.log.warn("{} icSourceCat sources matched only {} sourceCat "
925 "sources".format(numMatches, numUniqueSources))
927 self.log.info("Copying flags from icSourceCat to sourceCat for "
928 "%s sources" % (numMatches,))
930 # For each match: set the calibSourceKey flag and copy the desired
931 # fields
932 for icSrc, src, d in matches:
933 src.setFlag(self.calibSourceKey, True)
934 # src.assign copies the footprint from icSrc, which we don't want
935 # (DM-407)
936 # so set icSrc's footprint to src's footprint before src.assign,
937 # then restore it
938 icSrcFootprint = icSrc.getFootprint()
939 try:
940 icSrc.setFootprint(src.getFootprint())
941 src.assign(icSrc, self.schemaMapper)
942 finally:
943 icSrc.setFootprint(icSrcFootprint)