Coverage for python/lsst/pipe/tasks/calibrate.py : 16%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008-2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22import math
24from lsstDebug import getDebugFrame
25import lsst.pex.config as pexConfig
26import lsst.pipe.base as pipeBase
27import lsst.pipe.base.connectionTypes as cT
28import lsst.afw.table as afwTable
29from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches
30from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, SkyObjectsTask
31from lsst.obs.base import ExposureIdInfo
32import lsst.daf.base as dafBase
33from lsst.afw.math import BackgroundList
34from lsst.afw.table import IdFactory, SourceTable
35from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader
36from lsst.meas.base import (SingleFrameMeasurementTask,
37 ApplyApCorrTask,
38 CatalogCalculationTask)
39from lsst.meas.deblender import SourceDeblendTask
40from .fakes import BaseFakeSourcesTask
41from .photoCal import PhotoCalTask
43__all__ = ["CalibrateConfig", "CalibrateTask"]
46class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"),
47 defaultTemplates={}):
49 icSourceSchema = cT.InitInput(
50 doc="Schema produced by characterize image task, used to initialize this task",
51 name="icSrc_schema",
52 storageClass="SourceCatalog",
53 )
55 outputSchema = cT.InitOutput(
56 doc="Schema after CalibrateTask has been initialized",
57 name="src_schema",
58 storageClass="SourceCatalog",
59 )
61 exposure = cT.Input(
62 doc="Input image to calibrate",
63 name="icExp",
64 storageClass="ExposureF",
65 dimensions=("instrument", "visit", "detector"),
66 )
68 background = cT.Input(
69 doc="Backgrounds determined by characterize task",
70 name="icExpBackground",
71 storageClass="Background",
72 dimensions=("instrument", "visit", "detector"),
73 )
75 icSourceCat = cT.Input(
76 doc="Source catalog created by characterize task",
77 name="icSrc",
78 storageClass="SourceCatalog",
79 dimensions=("instrument", "visit", "detector"),
80 )
82 astromRefCat = cT.PrerequisiteInput(
83 doc="Reference catalog to use for astrometry",
84 name="cal_ref_cat",
85 storageClass="SimpleCatalog",
86 dimensions=("skypix",),
87 deferLoad=True,
88 multiple=True,
89 )
91 photoRefCat = cT.PrerequisiteInput(
92 doc="Reference catalog to use for photometric calibration",
93 name="cal_ref_cat",
94 storageClass="SimpleCatalog",
95 dimensions=("skypix",),
96 deferLoad=True,
97 multiple=True
98 )
100 outputExposure = cT.Output(
101 doc="Exposure after running calibration task",
102 name="calexp",
103 storageClass="ExposureF",
104 dimensions=("instrument", "visit", "detector"),
105 )
107 outputCat = cT.Output(
108 doc="Source catalog produced in calibrate task",
109 name="src",
110 storageClass="SourceCatalog",
111 dimensions=("instrument", "visit", "detector"),
112 )
114 outputBackground = cT.Output(
115 doc="Background models estimated in calibration task",
116 name="calexpBackground",
117 storageClass="Background",
118 dimensions=("instrument", "visit", "detector"),
119 )
121 matches = cT.Output(
122 doc="Source/refObj matches from the astrometry solver",
123 name="srcMatch",
124 storageClass="Catalog",
125 dimensions=("instrument", "visit", "detector"),
126 )
128 matchesDenormalized = cT.Output(
129 doc="Denormalized matches from astrometry solver",
130 name="srcMatchFull",
131 storageClass="Catalog",
132 dimensions=("instrument", "visit", "detector"),
133 )
135 def __init__(self, *, config=None):
136 super().__init__(config=config)
138 if config.doAstrometry is False:
139 self.prerequisiteInputs.remove("astromRefCat")
140 if config.doPhotoCal is False:
141 self.prerequisiteInputs.remove("photoRefCat")
143 if config.doWriteMatches is False or config.doAstrometry is False:
144 self.outputs.remove("matches")
145 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False:
146 self.outputs.remove("matchesDenormalized")
149class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections):
150 """Config for CalibrateTask"""
151 doWrite = pexConfig.Field(
152 dtype=bool,
153 default=True,
154 doc="Save calibration results?",
155 )
156 doWriteHeavyFootprintsInSources = pexConfig.Field(
157 dtype=bool,
158 default=True,
159 doc="Include HeavyFootprint data in source table? If false then heavy "
160 "footprints are saved as normal footprints, which saves some space"
161 )
162 doWriteMatches = pexConfig.Field(
163 dtype=bool,
164 default=True,
165 doc="Write reference matches (ignored if doWrite or doAstrometry false)?",
166 )
167 doWriteMatchesDenormalized = pexConfig.Field(
168 dtype=bool,
169 default=False,
170 doc=("Write reference matches in denormalized format? "
171 "This format uses more disk space, but is more convenient to "
172 "read. Ignored if doWriteMatches=False or doWrite=False."),
173 )
174 doAstrometry = pexConfig.Field(
175 dtype=bool,
176 default=True,
177 doc="Perform astrometric calibration?",
178 )
179 astromRefObjLoader = pexConfig.ConfigurableField(
180 target=LoadIndexedReferenceObjectsTask,
181 doc="reference object loader for astrometric calibration",
182 )
183 photoRefObjLoader = pexConfig.ConfigurableField(
184 target=LoadIndexedReferenceObjectsTask,
185 doc="reference object loader for photometric calibration",
186 )
187 astrometry = pexConfig.ConfigurableField(
188 target=AstrometryTask,
189 doc="Perform astrometric calibration to refine the WCS",
190 )
191 requireAstrometry = pexConfig.Field(
192 dtype=bool,
193 default=True,
194 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry "
195 "false."),
196 )
197 doPhotoCal = pexConfig.Field(
198 dtype=bool,
199 default=True,
200 doc="Perform phometric calibration?",
201 )
202 requirePhotoCal = pexConfig.Field(
203 dtype=bool,
204 default=True,
205 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal "
206 "false."),
207 )
208 photoCal = pexConfig.ConfigurableField(
209 target=PhotoCalTask,
210 doc="Perform photometric calibration",
211 )
212 icSourceFieldsToCopy = pexConfig.ListField(
213 dtype=str,
214 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
215 doc=("Fields to copy from the icSource catalog to the output catalog "
216 "for matching sources Any missing fields will trigger a "
217 "RuntimeError exception. Ignored if icSourceCat is not provided.")
218 )
219 matchRadiusPix = pexConfig.Field(
220 dtype=float,
221 default=3,
222 doc=("Match radius for matching icSourceCat objects to sourceCat "
223 "objects (pixels)"),
224 )
225 checkUnitsParseStrict = pexConfig.Field(
226 doc=("Strictness of Astropy unit compatibility check, can be 'raise', "
227 "'warn' or 'silent'"),
228 dtype=str,
229 default="raise",
230 )
231 detection = pexConfig.ConfigurableField(
232 target=SourceDetectionTask,
233 doc="Detect sources"
234 )
235 doDeblend = pexConfig.Field(
236 dtype=bool,
237 default=True,
238 doc="Run deblender input exposure"
239 )
240 deblend = pexConfig.ConfigurableField(
241 target=SourceDeblendTask,
242 doc="Split blended sources into their components"
243 )
244 doSkySources = pexConfig.Field(
245 dtype=bool,
246 default=True,
247 doc="Generate sky sources?",
248 )
249 skySources = pexConfig.ConfigurableField(
250 target=SkyObjectsTask,
251 doc="Generate sky sources",
252 )
253 measurement = pexConfig.ConfigurableField(
254 target=SingleFrameMeasurementTask,
255 doc="Measure sources"
256 )
257 doApCorr = pexConfig.Field(
258 dtype=bool,
259 default=True,
260 doc="Run subtask to apply aperture correction"
261 )
262 applyApCorr = pexConfig.ConfigurableField(
263 target=ApplyApCorrTask,
264 doc="Subtask to apply aperture corrections"
265 )
266 # If doApCorr is False, and the exposure does not have apcorrections
267 # already applied, the active plugins in catalogCalculation almost
268 # certainly should not contain the characterization plugin
269 catalogCalculation = pexConfig.ConfigurableField(
270 target=CatalogCalculationTask,
271 doc="Subtask to run catalogCalculation plugins on catalog"
272 )
273 doInsertFakes = pexConfig.Field(
274 dtype=bool,
275 default=False,
276 doc="Run fake sources injection task"
277 )
278 insertFakes = pexConfig.ConfigurableField(
279 target=BaseFakeSourcesTask,
280 doc="Injection of fake sources for testing purposes (must be "
281 "retargeted)"
282 )
283 doWriteExposure = pexConfig.Field(
284 dtype=bool,
285 default=True,
286 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a "
287 "normal calexp but as a fakes_calexp."
288 )
290 def setDefaults(self):
291 super().setDefaults()
292 self.detection.doTempLocalBackground = False
293 self.deblend.maxFootprintSize = 2000
294 self.measurement.plugins.names |= ["base_LocalPhotoCalib", "base_LocalWcs"]
296 def validate(self):
297 super().validate()
298 astromRefCatGen2 = getattr(self.astromRefObjLoader, "ref_dataset_name", None)
299 if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat:
300 raise ValueError(
301 f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference "
302 f"catalogs are different. These options must be kept in sync until Gen2 is retired."
303 )
304 photoRefCatGen2 = getattr(self.photoRefObjLoader, "ref_dataset_name", None)
305 if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat:
306 raise ValueError(
307 f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference "
308 f"catalogs are different. These options must be kept in sync until Gen2 is retired."
309 )
312## \addtogroup LSST_task_documentation
313## \{
314## \page CalibrateTask
315## \ref CalibrateTask_ "CalibrateTask"
316## \copybrief CalibrateTask
317## \}
319class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
320 r"""!Calibrate an exposure: measure sources and perform astrometric and
321 photometric calibration
323 @anchor CalibrateTask_
325 @section pipe_tasks_calibrate_Contents Contents
327 - @ref pipe_tasks_calibrate_Purpose
328 - @ref pipe_tasks_calibrate_Initialize
329 - @ref pipe_tasks_calibrate_IO
330 - @ref pipe_tasks_calibrate_Config
331 - @ref pipe_tasks_calibrate_Metadata
332 - @ref pipe_tasks_calibrate_Debug
335 @section pipe_tasks_calibrate_Purpose Description
337 Given an exposure with a good PSF model and aperture correction map
338 (e.g. as provided by @ref CharacterizeImageTask), perform the following
339 operations:
340 - Run detection and measurement
341 - Run astrometry subtask to fit an improved WCS
342 - Run photoCal subtask to fit the exposure's photometric zero-point
344 @section pipe_tasks_calibrate_Initialize Task initialisation
346 @copydoc \_\_init\_\_
348 @section pipe_tasks_calibrate_IO Invoking the Task
350 If you want this task to unpersist inputs or persist outputs, then call
351 the `runDataRef` method (a wrapper around the `run` method).
353 If you already have the inputs unpersisted and do not want to persist the
354 output then it is more direct to call the `run` method:
356 @section pipe_tasks_calibrate_Config Configuration parameters
358 See @ref CalibrateConfig
360 @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata
362 Exposure metadata
363 <dl>
364 <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task
365 <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal
366 task
367 <dt>COLORTERM1 <dd>?? (always 0.0)
368 <dt>COLORTERM2 <dd>?? (always 0.0)
369 <dt>COLORTERM3 <dd>?? (always 0.0)
370 </dl>
372 @section pipe_tasks_calibrate_Debug Debug variables
374 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink
375 interface supports a flag
376 `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug
377 for more about `debug.py`.
379 CalibrateTask has a debug dictionary containing one key:
380 <dl>
381 <dt>calibrate
382 <dd>frame (an int; <= 0 to not display) in which to display the exposure,
383 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for
384 the meaning of the various symbols.
385 </dl>
387 For example, put something like:
388 @code{.py}
389 import lsstDebug
390 def DebugInfo(name):
391 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would
392 # call us recursively
393 if name == "lsst.pipe.tasks.calibrate":
394 di.display = dict(
395 calibrate = 1,
396 )
398 return di
400 lsstDebug.Info = DebugInfo
401 @endcode
402 into your `debug.py` file and run `calibrateTask.py` with the `--debug`
403 flag.
405 Some subtasks may have their own debug variables; see individual Task
406 documentation.
407 """
409 # Example description used to live here, removed 2-20-2017 as per
410 # https://jira.lsstcorp.org/browse/DM-9520
412 ConfigClass = CalibrateConfig
413 _DefaultName = "calibrate"
414 RunnerClass = pipeBase.ButlerInitializedTaskRunner
416 def __init__(self, butler=None, astromRefObjLoader=None,
417 photoRefObjLoader=None, icSourceSchema=None,
418 initInputs=None, **kwargs):
419 """!Construct a CalibrateTask
421 @param[in] butler The butler is passed to the refObjLoader constructor
422 in case it is needed. Ignored if the refObjLoader argument
423 provides a loader directly.
424 @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks
425 that supplies an external reference catalog for astrometric
426 calibration. May be None if the desired loader can be constructed
427 from the butler argument or all steps requiring a reference catalog
428 are disabled.
429 @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks
430 that supplies an external reference catalog for photometric
431 calibration. May be None if the desired loader can be constructed
432 from the butler argument or all steps requiring a reference catalog
433 are disabled.
434 @param[in] icSourceSchema schema for icSource catalog, or None.
435 Schema values specified in config.icSourceFieldsToCopy will be
436 taken from this schema. If set to None, no values will be
437 propagated from the icSourceCatalog
438 @param[in,out] kwargs other keyword arguments for
439 lsst.pipe.base.CmdLineTask
440 """
441 super().__init__(**kwargs)
443 if icSourceSchema is None and butler is not None:
444 # Use butler to read icSourceSchema from disk.
445 icSourceSchema = butler.get("icSrc_schema", immediate=True).schema
447 if icSourceSchema is None and butler is None and initInputs is not None:
448 icSourceSchema = initInputs['icSourceSchema'].schema
450 if icSourceSchema is not None:
451 # use a schema mapper to avoid copying each field separately
452 self.schemaMapper = afwTable.SchemaMapper(icSourceSchema)
453 minimumSchema = afwTable.SourceTable.makeMinimalSchema()
454 self.schemaMapper.addMinimalSchema(minimumSchema, False)
456 # Add fields to copy from an icSource catalog
457 # and a field to indicate that the source matched a source in that
458 # catalog. If any fields are missing then raise an exception, but
459 # first find all missing fields in order to make the error message
460 # more useful.
461 self.calibSourceKey = self.schemaMapper.addOutputField(
462 afwTable.Field["Flag"]("calib_detected",
463 "Source was detected as an icSource"))
464 missingFieldNames = []
465 for fieldName in self.config.icSourceFieldsToCopy:
466 try:
467 schemaItem = icSourceSchema.find(fieldName)
468 except Exception:
469 missingFieldNames.append(fieldName)
470 else:
471 # field found; if addMapping fails then raise an exception
472 self.schemaMapper.addMapping(schemaItem.getKey())
474 if missingFieldNames:
475 raise RuntimeError("isSourceCat is missing fields {} "
476 "specified in icSourceFieldsToCopy"
477 .format(missingFieldNames))
479 # produce a temporary schema to pass to the subtasks; finalize it
480 # later
481 self.schema = self.schemaMapper.editOutputSchema()
482 else:
483 self.schemaMapper = None
484 self.schema = afwTable.SourceTable.makeMinimalSchema()
485 self.makeSubtask('detection', schema=self.schema)
487 self.algMetadata = dafBase.PropertyList()
489 # Only create a subtask for fakes if configuration option is set
490 # N.B. the config for fake object task must be retargeted to a child
491 # of BaseFakeSourcesTask
492 if self.config.doInsertFakes:
493 self.makeSubtask("insertFakes")
495 if self.config.doDeblend:
496 self.makeSubtask("deblend", schema=self.schema)
497 if self.config.doSkySources:
498 self.makeSubtask("skySources")
499 self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
500 self.makeSubtask('measurement', schema=self.schema,
501 algMetadata=self.algMetadata)
502 if self.config.doApCorr:
503 self.makeSubtask('applyApCorr', schema=self.schema)
504 self.makeSubtask('catalogCalculation', schema=self.schema)
506 if self.config.doAstrometry:
507 if astromRefObjLoader is None and butler is not None:
508 self.makeSubtask('astromRefObjLoader', butler=butler)
509 astromRefObjLoader = self.astromRefObjLoader
510 self.pixelMargin = astromRefObjLoader.config.pixelMargin
511 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader,
512 schema=self.schema)
513 if self.config.doPhotoCal:
514 if photoRefObjLoader is None and butler is not None:
515 self.makeSubtask('photoRefObjLoader', butler=butler)
516 photoRefObjLoader = self.photoRefObjLoader
517 self.pixelMargin = photoRefObjLoader.config.pixelMargin
518 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader,
519 schema=self.schema)
521 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None):
522 raise RuntimeError("PipelineTask form of this task should not be initialized with "
523 "reference object loaders.")
525 if self.schemaMapper is not None:
526 # finalize the schema
527 self.schema = self.schemaMapper.getOutputSchema()
528 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
530 sourceCatSchema = afwTable.SourceCatalog(self.schema)
531 sourceCatSchema.getTable().setMetadata(self.algMetadata)
532 self.outputSchema = sourceCatSchema
534 @pipeBase.timeMethod
535 def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None,
536 doUnpersist=True):
537 """!Calibrate an exposure, optionally unpersisting inputs and
538 persisting outputs.
540 This is a wrapper around the `run` method that unpersists inputs
541 (if `doUnpersist` true) and persists outputs (if `config.doWrite` true)
543 @param[in] dataRef butler data reference corresponding to a science
544 image
545 @param[in,out] exposure characterized exposure (an
546 lsst.afw.image.ExposureF or similar), or None to unpersist existing
547 icExp and icBackground. See `run` method for details of what is
548 read and written.
549 @param[in,out] background initial model of background already
550 subtracted from exposure (an lsst.afw.math.BackgroundList). May be
551 None if no background has been subtracted, though that is unusual
552 for calibration. A refined background model is output. Ignored if
553 exposure is None.
554 @param[in] icSourceCat catalog from which to copy the fields specified
555 by icSourceKeys, or None;
556 @param[in] doUnpersist unpersist data:
557 - if True, exposure, background and icSourceCat are read from
558 dataRef and those three arguments must all be None;
559 - if False the exposure must be provided; background and
560 icSourceCat are optional. True is intended for running as a
561 command-line task, False for running as a subtask
562 @return same data as the calibrate method
563 """
564 self.log.info("Processing %s" % (dataRef.dataId))
566 if doUnpersist:
567 if any(item is not None for item in (exposure, background,
568 icSourceCat)):
569 raise RuntimeError("doUnpersist true; exposure, background "
570 "and icSourceCat must all be None")
571 exposure = dataRef.get("icExp", immediate=True)
572 background = dataRef.get("icExpBackground", immediate=True)
573 icSourceCat = dataRef.get("icSrc", immediate=True)
574 elif exposure is None:
575 raise RuntimeError("doUnpersist false; exposure must be provided")
577 exposureIdInfo = dataRef.get("expIdInfo")
579 calRes = self.run(
580 exposure=exposure,
581 exposureIdInfo=exposureIdInfo,
582 background=background,
583 icSourceCat=icSourceCat,
584 )
586 if self.config.doWrite:
587 self.writeOutputs(
588 dataRef=dataRef,
589 exposure=calRes.exposure,
590 background=calRes.background,
591 sourceCat=calRes.sourceCat,
592 astromMatches=calRes.astromMatches,
593 matchMeta=calRes.matchMeta,
594 )
596 return calRes
598 def runQuantum(self, butlerQC, inputRefs, outputRefs):
599 inputs = butlerQC.get(inputRefs)
600 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
601 returnMaxBits=True)
602 inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
604 if self.config.doAstrometry:
605 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
606 for ref in inputRefs.astromRefCat],
607 refCats=inputs.pop('astromRefCat'),
608 config=self.config.astromRefObjLoader, log=self.log)
609 self.pixelMargin = refObjLoader.config.pixelMargin
610 self.astrometry.setRefObjLoader(refObjLoader)
612 if self.config.doPhotoCal:
613 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
614 for ref in inputRefs.photoRefCat],
615 refCats=inputs.pop('photoRefCat'),
616 config=self.config.photoRefObjLoader,
617 log=self.log)
618 self.pixelMargin = photoRefObjLoader.config.pixelMargin
619 self.photoCal.match.setRefObjLoader(photoRefObjLoader)
621 outputs = self.run(**inputs)
623 if self.config.doWriteMatches and self.config.doAstrometry:
624 normalizedMatches = afwTable.packMatches(outputs.astromMatches)
625 normalizedMatches.table.setMetadata(outputs.matchMeta)
626 if self.config.doWriteMatchesDenormalized:
627 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
628 outputs.matchesDenormalized = denormMatches
629 outputs.matches = normalizedMatches
630 butlerQC.put(outputs, outputRefs)
632 def run(self, exposure, exposureIdInfo=None, background=None,
633 icSourceCat=None):
634 """!Calibrate an exposure (science image or coadd)
636 @param[in,out] exposure exposure to calibrate (an
637 lsst.afw.image.ExposureF or similar);
638 in:
639 - MaskedImage
640 - Psf
641 out:
642 - MaskedImage has background subtracted
643 - Wcs is replaced
644 - PhotoCalib is replaced
645 @param[in] exposureIdInfo ID info for exposure (an
646 lsst.obs.base.ExposureIdInfo) If not provided, returned
647 SourceCatalog IDs will not be globally unique.
648 @param[in,out] background background model already subtracted from
649 exposure (an lsst.afw.math.BackgroundList). May be None if no
650 background has been subtracted, though that is unusual for
651 calibration. A refined background model is output.
652 @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
653 from which we can copy some fields.
655 @return pipe_base Struct containing these fields:
656 - exposure calibrate science exposure with refined WCS and PhotoCalib
657 - background model of background subtracted from exposure (an
658 lsst.afw.math.BackgroundList)
659 - sourceCat catalog of measured sources
660 - astromMatches list of source/refObj matches from the astrometry
661 solver
662 """
663 # detect, deblend and measure sources
664 if exposureIdInfo is None:
665 exposureIdInfo = ExposureIdInfo()
667 if background is None:
668 background = BackgroundList()
669 sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
670 exposureIdInfo.unusedBits)
671 table = SourceTable.make(self.schema, sourceIdFactory)
672 table.setMetadata(self.algMetadata)
674 detRes = self.detection.run(table=table, exposure=exposure,
675 doSmooth=True)
676 sourceCat = detRes.sources
677 if detRes.fpSets.background:
678 for bg in detRes.fpSets.background:
679 background.append(bg)
680 if self.config.doSkySources:
681 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId)
682 if skySourceFootprints:
683 for foot in skySourceFootprints:
684 s = sourceCat.addNew()
685 s.setFootprint(foot)
686 s.set(self.skySourceKey, True)
687 if self.config.doDeblend:
688 self.deblend.run(exposure=exposure, sources=sourceCat)
689 self.measurement.run(
690 measCat=sourceCat,
691 exposure=exposure,
692 exposureId=exposureIdInfo.expId
693 )
694 if self.config.doApCorr:
695 self.applyApCorr.run(
696 catalog=sourceCat,
697 apCorrMap=exposure.getInfo().getApCorrMap()
698 )
699 self.catalogCalculation.run(sourceCat)
701 if icSourceCat is not None and \
702 len(self.config.icSourceFieldsToCopy) > 0:
703 self.copyIcSourceFields(icSourceCat=icSourceCat,
704 sourceCat=sourceCat)
706 # TODO DM-11568: this contiguous check-and-copy could go away if we
707 # reserve enough space during SourceDetection and/or SourceDeblend.
708 # NOTE: sourceSelectors require contiguous catalogs, so ensure
709 # contiguity now, so views are preserved from here on.
710 if not sourceCat.isContiguous():
711 sourceCat = sourceCat.copy(deep=True)
713 # perform astrometry calibration:
714 # fit an improved WCS and update the exposure's WCS in place
715 astromMatches = None
716 matchMeta = None
717 if self.config.doAstrometry:
718 try:
719 astromRes = self.astrometry.run(
720 exposure=exposure,
721 sourceCat=sourceCat,
722 )
723 astromMatches = astromRes.matches
724 matchMeta = astromRes.matchMeta
725 except Exception as e:
726 if self.config.requireAstrometry:
727 raise
728 self.log.warn("Unable to perform astrometric calibration "
729 "(%s): attempting to proceed" % e)
731 # compute photometric calibration
732 if self.config.doPhotoCal:
733 try:
734 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
735 exposure.setPhotoCalib(photoRes.photoCalib)
736 # TODO: reword this to phrase it in terms of the calibration factor?
737 self.log.info("Photometric zero-point: %f" %
738 photoRes.photoCalib.instFluxToMagnitude(1.0))
739 self.setMetadata(exposure=exposure, photoRes=photoRes)
740 except Exception as e:
741 if self.config.requirePhotoCal:
742 raise
743 self.log.warn("Unable to perform photometric calibration "
744 "(%s): attempting to proceed" % e)
745 self.setMetadata(exposure=exposure, photoRes=None)
747 if self.config.doInsertFakes:
748 self.insertFakes.run(exposure, background=background)
750 table = SourceTable.make(self.schema, sourceIdFactory)
751 table.setMetadata(self.algMetadata)
753 detRes = self.detection.run(table=table, exposure=exposure,
754 doSmooth=True)
755 sourceCat = detRes.sources
756 if detRes.fpSets.background:
757 for bg in detRes.fpSets.background:
758 background.append(bg)
759 if self.config.doDeblend:
760 self.deblend.run(exposure=exposure, sources=sourceCat)
761 self.measurement.run(
762 measCat=sourceCat,
763 exposure=exposure,
764 exposureId=exposureIdInfo.expId
765 )
766 if self.config.doApCorr:
767 self.applyApCorr.run(
768 catalog=sourceCat,
769 apCorrMap=exposure.getInfo().getApCorrMap()
770 )
771 self.catalogCalculation.run(sourceCat)
773 if icSourceCat is not None and len(self.config.icSourceFieldsToCopy) > 0:
774 self.copyIcSourceFields(icSourceCat=icSourceCat,
775 sourceCat=sourceCat)
777 frame = getDebugFrame(self._display, "calibrate")
778 if frame:
779 displayAstrometry(
780 sourceCat=sourceCat,
781 exposure=exposure,
782 matches=astromMatches,
783 frame=frame,
784 pause=False,
785 )
787 return pipeBase.Struct(
788 exposure=exposure,
789 background=background,
790 sourceCat=sourceCat,
791 astromMatches=astromMatches,
792 matchMeta=matchMeta,
793 # These are duplicate entries with different names for use with
794 # gen3 middleware
795 outputExposure=exposure,
796 outputCat=sourceCat,
797 outputBackground=background,
798 )
800 def writeOutputs(self, dataRef, exposure, background, sourceCat,
801 astromMatches, matchMeta):
802 """Write output data to the output repository
804 @param[in] dataRef butler data reference corresponding to a science
805 image
806 @param[in] exposure exposure to write
807 @param[in] background background model for exposure
808 @param[in] sourceCat catalog of measured sources
809 @param[in] astromMatches list of source/refObj matches from the
810 astrometry solver
811 """
812 dataRef.put(sourceCat, "src")
813 if self.config.doWriteMatches and astromMatches is not None:
814 normalizedMatches = afwTable.packMatches(astromMatches)
815 normalizedMatches.table.setMetadata(matchMeta)
816 dataRef.put(normalizedMatches, "srcMatch")
817 if self.config.doWriteMatchesDenormalized:
818 denormMatches = denormalizeMatches(astromMatches, matchMeta)
819 dataRef.put(denormMatches, "srcMatchFull")
820 if self.config.doWriteExposure:
821 dataRef.put(exposure, "calexp")
822 dataRef.put(background, "calexpBackground")
824 def getSchemaCatalogs(self):
825 """Return a dict of empty catalogs for each catalog dataset produced
826 by this task.
827 """
828 sourceCat = afwTable.SourceCatalog(self.schema)
829 sourceCat.getTable().setMetadata(self.algMetadata)
830 return {"src": sourceCat}
832 def setMetadata(self, exposure, photoRes=None):
833 """!Set task and exposure metadata
835 Logs a warning and continues if needed data is missing.
837 @param[in,out] exposure exposure whose metadata is to be set
838 @param[in] photoRes results of running photoCal; if None then it was
839 not run
840 """
841 if photoRes is None:
842 return
844 metadata = exposure.getMetadata()
846 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata
847 try:
848 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
849 magZero = photoRes.zp - 2.5*math.log10(exposureTime)
850 except Exception:
851 self.log.warn("Could not set normalized MAGZERO in header: no "
852 "exposure time")
853 magZero = math.nan
855 try:
856 metadata.set('MAGZERO', magZero)
857 metadata.set('MAGZERO_RMS', photoRes.sigma)
858 metadata.set('MAGZERO_NOBJ', photoRes.ngood)
859 metadata.set('COLORTERM1', 0.0)
860 metadata.set('COLORTERM2', 0.0)
861 metadata.set('COLORTERM3', 0.0)
862 except Exception as e:
863 self.log.warn("Could not set exposure metadata: %s" % (e,))
865 def copyIcSourceFields(self, icSourceCat, sourceCat):
866 """!Match sources in icSourceCat and sourceCat and copy the specified fields
868 @param[in] icSourceCat catalog from which to copy fields
869 @param[in,out] sourceCat catalog to which to copy fields
871 The fields copied are those specified by `config.icSourceFieldsToCopy`
872 that actually exist in the schema. This was set up by the constructor
873 using self.schemaMapper.
874 """
875 if self.schemaMapper is None:
876 raise RuntimeError("To copy icSource fields you must specify "
877 "icSourceSchema nd icSourceKeys when "
878 "constructing this task")
879 if icSourceCat is None or sourceCat is None:
880 raise RuntimeError("icSourceCat and sourceCat must both be "
881 "specified")
882 if len(self.config.icSourceFieldsToCopy) == 0:
883 self.log.warn("copyIcSourceFields doing nothing because "
884 "icSourceFieldsToCopy is empty")
885 return
887 mc = afwTable.MatchControl()
888 mc.findOnlyClosest = False # return all matched objects
889 matches = afwTable.matchXy(icSourceCat, sourceCat,
890 self.config.matchRadiusPix, mc)
891 if self.config.doDeblend:
892 deblendKey = sourceCat.schema["deblend_nChild"].asKey()
893 # if deblended, keep children
894 matches = [m for m in matches if m[1].get(deblendKey) == 0]
896 # Because we had to allow multiple matches to handle parents, we now
897 # need to prune to the best matches
898 # closest matches as a dict of icSourceCat source ID:
899 # (icSourceCat source, sourceCat source, distance in pixels)
900 bestMatches = {}
901 for m0, m1, d in matches:
902 id0 = m0.getId()
903 match = bestMatches.get(id0)
904 if match is None or d <= match[2]:
905 bestMatches[id0] = (m0, m1, d)
906 matches = list(bestMatches.values())
908 # Check that no sourceCat sources are listed twice (we already know
909 # that each match has a unique icSourceCat source ID, due to using
910 # that ID as the key in bestMatches)
911 numMatches = len(matches)
912 numUniqueSources = len(set(m[1].getId() for m in matches))
913 if numUniqueSources != numMatches:
914 self.log.warn("{} icSourceCat sources matched only {} sourceCat "
915 "sources".format(numMatches, numUniqueSources))
917 self.log.info("Copying flags from icSourceCat to sourceCat for "
918 "%s sources" % (numMatches,))
920 # For each match: set the calibSourceKey flag and copy the desired
921 # fields
922 for icSrc, src, d in matches:
923 src.setFlag(self.calibSourceKey, True)
924 # src.assign copies the footprint from icSrc, which we don't want
925 # (DM-407)
926 # so set icSrc's footprint to src's footprint before src.assign,
927 # then restore it
928 icSrcFootprint = icSrc.getFootprint()
929 try:
930 icSrc.setFootprint(src.getFootprint())
931 src.assign(icSrc, self.schemaMapper)
932 finally:
933 icSrc.setFootprint(icSrcFootprint)