lsst.pipe.tasks gdf62c121a3+f00e3f5942
calibrate.py
Go to the documentation of this file.
2# LSST Data Management System
3# Copyright 2008-2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22import math
23
24from lsstDebug import getDebugFrame
25import lsst.pex.config as pexConfig
26import lsst.pipe.base as pipeBase
27import lsst.pipe.base.connectionTypes as cT
28import lsst.afw.table as afwTable
29from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches
30from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, SkyObjectsTask
31from lsst.obs.base import ExposureIdInfo
32import lsst.daf.base as dafBase
33from lsst.afw.math import BackgroundList
34from lsst.afw.table import SourceTable
35from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader
36from lsst.meas.base import (SingleFrameMeasurementTask,
37 ApplyApCorrTask,
38 CatalogCalculationTask)
39from lsst.meas.deblender import SourceDeblendTask
40from lsst.utils.timer import timeMethod
41from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask
42from .fakes import BaseFakeSourcesTask
43from .photoCal import PhotoCalTask
44from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask
45
46
47__all__ = ["CalibrateConfig", "CalibrateTask"]
48
49
50class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"),
51 defaultTemplates={}):
52
53 icSourceSchema = cT.InitInput(
54 doc="Schema produced by characterize image task, used to initialize this task",
55 name="icSrc_schema",
56 storageClass="SourceCatalog",
57 )
58
59 outputSchema = cT.InitOutput(
60 doc="Schema after CalibrateTask has been initialized",
61 name="src_schema",
62 storageClass="SourceCatalog",
63 )
64
65 exposure = cT.Input(
66 doc="Input image to calibrate",
67 name="icExp",
68 storageClass="ExposureF",
69 dimensions=("instrument", "visit", "detector"),
70 )
71
72 background = cT.Input(
73 doc="Backgrounds determined by characterize task",
74 name="icExpBackground",
75 storageClass="Background",
76 dimensions=("instrument", "visit", "detector"),
77 )
78
79 icSourceCat = cT.Input(
80 doc="Source catalog created by characterize task",
81 name="icSrc",
82 storageClass="SourceCatalog",
83 dimensions=("instrument", "visit", "detector"),
84 )
85
86 astromRefCat = cT.PrerequisiteInput(
87 doc="Reference catalog to use for astrometry",
88 name="cal_ref_cat",
89 storageClass="SimpleCatalog",
90 dimensions=("skypix",),
91 deferLoad=True,
92 multiple=True,
93 )
94
95 photoRefCat = cT.PrerequisiteInput(
96 doc="Reference catalog to use for photometric calibration",
97 name="cal_ref_cat",
98 storageClass="SimpleCatalog",
99 dimensions=("skypix",),
100 deferLoad=True,
101 multiple=True
102 )
103
104 outputExposure = cT.Output(
105 doc="Exposure after running calibration task",
106 name="calexp",
107 storageClass="ExposureF",
108 dimensions=("instrument", "visit", "detector"),
109 )
110
111 outputCat = cT.Output(
112 doc="Source catalog produced in calibrate task",
113 name="src",
114 storageClass="SourceCatalog",
115 dimensions=("instrument", "visit", "detector"),
116 )
117
118 outputBackground = cT.Output(
119 doc="Background models estimated in calibration task",
120 name="calexpBackground",
121 storageClass="Background",
122 dimensions=("instrument", "visit", "detector"),
123 )
124
125 matches = cT.Output(
126 doc="Source/refObj matches from the astrometry solver",
127 name="srcMatch",
128 storageClass="Catalog",
129 dimensions=("instrument", "visit", "detector"),
130 )
131
132 matchesDenormalized = cT.Output(
133 doc="Denormalized matches from astrometry solver",
134 name="srcMatchFull",
135 storageClass="Catalog",
136 dimensions=("instrument", "visit", "detector"),
137 )
138
139 def __init__(self, *, config=None):
140 super().__init__(config=config)
141
142 if config.doAstrometry is False:
143 self.prerequisiteInputs.remove("astromRefCat")
144 if config.doPhotoCal is False:
145 self.prerequisiteInputs.remove("photoRefCat")
146
147 if config.doWriteMatches is False or config.doAstrometry is False:
148 self.outputs.remove("matches")
149 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False:
150 self.outputs.remove("matchesDenormalized")
151
152
153class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections):
154 """Config for CalibrateTask"""
155 doWrite = pexConfig.Field(
156 dtype=bool,
157 default=True,
158 doc="Save calibration results?",
159 )
160 doWriteHeavyFootprintsInSources = pexConfig.Field(
161 dtype=bool,
162 default=True,
163 doc="Include HeavyFootprint data in source table? If false then heavy "
164 "footprints are saved as normal footprints, which saves some space"
165 )
166 doWriteMatches = pexConfig.Field(
167 dtype=bool,
168 default=True,
169 doc="Write reference matches (ignored if doWrite or doAstrometry false)?",
170 )
171 doWriteMatchesDenormalized = pexConfig.Field(
172 dtype=bool,
173 default=False,
174 doc=("Write reference matches in denormalized format? "
175 "This format uses more disk space, but is more convenient to "
176 "read. Ignored if doWriteMatches=False or doWrite=False."),
177 )
178 doAstrometry = pexConfig.Field(
179 dtype=bool,
180 default=True,
181 doc="Perform astrometric calibration?",
182 )
183 astromRefObjLoader = pexConfig.ConfigurableField(
184 target=LoadIndexedReferenceObjectsTask,
185 doc="reference object loader for astrometric calibration",
186 )
187 photoRefObjLoader = pexConfig.ConfigurableField(
188 target=LoadIndexedReferenceObjectsTask,
189 doc="reference object loader for photometric calibration",
190 )
191 astrometry = pexConfig.ConfigurableField(
192 target=AstrometryTask,
193 doc="Perform astrometric calibration to refine the WCS",
194 )
195 requireAstrometry = pexConfig.Field(
196 dtype=bool,
197 default=True,
198 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry "
199 "false."),
200 )
201 doPhotoCal = pexConfig.Field(
202 dtype=bool,
203 default=True,
204 doc="Perform phometric calibration?",
205 )
206 requirePhotoCal = pexConfig.Field(
207 dtype=bool,
208 default=True,
209 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal "
210 "false."),
211 )
212 photoCal = pexConfig.ConfigurableField(
213 target=PhotoCalTask,
214 doc="Perform photometric calibration",
215 )
216 icSourceFieldsToCopy = pexConfig.ListField(
217 dtype=str,
218 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
219 doc=("Fields to copy from the icSource catalog to the output catalog "
220 "for matching sources Any missing fields will trigger a "
221 "RuntimeError exception. Ignored if icSourceCat is not provided.")
222 )
223 matchRadiusPix = pexConfig.Field(
224 dtype=float,
225 default=3,
226 doc=("Match radius for matching icSourceCat objects to sourceCat "
227 "objects (pixels)"),
228 )
229 checkUnitsParseStrict = pexConfig.Field(
230 doc=("Strictness of Astropy unit compatibility check, can be 'raise', "
231 "'warn' or 'silent'"),
232 dtype=str,
233 default="raise",
234 )
235 detection = pexConfig.ConfigurableField(
236 target=SourceDetectionTask,
237 doc="Detect sources"
238 )
239 doDeblend = pexConfig.Field(
240 dtype=bool,
241 default=True,
242 doc="Run deblender input exposure"
243 )
244 deblend = pexConfig.ConfigurableField(
245 target=SourceDeblendTask,
246 doc="Split blended sources into their components"
247 )
248 doSkySources = pexConfig.Field(
249 dtype=bool,
250 default=True,
251 doc="Generate sky sources?",
252 )
253 skySources = pexConfig.ConfigurableField(
254 target=SkyObjectsTask,
255 doc="Generate sky sources",
256 )
257 measurement = pexConfig.ConfigurableField(
258 target=SingleFrameMeasurementTask,
259 doc="Measure sources"
260 )
261 postCalibrationMeasurement = pexConfig.ConfigurableField(
262 target=SingleFrameMeasurementTask,
263 doc="Second round of measurement for plugins that need to be run after photocal"
264 )
265 setPrimaryFlags = pexConfig.ConfigurableField(
266 target=SetPrimaryFlagsTask,
267 doc=("Set flags for primary source classification in single frame "
268 "processing. True if sources are not sky sources and not a parent.")
269 )
270 doApCorr = pexConfig.Field(
271 dtype=bool,
272 default=True,
273 doc="Run subtask to apply aperture correction"
274 )
275 applyApCorr = pexConfig.ConfigurableField(
276 target=ApplyApCorrTask,
277 doc="Subtask to apply aperture corrections"
278 )
279 # If doApCorr is False, and the exposure does not have apcorrections
280 # already applied, the active plugins in catalogCalculation almost
281 # certainly should not contain the characterization plugin
282 catalogCalculation = pexConfig.ConfigurableField(
283 target=CatalogCalculationTask,
284 doc="Subtask to run catalogCalculation plugins on catalog"
285 )
286 doInsertFakes = pexConfig.Field(
287 dtype=bool,
288 default=False,
289 doc="Run fake sources injection task"
290 )
291 insertFakes = pexConfig.ConfigurableField(
292 target=BaseFakeSourcesTask,
293 doc="Injection of fake sources for testing purposes (must be "
294 "retargeted)"
295 )
296 doComputeSummaryStats = pexConfig.Field(
297 dtype=bool,
298 default=True,
299 doc="Run subtask to measure exposure summary statistics?"
300 )
301 computeSummaryStats = pexConfig.ConfigurableField(
302 target=ComputeExposureSummaryStatsTask,
303 doc="Subtask to run computeSummaryStats on exposure"
304 )
305 doWriteExposure = pexConfig.Field(
306 dtype=bool,
307 default=True,
308 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a "
309 "normal calexp but as a fakes_calexp."
310 )
311
312 def setDefaults(self):
313 super().setDefaults()
314 self.detectiondetection.doTempLocalBackground = False
315 self.deblenddeblend.maxFootprintSize = 2000
316 self.postCalibrationMeasurementpostCalibrationMeasurement.plugins.names = ["base_LocalPhotoCalib", "base_LocalWcs"]
317 self.postCalibrationMeasurementpostCalibrationMeasurement.doReplaceWithNoise = False
318 for key in self.postCalibrationMeasurementpostCalibrationMeasurement.slots:
319 setattr(self.postCalibrationMeasurementpostCalibrationMeasurement.slots, key, None)
320
321 def validate(self):
322 super().validate()
323 astromRefCatGen2 = getattr(self.astromRefObjLoaderastromRefObjLoader, "ref_dataset_name", None)
324 if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat:
325 raise ValueError(
326 f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference "
327 f"catalogs are different. These options must be kept in sync until Gen2 is retired."
328 )
329 photoRefCatGen2 = getattr(self.photoRefObjLoaderphotoRefObjLoader, "ref_dataset_name", None)
330 if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat:
331 raise ValueError(
332 f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference "
333 f"catalogs are different. These options must be kept in sync until Gen2 is retired."
334 )
335
336
337
343
344class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
345 r"""!Calibrate an exposure: measure sources and perform astrometric and
346 photometric calibration
347
348 @anchor CalibrateTask_
349
350 @section pipe_tasks_calibrate_Contents Contents
351
352 - @ref pipe_tasks_calibrate_Purpose
353 - @ref pipe_tasks_calibrate_Initialize
354 - @ref pipe_tasks_calibrate_IO
355 - @ref pipe_tasks_calibrate_Config
356 - @ref pipe_tasks_calibrate_Metadata
357 - @ref pipe_tasks_calibrate_Debug
358
359
360 @section pipe_tasks_calibrate_Purpose Description
361
362 Given an exposure with a good PSF model and aperture correction map
363 (e.g. as provided by @ref CharacterizeImageTask), perform the following
364 operations:
365 - Run detection and measurement
366 - Run astrometry subtask to fit an improved WCS
367 - Run photoCal subtask to fit the exposure's photometric zero-point
368
369 @section pipe_tasks_calibrate_Initialize Task initialisation
370
371 @copydoc \_\_init\_\_
372
373 @section pipe_tasks_calibrate_IO Invoking the Task
374
375 If you want this task to unpersist inputs or persist outputs, then call
376 the `runDataRef` method (a wrapper around the `run` method).
377
378 If you already have the inputs unpersisted and do not want to persist the
379 output then it is more direct to call the `run` method:
380
381 @section pipe_tasks_calibrate_Config Configuration parameters
382
383 See @ref CalibrateConfig
384
385 @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata
386
387 Exposure metadata
388 <dl>
389 <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task
390 <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal
391 task
392 <dt>COLORTERM1 <dd>?? (always 0.0)
393 <dt>COLORTERM2 <dd>?? (always 0.0)
394 <dt>COLORTERM3 <dd>?? (always 0.0)
395 </dl>
396
397 @section pipe_tasks_calibrate_Debug Debug variables
398
399 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink
400 interface supports a flag
401 `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug
402 for more about `debug.py`.
403
404 CalibrateTask has a debug dictionary containing one key:
405 <dl>
406 <dt>calibrate
407 <dd>frame (an int; <= 0 to not display) in which to display the exposure,
408 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for
409 the meaning of the various symbols.
410 </dl>
411
412 For example, put something like:
413 @code{.py}
414 import lsstDebug
415 def DebugInfo(name):
416 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would
417 # call us recursively
418 if name == "lsst.pipe.tasks.calibrate":
419 di.display = dict(
420 calibrate = 1,
421 )
422
423 return di
424
425 lsstDebug.Info = DebugInfo
426 @endcode
427 into your `debug.py` file and run `calibrateTask.py` with the `--debug`
428 flag.
429
430 Some subtasks may have their own debug variables; see individual Task
431 documentation.
432 """
433
434 # Example description used to live here, removed 2-20-2017 as per
435 # https://jira.lsstcorp.org/browse/DM-9520
436
437 ConfigClass = CalibrateConfig
438 _DefaultName = "calibrate"
439 RunnerClass = pipeBase.ButlerInitializedTaskRunner
440
441 def __init__(self, butler=None, astromRefObjLoader=None,
442 photoRefObjLoader=None, icSourceSchema=None,
443 initInputs=None, **kwargs):
444 """!Construct a CalibrateTask
445
446 @param[in] butler The butler is passed to the refObjLoader constructor
447 in case it is needed. Ignored if the refObjLoader argument
448 provides a loader directly.
449 @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks
450 that supplies an external reference catalog for astrometric
451 calibration. May be None if the desired loader can be constructed
452 from the butler argument or all steps requiring a reference catalog
453 are disabled.
454 @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks
455 that supplies an external reference catalog for photometric
456 calibration. May be None if the desired loader can be constructed
457 from the butler argument or all steps requiring a reference catalog
458 are disabled.
459 @param[in] icSourceSchema schema for icSource catalog, or None.
460 Schema values specified in config.icSourceFieldsToCopy will be
461 taken from this schema. If set to None, no values will be
462 propagated from the icSourceCatalog
463 @param[in,out] kwargs other keyword arguments for
464 lsst.pipe.base.CmdLineTask
465 """
466 super().__init__(**kwargs)
467
468 if icSourceSchema is None and butler is not None:
469 # Use butler to read icSourceSchema from disk.
470 icSourceSchema = butler.get("icSrc_schema", immediate=True).schema
471
472 if icSourceSchema is None and butler is None and initInputs is not None:
473 icSourceSchema = initInputs['icSourceSchema'].schema
474
475 if icSourceSchema is not None:
476 # use a schema mapper to avoid copying each field separately
477 self.schemaMapperschemaMapper = afwTable.SchemaMapper(icSourceSchema)
478 minimumSchema = afwTable.SourceTable.makeMinimalSchema()
479 self.schemaMapperschemaMapper.addMinimalSchema(minimumSchema, False)
480
481 # Add fields to copy from an icSource catalog
482 # and a field to indicate that the source matched a source in that
483 # catalog. If any fields are missing then raise an exception, but
484 # first find all missing fields in order to make the error message
485 # more useful.
486 self.calibSourceKeycalibSourceKey = self.schemaMapperschemaMapper.addOutputField(
487 afwTable.Field["Flag"]("calib_detected",
488 "Source was detected as an icSource"))
489 missingFieldNames = []
490 for fieldName in self.config.icSourceFieldsToCopy:
491 try:
492 schemaItem = icSourceSchema.find(fieldName)
493 except Exception:
494 missingFieldNames.append(fieldName)
495 else:
496 # field found; if addMapping fails then raise an exception
497 self.schemaMapperschemaMapper.addMapping(schemaItem.getKey())
498
499 if missingFieldNames:
500 raise RuntimeError("isSourceCat is missing fields {} "
501 "specified in icSourceFieldsToCopy"
502 .format(missingFieldNames))
503
504 # produce a temporary schema to pass to the subtasks; finalize it
505 # later
506 self.schemaschema = self.schemaMapperschemaMapper.editOutputSchema()
507 else:
508 self.schemaMapperschemaMapper = None
509 self.schemaschema = afwTable.SourceTable.makeMinimalSchema()
510 self.makeSubtask('detection', schema=self.schemaschema)
511
512 self.algMetadataalgMetadata = dafBase.PropertyList()
513
514 # Only create a subtask for fakes if configuration option is set
515 # N.B. the config for fake object task must be retargeted to a child
516 # of BaseFakeSourcesTask
517 if self.config.doInsertFakes:
518 self.makeSubtask("insertFakes")
519
520 if self.config.doDeblend:
521 self.makeSubtask("deblend", schema=self.schemaschema)
522 if self.config.doSkySources:
523 self.makeSubtask("skySources")
524 self.skySourceKeyskySourceKey = self.schemaschema.addField("sky_source", type="Flag", doc="Sky objects.")
525 self.makeSubtask('measurement', schema=self.schemaschema,
526 algMetadata=self.algMetadataalgMetadata)
527 self.makeSubtask('postCalibrationMeasurement', schema=self.schemaschema,
528 algMetadata=self.algMetadataalgMetadata)
529 self.makeSubtask("setPrimaryFlags", schema=self.schemaschema, isSingleFrame=True)
530 if self.config.doApCorr:
531 self.makeSubtask('applyApCorr', schema=self.schemaschema)
532 self.makeSubtask('catalogCalculation', schema=self.schemaschema)
533
534 if self.config.doAstrometry:
535 if astromRefObjLoader is None and butler is not None:
536 self.makeSubtask('astromRefObjLoader', butler=butler)
537 astromRefObjLoader = self.astromRefObjLoader
538 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader,
539 schema=self.schemaschema)
540 if self.config.doPhotoCal:
541 if photoRefObjLoader is None and butler is not None:
542 self.makeSubtask('photoRefObjLoader', butler=butler)
543 photoRefObjLoader = self.photoRefObjLoader
544 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader,
545 schema=self.schemaschema)
546 if self.config.doComputeSummaryStats:
547 self.makeSubtask('computeSummaryStats')
548
549 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None):
550 raise RuntimeError("PipelineTask form of this task should not be initialized with "
551 "reference object loaders.")
552
553 if self.schemaMapperschemaMapper is not None:
554 # finalize the schema
555 self.schemaschema = self.schemaMapperschemaMapper.getOutputSchema()
556 self.schemaschema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
557
558 sourceCatSchema = afwTable.SourceCatalog(self.schemaschema)
559 sourceCatSchema.getTable().setMetadata(self.algMetadataalgMetadata)
560 self.outputSchemaoutputSchema = sourceCatSchema
561
562 @timeMethod
563 def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None,
564 doUnpersist=True):
565 """!Calibrate an exposure, optionally unpersisting inputs and
566 persisting outputs.
567
568 This is a wrapper around the `run` method that unpersists inputs
569 (if `doUnpersist` true) and persists outputs (if `config.doWrite` true)
570
571 @param[in] dataRef butler data reference corresponding to a science
572 image
573 @param[in,out] exposure characterized exposure (an
574 lsst.afw.image.ExposureF or similar), or None to unpersist existing
575 icExp and icBackground. See `run` method for details of what is
576 read and written.
577 @param[in,out] background initial model of background already
578 subtracted from exposure (an lsst.afw.math.BackgroundList). May be
579 None if no background has been subtracted, though that is unusual
580 for calibration. A refined background model is output. Ignored if
581 exposure is None.
582 @param[in] icSourceCat catalog from which to copy the fields specified
583 by icSourceKeys, or None;
584 @param[in] doUnpersist unpersist data:
585 - if True, exposure, background and icSourceCat are read from
586 dataRef and those three arguments must all be None;
587 - if False the exposure must be provided; background and
588 icSourceCat are optional. True is intended for running as a
589 command-line task, False for running as a subtask
590 @return same data as the calibrate method
591 """
592 self.log.info("Processing %s", dataRef.dataId)
593
594 if doUnpersist:
595 if any(item is not None for item in (exposure, background,
596 icSourceCat)):
597 raise RuntimeError("doUnpersist true; exposure, background "
598 "and icSourceCat must all be None")
599 exposure = dataRef.get("icExp", immediate=True)
600 background = dataRef.get("icExpBackground", immediate=True)
601 icSourceCat = dataRef.get("icSrc", immediate=True)
602 elif exposure is None:
603 raise RuntimeError("doUnpersist false; exposure must be provided")
604
605 exposureIdInfo = dataRef.get("expIdInfo")
606
607 calRes = self.runrun(
608 exposure=exposure,
609 exposureIdInfo=exposureIdInfo,
610 background=background,
611 icSourceCat=icSourceCat,
612 )
613
614 if self.config.doWrite:
615 self.writeOutputswriteOutputs(
616 dataRef=dataRef,
617 exposure=calRes.exposure,
618 background=calRes.background,
619 sourceCat=calRes.sourceCat,
620 astromMatches=calRes.astromMatches,
621 matchMeta=calRes.matchMeta,
622 )
623
624 return calRes
625
626 def runQuantum(self, butlerQC, inputRefs, outputRefs):
627 inputs = butlerQC.get(inputRefs)
628 inputs['exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "visit_detector")
629
630 if self.config.doAstrometry:
631 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
632 for ref in inputRefs.astromRefCat],
633 refCats=inputs.pop('astromRefCat'),
634 config=self.config.astromRefObjLoader, log=self.log)
635 self.astrometry.setRefObjLoader(refObjLoader)
636
637 if self.config.doPhotoCal:
638 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
639 for ref in inputRefs.photoRefCat],
640 refCats=inputs.pop('photoRefCat'),
641 config=self.config.photoRefObjLoader,
642 log=self.log)
643 self.photoCal.match.setRefObjLoader(photoRefObjLoader)
644
645 outputs = self.runrun(**inputs)
646
647 if self.config.doWriteMatches and self.config.doAstrometry:
648 normalizedMatches = afwTable.packMatches(outputs.astromMatches)
649 normalizedMatches.table.setMetadata(outputs.matchMeta)
650 if self.config.doWriteMatchesDenormalized:
651 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
652 outputs.matchesDenormalized = denormMatches
653 outputs.matches = normalizedMatches
654 butlerQC.put(outputs, outputRefs)
655
656 @timeMethod
657 def run(self, exposure, exposureIdInfo=None, background=None,
658 icSourceCat=None):
659 """!Calibrate an exposure (science image or coadd)
660
661 @param[in,out] exposure exposure to calibrate (an
662 lsst.afw.image.ExposureF or similar);
663 in:
664 - MaskedImage
665 - Psf
666 out:
667 - MaskedImage has background subtracted
668 - Wcs is replaced
669 - PhotoCalib is replaced
670 @param[in] exposureIdInfo ID info for exposure (an
671 lsst.obs.base.ExposureIdInfo) If not provided, returned
672 SourceCatalog IDs will not be globally unique.
673 @param[in,out] background background model already subtracted from
674 exposure (an lsst.afw.math.BackgroundList). May be None if no
675 background has been subtracted, though that is unusual for
676 calibration. A refined background model is output.
677 @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
678 from which we can copy some fields.
679
680 @return pipe_base Struct containing these fields:
681 - exposure calibrate science exposure with refined WCS and PhotoCalib
682 - background model of background subtracted from exposure (an
683 lsst.afw.math.BackgroundList)
684 - sourceCat catalog of measured sources
685 - astromMatches list of source/refObj matches from the astrometry
686 solver
687 """
688 # detect, deblend and measure sources
689 if exposureIdInfo is None:
690 exposureIdInfo = ExposureIdInfo()
691
692 if background is None:
693 background = BackgroundList()
694 sourceIdFactory = exposureIdInfo.makeSourceIdFactory()
695 table = SourceTable.make(self.schemaschema, sourceIdFactory)
696 table.setMetadata(self.algMetadataalgMetadata)
697
698 detRes = self.detection.run(table=table, exposure=exposure,
699 doSmooth=True)
700 sourceCat = detRes.sources
701 if detRes.fpSets.background:
702 for bg in detRes.fpSets.background:
703 background.append(bg)
704 if self.config.doSkySources:
705 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId)
706 if skySourceFootprints:
707 for foot in skySourceFootprints:
708 s = sourceCat.addNew()
709 s.setFootprint(foot)
710 s.set(self.skySourceKeyskySourceKey, True)
711 if self.config.doDeblend:
712 self.deblend.run(exposure=exposure, sources=sourceCat)
713 self.measurement.run(
714 measCat=sourceCat,
715 exposure=exposure,
716 exposureId=exposureIdInfo.expId
717 )
718 if self.config.doApCorr:
719 self.applyApCorr.run(
720 catalog=sourceCat,
721 apCorrMap=exposure.getInfo().getApCorrMap()
722 )
723 self.catalogCalculation.run(sourceCat)
724
725 self.setPrimaryFlags.run(sourceCat)
726
727 if icSourceCat is not None and \
728 len(self.config.icSourceFieldsToCopy) > 0:
729 self.copyIcSourceFieldscopyIcSourceFields(icSourceCat=icSourceCat,
730 sourceCat=sourceCat)
731
732 # TODO DM-11568: this contiguous check-and-copy could go away if we
733 # reserve enough space during SourceDetection and/or SourceDeblend.
734 # NOTE: sourceSelectors require contiguous catalogs, so ensure
735 # contiguity now, so views are preserved from here on.
736 if not sourceCat.isContiguous():
737 sourceCat = sourceCat.copy(deep=True)
738
739 # perform astrometry calibration:
740 # fit an improved WCS and update the exposure's WCS in place
741 astromMatches = None
742 matchMeta = None
743 if self.config.doAstrometry:
744 try:
745 astromRes = self.astrometry.run(
746 exposure=exposure,
747 sourceCat=sourceCat,
748 )
749 astromMatches = astromRes.matches
750 matchMeta = astromRes.matchMeta
751 except Exception as e:
752 if self.config.requireAstrometry:
753 raise
754 self.log.warning("Unable to perform astrometric calibration "
755 "(%s): attempting to proceed", e)
756
757 # compute photometric calibration
758 if self.config.doPhotoCal:
759 try:
760 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
761 exposure.setPhotoCalib(photoRes.photoCalib)
762 # TODO: reword this to phrase it in terms of the calibration factor?
763 self.log.info("Photometric zero-point: %f",
764 photoRes.photoCalib.instFluxToMagnitude(1.0))
765 self.setMetadatasetMetadata(exposure=exposure, photoRes=photoRes)
766 except Exception as e:
767 if self.config.requirePhotoCal:
768 raise
769 self.log.warning("Unable to perform photometric calibration "
770 "(%s): attempting to proceed", e)
771 self.setMetadatasetMetadata(exposure=exposure, photoRes=None)
772
773 self.postCalibrationMeasurement.run(
774 measCat=sourceCat,
775 exposure=exposure,
776 exposureId=exposureIdInfo.expId
777 )
778
779 if self.config.doInsertFakes:
780 self.insertFakes.run(exposure, background=background)
781
782 table = SourceTable.make(self.schemaschema, sourceIdFactory)
783 table.setMetadata(self.algMetadataalgMetadata)
784
785 detRes = self.detection.run(table=table, exposure=exposure,
786 doSmooth=True)
787 sourceCat = detRes.sources
788 if detRes.fpSets.background:
789 for bg in detRes.fpSets.background:
790 background.append(bg)
791 if self.config.doDeblend:
792 self.deblend.run(exposure=exposure, sources=sourceCat)
793 self.measurement.run(
794 measCat=sourceCat,
795 exposure=exposure,
796 exposureId=exposureIdInfo.expId
797 )
798 self.postCalibrationMeasurement.run(
799 measCat=sourceCat,
800 exposure=exposure,
801 exposureId=exposureIdInfo.expId
802 )
803 if self.config.doApCorr:
804 self.applyApCorr.run(
805 catalog=sourceCat,
806 apCorrMap=exposure.getInfo().getApCorrMap()
807 )
808 self.catalogCalculation.run(sourceCat)
809
810 if icSourceCat is not None and len(self.config.icSourceFieldsToCopy) > 0:
811 self.copyIcSourceFieldscopyIcSourceFields(icSourceCat=icSourceCat,
812 sourceCat=sourceCat)
813
814 if self.config.doComputeSummaryStats:
815 summary = self.computeSummaryStats.run(exposure=exposure,
816 sources=sourceCat,
817 background=background)
818 exposure.getInfo().setSummaryStats(summary)
819
820 frame = getDebugFrame(self._display, "calibrate")
821 if frame:
822 displayAstrometry(
823 sourceCat=sourceCat,
824 exposure=exposure,
825 matches=astromMatches,
826 frame=frame,
827 pause=False,
828 )
829
830 return pipeBase.Struct(
831 exposure=exposure,
832 background=background,
833 sourceCat=sourceCat,
834 astromMatches=astromMatches,
835 matchMeta=matchMeta,
836 # These are duplicate entries with different names for use with
837 # gen3 middleware
838 outputExposure=exposure,
839 outputCat=sourceCat,
840 outputBackground=background,
841 )
842
843 def writeOutputs(self, dataRef, exposure, background, sourceCat,
844 astromMatches, matchMeta):
845 """Write output data to the output repository
846
847 @param[in] dataRef butler data reference corresponding to a science
848 image
849 @param[in] exposure exposure to write
850 @param[in] background background model for exposure
851 @param[in] sourceCat catalog of measured sources
852 @param[in] astromMatches list of source/refObj matches from the
853 astrometry solver
854 """
855 dataRef.put(sourceCat, "src")
856 if self.config.doWriteMatches and astromMatches is not None:
857 normalizedMatches = afwTable.packMatches(astromMatches)
858 normalizedMatches.table.setMetadata(matchMeta)
859 dataRef.put(normalizedMatches, "srcMatch")
860 if self.config.doWriteMatchesDenormalized:
861 denormMatches = denormalizeMatches(astromMatches, matchMeta)
862 dataRef.put(denormMatches, "srcMatchFull")
863 if self.config.doWriteExposure:
864 dataRef.put(exposure, "calexp")
865 dataRef.put(background, "calexpBackground")
866
868 """Return a dict of empty catalogs for each catalog dataset produced
869 by this task.
870 """
871 sourceCat = afwTable.SourceCatalog(self.schemaschema)
872 sourceCat.getTable().setMetadata(self.algMetadataalgMetadata)
873 return {"src": sourceCat}
874
875 def setMetadata(self, exposure, photoRes=None):
876 """!Set task and exposure metadata
877
878 Logs a warning and continues if needed data is missing.
879
880 @param[in,out] exposure exposure whose metadata is to be set
881 @param[in] photoRes results of running photoCal; if None then it was
882 not run
883 """
884 if photoRes is None:
885 return
886
887 metadata = exposure.getMetadata()
888
889 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata
890 try:
891 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
892 magZero = photoRes.zp - 2.5*math.log10(exposureTime)
893 except Exception:
894 self.log.warning("Could not set normalized MAGZERO in header: no "
895 "exposure time")
896 magZero = math.nan
897
898 try:
899 metadata.set('MAGZERO', magZero)
900 metadata.set('MAGZERO_RMS', photoRes.sigma)
901 metadata.set('MAGZERO_NOBJ', photoRes.ngood)
902 metadata.set('COLORTERM1', 0.0)
903 metadata.set('COLORTERM2', 0.0)
904 metadata.set('COLORTERM3', 0.0)
905 except Exception as e:
906 self.log.warning("Could not set exposure metadata: %s", e)
907
908 def copyIcSourceFields(self, icSourceCat, sourceCat):
909 """!Match sources in icSourceCat and sourceCat and copy the specified fields
910
911 @param[in] icSourceCat catalog from which to copy fields
912 @param[in,out] sourceCat catalog to which to copy fields
913
914 The fields copied are those specified by `config.icSourceFieldsToCopy`
915 that actually exist in the schema. This was set up by the constructor
916 using self.schemaMapperschemaMapper.
917 """
918 if self.schemaMapperschemaMapper is None:
919 raise RuntimeError("To copy icSource fields you must specify "
920 "icSourceSchema nd icSourceKeys when "
921 "constructing this task")
922 if icSourceCat is None or sourceCat is None:
923 raise RuntimeError("icSourceCat and sourceCat must both be "
924 "specified")
925 if len(self.config.icSourceFieldsToCopy) == 0:
926 self.log.warning("copyIcSourceFields doing nothing because "
927 "icSourceFieldsToCopy is empty")
928 return
929
930 mc = afwTable.MatchControl()
931 mc.findOnlyClosest = False # return all matched objects
932 matches = afwTable.matchXy(icSourceCat, sourceCat,
933 self.config.matchRadiusPix, mc)
934 if self.config.doDeblend:
935 deblendKey = sourceCat.schema["deblend_nChild"].asKey()
936 # if deblended, keep children
937 matches = [m for m in matches if m[1].get(deblendKey) == 0]
938
939 # Because we had to allow multiple matches to handle parents, we now
940 # need to prune to the best matches
941 # closest matches as a dict of icSourceCat source ID:
942 # (icSourceCat source, sourceCat source, distance in pixels)
943 bestMatches = {}
944 for m0, m1, d in matches:
945 id0 = m0.getId()
946 match = bestMatches.get(id0)
947 if match is None or d <= match[2]:
948 bestMatches[id0] = (m0, m1, d)
949 matches = list(bestMatches.values())
950
951 # Check that no sourceCat sources are listed twice (we already know
952 # that each match has a unique icSourceCat source ID, due to using
953 # that ID as the key in bestMatches)
954 numMatches = len(matches)
955 numUniqueSources = len(set(m[1].getId() for m in matches))
956 if numUniqueSources != numMatches:
957 self.log.warning("%d icSourceCat sources matched only %d sourceCat "
958 "sources", numMatches, numUniqueSources)
959
960 self.log.info("Copying flags from icSourceCat to sourceCat for "
961 "%d sources", numMatches)
962
963 # For each match: set the calibSourceKey flag and copy the desired
964 # fields
965 for icSrc, src, d in matches:
966 src.setFlag(self.calibSourceKeycalibSourceKey, True)
967 # src.assign copies the footprint from icSrc, which we don't want
968 # (DM-407)
969 # so set icSrc's footprint to src's footprint before src.assign,
970 # then restore it
971 icSrcFootprint = icSrc.getFootprint()
972 try:
973 icSrc.setFootprint(src.getFootprint())
974 src.assign(icSrc, self.schemaMapperschemaMapper)
975 finally:
976 icSrc.setFootprint(icSrcFootprint)
Calibrate an exposure: measure sources and perform astrometric and photometric calibration.
Definition: calibrate.py:344
def writeOutputs(self, dataRef, exposure, background, sourceCat, astromMatches, matchMeta)
Definition: calibrate.py:844
def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None, doUnpersist=True)
Calibrate an exposure, optionally unpersisting inputs and persisting outputs.
Definition: calibrate.py:564
def __init__(self, butler=None, astromRefObjLoader=None, photoRefObjLoader=None, icSourceSchema=None, initInputs=None, **kwargs)
Construct a CalibrateTask.
Definition: calibrate.py:443
def setMetadata(self, exposure, photoRes=None)
Set task and exposure metadata.
Definition: calibrate.py:875
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: calibrate.py:626
def copyIcSourceFields(self, icSourceCat, sourceCat)
Match sources in icSourceCat and sourceCat and copy the specified fields.
Definition: calibrate.py:908
def run(self, exposure, exposureIdInfo=None, background=None, icSourceCat=None)
Calibrate an exposure (science image or coadd)
Definition: calibrate.py:658