lsst.pipe.tasks g226e71a814+02e4f8badd
calibrate.py
Go to the documentation of this file.
2# LSST Data Management System
3# Copyright 2008-2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22import math
23
24from lsstDebug import getDebugFrame
25import lsst.pex.config as pexConfig
26import lsst.pipe.base as pipeBase
27import lsst.pipe.base.connectionTypes as cT
28import lsst.afw.table as afwTable
29from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches
30from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, SkyObjectsTask
31from lsst.obs.base import ExposureIdInfo
32import lsst.daf.base as dafBase
33from lsst.afw.math import BackgroundList
34from lsst.afw.table import SourceTable
35from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader
36from lsst.meas.base import (SingleFrameMeasurementTask,
37 ApplyApCorrTask,
38 CatalogCalculationTask)
39from lsst.meas.deblender import SourceDeblendTask
40from lsst.utils.timer import timeMethod
41from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask
42from .fakes import BaseFakeSourcesTask
43from .photoCal import PhotoCalTask
44from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask
45
46
47__all__ = ["CalibrateConfig", "CalibrateTask"]
48
49
50class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"),
51 defaultTemplates={}):
52
53 icSourceSchema = cT.InitInput(
54 doc="Schema produced by characterize image task, used to initialize this task",
55 name="icSrc_schema",
56 storageClass="SourceCatalog",
57 )
58
59 outputSchema = cT.InitOutput(
60 doc="Schema after CalibrateTask has been initialized",
61 name="src_schema",
62 storageClass="SourceCatalog",
63 )
64
65 exposure = cT.Input(
66 doc="Input image to calibrate",
67 name="icExp",
68 storageClass="ExposureF",
69 dimensions=("instrument", "visit", "detector"),
70 )
71
72 background = cT.Input(
73 doc="Backgrounds determined by characterize task",
74 name="icExpBackground",
75 storageClass="Background",
76 dimensions=("instrument", "visit", "detector"),
77 )
78
79 icSourceCat = cT.Input(
80 doc="Source catalog created by characterize task",
81 name="icSrc",
82 storageClass="SourceCatalog",
83 dimensions=("instrument", "visit", "detector"),
84 )
85
86 astromRefCat = cT.PrerequisiteInput(
87 doc="Reference catalog to use for astrometry",
88 name="cal_ref_cat",
89 storageClass="SimpleCatalog",
90 dimensions=("skypix",),
91 deferLoad=True,
92 multiple=True,
93 )
94
95 photoRefCat = cT.PrerequisiteInput(
96 doc="Reference catalog to use for photometric calibration",
97 name="cal_ref_cat",
98 storageClass="SimpleCatalog",
99 dimensions=("skypix",),
100 deferLoad=True,
101 multiple=True
102 )
103
104 outputExposure = cT.Output(
105 doc="Exposure after running calibration task",
106 name="calexp",
107 storageClass="ExposureF",
108 dimensions=("instrument", "visit", "detector"),
109 )
110
111 outputCat = cT.Output(
112 doc="Source catalog produced in calibrate task",
113 name="src",
114 storageClass="SourceCatalog",
115 dimensions=("instrument", "visit", "detector"),
116 )
117
118 outputBackground = cT.Output(
119 doc="Background models estimated in calibration task",
120 name="calexpBackground",
121 storageClass="Background",
122 dimensions=("instrument", "visit", "detector"),
123 )
124
125 matches = cT.Output(
126 doc="Source/refObj matches from the astrometry solver",
127 name="srcMatch",
128 storageClass="Catalog",
129 dimensions=("instrument", "visit", "detector"),
130 )
131
132 matchesDenormalized = cT.Output(
133 doc="Denormalized matches from astrometry solver",
134 name="srcMatchFull",
135 storageClass="Catalog",
136 dimensions=("instrument", "visit", "detector"),
137 )
138
139 def __init__(self, *, config=None):
140 super().__init__(config=config)
141
142 if config.doAstrometry is False:
143 self.prerequisiteInputs.remove("astromRefCat")
144 if config.doPhotoCal is False:
145 self.prerequisiteInputs.remove("photoRefCat")
146
147 if config.doWriteMatches is False or config.doAstrometry is False:
148 self.outputs.remove("matches")
149 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False:
150 self.outputs.remove("matchesDenormalized")
151
152
153class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections):
154 """Config for CalibrateTask"""
155 doWrite = pexConfig.Field(
156 dtype=bool,
157 default=True,
158 doc="Save calibration results?",
159 )
160 doWriteHeavyFootprintsInSources = pexConfig.Field(
161 dtype=bool,
162 default=True,
163 doc="Include HeavyFootprint data in source table? If false then heavy "
164 "footprints are saved as normal footprints, which saves some space"
165 )
166 doWriteMatches = pexConfig.Field(
167 dtype=bool,
168 default=True,
169 doc="Write reference matches (ignored if doWrite or doAstrometry false)?",
170 )
171 doWriteMatchesDenormalized = pexConfig.Field(
172 dtype=bool,
173 default=False,
174 doc=("Write reference matches in denormalized format? "
175 "This format uses more disk space, but is more convenient to "
176 "read. Ignored if doWriteMatches=False or doWrite=False."),
177 )
178 doAstrometry = pexConfig.Field(
179 dtype=bool,
180 default=True,
181 doc="Perform astrometric calibration?",
182 )
183 astromRefObjLoader = pexConfig.ConfigurableField(
184 target=LoadIndexedReferenceObjectsTask,
185 doc="reference object loader for astrometric calibration",
186 )
187 photoRefObjLoader = pexConfig.ConfigurableField(
188 target=LoadIndexedReferenceObjectsTask,
189 doc="reference object loader for photometric calibration",
190 )
191 astrometry = pexConfig.ConfigurableField(
192 target=AstrometryTask,
193 doc="Perform astrometric calibration to refine the WCS",
194 )
195 requireAstrometry = pexConfig.Field(
196 dtype=bool,
197 default=True,
198 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry "
199 "false."),
200 )
201 doPhotoCal = pexConfig.Field(
202 dtype=bool,
203 default=True,
204 doc="Perform phometric calibration?",
205 )
206 requirePhotoCal = pexConfig.Field(
207 dtype=bool,
208 default=True,
209 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal "
210 "false."),
211 )
212 photoCal = pexConfig.ConfigurableField(
213 target=PhotoCalTask,
214 doc="Perform photometric calibration",
215 )
216 icSourceFieldsToCopy = pexConfig.ListField(
217 dtype=str,
218 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
219 doc=("Fields to copy from the icSource catalog to the output catalog "
220 "for matching sources Any missing fields will trigger a "
221 "RuntimeError exception. Ignored if icSourceCat is not provided.")
222 )
223 matchRadiusPix = pexConfig.Field(
224 dtype=float,
225 default=3,
226 doc=("Match radius for matching icSourceCat objects to sourceCat "
227 "objects (pixels)"),
228 )
229 checkUnitsParseStrict = pexConfig.Field(
230 doc=("Strictness of Astropy unit compatibility check, can be 'raise', "
231 "'warn' or 'silent'"),
232 dtype=str,
233 default="raise",
234 )
235 detection = pexConfig.ConfigurableField(
236 target=SourceDetectionTask,
237 doc="Detect sources"
238 )
239 doDeblend = pexConfig.Field(
240 dtype=bool,
241 default=True,
242 doc="Run deblender input exposure"
243 )
244 deblend = pexConfig.ConfigurableField(
245 target=SourceDeblendTask,
246 doc="Split blended sources into their components"
247 )
248 doSkySources = pexConfig.Field(
249 dtype=bool,
250 default=True,
251 doc="Generate sky sources?",
252 )
253 skySources = pexConfig.ConfigurableField(
254 target=SkyObjectsTask,
255 doc="Generate sky sources",
256 )
257 measurement = pexConfig.ConfigurableField(
258 target=SingleFrameMeasurementTask,
259 doc="Measure sources"
260 )
261 postCalibrationMeasurement = pexConfig.ConfigurableField(
262 target=SingleFrameMeasurementTask,
263 doc="Second round of measurement for plugins that need to be run after photocal"
264 )
265 setPrimaryFlags = pexConfig.ConfigurableField(
266 target=SetPrimaryFlagsTask,
267 doc=("Set flags for primary source classification in single frame "
268 "processing. True if sources are not sky sources and not a parent.")
269 )
270 doApCorr = pexConfig.Field(
271 dtype=bool,
272 default=True,
273 doc="Run subtask to apply aperture correction"
274 )
275 applyApCorr = pexConfig.ConfigurableField(
276 target=ApplyApCorrTask,
277 doc="Subtask to apply aperture corrections"
278 )
279 # If doApCorr is False, and the exposure does not have apcorrections
280 # already applied, the active plugins in catalogCalculation almost
281 # certainly should not contain the characterization plugin
282 catalogCalculation = pexConfig.ConfigurableField(
283 target=CatalogCalculationTask,
284 doc="Subtask to run catalogCalculation plugins on catalog"
285 )
286 doInsertFakes = pexConfig.Field(
287 dtype=bool,
288 default=False,
289 doc="Run fake sources injection task",
290 deprecated=("doInsertFakes is no longer supported. This config will be removed after v24. "
291 "Please use ProcessCcdWithFakesTask instead.")
292 )
293 insertFakes = pexConfig.ConfigurableField(
294 target=BaseFakeSourcesTask,
295 doc="Injection of fake sources for testing purposes (must be "
296 "retargeted)",
297 deprecated=("insertFakes is no longer supported. This config will be removed after v24. "
298 "Please use ProcessCcdWithFakesTask instead.")
299 )
300 doComputeSummaryStats = pexConfig.Field(
301 dtype=bool,
302 default=True,
303 doc="Run subtask to measure exposure summary statistics?"
304 )
305 computeSummaryStats = pexConfig.ConfigurableField(
306 target=ComputeExposureSummaryStatsTask,
307 doc="Subtask to run computeSummaryStats on exposure"
308 )
309 doWriteExposure = pexConfig.Field(
310 dtype=bool,
311 default=True,
312 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a "
313 "normal calexp but as a fakes_calexp."
314 )
315
316 def setDefaults(self):
317 super().setDefaults()
318 self.detectiondetection.doTempLocalBackground = False
319 self.deblenddeblend.maxFootprintSize = 2000
320 self.postCalibrationMeasurementpostCalibrationMeasurement.plugins.names = ["base_LocalPhotoCalib", "base_LocalWcs"]
321 self.postCalibrationMeasurementpostCalibrationMeasurement.doReplaceWithNoise = False
322 for key in self.postCalibrationMeasurementpostCalibrationMeasurement.slots:
323 setattr(self.postCalibrationMeasurementpostCalibrationMeasurement.slots, key, None)
324
325 def validate(self):
326 super().validate()
327 astromRefCatGen2 = getattr(self.astromRefObjLoaderastromRefObjLoader, "ref_dataset_name", None)
328 if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat:
329 raise ValueError(
330 f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference "
331 f"catalogs are different. These options must be kept in sync until Gen2 is retired."
332 )
333 photoRefCatGen2 = getattr(self.photoRefObjLoaderphotoRefObjLoader, "ref_dataset_name", None)
334 if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat:
335 raise ValueError(
336 f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference "
337 f"catalogs are different. These options must be kept in sync until Gen2 is retired."
338 )
339
340
341
347
348class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
349 r"""!Calibrate an exposure: measure sources and perform astrometric and
350 photometric calibration
351
352 @anchor CalibrateTask_
353
354 @section pipe_tasks_calibrate_Contents Contents
355
356 - @ref pipe_tasks_calibrate_Purpose
357 - @ref pipe_tasks_calibrate_Initialize
358 - @ref pipe_tasks_calibrate_IO
359 - @ref pipe_tasks_calibrate_Config
360 - @ref pipe_tasks_calibrate_Metadata
361 - @ref pipe_tasks_calibrate_Debug
362
363 @section pipe_tasks_calibrate_Purpose Description
364
365 Given an exposure with a good PSF model and aperture correction map
366 (e.g. as provided by @ref characterizeImage::CharacterizeImageTask "CharacterizeImageTask"),
367 perform the following operations:
368 - Run detection and measurement
369 - Run astrometry subtask to fit an improved WCS
370 - Run photoCal subtask to fit the exposure's photometric zero-point
371
372 @section pipe_tasks_calibrate_Initialize Task initialisation
373
374 @copydoc \_\_init\_\_
375
376 @section pipe_tasks_calibrate_IO Invoking the Task
377
378 If you want this task to unpersist inputs or persist outputs, then call
379 the `runDataRef` method (a wrapper around the `run` method).
380
381 If you already have the inputs unpersisted and do not want to persist the
382 output then it is more direct to call the `run` method:
383
384 @section pipe_tasks_calibrate_Config Configuration parameters
385
386 See @ref CalibrateConfig
387
388 @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata
389
390 Exposure metadata
391 <dl>
392 <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task
393 <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal
394 task
395 <dt>COLORTERM1 <dd>?? (always 0.0)
396 <dt>COLORTERM2 <dd>?? (always 0.0)
397 <dt>COLORTERM3 <dd>?? (always 0.0)
398 </dl>
399
400 @section pipe_tasks_calibrate_Debug Debug variables
401
402 The command line task
403 interface supports a flag
404 `--debug` to import `debug.py` from your `$PYTHONPATH`; see
405 <a href="https://pipelines.lsst.io/modules/lsstDebug/">the lsstDebug documentation</a>
406 for more about `debug.py`.
407
408 CalibrateTask has a debug dictionary containing one key:
409 <dl>
410 <dt>calibrate
411 <dd>frame (an int; <= 0 to not display) in which to display the exposure,
412 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for
413 the meaning of the various symbols.
414 </dl>
415
416 For example, put something like:
417 @code{.py}
418 import lsstDebug
419 def DebugInfo(name):
420 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would
421 # call us recursively
422 if name == "lsst.pipe.tasks.calibrate":
423 di.display = dict(
424 calibrate = 1,
425 )
426
427 return di
428
429 lsstDebug.Info = DebugInfo
430 @endcode
431 into your `debug.py` file and run `calibrateTask.py` with the `--debug`
432 flag.
433
434 Some subtasks may have their own debug variables; see individual Task
435 documentation.
436 """
437
438 # Example description used to live here, removed 2-20-2017 as per
439 # https://jira.lsstcorp.org/browse/DM-9520
440
441 ConfigClass = CalibrateConfig
442 _DefaultName = "calibrate"
443 RunnerClass = pipeBase.ButlerInitializedTaskRunner
444
445 def __init__(self, butler=None, astromRefObjLoader=None,
446 photoRefObjLoader=None, icSourceSchema=None,
447 initInputs=None, **kwargs):
448 """!Construct a CalibrateTask
449
450 @param[in] butler The butler is passed to the refObjLoader constructor
451 in case it is needed. Ignored if the refObjLoader argument
452 provides a loader directly.
453 @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks
454 that supplies an external reference catalog for astrometric
455 calibration. May be None if the desired loader can be constructed
456 from the butler argument or all steps requiring a reference catalog
457 are disabled.
458 @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks
459 that supplies an external reference catalog for photometric
460 calibration. May be None if the desired loader can be constructed
461 from the butler argument or all steps requiring a reference catalog
462 are disabled.
463 @param[in] icSourceSchema schema for icSource catalog, or None.
464 Schema values specified in config.icSourceFieldsToCopy will be
465 taken from this schema. If set to None, no values will be
466 propagated from the icSourceCatalog
467 @param[in,out] kwargs other keyword arguments for
468 lsst.pipe.base.CmdLineTask
469 """
470 super().__init__(**kwargs)
471
472 if icSourceSchema is None and butler is not None:
473 # Use butler to read icSourceSchema from disk.
474 icSourceSchema = butler.get("icSrc_schema", immediate=True).schema
475
476 if icSourceSchema is None and butler is None and initInputs is not None:
477 icSourceSchema = initInputs['icSourceSchema'].schema
478
479 if icSourceSchema is not None:
480 # use a schema mapper to avoid copying each field separately
481 self.schemaMapperschemaMapper = afwTable.SchemaMapper(icSourceSchema)
482 minimumSchema = afwTable.SourceTable.makeMinimalSchema()
483 self.schemaMapperschemaMapper.addMinimalSchema(minimumSchema, False)
484
485 # Add fields to copy from an icSource catalog
486 # and a field to indicate that the source matched a source in that
487 # catalog. If any fields are missing then raise an exception, but
488 # first find all missing fields in order to make the error message
489 # more useful.
490 self.calibSourceKeycalibSourceKey = self.schemaMapperschemaMapper.addOutputField(
491 afwTable.Field["Flag"]("calib_detected",
492 "Source was detected as an icSource"))
493 missingFieldNames = []
494 for fieldName in self.config.icSourceFieldsToCopy:
495 try:
496 schemaItem = icSourceSchema.find(fieldName)
497 except Exception:
498 missingFieldNames.append(fieldName)
499 else:
500 # field found; if addMapping fails then raise an exception
501 self.schemaMapperschemaMapper.addMapping(schemaItem.getKey())
502
503 if missingFieldNames:
504 raise RuntimeError("isSourceCat is missing fields {} "
505 "specified in icSourceFieldsToCopy"
506 .format(missingFieldNames))
507
508 # produce a temporary schema to pass to the subtasks; finalize it
509 # later
510 self.schemaschema = self.schemaMapperschemaMapper.editOutputSchema()
511 else:
512 self.schemaMapperschemaMapper = None
513 self.schemaschema = afwTable.SourceTable.makeMinimalSchema()
514 self.makeSubtask('detection', schema=self.schemaschema)
515
516 self.algMetadataalgMetadata = dafBase.PropertyList()
517
518 if self.config.doDeblend:
519 self.makeSubtask("deblend", schema=self.schemaschema)
520 if self.config.doSkySources:
521 self.makeSubtask("skySources")
522 self.skySourceKeyskySourceKey = self.schemaschema.addField("sky_source", type="Flag", doc="Sky objects.")
523 self.makeSubtask('measurement', schema=self.schemaschema,
524 algMetadata=self.algMetadataalgMetadata)
525 self.makeSubtask('postCalibrationMeasurement', schema=self.schemaschema,
526 algMetadata=self.algMetadataalgMetadata)
527 self.makeSubtask("setPrimaryFlags", schema=self.schemaschema, isSingleFrame=True)
528 if self.config.doApCorr:
529 self.makeSubtask('applyApCorr', schema=self.schemaschema)
530 self.makeSubtask('catalogCalculation', schema=self.schemaschema)
531
532 if self.config.doAstrometry:
533 if astromRefObjLoader is None and butler is not None:
534 self.makeSubtask('astromRefObjLoader', butler=butler)
535 astromRefObjLoader = self.astromRefObjLoader
536 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader,
537 schema=self.schemaschema)
538 if self.config.doPhotoCal:
539 if photoRefObjLoader is None and butler is not None:
540 self.makeSubtask('photoRefObjLoader', butler=butler)
541 photoRefObjLoader = self.photoRefObjLoader
542 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader,
543 schema=self.schemaschema)
544 if self.config.doComputeSummaryStats:
545 self.makeSubtask('computeSummaryStats')
546
547 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None):
548 raise RuntimeError("PipelineTask form of this task should not be initialized with "
549 "reference object loaders.")
550
551 if self.schemaMapperschemaMapper is not None:
552 # finalize the schema
553 self.schemaschema = self.schemaMapperschemaMapper.getOutputSchema()
554 self.schemaschema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
555
556 sourceCatSchema = afwTable.SourceCatalog(self.schemaschema)
557 sourceCatSchema.getTable().setMetadata(self.algMetadataalgMetadata)
558 self.outputSchemaoutputSchema = sourceCatSchema
559
560 @timeMethod
561 def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None,
562 doUnpersist=True):
563 """!Calibrate an exposure, optionally unpersisting inputs and
564 persisting outputs.
565
566 This is a wrapper around the `run` method that unpersists inputs
567 (if `doUnpersist` true) and persists outputs (if `config.doWrite` true)
568
569 @param[in] dataRef butler data reference corresponding to a science
570 image
571 @param[in,out] exposure characterized exposure (an
572 lsst.afw.image.ExposureF or similar), or None to unpersist existing
573 icExp and icBackground. See `run` method for details of what is
574 read and written.
575 @param[in,out] background initial model of background already
576 subtracted from exposure (an lsst.afw.math.BackgroundList). May be
577 None if no background has been subtracted, though that is unusual
578 for calibration. A refined background model is output. Ignored if
579 exposure is None.
580 @param[in] icSourceCat catalog from which to copy the fields specified
581 by icSourceKeys, or None;
582 @param[in] doUnpersist unpersist data:
583 - if True, exposure, background and icSourceCat are read from
584 dataRef and those three arguments must all be None;
585 - if False the exposure must be provided; background and
586 icSourceCat are optional. True is intended for running as a
587 command-line task, False for running as a subtask
588 @return same data as the calibrate method
589 """
590 self.log.info("Processing %s", dataRef.dataId)
591
592 if doUnpersist:
593 if any(item is not None for item in (exposure, background,
594 icSourceCat)):
595 raise RuntimeError("doUnpersist true; exposure, background "
596 "and icSourceCat must all be None")
597 exposure = dataRef.get("icExp", immediate=True)
598 background = dataRef.get("icExpBackground", immediate=True)
599 icSourceCat = dataRef.get("icSrc", immediate=True)
600 elif exposure is None:
601 raise RuntimeError("doUnpersist false; exposure must be provided")
602
603 exposureIdInfo = dataRef.get("expIdInfo")
604
605 calRes = self.runrun(
606 exposure=exposure,
607 exposureIdInfo=exposureIdInfo,
608 background=background,
609 icSourceCat=icSourceCat,
610 )
611
612 if self.config.doWrite:
613 self.writeOutputswriteOutputs(
614 dataRef=dataRef,
615 exposure=calRes.exposure,
616 background=calRes.background,
617 sourceCat=calRes.sourceCat,
618 astromMatches=calRes.astromMatches,
619 matchMeta=calRes.matchMeta,
620 )
621
622 return calRes
623
624 def runQuantum(self, butlerQC, inputRefs, outputRefs):
625 inputs = butlerQC.get(inputRefs)
626 inputs['exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "visit_detector")
627
628 if self.config.doAstrometry:
629 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
630 for ref in inputRefs.astromRefCat],
631 refCats=inputs.pop('astromRefCat'),
632 config=self.config.astromRefObjLoader, log=self.log)
633 self.astrometry.setRefObjLoader(refObjLoader)
634
635 if self.config.doPhotoCal:
636 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
637 for ref in inputRefs.photoRefCat],
638 refCats=inputs.pop('photoRefCat'),
639 config=self.config.photoRefObjLoader,
640 log=self.log)
641 self.photoCal.match.setRefObjLoader(photoRefObjLoader)
642
643 outputs = self.runrun(**inputs)
644
645 if self.config.doWriteMatches and self.config.doAstrometry:
646 normalizedMatches = afwTable.packMatches(outputs.astromMatches)
647 normalizedMatches.table.setMetadata(outputs.matchMeta)
648 if self.config.doWriteMatchesDenormalized:
649 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
650 outputs.matchesDenormalized = denormMatches
651 outputs.matches = normalizedMatches
652 butlerQC.put(outputs, outputRefs)
653
654 @timeMethod
655 def run(self, exposure, exposureIdInfo=None, background=None,
656 icSourceCat=None):
657 """!Calibrate an exposure (science image or coadd)
658
659 @param[in,out] exposure exposure to calibrate (an
660 lsst.afw.image.ExposureF or similar);
661 in:
662 - MaskedImage
663 - Psf
664 out:
665 - MaskedImage has background subtracted
666 - Wcs is replaced
667 - PhotoCalib is replaced
668 @param[in] exposureIdInfo ID info for exposure (an
669 lsst.obs.base.ExposureIdInfo) If not provided, returned
670 SourceCatalog IDs will not be globally unique.
671 @param[in,out] background background model already subtracted from
672 exposure (an lsst.afw.math.BackgroundList). May be None if no
673 background has been subtracted, though that is unusual for
674 calibration. A refined background model is output.
675 @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
676 from which we can copy some fields.
677
678 @return pipe_base Struct containing these fields:
679 - exposure calibrate science exposure with refined WCS and PhotoCalib
680 - background model of background subtracted from exposure (an
681 lsst.afw.math.BackgroundList)
682 - sourceCat catalog of measured sources
683 - astromMatches list of source/refObj matches from the astrometry
684 solver
685 """
686 # detect, deblend and measure sources
687 if exposureIdInfo is None:
688 exposureIdInfo = ExposureIdInfo()
689
690 if background is None:
691 background = BackgroundList()
692 sourceIdFactory = exposureIdInfo.makeSourceIdFactory()
693 table = SourceTable.make(self.schemaschema, sourceIdFactory)
694 table.setMetadata(self.algMetadataalgMetadata)
695
696 detRes = self.detection.run(table=table, exposure=exposure,
697 doSmooth=True)
698 sourceCat = detRes.sources
699 if detRes.fpSets.background:
700 for bg in detRes.fpSets.background:
701 background.append(bg)
702 if self.config.doSkySources:
703 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId)
704 if skySourceFootprints:
705 for foot in skySourceFootprints:
706 s = sourceCat.addNew()
707 s.setFootprint(foot)
708 s.set(self.skySourceKeyskySourceKey, True)
709 if self.config.doDeblend:
710 self.deblend.run(exposure=exposure, sources=sourceCat)
711 self.measurement.run(
712 measCat=sourceCat,
713 exposure=exposure,
714 exposureId=exposureIdInfo.expId
715 )
716 if self.config.doApCorr:
717 self.applyApCorr.run(
718 catalog=sourceCat,
719 apCorrMap=exposure.getInfo().getApCorrMap()
720 )
721 self.catalogCalculation.run(sourceCat)
722
723 self.setPrimaryFlags.run(sourceCat)
724
725 if icSourceCat is not None and \
726 len(self.config.icSourceFieldsToCopy) > 0:
727 self.copyIcSourceFieldscopyIcSourceFields(icSourceCat=icSourceCat,
728 sourceCat=sourceCat)
729
730 # TODO DM-11568: this contiguous check-and-copy could go away if we
731 # reserve enough space during SourceDetection and/or SourceDeblend.
732 # NOTE: sourceSelectors require contiguous catalogs, so ensure
733 # contiguity now, so views are preserved from here on.
734 if not sourceCat.isContiguous():
735 sourceCat = sourceCat.copy(deep=True)
736
737 # perform astrometry calibration:
738 # fit an improved WCS and update the exposure's WCS in place
739 astromMatches = None
740 matchMeta = None
741 if self.config.doAstrometry:
742 try:
743 astromRes = self.astrometry.run(
744 exposure=exposure,
745 sourceCat=sourceCat,
746 )
747 astromMatches = astromRes.matches
748 matchMeta = astromRes.matchMeta
749 except Exception as e:
750 if self.config.requireAstrometry:
751 raise
752 self.log.warning("Unable to perform astrometric calibration "
753 "(%s): attempting to proceed", e)
754
755 # compute photometric calibration
756 if self.config.doPhotoCal:
757 try:
758 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
759 exposure.setPhotoCalib(photoRes.photoCalib)
760 # TODO: reword this to phrase it in terms of the calibration factor?
761 self.log.info("Photometric zero-point: %f",
762 photoRes.photoCalib.instFluxToMagnitude(1.0))
763 self.setMetadatasetMetadata(exposure=exposure, photoRes=photoRes)
764 except Exception as e:
765 if self.config.requirePhotoCal:
766 raise
767 self.log.warning("Unable to perform photometric calibration "
768 "(%s): attempting to proceed", e)
769 self.setMetadatasetMetadata(exposure=exposure, photoRes=None)
770
771 self.postCalibrationMeasurement.run(
772 measCat=sourceCat,
773 exposure=exposure,
774 exposureId=exposureIdInfo.expId
775 )
776
777 if self.config.doComputeSummaryStats:
778 summary = self.computeSummaryStats.run(exposure=exposure,
779 sources=sourceCat,
780 background=background)
781 exposure.getInfo().setSummaryStats(summary)
782
783 frame = getDebugFrame(self._display, "calibrate")
784 if frame:
785 displayAstrometry(
786 sourceCat=sourceCat,
787 exposure=exposure,
788 matches=astromMatches,
789 frame=frame,
790 pause=False,
791 )
792
793 return pipeBase.Struct(
794 exposure=exposure,
795 background=background,
796 sourceCat=sourceCat,
797 astromMatches=astromMatches,
798 matchMeta=matchMeta,
799 # These are duplicate entries with different names for use with
800 # gen3 middleware
801 outputExposure=exposure,
802 outputCat=sourceCat,
803 outputBackground=background,
804 )
805
806 def writeOutputs(self, dataRef, exposure, background, sourceCat,
807 astromMatches, matchMeta):
808 """Write output data to the output repository
809
810 @param[in] dataRef butler data reference corresponding to a science
811 image
812 @param[in] exposure exposure to write
813 @param[in] background background model for exposure
814 @param[in] sourceCat catalog of measured sources
815 @param[in] astromMatches list of source/refObj matches from the
816 astrometry solver
817 """
818 dataRef.put(sourceCat, "src")
819 if self.config.doWriteMatches and astromMatches is not None:
820 normalizedMatches = afwTable.packMatches(astromMatches)
821 normalizedMatches.table.setMetadata(matchMeta)
822 dataRef.put(normalizedMatches, "srcMatch")
823 if self.config.doWriteMatchesDenormalized:
824 denormMatches = denormalizeMatches(astromMatches, matchMeta)
825 dataRef.put(denormMatches, "srcMatchFull")
826 if self.config.doWriteExposure:
827 dataRef.put(exposure, "calexp")
828 dataRef.put(background, "calexpBackground")
829
831 """Return a dict of empty catalogs for each catalog dataset produced
832 by this task.
833 """
834 sourceCat = afwTable.SourceCatalog(self.schemaschema)
835 sourceCat.getTable().setMetadata(self.algMetadataalgMetadata)
836 return {"src": sourceCat}
837
838 def setMetadata(self, exposure, photoRes=None):
839 """!Set task and exposure metadata
840
841 Logs a warning and continues if needed data is missing.
842
843 @param[in,out] exposure exposure whose metadata is to be set
844 @param[in] photoRes results of running photoCal; if None then it was
845 not run
846 """
847 if photoRes is None:
848 return
849
850 metadata = exposure.getMetadata()
851
852 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata
853 try:
854 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
855 magZero = photoRes.zp - 2.5*math.log10(exposureTime)
856 except Exception:
857 self.log.warning("Could not set normalized MAGZERO in header: no "
858 "exposure time")
859 magZero = math.nan
860
861 try:
862 metadata.set('MAGZERO', magZero)
863 metadata.set('MAGZERO_RMS', photoRes.sigma)
864 metadata.set('MAGZERO_NOBJ', photoRes.ngood)
865 metadata.set('COLORTERM1', 0.0)
866 metadata.set('COLORTERM2', 0.0)
867 metadata.set('COLORTERM3', 0.0)
868 except Exception as e:
869 self.log.warning("Could not set exposure metadata: %s", e)
870
871 def copyIcSourceFields(self, icSourceCat, sourceCat):
872 """!Match sources in icSourceCat and sourceCat and copy the specified fields
873
874 @param[in] icSourceCat catalog from which to copy fields
875 @param[in,out] sourceCat catalog to which to copy fields
876
877 The fields copied are those specified by `config.icSourceFieldsToCopy`
878 that actually exist in the schema. This was set up by the constructor
879 using self.schemaMapperschemaMapper.
880 """
881 if self.schemaMapperschemaMapper is None:
882 raise RuntimeError("To copy icSource fields you must specify "
883 "icSourceSchema nd icSourceKeys when "
884 "constructing this task")
885 if icSourceCat is None or sourceCat is None:
886 raise RuntimeError("icSourceCat and sourceCat must both be "
887 "specified")
888 if len(self.config.icSourceFieldsToCopy) == 0:
889 self.log.warning("copyIcSourceFields doing nothing because "
890 "icSourceFieldsToCopy is empty")
891 return
892
893 mc = afwTable.MatchControl()
894 mc.findOnlyClosest = False # return all matched objects
895 matches = afwTable.matchXy(icSourceCat, sourceCat,
896 self.config.matchRadiusPix, mc)
897 if self.config.doDeblend:
898 deblendKey = sourceCat.schema["deblend_nChild"].asKey()
899 # if deblended, keep children
900 matches = [m for m in matches if m[1].get(deblendKey) == 0]
901
902 # Because we had to allow multiple matches to handle parents, we now
903 # need to prune to the best matches
904 # closest matches as a dict of icSourceCat source ID:
905 # (icSourceCat source, sourceCat source, distance in pixels)
906 bestMatches = {}
907 for m0, m1, d in matches:
908 id0 = m0.getId()
909 match = bestMatches.get(id0)
910 if match is None or d <= match[2]:
911 bestMatches[id0] = (m0, m1, d)
912 matches = list(bestMatches.values())
913
914 # Check that no sourceCat sources are listed twice (we already know
915 # that each match has a unique icSourceCat source ID, due to using
916 # that ID as the key in bestMatches)
917 numMatches = len(matches)
918 numUniqueSources = len(set(m[1].getId() for m in matches))
919 if numUniqueSources != numMatches:
920 self.log.warning("%d icSourceCat sources matched only %d sourceCat "
921 "sources", numMatches, numUniqueSources)
922
923 self.log.info("Copying flags from icSourceCat to sourceCat for "
924 "%d sources", numMatches)
925
926 # For each match: set the calibSourceKey flag and copy the desired
927 # fields
928 for icSrc, src, d in matches:
929 src.setFlag(self.calibSourceKeycalibSourceKey, True)
930 # src.assign copies the footprint from icSrc, which we don't want
931 # (DM-407)
932 # so set icSrc's footprint to src's footprint before src.assign,
933 # then restore it
934 icSrcFootprint = icSrc.getFootprint()
935 try:
936 icSrc.setFootprint(src.getFootprint())
937 src.assign(icSrc, self.schemaMapperschemaMapper)
938 finally:
939 icSrc.setFootprint(icSrcFootprint)
Calibrate an exposure: measure sources and perform astrometric and photometric calibration.
Definition: calibrate.py:348
def writeOutputs(self, dataRef, exposure, background, sourceCat, astromMatches, matchMeta)
Definition: calibrate.py:807
def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None, doUnpersist=True)
Calibrate an exposure, optionally unpersisting inputs and persisting outputs.
Definition: calibrate.py:562
def __init__(self, butler=None, astromRefObjLoader=None, photoRefObjLoader=None, icSourceSchema=None, initInputs=None, **kwargs)
Construct a CalibrateTask.
Definition: calibrate.py:447
def setMetadata(self, exposure, photoRes=None)
Set task and exposure metadata.
Definition: calibrate.py:838
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: calibrate.py:624
def copyIcSourceFields(self, icSourceCat, sourceCat)
Match sources in icSourceCat and sourceCat and copy the specified fields.
Definition: calibrate.py:871
def run(self, exposure, exposureIdInfo=None, background=None, icSourceCat=None)
Calibrate an exposure (science image or coadd)
Definition: calibrate.py:656