lsst.pipe.tasks gb4d8b8e895+227c5305aa
calibrate.py
Go to the documentation of this file.
2# LSST Data Management System
3# Copyright 2008-2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22import math
23
24from lsstDebug import getDebugFrame
25import lsst.pex.config as pexConfig
26import lsst.pipe.base as pipeBase
27import lsst.pipe.base.connectionTypes as cT
28import lsst.afw.table as afwTable
29from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches
30from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, SkyObjectsTask
31from lsst.obs.base import ExposureIdInfo
32import lsst.daf.base as dafBase
33from lsst.afw.math import BackgroundList
34from lsst.afw.table import SourceTable
35from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader
36from lsst.meas.base import (SingleFrameMeasurementTask,
37 ApplyApCorrTask,
38 CatalogCalculationTask)
39from lsst.meas.deblender import SourceDeblendTask
40from lsst.utils.timer import timeMethod
41from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask
42from .fakes import BaseFakeSourcesTask
43from .photoCal import PhotoCalTask
44from .computeExposureSummaryStats import ComputeExposureSummaryStatsTask
45
46
47__all__ = ["CalibrateConfig", "CalibrateTask"]
48
49
50class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"),
51 defaultTemplates={}):
52
53 icSourceSchema = cT.InitInput(
54 doc="Schema produced by characterize image task, used to initialize this task",
55 name="icSrc_schema",
56 storageClass="SourceCatalog",
57 )
58
59 outputSchema = cT.InitOutput(
60 doc="Schema after CalibrateTask has been initialized",
61 name="src_schema",
62 storageClass="SourceCatalog",
63 )
64
65 exposure = cT.Input(
66 doc="Input image to calibrate",
67 name="icExp",
68 storageClass="ExposureF",
69 dimensions=("instrument", "visit", "detector"),
70 )
71
72 background = cT.Input(
73 doc="Backgrounds determined by characterize task",
74 name="icExpBackground",
75 storageClass="Background",
76 dimensions=("instrument", "visit", "detector"),
77 )
78
79 icSourceCat = cT.Input(
80 doc="Source catalog created by characterize task",
81 name="icSrc",
82 storageClass="SourceCatalog",
83 dimensions=("instrument", "visit", "detector"),
84 )
85
86 astromRefCat = cT.PrerequisiteInput(
87 doc="Reference catalog to use for astrometry",
88 name="cal_ref_cat",
89 storageClass="SimpleCatalog",
90 dimensions=("skypix",),
91 deferLoad=True,
92 multiple=True,
93 )
94
95 photoRefCat = cT.PrerequisiteInput(
96 doc="Reference catalog to use for photometric calibration",
97 name="cal_ref_cat",
98 storageClass="SimpleCatalog",
99 dimensions=("skypix",),
100 deferLoad=True,
101 multiple=True
102 )
103
104 outputExposure = cT.Output(
105 doc="Exposure after running calibration task",
106 name="calexp",
107 storageClass="ExposureF",
108 dimensions=("instrument", "visit", "detector"),
109 )
110
111 outputCat = cT.Output(
112 doc="Source catalog produced in calibrate task",
113 name="src",
114 storageClass="SourceCatalog",
115 dimensions=("instrument", "visit", "detector"),
116 )
117
118 outputBackground = cT.Output(
119 doc="Background models estimated in calibration task",
120 name="calexpBackground",
121 storageClass="Background",
122 dimensions=("instrument", "visit", "detector"),
123 )
124
125 matches = cT.Output(
126 doc="Source/refObj matches from the astrometry solver",
127 name="srcMatch",
128 storageClass="Catalog",
129 dimensions=("instrument", "visit", "detector"),
130 )
131
132 matchesDenormalized = cT.Output(
133 doc="Denormalized matches from astrometry solver",
134 name="srcMatchFull",
135 storageClass="Catalog",
136 dimensions=("instrument", "visit", "detector"),
137 )
138
139 def __init__(self, *, config=None):
140 super().__init__(config=config)
141
142 if config.doAstrometry is False:
143 self.prerequisiteInputs.remove("astromRefCat")
144 if config.doPhotoCal is False:
145 self.prerequisiteInputs.remove("photoRefCat")
146
147 if config.doWriteMatches is False or config.doAstrometry is False:
148 self.outputs.remove("matches")
149 if config.doWriteMatchesDenormalized is False or config.doAstrometry is False:
150 self.outputs.remove("matchesDenormalized")
151
152
153class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections):
154 """Config for CalibrateTask"""
155 doWrite = pexConfig.Field(
156 dtype=bool,
157 default=True,
158 doc="Save calibration results?",
159 )
160 doWriteHeavyFootprintsInSources = pexConfig.Field(
161 dtype=bool,
162 default=True,
163 doc="Include HeavyFootprint data in source table? If false then heavy "
164 "footprints are saved as normal footprints, which saves some space"
165 )
166 doWriteMatches = pexConfig.Field(
167 dtype=bool,
168 default=True,
169 doc="Write reference matches (ignored if doWrite or doAstrometry false)?",
170 )
171 doWriteMatchesDenormalized = pexConfig.Field(
172 dtype=bool,
173 default=False,
174 doc=("Write reference matches in denormalized format? "
175 "This format uses more disk space, but is more convenient to "
176 "read. Ignored if doWriteMatches=False or doWrite=False."),
177 )
178 doAstrometry = pexConfig.Field(
179 dtype=bool,
180 default=True,
181 doc="Perform astrometric calibration?",
182 )
183 astromRefObjLoader = pexConfig.ConfigurableField(
184 target=LoadIndexedReferenceObjectsTask,
185 doc="reference object loader for astrometric calibration",
186 )
187 photoRefObjLoader = pexConfig.ConfigurableField(
188 target=LoadIndexedReferenceObjectsTask,
189 doc="reference object loader for photometric calibration",
190 )
191 astrometry = pexConfig.ConfigurableField(
192 target=AstrometryTask,
193 doc="Perform astrometric calibration to refine the WCS",
194 )
195 requireAstrometry = pexConfig.Field(
196 dtype=bool,
197 default=True,
198 doc=("Raise an exception if astrometry fails? Ignored if doAstrometry "
199 "false."),
200 )
201 doPhotoCal = pexConfig.Field(
202 dtype=bool,
203 default=True,
204 doc="Perform phometric calibration?",
205 )
206 requirePhotoCal = pexConfig.Field(
207 dtype=bool,
208 default=True,
209 doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal "
210 "false."),
211 )
212 photoCal = pexConfig.ConfigurableField(
213 target=PhotoCalTask,
214 doc="Perform photometric calibration",
215 )
216 icSourceFieldsToCopy = pexConfig.ListField(
217 dtype=str,
218 default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
219 doc=("Fields to copy from the icSource catalog to the output catalog "
220 "for matching sources Any missing fields will trigger a "
221 "RuntimeError exception. Ignored if icSourceCat is not provided.")
222 )
223 matchRadiusPix = pexConfig.Field(
224 dtype=float,
225 default=3,
226 doc=("Match radius for matching icSourceCat objects to sourceCat "
227 "objects (pixels)"),
228 )
229 checkUnitsParseStrict = pexConfig.Field(
230 doc=("Strictness of Astropy unit compatibility check, can be 'raise', "
231 "'warn' or 'silent'"),
232 dtype=str,
233 default="raise",
234 )
235 detection = pexConfig.ConfigurableField(
236 target=SourceDetectionTask,
237 doc="Detect sources"
238 )
239 doDeblend = pexConfig.Field(
240 dtype=bool,
241 default=True,
242 doc="Run deblender input exposure"
243 )
244 deblend = pexConfig.ConfigurableField(
245 target=SourceDeblendTask,
246 doc="Split blended sources into their components"
247 )
248 doSkySources = pexConfig.Field(
249 dtype=bool,
250 default=True,
251 doc="Generate sky sources?",
252 )
253 skySources = pexConfig.ConfigurableField(
254 target=SkyObjectsTask,
255 doc="Generate sky sources",
256 )
257 measurement = pexConfig.ConfigurableField(
258 target=SingleFrameMeasurementTask,
259 doc="Measure sources"
260 )
261 setPrimaryFlags = pexConfig.ConfigurableField(
262 target=SetPrimaryFlagsTask,
263 doc=("Set flags for primary source classification in single frame "
264 "processing. True if sources are not sky sources and not a parent.")
265 )
266 doApCorr = pexConfig.Field(
267 dtype=bool,
268 default=True,
269 doc="Run subtask to apply aperture correction"
270 )
271 applyApCorr = pexConfig.ConfigurableField(
272 target=ApplyApCorrTask,
273 doc="Subtask to apply aperture corrections"
274 )
275 # If doApCorr is False, and the exposure does not have apcorrections
276 # already applied, the active plugins in catalogCalculation almost
277 # certainly should not contain the characterization plugin
278 catalogCalculation = pexConfig.ConfigurableField(
279 target=CatalogCalculationTask,
280 doc="Subtask to run catalogCalculation plugins on catalog"
281 )
282 doInsertFakes = pexConfig.Field(
283 dtype=bool,
284 default=False,
285 doc="Run fake sources injection task"
286 )
287 insertFakes = pexConfig.ConfigurableField(
288 target=BaseFakeSourcesTask,
289 doc="Injection of fake sources for testing purposes (must be "
290 "retargeted)"
291 )
292 doComputeSummaryStats = pexConfig.Field(
293 dtype=bool,
294 default=True,
295 doc="Run subtask to measure exposure summary statistics?"
296 )
297 computeSummaryStats = pexConfig.ConfigurableField(
298 target=ComputeExposureSummaryStatsTask,
299 doc="Subtask to run computeSummaryStats on exposure"
300 )
301 doWriteExposure = pexConfig.Field(
302 dtype=bool,
303 default=True,
304 doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a "
305 "normal calexp but as a fakes_calexp."
306 )
307
308 def setDefaults(self):
309 super().setDefaults()
310 self.detectiondetection.doTempLocalBackground = False
311 self.deblenddeblend.maxFootprintSize = 2000
312 self.measurementmeasurement.plugins.names |= ["base_LocalPhotoCalib", "base_LocalWcs"]
313
314 def validate(self):
315 super().validate()
316 astromRefCatGen2 = getattr(self.astromRefObjLoaderastromRefObjLoader, "ref_dataset_name", None)
317 if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat:
318 raise ValueError(
319 f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference "
320 f"catalogs are different. These options must be kept in sync until Gen2 is retired."
321 )
322 photoRefCatGen2 = getattr(self.photoRefObjLoaderphotoRefObjLoader, "ref_dataset_name", None)
323 if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat:
324 raise ValueError(
325 f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference "
326 f"catalogs are different. These options must be kept in sync until Gen2 is retired."
327 )
328
329
330
336
337class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
338 r"""!Calibrate an exposure: measure sources and perform astrometric and
339 photometric calibration
340
341 @anchor CalibrateTask_
342
343 @section pipe_tasks_calibrate_Contents Contents
344
345 - @ref pipe_tasks_calibrate_Purpose
346 - @ref pipe_tasks_calibrate_Initialize
347 - @ref pipe_tasks_calibrate_IO
348 - @ref pipe_tasks_calibrate_Config
349 - @ref pipe_tasks_calibrate_Metadata
350 - @ref pipe_tasks_calibrate_Debug
351
352
353 @section pipe_tasks_calibrate_Purpose Description
354
355 Given an exposure with a good PSF model and aperture correction map
356 (e.g. as provided by @ref CharacterizeImageTask), perform the following
357 operations:
358 - Run detection and measurement
359 - Run astrometry subtask to fit an improved WCS
360 - Run photoCal subtask to fit the exposure's photometric zero-point
361
362 @section pipe_tasks_calibrate_Initialize Task initialisation
363
364 @copydoc \_\_init\_\_
365
366 @section pipe_tasks_calibrate_IO Invoking the Task
367
368 If you want this task to unpersist inputs or persist outputs, then call
369 the `runDataRef` method (a wrapper around the `run` method).
370
371 If you already have the inputs unpersisted and do not want to persist the
372 output then it is more direct to call the `run` method:
373
374 @section pipe_tasks_calibrate_Config Configuration parameters
375
376 See @ref CalibrateConfig
377
378 @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata
379
380 Exposure metadata
381 <dl>
382 <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task
383 <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal
384 task
385 <dt>COLORTERM1 <dd>?? (always 0.0)
386 <dt>COLORTERM2 <dd>?? (always 0.0)
387 <dt>COLORTERM3 <dd>?? (always 0.0)
388 </dl>
389
390 @section pipe_tasks_calibrate_Debug Debug variables
391
392 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink
393 interface supports a flag
394 `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug
395 for more about `debug.py`.
396
397 CalibrateTask has a debug dictionary containing one key:
398 <dl>
399 <dt>calibrate
400 <dd>frame (an int; <= 0 to not display) in which to display the exposure,
401 sources and matches. See @ref lsst.meas.astrom.displayAstrometry for
402 the meaning of the various symbols.
403 </dl>
404
405 For example, put something like:
406 @code{.py}
407 import lsstDebug
408 def DebugInfo(name):
409 di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would
410 # call us recursively
411 if name == "lsst.pipe.tasks.calibrate":
412 di.display = dict(
413 calibrate = 1,
414 )
415
416 return di
417
418 lsstDebug.Info = DebugInfo
419 @endcode
420 into your `debug.py` file and run `calibrateTask.py` with the `--debug`
421 flag.
422
423 Some subtasks may have their own debug variables; see individual Task
424 documentation.
425 """
426
427 # Example description used to live here, removed 2-20-2017 as per
428 # https://jira.lsstcorp.org/browse/DM-9520
429
430 ConfigClass = CalibrateConfig
431 _DefaultName = "calibrate"
432 RunnerClass = pipeBase.ButlerInitializedTaskRunner
433
434 def __init__(self, butler=None, astromRefObjLoader=None,
435 photoRefObjLoader=None, icSourceSchema=None,
436 initInputs=None, **kwargs):
437 """!Construct a CalibrateTask
438
439 @param[in] butler The butler is passed to the refObjLoader constructor
440 in case it is needed. Ignored if the refObjLoader argument
441 provides a loader directly.
442 @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks
443 that supplies an external reference catalog for astrometric
444 calibration. May be None if the desired loader can be constructed
445 from the butler argument or all steps requiring a reference catalog
446 are disabled.
447 @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks
448 that supplies an external reference catalog for photometric
449 calibration. May be None if the desired loader can be constructed
450 from the butler argument or all steps requiring a reference catalog
451 are disabled.
452 @param[in] icSourceSchema schema for icSource catalog, or None.
453 Schema values specified in config.icSourceFieldsToCopy will be
454 taken from this schema. If set to None, no values will be
455 propagated from the icSourceCatalog
456 @param[in,out] kwargs other keyword arguments for
457 lsst.pipe.base.CmdLineTask
458 """
459 super().__init__(**kwargs)
460
461 if icSourceSchema is None and butler is not None:
462 # Use butler to read icSourceSchema from disk.
463 icSourceSchema = butler.get("icSrc_schema", immediate=True).schema
464
465 if icSourceSchema is None and butler is None and initInputs is not None:
466 icSourceSchema = initInputs['icSourceSchema'].schema
467
468 if icSourceSchema is not None:
469 # use a schema mapper to avoid copying each field separately
470 self.schemaMapperschemaMapper = afwTable.SchemaMapper(icSourceSchema)
471 minimumSchema = afwTable.SourceTable.makeMinimalSchema()
472 self.schemaMapperschemaMapper.addMinimalSchema(minimumSchema, False)
473
474 # Add fields to copy from an icSource catalog
475 # and a field to indicate that the source matched a source in that
476 # catalog. If any fields are missing then raise an exception, but
477 # first find all missing fields in order to make the error message
478 # more useful.
479 self.calibSourceKeycalibSourceKey = self.schemaMapperschemaMapper.addOutputField(
480 afwTable.Field["Flag"]("calib_detected",
481 "Source was detected as an icSource"))
482 missingFieldNames = []
483 for fieldName in self.config.icSourceFieldsToCopy:
484 try:
485 schemaItem = icSourceSchema.find(fieldName)
486 except Exception:
487 missingFieldNames.append(fieldName)
488 else:
489 # field found; if addMapping fails then raise an exception
490 self.schemaMapperschemaMapper.addMapping(schemaItem.getKey())
491
492 if missingFieldNames:
493 raise RuntimeError("isSourceCat is missing fields {} "
494 "specified in icSourceFieldsToCopy"
495 .format(missingFieldNames))
496
497 # produce a temporary schema to pass to the subtasks; finalize it
498 # later
499 self.schemaschema = self.schemaMapperschemaMapper.editOutputSchema()
500 else:
501 self.schemaMapperschemaMapper = None
502 self.schemaschema = afwTable.SourceTable.makeMinimalSchema()
503 self.makeSubtask('detection', schema=self.schemaschema)
504
505 self.algMetadataalgMetadata = dafBase.PropertyList()
506
507 # Only create a subtask for fakes if configuration option is set
508 # N.B. the config for fake object task must be retargeted to a child
509 # of BaseFakeSourcesTask
510 if self.config.doInsertFakes:
511 self.makeSubtask("insertFakes")
512
513 if self.config.doDeblend:
514 self.makeSubtask("deblend", schema=self.schemaschema)
515 if self.config.doSkySources:
516 self.makeSubtask("skySources")
517 self.skySourceKeyskySourceKey = self.schemaschema.addField("sky_source", type="Flag", doc="Sky objects.")
518 self.makeSubtask('measurement', schema=self.schemaschema,
519 algMetadata=self.algMetadataalgMetadata)
520 self.makeSubtask("setPrimaryFlags", schema=self.schemaschema, isSingleFrame=True)
521 if self.config.doApCorr:
522 self.makeSubtask('applyApCorr', schema=self.schemaschema)
523 self.makeSubtask('catalogCalculation', schema=self.schemaschema)
524
525 if self.config.doAstrometry:
526 if astromRefObjLoader is None and butler is not None:
527 self.makeSubtask('astromRefObjLoader', butler=butler)
528 astromRefObjLoader = self.astromRefObjLoader
529 self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader,
530 schema=self.schemaschema)
531 if self.config.doPhotoCal:
532 if photoRefObjLoader is None and butler is not None:
533 self.makeSubtask('photoRefObjLoader', butler=butler)
534 photoRefObjLoader = self.photoRefObjLoader
535 self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader,
536 schema=self.schemaschema)
537 if self.config.doComputeSummaryStats:
538 self.makeSubtask('computeSummaryStats')
539
540 if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None):
541 raise RuntimeError("PipelineTask form of this task should not be initialized with "
542 "reference object loaders.")
543
544 if self.schemaMapperschemaMapper is not None:
545 # finalize the schema
546 self.schemaschema = self.schemaMapperschemaMapper.getOutputSchema()
547 self.schemaschema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
548
549 sourceCatSchema = afwTable.SourceCatalog(self.schemaschema)
550 sourceCatSchema.getTable().setMetadata(self.algMetadataalgMetadata)
551 self.outputSchemaoutputSchema = sourceCatSchema
552
553 @timeMethod
554 def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None,
555 doUnpersist=True):
556 """!Calibrate an exposure, optionally unpersisting inputs and
557 persisting outputs.
558
559 This is a wrapper around the `run` method that unpersists inputs
560 (if `doUnpersist` true) and persists outputs (if `config.doWrite` true)
561
562 @param[in] dataRef butler data reference corresponding to a science
563 image
564 @param[in,out] exposure characterized exposure (an
565 lsst.afw.image.ExposureF or similar), or None to unpersist existing
566 icExp and icBackground. See `run` method for details of what is
567 read and written.
568 @param[in,out] background initial model of background already
569 subtracted from exposure (an lsst.afw.math.BackgroundList). May be
570 None if no background has been subtracted, though that is unusual
571 for calibration. A refined background model is output. Ignored if
572 exposure is None.
573 @param[in] icSourceCat catalog from which to copy the fields specified
574 by icSourceKeys, or None;
575 @param[in] doUnpersist unpersist data:
576 - if True, exposure, background and icSourceCat are read from
577 dataRef and those three arguments must all be None;
578 - if False the exposure must be provided; background and
579 icSourceCat are optional. True is intended for running as a
580 command-line task, False for running as a subtask
581 @return same data as the calibrate method
582 """
583 self.log.info("Processing %s", dataRef.dataId)
584
585 if doUnpersist:
586 if any(item is not None for item in (exposure, background,
587 icSourceCat)):
588 raise RuntimeError("doUnpersist true; exposure, background "
589 "and icSourceCat must all be None")
590 exposure = dataRef.get("icExp", immediate=True)
591 background = dataRef.get("icExpBackground", immediate=True)
592 icSourceCat = dataRef.get("icSrc", immediate=True)
593 elif exposure is None:
594 raise RuntimeError("doUnpersist false; exposure must be provided")
595
596 exposureIdInfo = dataRef.get("expIdInfo")
597
598 calRes = self.runrun(
599 exposure=exposure,
600 exposureIdInfo=exposureIdInfo,
601 background=background,
602 icSourceCat=icSourceCat,
603 )
604
605 if self.config.doWrite:
606 self.writeOutputswriteOutputs(
607 dataRef=dataRef,
608 exposure=calRes.exposure,
609 background=calRes.background,
610 sourceCat=calRes.sourceCat,
611 astromMatches=calRes.astromMatches,
612 matchMeta=calRes.matchMeta,
613 )
614
615 return calRes
616
617 def runQuantum(self, butlerQC, inputRefs, outputRefs):
618 inputs = butlerQC.get(inputRefs)
619 inputs['exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "visit_detector")
620
621 if self.config.doAstrometry:
622 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
623 for ref in inputRefs.astromRefCat],
624 refCats=inputs.pop('astromRefCat'),
625 config=self.config.astromRefObjLoader, log=self.log)
626 self.astrometry.setRefObjLoader(refObjLoader)
627
628 if self.config.doPhotoCal:
629 photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
630 for ref in inputRefs.photoRefCat],
631 refCats=inputs.pop('photoRefCat'),
632 config=self.config.photoRefObjLoader,
633 log=self.log)
634 self.photoCal.match.setRefObjLoader(photoRefObjLoader)
635
636 outputs = self.runrun(**inputs)
637
638 if self.config.doWriteMatches and self.config.doAstrometry:
639 normalizedMatches = afwTable.packMatches(outputs.astromMatches)
640 normalizedMatches.table.setMetadata(outputs.matchMeta)
641 if self.config.doWriteMatchesDenormalized:
642 denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
643 outputs.matchesDenormalized = denormMatches
644 outputs.matches = normalizedMatches
645 butlerQC.put(outputs, outputRefs)
646
647 @timeMethod
648 def run(self, exposure, exposureIdInfo=None, background=None,
649 icSourceCat=None):
650 """!Calibrate an exposure (science image or coadd)
651
652 @param[in,out] exposure exposure to calibrate (an
653 lsst.afw.image.ExposureF or similar);
654 in:
655 - MaskedImage
656 - Psf
657 out:
658 - MaskedImage has background subtracted
659 - Wcs is replaced
660 - PhotoCalib is replaced
661 @param[in] exposureIdInfo ID info for exposure (an
662 lsst.obs.base.ExposureIdInfo) If not provided, returned
663 SourceCatalog IDs will not be globally unique.
664 @param[in,out] background background model already subtracted from
665 exposure (an lsst.afw.math.BackgroundList). May be None if no
666 background has been subtracted, though that is unusual for
667 calibration. A refined background model is output.
668 @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
669 from which we can copy some fields.
670
671 @return pipe_base Struct containing these fields:
672 - exposure calibrate science exposure with refined WCS and PhotoCalib
673 - background model of background subtracted from exposure (an
674 lsst.afw.math.BackgroundList)
675 - sourceCat catalog of measured sources
676 - astromMatches list of source/refObj matches from the astrometry
677 solver
678 """
679 # detect, deblend and measure sources
680 if exposureIdInfo is None:
681 exposureIdInfo = ExposureIdInfo()
682
683 if background is None:
684 background = BackgroundList()
685 sourceIdFactory = exposureIdInfo.makeSourceIdFactory()
686 table = SourceTable.make(self.schemaschema, sourceIdFactory)
687 table.setMetadata(self.algMetadataalgMetadata)
688
689 detRes = self.detection.run(table=table, exposure=exposure,
690 doSmooth=True)
691 sourceCat = detRes.sources
692 if detRes.fpSets.background:
693 for bg in detRes.fpSets.background:
694 background.append(bg)
695 if self.config.doSkySources:
696 skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId)
697 if skySourceFootprints:
698 for foot in skySourceFootprints:
699 s = sourceCat.addNew()
700 s.setFootprint(foot)
701 s.set(self.skySourceKeyskySourceKey, True)
702 if self.config.doDeblend:
703 self.deblend.run(exposure=exposure, sources=sourceCat)
704 self.measurement.run(
705 measCat=sourceCat,
706 exposure=exposure,
707 exposureId=exposureIdInfo.expId
708 )
709 if self.config.doApCorr:
710 self.applyApCorr.run(
711 catalog=sourceCat,
712 apCorrMap=exposure.getInfo().getApCorrMap()
713 )
714 self.catalogCalculation.run(sourceCat)
715
716 self.setPrimaryFlags.run(sourceCat)
717
718 if icSourceCat is not None and \
719 len(self.config.icSourceFieldsToCopy) > 0:
720 self.copyIcSourceFieldscopyIcSourceFields(icSourceCat=icSourceCat,
721 sourceCat=sourceCat)
722
723 # TODO DM-11568: this contiguous check-and-copy could go away if we
724 # reserve enough space during SourceDetection and/or SourceDeblend.
725 # NOTE: sourceSelectors require contiguous catalogs, so ensure
726 # contiguity now, so views are preserved from here on.
727 if not sourceCat.isContiguous():
728 sourceCat = sourceCat.copy(deep=True)
729
730 # perform astrometry calibration:
731 # fit an improved WCS and update the exposure's WCS in place
732 astromMatches = None
733 matchMeta = None
734 if self.config.doAstrometry:
735 try:
736 astromRes = self.astrometry.run(
737 exposure=exposure,
738 sourceCat=sourceCat,
739 )
740 astromMatches = astromRes.matches
741 matchMeta = astromRes.matchMeta
742 except Exception as e:
743 if self.config.requireAstrometry:
744 raise
745 self.log.warning("Unable to perform astrometric calibration "
746 "(%s): attempting to proceed", e)
747
748 # compute photometric calibration
749 if self.config.doPhotoCal:
750 try:
751 photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
752 exposure.setPhotoCalib(photoRes.photoCalib)
753 # TODO: reword this to phrase it in terms of the calibration factor?
754 self.log.info("Photometric zero-point: %f",
755 photoRes.photoCalib.instFluxToMagnitude(1.0))
756 self.setMetadatasetMetadata(exposure=exposure, photoRes=photoRes)
757 except Exception as e:
758 if self.config.requirePhotoCal:
759 raise
760 self.log.warning("Unable to perform photometric calibration "
761 "(%s): attempting to proceed", e)
762 self.setMetadatasetMetadata(exposure=exposure, photoRes=None)
763
764 if self.config.doInsertFakes:
765 self.insertFakes.run(exposure, background=background)
766
767 table = SourceTable.make(self.schemaschema, sourceIdFactory)
768 table.setMetadata(self.algMetadataalgMetadata)
769
770 detRes = self.detection.run(table=table, exposure=exposure,
771 doSmooth=True)
772 sourceCat = detRes.sources
773 if detRes.fpSets.background:
774 for bg in detRes.fpSets.background:
775 background.append(bg)
776 if self.config.doDeblend:
777 self.deblend.run(exposure=exposure, sources=sourceCat)
778 self.measurement.run(
779 measCat=sourceCat,
780 exposure=exposure,
781 exposureId=exposureIdInfo.expId
782 )
783 if self.config.doApCorr:
784 self.applyApCorr.run(
785 catalog=sourceCat,
786 apCorrMap=exposure.getInfo().getApCorrMap()
787 )
788 self.catalogCalculation.run(sourceCat)
789
790 if icSourceCat is not None and len(self.config.icSourceFieldsToCopy) > 0:
791 self.copyIcSourceFieldscopyIcSourceFields(icSourceCat=icSourceCat,
792 sourceCat=sourceCat)
793
794 if self.config.doComputeSummaryStats:
795 summary = self.computeSummaryStats.run(exposure=exposure,
796 sources=sourceCat,
797 background=background)
798 exposure.getInfo().setSummaryStats(summary)
799
800 frame = getDebugFrame(self._display, "calibrate")
801 if frame:
802 displayAstrometry(
803 sourceCat=sourceCat,
804 exposure=exposure,
805 matches=astromMatches,
806 frame=frame,
807 pause=False,
808 )
809
810 return pipeBase.Struct(
811 exposure=exposure,
812 background=background,
813 sourceCat=sourceCat,
814 astromMatches=astromMatches,
815 matchMeta=matchMeta,
816 # These are duplicate entries with different names for use with
817 # gen3 middleware
818 outputExposure=exposure,
819 outputCat=sourceCat,
820 outputBackground=background,
821 )
822
823 def writeOutputs(self, dataRef, exposure, background, sourceCat,
824 astromMatches, matchMeta):
825 """Write output data to the output repository
826
827 @param[in] dataRef butler data reference corresponding to a science
828 image
829 @param[in] exposure exposure to write
830 @param[in] background background model for exposure
831 @param[in] sourceCat catalog of measured sources
832 @param[in] astromMatches list of source/refObj matches from the
833 astrometry solver
834 """
835 dataRef.put(sourceCat, "src")
836 if self.config.doWriteMatches and astromMatches is not None:
837 normalizedMatches = afwTable.packMatches(astromMatches)
838 normalizedMatches.table.setMetadata(matchMeta)
839 dataRef.put(normalizedMatches, "srcMatch")
840 if self.config.doWriteMatchesDenormalized:
841 denormMatches = denormalizeMatches(astromMatches, matchMeta)
842 dataRef.put(denormMatches, "srcMatchFull")
843 if self.config.doWriteExposure:
844 dataRef.put(exposure, "calexp")
845 dataRef.put(background, "calexpBackground")
846
848 """Return a dict of empty catalogs for each catalog dataset produced
849 by this task.
850 """
851 sourceCat = afwTable.SourceCatalog(self.schemaschema)
852 sourceCat.getTable().setMetadata(self.algMetadataalgMetadata)
853 return {"src": sourceCat}
854
855 def setMetadata(self, exposure, photoRes=None):
856 """!Set task and exposure metadata
857
858 Logs a warning and continues if needed data is missing.
859
860 @param[in,out] exposure exposure whose metadata is to be set
861 @param[in] photoRes results of running photoCal; if None then it was
862 not run
863 """
864 if photoRes is None:
865 return
866
867 metadata = exposure.getMetadata()
868
869 # convert zero-point to (mag/sec/adu) for task MAGZERO metadata
870 try:
871 exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
872 magZero = photoRes.zp - 2.5*math.log10(exposureTime)
873 except Exception:
874 self.log.warning("Could not set normalized MAGZERO in header: no "
875 "exposure time")
876 magZero = math.nan
877
878 try:
879 metadata.set('MAGZERO', magZero)
880 metadata.set('MAGZERO_RMS', photoRes.sigma)
881 metadata.set('MAGZERO_NOBJ', photoRes.ngood)
882 metadata.set('COLORTERM1', 0.0)
883 metadata.set('COLORTERM2', 0.0)
884 metadata.set('COLORTERM3', 0.0)
885 except Exception as e:
886 self.log.warning("Could not set exposure metadata: %s", e)
887
888 def copyIcSourceFields(self, icSourceCat, sourceCat):
889 """!Match sources in icSourceCat and sourceCat and copy the specified fields
890
891 @param[in] icSourceCat catalog from which to copy fields
892 @param[in,out] sourceCat catalog to which to copy fields
893
894 The fields copied are those specified by `config.icSourceFieldsToCopy`
895 that actually exist in the schema. This was set up by the constructor
896 using self.schemaMapperschemaMapper.
897 """
898 if self.schemaMapperschemaMapper is None:
899 raise RuntimeError("To copy icSource fields you must specify "
900 "icSourceSchema nd icSourceKeys when "
901 "constructing this task")
902 if icSourceCat is None or sourceCat is None:
903 raise RuntimeError("icSourceCat and sourceCat must both be "
904 "specified")
905 if len(self.config.icSourceFieldsToCopy) == 0:
906 self.log.warning("copyIcSourceFields doing nothing because "
907 "icSourceFieldsToCopy is empty")
908 return
909
910 mc = afwTable.MatchControl()
911 mc.findOnlyClosest = False # return all matched objects
912 matches = afwTable.matchXy(icSourceCat, sourceCat,
913 self.config.matchRadiusPix, mc)
914 if self.config.doDeblend:
915 deblendKey = sourceCat.schema["deblend_nChild"].asKey()
916 # if deblended, keep children
917 matches = [m for m in matches if m[1].get(deblendKey) == 0]
918
919 # Because we had to allow multiple matches to handle parents, we now
920 # need to prune to the best matches
921 # closest matches as a dict of icSourceCat source ID:
922 # (icSourceCat source, sourceCat source, distance in pixels)
923 bestMatches = {}
924 for m0, m1, d in matches:
925 id0 = m0.getId()
926 match = bestMatches.get(id0)
927 if match is None or d <= match[2]:
928 bestMatches[id0] = (m0, m1, d)
929 matches = list(bestMatches.values())
930
931 # Check that no sourceCat sources are listed twice (we already know
932 # that each match has a unique icSourceCat source ID, due to using
933 # that ID as the key in bestMatches)
934 numMatches = len(matches)
935 numUniqueSources = len(set(m[1].getId() for m in matches))
936 if numUniqueSources != numMatches:
937 self.log.warning("%d icSourceCat sources matched only %d sourceCat "
938 "sources", numMatches, numUniqueSources)
939
940 self.log.info("Copying flags from icSourceCat to sourceCat for "
941 "%d sources", numMatches)
942
943 # For each match: set the calibSourceKey flag and copy the desired
944 # fields
945 for icSrc, src, d in matches:
946 src.setFlag(self.calibSourceKeycalibSourceKey, True)
947 # src.assign copies the footprint from icSrc, which we don't want
948 # (DM-407)
949 # so set icSrc's footprint to src's footprint before src.assign,
950 # then restore it
951 icSrcFootprint = icSrc.getFootprint()
952 try:
953 icSrc.setFootprint(src.getFootprint())
954 src.assign(icSrc, self.schemaMapperschemaMapper)
955 finally:
956 icSrc.setFootprint(icSrcFootprint)
Calibrate an exposure: measure sources and perform astrometric and photometric calibration.
Definition: calibrate.py:337
def writeOutputs(self, dataRef, exposure, background, sourceCat, astromMatches, matchMeta)
Definition: calibrate.py:824
def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None, doUnpersist=True)
Calibrate an exposure, optionally unpersisting inputs and persisting outputs.
Definition: calibrate.py:555
def __init__(self, butler=None, astromRefObjLoader=None, photoRefObjLoader=None, icSourceSchema=None, initInputs=None, **kwargs)
Construct a CalibrateTask.
Definition: calibrate.py:436
def setMetadata(self, exposure, photoRes=None)
Set task and exposure metadata.
Definition: calibrate.py:855
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: calibrate.py:617
def copyIcSourceFields(self, icSourceCat, sourceCat)
Match sources in icSourceCat and sourceCat and copy the specified fields.
Definition: calibrate.py:888
def run(self, exposure, exposureIdInfo=None, background=None, icSourceCat=None)
Calibrate an exposure (science image or coadd)
Definition: calibrate.py:649