lsst.pipe.tasks  19.0.0-23-g5d8da22d
calibrate.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2016 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import math
23 
24 from lsstDebug import getDebugFrame
25 import lsst.pex.config as pexConfig
26 import lsst.pipe.base as pipeBase
28 import lsst.afw.table as afwTable
29 from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches
30 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
31 from lsst.obs.base import ExposureIdInfo
32 import lsst.daf.base as dafBase
33 from lsst.afw.math import BackgroundList
34 from lsst.afw.table import IdFactory, SourceTable
35 from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader
36 from lsst.meas.base import (SingleFrameMeasurementTask,
37  ApplyApCorrTask,
38  CatalogCalculationTask,
39  EvaluateLocalCalibrationTask)
40 from lsst.meas.deblender import SourceDeblendTask
41 from .fakes import BaseFakeSourcesTask
42 from .photoCal import PhotoCalTask
43 
44 __all__ = ["CalibrateConfig", "CalibrateTask"]
45 
46 
47 class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"),
48  defaultTemplates={}):
49 
50  icSourceSchema = cT.InitInput(
51  doc="Schema produced by characterize image task, used to initialize this task",
52  name="icSrc_schema",
53  storageClass="SourceCatalog",
54  )
55 
56  outputSchema = cT.InitOutput(
57  doc="Schema after CalibrateTask has been initialized",
58  name="src_schema",
59  storageClass="SourceCatalog",
60  )
61 
62  exposure = cT.Input(
63  doc="Input image to calibrate",
64  name="icExp",
65  storageClass="ExposureF",
66  dimensions=("instrument", "visit", "detector"),
67  )
68 
69  background = cT.Input(
70  doc="Backgrounds determined by characterize task",
71  name="icExpBackground",
72  storageClass="Background",
73  dimensions=("instrument", "visit", "detector"),
74  )
75 
76  icSourceCat = cT.Input(
77  doc="Source catalog created by characterize task",
78  name="icSrc",
79  storageClass="SourceCatalog",
80  dimensions=("instrument", "visit", "detector"),
81  )
82 
83  astromRefCat = cT.PrerequisiteInput(
84  doc="Reference catalog to use for astrometry",
85  name="cal_ref_cat",
86  storageClass="SimpleCatalog",
87  dimensions=("skypix",),
88  deferLoad=True,
89  multiple=True,
90  )
91 
92  photoRefCat = cT.PrerequisiteInput(
93  doc="Reference catalog to use for photometric calibration",
94  name="cal_ref_cat",
95  storageClass="SimpleCatalog",
96  dimensions=("skypix",),
97  deferLoad=True,
98  multiple=True
99  )
100 
101  outputExposure = cT.Output(
102  doc="Exposure after running calibration task",
103  name="calexp",
104  storageClass="ExposureF",
105  dimensions=("instrument", "visit", "detector"),
106  )
107 
108  outputCat = cT.Output(
109  doc="Source catalog produced in calibrate task",
110  name="src",
111  storageClass="SourceCatalog",
112  dimensions=("instrument", "visit", "detector"),
113  )
114 
115  outputBackground = cT.Output(
116  doc="Background models estimated in calibration task",
117  name="calexpBackground",
118  storageClass="Background",
119  dimensions=("instrument", "visit", "detector"),
120  )
121 
122  matches = cT.Output(
123  doc="Source/refObj matches from the astrometry solver",
124  name="srcMatch",
125  storageClass="Catalog",
126  dimensions=("instrument", "visit", "detector"),
127  )
128 
129  matchesDenormalized = cT.Output(
130  doc="Denormalized matches from astrometry solver",
131  name="srcMatchFull",
132  storageClass="Catalog",
133  dimensions=("instrument", "visit", "detector"),
134  )
135 
136  def __init__(self, *, config=None):
137  super().__init__(config=config)
138  if config.doWriteMatches is False:
139  self.outputs.remove("matches")
140  if config.doWriteMatchesDenormalized is False:
141  self.outputs.remove("matchesDenormalized")
142 
143 
144 class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections):
145  """Config for CalibrateTask"""
146  doWrite = pexConfig.Field(
147  dtype=bool,
148  default=True,
149  doc="Save calibration results?",
150  )
151  doWriteHeavyFootprintsInSources = pexConfig.Field(
152  dtype=bool,
153  default=True,
154  doc="Include HeavyFootprint data in source table? If false then heavy "
155  "footprints are saved as normal footprints, which saves some space"
156  )
157  doWriteMatches = pexConfig.Field(
158  dtype=bool,
159  default=True,
160  doc="Write reference matches (ignored if doWrite false)?",
161  )
162  doWriteMatchesDenormalized = pexConfig.Field(
163  dtype=bool,
164  default=False,
165  doc=("Write reference matches in denormalized format? "
166  "This format uses more disk space, but is more convenient to "
167  "read. Ignored if doWriteMatches=False or doWrite=False."),
168  )
169  doAstrometry = pexConfig.Field(
170  dtype=bool,
171  default=True,
172  doc="Perform astrometric calibration?",
173  )
174  astromRefObjLoader = pexConfig.ConfigurableField(
175  target=LoadIndexedReferenceObjectsTask,
176  doc="reference object loader for astrometric calibration",
177  )
178  photoRefObjLoader = pexConfig.ConfigurableField(
179  target=LoadIndexedReferenceObjectsTask,
180  doc="reference object loader for photometric calibration",
181  )
182  astrometry = pexConfig.ConfigurableField(
183  target=AstrometryTask,
184  doc="Perform astrometric calibration to refine the WCS",
185  )
186  requireAstrometry = pexConfig.Field(
187  dtype=bool,
188  default=True,
189  doc=("Raise an exception if astrometry fails? Ignored if doAstrometry "
190  "false."),
191  )
192  doPhotoCal = pexConfig.Field(
193  dtype=bool,
194  default=True,
195  doc="Perform phometric calibration?",
196  )
197  requirePhotoCal = pexConfig.Field(
198  dtype=bool,
199  default=True,
200  doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal "
201  "false."),
202  )
203  photoCal = pexConfig.ConfigurableField(
204  target=PhotoCalTask,
205  doc="Perform photometric calibration",
206  )
207  doEvalLocCalibration = pexConfig.Field(
208  dtype=bool,
209  default=True,
210  doc="Store calibration products (local wcs and PhotoCalib) in output "
211  "source catalog."
212  )
213  evalLocCalib = pexConfig.ConfigurableField(
214  target=EvaluateLocalCalibrationTask,
215  doc="Task to strip calibrations from an exposure and store their "
216  "local values in the output source catalog."
217  )
218  icSourceFieldsToCopy = pexConfig.ListField(
219  dtype=str,
220  default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
221  doc=("Fields to copy from the icSource catalog to the output catalog "
222  "for matching sources Any missing fields will trigger a "
223  "RuntimeError exception. Ignored if icSourceCat is not provided.")
224  )
225  matchRadiusPix = pexConfig.Field(
226  dtype=float,
227  default=3,
228  doc=("Match radius for matching icSourceCat objects to sourceCat "
229  "objects (pixels)"),
230  )
231  checkUnitsParseStrict = pexConfig.Field(
232  doc=("Strictness of Astropy unit compatibility check, can be 'raise', "
233  "'warn' or 'silent'"),
234  dtype=str,
235  default="raise",
236  )
237  detection = pexConfig.ConfigurableField(
238  target=SourceDetectionTask,
239  doc="Detect sources"
240  )
241  doDeblend = pexConfig.Field(
242  dtype=bool,
243  default=True,
244  doc="Run deblender input exposure"
245  )
246  deblend = pexConfig.ConfigurableField(
247  target=SourceDeblendTask,
248  doc="Split blended sources into their components"
249  )
250  measurement = pexConfig.ConfigurableField(
251  target=SingleFrameMeasurementTask,
252  doc="Measure sources"
253  )
254  doApCorr = pexConfig.Field(
255  dtype=bool,
256  default=True,
257  doc="Run subtask to apply aperture correction"
258  )
259  applyApCorr = pexConfig.ConfigurableField(
260  target=ApplyApCorrTask,
261  doc="Subtask to apply aperture corrections"
262  )
263  # If doApCorr is False, and the exposure does not have apcorrections
264  # already applied, the active plugins in catalogCalculation almost
265  # certainly should not contain the characterization plugin
266  catalogCalculation = pexConfig.ConfigurableField(
267  target=CatalogCalculationTask,
268  doc="Subtask to run catalogCalculation plugins on catalog"
269  )
270  doInsertFakes = pexConfig.Field(
271  dtype=bool,
272  default=False,
273  doc="Run fake sources injection task"
274  )
275  insertFakes = pexConfig.ConfigurableField(
276  target=BaseFakeSourcesTask,
277  doc="Injection of fake sources for testing purposes (must be "
278  "retargeted)"
279  )
280  doWriteExposure = pexConfig.Field(
281  dtype=bool,
282  default=True,
283  doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a "
284  "normal calexp but as a fakes_calexp."
285  )
286 
287  def setDefaults(self):
288  super().setDefaults()
289  self.detection.doTempLocalBackground = False
290  self.deblend.maxFootprintSize = 2000
291 
292  def validate(self):
293  super().validate()
294  astromRefCatGen2 = getattr(self.astromRefObjLoader, "ref_dataset_name", None)
295  if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat:
296  raise ValueError(
297  f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference "
298  f"catalogs are different. These options must be kept in sync until Gen2 is retired."
299  )
300  photoRefCatGen2 = getattr(self.photoRefObjLoader, "ref_dataset_name", None)
301  if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat:
302  raise ValueError(
303  f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference "
304  f"catalogs are different. These options must be kept in sync until Gen2 is retired."
305  )
306 
307 
308 
314 
315 class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
316  r"""!Calibrate an exposure: measure sources and perform astrometric and
317  photometric calibration
318 
319  @anchor CalibrateTask_
320 
321  @section pipe_tasks_calibrate_Contents Contents
322 
323  - @ref pipe_tasks_calibrate_Purpose
324  - @ref pipe_tasks_calibrate_Initialize
325  - @ref pipe_tasks_calibrate_IO
326  - @ref pipe_tasks_calibrate_Config
327  - @ref pipe_tasks_calibrate_Metadata
328  - @ref pipe_tasks_calibrate_Debug
329 
330 
331  @section pipe_tasks_calibrate_Purpose Description
332 
333  Given an exposure with a good PSF model and aperture correction map
334  (e.g. as provided by @ref CharacterizeImageTask), perform the following
335  operations:
336  - Run detection and measurement
337  - Run astrometry subtask to fit an improved WCS
338  - Run photoCal subtask to fit the exposure's photometric zero-point
339 
340  @section pipe_tasks_calibrate_Initialize Task initialisation
341 
342  @copydoc \_\_init\_\_
343 
344  @section pipe_tasks_calibrate_IO Invoking the Task
345 
346  If you want this task to unpersist inputs or persist outputs, then call
347  the `runDataRef` method (a wrapper around the `run` method).
348 
349  If you already have the inputs unpersisted and do not want to persist the
350  output then it is more direct to call the `run` method:
351 
352  @section pipe_tasks_calibrate_Config Configuration parameters
353 
354  See @ref CalibrateConfig
355 
356  @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata
357 
358  Exposure metadata
359  <dl>
360  <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task
361  <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal
362  task
363  <dt>COLORTERM1 <dd>?? (always 0.0)
364  <dt>COLORTERM2 <dd>?? (always 0.0)
365  <dt>COLORTERM3 <dd>?? (always 0.0)
366  </dl>
367 
368  @section pipe_tasks_calibrate_Debug Debug variables
369 
370  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink
371  interface supports a flag
372  `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug
373  for more about `debug.py`.
374 
375  CalibrateTask has a debug dictionary containing one key:
376  <dl>
377  <dt>calibrate
378  <dd>frame (an int; <= 0 to not display) in which to display the exposure,
379  sources and matches. See @ref lsst.meas.astrom.displayAstrometry for
380  the meaning of the various symbols.
381  </dl>
382 
383  For example, put something like:
384  @code{.py}
385  import lsstDebug
386  def DebugInfo(name):
387  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would
388  # call us recursively
389  if name == "lsst.pipe.tasks.calibrate":
390  di.display = dict(
391  calibrate = 1,
392  )
393 
394  return di
395 
396  lsstDebug.Info = DebugInfo
397  @endcode
398  into your `debug.py` file and run `calibrateTask.py` with the `--debug`
399  flag.
400 
401  Some subtasks may have their own debug variables; see individual Task
402  documentation.
403  """
404 
405  # Example description used to live here, removed 2-20-2017 as per
406  # https://jira.lsstcorp.org/browse/DM-9520
407 
408  ConfigClass = CalibrateConfig
409  _DefaultName = "calibrate"
410  RunnerClass = pipeBase.ButlerInitializedTaskRunner
411 
412  def __init__(self, butler=None, astromRefObjLoader=None,
413  photoRefObjLoader=None, icSourceSchema=None,
414  initInputs=None, **kwargs):
415  """!Construct a CalibrateTask
416 
417  @param[in] butler The butler is passed to the refObjLoader constructor
418  in case it is needed. Ignored if the refObjLoader argument
419  provides a loader directly.
420  @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks
421  that supplies an external reference catalog for astrometric
422  calibration. May be None if the desired loader can be constructed
423  from the butler argument or all steps requiring a reference catalog
424  are disabled.
425  @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks
426  that supplies an external reference catalog for photometric
427  calibration. May be None if the desired loader can be constructed
428  from the butler argument or all steps requiring a reference catalog
429  are disabled.
430  @param[in] icSourceSchema schema for icSource catalog, or None.
431  Schema values specified in config.icSourceFieldsToCopy will be
432  taken from this schema. If set to None, no values will be
433  propagated from the icSourceCatalog
434  @param[in,out] kwargs other keyword arguments for
435  lsst.pipe.base.CmdLineTask
436  """
437  super().__init__(**kwargs)
438 
439  if icSourceSchema is None and butler is not None:
440  # Use butler to read icSourceSchema from disk.
441  icSourceSchema = butler.get("icSrc_schema", immediate=True).schema
442 
443  if icSourceSchema is None and butler is None and initInputs is not None:
444  icSourceSchema = initInputs['icSourceSchema'].schema
445 
446  if icSourceSchema is not None:
447  # use a schema mapper to avoid copying each field separately
448  self.schemaMapper = afwTable.SchemaMapper(icSourceSchema)
449  minimumSchema = afwTable.SourceTable.makeMinimalSchema()
450  self.schemaMapper.addMinimalSchema(minimumSchema, False)
451 
452  # Add fields to copy from an icSource catalog
453  # and a field to indicate that the source matched a source in that
454  # catalog. If any fields are missing then raise an exception, but
455  # first find all missing fields in order to make the error message
456  # more useful.
457  self.calibSourceKey = self.schemaMapper.addOutputField(
458  afwTable.Field["Flag"]("calib_detected",
459  "Source was detected as an icSource"))
460  missingFieldNames = []
461  for fieldName in self.config.icSourceFieldsToCopy:
462  try:
463  schemaItem = icSourceSchema.find(fieldName)
464  except Exception:
465  missingFieldNames.append(fieldName)
466  else:
467  # field found; if addMapping fails then raise an exception
468  self.schemaMapper.addMapping(schemaItem.getKey())
469 
470  if missingFieldNames:
471  raise RuntimeError("isSourceCat is missing fields {} "
472  "specified in icSourceFieldsToCopy"
473  .format(missingFieldNames))
474 
475  # produce a temporary schema to pass to the subtasks; finalize it
476  # later
477  self.schema = self.schemaMapper.editOutputSchema()
478  else:
479  self.schemaMapper = None
480  self.schema = afwTable.SourceTable.makeMinimalSchema()
481  self.makeSubtask('detection', schema=self.schema)
482 
483  self.algMetadata = dafBase.PropertyList()
484 
485  # Only create a subtask for fakes if configuration option is set
486  # N.B. the config for fake object task must be retargeted to a child
487  # of BaseFakeSourcesTask
488  if self.config.doInsertFakes:
489  self.makeSubtask("insertFakes")
490 
491  if self.config.doDeblend:
492  self.makeSubtask("deblend", schema=self.schema)
493  self.makeSubtask('measurement', schema=self.schema,
494  algMetadata=self.algMetadata)
495  if self.config.doApCorr:
496  self.makeSubtask('applyApCorr', schema=self.schema)
497  self.makeSubtask('catalogCalculation', schema=self.schema)
498 
499  if self.config.doAstrometry:
500  if astromRefObjLoader is None and butler is not None:
501  self.makeSubtask('astromRefObjLoader', butler=butler)
502  astromRefObjLoader = self.astromRefObjLoader
503  self.pixelMargin = astromRefObjLoader.config.pixelMargin
504  self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader,
505  schema=self.schema)
506  if self.config.doPhotoCal:
507  if photoRefObjLoader is None and butler is not None:
508  self.makeSubtask('photoRefObjLoader', butler=butler)
509  photoRefObjLoader = self.photoRefObjLoader
510  self.pixelMargin = photoRefObjLoader.config.pixelMargin
511  self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader,
512  schema=self.schema)
513 
514  if self.config.doEvalLocCalibration and self.config.doAstrometry and self.config.doPhotoCal:
515  self.makeSubtask("evalLocCalib", schema=self.schema)
516 
517  if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None):
518  raise RuntimeError("PipelineTask form of this task should not be initialized with "
519  "reference object loaders.")
520 
521  if self.schemaMapper is not None:
522  # finalize the schema
523  self.schema = self.schemaMapper.getOutputSchema()
524  self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
525 
526  sourceCatSchema = afwTable.SourceCatalog(self.schema)
527  sourceCatSchema.getTable().setMetadata(self.algMetadata)
528  self.outputSchema = sourceCatSchema
529 
530  @pipeBase.timeMethod
531  def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None,
532  doUnpersist=True):
533  """!Calibrate an exposure, optionally unpersisting inputs and
534  persisting outputs.
535 
536  This is a wrapper around the `run` method that unpersists inputs
537  (if `doUnpersist` true) and persists outputs (if `config.doWrite` true)
538 
539  @param[in] dataRef butler data reference corresponding to a science
540  image
541  @param[in,out] exposure characterized exposure (an
542  lsst.afw.image.ExposureF or similar), or None to unpersist existing
543  icExp and icBackground. See `run` method for details of what is
544  read and written.
545  @param[in,out] background initial model of background already
546  subtracted from exposure (an lsst.afw.math.BackgroundList). May be
547  None if no background has been subtracted, though that is unusual
548  for calibration. A refined background model is output. Ignored if
549  exposure is None.
550  @param[in] icSourceCat catalog from which to copy the fields specified
551  by icSourceKeys, or None;
552  @param[in] doUnpersist unpersist data:
553  - if True, exposure, background and icSourceCat are read from
554  dataRef and those three arguments must all be None;
555  - if False the exposure must be provided; background and
556  icSourceCat are optional. True is intended for running as a
557  command-line task, False for running as a subtask
558  @return same data as the calibrate method
559  """
560  self.log.info("Processing %s" % (dataRef.dataId))
561 
562  if doUnpersist:
563  if any(item is not None for item in (exposure, background,
564  icSourceCat)):
565  raise RuntimeError("doUnpersist true; exposure, background "
566  "and icSourceCat must all be None")
567  exposure = dataRef.get("icExp", immediate=True)
568  background = dataRef.get("icExpBackground", immediate=True)
569  icSourceCat = dataRef.get("icSrc", immediate=True)
570  elif exposure is None:
571  raise RuntimeError("doUnpersist false; exposure must be provided")
572 
573  exposureIdInfo = dataRef.get("expIdInfo")
574 
575  calRes = self.run(
576  exposure=exposure,
577  exposureIdInfo=exposureIdInfo,
578  background=background,
579  icSourceCat=icSourceCat,
580  )
581 
582  if self.config.doWrite:
583  self.writeOutputs(
584  dataRef=dataRef,
585  exposure=calRes.exposure,
586  background=calRes.background,
587  sourceCat=calRes.sourceCat,
588  astromMatches=calRes.astromMatches,
589  matchMeta=calRes.matchMeta,
590  )
591 
592  return calRes
593 
594  def runQuantum(self, butlerQC, inputRefs, outputRefs):
595  inputs = butlerQC.get(inputRefs)
596  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
597  returnMaxBits=True)
598  inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
599 
600  if self.config.doAstrometry:
601  refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
602  for ref in inputRefs.astromRefCat],
603  refCats=inputs.pop('astromRefCat'),
604  config=self.config.astromRefObjLoader, log=self.log)
605  self.pixelMargin = refObjLoader.config.pixelMargin
606  self.astrometry.setRefObjLoader(refObjLoader)
607 
608  if self.config.doPhotoCal:
609  photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
610  for ref in inputRefs.photoRefCat],
611  refCats=inputs.pop('photoRefCat'),
612  config=self.config.photoRefObjLoader,
613  log=self.log)
614  self.pixelMargin = photoRefObjLoader.config.pixelMargin
615  self.photoCal.match.setRefObjLoader(photoRefObjLoader)
616 
617  outputs = self.run(**inputs)
618 
619  if self.config.doWriteMatches:
620  normalizedMatches = afwTable.packMatches(outputs.astromMatches)
621  normalizedMatches.table.setMetadata(outputs.matchMeta)
622  if self.config.doWriteMatchesDenormalized:
623  denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
624  outputs.matchesDenormalized = denormMatches
625  outputs.matches = normalizedMatches
626  butlerQC.put(outputs, outputRefs)
627 
628  def run(self, exposure, exposureIdInfo=None, background=None,
629  icSourceCat=None):
630  """!Calibrate an exposure (science image or coadd)
631 
632  @param[in,out] exposure exposure to calibrate (an
633  lsst.afw.image.ExposureF or similar);
634  in:
635  - MaskedImage
636  - Psf
637  out:
638  - MaskedImage has background subtracted
639  - Wcs is replaced
640  - PhotoCalib is replaced
641  @param[in] exposureIdInfo ID info for exposure (an
642  lsst.obs.base.ExposureIdInfo) If not provided, returned
643  SourceCatalog IDs will not be globally unique.
644  @param[in,out] background background model already subtracted from
645  exposure (an lsst.afw.math.BackgroundList). May be None if no
646  background has been subtracted, though that is unusual for
647  calibration. A refined background model is output.
648  @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
649  from which we can copy some fields.
650 
651  @return pipe_base Struct containing these fields:
652  - exposure calibrate science exposure with refined WCS and PhotoCalib
653  - background model of background subtracted from exposure (an
654  lsst.afw.math.BackgroundList)
655  - sourceCat catalog of measured sources
656  - astromMatches list of source/refObj matches from the astrometry
657  solver
658  """
659  # detect, deblend and measure sources
660  if exposureIdInfo is None:
661  exposureIdInfo = ExposureIdInfo()
662 
663  if background is None:
664  background = BackgroundList()
665  sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
666  exposureIdInfo.unusedBits)
667  table = SourceTable.make(self.schema, sourceIdFactory)
668  table.setMetadata(self.algMetadata)
669 
670  detRes = self.detection.run(table=table, exposure=exposure,
671  doSmooth=True)
672  sourceCat = detRes.sources
673  if detRes.fpSets.background:
674  for bg in detRes.fpSets.background:
675  background.append(bg)
676  if self.config.doDeblend:
677  self.deblend.run(exposure=exposure, sources=sourceCat)
678  self.measurement.run(
679  measCat=sourceCat,
680  exposure=exposure,
681  exposureId=exposureIdInfo.expId
682  )
683  if self.config.doApCorr:
684  self.applyApCorr.run(
685  catalog=sourceCat,
686  apCorrMap=exposure.getInfo().getApCorrMap()
687  )
688  self.catalogCalculation.run(sourceCat)
689 
690  if icSourceCat is not None and \
691  len(self.config.icSourceFieldsToCopy) > 0:
692  self.copyIcSourceFields(icSourceCat=icSourceCat,
693  sourceCat=sourceCat)
694 
695  # TODO DM-11568: this contiguous check-and-copy could go away if we
696  # reserve enough space during SourceDetection and/or SourceDeblend.
697  # NOTE: sourceSelectors require contiguous catalogs, so ensure
698  # contiguity now, so views are preserved from here on.
699  if not sourceCat.isContiguous():
700  sourceCat = sourceCat.copy(deep=True)
701 
702  # perform astrometry calibration:
703  # fit an improved WCS and update the exposure's WCS in place
704  astromMatches = None
705  matchMeta = None
706  if self.config.doAstrometry:
707  try:
708  astromRes = self.astrometry.run(
709  exposure=exposure,
710  sourceCat=sourceCat,
711  )
712  astromMatches = astromRes.matches
713  matchMeta = astromRes.matchMeta
714  except Exception as e:
715  if self.config.requireAstrometry:
716  raise
717  self.log.warn("Unable to perform astrometric calibration "
718  "(%s): attempting to proceed" % e)
719 
720  # compute photometric calibration
721  if self.config.doPhotoCal:
722  try:
723  photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
724  exposure.setPhotoCalib(photoRes.photoCalib)
725  # TODO: reword this to phrase it in terms of the calibration factor?
726  self.log.info("Photometric zero-point: %f" %
727  photoRes.photoCalib.instFluxToMagnitude(1.0))
728  self.setMetadata(exposure=exposure, photoRes=photoRes)
729  except Exception as e:
730  if self.config.requirePhotoCal:
731  raise
732  self.log.warn("Unable to perform photometric calibration "
733  "(%s): attempting to proceed" % e)
734  self.setMetadata(exposure=exposure, photoRes=None)
735 
736  if self.config.doEvalLocCalibration and self.config.doAstrometry and self.config.doPhotoCal:
737  self.evalLocCalib.run(sourceCat, exposure)
738 
739  if self.config.doInsertFakes:
740  self.insertFakes.run(exposure, background=background)
741 
742  table = SourceTable.make(self.schema, sourceIdFactory)
743  table.setMetadata(self.algMetadata)
744 
745  detRes = self.detection.run(table=table, exposure=exposure,
746  doSmooth=True)
747  sourceCat = detRes.sources
748  if detRes.fpSets.background:
749  for bg in detRes.fpSets.background:
750  background.append(bg)
751  if self.config.doDeblend:
752  self.deblend.run(exposure=exposure, sources=sourceCat)
753  self.measurement.run(
754  measCat=sourceCat,
755  exposure=exposure,
756  exposureId=exposureIdInfo.expId
757  )
758  if self.config.doApCorr:
759  self.applyApCorr.run(
760  catalog=sourceCat,
761  apCorrMap=exposure.getInfo().getApCorrMap()
762  )
763  self.catalogCalculation.run(sourceCat)
764 
765  if icSourceCat is not None and len(self.config.icSourceFieldsToCopy) > 0:
766  self.copyIcSourceFields(icSourceCat=icSourceCat,
767  sourceCat=sourceCat)
768 
769  frame = getDebugFrame(self._display, "calibrate")
770  if frame:
771  displayAstrometry(
772  sourceCat=sourceCat,
773  exposure=exposure,
774  matches=astromMatches,
775  frame=frame,
776  pause=False,
777  )
778 
779  return pipeBase.Struct(
780  exposure=exposure,
781  background=background,
782  sourceCat=sourceCat,
783  astromMatches=astromMatches,
784  matchMeta=matchMeta,
785  # These are duplicate entries with different names for use with
786  # gen3 middleware
787  outputExposure=exposure,
788  outputCat=sourceCat,
789  outputBackground=background,
790  )
791 
792  def writeOutputs(self, dataRef, exposure, background, sourceCat,
793  astromMatches, matchMeta):
794  """Write output data to the output repository
795 
796  @param[in] dataRef butler data reference corresponding to a science
797  image
798  @param[in] exposure exposure to write
799  @param[in] background background model for exposure
800  @param[in] sourceCat catalog of measured sources
801  @param[in] astromMatches list of source/refObj matches from the
802  astrometry solver
803  """
804  dataRef.put(sourceCat, "src")
805  if self.config.doWriteMatches and astromMatches is not None:
806  normalizedMatches = afwTable.packMatches(astromMatches)
807  normalizedMatches.table.setMetadata(matchMeta)
808  dataRef.put(normalizedMatches, "srcMatch")
809  if self.config.doWriteMatchesDenormalized:
810  denormMatches = denormalizeMatches(astromMatches, matchMeta)
811  dataRef.put(denormMatches, "srcMatchFull")
812  if self.config.doWriteExposure:
813  dataRef.put(exposure, "calexp")
814  dataRef.put(background, "calexpBackground")
815 
816  def getSchemaCatalogs(self):
817  """Return a dict of empty catalogs for each catalog dataset produced
818  by this task.
819  """
820  sourceCat = afwTable.SourceCatalog(self.schema)
821  sourceCat.getTable().setMetadata(self.algMetadata)
822  return {"src": sourceCat}
823 
824  def setMetadata(self, exposure, photoRes=None):
825  """!Set task and exposure metadata
826 
827  Logs a warning and continues if needed data is missing.
828 
829  @param[in,out] exposure exposure whose metadata is to be set
830  @param[in] photoRes results of running photoCal; if None then it was
831  not run
832  """
833  if photoRes is None:
834  return
835 
836  metadata = exposure.getMetadata()
837 
838  # convert zero-point to (mag/sec/adu) for task MAGZERO metadata
839  try:
840  exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
841  magZero = photoRes.zp - 2.5*math.log10(exposureTime)
842  except Exception:
843  self.log.warn("Could not set normalized MAGZERO in header: no "
844  "exposure time")
845  magZero = math.nan
846 
847  try:
848  metadata.set('MAGZERO', magZero)
849  metadata.set('MAGZERO_RMS', photoRes.sigma)
850  metadata.set('MAGZERO_NOBJ', photoRes.ngood)
851  metadata.set('COLORTERM1', 0.0)
852  metadata.set('COLORTERM2', 0.0)
853  metadata.set('COLORTERM3', 0.0)
854  except Exception as e:
855  self.log.warn("Could not set exposure metadata: %s" % (e,))
856 
857  def copyIcSourceFields(self, icSourceCat, sourceCat):
858  """!Match sources in icSourceCat and sourceCat and copy the specified fields
859 
860  @param[in] icSourceCat catalog from which to copy fields
861  @param[in,out] sourceCat catalog to which to copy fields
862 
863  The fields copied are those specified by `config.icSourceFieldsToCopy`
864  that actually exist in the schema. This was set up by the constructor
865  using self.schemaMapper.
866  """
867  if self.schemaMapper is None:
868  raise RuntimeError("To copy icSource fields you must specify "
869  "icSourceSchema nd icSourceKeys when "
870  "constructing this task")
871  if icSourceCat is None or sourceCat is None:
872  raise RuntimeError("icSourceCat and sourceCat must both be "
873  "specified")
874  if len(self.config.icSourceFieldsToCopy) == 0:
875  self.log.warn("copyIcSourceFields doing nothing because "
876  "icSourceFieldsToCopy is empty")
877  return
878 
879  mc = afwTable.MatchControl()
880  mc.findOnlyClosest = False # return all matched objects
881  matches = afwTable.matchXy(icSourceCat, sourceCat,
882  self.config.matchRadiusPix, mc)
883  if self.config.doDeblend:
884  deblendKey = sourceCat.schema["deblend_nChild"].asKey()
885  # if deblended, keep children
886  matches = [m for m in matches if m[1].get(deblendKey) == 0]
887 
888  # Because we had to allow multiple matches to handle parents, we now
889  # need to prune to the best matches
890  # closest matches as a dict of icSourceCat source ID:
891  # (icSourceCat source, sourceCat source, distance in pixels)
892  bestMatches = {}
893  for m0, m1, d in matches:
894  id0 = m0.getId()
895  match = bestMatches.get(id0)
896  if match is None or d <= match[2]:
897  bestMatches[id0] = (m0, m1, d)
898  matches = list(bestMatches.values())
899 
900  # Check that no sourceCat sources are listed twice (we already know
901  # that each match has a unique icSourceCat source ID, due to using
902  # that ID as the key in bestMatches)
903  numMatches = len(matches)
904  numUniqueSources = len(set(m[1].getId() for m in matches))
905  if numUniqueSources != numMatches:
906  self.log.warn("{} icSourceCat sources matched only {} sourceCat "
907  "sources".format(numMatches, numUniqueSources))
908 
909  self.log.info("Copying flags from icSourceCat to sourceCat for "
910  "%s sources" % (numMatches,))
911 
912  # For each match: set the calibSourceKey flag and copy the desired
913  # fields
914  for icSrc, src, d in matches:
915  src.setFlag(self.calibSourceKey, True)
916  # src.assign copies the footprint from icSrc, which we don't want
917  # (DM-407)
918  # so set icSrc's footprint to src's footprint before src.assign,
919  # then restore it
920  icSrcFootprint = icSrc.getFootprint()
921  try:
922  icSrc.setFootprint(src.getFootprint())
923  src.assign(icSrc, self.schemaMapper)
924  finally:
925  icSrc.setFootprint(icSrcFootprint)
def copyIcSourceFields(self, icSourceCat, sourceCat)
Match sources in icSourceCat and sourceCat and copy the specified fields.
Definition: calibrate.py:857
def __init__(self, butler=None, astromRefObjLoader=None, photoRefObjLoader=None, icSourceSchema=None, initInputs=None, kwargs)
Construct a CalibrateTask.
Definition: calibrate.py:414
def run(self, exposure, exposureIdInfo=None, background=None, icSourceCat=None)
Calibrate an exposure (science image or coadd)
Definition: calibrate.py:629
def writeOutputs(self, dataRef, exposure, background, sourceCat, astromMatches, matchMeta)
Definition: calibrate.py:793
def setMetadata(self, exposure, photoRes=None)
Set task and exposure metadata.
Definition: calibrate.py:824
def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None, doUnpersist=True)
Calibrate an exposure, optionally unpersisting inputs and persisting outputs.
Definition: calibrate.py:532
Calibrate an exposure: measure sources and perform astrometric and photometric calibration.
Definition: calibrate.py:315
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: calibrate.py:594