lsst.pipe.tasks  19.0.0-32-gedca71e0+2
calibrate.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2016 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import math
23 
24 from lsstDebug import getDebugFrame
25 import lsst.pex.config as pexConfig
26 import lsst.pipe.base as pipeBase
28 import lsst.afw.table as afwTable
29 from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches
30 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
31 from lsst.obs.base import ExposureIdInfo
32 import lsst.daf.base as dafBase
33 from lsst.afw.math import BackgroundList
34 from lsst.afw.table import IdFactory, SourceTable
35 from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader
36 from lsst.meas.base import (SingleFrameMeasurementTask,
37  ApplyApCorrTask,
38  CatalogCalculationTask)
39 from lsst.meas.deblender import SourceDeblendTask
40 from .fakes import BaseFakeSourcesTask
41 from .photoCal import PhotoCalTask
42 
43 __all__ = ["CalibrateConfig", "CalibrateTask"]
44 
45 
46 class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"),
47  defaultTemplates={}):
48 
49  icSourceSchema = cT.InitInput(
50  doc="Schema produced by characterize image task, used to initialize this task",
51  name="icSrc_schema",
52  storageClass="SourceCatalog",
53  )
54 
55  outputSchema = cT.InitOutput(
56  doc="Schema after CalibrateTask has been initialized",
57  name="src_schema",
58  storageClass="SourceCatalog",
59  )
60 
61  exposure = cT.Input(
62  doc="Input image to calibrate",
63  name="icExp",
64  storageClass="ExposureF",
65  dimensions=("instrument", "visit", "detector"),
66  )
67 
68  background = cT.Input(
69  doc="Backgrounds determined by characterize task",
70  name="icExpBackground",
71  storageClass="Background",
72  dimensions=("instrument", "visit", "detector"),
73  )
74 
75  icSourceCat = cT.Input(
76  doc="Source catalog created by characterize task",
77  name="icSrc",
78  storageClass="SourceCatalog",
79  dimensions=("instrument", "visit", "detector"),
80  )
81 
82  astromRefCat = cT.PrerequisiteInput(
83  doc="Reference catalog to use for astrometry",
84  name="cal_ref_cat",
85  storageClass="SimpleCatalog",
86  dimensions=("skypix",),
87  deferLoad=True,
88  multiple=True,
89  )
90 
91  photoRefCat = cT.PrerequisiteInput(
92  doc="Reference catalog to use for photometric calibration",
93  name="cal_ref_cat",
94  storageClass="SimpleCatalog",
95  dimensions=("skypix",),
96  deferLoad=True,
97  multiple=True
98  )
99 
100  outputExposure = cT.Output(
101  doc="Exposure after running calibration task",
102  name="calexp",
103  storageClass="ExposureF",
104  dimensions=("instrument", "visit", "detector"),
105  )
106 
107  outputCat = cT.Output(
108  doc="Source catalog produced in calibrate task",
109  name="src",
110  storageClass="SourceCatalog",
111  dimensions=("instrument", "visit", "detector"),
112  )
113 
114  outputBackground = cT.Output(
115  doc="Background models estimated in calibration task",
116  name="calexpBackground",
117  storageClass="Background",
118  dimensions=("instrument", "visit", "detector"),
119  )
120 
121  matches = cT.Output(
122  doc="Source/refObj matches from the astrometry solver",
123  name="srcMatch",
124  storageClass="Catalog",
125  dimensions=("instrument", "visit", "detector"),
126  )
127 
128  matchesDenormalized = cT.Output(
129  doc="Denormalized matches from astrometry solver",
130  name="srcMatchFull",
131  storageClass="Catalog",
132  dimensions=("instrument", "visit", "detector"),
133  )
134 
135  def __init__(self, *, config=None):
136  super().__init__(config=config)
137  if config.doWriteMatches is False:
138  self.outputs.remove("matches")
139  if config.doWriteMatchesDenormalized is False:
140  self.outputs.remove("matchesDenormalized")
141 
142 
143 class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections):
144  """Config for CalibrateTask"""
145  doWrite = pexConfig.Field(
146  dtype=bool,
147  default=True,
148  doc="Save calibration results?",
149  )
150  doWriteHeavyFootprintsInSources = pexConfig.Field(
151  dtype=bool,
152  default=True,
153  doc="Include HeavyFootprint data in source table? If false then heavy "
154  "footprints are saved as normal footprints, which saves some space"
155  )
156  doWriteMatches = pexConfig.Field(
157  dtype=bool,
158  default=True,
159  doc="Write reference matches (ignored if doWrite false)?",
160  )
161  doWriteMatchesDenormalized = pexConfig.Field(
162  dtype=bool,
163  default=False,
164  doc=("Write reference matches in denormalized format? "
165  "This format uses more disk space, but is more convenient to "
166  "read. Ignored if doWriteMatches=False or doWrite=False."),
167  )
168  doAstrometry = pexConfig.Field(
169  dtype=bool,
170  default=True,
171  doc="Perform astrometric calibration?",
172  )
173  astromRefObjLoader = pexConfig.ConfigurableField(
174  target=LoadIndexedReferenceObjectsTask,
175  doc="reference object loader for astrometric calibration",
176  )
177  photoRefObjLoader = pexConfig.ConfigurableField(
178  target=LoadIndexedReferenceObjectsTask,
179  doc="reference object loader for photometric calibration",
180  )
181  astrometry = pexConfig.ConfigurableField(
182  target=AstrometryTask,
183  doc="Perform astrometric calibration to refine the WCS",
184  )
185  requireAstrometry = pexConfig.Field(
186  dtype=bool,
187  default=True,
188  doc=("Raise an exception if astrometry fails? Ignored if doAstrometry "
189  "false."),
190  )
191  doPhotoCal = pexConfig.Field(
192  dtype=bool,
193  default=True,
194  doc="Perform phometric calibration?",
195  )
196  requirePhotoCal = pexConfig.Field(
197  dtype=bool,
198  default=True,
199  doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal "
200  "false."),
201  )
202  photoCal = pexConfig.ConfigurableField(
203  target=PhotoCalTask,
204  doc="Perform photometric calibration",
205  )
206  icSourceFieldsToCopy = pexConfig.ListField(
207  dtype=str,
208  default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
209  doc=("Fields to copy from the icSource catalog to the output catalog "
210  "for matching sources Any missing fields will trigger a "
211  "RuntimeError exception. Ignored if icSourceCat is not provided.")
212  )
213  matchRadiusPix = pexConfig.Field(
214  dtype=float,
215  default=3,
216  doc=("Match radius for matching icSourceCat objects to sourceCat "
217  "objects (pixels)"),
218  )
219  checkUnitsParseStrict = pexConfig.Field(
220  doc=("Strictness of Astropy unit compatibility check, can be 'raise', "
221  "'warn' or 'silent'"),
222  dtype=str,
223  default="raise",
224  )
225  detection = pexConfig.ConfigurableField(
226  target=SourceDetectionTask,
227  doc="Detect sources"
228  )
229  doDeblend = pexConfig.Field(
230  dtype=bool,
231  default=True,
232  doc="Run deblender input exposure"
233  )
234  deblend = pexConfig.ConfigurableField(
235  target=SourceDeblendTask,
236  doc="Split blended sources into their components"
237  )
238  measurement = pexConfig.ConfigurableField(
239  target=SingleFrameMeasurementTask,
240  doc="Measure sources"
241  )
242  doApCorr = pexConfig.Field(
243  dtype=bool,
244  default=True,
245  doc="Run subtask to apply aperture correction"
246  )
247  applyApCorr = pexConfig.ConfigurableField(
248  target=ApplyApCorrTask,
249  doc="Subtask to apply aperture corrections"
250  )
251  # If doApCorr is False, and the exposure does not have apcorrections
252  # already applied, the active plugins in catalogCalculation almost
253  # certainly should not contain the characterization plugin
254  catalogCalculation = pexConfig.ConfigurableField(
255  target=CatalogCalculationTask,
256  doc="Subtask to run catalogCalculation plugins on catalog"
257  )
258  doInsertFakes = pexConfig.Field(
259  dtype=bool,
260  default=False,
261  doc="Run fake sources injection task"
262  )
263  insertFakes = pexConfig.ConfigurableField(
264  target=BaseFakeSourcesTask,
265  doc="Injection of fake sources for testing purposes (must be "
266  "retargeted)"
267  )
268  doWriteExposure = pexConfig.Field(
269  dtype=bool,
270  default=True,
271  doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a "
272  "normal calexp but as a fakes_calexp."
273  )
274 
275  def setDefaults(self):
276  super().setDefaults()
277  self.detection.doTempLocalBackground = False
278  self.deblend.maxFootprintSize = 2000
279 
280  def validate(self):
281  super().validate()
282  astromRefCatGen2 = getattr(self.astromRefObjLoader, "ref_dataset_name", None)
283  if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat:
284  raise ValueError(
285  f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference "
286  f"catalogs are different. These options must be kept in sync until Gen2 is retired."
287  )
288  photoRefCatGen2 = getattr(self.photoRefObjLoader, "ref_dataset_name", None)
289  if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat:
290  raise ValueError(
291  f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference "
292  f"catalogs are different. These options must be kept in sync until Gen2 is retired."
293  )
294 
295 
296 
302 
303 class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
304  r"""!Calibrate an exposure: measure sources and perform astrometric and
305  photometric calibration
306 
307  @anchor CalibrateTask_
308 
309  @section pipe_tasks_calibrate_Contents Contents
310 
311  - @ref pipe_tasks_calibrate_Purpose
312  - @ref pipe_tasks_calibrate_Initialize
313  - @ref pipe_tasks_calibrate_IO
314  - @ref pipe_tasks_calibrate_Config
315  - @ref pipe_tasks_calibrate_Metadata
316  - @ref pipe_tasks_calibrate_Debug
317 
318 
319  @section pipe_tasks_calibrate_Purpose Description
320 
321  Given an exposure with a good PSF model and aperture correction map
322  (e.g. as provided by @ref CharacterizeImageTask), perform the following
323  operations:
324  - Run detection and measurement
325  - Run astrometry subtask to fit an improved WCS
326  - Run photoCal subtask to fit the exposure's photometric zero-point
327 
328  @section pipe_tasks_calibrate_Initialize Task initialisation
329 
330  @copydoc \_\_init\_\_
331 
332  @section pipe_tasks_calibrate_IO Invoking the Task
333 
334  If you want this task to unpersist inputs or persist outputs, then call
335  the `runDataRef` method (a wrapper around the `run` method).
336 
337  If you already have the inputs unpersisted and do not want to persist the
338  output then it is more direct to call the `run` method:
339 
340  @section pipe_tasks_calibrate_Config Configuration parameters
341 
342  See @ref CalibrateConfig
343 
344  @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata
345 
346  Exposure metadata
347  <dl>
348  <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task
349  <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal
350  task
351  <dt>COLORTERM1 <dd>?? (always 0.0)
352  <dt>COLORTERM2 <dd>?? (always 0.0)
353  <dt>COLORTERM3 <dd>?? (always 0.0)
354  </dl>
355 
356  @section pipe_tasks_calibrate_Debug Debug variables
357 
358  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink
359  interface supports a flag
360  `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug
361  for more about `debug.py`.
362 
363  CalibrateTask has a debug dictionary containing one key:
364  <dl>
365  <dt>calibrate
366  <dd>frame (an int; <= 0 to not display) in which to display the exposure,
367  sources and matches. See @ref lsst.meas.astrom.displayAstrometry for
368  the meaning of the various symbols.
369  </dl>
370 
371  For example, put something like:
372  @code{.py}
373  import lsstDebug
374  def DebugInfo(name):
375  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would
376  # call us recursively
377  if name == "lsst.pipe.tasks.calibrate":
378  di.display = dict(
379  calibrate = 1,
380  )
381 
382  return di
383 
384  lsstDebug.Info = DebugInfo
385  @endcode
386  into your `debug.py` file and run `calibrateTask.py` with the `--debug`
387  flag.
388 
389  Some subtasks may have their own debug variables; see individual Task
390  documentation.
391  """
392 
393  # Example description used to live here, removed 2-20-2017 as per
394  # https://jira.lsstcorp.org/browse/DM-9520
395 
396  ConfigClass = CalibrateConfig
397  _DefaultName = "calibrate"
398  RunnerClass = pipeBase.ButlerInitializedTaskRunner
399 
400  def __init__(self, butler=None, astromRefObjLoader=None,
401  photoRefObjLoader=None, icSourceSchema=None,
402  initInputs=None, **kwargs):
403  """!Construct a CalibrateTask
404 
405  @param[in] butler The butler is passed to the refObjLoader constructor
406  in case it is needed. Ignored if the refObjLoader argument
407  provides a loader directly.
408  @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks
409  that supplies an external reference catalog for astrometric
410  calibration. May be None if the desired loader can be constructed
411  from the butler argument or all steps requiring a reference catalog
412  are disabled.
413  @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks
414  that supplies an external reference catalog for photometric
415  calibration. May be None if the desired loader can be constructed
416  from the butler argument or all steps requiring a reference catalog
417  are disabled.
418  @param[in] icSourceSchema schema for icSource catalog, or None.
419  Schema values specified in config.icSourceFieldsToCopy will be
420  taken from this schema. If set to None, no values will be
421  propagated from the icSourceCatalog
422  @param[in,out] kwargs other keyword arguments for
423  lsst.pipe.base.CmdLineTask
424  """
425  super().__init__(**kwargs)
426 
427  if icSourceSchema is None and butler is not None:
428  # Use butler to read icSourceSchema from disk.
429  icSourceSchema = butler.get("icSrc_schema", immediate=True).schema
430 
431  if icSourceSchema is None and butler is None and initInputs is not None:
432  icSourceSchema = initInputs['icSourceSchema'].schema
433 
434  if icSourceSchema is not None:
435  # use a schema mapper to avoid copying each field separately
436  self.schemaMapper = afwTable.SchemaMapper(icSourceSchema)
437  minimumSchema = afwTable.SourceTable.makeMinimalSchema()
438  self.schemaMapper.addMinimalSchema(minimumSchema, False)
439 
440  # Add fields to copy from an icSource catalog
441  # and a field to indicate that the source matched a source in that
442  # catalog. If any fields are missing then raise an exception, but
443  # first find all missing fields in order to make the error message
444  # more useful.
445  self.calibSourceKey = self.schemaMapper.addOutputField(
446  afwTable.Field["Flag"]("calib_detected",
447  "Source was detected as an icSource"))
448  missingFieldNames = []
449  for fieldName in self.config.icSourceFieldsToCopy:
450  try:
451  schemaItem = icSourceSchema.find(fieldName)
452  except Exception:
453  missingFieldNames.append(fieldName)
454  else:
455  # field found; if addMapping fails then raise an exception
456  self.schemaMapper.addMapping(schemaItem.getKey())
457 
458  if missingFieldNames:
459  raise RuntimeError("isSourceCat is missing fields {} "
460  "specified in icSourceFieldsToCopy"
461  .format(missingFieldNames))
462 
463  # produce a temporary schema to pass to the subtasks; finalize it
464  # later
465  self.schema = self.schemaMapper.editOutputSchema()
466  else:
467  self.schemaMapper = None
468  self.schema = afwTable.SourceTable.makeMinimalSchema()
469  self.makeSubtask('detection', schema=self.schema)
470 
471  self.algMetadata = dafBase.PropertyList()
472 
473  # Only create a subtask for fakes if configuration option is set
474  # N.B. the config for fake object task must be retargeted to a child
475  # of BaseFakeSourcesTask
476  if self.config.doInsertFakes:
477  self.makeSubtask("insertFakes")
478 
479  if self.config.doDeblend:
480  self.makeSubtask("deblend", schema=self.schema)
481  self.makeSubtask('measurement', schema=self.schema,
482  algMetadata=self.algMetadata)
483  if self.config.doApCorr:
484  self.makeSubtask('applyApCorr', schema=self.schema)
485  self.makeSubtask('catalogCalculation', schema=self.schema)
486 
487  if self.config.doAstrometry:
488  if astromRefObjLoader is None and butler is not None:
489  self.makeSubtask('astromRefObjLoader', butler=butler)
490  astromRefObjLoader = self.astromRefObjLoader
491  self.pixelMargin = astromRefObjLoader.config.pixelMargin
492  self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader,
493  schema=self.schema)
494  if self.config.doPhotoCal:
495  if photoRefObjLoader is None and butler is not None:
496  self.makeSubtask('photoRefObjLoader', butler=butler)
497  photoRefObjLoader = self.photoRefObjLoader
498  self.pixelMargin = photoRefObjLoader.config.pixelMargin
499  self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader,
500  schema=self.schema)
501 
502  if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None):
503  raise RuntimeError("PipelineTask form of this task should not be initialized with "
504  "reference object loaders.")
505 
506  if self.schemaMapper is not None:
507  # finalize the schema
508  self.schema = self.schemaMapper.getOutputSchema()
509  self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
510 
511  sourceCatSchema = afwTable.SourceCatalog(self.schema)
512  sourceCatSchema.getTable().setMetadata(self.algMetadata)
513  self.outputSchema = sourceCatSchema
514 
515  @pipeBase.timeMethod
516  def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None,
517  doUnpersist=True):
518  """!Calibrate an exposure, optionally unpersisting inputs and
519  persisting outputs.
520 
521  This is a wrapper around the `run` method that unpersists inputs
522  (if `doUnpersist` true) and persists outputs (if `config.doWrite` true)
523 
524  @param[in] dataRef butler data reference corresponding to a science
525  image
526  @param[in,out] exposure characterized exposure (an
527  lsst.afw.image.ExposureF or similar), or None to unpersist existing
528  icExp and icBackground. See `run` method for details of what is
529  read and written.
530  @param[in,out] background initial model of background already
531  subtracted from exposure (an lsst.afw.math.BackgroundList). May be
532  None if no background has been subtracted, though that is unusual
533  for calibration. A refined background model is output. Ignored if
534  exposure is None.
535  @param[in] icSourceCat catalog from which to copy the fields specified
536  by icSourceKeys, or None;
537  @param[in] doUnpersist unpersist data:
538  - if True, exposure, background and icSourceCat are read from
539  dataRef and those three arguments must all be None;
540  - if False the exposure must be provided; background and
541  icSourceCat are optional. True is intended for running as a
542  command-line task, False for running as a subtask
543  @return same data as the calibrate method
544  """
545  self.log.info("Processing %s" % (dataRef.dataId))
546 
547  if doUnpersist:
548  if any(item is not None for item in (exposure, background,
549  icSourceCat)):
550  raise RuntimeError("doUnpersist true; exposure, background "
551  "and icSourceCat must all be None")
552  exposure = dataRef.get("icExp", immediate=True)
553  background = dataRef.get("icExpBackground", immediate=True)
554  icSourceCat = dataRef.get("icSrc", immediate=True)
555  elif exposure is None:
556  raise RuntimeError("doUnpersist false; exposure must be provided")
557 
558  exposureIdInfo = dataRef.get("expIdInfo")
559 
560  calRes = self.run(
561  exposure=exposure,
562  exposureIdInfo=exposureIdInfo,
563  background=background,
564  icSourceCat=icSourceCat,
565  )
566 
567  if self.config.doWrite:
568  self.writeOutputs(
569  dataRef=dataRef,
570  exposure=calRes.exposure,
571  background=calRes.background,
572  sourceCat=calRes.sourceCat,
573  astromMatches=calRes.astromMatches,
574  matchMeta=calRes.matchMeta,
575  )
576 
577  return calRes
578 
579  def runQuantum(self, butlerQC, inputRefs, outputRefs):
580  inputs = butlerQC.get(inputRefs)
581  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
582  returnMaxBits=True)
583  inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
584 
585  if self.config.doAstrometry:
586  refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
587  for ref in inputRefs.astromRefCat],
588  refCats=inputs.pop('astromRefCat'),
589  config=self.config.astromRefObjLoader, log=self.log)
590  self.pixelMargin = refObjLoader.config.pixelMargin
591  self.astrometry.setRefObjLoader(refObjLoader)
592 
593  if self.config.doPhotoCal:
594  photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
595  for ref in inputRefs.photoRefCat],
596  refCats=inputs.pop('photoRefCat'),
597  config=self.config.photoRefObjLoader,
598  log=self.log)
599  self.pixelMargin = photoRefObjLoader.config.pixelMargin
600  self.photoCal.match.setRefObjLoader(photoRefObjLoader)
601 
602  outputs = self.run(**inputs)
603 
604  if self.config.doWriteMatches:
605  normalizedMatches = afwTable.packMatches(outputs.astromMatches)
606  normalizedMatches.table.setMetadata(outputs.matchMeta)
607  if self.config.doWriteMatchesDenormalized:
608  denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
609  outputs.matchesDenormalized = denormMatches
610  outputs.matches = normalizedMatches
611  butlerQC.put(outputs, outputRefs)
612 
613  def run(self, exposure, exposureIdInfo=None, background=None,
614  icSourceCat=None):
615  """!Calibrate an exposure (science image or coadd)
616 
617  @param[in,out] exposure exposure to calibrate (an
618  lsst.afw.image.ExposureF or similar);
619  in:
620  - MaskedImage
621  - Psf
622  out:
623  - MaskedImage has background subtracted
624  - Wcs is replaced
625  - PhotoCalib is replaced
626  @param[in] exposureIdInfo ID info for exposure (an
627  lsst.obs.base.ExposureIdInfo) If not provided, returned
628  SourceCatalog IDs will not be globally unique.
629  @param[in,out] background background model already subtracted from
630  exposure (an lsst.afw.math.BackgroundList). May be None if no
631  background has been subtracted, though that is unusual for
632  calibration. A refined background model is output.
633  @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
634  from which we can copy some fields.
635 
636  @return pipe_base Struct containing these fields:
637  - exposure calibrate science exposure with refined WCS and PhotoCalib
638  - background model of background subtracted from exposure (an
639  lsst.afw.math.BackgroundList)
640  - sourceCat catalog of measured sources
641  - astromMatches list of source/refObj matches from the astrometry
642  solver
643  """
644  # detect, deblend and measure sources
645  if exposureIdInfo is None:
646  exposureIdInfo = ExposureIdInfo()
647 
648  if background is None:
649  background = BackgroundList()
650  sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
651  exposureIdInfo.unusedBits)
652  table = SourceTable.make(self.schema, sourceIdFactory)
653  table.setMetadata(self.algMetadata)
654 
655  detRes = self.detection.run(table=table, exposure=exposure,
656  doSmooth=True)
657  sourceCat = detRes.sources
658  if detRes.fpSets.background:
659  for bg in detRes.fpSets.background:
660  background.append(bg)
661  if self.config.doDeblend:
662  self.deblend.run(exposure=exposure, sources=sourceCat)
663  self.measurement.run(
664  measCat=sourceCat,
665  exposure=exposure,
666  exposureId=exposureIdInfo.expId
667  )
668  if self.config.doApCorr:
669  self.applyApCorr.run(
670  catalog=sourceCat,
671  apCorrMap=exposure.getInfo().getApCorrMap()
672  )
673  self.catalogCalculation.run(sourceCat)
674 
675  if icSourceCat is not None and \
676  len(self.config.icSourceFieldsToCopy) > 0:
677  self.copyIcSourceFields(icSourceCat=icSourceCat,
678  sourceCat=sourceCat)
679 
680  # TODO DM-11568: this contiguous check-and-copy could go away if we
681  # reserve enough space during SourceDetection and/or SourceDeblend.
682  # NOTE: sourceSelectors require contiguous catalogs, so ensure
683  # contiguity now, so views are preserved from here on.
684  if not sourceCat.isContiguous():
685  sourceCat = sourceCat.copy(deep=True)
686 
687  # perform astrometry calibration:
688  # fit an improved WCS and update the exposure's WCS in place
689  astromMatches = None
690  matchMeta = None
691  if self.config.doAstrometry:
692  try:
693  astromRes = self.astrometry.run(
694  exposure=exposure,
695  sourceCat=sourceCat,
696  )
697  astromMatches = astromRes.matches
698  matchMeta = astromRes.matchMeta
699  except Exception as e:
700  if self.config.requireAstrometry:
701  raise
702  self.log.warn("Unable to perform astrometric calibration "
703  "(%s): attempting to proceed" % e)
704 
705  # compute photometric calibration
706  if self.config.doPhotoCal:
707  try:
708  photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
709  exposure.setPhotoCalib(photoRes.photoCalib)
710  # TODO: reword this to phrase it in terms of the calibration factor?
711  self.log.info("Photometric zero-point: %f" %
712  photoRes.photoCalib.instFluxToMagnitude(1.0))
713  self.setMetadata(exposure=exposure, photoRes=photoRes)
714  except Exception as e:
715  if self.config.requirePhotoCal:
716  raise
717  self.log.warn("Unable to perform photometric calibration "
718  "(%s): attempting to proceed" % e)
719  self.setMetadata(exposure=exposure, photoRes=None)
720 
721  if self.config.doInsertFakes:
722  self.insertFakes.run(exposure, background=background)
723 
724  table = SourceTable.make(self.schema, sourceIdFactory)
725  table.setMetadata(self.algMetadata)
726 
727  detRes = self.detection.run(table=table, exposure=exposure,
728  doSmooth=True)
729  sourceCat = detRes.sources
730  if detRes.fpSets.background:
731  for bg in detRes.fpSets.background:
732  background.append(bg)
733  if self.config.doDeblend:
734  self.deblend.run(exposure=exposure, sources=sourceCat)
735  self.measurement.run(
736  measCat=sourceCat,
737  exposure=exposure,
738  exposureId=exposureIdInfo.expId
739  )
740  if self.config.doApCorr:
741  self.applyApCorr.run(
742  catalog=sourceCat,
743  apCorrMap=exposure.getInfo().getApCorrMap()
744  )
745  self.catalogCalculation.run(sourceCat)
746 
747  if icSourceCat is not None and len(self.config.icSourceFieldsToCopy) > 0:
748  self.copyIcSourceFields(icSourceCat=icSourceCat,
749  sourceCat=sourceCat)
750 
751  frame = getDebugFrame(self._display, "calibrate")
752  if frame:
753  displayAstrometry(
754  sourceCat=sourceCat,
755  exposure=exposure,
756  matches=astromMatches,
757  frame=frame,
758  pause=False,
759  )
760 
761  return pipeBase.Struct(
762  exposure=exposure,
763  background=background,
764  sourceCat=sourceCat,
765  astromMatches=astromMatches,
766  matchMeta=matchMeta,
767  # These are duplicate entries with different names for use with
768  # gen3 middleware
769  outputExposure=exposure,
770  outputCat=sourceCat,
771  outputBackground=background,
772  )
773 
774  def writeOutputs(self, dataRef, exposure, background, sourceCat,
775  astromMatches, matchMeta):
776  """Write output data to the output repository
777 
778  @param[in] dataRef butler data reference corresponding to a science
779  image
780  @param[in] exposure exposure to write
781  @param[in] background background model for exposure
782  @param[in] sourceCat catalog of measured sources
783  @param[in] astromMatches list of source/refObj matches from the
784  astrometry solver
785  """
786  dataRef.put(sourceCat, "src")
787  if self.config.doWriteMatches and astromMatches is not None:
788  normalizedMatches = afwTable.packMatches(astromMatches)
789  normalizedMatches.table.setMetadata(matchMeta)
790  dataRef.put(normalizedMatches, "srcMatch")
791  if self.config.doWriteMatchesDenormalized:
792  denormMatches = denormalizeMatches(astromMatches, matchMeta)
793  dataRef.put(denormMatches, "srcMatchFull")
794  if self.config.doWriteExposure:
795  dataRef.put(exposure, "calexp")
796  dataRef.put(background, "calexpBackground")
797 
798  def getSchemaCatalogs(self):
799  """Return a dict of empty catalogs for each catalog dataset produced
800  by this task.
801  """
802  sourceCat = afwTable.SourceCatalog(self.schema)
803  sourceCat.getTable().setMetadata(self.algMetadata)
804  return {"src": sourceCat}
805 
806  def setMetadata(self, exposure, photoRes=None):
807  """!Set task and exposure metadata
808 
809  Logs a warning and continues if needed data is missing.
810 
811  @param[in,out] exposure exposure whose metadata is to be set
812  @param[in] photoRes results of running photoCal; if None then it was
813  not run
814  """
815  if photoRes is None:
816  return
817 
818  metadata = exposure.getMetadata()
819 
820  # convert zero-point to (mag/sec/adu) for task MAGZERO metadata
821  try:
822  exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
823  magZero = photoRes.zp - 2.5*math.log10(exposureTime)
824  except Exception:
825  self.log.warn("Could not set normalized MAGZERO in header: no "
826  "exposure time")
827  magZero = math.nan
828 
829  try:
830  metadata.set('MAGZERO', magZero)
831  metadata.set('MAGZERO_RMS', photoRes.sigma)
832  metadata.set('MAGZERO_NOBJ', photoRes.ngood)
833  metadata.set('COLORTERM1', 0.0)
834  metadata.set('COLORTERM2', 0.0)
835  metadata.set('COLORTERM3', 0.0)
836  except Exception as e:
837  self.log.warn("Could not set exposure metadata: %s" % (e,))
838 
839  def copyIcSourceFields(self, icSourceCat, sourceCat):
840  """!Match sources in icSourceCat and sourceCat and copy the specified fields
841 
842  @param[in] icSourceCat catalog from which to copy fields
843  @param[in,out] sourceCat catalog to which to copy fields
844 
845  The fields copied are those specified by `config.icSourceFieldsToCopy`
846  that actually exist in the schema. This was set up by the constructor
847  using self.schemaMapper.
848  """
849  if self.schemaMapper is None:
850  raise RuntimeError("To copy icSource fields you must specify "
851  "icSourceSchema nd icSourceKeys when "
852  "constructing this task")
853  if icSourceCat is None or sourceCat is None:
854  raise RuntimeError("icSourceCat and sourceCat must both be "
855  "specified")
856  if len(self.config.icSourceFieldsToCopy) == 0:
857  self.log.warn("copyIcSourceFields doing nothing because "
858  "icSourceFieldsToCopy is empty")
859  return
860 
861  mc = afwTable.MatchControl()
862  mc.findOnlyClosest = False # return all matched objects
863  matches = afwTable.matchXy(icSourceCat, sourceCat,
864  self.config.matchRadiusPix, mc)
865  if self.config.doDeblend:
866  deblendKey = sourceCat.schema["deblend_nChild"].asKey()
867  # if deblended, keep children
868  matches = [m for m in matches if m[1].get(deblendKey) == 0]
869 
870  # Because we had to allow multiple matches to handle parents, we now
871  # need to prune to the best matches
872  # closest matches as a dict of icSourceCat source ID:
873  # (icSourceCat source, sourceCat source, distance in pixels)
874  bestMatches = {}
875  for m0, m1, d in matches:
876  id0 = m0.getId()
877  match = bestMatches.get(id0)
878  if match is None or d <= match[2]:
879  bestMatches[id0] = (m0, m1, d)
880  matches = list(bestMatches.values())
881 
882  # Check that no sourceCat sources are listed twice (we already know
883  # that each match has a unique icSourceCat source ID, due to using
884  # that ID as the key in bestMatches)
885  numMatches = len(matches)
886  numUniqueSources = len(set(m[1].getId() for m in matches))
887  if numUniqueSources != numMatches:
888  self.log.warn("{} icSourceCat sources matched only {} sourceCat "
889  "sources".format(numMatches, numUniqueSources))
890 
891  self.log.info("Copying flags from icSourceCat to sourceCat for "
892  "%s sources" % (numMatches,))
893 
894  # For each match: set the calibSourceKey flag and copy the desired
895  # fields
896  for icSrc, src, d in matches:
897  src.setFlag(self.calibSourceKey, True)
898  # src.assign copies the footprint from icSrc, which we don't want
899  # (DM-407)
900  # so set icSrc's footprint to src's footprint before src.assign,
901  # then restore it
902  icSrcFootprint = icSrc.getFootprint()
903  try:
904  icSrc.setFootprint(src.getFootprint())
905  src.assign(icSrc, self.schemaMapper)
906  finally:
907  icSrc.setFootprint(icSrcFootprint)
def copyIcSourceFields(self, icSourceCat, sourceCat)
Match sources in icSourceCat and sourceCat and copy the specified fields.
Definition: calibrate.py:839
def __init__(self, butler=None, astromRefObjLoader=None, photoRefObjLoader=None, icSourceSchema=None, initInputs=None, kwargs)
Construct a CalibrateTask.
Definition: calibrate.py:402
def run(self, exposure, exposureIdInfo=None, background=None, icSourceCat=None)
Calibrate an exposure (science image or coadd)
Definition: calibrate.py:614
def writeOutputs(self, dataRef, exposure, background, sourceCat, astromMatches, matchMeta)
Definition: calibrate.py:775
def setMetadata(self, exposure, photoRes=None)
Set task and exposure metadata.
Definition: calibrate.py:806
def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None, doUnpersist=True)
Calibrate an exposure, optionally unpersisting inputs and persisting outputs.
Definition: calibrate.py:517
Calibrate an exposure: measure sources and perform astrometric and photometric calibration.
Definition: calibrate.py:303
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: calibrate.py:579