lsst.pipe.tasks  18.1.0-15-gc153667b+1
calibrate.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2016 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import math
23 
24 from lsstDebug import getDebugFrame
25 import lsst.pex.config as pexConfig
26 import lsst.pipe.base as pipeBase
28 import lsst.afw.table as afwTable
29 from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches
30 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
31 from lsst.obs.base import ExposureIdInfo
32 import lsst.daf.base as dafBase
33 from lsst.afw.math import BackgroundList
34 from lsst.afw.table import IdFactory, SourceTable
35 from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader
36 from lsst.meas.base import (SingleFrameMeasurementTask, ApplyApCorrTask,
37  CatalogCalculationTask)
38 from lsst.meas.deblender import SourceDeblendTask
39 from .fakes import BaseFakeSourcesTask
40 from .photoCal import PhotoCalTask
41 
42 __all__ = ["CalibrateConfig", "CalibrateTask"]
43 
44 
45 class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"),
46  defaultTemplates={}):
47 
48  icSourceSchema = cT.InitInput(
49  doc="Schema produced by characterize image task, used to initialize this task",
50  name="icSrc_schema",
51  storageClass="SourceCatalog",
52  multiple=True
53  )
54 
55  outputSchema = cT.InitOutput(
56  doc="Schema after CalibrateTask has been initialized",
57  name="src_schema",
58  storageClass="SourceCatalog",
59  multiple=True
60  )
61 
62  exposure = cT.Input(
63  doc="Input image to calibrate",
64  name="icExp",
65  storageClass="ExposureF",
66  dimensions=("instrument", "visit", "detector"),
67  )
68 
69  background = cT.Input(
70  doc="Backgrounds determined by characterize task",
71  name="icExpBackground",
72  storageClass="Background",
73  dimensions=("instrument", "visit", "detector"),
74  )
75 
76  icSourceCat = cT.Input(
77  doc="Source catalog created by characterize task",
78  name="icSrc",
79  storageClass="SourceCatalog",
80  dimensions=("instrument", "visit", "detector"),
81  )
82 
83  astromRefCat = cT.PrerequisiteInput(
84  doc="Reference catalog to use for astrometry",
85  name="ref_cat",
86  storageClass="SimpleCatalog",
87  dimensions=("skypix",),
88  deferLoad=True,
89  multiple=True,
90  )
91 
92  photoRefCat = cT.PrerequisiteInput(
93  doc="Reference catalog to use for photometric calibration",
94  name="ref_cat",
95  storageClass="SimpleCatalog",
96  dimensions=("skypix",),
97  deferLoad=True,
98  multiple=True
99  )
100 
101  outputExposure = cT.Output(
102  doc="Exposure after running calibration task",
103  name="calexp",
104  storageClass="ExposureF",
105  dimensions=("instrument", "visit", "detector"),
106  )
107 
108  outputCat = cT.Output(
109  doc="Source catalog produced in calibrate task",
110  name="src",
111  storageClass="SourceCatalog",
112  dimensions=("instrument", "visit", "detector"),
113  )
114 
115  outputBackground = cT.Output(
116  doc="Background models estimated in calibration task",
117  name="calexpBackground",
118  storageClass="Background",
119  dimensions=("instrument", "visit", "detector"),
120  )
121 
122  matches = cT.Output(
123  doc="Source/refObj matches from the astrometry solver",
124  name="srcMatch",
125  storageClass="Catalog",
126  dimensions=("instrument", "visit", "detector"),
127  )
128 
129  matchesDenormalized = cT.Output(
130  doc="Denormalized matches from astrometry solver",
131  name="srcMatchFull",
132  storageClass="Catalog",
133  dimensions=("instrument", "visit", "detector"),
134  )
135 
136  def __init__(self, *, config=None):
137  super().__init__(config=config)
138  if config.doWriteMatches is False:
139  self.outputs.remove("matches")
140  if config.doWriteMatchesDenormalized is False:
141  self.outputs.remove("matchesDenormalized")
142 
143 
144 class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections):
145  """Config for CalibrateTask"""
146  doWrite = pexConfig.Field(
147  dtype=bool,
148  default=True,
149  doc="Save calibration results?",
150  )
151  doWriteHeavyFootprintsInSources = pexConfig.Field(
152  dtype=bool,
153  default=True,
154  doc="Include HeavyFootprint data in source table? If false then heavy "
155  "footprints are saved as normal footprints, which saves some space"
156  )
157  doWriteMatches = pexConfig.Field(
158  dtype=bool,
159  default=True,
160  doc="Write reference matches (ignored if doWrite false)?",
161  )
162  doWriteMatchesDenormalized = pexConfig.Field(
163  dtype=bool,
164  default=False,
165  doc=("Write reference matches in denormalized format? "
166  "This format uses more disk space, but is more convenient to "
167  "read. Ignored if doWriteMatches=False or doWrite=False."),
168  )
169  doAstrometry = pexConfig.Field(
170  dtype=bool,
171  default=True,
172  doc="Perform astrometric calibration?",
173  )
174  astromRefObjLoader = pexConfig.ConfigurableField(
175  target=LoadIndexedReferenceObjectsTask,
176  doc="reference object loader for astrometric calibration",
177  )
178  photoRefObjLoader = pexConfig.ConfigurableField(
179  target=LoadIndexedReferenceObjectsTask,
180  doc="reference object loader for photometric calibration",
181  )
182  astrometry = pexConfig.ConfigurableField(
183  target=AstrometryTask,
184  doc="Perform astrometric calibration to refine the WCS",
185  )
186  requireAstrometry = pexConfig.Field(
187  dtype=bool,
188  default=True,
189  doc=("Raise an exception if astrometry fails? Ignored if doAstrometry "
190  "false."),
191  )
192  doPhotoCal = pexConfig.Field(
193  dtype=bool,
194  default=True,
195  doc="Perform phometric calibration?",
196  )
197  requirePhotoCal = pexConfig.Field(
198  dtype=bool,
199  default=True,
200  doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal "
201  "false."),
202  )
203  photoCal = pexConfig.ConfigurableField(
204  target=PhotoCalTask,
205  doc="Perform photometric calibration",
206  )
207  icSourceFieldsToCopy = pexConfig.ListField(
208  dtype=str,
209  default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
210  doc=("Fields to copy from the icSource catalog to the output catalog "
211  "for matching sources Any missing fields will trigger a "
212  "RuntimeError exception. Ignored if icSourceCat is not provided.")
213  )
214  matchRadiusPix = pexConfig.Field(
215  dtype=float,
216  default=3,
217  doc=("Match radius for matching icSourceCat objects to sourceCat "
218  "objects (pixels)"),
219  )
220  checkUnitsParseStrict = pexConfig.Field(
221  doc=("Strictness of Astropy unit compatibility check, can be 'raise', "
222  "'warn' or 'silent'"),
223  dtype=str,
224  default="raise",
225  )
226  detection = pexConfig.ConfigurableField(
227  target=SourceDetectionTask,
228  doc="Detect sources"
229  )
230  doDeblend = pexConfig.Field(
231  dtype=bool,
232  default=True,
233  doc="Run deblender input exposure"
234  )
235  deblend = pexConfig.ConfigurableField(
236  target=SourceDeblendTask,
237  doc="Split blended sources into their components"
238  )
239  measurement = pexConfig.ConfigurableField(
240  target=SingleFrameMeasurementTask,
241  doc="Measure sources"
242  )
243  doApCorr = pexConfig.Field(
244  dtype=bool,
245  default=True,
246  doc="Run subtask to apply aperture correction"
247  )
248  applyApCorr = pexConfig.ConfigurableField(
249  target=ApplyApCorrTask,
250  doc="Subtask to apply aperture corrections"
251  )
252  # If doApCorr is False, and the exposure does not have apcorrections
253  # already applied, the active plugins in catalogCalculation almost
254  # certainly should not contain the characterization plugin
255  catalogCalculation = pexConfig.ConfigurableField(
256  target=CatalogCalculationTask,
257  doc="Subtask to run catalogCalculation plugins on catalog"
258  )
259  doInsertFakes = pexConfig.Field(
260  dtype=bool,
261  default=False,
262  doc="Run fake sources injection task"
263  )
264  insertFakes = pexConfig.ConfigurableField(
265  target=BaseFakeSourcesTask,
266  doc="Injection of fake sources for testing purposes (must be "
267  "retargeted)"
268  )
269  doWriteExposure = pexConfig.Field(
270  dtype=bool,
271  default=True,
272  doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a "
273  "normal calexp but as a fakes_calexp."
274  )
275 
276 
277 
283 
284 class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
285  r"""!Calibrate an exposure: measure sources and perform astrometric and
286  photometric calibration
287 
288  @anchor CalibrateTask_
289 
290  @section pipe_tasks_calibrate_Contents Contents
291 
292  - @ref pipe_tasks_calibrate_Purpose
293  - @ref pipe_tasks_calibrate_Initialize
294  - @ref pipe_tasks_calibrate_IO
295  - @ref pipe_tasks_calibrate_Config
296  - @ref pipe_tasks_calibrate_Metadata
297  - @ref pipe_tasks_calibrate_Debug
298 
299 
300  @section pipe_tasks_calibrate_Purpose Description
301 
302  Given an exposure with a good PSF model and aperture correction map
303  (e.g. as provided by @ref CharacterizeImageTask), perform the following
304  operations:
305  - Run detection and measurement
306  - Run astrometry subtask to fit an improved WCS
307  - Run photoCal subtask to fit the exposure's photometric zero-point
308 
309  @section pipe_tasks_calibrate_Initialize Task initialisation
310 
311  @copydoc \_\_init\_\_
312 
313  @section pipe_tasks_calibrate_IO Invoking the Task
314 
315  If you want this task to unpersist inputs or persist outputs, then call
316  the `runDataRef` method (a wrapper around the `run` method).
317 
318  If you already have the inputs unpersisted and do not want to persist the
319  output then it is more direct to call the `run` method:
320 
321  @section pipe_tasks_calibrate_Config Configuration parameters
322 
323  See @ref CalibrateConfig
324 
325  @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata
326 
327  Exposure metadata
328  <dl>
329  <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task
330  <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal
331  task
332  <dt>COLORTERM1 <dd>?? (always 0.0)
333  <dt>COLORTERM2 <dd>?? (always 0.0)
334  <dt>COLORTERM3 <dd>?? (always 0.0)
335  </dl>
336 
337  @section pipe_tasks_calibrate_Debug Debug variables
338 
339  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink
340  interface supports a flag
341  `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug
342  for more about `debug.py`.
343 
344  CalibrateTask has a debug dictionary containing one key:
345  <dl>
346  <dt>calibrate
347  <dd>frame (an int; <= 0 to not display) in which to display the exposure,
348  sources and matches. See @ref lsst.meas.astrom.displayAstrometry for
349  the meaning of the various symbols.
350  </dl>
351 
352  For example, put something like:
353  @code{.py}
354  import lsstDebug
355  def DebugInfo(name):
356  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would
357  # call us recursively
358  if name == "lsst.pipe.tasks.calibrate":
359  di.display = dict(
360  calibrate = 1,
361  )
362 
363  return di
364 
365  lsstDebug.Info = DebugInfo
366  @endcode
367  into your `debug.py` file and run `calibrateTask.py` with the `--debug`
368  flag.
369 
370  Some subtasks may have their own debug variables; see individual Task
371  documentation.
372  """
373 
374  # Example description used to live here, removed 2-20-2017 as per
375  # https://jira.lsstcorp.org/browse/DM-9520
376 
377  ConfigClass = CalibrateConfig
378  _DefaultName = "calibrate"
379  RunnerClass = pipeBase.ButlerInitializedTaskRunner
380 
381  def __init__(self, butler=None, astromRefObjLoader=None,
382  photoRefObjLoader=None, icSourceSchema=None,
383  initInputs=None, **kwargs):
384  """!Construct a CalibrateTask
385 
386  @param[in] butler The butler is passed to the refObjLoader constructor
387  in case it is needed. Ignored if the refObjLoader argument
388  provides a loader directly.
389  @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks
390  that supplies an external reference catalog for astrometric
391  calibration. May be None if the desired loader can be constructed
392  from the butler argument or all steps requiring a reference catalog
393  are disabled.
394  @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks
395  that supplies an external reference catalog for photometric
396  calibration. May be None if the desired loader can be constructed
397  from the butler argument or all steps requiring a reference catalog
398  are disabled.
399  @param[in] icSourceSchema schema for icSource catalog, or None.
400  Schema values specified in config.icSourceFieldsToCopy will be
401  taken from this schema. If set to None, no values will be
402  propagated from the icSourceCatalog
403  @param[in,out] kwargs other keyword arguments for
404  lsst.pipe.base.CmdLineTask
405  """
406  super().__init__(**kwargs)
407 
408  if icSourceSchema is None and butler is not None:
409  # Use butler to read icSourceSchema from disk.
410  icSourceSchema = butler.get("icSrc_schema", immediate=True).schema
411 
412  if icSourceSchema is None and butler is None and initInputs is not None:
413  icSourceSchema = initInputs['icSourceSchema'].schema
414 
415  if icSourceSchema is not None:
416  # use a schema mapper to avoid copying each field separately
417  self.schemaMapper = afwTable.SchemaMapper(icSourceSchema)
418  minimumSchema = afwTable.SourceTable.makeMinimalSchema()
419  self.schemaMapper.addMinimalSchema(minimumSchema, False)
420 
421  # Add fields to copy from an icSource catalog
422  # and a field to indicate that the source matched a source in that
423  # catalog. If any fields are missing then raise an exception, but
424  # first find all missing fields in order to make the error message
425  # more useful.
426  self.calibSourceKey = self.schemaMapper.addOutputField(
427  afwTable.Field["Flag"]("calib_detected",
428  "Source was detected as an icSource"))
429  missingFieldNames = []
430  for fieldName in self.config.icSourceFieldsToCopy:
431  try:
432  schemaItem = icSourceSchema.find(fieldName)
433  except Exception:
434  missingFieldNames.append(fieldName)
435  else:
436  # field found; if addMapping fails then raise an exception
437  self.schemaMapper.addMapping(schemaItem.getKey())
438 
439  if missingFieldNames:
440  raise RuntimeError("isSourceCat is missing fields {} "
441  "specified in icSourceFieldsToCopy"
442  .format(missingFieldNames))
443 
444  # produce a temporary schema to pass to the subtasks; finalize it
445  # later
446  self.schema = self.schemaMapper.editOutputSchema()
447  else:
448  self.schemaMapper = None
449  self.schema = afwTable.SourceTable.makeMinimalSchema()
450  self.makeSubtask('detection', schema=self.schema)
451 
452  self.algMetadata = dafBase.PropertyList()
453 
454  # Only create a subtask for fakes if configuration option is set
455  # N.B. the config for fake object task must be retargeted to a child
456  # of BaseFakeSourcesTask
457  if self.config.doInsertFakes:
458  self.makeSubtask("insertFakes")
459 
460  if self.config.doDeblend:
461  self.makeSubtask("deblend", schema=self.schema)
462  self.makeSubtask('measurement', schema=self.schema,
463  algMetadata=self.algMetadata)
464  if self.config.doApCorr:
465  self.makeSubtask('applyApCorr', schema=self.schema)
466  self.makeSubtask('catalogCalculation', schema=self.schema)
467 
468  if self.config.doAstrometry:
469  if astromRefObjLoader is None and butler is not None:
470  self.makeSubtask('astromRefObjLoader', butler=butler)
471  astromRefObjLoader = self.astromRefObjLoader
472  self.pixelMargin = astromRefObjLoader.config.pixelMargin
473  self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader,
474  schema=self.schema)
475  if self.config.doPhotoCal:
476  if photoRefObjLoader is None and butler is not None:
477  self.makeSubtask('photoRefObjLoader', butler=butler)
478  photoRefObjLoader = self.photoRefObjLoader
479  self.pixelMargin = photoRefObjLoader.config.pixelMargin
480  self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader,
481  schema=self.schema)
482 
483  if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None):
484  raise RuntimeError("PipelineTask form of this task should not be initialized with "
485  "reference object loaders.")
486 
487  if self.schemaMapper is not None:
488  # finalize the schema
489  self.schema = self.schemaMapper.getOutputSchema()
490  self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
491 
492  sourceCatSchema = afwTable.SourceCatalog(self.schema)
493  sourceCatSchema.getTable().setMetadata(self.algMetadata)
494  self.outputSchema = sourceCatSchema
495 
496  @pipeBase.timeMethod
497  def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None,
498  doUnpersist=True):
499  """!Calibrate an exposure, optionally unpersisting inputs and
500  persisting outputs.
501 
502  This is a wrapper around the `run` method that unpersists inputs
503  (if `doUnpersist` true) and persists outputs (if `config.doWrite` true)
504 
505  @param[in] dataRef butler data reference corresponding to a science
506  image
507  @param[in,out] exposure characterized exposure (an
508  lsst.afw.image.ExposureF or similar), or None to unpersist existing
509  icExp and icBackground. See `run` method for details of what is
510  read and written.
511  @param[in,out] background initial model of background already
512  subtracted from exposure (an lsst.afw.math.BackgroundList). May be
513  None if no background has been subtracted, though that is unusual
514  for calibration. A refined background model is output. Ignored if
515  exposure is None.
516  @param[in] icSourceCat catalog from which to copy the fields specified
517  by icSourceKeys, or None;
518  @param[in] doUnpersist unpersist data:
519  - if True, exposure, background and icSourceCat are read from
520  dataRef and those three arguments must all be None;
521  - if False the exposure must be provided; background and
522  icSourceCat are optional. True is intended for running as a
523  command-line task, False for running as a subtask
524  @return same data as the calibrate method
525  """
526  self.log.info("Processing %s" % (dataRef.dataId))
527 
528  if doUnpersist:
529  if any(item is not None for item in (exposure, background,
530  icSourceCat)):
531  raise RuntimeError("doUnpersist true; exposure, background "
532  "and icSourceCat must all be None")
533  exposure = dataRef.get("icExp", immediate=True)
534  background = dataRef.get("icExpBackground", immediate=True)
535  icSourceCat = dataRef.get("icSrc", immediate=True)
536  elif exposure is None:
537  raise RuntimeError("doUnpersist false; exposure must be provided")
538 
539  exposureIdInfo = dataRef.get("expIdInfo")
540 
541  calRes = self.run(
542  exposure=exposure,
543  exposureIdInfo=exposureIdInfo,
544  background=background,
545  icSourceCat=icSourceCat,
546  )
547 
548  if self.config.doWrite:
549  self.writeOutputs(
550  dataRef=dataRef,
551  exposure=calRes.exposure,
552  background=calRes.background,
553  sourceCat=calRes.sourceCat,
554  astromMatches=calRes.astromMatches,
555  matchMeta=calRes.matchMeta,
556  )
557 
558  return calRes
559 
560  def runQuantum(self, butlerQC, inputRefs, outputRefs):
561  inputs = butlerQC.get(inputRefs)
562  expId, expBits = butlerQC.registry.packDataId("visit_detector",
563  butlerQC.quantum.dataId,
564  returnMaxBits=True)
565  inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
566 
567  if self.config.doAstrometry:
568  refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
569  for ref in inputRefs.astromRefCat],
570  refCats=inputs.pop('astromRefCat'),
571  config=self.config.astromRefObjLoader, log=self.log)
572  self.pixelMargin = refObjLoader.config.pixelMargin
573  self.astrometry.setRefObjLoader(refObjLoader)
574 
575  if self.config.doPhotoCal:
576  photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
577  for ref in inputRefs.photoRefCat],
578  refCats=inputs.pop('photoRefCat'),
579  config=self.config.photoRefObjLoader,
580  log=self.log)
581  self.pixelMargin = photoRefObjLoader.config.pixelMargin
582  self.photoCal.match.setRefObjLoader(photoRefObjLoader)
583 
584  outputs = self.run(**inputs)
585 
586  if self.config.doWriteMatches:
587  normalizedMatches = afwTable.packMatches(outputs.astromMatches)
588  normalizedMatches.table.setMetadata(outputs.matchMeta)
589  if self.config.doWriteMatchesDenormalized:
590  denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
591  outputs.matchesDenormalized = denormMatches
592  outputs.matches = normalizedMatches
593  butlerQC.put(outputs, outputRefs)
594 
595  def run(self, exposure, exposureIdInfo=None, background=None,
596  icSourceCat=None):
597  """!Calibrate an exposure (science image or coadd)
598 
599  @param[in,out] exposure exposure to calibrate (an
600  lsst.afw.image.ExposureF or similar);
601  in:
602  - MaskedImage
603  - Psf
604  out:
605  - MaskedImage has background subtracted
606  - Wcs is replaced
607  - PhotoCalib is replaced
608  @param[in] exposureIdInfo ID info for exposure (an
609  lsst.obs.base.ExposureIdInfo) If not provided, returned
610  SourceCatalog IDs will not be globally unique.
611  @param[in,out] background background model already subtracted from
612  exposure (an lsst.afw.math.BackgroundList). May be None if no
613  background has been subtracted, though that is unusual for
614  calibration. A refined background model is output.
615  @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
616  from which we can copy some fields.
617 
618  @return pipe_base Struct containing these fields:
619  - exposure calibrate science exposure with refined WCS and PhotoCalib
620  - background model of background subtracted from exposure (an
621  lsst.afw.math.BackgroundList)
622  - sourceCat catalog of measured sources
623  - astromMatches list of source/refObj matches from the astrometry
624  solver
625  """
626  # detect, deblend and measure sources
627  if exposureIdInfo is None:
628  exposureIdInfo = ExposureIdInfo()
629 
630  if background is None:
631  background = BackgroundList()
632  sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
633  exposureIdInfo.unusedBits)
634  table = SourceTable.make(self.schema, sourceIdFactory)
635  table.setMetadata(self.algMetadata)
636 
637  detRes = self.detection.run(table=table, exposure=exposure,
638  doSmooth=True)
639  sourceCat = detRes.sources
640  if detRes.fpSets.background:
641  for bg in detRes.fpSets.background:
642  background.append(bg)
643  if self.config.doDeblend:
644  self.deblend.run(exposure=exposure, sources=sourceCat)
645  self.measurement.run(
646  measCat=sourceCat,
647  exposure=exposure,
648  exposureId=exposureIdInfo.expId
649  )
650  if self.config.doApCorr:
651  self.applyApCorr.run(
652  catalog=sourceCat,
653  apCorrMap=exposure.getInfo().getApCorrMap()
654  )
655  self.catalogCalculation.run(sourceCat)
656 
657  if icSourceCat is not None and \
658  len(self.config.icSourceFieldsToCopy) > 0:
659  self.copyIcSourceFields(icSourceCat=icSourceCat,
660  sourceCat=sourceCat)
661 
662  # TODO DM-11568: this contiguous check-and-copy could go away if we
663  # reserve enough space during SourceDetection and/or SourceDeblend.
664  # NOTE: sourceSelectors require contiguous catalogs, so ensure
665  # contiguity now, so views are preserved from here on.
666  if not sourceCat.isContiguous():
667  sourceCat = sourceCat.copy(deep=True)
668 
669  # perform astrometry calibration:
670  # fit an improved WCS and update the exposure's WCS in place
671  astromMatches = None
672  matchMeta = None
673  if self.config.doAstrometry:
674  try:
675  astromRes = self.astrometry.run(
676  exposure=exposure,
677  sourceCat=sourceCat,
678  )
679  astromMatches = astromRes.matches
680  matchMeta = astromRes.matchMeta
681  except Exception as e:
682  if self.config.requireAstrometry:
683  raise
684  self.log.warn("Unable to perform astrometric calibration "
685  "(%s): attempting to proceed" % e)
686 
687  # compute photometric calibration
688  if self.config.doPhotoCal:
689  try:
690  photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
691  exposure.setPhotoCalib(photoRes.photoCalib)
692  # TODO: reword this to phrase it in terms of the calibration factor?
693  self.log.info("Photometric zero-point: %f" %
694  photoRes.photoCalib.instFluxToMagnitude(1.0))
695  self.setMetadata(exposure=exposure, photoRes=photoRes)
696  except Exception as e:
697  if self.config.requirePhotoCal:
698  raise
699  self.log.warn("Unable to perform photometric calibration "
700  "(%s): attempting to proceed" % e)
701  self.setMetadata(exposure=exposure, photoRes=None)
702 
703  if self.config.doInsertFakes:
704  self.insertFakes.run(exposure, background=background)
705 
706  table = SourceTable.make(self.schema, sourceIdFactory)
707  table.setMetadata(self.algMetadata)
708 
709  detRes = self.detection.run(table=table, exposure=exposure,
710  doSmooth=True)
711  sourceCat = detRes.sources
712  if detRes.fpSets.background:
713  for bg in detRes.fpSets.background:
714  background.append(bg)
715  if self.config.doDeblend:
716  self.deblend.run(exposure=exposure, sources=sourceCat)
717  self.measurement.run(
718  measCat=sourceCat,
719  exposure=exposure,
720  exposureId=exposureIdInfo.expId
721  )
722  if self.config.doApCorr:
723  self.applyApCorr.run(
724  catalog=sourceCat,
725  apCorrMap=exposure.getInfo().getApCorrMap()
726  )
727  self.catalogCalculation.run(sourceCat)
728 
729  if icSourceCat is not None and \
730  len(self.config.icSourceFieldsToCopy) > 0:
731  self.copyIcSourceFields(icSourceCat=icSourceCat,
732  sourceCat=sourceCat)
733 
734  frame = getDebugFrame(self._display, "calibrate")
735  if frame:
736  displayAstrometry(
737  sourceCat=sourceCat,
738  exposure=exposure,
739  matches=astromMatches,
740  frame=frame,
741  pause=False,
742  )
743 
744  return pipeBase.Struct(
745  exposure=exposure,
746  background=background,
747  sourceCat=sourceCat,
748  astromMatches=astromMatches,
749  matchMeta=matchMeta,
750  # These are duplicate entries with different names for use with
751  # gen3 middleware
752  outputExposure=exposure,
753  outputCat=sourceCat,
754  outputBackground=background,
755  )
756 
757  def writeOutputs(self, dataRef, exposure, background, sourceCat,
758  astromMatches, matchMeta):
759  """Write output data to the output repository
760 
761  @param[in] dataRef butler data reference corresponding to a science
762  image
763  @param[in] exposure exposure to write
764  @param[in] background background model for exposure
765  @param[in] sourceCat catalog of measured sources
766  @param[in] astromMatches list of source/refObj matches from the
767  astrometry solver
768  """
769  dataRef.put(sourceCat, "src")
770  if self.config.doWriteMatches and astromMatches is not None:
771  normalizedMatches = afwTable.packMatches(astromMatches)
772  normalizedMatches.table.setMetadata(matchMeta)
773  dataRef.put(normalizedMatches, "srcMatch")
774  if self.config.doWriteMatchesDenormalized:
775  denormMatches = denormalizeMatches(astromMatches, matchMeta)
776  dataRef.put(denormMatches, "srcMatchFull")
777  if self.config.doWriteExposure:
778  dataRef.put(exposure, "calexp")
779  dataRef.put(background, "calexpBackground")
780 
781  def getSchemaCatalogs(self):
782  """Return a dict of empty catalogs for each catalog dataset produced
783  by this task.
784  """
785  sourceCat = afwTable.SourceCatalog(self.schema)
786  sourceCat.getTable().setMetadata(self.algMetadata)
787  return {"src": sourceCat}
788 
789  def setMetadata(self, exposure, photoRes=None):
790  """!Set task and exposure metadata
791 
792  Logs a warning and continues if needed data is missing.
793 
794  @param[in,out] exposure exposure whose metadata is to be set
795  @param[in] photoRes results of running photoCal; if None then it was
796  not run
797  """
798  if photoRes is None:
799  return
800 
801  # convert zero-point to (mag/sec/adu) for task MAGZERO metadata
802  try:
803  exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
804  magZero = photoRes.zp - 2.5*math.log10(exposureTime)
805  self.metadata.set('MAGZERO', magZero)
806  except Exception:
807  self.log.warn("Could not set normalized MAGZERO in header: no "
808  "exposure time")
809 
810  try:
811  metadata = exposure.getMetadata()
812  metadata.set('MAGZERO_RMS', photoRes.sigma)
813  metadata.set('MAGZERO_NOBJ', photoRes.ngood)
814  metadata.set('COLORTERM1', 0.0)
815  metadata.set('COLORTERM2', 0.0)
816  metadata.set('COLORTERM3', 0.0)
817  except Exception as e:
818  self.log.warn("Could not set exposure metadata: %s" % (e,))
819 
820  def copyIcSourceFields(self, icSourceCat, sourceCat):
821  """!Match sources in icSourceCat and sourceCat and copy the specified fields
822 
823  @param[in] icSourceCat catalog from which to copy fields
824  @param[in,out] sourceCat catalog to which to copy fields
825 
826  The fields copied are those specified by `config.icSourceFieldsToCopy`
827  that actually exist in the schema. This was set up by the constructor
828  using self.schemaMapper.
829  """
830  if self.schemaMapper is None:
831  raise RuntimeError("To copy icSource fields you must specify "
832  "icSourceSchema nd icSourceKeys when "
833  "constructing this task")
834  if icSourceCat is None or sourceCat is None:
835  raise RuntimeError("icSourceCat and sourceCat must both be "
836  "specified")
837  if len(self.config.icSourceFieldsToCopy) == 0:
838  self.log.warn("copyIcSourceFields doing nothing because "
839  "icSourceFieldsToCopy is empty")
840  return
841 
842  mc = afwTable.MatchControl()
843  mc.findOnlyClosest = False # return all matched objects
844  matches = afwTable.matchXy(icSourceCat, sourceCat,
845  self.config.matchRadiusPix, mc)
846  if self.config.doDeblend:
847  deblendKey = sourceCat.schema["deblend_nChild"].asKey()
848  # if deblended, keep children
849  matches = [m for m in matches if m[1].get(deblendKey) == 0]
850 
851  # Because we had to allow multiple matches to handle parents, we now
852  # need to prune to the best matches
853  # closest matches as a dict of icSourceCat source ID:
854  # (icSourceCat source, sourceCat source, distance in pixels)
855  bestMatches = {}
856  for m0, m1, d in matches:
857  id0 = m0.getId()
858  match = bestMatches.get(id0)
859  if match is None or d <= match[2]:
860  bestMatches[id0] = (m0, m1, d)
861  matches = list(bestMatches.values())
862 
863  # Check that no sourceCat sources are listed twice (we already know
864  # that each match has a unique icSourceCat source ID, due to using
865  # that ID as the key in bestMatches)
866  numMatches = len(matches)
867  numUniqueSources = len(set(m[1].getId() for m in matches))
868  if numUniqueSources != numMatches:
869  self.log.warn("{} icSourceCat sources matched only {} sourceCat "
870  "sources".format(numMatches, numUniqueSources))
871 
872  self.log.info("Copying flags from icSourceCat to sourceCat for "
873  "%s sources" % (numMatches,))
874 
875  # For each match: set the calibSourceKey flag and copy the desired
876  # fields
877  for icSrc, src, d in matches:
878  src.setFlag(self.calibSourceKey, True)
879  # src.assign copies the footprint from icSrc, which we don't want
880  # (DM-407)
881  # so set icSrc's footprint to src's footprint before src.assign,
882  # then restore it
883  icSrcFootprint = icSrc.getFootprint()
884  try:
885  icSrc.setFootprint(src.getFootprint())
886  src.assign(icSrc, self.schemaMapper)
887  finally:
888  icSrc.setFootprint(icSrcFootprint)
def copyIcSourceFields(self, icSourceCat, sourceCat)
Match sources in icSourceCat and sourceCat and copy the specified fields.
Definition: calibrate.py:820
def __init__(self, butler=None, astromRefObjLoader=None, photoRefObjLoader=None, icSourceSchema=None, initInputs=None, kwargs)
Construct a CalibrateTask.
Definition: calibrate.py:383
def run(self, exposure, exposureIdInfo=None, background=None, icSourceCat=None)
Calibrate an exposure (science image or coadd)
Definition: calibrate.py:596
def writeOutputs(self, dataRef, exposure, background, sourceCat, astromMatches, matchMeta)
Definition: calibrate.py:758
def setMetadata(self, exposure, photoRes=None)
Set task and exposure metadata.
Definition: calibrate.py:789
def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None, doUnpersist=True)
Calibrate an exposure, optionally unpersisting inputs and persisting outputs.
Definition: calibrate.py:498
Calibrate an exposure: measure sources and perform astrometric and photometric calibration.
Definition: calibrate.py:284
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: calibrate.py:560