lsst.pipe.tasks  19.0.0-39-g09c7e8d9+4
calibrate.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2016 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import math
23 
24 from lsstDebug import getDebugFrame
25 import lsst.pex.config as pexConfig
26 import lsst.pipe.base as pipeBase
28 import lsst.afw.table as afwTable
29 from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches
30 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
31 from lsst.obs.base import ExposureIdInfo
32 import lsst.daf.base as dafBase
33 from lsst.afw.math import BackgroundList
34 from lsst.afw.table import IdFactory, SourceTable
35 from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader
36 from lsst.meas.base import (SingleFrameMeasurementTask,
37  ApplyApCorrTask,
38  CatalogCalculationTask)
39 from lsst.meas.deblender import SourceDeblendTask
40 from .fakes import BaseFakeSourcesTask
41 from .photoCal import PhotoCalTask
42 
43 __all__ = ["CalibrateConfig", "CalibrateTask"]
44 
45 
46 class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"),
47  defaultTemplates={}):
48 
49  icSourceSchema = cT.InitInput(
50  doc="Schema produced by characterize image task, used to initialize this task",
51  name="icSrc_schema",
52  storageClass="SourceCatalog",
53  )
54 
55  outputSchema = cT.InitOutput(
56  doc="Schema after CalibrateTask has been initialized",
57  name="src_schema",
58  storageClass="SourceCatalog",
59  )
60 
61  exposure = cT.Input(
62  doc="Input image to calibrate",
63  name="icExp",
64  storageClass="ExposureF",
65  dimensions=("instrument", "visit", "detector"),
66  )
67 
68  background = cT.Input(
69  doc="Backgrounds determined by characterize task",
70  name="icExpBackground",
71  storageClass="Background",
72  dimensions=("instrument", "visit", "detector"),
73  )
74 
75  icSourceCat = cT.Input(
76  doc="Source catalog created by characterize task",
77  name="icSrc",
78  storageClass="SourceCatalog",
79  dimensions=("instrument", "visit", "detector"),
80  )
81 
82  astromRefCat = cT.PrerequisiteInput(
83  doc="Reference catalog to use for astrometry",
84  name="cal_ref_cat",
85  storageClass="SimpleCatalog",
86  dimensions=("skypix",),
87  deferLoad=True,
88  multiple=True,
89  )
90 
91  photoRefCat = cT.PrerequisiteInput(
92  doc="Reference catalog to use for photometric calibration",
93  name="cal_ref_cat",
94  storageClass="SimpleCatalog",
95  dimensions=("skypix",),
96  deferLoad=True,
97  multiple=True
98  )
99 
100  outputExposure = cT.Output(
101  doc="Exposure after running calibration task",
102  name="calexp",
103  storageClass="ExposureF",
104  dimensions=("instrument", "visit", "detector"),
105  )
106 
107  outputCat = cT.Output(
108  doc="Source catalog produced in calibrate task",
109  name="src",
110  storageClass="SourceCatalog",
111  dimensions=("instrument", "visit", "detector"),
112  )
113 
114  outputBackground = cT.Output(
115  doc="Background models estimated in calibration task",
116  name="calexpBackground",
117  storageClass="Background",
118  dimensions=("instrument", "visit", "detector"),
119  )
120 
121  matches = cT.Output(
122  doc="Source/refObj matches from the astrometry solver",
123  name="srcMatch",
124  storageClass="Catalog",
125  dimensions=("instrument", "visit", "detector"),
126  )
127 
128  matchesDenormalized = cT.Output(
129  doc="Denormalized matches from astrometry solver",
130  name="srcMatchFull",
131  storageClass="Catalog",
132  dimensions=("instrument", "visit", "detector"),
133  )
134 
135  def __init__(self, *, config=None):
136  super().__init__(config=config)
137 
138  if config.doAstrometry is False:
139  self.prerequisiteInputs.remove("astromRefCat")
140  if config.doPhotoCal is False:
141  self.prerequisiteInputs.remove("photoRefCat")
142 
143  if config.doWriteMatches is False or config.doAstrometry is False:
144  self.outputs.remove("matches")
145  if config.doWriteMatchesDenormalized is False or config.doAstrometry is False:
146  self.outputs.remove("matchesDenormalized")
147 
148 
149 class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections):
150  """Config for CalibrateTask"""
151  doWrite = pexConfig.Field(
152  dtype=bool,
153  default=True,
154  doc="Save calibration results?",
155  )
156  doWriteHeavyFootprintsInSources = pexConfig.Field(
157  dtype=bool,
158  default=True,
159  doc="Include HeavyFootprint data in source table? If false then heavy "
160  "footprints are saved as normal footprints, which saves some space"
161  )
162  doWriteMatches = pexConfig.Field(
163  dtype=bool,
164  default=True,
165  doc="Write reference matches (ignored if doWrite or doAstrometry false)?",
166  )
167  doWriteMatchesDenormalized = pexConfig.Field(
168  dtype=bool,
169  default=False,
170  doc=("Write reference matches in denormalized format? "
171  "This format uses more disk space, but is more convenient to "
172  "read. Ignored if doWriteMatches=False or doWrite=False."),
173  )
174  doAstrometry = pexConfig.Field(
175  dtype=bool,
176  default=True,
177  doc="Perform astrometric calibration?",
178  )
179  astromRefObjLoader = pexConfig.ConfigurableField(
180  target=LoadIndexedReferenceObjectsTask,
181  doc="reference object loader for astrometric calibration",
182  )
183  photoRefObjLoader = pexConfig.ConfigurableField(
184  target=LoadIndexedReferenceObjectsTask,
185  doc="reference object loader for photometric calibration",
186  )
187  astrometry = pexConfig.ConfigurableField(
188  target=AstrometryTask,
189  doc="Perform astrometric calibration to refine the WCS",
190  )
191  requireAstrometry = pexConfig.Field(
192  dtype=bool,
193  default=True,
194  doc=("Raise an exception if astrometry fails? Ignored if doAstrometry "
195  "false."),
196  )
197  doPhotoCal = pexConfig.Field(
198  dtype=bool,
199  default=True,
200  doc="Perform phometric calibration?",
201  )
202  requirePhotoCal = pexConfig.Field(
203  dtype=bool,
204  default=True,
205  doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal "
206  "false."),
207  )
208  photoCal = pexConfig.ConfigurableField(
209  target=PhotoCalTask,
210  doc="Perform photometric calibration",
211  )
212  icSourceFieldsToCopy = pexConfig.ListField(
213  dtype=str,
214  default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
215  doc=("Fields to copy from the icSource catalog to the output catalog "
216  "for matching sources Any missing fields will trigger a "
217  "RuntimeError exception. Ignored if icSourceCat is not provided.")
218  )
219  matchRadiusPix = pexConfig.Field(
220  dtype=float,
221  default=3,
222  doc=("Match radius for matching icSourceCat objects to sourceCat "
223  "objects (pixels)"),
224  )
225  checkUnitsParseStrict = pexConfig.Field(
226  doc=("Strictness of Astropy unit compatibility check, can be 'raise', "
227  "'warn' or 'silent'"),
228  dtype=str,
229  default="raise",
230  )
231  detection = pexConfig.ConfigurableField(
232  target=SourceDetectionTask,
233  doc="Detect sources"
234  )
235  doDeblend = pexConfig.Field(
236  dtype=bool,
237  default=True,
238  doc="Run deblender input exposure"
239  )
240  deblend = pexConfig.ConfigurableField(
241  target=SourceDeblendTask,
242  doc="Split blended sources into their components"
243  )
244  measurement = pexConfig.ConfigurableField(
245  target=SingleFrameMeasurementTask,
246  doc="Measure sources"
247  )
248  doApCorr = pexConfig.Field(
249  dtype=bool,
250  default=True,
251  doc="Run subtask to apply aperture correction"
252  )
253  applyApCorr = pexConfig.ConfigurableField(
254  target=ApplyApCorrTask,
255  doc="Subtask to apply aperture corrections"
256  )
257  # If doApCorr is False, and the exposure does not have apcorrections
258  # already applied, the active plugins in catalogCalculation almost
259  # certainly should not contain the characterization plugin
260  catalogCalculation = pexConfig.ConfigurableField(
261  target=CatalogCalculationTask,
262  doc="Subtask to run catalogCalculation plugins on catalog"
263  )
264  doInsertFakes = pexConfig.Field(
265  dtype=bool,
266  default=False,
267  doc="Run fake sources injection task"
268  )
269  insertFakes = pexConfig.ConfigurableField(
270  target=BaseFakeSourcesTask,
271  doc="Injection of fake sources for testing purposes (must be "
272  "retargeted)"
273  )
274  doWriteExposure = pexConfig.Field(
275  dtype=bool,
276  default=True,
277  doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a "
278  "normal calexp but as a fakes_calexp."
279  )
280 
281  def setDefaults(self):
282  super().setDefaults()
283  self.detection.doTempLocalBackground = False
284  self.deblend.maxFootprintSize = 2000
285 
286  def validate(self):
287  super().validate()
288  astromRefCatGen2 = getattr(self.astromRefObjLoader, "ref_dataset_name", None)
289  if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat:
290  raise ValueError(
291  f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference "
292  f"catalogs are different. These options must be kept in sync until Gen2 is retired."
293  )
294  photoRefCatGen2 = getattr(self.photoRefObjLoader, "ref_dataset_name", None)
295  if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat:
296  raise ValueError(
297  f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference "
298  f"catalogs are different. These options must be kept in sync until Gen2 is retired."
299  )
300 
301 
302 
308 
309 class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
310  r"""!Calibrate an exposure: measure sources and perform astrometric and
311  photometric calibration
312 
313  @anchor CalibrateTask_
314 
315  @section pipe_tasks_calibrate_Contents Contents
316 
317  - @ref pipe_tasks_calibrate_Purpose
318  - @ref pipe_tasks_calibrate_Initialize
319  - @ref pipe_tasks_calibrate_IO
320  - @ref pipe_tasks_calibrate_Config
321  - @ref pipe_tasks_calibrate_Metadata
322  - @ref pipe_tasks_calibrate_Debug
323 
324 
325  @section pipe_tasks_calibrate_Purpose Description
326 
327  Given an exposure with a good PSF model and aperture correction map
328  (e.g. as provided by @ref CharacterizeImageTask), perform the following
329  operations:
330  - Run detection and measurement
331  - Run astrometry subtask to fit an improved WCS
332  - Run photoCal subtask to fit the exposure's photometric zero-point
333 
334  @section pipe_tasks_calibrate_Initialize Task initialisation
335 
336  @copydoc \_\_init\_\_
337 
338  @section pipe_tasks_calibrate_IO Invoking the Task
339 
340  If you want this task to unpersist inputs or persist outputs, then call
341  the `runDataRef` method (a wrapper around the `run` method).
342 
343  If you already have the inputs unpersisted and do not want to persist the
344  output then it is more direct to call the `run` method:
345 
346  @section pipe_tasks_calibrate_Config Configuration parameters
347 
348  See @ref CalibrateConfig
349 
350  @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata
351 
352  Exposure metadata
353  <dl>
354  <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task
355  <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal
356  task
357  <dt>COLORTERM1 <dd>?? (always 0.0)
358  <dt>COLORTERM2 <dd>?? (always 0.0)
359  <dt>COLORTERM3 <dd>?? (always 0.0)
360  </dl>
361 
362  @section pipe_tasks_calibrate_Debug Debug variables
363 
364  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink
365  interface supports a flag
366  `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug
367  for more about `debug.py`.
368 
369  CalibrateTask has a debug dictionary containing one key:
370  <dl>
371  <dt>calibrate
372  <dd>frame (an int; <= 0 to not display) in which to display the exposure,
373  sources and matches. See @ref lsst.meas.astrom.displayAstrometry for
374  the meaning of the various symbols.
375  </dl>
376 
377  For example, put something like:
378  @code{.py}
379  import lsstDebug
380  def DebugInfo(name):
381  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would
382  # call us recursively
383  if name == "lsst.pipe.tasks.calibrate":
384  di.display = dict(
385  calibrate = 1,
386  )
387 
388  return di
389 
390  lsstDebug.Info = DebugInfo
391  @endcode
392  into your `debug.py` file and run `calibrateTask.py` with the `--debug`
393  flag.
394 
395  Some subtasks may have their own debug variables; see individual Task
396  documentation.
397  """
398 
399  # Example description used to live here, removed 2-20-2017 as per
400  # https://jira.lsstcorp.org/browse/DM-9520
401 
402  ConfigClass = CalibrateConfig
403  _DefaultName = "calibrate"
404  RunnerClass = pipeBase.ButlerInitializedTaskRunner
405 
406  def __init__(self, butler=None, astromRefObjLoader=None,
407  photoRefObjLoader=None, icSourceSchema=None,
408  initInputs=None, **kwargs):
409  """!Construct a CalibrateTask
410 
411  @param[in] butler The butler is passed to the refObjLoader constructor
412  in case it is needed. Ignored if the refObjLoader argument
413  provides a loader directly.
414  @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks
415  that supplies an external reference catalog for astrometric
416  calibration. May be None if the desired loader can be constructed
417  from the butler argument or all steps requiring a reference catalog
418  are disabled.
419  @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks
420  that supplies an external reference catalog for photometric
421  calibration. May be None if the desired loader can be constructed
422  from the butler argument or all steps requiring a reference catalog
423  are disabled.
424  @param[in] icSourceSchema schema for icSource catalog, or None.
425  Schema values specified in config.icSourceFieldsToCopy will be
426  taken from this schema. If set to None, no values will be
427  propagated from the icSourceCatalog
428  @param[in,out] kwargs other keyword arguments for
429  lsst.pipe.base.CmdLineTask
430  """
431  super().__init__(**kwargs)
432 
433  if icSourceSchema is None and butler is not None:
434  # Use butler to read icSourceSchema from disk.
435  icSourceSchema = butler.get("icSrc_schema", immediate=True).schema
436 
437  if icSourceSchema is None and butler is None and initInputs is not None:
438  icSourceSchema = initInputs['icSourceSchema'].schema
439 
440  if icSourceSchema is not None:
441  # use a schema mapper to avoid copying each field separately
442  self.schemaMapper = afwTable.SchemaMapper(icSourceSchema)
443  minimumSchema = afwTable.SourceTable.makeMinimalSchema()
444  self.schemaMapper.addMinimalSchema(minimumSchema, False)
445 
446  # Add fields to copy from an icSource catalog
447  # and a field to indicate that the source matched a source in that
448  # catalog. If any fields are missing then raise an exception, but
449  # first find all missing fields in order to make the error message
450  # more useful.
451  self.calibSourceKey = self.schemaMapper.addOutputField(
452  afwTable.Field["Flag"]("calib_detected",
453  "Source was detected as an icSource"))
454  missingFieldNames = []
455  for fieldName in self.config.icSourceFieldsToCopy:
456  try:
457  schemaItem = icSourceSchema.find(fieldName)
458  except Exception:
459  missingFieldNames.append(fieldName)
460  else:
461  # field found; if addMapping fails then raise an exception
462  self.schemaMapper.addMapping(schemaItem.getKey())
463 
464  if missingFieldNames:
465  raise RuntimeError("isSourceCat is missing fields {} "
466  "specified in icSourceFieldsToCopy"
467  .format(missingFieldNames))
468 
469  # produce a temporary schema to pass to the subtasks; finalize it
470  # later
471  self.schema = self.schemaMapper.editOutputSchema()
472  else:
473  self.schemaMapper = None
474  self.schema = afwTable.SourceTable.makeMinimalSchema()
475  self.makeSubtask('detection', schema=self.schema)
476 
477  self.algMetadata = dafBase.PropertyList()
478 
479  # Only create a subtask for fakes if configuration option is set
480  # N.B. the config for fake object task must be retargeted to a child
481  # of BaseFakeSourcesTask
482  if self.config.doInsertFakes:
483  self.makeSubtask("insertFakes")
484 
485  if self.config.doDeblend:
486  self.makeSubtask("deblend", schema=self.schema)
487  self.makeSubtask('measurement', schema=self.schema,
488  algMetadata=self.algMetadata)
489  if self.config.doApCorr:
490  self.makeSubtask('applyApCorr', schema=self.schema)
491  self.makeSubtask('catalogCalculation', schema=self.schema)
492 
493  if self.config.doAstrometry:
494  if astromRefObjLoader is None and butler is not None:
495  self.makeSubtask('astromRefObjLoader', butler=butler)
496  astromRefObjLoader = self.astromRefObjLoader
497  self.pixelMargin = astromRefObjLoader.config.pixelMargin
498  self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader,
499  schema=self.schema)
500  if self.config.doPhotoCal:
501  if photoRefObjLoader is None and butler is not None:
502  self.makeSubtask('photoRefObjLoader', butler=butler)
503  photoRefObjLoader = self.photoRefObjLoader
504  self.pixelMargin = photoRefObjLoader.config.pixelMargin
505  self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader,
506  schema=self.schema)
507 
508  if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None):
509  raise RuntimeError("PipelineTask form of this task should not be initialized with "
510  "reference object loaders.")
511 
512  if self.schemaMapper is not None:
513  # finalize the schema
514  self.schema = self.schemaMapper.getOutputSchema()
515  self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
516 
517  sourceCatSchema = afwTable.SourceCatalog(self.schema)
518  sourceCatSchema.getTable().setMetadata(self.algMetadata)
519  self.outputSchema = sourceCatSchema
520 
521  @pipeBase.timeMethod
522  def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None,
523  doUnpersist=True):
524  """!Calibrate an exposure, optionally unpersisting inputs and
525  persisting outputs.
526 
527  This is a wrapper around the `run` method that unpersists inputs
528  (if `doUnpersist` true) and persists outputs (if `config.doWrite` true)
529 
530  @param[in] dataRef butler data reference corresponding to a science
531  image
532  @param[in,out] exposure characterized exposure (an
533  lsst.afw.image.ExposureF or similar), or None to unpersist existing
534  icExp and icBackground. See `run` method for details of what is
535  read and written.
536  @param[in,out] background initial model of background already
537  subtracted from exposure (an lsst.afw.math.BackgroundList). May be
538  None if no background has been subtracted, though that is unusual
539  for calibration. A refined background model is output. Ignored if
540  exposure is None.
541  @param[in] icSourceCat catalog from which to copy the fields specified
542  by icSourceKeys, or None;
543  @param[in] doUnpersist unpersist data:
544  - if True, exposure, background and icSourceCat are read from
545  dataRef and those three arguments must all be None;
546  - if False the exposure must be provided; background and
547  icSourceCat are optional. True is intended for running as a
548  command-line task, False for running as a subtask
549  @return same data as the calibrate method
550  """
551  self.log.info("Processing %s" % (dataRef.dataId))
552 
553  if doUnpersist:
554  if any(item is not None for item in (exposure, background,
555  icSourceCat)):
556  raise RuntimeError("doUnpersist true; exposure, background "
557  "and icSourceCat must all be None")
558  exposure = dataRef.get("icExp", immediate=True)
559  background = dataRef.get("icExpBackground", immediate=True)
560  icSourceCat = dataRef.get("icSrc", immediate=True)
561  elif exposure is None:
562  raise RuntimeError("doUnpersist false; exposure must be provided")
563 
564  exposureIdInfo = dataRef.get("expIdInfo")
565 
566  calRes = self.run(
567  exposure=exposure,
568  exposureIdInfo=exposureIdInfo,
569  background=background,
570  icSourceCat=icSourceCat,
571  )
572 
573  if self.config.doWrite:
574  self.writeOutputs(
575  dataRef=dataRef,
576  exposure=calRes.exposure,
577  background=calRes.background,
578  sourceCat=calRes.sourceCat,
579  astromMatches=calRes.astromMatches,
580  matchMeta=calRes.matchMeta,
581  )
582 
583  return calRes
584 
585  def runQuantum(self, butlerQC, inputRefs, outputRefs):
586  inputs = butlerQC.get(inputRefs)
587  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
588  returnMaxBits=True)
589  inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
590 
591  if self.config.doAstrometry:
592  refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
593  for ref in inputRefs.astromRefCat],
594  refCats=inputs.pop('astromRefCat'),
595  config=self.config.astromRefObjLoader, log=self.log)
596  self.pixelMargin = refObjLoader.config.pixelMargin
597  self.astrometry.setRefObjLoader(refObjLoader)
598 
599  if self.config.doPhotoCal:
600  photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
601  for ref in inputRefs.photoRefCat],
602  refCats=inputs.pop('photoRefCat'),
603  config=self.config.photoRefObjLoader,
604  log=self.log)
605  self.pixelMargin = photoRefObjLoader.config.pixelMargin
606  self.photoCal.match.setRefObjLoader(photoRefObjLoader)
607 
608  outputs = self.run(**inputs)
609 
610  if self.config.doWriteMatches and self.config.doAstrometry:
611  normalizedMatches = afwTable.packMatches(outputs.astromMatches)
612  normalizedMatches.table.setMetadata(outputs.matchMeta)
613  if self.config.doWriteMatchesDenormalized:
614  denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
615  outputs.matchesDenormalized = denormMatches
616  outputs.matches = normalizedMatches
617  butlerQC.put(outputs, outputRefs)
618 
619  def run(self, exposure, exposureIdInfo=None, background=None,
620  icSourceCat=None):
621  """!Calibrate an exposure (science image or coadd)
622 
623  @param[in,out] exposure exposure to calibrate (an
624  lsst.afw.image.ExposureF or similar);
625  in:
626  - MaskedImage
627  - Psf
628  out:
629  - MaskedImage has background subtracted
630  - Wcs is replaced
631  - PhotoCalib is replaced
632  @param[in] exposureIdInfo ID info for exposure (an
633  lsst.obs.base.ExposureIdInfo) If not provided, returned
634  SourceCatalog IDs will not be globally unique.
635  @param[in,out] background background model already subtracted from
636  exposure (an lsst.afw.math.BackgroundList). May be None if no
637  background has been subtracted, though that is unusual for
638  calibration. A refined background model is output.
639  @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
640  from which we can copy some fields.
641 
642  @return pipe_base Struct containing these fields:
643  - exposure calibrate science exposure with refined WCS and PhotoCalib
644  - background model of background subtracted from exposure (an
645  lsst.afw.math.BackgroundList)
646  - sourceCat catalog of measured sources
647  - astromMatches list of source/refObj matches from the astrometry
648  solver
649  """
650  # detect, deblend and measure sources
651  if exposureIdInfo is None:
652  exposureIdInfo = ExposureIdInfo()
653 
654  if background is None:
655  background = BackgroundList()
656  sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
657  exposureIdInfo.unusedBits)
658  table = SourceTable.make(self.schema, sourceIdFactory)
659  table.setMetadata(self.algMetadata)
660 
661  detRes = self.detection.run(table=table, exposure=exposure,
662  doSmooth=True)
663  sourceCat = detRes.sources
664  if detRes.fpSets.background:
665  for bg in detRes.fpSets.background:
666  background.append(bg)
667  if self.config.doDeblend:
668  self.deblend.run(exposure=exposure, sources=sourceCat)
669  self.measurement.run(
670  measCat=sourceCat,
671  exposure=exposure,
672  exposureId=exposureIdInfo.expId
673  )
674  if self.config.doApCorr:
675  self.applyApCorr.run(
676  catalog=sourceCat,
677  apCorrMap=exposure.getInfo().getApCorrMap()
678  )
679  self.catalogCalculation.run(sourceCat)
680 
681  if icSourceCat is not None and \
682  len(self.config.icSourceFieldsToCopy) > 0:
683  self.copyIcSourceFields(icSourceCat=icSourceCat,
684  sourceCat=sourceCat)
685 
686  # TODO DM-11568: this contiguous check-and-copy could go away if we
687  # reserve enough space during SourceDetection and/or SourceDeblend.
688  # NOTE: sourceSelectors require contiguous catalogs, so ensure
689  # contiguity now, so views are preserved from here on.
690  if not sourceCat.isContiguous():
691  sourceCat = sourceCat.copy(deep=True)
692 
693  # perform astrometry calibration:
694  # fit an improved WCS and update the exposure's WCS in place
695  astromMatches = None
696  matchMeta = None
697  if self.config.doAstrometry:
698  try:
699  astromRes = self.astrometry.run(
700  exposure=exposure,
701  sourceCat=sourceCat,
702  )
703  astromMatches = astromRes.matches
704  matchMeta = astromRes.matchMeta
705  except Exception as e:
706  if self.config.requireAstrometry:
707  raise
708  self.log.warn("Unable to perform astrometric calibration "
709  "(%s): attempting to proceed" % e)
710 
711  # compute photometric calibration
712  if self.config.doPhotoCal:
713  try:
714  photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
715  exposure.setPhotoCalib(photoRes.photoCalib)
716  # TODO: reword this to phrase it in terms of the calibration factor?
717  self.log.info("Photometric zero-point: %f" %
718  photoRes.photoCalib.instFluxToMagnitude(1.0))
719  self.setMetadata(exposure=exposure, photoRes=photoRes)
720  except Exception as e:
721  if self.config.requirePhotoCal:
722  raise
723  self.log.warn("Unable to perform photometric calibration "
724  "(%s): attempting to proceed" % e)
725  self.setMetadata(exposure=exposure, photoRes=None)
726 
727  if self.config.doInsertFakes:
728  self.insertFakes.run(exposure, background=background)
729 
730  table = SourceTable.make(self.schema, sourceIdFactory)
731  table.setMetadata(self.algMetadata)
732 
733  detRes = self.detection.run(table=table, exposure=exposure,
734  doSmooth=True)
735  sourceCat = detRes.sources
736  if detRes.fpSets.background:
737  for bg in detRes.fpSets.background:
738  background.append(bg)
739  if self.config.doDeblend:
740  self.deblend.run(exposure=exposure, sources=sourceCat)
741  self.measurement.run(
742  measCat=sourceCat,
743  exposure=exposure,
744  exposureId=exposureIdInfo.expId
745  )
746  if self.config.doApCorr:
747  self.applyApCorr.run(
748  catalog=sourceCat,
749  apCorrMap=exposure.getInfo().getApCorrMap()
750  )
751  self.catalogCalculation.run(sourceCat)
752 
753  if icSourceCat is not None and len(self.config.icSourceFieldsToCopy) > 0:
754  self.copyIcSourceFields(icSourceCat=icSourceCat,
755  sourceCat=sourceCat)
756 
757  frame = getDebugFrame(self._display, "calibrate")
758  if frame:
759  displayAstrometry(
760  sourceCat=sourceCat,
761  exposure=exposure,
762  matches=astromMatches,
763  frame=frame,
764  pause=False,
765  )
766 
767  return pipeBase.Struct(
768  exposure=exposure,
769  background=background,
770  sourceCat=sourceCat,
771  astromMatches=astromMatches,
772  matchMeta=matchMeta,
773  # These are duplicate entries with different names for use with
774  # gen3 middleware
775  outputExposure=exposure,
776  outputCat=sourceCat,
777  outputBackground=background,
778  )
779 
780  def writeOutputs(self, dataRef, exposure, background, sourceCat,
781  astromMatches, matchMeta):
782  """Write output data to the output repository
783 
784  @param[in] dataRef butler data reference corresponding to a science
785  image
786  @param[in] exposure exposure to write
787  @param[in] background background model for exposure
788  @param[in] sourceCat catalog of measured sources
789  @param[in] astromMatches list of source/refObj matches from the
790  astrometry solver
791  """
792  dataRef.put(sourceCat, "src")
793  if self.config.doWriteMatches and astromMatches is not None:
794  normalizedMatches = afwTable.packMatches(astromMatches)
795  normalizedMatches.table.setMetadata(matchMeta)
796  dataRef.put(normalizedMatches, "srcMatch")
797  if self.config.doWriteMatchesDenormalized:
798  denormMatches = denormalizeMatches(astromMatches, matchMeta)
799  dataRef.put(denormMatches, "srcMatchFull")
800  if self.config.doWriteExposure:
801  dataRef.put(exposure, "calexp")
802  dataRef.put(background, "calexpBackground")
803 
804  def getSchemaCatalogs(self):
805  """Return a dict of empty catalogs for each catalog dataset produced
806  by this task.
807  """
808  sourceCat = afwTable.SourceCatalog(self.schema)
809  sourceCat.getTable().setMetadata(self.algMetadata)
810  return {"src": sourceCat}
811 
812  def setMetadata(self, exposure, photoRes=None):
813  """!Set task and exposure metadata
814 
815  Logs a warning and continues if needed data is missing.
816 
817  @param[in,out] exposure exposure whose metadata is to be set
818  @param[in] photoRes results of running photoCal; if None then it was
819  not run
820  """
821  if photoRes is None:
822  return
823 
824  metadata = exposure.getMetadata()
825 
826  # convert zero-point to (mag/sec/adu) for task MAGZERO metadata
827  try:
828  exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
829  magZero = photoRes.zp - 2.5*math.log10(exposureTime)
830  except Exception:
831  self.log.warn("Could not set normalized MAGZERO in header: no "
832  "exposure time")
833  magZero = math.nan
834 
835  try:
836  metadata.set('MAGZERO', magZero)
837  metadata.set('MAGZERO_RMS', photoRes.sigma)
838  metadata.set('MAGZERO_NOBJ', photoRes.ngood)
839  metadata.set('COLORTERM1', 0.0)
840  metadata.set('COLORTERM2', 0.0)
841  metadata.set('COLORTERM3', 0.0)
842  except Exception as e:
843  self.log.warn("Could not set exposure metadata: %s" % (e,))
844 
845  def copyIcSourceFields(self, icSourceCat, sourceCat):
846  """!Match sources in icSourceCat and sourceCat and copy the specified fields
847 
848  @param[in] icSourceCat catalog from which to copy fields
849  @param[in,out] sourceCat catalog to which to copy fields
850 
851  The fields copied are those specified by `config.icSourceFieldsToCopy`
852  that actually exist in the schema. This was set up by the constructor
853  using self.schemaMapper.
854  """
855  if self.schemaMapper is None:
856  raise RuntimeError("To copy icSource fields you must specify "
857  "icSourceSchema nd icSourceKeys when "
858  "constructing this task")
859  if icSourceCat is None or sourceCat is None:
860  raise RuntimeError("icSourceCat and sourceCat must both be "
861  "specified")
862  if len(self.config.icSourceFieldsToCopy) == 0:
863  self.log.warn("copyIcSourceFields doing nothing because "
864  "icSourceFieldsToCopy is empty")
865  return
866 
867  mc = afwTable.MatchControl()
868  mc.findOnlyClosest = False # return all matched objects
869  matches = afwTable.matchXy(icSourceCat, sourceCat,
870  self.config.matchRadiusPix, mc)
871  if self.config.doDeblend:
872  deblendKey = sourceCat.schema["deblend_nChild"].asKey()
873  # if deblended, keep children
874  matches = [m for m in matches if m[1].get(deblendKey) == 0]
875 
876  # Because we had to allow multiple matches to handle parents, we now
877  # need to prune to the best matches
878  # closest matches as a dict of icSourceCat source ID:
879  # (icSourceCat source, sourceCat source, distance in pixels)
880  bestMatches = {}
881  for m0, m1, d in matches:
882  id0 = m0.getId()
883  match = bestMatches.get(id0)
884  if match is None or d <= match[2]:
885  bestMatches[id0] = (m0, m1, d)
886  matches = list(bestMatches.values())
887 
888  # Check that no sourceCat sources are listed twice (we already know
889  # that each match has a unique icSourceCat source ID, due to using
890  # that ID as the key in bestMatches)
891  numMatches = len(matches)
892  numUniqueSources = len(set(m[1].getId() for m in matches))
893  if numUniqueSources != numMatches:
894  self.log.warn("{} icSourceCat sources matched only {} sourceCat "
895  "sources".format(numMatches, numUniqueSources))
896 
897  self.log.info("Copying flags from icSourceCat to sourceCat for "
898  "%s sources" % (numMatches,))
899 
900  # For each match: set the calibSourceKey flag and copy the desired
901  # fields
902  for icSrc, src, d in matches:
903  src.setFlag(self.calibSourceKey, True)
904  # src.assign copies the footprint from icSrc, which we don't want
905  # (DM-407)
906  # so set icSrc's footprint to src's footprint before src.assign,
907  # then restore it
908  icSrcFootprint = icSrc.getFootprint()
909  try:
910  icSrc.setFootprint(src.getFootprint())
911  src.assign(icSrc, self.schemaMapper)
912  finally:
913  icSrc.setFootprint(icSrcFootprint)
def copyIcSourceFields(self, icSourceCat, sourceCat)
Match sources in icSourceCat and sourceCat and copy the specified fields.
Definition: calibrate.py:845
def __init__(self, butler=None, astromRefObjLoader=None, photoRefObjLoader=None, icSourceSchema=None, initInputs=None, kwargs)
Construct a CalibrateTask.
Definition: calibrate.py:408
def run(self, exposure, exposureIdInfo=None, background=None, icSourceCat=None)
Calibrate an exposure (science image or coadd)
Definition: calibrate.py:620
def writeOutputs(self, dataRef, exposure, background, sourceCat, astromMatches, matchMeta)
Definition: calibrate.py:781
def setMetadata(self, exposure, photoRes=None)
Set task and exposure metadata.
Definition: calibrate.py:812
def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None, doUnpersist=True)
Calibrate an exposure, optionally unpersisting inputs and persisting outputs.
Definition: calibrate.py:523
Calibrate an exposure: measure sources and perform astrometric and photometric calibration.
Definition: calibrate.py:309
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: calibrate.py:585