lsst.pipe.tasks  21.0.0-51-gd3b42663+3c7bb0c660
calibrate.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2016 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import math
23 
24 from lsstDebug import getDebugFrame
25 import lsst.pex.config as pexConfig
26 import lsst.pipe.base as pipeBase
28 import lsst.afw.table as afwTable
29 from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches
30 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, SkyObjectsTask
31 from lsst.obs.base import ExposureIdInfo
32 import lsst.daf.base as dafBase
33 from lsst.afw.math import BackgroundList
34 from lsst.afw.table import SourceTable
35 from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader
36 from lsst.meas.base import (SingleFrameMeasurementTask,
37  ApplyApCorrTask,
38  CatalogCalculationTask)
39 from lsst.meas.deblender import SourceDeblendTask
40 from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask
41 from .fakes import BaseFakeSourcesTask
42 from .photoCal import PhotoCalTask
43 
44 __all__ = ["CalibrateConfig", "CalibrateTask"]
45 
46 
47 class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"),
48  defaultTemplates={}):
49 
50  icSourceSchema = cT.InitInput(
51  doc="Schema produced by characterize image task, used to initialize this task",
52  name="icSrc_schema",
53  storageClass="SourceCatalog",
54  )
55 
56  outputSchema = cT.InitOutput(
57  doc="Schema after CalibrateTask has been initialized",
58  name="src_schema",
59  storageClass="SourceCatalog",
60  )
61 
62  exposure = cT.Input(
63  doc="Input image to calibrate",
64  name="icExp",
65  storageClass="ExposureF",
66  dimensions=("instrument", "visit", "detector"),
67  )
68 
69  background = cT.Input(
70  doc="Backgrounds determined by characterize task",
71  name="icExpBackground",
72  storageClass="Background",
73  dimensions=("instrument", "visit", "detector"),
74  )
75 
76  icSourceCat = cT.Input(
77  doc="Source catalog created by characterize task",
78  name="icSrc",
79  storageClass="SourceCatalog",
80  dimensions=("instrument", "visit", "detector"),
81  )
82 
83  astromRefCat = cT.PrerequisiteInput(
84  doc="Reference catalog to use for astrometry",
85  name="cal_ref_cat",
86  storageClass="SimpleCatalog",
87  dimensions=("skypix",),
88  deferLoad=True,
89  multiple=True,
90  )
91 
92  photoRefCat = cT.PrerequisiteInput(
93  doc="Reference catalog to use for photometric calibration",
94  name="cal_ref_cat",
95  storageClass="SimpleCatalog",
96  dimensions=("skypix",),
97  deferLoad=True,
98  multiple=True
99  )
100 
101  outputExposure = cT.Output(
102  doc="Exposure after running calibration task",
103  name="calexp",
104  storageClass="ExposureF",
105  dimensions=("instrument", "visit", "detector"),
106  )
107 
108  outputCat = cT.Output(
109  doc="Source catalog produced in calibrate task",
110  name="src",
111  storageClass="SourceCatalog",
112  dimensions=("instrument", "visit", "detector"),
113  )
114 
115  outputBackground = cT.Output(
116  doc="Background models estimated in calibration task",
117  name="calexpBackground",
118  storageClass="Background",
119  dimensions=("instrument", "visit", "detector"),
120  )
121 
122  matches = cT.Output(
123  doc="Source/refObj matches from the astrometry solver",
124  name="srcMatch",
125  storageClass="Catalog",
126  dimensions=("instrument", "visit", "detector"),
127  )
128 
129  matchesDenormalized = cT.Output(
130  doc="Denormalized matches from astrometry solver",
131  name="srcMatchFull",
132  storageClass="Catalog",
133  dimensions=("instrument", "visit", "detector"),
134  )
135 
136  def __init__(self, *, config=None):
137  super().__init__(config=config)
138 
139  if config.doAstrometry is False:
140  self.prerequisiteInputs.remove("astromRefCat")
141  if config.doPhotoCal is False:
142  self.prerequisiteInputs.remove("photoRefCat")
143 
144  if config.doWriteMatches is False or config.doAstrometry is False:
145  self.outputs.remove("matches")
146  if config.doWriteMatchesDenormalized is False or config.doAstrometry is False:
147  self.outputs.remove("matchesDenormalized")
148 
149 
150 class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections):
151  """Config for CalibrateTask"""
152  doWrite = pexConfig.Field(
153  dtype=bool,
154  default=True,
155  doc="Save calibration results?",
156  )
157  doWriteHeavyFootprintsInSources = pexConfig.Field(
158  dtype=bool,
159  default=True,
160  doc="Include HeavyFootprint data in source table? If false then heavy "
161  "footprints are saved as normal footprints, which saves some space"
162  )
163  doWriteMatches = pexConfig.Field(
164  dtype=bool,
165  default=True,
166  doc="Write reference matches (ignored if doWrite or doAstrometry false)?",
167  )
168  doWriteMatchesDenormalized = pexConfig.Field(
169  dtype=bool,
170  default=False,
171  doc=("Write reference matches in denormalized format? "
172  "This format uses more disk space, but is more convenient to "
173  "read. Ignored if doWriteMatches=False or doWrite=False."),
174  )
175  doAstrometry = pexConfig.Field(
176  dtype=bool,
177  default=True,
178  doc="Perform astrometric calibration?",
179  )
180  astromRefObjLoader = pexConfig.ConfigurableField(
181  target=LoadIndexedReferenceObjectsTask,
182  doc="reference object loader for astrometric calibration",
183  )
184  photoRefObjLoader = pexConfig.ConfigurableField(
185  target=LoadIndexedReferenceObjectsTask,
186  doc="reference object loader for photometric calibration",
187  )
188  astrometry = pexConfig.ConfigurableField(
189  target=AstrometryTask,
190  doc="Perform astrometric calibration to refine the WCS",
191  )
192  requireAstrometry = pexConfig.Field(
193  dtype=bool,
194  default=True,
195  doc=("Raise an exception if astrometry fails? Ignored if doAstrometry "
196  "false."),
197  )
198  doPhotoCal = pexConfig.Field(
199  dtype=bool,
200  default=True,
201  doc="Perform phometric calibration?",
202  )
203  requirePhotoCal = pexConfig.Field(
204  dtype=bool,
205  default=True,
206  doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal "
207  "false."),
208  )
209  photoCal = pexConfig.ConfigurableField(
210  target=PhotoCalTask,
211  doc="Perform photometric calibration",
212  )
213  icSourceFieldsToCopy = pexConfig.ListField(
214  dtype=str,
215  default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
216  doc=("Fields to copy from the icSource catalog to the output catalog "
217  "for matching sources Any missing fields will trigger a "
218  "RuntimeError exception. Ignored if icSourceCat is not provided.")
219  )
220  matchRadiusPix = pexConfig.Field(
221  dtype=float,
222  default=3,
223  doc=("Match radius for matching icSourceCat objects to sourceCat "
224  "objects (pixels)"),
225  )
226  checkUnitsParseStrict = pexConfig.Field(
227  doc=("Strictness of Astropy unit compatibility check, can be 'raise', "
228  "'warn' or 'silent'"),
229  dtype=str,
230  default="raise",
231  )
232  detection = pexConfig.ConfigurableField(
233  target=SourceDetectionTask,
234  doc="Detect sources"
235  )
236  doDeblend = pexConfig.Field(
237  dtype=bool,
238  default=True,
239  doc="Run deblender input exposure"
240  )
241  deblend = pexConfig.ConfigurableField(
242  target=SourceDeblendTask,
243  doc="Split blended sources into their components"
244  )
245  doSkySources = pexConfig.Field(
246  dtype=bool,
247  default=True,
248  doc="Generate sky sources?",
249  )
250  skySources = pexConfig.ConfigurableField(
251  target=SkyObjectsTask,
252  doc="Generate sky sources",
253  )
254  measurement = pexConfig.ConfigurableField(
255  target=SingleFrameMeasurementTask,
256  doc="Measure sources"
257  )
258  setPrimaryFlags = pexConfig.ConfigurableField(
259  target=SetPrimaryFlagsTask,
260  doc=("Set flags for primary source classification in single frame "
261  "processing. True if sources are not sky sources and not a parent.")
262  )
263  doApCorr = pexConfig.Field(
264  dtype=bool,
265  default=True,
266  doc="Run subtask to apply aperture correction"
267  )
268  applyApCorr = pexConfig.ConfigurableField(
269  target=ApplyApCorrTask,
270  doc="Subtask to apply aperture corrections"
271  )
272  # If doApCorr is False, and the exposure does not have apcorrections
273  # already applied, the active plugins in catalogCalculation almost
274  # certainly should not contain the characterization plugin
275  catalogCalculation = pexConfig.ConfigurableField(
276  target=CatalogCalculationTask,
277  doc="Subtask to run catalogCalculation plugins on catalog"
278  )
279  doInsertFakes = pexConfig.Field(
280  dtype=bool,
281  default=False,
282  doc="Run fake sources injection task"
283  )
284  insertFakes = pexConfig.ConfigurableField(
285  target=BaseFakeSourcesTask,
286  doc="Injection of fake sources for testing purposes (must be "
287  "retargeted)"
288  )
289  doWriteExposure = pexConfig.Field(
290  dtype=bool,
291  default=True,
292  doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a "
293  "normal calexp but as a fakes_calexp."
294  )
295 
296  def setDefaults(self):
297  super().setDefaults()
298  self.detectiondetection.doTempLocalBackground = False
299  self.deblenddeblend.maxFootprintSize = 2000
300  self.measurementmeasurement.plugins.names |= ["base_LocalPhotoCalib", "base_LocalWcs"]
301 
302  def validate(self):
303  super().validate()
304  astromRefCatGen2 = getattr(self.astromRefObjLoaderastromRefObjLoader, "ref_dataset_name", None)
305  if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat:
306  raise ValueError(
307  f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference "
308  f"catalogs are different. These options must be kept in sync until Gen2 is retired."
309  )
310  photoRefCatGen2 = getattr(self.photoRefObjLoaderphotoRefObjLoader, "ref_dataset_name", None)
311  if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat:
312  raise ValueError(
313  f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference "
314  f"catalogs are different. These options must be kept in sync until Gen2 is retired."
315  )
316 
317 
318 
324 
325 class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
326  r"""!Calibrate an exposure: measure sources and perform astrometric and
327  photometric calibration
328 
329  @anchor CalibrateTask_
330 
331  @section pipe_tasks_calibrate_Contents Contents
332 
333  - @ref pipe_tasks_calibrate_Purpose
334  - @ref pipe_tasks_calibrate_Initialize
335  - @ref pipe_tasks_calibrate_IO
336  - @ref pipe_tasks_calibrate_Config
337  - @ref pipe_tasks_calibrate_Metadata
338  - @ref pipe_tasks_calibrate_Debug
339 
340 
341  @section pipe_tasks_calibrate_Purpose Description
342 
343  Given an exposure with a good PSF model and aperture correction map
344  (e.g. as provided by @ref CharacterizeImageTask), perform the following
345  operations:
346  - Run detection and measurement
347  - Run astrometry subtask to fit an improved WCS
348  - Run photoCal subtask to fit the exposure's photometric zero-point
349 
350  @section pipe_tasks_calibrate_Initialize Task initialisation
351 
352  @copydoc \_\_init\_\_
353 
354  @section pipe_tasks_calibrate_IO Invoking the Task
355 
356  If you want this task to unpersist inputs or persist outputs, then call
357  the `runDataRef` method (a wrapper around the `run` method).
358 
359  If you already have the inputs unpersisted and do not want to persist the
360  output then it is more direct to call the `run` method:
361 
362  @section pipe_tasks_calibrate_Config Configuration parameters
363 
364  See @ref CalibrateConfig
365 
366  @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata
367 
368  Exposure metadata
369  <dl>
370  <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task
371  <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal
372  task
373  <dt>COLORTERM1 <dd>?? (always 0.0)
374  <dt>COLORTERM2 <dd>?? (always 0.0)
375  <dt>COLORTERM3 <dd>?? (always 0.0)
376  </dl>
377 
378  @section pipe_tasks_calibrate_Debug Debug variables
379 
380  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink
381  interface supports a flag
382  `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug
383  for more about `debug.py`.
384 
385  CalibrateTask has a debug dictionary containing one key:
386  <dl>
387  <dt>calibrate
388  <dd>frame (an int; <= 0 to not display) in which to display the exposure,
389  sources and matches. See @ref lsst.meas.astrom.displayAstrometry for
390  the meaning of the various symbols.
391  </dl>
392 
393  For example, put something like:
394  @code{.py}
395  import lsstDebug
396  def DebugInfo(name):
397  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would
398  # call us recursively
399  if name == "lsst.pipe.tasks.calibrate":
400  di.display = dict(
401  calibrate = 1,
402  )
403 
404  return di
405 
406  lsstDebug.Info = DebugInfo
407  @endcode
408  into your `debug.py` file and run `calibrateTask.py` with the `--debug`
409  flag.
410 
411  Some subtasks may have their own debug variables; see individual Task
412  documentation.
413  """
414 
415  # Example description used to live here, removed 2-20-2017 as per
416  # https://jira.lsstcorp.org/browse/DM-9520
417 
418  ConfigClass = CalibrateConfig
419  _DefaultName = "calibrate"
420  RunnerClass = pipeBase.ButlerInitializedTaskRunner
421 
422  def __init__(self, butler=None, astromRefObjLoader=None,
423  photoRefObjLoader=None, icSourceSchema=None,
424  initInputs=None, **kwargs):
425  """!Construct a CalibrateTask
426 
427  @param[in] butler The butler is passed to the refObjLoader constructor
428  in case it is needed. Ignored if the refObjLoader argument
429  provides a loader directly.
430  @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks
431  that supplies an external reference catalog for astrometric
432  calibration. May be None if the desired loader can be constructed
433  from the butler argument or all steps requiring a reference catalog
434  are disabled.
435  @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks
436  that supplies an external reference catalog for photometric
437  calibration. May be None if the desired loader can be constructed
438  from the butler argument or all steps requiring a reference catalog
439  are disabled.
440  @param[in] icSourceSchema schema for icSource catalog, or None.
441  Schema values specified in config.icSourceFieldsToCopy will be
442  taken from this schema. If set to None, no values will be
443  propagated from the icSourceCatalog
444  @param[in,out] kwargs other keyword arguments for
445  lsst.pipe.base.CmdLineTask
446  """
447  super().__init__(**kwargs)
448 
449  if icSourceSchema is None and butler is not None:
450  # Use butler to read icSourceSchema from disk.
451  icSourceSchema = butler.get("icSrc_schema", immediate=True).schema
452 
453  if icSourceSchema is None and butler is None and initInputs is not None:
454  icSourceSchema = initInputs['icSourceSchema'].schema
455 
456  if icSourceSchema is not None:
457  # use a schema mapper to avoid copying each field separately
458  self.schemaMapperschemaMapper = afwTable.SchemaMapper(icSourceSchema)
459  minimumSchema = afwTable.SourceTable.makeMinimalSchema()
460  self.schemaMapperschemaMapper.addMinimalSchema(minimumSchema, False)
461 
462  # Add fields to copy from an icSource catalog
463  # and a field to indicate that the source matched a source in that
464  # catalog. If any fields are missing then raise an exception, but
465  # first find all missing fields in order to make the error message
466  # more useful.
467  self.calibSourceKeycalibSourceKey = self.schemaMapperschemaMapper.addOutputField(
468  afwTable.Field["Flag"]("calib_detected",
469  "Source was detected as an icSource"))
470  missingFieldNames = []
471  for fieldName in self.config.icSourceFieldsToCopy:
472  try:
473  schemaItem = icSourceSchema.find(fieldName)
474  except Exception:
475  missingFieldNames.append(fieldName)
476  else:
477  # field found; if addMapping fails then raise an exception
478  self.schemaMapperschemaMapper.addMapping(schemaItem.getKey())
479 
480  if missingFieldNames:
481  raise RuntimeError("isSourceCat is missing fields {} "
482  "specified in icSourceFieldsToCopy"
483  .format(missingFieldNames))
484 
485  # produce a temporary schema to pass to the subtasks; finalize it
486  # later
487  self.schemaschema = self.schemaMapperschemaMapper.editOutputSchema()
488  else:
489  self.schemaMapperschemaMapper = None
490  self.schemaschema = afwTable.SourceTable.makeMinimalSchema()
491  self.makeSubtask('detection', schema=self.schemaschema)
492 
493  self.algMetadataalgMetadata = dafBase.PropertyList()
494 
495  # Only create a subtask for fakes if configuration option is set
496  # N.B. the config for fake object task must be retargeted to a child
497  # of BaseFakeSourcesTask
498  if self.config.doInsertFakes:
499  self.makeSubtask("insertFakes")
500 
501  if self.config.doDeblend:
502  self.makeSubtask("deblend", schema=self.schemaschema)
503  if self.config.doSkySources:
504  self.makeSubtask("skySources")
505  self.skySourceKeyskySourceKey = self.schemaschema.addField("sky_source", type="Flag", doc="Sky objects.")
506  self.makeSubtask('measurement', schema=self.schemaschema,
507  algMetadata=self.algMetadataalgMetadata)
508  self.makeSubtask("setPrimaryFlags", schema=self.schemaschema, isSingleFrame=True)
509  if self.config.doApCorr:
510  self.makeSubtask('applyApCorr', schema=self.schemaschema)
511  self.makeSubtask('catalogCalculation', schema=self.schemaschema)
512 
513  if self.config.doAstrometry:
514  if astromRefObjLoader is None and butler is not None:
515  self.makeSubtask('astromRefObjLoader', butler=butler)
516  astromRefObjLoader = self.astromRefObjLoader
517  self.pixelMarginpixelMargin = astromRefObjLoader.config.pixelMargin
518  self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader,
519  schema=self.schemaschema)
520  if self.config.doPhotoCal:
521  if photoRefObjLoader is None and butler is not None:
522  self.makeSubtask('photoRefObjLoader', butler=butler)
523  photoRefObjLoader = self.photoRefObjLoader
524  self.pixelMarginpixelMargin = photoRefObjLoader.config.pixelMargin
525  self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader,
526  schema=self.schemaschema)
527 
528  if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None):
529  raise RuntimeError("PipelineTask form of this task should not be initialized with "
530  "reference object loaders.")
531 
532  if self.schemaMapperschemaMapper is not None:
533  # finalize the schema
534  self.schemaschema = self.schemaMapperschemaMapper.getOutputSchema()
535  self.schemaschema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
536 
537  sourceCatSchema = afwTable.SourceCatalog(self.schemaschema)
538  sourceCatSchema.getTable().setMetadata(self.algMetadataalgMetadata)
539  self.outputSchemaoutputSchema = sourceCatSchema
540 
541  @pipeBase.timeMethod
542  def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None,
543  doUnpersist=True):
544  """!Calibrate an exposure, optionally unpersisting inputs and
545  persisting outputs.
546 
547  This is a wrapper around the `run` method that unpersists inputs
548  (if `doUnpersist` true) and persists outputs (if `config.doWrite` true)
549 
550  @param[in] dataRef butler data reference corresponding to a science
551  image
552  @param[in,out] exposure characterized exposure (an
553  lsst.afw.image.ExposureF or similar), or None to unpersist existing
554  icExp and icBackground. See `run` method for details of what is
555  read and written.
556  @param[in,out] background initial model of background already
557  subtracted from exposure (an lsst.afw.math.BackgroundList). May be
558  None if no background has been subtracted, though that is unusual
559  for calibration. A refined background model is output. Ignored if
560  exposure is None.
561  @param[in] icSourceCat catalog from which to copy the fields specified
562  by icSourceKeys, or None;
563  @param[in] doUnpersist unpersist data:
564  - if True, exposure, background and icSourceCat are read from
565  dataRef and those three arguments must all be None;
566  - if False the exposure must be provided; background and
567  icSourceCat are optional. True is intended for running as a
568  command-line task, False for running as a subtask
569  @return same data as the calibrate method
570  """
571  self.log.info("Processing %s" % (dataRef.dataId))
572 
573  if doUnpersist:
574  if any(item is not None for item in (exposure, background,
575  icSourceCat)):
576  raise RuntimeError("doUnpersist true; exposure, background "
577  "and icSourceCat must all be None")
578  exposure = dataRef.get("icExp", immediate=True)
579  background = dataRef.get("icExpBackground", immediate=True)
580  icSourceCat = dataRef.get("icSrc", immediate=True)
581  elif exposure is None:
582  raise RuntimeError("doUnpersist false; exposure must be provided")
583 
584  exposureIdInfo = dataRef.get("expIdInfo")
585 
586  calRes = self.runrun(
587  exposure=exposure,
588  exposureIdInfo=exposureIdInfo,
589  background=background,
590  icSourceCat=icSourceCat,
591  )
592 
593  if self.config.doWrite:
594  self.writeOutputswriteOutputs(
595  dataRef=dataRef,
596  exposure=calRes.exposure,
597  background=calRes.background,
598  sourceCat=calRes.sourceCat,
599  astromMatches=calRes.astromMatches,
600  matchMeta=calRes.matchMeta,
601  )
602 
603  return calRes
604 
605  def runQuantum(self, butlerQC, inputRefs, outputRefs):
606  inputs = butlerQC.get(inputRefs)
607  inputs['exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "visit_detector")
608 
609  if self.config.doAstrometry:
610  refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
611  for ref in inputRefs.astromRefCat],
612  refCats=inputs.pop('astromRefCat'),
613  config=self.config.astromRefObjLoader, log=self.log)
614  self.pixelMarginpixelMargin = refObjLoader.config.pixelMargin
615  self.astrometry.setRefObjLoader(refObjLoader)
616 
617  if self.config.doPhotoCal:
618  photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
619  for ref in inputRefs.photoRefCat],
620  refCats=inputs.pop('photoRefCat'),
621  config=self.config.photoRefObjLoader,
622  log=self.log)
623  self.pixelMarginpixelMargin = photoRefObjLoader.config.pixelMargin
624  self.photoCal.match.setRefObjLoader(photoRefObjLoader)
625 
626  outputs = self.runrun(**inputs)
627 
628  if self.config.doWriteMatches and self.config.doAstrometry:
629  normalizedMatches = afwTable.packMatches(outputs.astromMatches)
630  normalizedMatches.table.setMetadata(outputs.matchMeta)
631  if self.config.doWriteMatchesDenormalized:
632  denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
633  outputs.matchesDenormalized = denormMatches
634  outputs.matches = normalizedMatches
635  butlerQC.put(outputs, outputRefs)
636 
637  @pipeBase.timeMethod
638  def run(self, exposure, exposureIdInfo=None, background=None,
639  icSourceCat=None):
640  """!Calibrate an exposure (science image or coadd)
641 
642  @param[in,out] exposure exposure to calibrate (an
643  lsst.afw.image.ExposureF or similar);
644  in:
645  - MaskedImage
646  - Psf
647  out:
648  - MaskedImage has background subtracted
649  - Wcs is replaced
650  - PhotoCalib is replaced
651  @param[in] exposureIdInfo ID info for exposure (an
652  lsst.obs.base.ExposureIdInfo) If not provided, returned
653  SourceCatalog IDs will not be globally unique.
654  @param[in,out] background background model already subtracted from
655  exposure (an lsst.afw.math.BackgroundList). May be None if no
656  background has been subtracted, though that is unusual for
657  calibration. A refined background model is output.
658  @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
659  from which we can copy some fields.
660 
661  @return pipe_base Struct containing these fields:
662  - exposure calibrate science exposure with refined WCS and PhotoCalib
663  - background model of background subtracted from exposure (an
664  lsst.afw.math.BackgroundList)
665  - sourceCat catalog of measured sources
666  - astromMatches list of source/refObj matches from the astrometry
667  solver
668  """
669  # detect, deblend and measure sources
670  if exposureIdInfo is None:
671  exposureIdInfo = ExposureIdInfo()
672 
673  if background is None:
674  background = BackgroundList()
675  sourceIdFactory = exposureIdInfo.makeSourceIdFactory()
676  table = SourceTable.make(self.schemaschema, sourceIdFactory)
677  table.setMetadata(self.algMetadataalgMetadata)
678 
679  detRes = self.detection.run(table=table, exposure=exposure,
680  doSmooth=True)
681  sourceCat = detRes.sources
682  if detRes.fpSets.background:
683  for bg in detRes.fpSets.background:
684  background.append(bg)
685  if self.config.doSkySources:
686  skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId)
687  if skySourceFootprints:
688  for foot in skySourceFootprints:
689  s = sourceCat.addNew()
690  s.setFootprint(foot)
691  s.set(self.skySourceKeyskySourceKey, True)
692  if self.config.doDeblend:
693  self.deblend.run(exposure=exposure, sources=sourceCat)
694  self.measurement.run(
695  measCat=sourceCat,
696  exposure=exposure,
697  exposureId=exposureIdInfo.expId
698  )
699  if self.config.doApCorr:
700  self.applyApCorr.run(
701  catalog=sourceCat,
702  apCorrMap=exposure.getInfo().getApCorrMap()
703  )
704  self.catalogCalculation.run(sourceCat)
705 
706  self.setPrimaryFlags.run(sourceCat, includeDeblend=self.config.doDeblend)
707 
708  if icSourceCat is not None and \
709  len(self.config.icSourceFieldsToCopy) > 0:
710  self.copyIcSourceFieldscopyIcSourceFields(icSourceCat=icSourceCat,
711  sourceCat=sourceCat)
712 
713  # TODO DM-11568: this contiguous check-and-copy could go away if we
714  # reserve enough space during SourceDetection and/or SourceDeblend.
715  # NOTE: sourceSelectors require contiguous catalogs, so ensure
716  # contiguity now, so views are preserved from here on.
717  if not sourceCat.isContiguous():
718  sourceCat = sourceCat.copy(deep=True)
719 
720  # perform astrometry calibration:
721  # fit an improved WCS and update the exposure's WCS in place
722  astromMatches = None
723  matchMeta = None
724  if self.config.doAstrometry:
725  try:
726  astromRes = self.astrometry.run(
727  exposure=exposure,
728  sourceCat=sourceCat,
729  )
730  astromMatches = astromRes.matches
731  matchMeta = astromRes.matchMeta
732  except Exception as e:
733  if self.config.requireAstrometry:
734  raise
735  self.log.warn("Unable to perform astrometric calibration "
736  "(%s): attempting to proceed" % e)
737 
738  # compute photometric calibration
739  if self.config.doPhotoCal:
740  try:
741  photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
742  exposure.setPhotoCalib(photoRes.photoCalib)
743  # TODO: reword this to phrase it in terms of the calibration factor?
744  self.log.info("Photometric zero-point: %f" %
745  photoRes.photoCalib.instFluxToMagnitude(1.0))
746  self.setMetadatasetMetadata(exposure=exposure, photoRes=photoRes)
747  except Exception as e:
748  if self.config.requirePhotoCal:
749  raise
750  self.log.warn("Unable to perform photometric calibration "
751  "(%s): attempting to proceed" % e)
752  self.setMetadatasetMetadata(exposure=exposure, photoRes=None)
753 
754  if self.config.doInsertFakes:
755  self.insertFakes.run(exposure, background=background)
756 
757  table = SourceTable.make(self.schemaschema, sourceIdFactory)
758  table.setMetadata(self.algMetadataalgMetadata)
759 
760  detRes = self.detection.run(table=table, exposure=exposure,
761  doSmooth=True)
762  sourceCat = detRes.sources
763  if detRes.fpSets.background:
764  for bg in detRes.fpSets.background:
765  background.append(bg)
766  if self.config.doDeblend:
767  self.deblend.run(exposure=exposure, sources=sourceCat)
768  self.measurement.run(
769  measCat=sourceCat,
770  exposure=exposure,
771  exposureId=exposureIdInfo.expId
772  )
773  if self.config.doApCorr:
774  self.applyApCorr.run(
775  catalog=sourceCat,
776  apCorrMap=exposure.getInfo().getApCorrMap()
777  )
778  self.catalogCalculation.run(sourceCat)
779 
780  if icSourceCat is not None and len(self.config.icSourceFieldsToCopy) > 0:
781  self.copyIcSourceFieldscopyIcSourceFields(icSourceCat=icSourceCat,
782  sourceCat=sourceCat)
783 
784  frame = getDebugFrame(self._display, "calibrate")
785  if frame:
786  displayAstrometry(
787  sourceCat=sourceCat,
788  exposure=exposure,
789  matches=astromMatches,
790  frame=frame,
791  pause=False,
792  )
793 
794  return pipeBase.Struct(
795  exposure=exposure,
796  background=background,
797  sourceCat=sourceCat,
798  astromMatches=astromMatches,
799  matchMeta=matchMeta,
800  # These are duplicate entries with different names for use with
801  # gen3 middleware
802  outputExposure=exposure,
803  outputCat=sourceCat,
804  outputBackground=background,
805  )
806 
807  def writeOutputs(self, dataRef, exposure, background, sourceCat,
808  astromMatches, matchMeta):
809  """Write output data to the output repository
810 
811  @param[in] dataRef butler data reference corresponding to a science
812  image
813  @param[in] exposure exposure to write
814  @param[in] background background model for exposure
815  @param[in] sourceCat catalog of measured sources
816  @param[in] astromMatches list of source/refObj matches from the
817  astrometry solver
818  """
819  dataRef.put(sourceCat, "src")
820  if self.config.doWriteMatches and astromMatches is not None:
821  normalizedMatches = afwTable.packMatches(astromMatches)
822  normalizedMatches.table.setMetadata(matchMeta)
823  dataRef.put(normalizedMatches, "srcMatch")
824  if self.config.doWriteMatchesDenormalized:
825  denormMatches = denormalizeMatches(astromMatches, matchMeta)
826  dataRef.put(denormMatches, "srcMatchFull")
827  if self.config.doWriteExposure:
828  dataRef.put(exposure, "calexp")
829  dataRef.put(background, "calexpBackground")
830 
831  def getSchemaCatalogs(self):
832  """Return a dict of empty catalogs for each catalog dataset produced
833  by this task.
834  """
835  sourceCat = afwTable.SourceCatalog(self.schemaschema)
836  sourceCat.getTable().setMetadata(self.algMetadataalgMetadata)
837  return {"src": sourceCat}
838 
839  def setMetadata(self, exposure, photoRes=None):
840  """!Set task and exposure metadata
841 
842  Logs a warning and continues if needed data is missing.
843 
844  @param[in,out] exposure exposure whose metadata is to be set
845  @param[in] photoRes results of running photoCal; if None then it was
846  not run
847  """
848  if photoRes is None:
849  return
850 
851  metadata = exposure.getMetadata()
852 
853  # convert zero-point to (mag/sec/adu) for task MAGZERO metadata
854  try:
855  exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
856  magZero = photoRes.zp - 2.5*math.log10(exposureTime)
857  except Exception:
858  self.log.warn("Could not set normalized MAGZERO in header: no "
859  "exposure time")
860  magZero = math.nan
861 
862  try:
863  metadata.set('MAGZERO', magZero)
864  metadata.set('MAGZERO_RMS', photoRes.sigma)
865  metadata.set('MAGZERO_NOBJ', photoRes.ngood)
866  metadata.set('COLORTERM1', 0.0)
867  metadata.set('COLORTERM2', 0.0)
868  metadata.set('COLORTERM3', 0.0)
869  except Exception as e:
870  self.log.warn("Could not set exposure metadata: %s" % (e,))
871 
872  def copyIcSourceFields(self, icSourceCat, sourceCat):
873  """!Match sources in icSourceCat and sourceCat and copy the specified fields
874 
875  @param[in] icSourceCat catalog from which to copy fields
876  @param[in,out] sourceCat catalog to which to copy fields
877 
878  The fields copied are those specified by `config.icSourceFieldsToCopy`
879  that actually exist in the schema. This was set up by the constructor
880  using self.schemaMapper.
881  """
882  if self.schemaMapperschemaMapper is None:
883  raise RuntimeError("To copy icSource fields you must specify "
884  "icSourceSchema nd icSourceKeys when "
885  "constructing this task")
886  if icSourceCat is None or sourceCat is None:
887  raise RuntimeError("icSourceCat and sourceCat must both be "
888  "specified")
889  if len(self.config.icSourceFieldsToCopy) == 0:
890  self.log.warn("copyIcSourceFields doing nothing because "
891  "icSourceFieldsToCopy is empty")
892  return
893 
894  mc = afwTable.MatchControl()
895  mc.findOnlyClosest = False # return all matched objects
896  matches = afwTable.matchXy(icSourceCat, sourceCat,
897  self.config.matchRadiusPix, mc)
898  if self.config.doDeblend:
899  deblendKey = sourceCat.schema["deblend_nChild"].asKey()
900  # if deblended, keep children
901  matches = [m for m in matches if m[1].get(deblendKey) == 0]
902 
903  # Because we had to allow multiple matches to handle parents, we now
904  # need to prune to the best matches
905  # closest matches as a dict of icSourceCat source ID:
906  # (icSourceCat source, sourceCat source, distance in pixels)
907  bestMatches = {}
908  for m0, m1, d in matches:
909  id0 = m0.getId()
910  match = bestMatches.get(id0)
911  if match is None or d <= match[2]:
912  bestMatches[id0] = (m0, m1, d)
913  matches = list(bestMatches.values())
914 
915  # Check that no sourceCat sources are listed twice (we already know
916  # that each match has a unique icSourceCat source ID, due to using
917  # that ID as the key in bestMatches)
918  numMatches = len(matches)
919  numUniqueSources = len(set(m[1].getId() for m in matches))
920  if numUniqueSources != numMatches:
921  self.log.warn("{} icSourceCat sources matched only {} sourceCat "
922  "sources".format(numMatches, numUniqueSources))
923 
924  self.log.info("Copying flags from icSourceCat to sourceCat for "
925  "%s sources" % (numMatches,))
926 
927  # For each match: set the calibSourceKey flag and copy the desired
928  # fields
929  for icSrc, src, d in matches:
930  src.setFlag(self.calibSourceKeycalibSourceKey, True)
931  # src.assign copies the footprint from icSrc, which we don't want
932  # (DM-407)
933  # so set icSrc's footprint to src's footprint before src.assign,
934  # then restore it
935  icSrcFootprint = icSrc.getFootprint()
936  try:
937  icSrc.setFootprint(src.getFootprint())
938  src.assign(icSrc, self.schemaMapperschemaMapper)
939  finally:
940  icSrc.setFootprint(icSrcFootprint)
Calibrate an exposure: measure sources and perform astrometric and photometric calibration.
Definition: calibrate.py:325
def writeOutputs(self, dataRef, exposure, background, sourceCat, astromMatches, matchMeta)
Definition: calibrate.py:808
def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None, doUnpersist=True)
Calibrate an exposure, optionally unpersisting inputs and persisting outputs.
Definition: calibrate.py:543
def __init__(self, butler=None, astromRefObjLoader=None, photoRefObjLoader=None, icSourceSchema=None, initInputs=None, **kwargs)
Construct a CalibrateTask.
Definition: calibrate.py:424
def setMetadata(self, exposure, photoRes=None)
Set task and exposure metadata.
Definition: calibrate.py:839
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: calibrate.py:605
def copyIcSourceFields(self, icSourceCat, sourceCat)
Match sources in icSourceCat and sourceCat and copy the specified fields.
Definition: calibrate.py:872
def run(self, exposure, exposureIdInfo=None, background=None, icSourceCat=None)
Calibrate an exposure (science image or coadd)
Definition: calibrate.py:639