lsst.pipe.tasks  19.0.0-55-g5222f1c4
calibrate.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2016 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import math
23 
24 from lsstDebug import getDebugFrame
25 import lsst.pex.config as pexConfig
26 import lsst.pipe.base as pipeBase
28 import lsst.afw.table as afwTable
29 from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches
30 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
31 from lsst.obs.base import ExposureIdInfo
32 import lsst.daf.base as dafBase
33 from lsst.afw.math import BackgroundList
34 from lsst.afw.table import IdFactory, SourceTable
35 from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader
36 from lsst.meas.base import (SingleFrameMeasurementTask,
37  ApplyApCorrTask,
38  CatalogCalculationTask)
39 from lsst.meas.deblender import SourceDeblendTask
40 from .fakes import BaseFakeSourcesTask
41 from .photoCal import PhotoCalTask
42 
43 __all__ = ["CalibrateConfig", "CalibrateTask"]
44 
45 
46 class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"),
47  defaultTemplates={}):
48 
49  icSourceSchema = cT.InitInput(
50  doc="Schema produced by characterize image task, used to initialize this task",
51  name="icSrc_schema",
52  storageClass="SourceCatalog",
53  )
54 
55  outputSchema = cT.InitOutput(
56  doc="Schema after CalibrateTask has been initialized",
57  name="src_schema",
58  storageClass="SourceCatalog",
59  )
60 
61  exposure = cT.Input(
62  doc="Input image to calibrate",
63  name="icExp",
64  storageClass="ExposureF",
65  dimensions=("instrument", "visit", "detector"),
66  )
67 
68  background = cT.Input(
69  doc="Backgrounds determined by characterize task",
70  name="icExpBackground",
71  storageClass="Background",
72  dimensions=("instrument", "visit", "detector"),
73  )
74 
75  icSourceCat = cT.Input(
76  doc="Source catalog created by characterize task",
77  name="icSrc",
78  storageClass="SourceCatalog",
79  dimensions=("instrument", "visit", "detector"),
80  )
81 
82  astromRefCat = cT.PrerequisiteInput(
83  doc="Reference catalog to use for astrometry",
84  name="cal_ref_cat",
85  storageClass="SimpleCatalog",
86  dimensions=("skypix",),
87  deferLoad=True,
88  multiple=True,
89  )
90 
91  photoRefCat = cT.PrerequisiteInput(
92  doc="Reference catalog to use for photometric calibration",
93  name="cal_ref_cat",
94  storageClass="SimpleCatalog",
95  dimensions=("skypix",),
96  deferLoad=True,
97  multiple=True
98  )
99 
100  outputExposure = cT.Output(
101  doc="Exposure after running calibration task",
102  name="calexp",
103  storageClass="ExposureF",
104  dimensions=("instrument", "visit", "detector"),
105  )
106 
107  outputCat = cT.Output(
108  doc="Source catalog produced in calibrate task",
109  name="src",
110  storageClass="SourceCatalog",
111  dimensions=("instrument", "visit", "detector"),
112  )
113 
114  outputBackground = cT.Output(
115  doc="Background models estimated in calibration task",
116  name="calexpBackground",
117  storageClass="Background",
118  dimensions=("instrument", "visit", "detector"),
119  )
120 
121  matches = cT.Output(
122  doc="Source/refObj matches from the astrometry solver",
123  name="srcMatch",
124  storageClass="Catalog",
125  dimensions=("instrument", "visit", "detector"),
126  )
127 
128  matchesDenormalized = cT.Output(
129  doc="Denormalized matches from astrometry solver",
130  name="srcMatchFull",
131  storageClass="Catalog",
132  dimensions=("instrument", "visit", "detector"),
133  )
134 
135  def __init__(self, *, config=None):
136  super().__init__(config=config)
137 
138  if config.doAstrometry is False:
139  self.prerequisiteInputs.remove("astromRefCat")
140  if config.doPhotoCal is False:
141  self.prerequisiteInputs.remove("photoRefCat")
142 
143  if config.doWriteMatches is False or config.doAstrometry is False:
144  self.outputs.remove("matches")
145  if config.doWriteMatchesDenormalized is False or config.doAstrometry is False:
146  self.outputs.remove("matchesDenormalized")
147 
148 
149 class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections):
150  """Config for CalibrateTask"""
151  doWrite = pexConfig.Field(
152  dtype=bool,
153  default=True,
154  doc="Save calibration results?",
155  )
156  doWriteHeavyFootprintsInSources = pexConfig.Field(
157  dtype=bool,
158  default=True,
159  doc="Include HeavyFootprint data in source table? If false then heavy "
160  "footprints are saved as normal footprints, which saves some space"
161  )
162  doWriteMatches = pexConfig.Field(
163  dtype=bool,
164  default=True,
165  doc="Write reference matches (ignored if doWrite or doAstrometry false)?",
166  )
167  doWriteMatchesDenormalized = pexConfig.Field(
168  dtype=bool,
169  default=False,
170  doc=("Write reference matches in denormalized format? "
171  "This format uses more disk space, but is more convenient to "
172  "read. Ignored if doWriteMatches=False or doWrite=False."),
173  )
174  doAstrometry = pexConfig.Field(
175  dtype=bool,
176  default=True,
177  doc="Perform astrometric calibration?",
178  )
179  astromRefObjLoader = pexConfig.ConfigurableField(
180  target=LoadIndexedReferenceObjectsTask,
181  doc="reference object loader for astrometric calibration",
182  )
183  photoRefObjLoader = pexConfig.ConfigurableField(
184  target=LoadIndexedReferenceObjectsTask,
185  doc="reference object loader for photometric calibration",
186  )
187  astrometry = pexConfig.ConfigurableField(
188  target=AstrometryTask,
189  doc="Perform astrometric calibration to refine the WCS",
190  )
191  requireAstrometry = pexConfig.Field(
192  dtype=bool,
193  default=True,
194  doc=("Raise an exception if astrometry fails? Ignored if doAstrometry "
195  "false."),
196  )
197  doPhotoCal = pexConfig.Field(
198  dtype=bool,
199  default=True,
200  doc="Perform phometric calibration?",
201  )
202  requirePhotoCal = pexConfig.Field(
203  dtype=bool,
204  default=True,
205  doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal "
206  "false."),
207  )
208  photoCal = pexConfig.ConfigurableField(
209  target=PhotoCalTask,
210  doc="Perform photometric calibration",
211  )
212  icSourceFieldsToCopy = pexConfig.ListField(
213  dtype=str,
214  default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
215  doc=("Fields to copy from the icSource catalog to the output catalog "
216  "for matching sources Any missing fields will trigger a "
217  "RuntimeError exception. Ignored if icSourceCat is not provided.")
218  )
219  matchRadiusPix = pexConfig.Field(
220  dtype=float,
221  default=3,
222  doc=("Match radius for matching icSourceCat objects to sourceCat "
223  "objects (pixels)"),
224  )
225  checkUnitsParseStrict = pexConfig.Field(
226  doc=("Strictness of Astropy unit compatibility check, can be 'raise', "
227  "'warn' or 'silent'"),
228  dtype=str,
229  default="raise",
230  )
231  detection = pexConfig.ConfigurableField(
232  target=SourceDetectionTask,
233  doc="Detect sources"
234  )
235  doDeblend = pexConfig.Field(
236  dtype=bool,
237  default=True,
238  doc="Run deblender input exposure"
239  )
240  deblend = pexConfig.ConfigurableField(
241  target=SourceDeblendTask,
242  doc="Split blended sources into their components"
243  )
244  measurement = pexConfig.ConfigurableField(
245  target=SingleFrameMeasurementTask,
246  doc="Measure sources"
247  )
248  doApCorr = pexConfig.Field(
249  dtype=bool,
250  default=True,
251  doc="Run subtask to apply aperture correction"
252  )
253  applyApCorr = pexConfig.ConfigurableField(
254  target=ApplyApCorrTask,
255  doc="Subtask to apply aperture corrections"
256  )
257  # If doApCorr is False, and the exposure does not have apcorrections
258  # already applied, the active plugins in catalogCalculation almost
259  # certainly should not contain the characterization plugin
260  catalogCalculation = pexConfig.ConfigurableField(
261  target=CatalogCalculationTask,
262  doc="Subtask to run catalogCalculation plugins on catalog"
263  )
264  doInsertFakes = pexConfig.Field(
265  dtype=bool,
266  default=False,
267  doc="Run fake sources injection task"
268  )
269  insertFakes = pexConfig.ConfigurableField(
270  target=BaseFakeSourcesTask,
271  doc="Injection of fake sources for testing purposes (must be "
272  "retargeted)"
273  )
274  doWriteExposure = pexConfig.Field(
275  dtype=bool,
276  default=True,
277  doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a "
278  "normal calexp but as a fakes_calexp."
279  )
280 
281  def setDefaults(self):
282  super().setDefaults()
283  self.detection.doTempLocalBackground = False
284  self.deblend.maxFootprintSize = 2000
285  self.measurement.plugins.names |= ["base_LocalPhotoCalib", "base_LocalWcs"]
286 
287  def validate(self):
288  super().validate()
289  astromRefCatGen2 = getattr(self.astromRefObjLoader, "ref_dataset_name", None)
290  if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat:
291  raise ValueError(
292  f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference "
293  f"catalogs are different. These options must be kept in sync until Gen2 is retired."
294  )
295  photoRefCatGen2 = getattr(self.photoRefObjLoader, "ref_dataset_name", None)
296  if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat:
297  raise ValueError(
298  f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference "
299  f"catalogs are different. These options must be kept in sync until Gen2 is retired."
300  )
301 
302 
303 
309 
310 class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
311  r"""!Calibrate an exposure: measure sources and perform astrometric and
312  photometric calibration
313 
314  @anchor CalibrateTask_
315 
316  @section pipe_tasks_calibrate_Contents Contents
317 
318  - @ref pipe_tasks_calibrate_Purpose
319  - @ref pipe_tasks_calibrate_Initialize
320  - @ref pipe_tasks_calibrate_IO
321  - @ref pipe_tasks_calibrate_Config
322  - @ref pipe_tasks_calibrate_Metadata
323  - @ref pipe_tasks_calibrate_Debug
324 
325 
326  @section pipe_tasks_calibrate_Purpose Description
327 
328  Given an exposure with a good PSF model and aperture correction map
329  (e.g. as provided by @ref CharacterizeImageTask), perform the following
330  operations:
331  - Run detection and measurement
332  - Run astrometry subtask to fit an improved WCS
333  - Run photoCal subtask to fit the exposure's photometric zero-point
334 
335  @section pipe_tasks_calibrate_Initialize Task initialisation
336 
337  @copydoc \_\_init\_\_
338 
339  @section pipe_tasks_calibrate_IO Invoking the Task
340 
341  If you want this task to unpersist inputs or persist outputs, then call
342  the `runDataRef` method (a wrapper around the `run` method).
343 
344  If you already have the inputs unpersisted and do not want to persist the
345  output then it is more direct to call the `run` method:
346 
347  @section pipe_tasks_calibrate_Config Configuration parameters
348 
349  See @ref CalibrateConfig
350 
351  @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata
352 
353  Exposure metadata
354  <dl>
355  <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task
356  <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal
357  task
358  <dt>COLORTERM1 <dd>?? (always 0.0)
359  <dt>COLORTERM2 <dd>?? (always 0.0)
360  <dt>COLORTERM3 <dd>?? (always 0.0)
361  </dl>
362 
363  @section pipe_tasks_calibrate_Debug Debug variables
364 
365  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink
366  interface supports a flag
367  `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug
368  for more about `debug.py`.
369 
370  CalibrateTask has a debug dictionary containing one key:
371  <dl>
372  <dt>calibrate
373  <dd>frame (an int; <= 0 to not display) in which to display the exposure,
374  sources and matches. See @ref lsst.meas.astrom.displayAstrometry for
375  the meaning of the various symbols.
376  </dl>
377 
378  For example, put something like:
379  @code{.py}
380  import lsstDebug
381  def DebugInfo(name):
382  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would
383  # call us recursively
384  if name == "lsst.pipe.tasks.calibrate":
385  di.display = dict(
386  calibrate = 1,
387  )
388 
389  return di
390 
391  lsstDebug.Info = DebugInfo
392  @endcode
393  into your `debug.py` file and run `calibrateTask.py` with the `--debug`
394  flag.
395 
396  Some subtasks may have their own debug variables; see individual Task
397  documentation.
398  """
399 
400  # Example description used to live here, removed 2-20-2017 as per
401  # https://jira.lsstcorp.org/browse/DM-9520
402 
403  ConfigClass = CalibrateConfig
404  _DefaultName = "calibrate"
405  RunnerClass = pipeBase.ButlerInitializedTaskRunner
406 
407  def __init__(self, butler=None, astromRefObjLoader=None,
408  photoRefObjLoader=None, icSourceSchema=None,
409  initInputs=None, **kwargs):
410  """!Construct a CalibrateTask
411 
412  @param[in] butler The butler is passed to the refObjLoader constructor
413  in case it is needed. Ignored if the refObjLoader argument
414  provides a loader directly.
415  @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks
416  that supplies an external reference catalog for astrometric
417  calibration. May be None if the desired loader can be constructed
418  from the butler argument or all steps requiring a reference catalog
419  are disabled.
420  @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks
421  that supplies an external reference catalog for photometric
422  calibration. May be None if the desired loader can be constructed
423  from the butler argument or all steps requiring a reference catalog
424  are disabled.
425  @param[in] icSourceSchema schema for icSource catalog, or None.
426  Schema values specified in config.icSourceFieldsToCopy will be
427  taken from this schema. If set to None, no values will be
428  propagated from the icSourceCatalog
429  @param[in,out] kwargs other keyword arguments for
430  lsst.pipe.base.CmdLineTask
431  """
432  super().__init__(**kwargs)
433 
434  if icSourceSchema is None and butler is not None:
435  # Use butler to read icSourceSchema from disk.
436  icSourceSchema = butler.get("icSrc_schema", immediate=True).schema
437 
438  if icSourceSchema is None and butler is None and initInputs is not None:
439  icSourceSchema = initInputs['icSourceSchema'].schema
440 
441  if icSourceSchema is not None:
442  # use a schema mapper to avoid copying each field separately
443  self.schemaMapper = afwTable.SchemaMapper(icSourceSchema)
444  minimumSchema = afwTable.SourceTable.makeMinimalSchema()
445  self.schemaMapper.addMinimalSchema(minimumSchema, False)
446 
447  # Add fields to copy from an icSource catalog
448  # and a field to indicate that the source matched a source in that
449  # catalog. If any fields are missing then raise an exception, but
450  # first find all missing fields in order to make the error message
451  # more useful.
452  self.calibSourceKey = self.schemaMapper.addOutputField(
453  afwTable.Field["Flag"]("calib_detected",
454  "Source was detected as an icSource"))
455  missingFieldNames = []
456  for fieldName in self.config.icSourceFieldsToCopy:
457  try:
458  schemaItem = icSourceSchema.find(fieldName)
459  except Exception:
460  missingFieldNames.append(fieldName)
461  else:
462  # field found; if addMapping fails then raise an exception
463  self.schemaMapper.addMapping(schemaItem.getKey())
464 
465  if missingFieldNames:
466  raise RuntimeError("isSourceCat is missing fields {} "
467  "specified in icSourceFieldsToCopy"
468  .format(missingFieldNames))
469 
470  # produce a temporary schema to pass to the subtasks; finalize it
471  # later
472  self.schema = self.schemaMapper.editOutputSchema()
473  else:
474  self.schemaMapper = None
475  self.schema = afwTable.SourceTable.makeMinimalSchema()
476  self.makeSubtask('detection', schema=self.schema)
477 
478  self.algMetadata = dafBase.PropertyList()
479 
480  # Only create a subtask for fakes if configuration option is set
481  # N.B. the config for fake object task must be retargeted to a child
482  # of BaseFakeSourcesTask
483  if self.config.doInsertFakes:
484  self.makeSubtask("insertFakes")
485 
486  if self.config.doDeblend:
487  self.makeSubtask("deblend", schema=self.schema)
488  self.makeSubtask('measurement', schema=self.schema,
489  algMetadata=self.algMetadata)
490  if self.config.doApCorr:
491  self.makeSubtask('applyApCorr', schema=self.schema)
492  self.makeSubtask('catalogCalculation', schema=self.schema)
493 
494  if self.config.doAstrometry:
495  if astromRefObjLoader is None and butler is not None:
496  self.makeSubtask('astromRefObjLoader', butler=butler)
497  astromRefObjLoader = self.astromRefObjLoader
498  self.pixelMargin = astromRefObjLoader.config.pixelMargin
499  self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader,
500  schema=self.schema)
501  if self.config.doPhotoCal:
502  if photoRefObjLoader is None and butler is not None:
503  self.makeSubtask('photoRefObjLoader', butler=butler)
504  photoRefObjLoader = self.photoRefObjLoader
505  self.pixelMargin = photoRefObjLoader.config.pixelMargin
506  self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader,
507  schema=self.schema)
508 
509  if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None):
510  raise RuntimeError("PipelineTask form of this task should not be initialized with "
511  "reference object loaders.")
512 
513  if self.schemaMapper is not None:
514  # finalize the schema
515  self.schema = self.schemaMapper.getOutputSchema()
516  self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
517 
518  sourceCatSchema = afwTable.SourceCatalog(self.schema)
519  sourceCatSchema.getTable().setMetadata(self.algMetadata)
520  self.outputSchema = sourceCatSchema
521 
522  @pipeBase.timeMethod
523  def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None,
524  doUnpersist=True):
525  """!Calibrate an exposure, optionally unpersisting inputs and
526  persisting outputs.
527 
528  This is a wrapper around the `run` method that unpersists inputs
529  (if `doUnpersist` true) and persists outputs (if `config.doWrite` true)
530 
531  @param[in] dataRef butler data reference corresponding to a science
532  image
533  @param[in,out] exposure characterized exposure (an
534  lsst.afw.image.ExposureF or similar), or None to unpersist existing
535  icExp and icBackground. See `run` method for details of what is
536  read and written.
537  @param[in,out] background initial model of background already
538  subtracted from exposure (an lsst.afw.math.BackgroundList). May be
539  None if no background has been subtracted, though that is unusual
540  for calibration. A refined background model is output. Ignored if
541  exposure is None.
542  @param[in] icSourceCat catalog from which to copy the fields specified
543  by icSourceKeys, or None;
544  @param[in] doUnpersist unpersist data:
545  - if True, exposure, background and icSourceCat are read from
546  dataRef and those three arguments must all be None;
547  - if False the exposure must be provided; background and
548  icSourceCat are optional. True is intended for running as a
549  command-line task, False for running as a subtask
550  @return same data as the calibrate method
551  """
552  self.log.info("Processing %s" % (dataRef.dataId))
553 
554  if doUnpersist:
555  if any(item is not None for item in (exposure, background,
556  icSourceCat)):
557  raise RuntimeError("doUnpersist true; exposure, background "
558  "and icSourceCat must all be None")
559  exposure = dataRef.get("icExp", immediate=True)
560  background = dataRef.get("icExpBackground", immediate=True)
561  icSourceCat = dataRef.get("icSrc", immediate=True)
562  elif exposure is None:
563  raise RuntimeError("doUnpersist false; exposure must be provided")
564 
565  exposureIdInfo = dataRef.get("expIdInfo")
566 
567  calRes = self.run(
568  exposure=exposure,
569  exposureIdInfo=exposureIdInfo,
570  background=background,
571  icSourceCat=icSourceCat,
572  )
573 
574  if self.config.doWrite:
575  self.writeOutputs(
576  dataRef=dataRef,
577  exposure=calRes.exposure,
578  background=calRes.background,
579  sourceCat=calRes.sourceCat,
580  astromMatches=calRes.astromMatches,
581  matchMeta=calRes.matchMeta,
582  )
583 
584  return calRes
585 
586  def runQuantum(self, butlerQC, inputRefs, outputRefs):
587  inputs = butlerQC.get(inputRefs)
588  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
589  returnMaxBits=True)
590  inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
591 
592  if self.config.doAstrometry:
593  refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
594  for ref in inputRefs.astromRefCat],
595  refCats=inputs.pop('astromRefCat'),
596  config=self.config.astromRefObjLoader, log=self.log)
597  self.pixelMargin = refObjLoader.config.pixelMargin
598  self.astrometry.setRefObjLoader(refObjLoader)
599 
600  if self.config.doPhotoCal:
601  photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
602  for ref in inputRefs.photoRefCat],
603  refCats=inputs.pop('photoRefCat'),
604  config=self.config.photoRefObjLoader,
605  log=self.log)
606  self.pixelMargin = photoRefObjLoader.config.pixelMargin
607  self.photoCal.match.setRefObjLoader(photoRefObjLoader)
608 
609  outputs = self.run(**inputs)
610 
611  if self.config.doWriteMatches and self.config.doAstrometry:
612  normalizedMatches = afwTable.packMatches(outputs.astromMatches)
613  normalizedMatches.table.setMetadata(outputs.matchMeta)
614  if self.config.doWriteMatchesDenormalized:
615  denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
616  outputs.matchesDenormalized = denormMatches
617  outputs.matches = normalizedMatches
618  butlerQC.put(outputs, outputRefs)
619 
620  def run(self, exposure, exposureIdInfo=None, background=None,
621  icSourceCat=None):
622  """!Calibrate an exposure (science image or coadd)
623 
624  @param[in,out] exposure exposure to calibrate (an
625  lsst.afw.image.ExposureF or similar);
626  in:
627  - MaskedImage
628  - Psf
629  out:
630  - MaskedImage has background subtracted
631  - Wcs is replaced
632  - PhotoCalib is replaced
633  @param[in] exposureIdInfo ID info for exposure (an
634  lsst.obs.base.ExposureIdInfo) If not provided, returned
635  SourceCatalog IDs will not be globally unique.
636  @param[in,out] background background model already subtracted from
637  exposure (an lsst.afw.math.BackgroundList). May be None if no
638  background has been subtracted, though that is unusual for
639  calibration. A refined background model is output.
640  @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
641  from which we can copy some fields.
642 
643  @return pipe_base Struct containing these fields:
644  - exposure calibrate science exposure with refined WCS and PhotoCalib
645  - background model of background subtracted from exposure (an
646  lsst.afw.math.BackgroundList)
647  - sourceCat catalog of measured sources
648  - astromMatches list of source/refObj matches from the astrometry
649  solver
650  """
651  # detect, deblend and measure sources
652  if exposureIdInfo is None:
653  exposureIdInfo = ExposureIdInfo()
654 
655  if background is None:
656  background = BackgroundList()
657  sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
658  exposureIdInfo.unusedBits)
659  table = SourceTable.make(self.schema, sourceIdFactory)
660  table.setMetadata(self.algMetadata)
661 
662  detRes = self.detection.run(table=table, exposure=exposure,
663  doSmooth=True)
664  sourceCat = detRes.sources
665  if detRes.fpSets.background:
666  for bg in detRes.fpSets.background:
667  background.append(bg)
668  if self.config.doDeblend:
669  self.deblend.run(exposure=exposure, sources=sourceCat)
670  self.measurement.run(
671  measCat=sourceCat,
672  exposure=exposure,
673  exposureId=exposureIdInfo.expId
674  )
675  if self.config.doApCorr:
676  self.applyApCorr.run(
677  catalog=sourceCat,
678  apCorrMap=exposure.getInfo().getApCorrMap()
679  )
680  self.catalogCalculation.run(sourceCat)
681 
682  if icSourceCat is not None and \
683  len(self.config.icSourceFieldsToCopy) > 0:
684  self.copyIcSourceFields(icSourceCat=icSourceCat,
685  sourceCat=sourceCat)
686 
687  # TODO DM-11568: this contiguous check-and-copy could go away if we
688  # reserve enough space during SourceDetection and/or SourceDeblend.
689  # NOTE: sourceSelectors require contiguous catalogs, so ensure
690  # contiguity now, so views are preserved from here on.
691  if not sourceCat.isContiguous():
692  sourceCat = sourceCat.copy(deep=True)
693 
694  # perform astrometry calibration:
695  # fit an improved WCS and update the exposure's WCS in place
696  astromMatches = None
697  matchMeta = None
698  if self.config.doAstrometry:
699  try:
700  astromRes = self.astrometry.run(
701  exposure=exposure,
702  sourceCat=sourceCat,
703  )
704  astromMatches = astromRes.matches
705  matchMeta = astromRes.matchMeta
706  except Exception as e:
707  if self.config.requireAstrometry:
708  raise
709  self.log.warn("Unable to perform astrometric calibration "
710  "(%s): attempting to proceed" % e)
711 
712  # compute photometric calibration
713  if self.config.doPhotoCal:
714  try:
715  photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
716  exposure.setPhotoCalib(photoRes.photoCalib)
717  # TODO: reword this to phrase it in terms of the calibration factor?
718  self.log.info("Photometric zero-point: %f" %
719  photoRes.photoCalib.instFluxToMagnitude(1.0))
720  self.setMetadata(exposure=exposure, photoRes=photoRes)
721  except Exception as e:
722  if self.config.requirePhotoCal:
723  raise
724  self.log.warn("Unable to perform photometric calibration "
725  "(%s): attempting to proceed" % e)
726  self.setMetadata(exposure=exposure, photoRes=None)
727 
728  if self.config.doInsertFakes:
729  self.insertFakes.run(exposure, background=background)
730 
731  table = SourceTable.make(self.schema, sourceIdFactory)
732  table.setMetadata(self.algMetadata)
733 
734  detRes = self.detection.run(table=table, exposure=exposure,
735  doSmooth=True)
736  sourceCat = detRes.sources
737  if detRes.fpSets.background:
738  for bg in detRes.fpSets.background:
739  background.append(bg)
740  if self.config.doDeblend:
741  self.deblend.run(exposure=exposure, sources=sourceCat)
742  self.measurement.run(
743  measCat=sourceCat,
744  exposure=exposure,
745  exposureId=exposureIdInfo.expId
746  )
747  if self.config.doApCorr:
748  self.applyApCorr.run(
749  catalog=sourceCat,
750  apCorrMap=exposure.getInfo().getApCorrMap()
751  )
752  self.catalogCalculation.run(sourceCat)
753 
754  if icSourceCat is not None and len(self.config.icSourceFieldsToCopy) > 0:
755  self.copyIcSourceFields(icSourceCat=icSourceCat,
756  sourceCat=sourceCat)
757 
758  frame = getDebugFrame(self._display, "calibrate")
759  if frame:
760  displayAstrometry(
761  sourceCat=sourceCat,
762  exposure=exposure,
763  matches=astromMatches,
764  frame=frame,
765  pause=False,
766  )
767 
768  return pipeBase.Struct(
769  exposure=exposure,
770  background=background,
771  sourceCat=sourceCat,
772  astromMatches=astromMatches,
773  matchMeta=matchMeta,
774  # These are duplicate entries with different names for use with
775  # gen3 middleware
776  outputExposure=exposure,
777  outputCat=sourceCat,
778  outputBackground=background,
779  )
780 
781  def writeOutputs(self, dataRef, exposure, background, sourceCat,
782  astromMatches, matchMeta):
783  """Write output data to the output repository
784 
785  @param[in] dataRef butler data reference corresponding to a science
786  image
787  @param[in] exposure exposure to write
788  @param[in] background background model for exposure
789  @param[in] sourceCat catalog of measured sources
790  @param[in] astromMatches list of source/refObj matches from the
791  astrometry solver
792  """
793  dataRef.put(sourceCat, "src")
794  if self.config.doWriteMatches and astromMatches is not None:
795  normalizedMatches = afwTable.packMatches(astromMatches)
796  normalizedMatches.table.setMetadata(matchMeta)
797  dataRef.put(normalizedMatches, "srcMatch")
798  if self.config.doWriteMatchesDenormalized:
799  denormMatches = denormalizeMatches(astromMatches, matchMeta)
800  dataRef.put(denormMatches, "srcMatchFull")
801  if self.config.doWriteExposure:
802  dataRef.put(exposure, "calexp")
803  dataRef.put(background, "calexpBackground")
804 
805  def getSchemaCatalogs(self):
806  """Return a dict of empty catalogs for each catalog dataset produced
807  by this task.
808  """
809  sourceCat = afwTable.SourceCatalog(self.schema)
810  sourceCat.getTable().setMetadata(self.algMetadata)
811  return {"src": sourceCat}
812 
813  def setMetadata(self, exposure, photoRes=None):
814  """!Set task and exposure metadata
815 
816  Logs a warning and continues if needed data is missing.
817 
818  @param[in,out] exposure exposure whose metadata is to be set
819  @param[in] photoRes results of running photoCal; if None then it was
820  not run
821  """
822  if photoRes is None:
823  return
824 
825  metadata = exposure.getMetadata()
826 
827  # convert zero-point to (mag/sec/adu) for task MAGZERO metadata
828  try:
829  exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
830  magZero = photoRes.zp - 2.5*math.log10(exposureTime)
831  except Exception:
832  self.log.warn("Could not set normalized MAGZERO in header: no "
833  "exposure time")
834  magZero = math.nan
835 
836  try:
837  metadata.set('MAGZERO', magZero)
838  metadata.set('MAGZERO_RMS', photoRes.sigma)
839  metadata.set('MAGZERO_NOBJ', photoRes.ngood)
840  metadata.set('COLORTERM1', 0.0)
841  metadata.set('COLORTERM2', 0.0)
842  metadata.set('COLORTERM3', 0.0)
843  except Exception as e:
844  self.log.warn("Could not set exposure metadata: %s" % (e,))
845 
846  def copyIcSourceFields(self, icSourceCat, sourceCat):
847  """!Match sources in icSourceCat and sourceCat and copy the specified fields
848 
849  @param[in] icSourceCat catalog from which to copy fields
850  @param[in,out] sourceCat catalog to which to copy fields
851 
852  The fields copied are those specified by `config.icSourceFieldsToCopy`
853  that actually exist in the schema. This was set up by the constructor
854  using self.schemaMapper.
855  """
856  if self.schemaMapper is None:
857  raise RuntimeError("To copy icSource fields you must specify "
858  "icSourceSchema nd icSourceKeys when "
859  "constructing this task")
860  if icSourceCat is None or sourceCat is None:
861  raise RuntimeError("icSourceCat and sourceCat must both be "
862  "specified")
863  if len(self.config.icSourceFieldsToCopy) == 0:
864  self.log.warn("copyIcSourceFields doing nothing because "
865  "icSourceFieldsToCopy is empty")
866  return
867 
868  mc = afwTable.MatchControl()
869  mc.findOnlyClosest = False # return all matched objects
870  matches = afwTable.matchXy(icSourceCat, sourceCat,
871  self.config.matchRadiusPix, mc)
872  if self.config.doDeblend:
873  deblendKey = sourceCat.schema["deblend_nChild"].asKey()
874  # if deblended, keep children
875  matches = [m for m in matches if m[1].get(deblendKey) == 0]
876 
877  # Because we had to allow multiple matches to handle parents, we now
878  # need to prune to the best matches
879  # closest matches as a dict of icSourceCat source ID:
880  # (icSourceCat source, sourceCat source, distance in pixels)
881  bestMatches = {}
882  for m0, m1, d in matches:
883  id0 = m0.getId()
884  match = bestMatches.get(id0)
885  if match is None or d <= match[2]:
886  bestMatches[id0] = (m0, m1, d)
887  matches = list(bestMatches.values())
888 
889  # Check that no sourceCat sources are listed twice (we already know
890  # that each match has a unique icSourceCat source ID, due to using
891  # that ID as the key in bestMatches)
892  numMatches = len(matches)
893  numUniqueSources = len(set(m[1].getId() for m in matches))
894  if numUniqueSources != numMatches:
895  self.log.warn("{} icSourceCat sources matched only {} sourceCat "
896  "sources".format(numMatches, numUniqueSources))
897 
898  self.log.info("Copying flags from icSourceCat to sourceCat for "
899  "%s sources" % (numMatches,))
900 
901  # For each match: set the calibSourceKey flag and copy the desired
902  # fields
903  for icSrc, src, d in matches:
904  src.setFlag(self.calibSourceKey, True)
905  # src.assign copies the footprint from icSrc, which we don't want
906  # (DM-407)
907  # so set icSrc's footprint to src's footprint before src.assign,
908  # then restore it
909  icSrcFootprint = icSrc.getFootprint()
910  try:
911  icSrc.setFootprint(src.getFootprint())
912  src.assign(icSrc, self.schemaMapper)
913  finally:
914  icSrc.setFootprint(icSrcFootprint)
lsst.pipe.tasks.calibrate.CalibrateConfig.measurement
measurement
Definition: calibrate.py:244
lsst.pipe.tasks.calibrate.CalibrateTask.runQuantum
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: calibrate.py:586
lsst::meas::astrom
lsst.pipe.tasks.calibrate.CalibrateConfig.setDefaults
def setDefaults(self)
Definition: calibrate.py:281
lsst.pipe.tasks.calibrate.CalibrateConnections.__init__
def __init__(self, *config=None)
Definition: calibrate.py:135
lsst.pipe.tasks.calibrate.CalibrateTask.outputSchema
outputSchema
Definition: calibrate.py:518
lsst.pipe.tasks.calibrate.CalibrateConfig
Definition: calibrate.py:149
lsst.pipe.tasks.calibrate.CalibrateTask.schemaMapper
schemaMapper
Definition: calibrate.py:441
lsst.pipe.tasks.calibrate.CalibrateTask
Calibrate an exposure: measure sources and perform astrometric and photometric calibration.
Definition: calibrate.py:310
lsst.pipe.tasks.calibrate.CalibrateTask.writeOutputs
def writeOutputs(self, dataRef, exposure, background, sourceCat, astromMatches, matchMeta)
Definition: calibrate.py:781
lsst.pipe.tasks.calibrate.CalibrateTask.pixelMargin
pixelMargin
Definition: calibrate.py:496
lsst.pipe.tasks.calibrate.CalibrateTask.copyIcSourceFields
def copyIcSourceFields(self, icSourceCat, sourceCat)
Match sources in icSourceCat and sourceCat and copy the specified fields.
Definition: calibrate.py:846
lsst::meas::base
lsst.pipe.tasks.calibrate.CalibrateConfig.astromRefObjLoader
astromRefObjLoader
Definition: calibrate.py:179
lsst.pipe.tasks.calibrate.CalibrateConfig.photoRefObjLoader
photoRefObjLoader
Definition: calibrate.py:183
lsst::afw::table
lsst::meas::deblender
lsst.pipe.tasks.calibrate.CalibrateConfig.validate
def validate(self)
Definition: calibrate.py:287
lsst::daf::base
lsst.pipe.tasks.calibrate.CalibrateTask.schema
schema
Definition: calibrate.py:470
lsst.pipe.tasks.calibrate.CalibrateTask.__init__
def __init__(self, butler=None, astromRefObjLoader=None, photoRefObjLoader=None, icSourceSchema=None, initInputs=None, **kwargs)
Construct a CalibrateTask.
Definition: calibrate.py:407
lsst.pipe.tasks.calibrate.CalibrateTask.calibSourceKey
calibSourceKey
Definition: calibrate.py:450
lsst::afw::math
lsst.pipe.tasks.calibrate.CalibrateTask.runDataRef
def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None, doUnpersist=True)
Calibrate an exposure, optionally unpersisting inputs and persisting outputs.
Definition: calibrate.py:523
lsst.pipe.tasks.calibrate.CalibrateConnections
Definition: calibrate.py:47
lsst.pipe.tasks.calibrate.CalibrateTask.getSchemaCatalogs
def getSchemaCatalogs(self)
Definition: calibrate.py:805
lsst.pipe.tasks.calibrate.CalibrateTask.setMetadata
def setMetadata(self, exposure, photoRes=None)
Set task and exposure metadata.
Definition: calibrate.py:813
lsst.pipe.tasks.calibrate.CalibrateConfig.deblend
deblend
Definition: calibrate.py:240
lsst.pipe::base
lsst::meas::algorithms
lsst.pipe.tasks.calibrate.CalibrateTask.algMetadata
algMetadata
Definition: calibrate.py:476
lsst.pipe::base::connectionTypes
lsst.pipe.tasks.calibrate.CalibrateTask.run
def run(self, exposure, exposureIdInfo=None, background=None, icSourceCat=None)
Calibrate an exposure (science image or coadd)
Definition: calibrate.py:620
lsst::obs::base
lsst.pipe.tasks.calibrate.CalibrateConfig.detection
detection
Definition: calibrate.py:231