lsst.pipe.tasks  20.0.0-16-g6654009b+03b8a8f522
calibrate.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2016 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import math
23 
24 from lsstDebug import getDebugFrame
25 import lsst.pex.config as pexConfig
26 import lsst.pipe.base as pipeBase
28 import lsst.afw.table as afwTable
29 from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches
30 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, SkyObjectsTask
31 from lsst.obs.base import ExposureIdInfo
32 import lsst.daf.base as dafBase
33 from lsst.afw.math import BackgroundList
34 from lsst.afw.table import IdFactory, SourceTable
35 from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader
36 from lsst.meas.base import (SingleFrameMeasurementTask,
37  ApplyApCorrTask,
38  CatalogCalculationTask)
39 from lsst.meas.deblender import SourceDeblendTask
40 from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask
41 from .fakes import BaseFakeSourcesTask
42 from .photoCal import PhotoCalTask
43 
44 __all__ = ["CalibrateConfig", "CalibrateTask"]
45 
46 
47 class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"),
48  defaultTemplates={}):
49 
50  icSourceSchema = cT.InitInput(
51  doc="Schema produced by characterize image task, used to initialize this task",
52  name="icSrc_schema",
53  storageClass="SourceCatalog",
54  )
55 
56  outputSchema = cT.InitOutput(
57  doc="Schema after CalibrateTask has been initialized",
58  name="src_schema",
59  storageClass="SourceCatalog",
60  )
61 
62  exposure = cT.Input(
63  doc="Input image to calibrate",
64  name="icExp",
65  storageClass="ExposureF",
66  dimensions=("instrument", "visit", "detector"),
67  )
68 
69  background = cT.Input(
70  doc="Backgrounds determined by characterize task",
71  name="icExpBackground",
72  storageClass="Background",
73  dimensions=("instrument", "visit", "detector"),
74  )
75 
76  icSourceCat = cT.Input(
77  doc="Source catalog created by characterize task",
78  name="icSrc",
79  storageClass="SourceCatalog",
80  dimensions=("instrument", "visit", "detector"),
81  )
82 
83  astromRefCat = cT.PrerequisiteInput(
84  doc="Reference catalog to use for astrometry",
85  name="cal_ref_cat",
86  storageClass="SimpleCatalog",
87  dimensions=("skypix",),
88  deferLoad=True,
89  multiple=True,
90  )
91 
92  photoRefCat = cT.PrerequisiteInput(
93  doc="Reference catalog to use for photometric calibration",
94  name="cal_ref_cat",
95  storageClass="SimpleCatalog",
96  dimensions=("skypix",),
97  deferLoad=True,
98  multiple=True
99  )
100 
101  outputExposure = cT.Output(
102  doc="Exposure after running calibration task",
103  name="calexp",
104  storageClass="ExposureF",
105  dimensions=("instrument", "visit", "detector"),
106  )
107 
108  outputCat = cT.Output(
109  doc="Source catalog produced in calibrate task",
110  name="src",
111  storageClass="SourceCatalog",
112  dimensions=("instrument", "visit", "detector"),
113  )
114 
115  outputBackground = cT.Output(
116  doc="Background models estimated in calibration task",
117  name="calexpBackground",
118  storageClass="Background",
119  dimensions=("instrument", "visit", "detector"),
120  )
121 
122  matches = cT.Output(
123  doc="Source/refObj matches from the astrometry solver",
124  name="srcMatch",
125  storageClass="Catalog",
126  dimensions=("instrument", "visit", "detector"),
127  )
128 
129  matchesDenormalized = cT.Output(
130  doc="Denormalized matches from astrometry solver",
131  name="srcMatchFull",
132  storageClass="Catalog",
133  dimensions=("instrument", "visit", "detector"),
134  )
135 
136  def __init__(self, *, config=None):
137  super().__init__(config=config)
138 
139  if config.doAstrometry is False:
140  self.prerequisiteInputs.remove("astromRefCat")
141  if config.doPhotoCal is False:
142  self.prerequisiteInputs.remove("photoRefCat")
143 
144  if config.doWriteMatches is False or config.doAstrometry is False:
145  self.outputs.remove("matches")
146  if config.doWriteMatchesDenormalized is False or config.doAstrometry is False:
147  self.outputs.remove("matchesDenormalized")
148 
149 
150 class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections):
151  """Config for CalibrateTask"""
152  doWrite = pexConfig.Field(
153  dtype=bool,
154  default=True,
155  doc="Save calibration results?",
156  )
157  doWriteHeavyFootprintsInSources = pexConfig.Field(
158  dtype=bool,
159  default=True,
160  doc="Include HeavyFootprint data in source table? If false then heavy "
161  "footprints are saved as normal footprints, which saves some space"
162  )
163  doWriteMatches = pexConfig.Field(
164  dtype=bool,
165  default=True,
166  doc="Write reference matches (ignored if doWrite or doAstrometry false)?",
167  )
168  doWriteMatchesDenormalized = pexConfig.Field(
169  dtype=bool,
170  default=False,
171  doc=("Write reference matches in denormalized format? "
172  "This format uses more disk space, but is more convenient to "
173  "read. Ignored if doWriteMatches=False or doWrite=False."),
174  )
175  doAstrometry = pexConfig.Field(
176  dtype=bool,
177  default=True,
178  doc="Perform astrometric calibration?",
179  )
180  astromRefObjLoader = pexConfig.ConfigurableField(
181  target=LoadIndexedReferenceObjectsTask,
182  doc="reference object loader for astrometric calibration",
183  )
184  photoRefObjLoader = pexConfig.ConfigurableField(
185  target=LoadIndexedReferenceObjectsTask,
186  doc="reference object loader for photometric calibration",
187  )
188  astrometry = pexConfig.ConfigurableField(
189  target=AstrometryTask,
190  doc="Perform astrometric calibration to refine the WCS",
191  )
192  requireAstrometry = pexConfig.Field(
193  dtype=bool,
194  default=True,
195  doc=("Raise an exception if astrometry fails? Ignored if doAstrometry "
196  "false."),
197  )
198  doPhotoCal = pexConfig.Field(
199  dtype=bool,
200  default=True,
201  doc="Perform phometric calibration?",
202  )
203  requirePhotoCal = pexConfig.Field(
204  dtype=bool,
205  default=True,
206  doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal "
207  "false."),
208  )
209  photoCal = pexConfig.ConfigurableField(
210  target=PhotoCalTask,
211  doc="Perform photometric calibration",
212  )
213  icSourceFieldsToCopy = pexConfig.ListField(
214  dtype=str,
215  default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
216  doc=("Fields to copy from the icSource catalog to the output catalog "
217  "for matching sources Any missing fields will trigger a "
218  "RuntimeError exception. Ignored if icSourceCat is not provided.")
219  )
220  matchRadiusPix = pexConfig.Field(
221  dtype=float,
222  default=3,
223  doc=("Match radius for matching icSourceCat objects to sourceCat "
224  "objects (pixels)"),
225  )
226  checkUnitsParseStrict = pexConfig.Field(
227  doc=("Strictness of Astropy unit compatibility check, can be 'raise', "
228  "'warn' or 'silent'"),
229  dtype=str,
230  default="raise",
231  )
232  detection = pexConfig.ConfigurableField(
233  target=SourceDetectionTask,
234  doc="Detect sources"
235  )
236  doDeblend = pexConfig.Field(
237  dtype=bool,
238  default=True,
239  doc="Run deblender input exposure"
240  )
241  deblend = pexConfig.ConfigurableField(
242  target=SourceDeblendTask,
243  doc="Split blended sources into their components"
244  )
245  doSkySources = pexConfig.Field(
246  dtype=bool,
247  default=True,
248  doc="Generate sky sources?",
249  )
250  skySources = pexConfig.ConfigurableField(
251  target=SkyObjectsTask,
252  doc="Generate sky sources",
253  )
254  measurement = pexConfig.ConfigurableField(
255  target=SingleFrameMeasurementTask,
256  doc="Measure sources"
257  )
258  setPrimaryFlags = pexConfig.ConfigurableField(
259  target=SetPrimaryFlagsTask,
260  doc=("Set flags for primary source classification in single frame "
261  "processing. True if sources are not sky sources and not a parent.")
262  )
263  doApCorr = pexConfig.Field(
264  dtype=bool,
265  default=True,
266  doc="Run subtask to apply aperture correction"
267  )
268  applyApCorr = pexConfig.ConfigurableField(
269  target=ApplyApCorrTask,
270  doc="Subtask to apply aperture corrections"
271  )
272  # If doApCorr is False, and the exposure does not have apcorrections
273  # already applied, the active plugins in catalogCalculation almost
274  # certainly should not contain the characterization plugin
275  catalogCalculation = pexConfig.ConfigurableField(
276  target=CatalogCalculationTask,
277  doc="Subtask to run catalogCalculation plugins on catalog"
278  )
279  doInsertFakes = pexConfig.Field(
280  dtype=bool,
281  default=False,
282  doc="Run fake sources injection task"
283  )
284  insertFakes = pexConfig.ConfigurableField(
285  target=BaseFakeSourcesTask,
286  doc="Injection of fake sources for testing purposes (must be "
287  "retargeted)"
288  )
289  doWriteExposure = pexConfig.Field(
290  dtype=bool,
291  default=True,
292  doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a "
293  "normal calexp but as a fakes_calexp."
294  )
295 
296  def setDefaults(self):
297  super().setDefaults()
298  self.detection.doTempLocalBackground = False
299  self.deblend.maxFootprintSize = 2000
300  self.measurement.plugins.names |= ["base_LocalPhotoCalib", "base_LocalWcs"]
301 
302  def validate(self):
303  super().validate()
304  astromRefCatGen2 = getattr(self.astromRefObjLoader, "ref_dataset_name", None)
305  if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat:
306  raise ValueError(
307  f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference "
308  f"catalogs are different. These options must be kept in sync until Gen2 is retired."
309  )
310  photoRefCatGen2 = getattr(self.photoRefObjLoader, "ref_dataset_name", None)
311  if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat:
312  raise ValueError(
313  f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference "
314  f"catalogs are different. These options must be kept in sync until Gen2 is retired."
315  )
316 
317 
318 
324 
325 class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
326  r"""!Calibrate an exposure: measure sources and perform astrometric and
327  photometric calibration
328 
329  @anchor CalibrateTask_
330 
331  @section pipe_tasks_calibrate_Contents Contents
332 
333  - @ref pipe_tasks_calibrate_Purpose
334  - @ref pipe_tasks_calibrate_Initialize
335  - @ref pipe_tasks_calibrate_IO
336  - @ref pipe_tasks_calibrate_Config
337  - @ref pipe_tasks_calibrate_Metadata
338  - @ref pipe_tasks_calibrate_Debug
339 
340 
341  @section pipe_tasks_calibrate_Purpose Description
342 
343  Given an exposure with a good PSF model and aperture correction map
344  (e.g. as provided by @ref CharacterizeImageTask), perform the following
345  operations:
346  - Run detection and measurement
347  - Run astrometry subtask to fit an improved WCS
348  - Run photoCal subtask to fit the exposure's photometric zero-point
349 
350  @section pipe_tasks_calibrate_Initialize Task initialisation
351 
352  @copydoc \_\_init\_\_
353 
354  @section pipe_tasks_calibrate_IO Invoking the Task
355 
356  If you want this task to unpersist inputs or persist outputs, then call
357  the `runDataRef` method (a wrapper around the `run` method).
358 
359  If you already have the inputs unpersisted and do not want to persist the
360  output then it is more direct to call the `run` method:
361 
362  @section pipe_tasks_calibrate_Config Configuration parameters
363 
364  See @ref CalibrateConfig
365 
366  @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata
367 
368  Exposure metadata
369  <dl>
370  <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task
371  <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal
372  task
373  <dt>COLORTERM1 <dd>?? (always 0.0)
374  <dt>COLORTERM2 <dd>?? (always 0.0)
375  <dt>COLORTERM3 <dd>?? (always 0.0)
376  </dl>
377 
378  @section pipe_tasks_calibrate_Debug Debug variables
379 
380  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink
381  interface supports a flag
382  `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug
383  for more about `debug.py`.
384 
385  CalibrateTask has a debug dictionary containing one key:
386  <dl>
387  <dt>calibrate
388  <dd>frame (an int; <= 0 to not display) in which to display the exposure,
389  sources and matches. See @ref lsst.meas.astrom.displayAstrometry for
390  the meaning of the various symbols.
391  </dl>
392 
393  For example, put something like:
394  @code{.py}
395  import lsstDebug
396  def DebugInfo(name):
397  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would
398  # call us recursively
399  if name == "lsst.pipe.tasks.calibrate":
400  di.display = dict(
401  calibrate = 1,
402  )
403 
404  return di
405 
406  lsstDebug.Info = DebugInfo
407  @endcode
408  into your `debug.py` file and run `calibrateTask.py` with the `--debug`
409  flag.
410 
411  Some subtasks may have their own debug variables; see individual Task
412  documentation.
413  """
414 
415  # Example description used to live here, removed 2-20-2017 as per
416  # https://jira.lsstcorp.org/browse/DM-9520
417 
418  ConfigClass = CalibrateConfig
419  _DefaultName = "calibrate"
420  RunnerClass = pipeBase.ButlerInitializedTaskRunner
421 
422  def __init__(self, butler=None, astromRefObjLoader=None,
423  photoRefObjLoader=None, icSourceSchema=None,
424  initInputs=None, **kwargs):
425  """!Construct a CalibrateTask
426 
427  @param[in] butler The butler is passed to the refObjLoader constructor
428  in case it is needed. Ignored if the refObjLoader argument
429  provides a loader directly.
430  @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks
431  that supplies an external reference catalog for astrometric
432  calibration. May be None if the desired loader can be constructed
433  from the butler argument or all steps requiring a reference catalog
434  are disabled.
435  @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks
436  that supplies an external reference catalog for photometric
437  calibration. May be None if the desired loader can be constructed
438  from the butler argument or all steps requiring a reference catalog
439  are disabled.
440  @param[in] icSourceSchema schema for icSource catalog, or None.
441  Schema values specified in config.icSourceFieldsToCopy will be
442  taken from this schema. If set to None, no values will be
443  propagated from the icSourceCatalog
444  @param[in,out] kwargs other keyword arguments for
445  lsst.pipe.base.CmdLineTask
446  """
447  super().__init__(**kwargs)
448 
449  if icSourceSchema is None and butler is not None:
450  # Use butler to read icSourceSchema from disk.
451  icSourceSchema = butler.get("icSrc_schema", immediate=True).schema
452 
453  if icSourceSchema is None and butler is None and initInputs is not None:
454  icSourceSchema = initInputs['icSourceSchema'].schema
455 
456  if icSourceSchema is not None:
457  # use a schema mapper to avoid copying each field separately
458  self.schemaMapper = afwTable.SchemaMapper(icSourceSchema)
459  minimumSchema = afwTable.SourceTable.makeMinimalSchema()
460  self.schemaMapper.addMinimalSchema(minimumSchema, False)
461 
462  # Add fields to copy from an icSource catalog
463  # and a field to indicate that the source matched a source in that
464  # catalog. If any fields are missing then raise an exception, but
465  # first find all missing fields in order to make the error message
466  # more useful.
467  self.calibSourceKey = self.schemaMapper.addOutputField(
468  afwTable.Field["Flag"]("calib_detected",
469  "Source was detected as an icSource"))
470  missingFieldNames = []
471  for fieldName in self.config.icSourceFieldsToCopy:
472  try:
473  schemaItem = icSourceSchema.find(fieldName)
474  except Exception:
475  missingFieldNames.append(fieldName)
476  else:
477  # field found; if addMapping fails then raise an exception
478  self.schemaMapper.addMapping(schemaItem.getKey())
479 
480  if missingFieldNames:
481  raise RuntimeError("isSourceCat is missing fields {} "
482  "specified in icSourceFieldsToCopy"
483  .format(missingFieldNames))
484 
485  # produce a temporary schema to pass to the subtasks; finalize it
486  # later
487  self.schema = self.schemaMapper.editOutputSchema()
488  else:
489  self.schemaMapper = None
490  self.schema = afwTable.SourceTable.makeMinimalSchema()
491  self.makeSubtask('detection', schema=self.schema)
492 
493  self.algMetadata = dafBase.PropertyList()
494 
495  # Only create a subtask for fakes if configuration option is set
496  # N.B. the config for fake object task must be retargeted to a child
497  # of BaseFakeSourcesTask
498  if self.config.doInsertFakes:
499  self.makeSubtask("insertFakes")
500 
501  if self.config.doDeblend:
502  self.makeSubtask("deblend", schema=self.schema)
503  if self.config.doSkySources:
504  self.makeSubtask("skySources")
505  self.skySourceKey = self.schema.addField("sky_source", type="Flag", doc="Sky objects.")
506  self.makeSubtask('measurement', schema=self.schema,
507  algMetadata=self.algMetadata)
508  self.makeSubtask("setPrimaryFlags", schema=self.schema, isSingleFrame=True)
509  if self.config.doApCorr:
510  self.makeSubtask('applyApCorr', schema=self.schema)
511  self.makeSubtask('catalogCalculation', schema=self.schema)
512 
513  if self.config.doAstrometry:
514  if astromRefObjLoader is None and butler is not None:
515  self.makeSubtask('astromRefObjLoader', butler=butler)
516  astromRefObjLoader = self.astromRefObjLoader
517  self.pixelMargin = astromRefObjLoader.config.pixelMargin
518  self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader,
519  schema=self.schema)
520  if self.config.doPhotoCal:
521  if photoRefObjLoader is None and butler is not None:
522  self.makeSubtask('photoRefObjLoader', butler=butler)
523  photoRefObjLoader = self.photoRefObjLoader
524  self.pixelMargin = photoRefObjLoader.config.pixelMargin
525  self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader,
526  schema=self.schema)
527 
528  if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None):
529  raise RuntimeError("PipelineTask form of this task should not be initialized with "
530  "reference object loaders.")
531 
532  if self.schemaMapper is not None:
533  # finalize the schema
534  self.schema = self.schemaMapper.getOutputSchema()
535  self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
536 
537  sourceCatSchema = afwTable.SourceCatalog(self.schema)
538  sourceCatSchema.getTable().setMetadata(self.algMetadata)
539  self.outputSchema = sourceCatSchema
540 
541  @pipeBase.timeMethod
542  def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None,
543  doUnpersist=True):
544  """!Calibrate an exposure, optionally unpersisting inputs and
545  persisting outputs.
546 
547  This is a wrapper around the `run` method that unpersists inputs
548  (if `doUnpersist` true) and persists outputs (if `config.doWrite` true)
549 
550  @param[in] dataRef butler data reference corresponding to a science
551  image
552  @param[in,out] exposure characterized exposure (an
553  lsst.afw.image.ExposureF or similar), or None to unpersist existing
554  icExp and icBackground. See `run` method for details of what is
555  read and written.
556  @param[in,out] background initial model of background already
557  subtracted from exposure (an lsst.afw.math.BackgroundList). May be
558  None if no background has been subtracted, though that is unusual
559  for calibration. A refined background model is output. Ignored if
560  exposure is None.
561  @param[in] icSourceCat catalog from which to copy the fields specified
562  by icSourceKeys, or None;
563  @param[in] doUnpersist unpersist data:
564  - if True, exposure, background and icSourceCat are read from
565  dataRef and those three arguments must all be None;
566  - if False the exposure must be provided; background and
567  icSourceCat are optional. True is intended for running as a
568  command-line task, False for running as a subtask
569  @return same data as the calibrate method
570  """
571  self.log.info("Processing %s" % (dataRef.dataId))
572 
573  if doUnpersist:
574  if any(item is not None for item in (exposure, background,
575  icSourceCat)):
576  raise RuntimeError("doUnpersist true; exposure, background "
577  "and icSourceCat must all be None")
578  exposure = dataRef.get("icExp", immediate=True)
579  background = dataRef.get("icExpBackground", immediate=True)
580  icSourceCat = dataRef.get("icSrc", immediate=True)
581  elif exposure is None:
582  raise RuntimeError("doUnpersist false; exposure must be provided")
583 
584  exposureIdInfo = dataRef.get("expIdInfo")
585 
586  calRes = self.run(
587  exposure=exposure,
588  exposureIdInfo=exposureIdInfo,
589  background=background,
590  icSourceCat=icSourceCat,
591  )
592 
593  if self.config.doWrite:
594  self.writeOutputs(
595  dataRef=dataRef,
596  exposure=calRes.exposure,
597  background=calRes.background,
598  sourceCat=calRes.sourceCat,
599  astromMatches=calRes.astromMatches,
600  matchMeta=calRes.matchMeta,
601  )
602 
603  return calRes
604 
605  def runQuantum(self, butlerQC, inputRefs, outputRefs):
606  inputs = butlerQC.get(inputRefs)
607  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
608  returnMaxBits=True)
609  inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
610 
611  if self.config.doAstrometry:
612  refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
613  for ref in inputRefs.astromRefCat],
614  refCats=inputs.pop('astromRefCat'),
615  config=self.config.astromRefObjLoader, log=self.log)
616  self.pixelMargin = refObjLoader.config.pixelMargin
617  self.astrometry.setRefObjLoader(refObjLoader)
618 
619  if self.config.doPhotoCal:
620  photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
621  for ref in inputRefs.photoRefCat],
622  refCats=inputs.pop('photoRefCat'),
623  config=self.config.photoRefObjLoader,
624  log=self.log)
625  self.pixelMargin = photoRefObjLoader.config.pixelMargin
626  self.photoCal.match.setRefObjLoader(photoRefObjLoader)
627 
628  outputs = self.run(**inputs)
629 
630  if self.config.doWriteMatches and self.config.doAstrometry:
631  normalizedMatches = afwTable.packMatches(outputs.astromMatches)
632  normalizedMatches.table.setMetadata(outputs.matchMeta)
633  if self.config.doWriteMatchesDenormalized:
634  denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
635  outputs.matchesDenormalized = denormMatches
636  outputs.matches = normalizedMatches
637  butlerQC.put(outputs, outputRefs)
638 
639  def run(self, exposure, exposureIdInfo=None, background=None,
640  icSourceCat=None):
641  """!Calibrate an exposure (science image or coadd)
642 
643  @param[in,out] exposure exposure to calibrate (an
644  lsst.afw.image.ExposureF or similar);
645  in:
646  - MaskedImage
647  - Psf
648  out:
649  - MaskedImage has background subtracted
650  - Wcs is replaced
651  - PhotoCalib is replaced
652  @param[in] exposureIdInfo ID info for exposure (an
653  lsst.obs.base.ExposureIdInfo) If not provided, returned
654  SourceCatalog IDs will not be globally unique.
655  @param[in,out] background background model already subtracted from
656  exposure (an lsst.afw.math.BackgroundList). May be None if no
657  background has been subtracted, though that is unusual for
658  calibration. A refined background model is output.
659  @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
660  from which we can copy some fields.
661 
662  @return pipe_base Struct containing these fields:
663  - exposure calibrate science exposure with refined WCS and PhotoCalib
664  - background model of background subtracted from exposure (an
665  lsst.afw.math.BackgroundList)
666  - sourceCat catalog of measured sources
667  - astromMatches list of source/refObj matches from the astrometry
668  solver
669  """
670  # detect, deblend and measure sources
671  if exposureIdInfo is None:
672  exposureIdInfo = ExposureIdInfo()
673 
674  if background is None:
675  background = BackgroundList()
676  sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
677  exposureIdInfo.unusedBits)
678  table = SourceTable.make(self.schema, sourceIdFactory)
679  table.setMetadata(self.algMetadata)
680 
681  detRes = self.detection.run(table=table, exposure=exposure,
682  doSmooth=True)
683  sourceCat = detRes.sources
684  if detRes.fpSets.background:
685  for bg in detRes.fpSets.background:
686  background.append(bg)
687  if self.config.doSkySources:
688  skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId)
689  if skySourceFootprints:
690  for foot in skySourceFootprints:
691  s = sourceCat.addNew()
692  s.setFootprint(foot)
693  s.set(self.skySourceKey, True)
694  if self.config.doDeblend:
695  self.deblend.run(exposure=exposure, sources=sourceCat)
696  self.measurement.run(
697  measCat=sourceCat,
698  exposure=exposure,
699  exposureId=exposureIdInfo.expId
700  )
701  if self.config.doApCorr:
702  self.applyApCorr.run(
703  catalog=sourceCat,
704  apCorrMap=exposure.getInfo().getApCorrMap()
705  )
706  self.catalogCalculation.run(sourceCat)
707 
708  self.setPrimaryFlags.run(sourceCat, includeDeblend=self.config.doDeblend)
709 
710  if icSourceCat is not None and \
711  len(self.config.icSourceFieldsToCopy) > 0:
712  self.copyIcSourceFields(icSourceCat=icSourceCat,
713  sourceCat=sourceCat)
714 
715  # TODO DM-11568: this contiguous check-and-copy could go away if we
716  # reserve enough space during SourceDetection and/or SourceDeblend.
717  # NOTE: sourceSelectors require contiguous catalogs, so ensure
718  # contiguity now, so views are preserved from here on.
719  if not sourceCat.isContiguous():
720  sourceCat = sourceCat.copy(deep=True)
721 
722  # perform astrometry calibration:
723  # fit an improved WCS and update the exposure's WCS in place
724  astromMatches = None
725  matchMeta = None
726  if self.config.doAstrometry:
727  try:
728  astromRes = self.astrometry.run(
729  exposure=exposure,
730  sourceCat=sourceCat,
731  )
732  astromMatches = astromRes.matches
733  matchMeta = astromRes.matchMeta
734  except Exception as e:
735  if self.config.requireAstrometry:
736  raise
737  self.log.warn("Unable to perform astrometric calibration "
738  "(%s): attempting to proceed" % e)
739 
740  # compute photometric calibration
741  if self.config.doPhotoCal:
742  try:
743  photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
744  exposure.setPhotoCalib(photoRes.photoCalib)
745  # TODO: reword this to phrase it in terms of the calibration factor?
746  self.log.info("Photometric zero-point: %f" %
747  photoRes.photoCalib.instFluxToMagnitude(1.0))
748  self.setMetadata(exposure=exposure, photoRes=photoRes)
749  except Exception as e:
750  if self.config.requirePhotoCal:
751  raise
752  self.log.warn("Unable to perform photometric calibration "
753  "(%s): attempting to proceed" % e)
754  self.setMetadata(exposure=exposure, photoRes=None)
755 
756  if self.config.doInsertFakes:
757  self.insertFakes.run(exposure, background=background)
758 
759  table = SourceTable.make(self.schema, sourceIdFactory)
760  table.setMetadata(self.algMetadata)
761 
762  detRes = self.detection.run(table=table, exposure=exposure,
763  doSmooth=True)
764  sourceCat = detRes.sources
765  if detRes.fpSets.background:
766  for bg in detRes.fpSets.background:
767  background.append(bg)
768  if self.config.doDeblend:
769  self.deblend.run(exposure=exposure, sources=sourceCat)
770  self.measurement.run(
771  measCat=sourceCat,
772  exposure=exposure,
773  exposureId=exposureIdInfo.expId
774  )
775  if self.config.doApCorr:
776  self.applyApCorr.run(
777  catalog=sourceCat,
778  apCorrMap=exposure.getInfo().getApCorrMap()
779  )
780  self.catalogCalculation.run(sourceCat)
781 
782  if icSourceCat is not None and len(self.config.icSourceFieldsToCopy) > 0:
783  self.copyIcSourceFields(icSourceCat=icSourceCat,
784  sourceCat=sourceCat)
785 
786  frame = getDebugFrame(self._display, "calibrate")
787  if frame:
788  displayAstrometry(
789  sourceCat=sourceCat,
790  exposure=exposure,
791  matches=astromMatches,
792  frame=frame,
793  pause=False,
794  )
795 
796  return pipeBase.Struct(
797  exposure=exposure,
798  background=background,
799  sourceCat=sourceCat,
800  astromMatches=astromMatches,
801  matchMeta=matchMeta,
802  # These are duplicate entries with different names for use with
803  # gen3 middleware
804  outputExposure=exposure,
805  outputCat=sourceCat,
806  outputBackground=background,
807  )
808 
809  def writeOutputs(self, dataRef, exposure, background, sourceCat,
810  astromMatches, matchMeta):
811  """Write output data to the output repository
812 
813  @param[in] dataRef butler data reference corresponding to a science
814  image
815  @param[in] exposure exposure to write
816  @param[in] background background model for exposure
817  @param[in] sourceCat catalog of measured sources
818  @param[in] astromMatches list of source/refObj matches from the
819  astrometry solver
820  """
821  dataRef.put(sourceCat, "src")
822  if self.config.doWriteMatches and astromMatches is not None:
823  normalizedMatches = afwTable.packMatches(astromMatches)
824  normalizedMatches.table.setMetadata(matchMeta)
825  dataRef.put(normalizedMatches, "srcMatch")
826  if self.config.doWriteMatchesDenormalized:
827  denormMatches = denormalizeMatches(astromMatches, matchMeta)
828  dataRef.put(denormMatches, "srcMatchFull")
829  if self.config.doWriteExposure:
830  dataRef.put(exposure, "calexp")
831  dataRef.put(background, "calexpBackground")
832 
833  def getSchemaCatalogs(self):
834  """Return a dict of empty catalogs for each catalog dataset produced
835  by this task.
836  """
837  sourceCat = afwTable.SourceCatalog(self.schema)
838  sourceCat.getTable().setMetadata(self.algMetadata)
839  return {"src": sourceCat}
840 
841  def setMetadata(self, exposure, photoRes=None):
842  """!Set task and exposure metadata
843 
844  Logs a warning and continues if needed data is missing.
845 
846  @param[in,out] exposure exposure whose metadata is to be set
847  @param[in] photoRes results of running photoCal; if None then it was
848  not run
849  """
850  if photoRes is None:
851  return
852 
853  metadata = exposure.getMetadata()
854 
855  # convert zero-point to (mag/sec/adu) for task MAGZERO metadata
856  try:
857  exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
858  magZero = photoRes.zp - 2.5*math.log10(exposureTime)
859  except Exception:
860  self.log.warn("Could not set normalized MAGZERO in header: no "
861  "exposure time")
862  magZero = math.nan
863 
864  try:
865  metadata.set('MAGZERO', magZero)
866  metadata.set('MAGZERO_RMS', photoRes.sigma)
867  metadata.set('MAGZERO_NOBJ', photoRes.ngood)
868  metadata.set('COLORTERM1', 0.0)
869  metadata.set('COLORTERM2', 0.0)
870  metadata.set('COLORTERM3', 0.0)
871  except Exception as e:
872  self.log.warn("Could not set exposure metadata: %s" % (e,))
873 
874  def copyIcSourceFields(self, icSourceCat, sourceCat):
875  """!Match sources in icSourceCat and sourceCat and copy the specified fields
876 
877  @param[in] icSourceCat catalog from which to copy fields
878  @param[in,out] sourceCat catalog to which to copy fields
879 
880  The fields copied are those specified by `config.icSourceFieldsToCopy`
881  that actually exist in the schema. This was set up by the constructor
882  using self.schemaMapper.
883  """
884  if self.schemaMapper is None:
885  raise RuntimeError("To copy icSource fields you must specify "
886  "icSourceSchema nd icSourceKeys when "
887  "constructing this task")
888  if icSourceCat is None or sourceCat is None:
889  raise RuntimeError("icSourceCat and sourceCat must both be "
890  "specified")
891  if len(self.config.icSourceFieldsToCopy) == 0:
892  self.log.warn("copyIcSourceFields doing nothing because "
893  "icSourceFieldsToCopy is empty")
894  return
895 
896  mc = afwTable.MatchControl()
897  mc.findOnlyClosest = False # return all matched objects
898  matches = afwTable.matchXy(icSourceCat, sourceCat,
899  self.config.matchRadiusPix, mc)
900  if self.config.doDeblend:
901  deblendKey = sourceCat.schema["deblend_nChild"].asKey()
902  # if deblended, keep children
903  matches = [m for m in matches if m[1].get(deblendKey) == 0]
904 
905  # Because we had to allow multiple matches to handle parents, we now
906  # need to prune to the best matches
907  # closest matches as a dict of icSourceCat source ID:
908  # (icSourceCat source, sourceCat source, distance in pixels)
909  bestMatches = {}
910  for m0, m1, d in matches:
911  id0 = m0.getId()
912  match = bestMatches.get(id0)
913  if match is None or d <= match[2]:
914  bestMatches[id0] = (m0, m1, d)
915  matches = list(bestMatches.values())
916 
917  # Check that no sourceCat sources are listed twice (we already know
918  # that each match has a unique icSourceCat source ID, due to using
919  # that ID as the key in bestMatches)
920  numMatches = len(matches)
921  numUniqueSources = len(set(m[1].getId() for m in matches))
922  if numUniqueSources != numMatches:
923  self.log.warn("{} icSourceCat sources matched only {} sourceCat "
924  "sources".format(numMatches, numUniqueSources))
925 
926  self.log.info("Copying flags from icSourceCat to sourceCat for "
927  "%s sources" % (numMatches,))
928 
929  # For each match: set the calibSourceKey flag and copy the desired
930  # fields
931  for icSrc, src, d in matches:
932  src.setFlag(self.calibSourceKey, True)
933  # src.assign copies the footprint from icSrc, which we don't want
934  # (DM-407)
935  # so set icSrc's footprint to src's footprint before src.assign,
936  # then restore it
937  icSrcFootprint = icSrc.getFootprint()
938  try:
939  icSrc.setFootprint(src.getFootprint())
940  src.assign(icSrc, self.schemaMapper)
941  finally:
942  icSrc.setFootprint(icSrcFootprint)
lsst.pipe.tasks.calibrate.CalibrateConfig.measurement
measurement
Definition: calibrate.py:254
lsst.pipe.tasks.calibrate.CalibrateTask.skySourceKey
skySourceKey
Definition: calibrate.py:503
lsst.pipe.tasks.calibrate.CalibrateTask.runQuantum
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: calibrate.py:605
lsst::meas::astrom
lsst.pipe.tasks.calibrate.CalibrateConfig.setDefaults
def setDefaults(self)
Definition: calibrate.py:296
lsst.pipe.tasks.calibrate.CalibrateConnections.__init__
def __init__(self, *config=None)
Definition: calibrate.py:136
lsst.pipe.tasks.calibrate.CalibrateTask.outputSchema
outputSchema
Definition: calibrate.py:537
lsst.pipe.tasks.calibrate.CalibrateConfig
Definition: calibrate.py:150
lsst.pipe.tasks.calibrate.CalibrateTask.schemaMapper
schemaMapper
Definition: calibrate.py:456
lsst.pipe.tasks.calibrate.CalibrateTask
Calibrate an exposure: measure sources and perform astrometric and photometric calibration.
Definition: calibrate.py:325
lsst.pipe.tasks.calibrate.CalibrateTask.writeOutputs
def writeOutputs(self, dataRef, exposure, background, sourceCat, astromMatches, matchMeta)
Definition: calibrate.py:809
lsst.pipe.tasks.setPrimaryFlags
Definition: setPrimaryFlags.py:1
lsst.pipe.tasks.calibrate.CalibrateTask.pixelMargin
pixelMargin
Definition: calibrate.py:515
lsst.pipe.tasks.calibrate.CalibrateTask.copyIcSourceFields
def copyIcSourceFields(self, icSourceCat, sourceCat)
Match sources in icSourceCat and sourceCat and copy the specified fields.
Definition: calibrate.py:874
lsst::meas::base
lsst.pipe.tasks.calibrate.CalibrateConfig.astromRefObjLoader
astromRefObjLoader
Definition: calibrate.py:180
lsst.pipe.tasks.calibrate.CalibrateConfig.photoRefObjLoader
photoRefObjLoader
Definition: calibrate.py:184
lsst::afw::table
lsst::meas::deblender
lsst.pipe.tasks.calibrate.CalibrateConfig.validate
def validate(self)
Definition: calibrate.py:302
lsst::daf::base
lsst.pipe.tasks.calibrate.CalibrateTask.schema
schema
Definition: calibrate.py:485
lsst.pipe.tasks.calibrate.CalibrateTask.__init__
def __init__(self, butler=None, astromRefObjLoader=None, photoRefObjLoader=None, icSourceSchema=None, initInputs=None, **kwargs)
Construct a CalibrateTask.
Definition: calibrate.py:422
lsst.pipe.tasks.calibrate.CalibrateTask.calibSourceKey
calibSourceKey
Definition: calibrate.py:465
lsst::afw::math
lsst.pipe.tasks.calibrate.CalibrateTask.runDataRef
def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None, doUnpersist=True)
Calibrate an exposure, optionally unpersisting inputs and persisting outputs.
Definition: calibrate.py:542
lsst.pipe.tasks.calibrate.CalibrateConnections
Definition: calibrate.py:48
lsst.pipe.tasks.calibrate.CalibrateTask.getSchemaCatalogs
def getSchemaCatalogs(self)
Definition: calibrate.py:833
lsst.pipe.tasks.calibrate.CalibrateTask.setMetadata
def setMetadata(self, exposure, photoRes=None)
Set task and exposure metadata.
Definition: calibrate.py:841
lsst.pipe.tasks.calibrate.CalibrateConfig.deblend
deblend
Definition: calibrate.py:241
lsst.pipe::base
lsst::meas::algorithms
lsst.pipe.tasks.calibrate.CalibrateTask.algMetadata
algMetadata
Definition: calibrate.py:491
lsst.pipe::base::connectionTypes
lsst.pipe.tasks.calibrate.CalibrateTask.run
def run(self, exposure, exposureIdInfo=None, background=None, icSourceCat=None)
Calibrate an exposure (science image or coadd)
Definition: calibrate.py:639
lsst::obs::base
lsst.pipe.tasks.calibrate.CalibrateConfig.detection
detection
Definition: calibrate.py:232