lsst.pipe.tasks  13.0-66-gfbf2f2ce+5
characterizeImage.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2015 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 from __future__ import absolute_import, division, print_function
23 from builtins import range
24 import numpy as np
25 
26 from lsstDebug import getDebugFrame
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 import lsst.daf.base as dafBase
30 from lsst.afw.math import BackgroundList
31 from lsst.afw.table import SourceTable, SourceCatalog, IdFactory
32 from lsst.meas.algorithms import SubtractBackgroundTask, SourceDetectionTask, MeasureApCorrTask
33 from lsst.meas.algorithms.installGaussianPsf import InstallGaussianPsfTask
34 from lsst.meas.astrom import RefMatchTask, displayAstrometry
35 from lsst.meas.extensions.astrometryNet import LoadAstrometryNetObjectsTask
36 from lsst.obs.base import ExposureIdInfo
37 from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
38 from lsst.meas.deblender import SourceDeblendTask
39 from .measurePsf import MeasurePsfTask
40 from .repair import RepairTask
41 
42 __all__ = ["CharacterizeImageConfig", "CharacterizeImageTask"]
43 
44 
45 class CharacterizeImageConfig(pexConfig.Config):
46  """!Config for CharacterizeImageTask"""
47  doMeasurePsf = pexConfig.Field(
48  dtype = bool,
49  default = True,
50  doc = "Measure PSF? If False then for all subsequent operations use either existing PSF "
51  "model when present, or install simple PSF model when not (see installSimplePsf "
52  "config options)"
53  )
54  doWrite = pexConfig.Field(
55  dtype = bool,
56  default = True,
57  doc = "Persist results?",
58  )
59  doWriteExposure = pexConfig.Field(
60  dtype = bool,
61  default = True,
62  doc = "Write icExp and icExpBackground in addition to icSrc? Ignored if doWrite False.",
63  )
64  psfIterations = pexConfig.RangeField(
65  dtype = int,
66  default = 2,
67  min = 1,
68  doc = "Number of iterations of detect sources, measure sources, "
69  "estimate PSF. If useSimplePsf is True then 2 should be plenty; "
70  "otherwise more may be wanted.",
71  )
72  background = pexConfig.ConfigurableField(
73  target = SubtractBackgroundTask,
74  doc = "Configuration for initial background estimation",
75  )
76  detection = pexConfig.ConfigurableField(
77  target = SourceDetectionTask,
78  doc = "Detect sources"
79  )
80  doDeblend = pexConfig.Field(
81  dtype = bool,
82  default = True,
83  doc = "Run deblender input exposure"
84  )
85  deblend = pexConfig.ConfigurableField(
86  target = SourceDeblendTask,
87  doc = "Split blended source into their components"
88  )
89  measurement = pexConfig.ConfigurableField(
90  target = SingleFrameMeasurementTask,
91  doc = "Measure sources"
92  )
93  doApCorr = pexConfig.Field(
94  dtype = bool,
95  default = True,
96  doc = "Run subtasks to measure and apply aperture corrections"
97  )
98  measureApCorr = pexConfig.ConfigurableField(
99  target = MeasureApCorrTask,
100  doc = "Subtask to measure aperture corrections"
101  )
102  applyApCorr = pexConfig.ConfigurableField(
103  target = ApplyApCorrTask,
104  doc = "Subtask to apply aperture corrections"
105  )
106  # If doApCorr is False, and the exposure does not have apcorrections already applied, the
107  # active plugins in catalogCalculation almost certainly should not contain the characterization plugin
108  catalogCalculation = pexConfig.ConfigurableField(
109  target = CatalogCalculationTask,
110  doc = "Subtask to run catalogCalculation plugins on catalog"
111  )
112  useSimplePsf = pexConfig.Field(
113  dtype = bool,
114  default = True,
115  doc = "Replace the existing PSF model with a simplified version that has the same sigma "
116  "at the start of each PSF determination iteration? Doing so makes PSF determination "
117  "converge more robustly and quickly.",
118  )
119  installSimplePsf = pexConfig.ConfigurableField(
120  target = InstallGaussianPsfTask,
121  doc = "Install a simple PSF model",
122  )
123  refObjLoader = pexConfig.ConfigurableField(
124  target = LoadAstrometryNetObjectsTask,
125  doc = "reference object loader",
126  )
127  ref_match = pexConfig.ConfigurableField(
128  target = RefMatchTask,
129  doc = "Task to load and match reference objects. Only used if measurePsf can use matches. "
130  "Warning: matching will only work well if the initial WCS is accurate enough "
131  "to give good matches (roughly: good to 3 arcsec across the CCD).",
132  )
133  measurePsf = pexConfig.ConfigurableField(
134  target = MeasurePsfTask,
135  doc = "Measure PSF",
136  )
137  repair = pexConfig.ConfigurableField(
138  target = RepairTask,
139  doc = "Remove cosmic rays",
140  )
141  checkUnitsParseStrict = pexConfig.Field(
142  doc = "Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
143  dtype = str,
144  default = "raise",
145  )
146 
147  def setDefaults(self):
148  pexConfig.Config.setDefaults(self)
149  # just detect bright stars; includeThresholdMultipler=10 seems large,
150  # but these are the values we have been using
151  self.detection.thresholdValue = 5.0
152  self.detection.includeThresholdMultiplier = 10.0
153  # do not deblend, as it makes a mess
154  self.doDeblend = False
155  # measure and apply aperture correction; note: measuring and applying aperture
156  # correction are disabled until the final measurement, after PSF is measured
157  self.doApCorr = True
158  # minimal set of measurements needed to determine PSF
159  self.measurement.plugins.names = [
160  "base_PixelFlags",
161  "base_SdssCentroid",
162  "base_SdssShape",
163  "base_GaussianFlux",
164  "base_PsfFlux",
165  "base_CircularApertureFlux",
166  ]
167 
168  def validate(self):
169  if self.doApCorr and not self.measurePsf:
170  raise RuntimeError("Must measure PSF to measure aperture correction, "
171  "because flags determined by PSF measurement are used to identify "
172  "sources used to measure aperture correction")
173 
174 
180 
181 
182 class CharacterizeImageTask(pipeBase.CmdLineTask):
183  """!Measure bright sources and use this to estimate background and PSF of an exposure
184 
185  @anchor CharacterizeImageTask_
186 
187  @section pipe_tasks_characterizeImage_Contents Contents
188 
189  - @ref pipe_tasks_characterizeImage_Purpose
190  - @ref pipe_tasks_characterizeImage_Initialize
191  - @ref pipe_tasks_characterizeImage_IO
192  - @ref pipe_tasks_characterizeImage_Config
193  - @ref pipe_tasks_characterizeImage_Debug
194 
195 
196  @section pipe_tasks_characterizeImage_Purpose Description
197 
198  Given an exposure with defects repaired (masked and interpolated over, e.g. as output by IsrTask):
199  - detect and measure bright sources
200  - repair cosmic rays
201  - measure and subtract background
202  - measure PSF
203 
204  @section pipe_tasks_characterizeImage_Initialize Task initialisation
205 
206  @copydoc \_\_init\_\_
207 
208  @section pipe_tasks_characterizeImage_IO Invoking the Task
209 
210  If you want this task to unpersist inputs or persist outputs, then call
211  the `run` method (a thin wrapper around the `characterize` method).
212 
213  If you already have the inputs unpersisted and do not want to persist the output
214  then it is more direct to call the `characterize` method:
215 
216  @section pipe_tasks_characterizeImage_Config Configuration parameters
217 
218  See @ref CharacterizeImageConfig
219 
220  @section pipe_tasks_characterizeImage_Debug Debug variables
221 
222  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a flag
223  `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug for more about `debug.py`.
224 
225  CharacterizeImageTask has a debug dictionary with the following keys:
226  <dl>
227  <dt>frame
228  <dd>int: if specified, the frame of first debug image displayed (defaults to 1)
229  <dt>repair_iter
230  <dd>bool; if True display image after each repair in the measure PSF loop
231  <dt>background_iter
232  <dd>bool; if True display image after each background subtraction in the measure PSF loop
233  <dt>measure_iter
234  <dd>bool; if True display image and sources at the end of each iteration of the measure PSF loop
235  See @ref lsst.meas.astrom.displayAstrometry for the meaning of the various symbols.
236  <dt>psf
237  <dd>bool; if True display image and sources after PSF is measured;
238  this will be identical to the final image displayed by measure_iter if measure_iter is true
239  <dt>repair
240  <dd>bool; if True display image and sources after final repair
241  <dt>measure
242  <dd>bool; if True display image and sources after final measurement
243  </dl>
244 
245  For example, put something like:
246  @code{.py}
247  import lsstDebug
248  def DebugInfo(name):
249  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
250  if name == "lsst.pipe.tasks.characterizeImage":
251  di.display = dict(
252  repair = True,
253  )
254 
255  return di
256 
257  lsstDebug.Info = DebugInfo
258  @endcode
259  into your `debug.py` file and run `calibrateTask.py` with the `--debug` flag.
260 
261  Some subtasks may have their own debug variables; see individual Task documentation.
262  """
263 
264  # Example description used to live here, removed 2-20-2017 by MSSG
265 
266  ConfigClass = CharacterizeImageConfig
267  _DefaultName = "characterizeImage"
268  RunnerClass = pipeBase.ButlerInitializedTaskRunner
269 
270  def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs):
271  """!Construct a CharacterizeImageTask
272 
273  @param[in] butler A butler object is passed to the refObjLoader constructor in case
274  it is needed to load catalogs. May be None if a catalog-based star selector is
275  not used, if the reference object loader constructor does not require a butler,
276  or if a reference object loader is passed directly via the refObjLoader argument.
277  @param[in] refObjLoader An instance of LoadReferenceObjectsTasks that supplies an
278  external reference catalog to a catalog-based star selector. May be None if a
279  catalog star selector is not used or the loader can be constructed from the
280  butler argument.
281  @param[in,out] schema initial schema (an lsst.afw.table.SourceTable), or None
282  @param[in,out] kwargs other keyword arguments for lsst.pipe.base.CmdLineTask
283  """
284  pipeBase.CmdLineTask.__init__(self, **kwargs)
285  if schema is None:
286  schema = SourceTable.makeMinimalSchema()
287  self.schema = schema
288  self.makeSubtask("background")
289  self.makeSubtask("installSimplePsf")
290  self.makeSubtask("repair")
291  self.makeSubtask("measurePsf", schema=self.schema)
292  if self.config.doMeasurePsf and self.measurePsf.usesMatches:
293  if not refObjLoader:
294  self.makeSubtask('refObjLoader', butler=butler)
295  refObjLoader = self.refObjLoader
296  self.makeSubtask("ref_match", refObjLoader=refObjLoader)
297  self.algMetadata = dafBase.PropertyList()
298  self.makeSubtask('detection', schema=self.schema)
299  if self.config.doDeblend:
300  self.makeSubtask("deblend", schema=self.schema)
301  self.makeSubtask('measurement', schema=self.schema, algMetadata=self.algMetadata)
302  if self.config.doApCorr:
303  self.makeSubtask('measureApCorr', schema=self.schema)
304  self.makeSubtask('applyApCorr', schema=self.schema)
305  self.makeSubtask('catalogCalculation', schema=self.schema)
306  self._initialFrame = getDebugFrame(self._display, "frame") or 1
307  self._frame = self._initialFrame
308  self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
309 
310  @pipeBase.timeMethod
311  def run(self, dataRef, exposure=None, background=None, doUnpersist=True):
312  """!Characterize a science image and, if wanted, persist the results
313 
314  This simply unpacks the exposure and passes it to the characterize method to do the work.
315 
316  @param[in] dataRef: butler data reference for science exposure
317  @param[in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF or similar).
318  If None then unpersist from "postISRCCD".
319  The following changes are made, depending on the config:
320  - set psf to the measured PSF
321  - set apCorrMap to the measured aperture correction
322  - subtract background
323  - interpolate over cosmic rays
324  - update detection and cosmic ray mask planes
325  @param[in,out] background initial model of background already subtracted from exposure
326  (an lsst.afw.math.BackgroundList). May be None if no background has been subtracted,
327  which is typical for image characterization.
328  A refined background model is output.
329  @param[in] doUnpersist if True the exposure is read from the repository
330  and the exposure and background arguments must be None;
331  if False the exposure must be provided.
332  True is intended for running as a command-line task, False for running as a subtask
333 
334  @return same data as the characterize method
335  """
336  self._frame = self._initialFrame # reset debug display frame
337  self.log.info("Processing %s" % (dataRef.dataId))
338 
339  if doUnpersist:
340  if exposure is not None or background is not None:
341  raise RuntimeError("doUnpersist true; exposure and background must be None")
342  exposure = dataRef.get("postISRCCD", immediate=True)
343  elif exposure is None:
344  raise RuntimeError("doUnpersist false; exposure must be provided")
345 
346  exposureIdInfo = dataRef.get("expIdInfo")
347 
348  charRes = self.characterize(
349  exposure = exposure,
350  exposureIdInfo = exposureIdInfo,
351  background = background,
352  )
353 
354  if self.config.doWrite:
355  dataRef.put(charRes.sourceCat, "icSrc")
356  if self.config.doWriteExposure:
357  dataRef.put(charRes.exposure, "icExp")
358  dataRef.put(charRes.background, "icExpBackground")
359 
360  return charRes
361 
362  @pipeBase.timeMethod
363  def characterize(self, exposure, exposureIdInfo=None, background=None):
364  """!Characterize a science image
365 
366  Peforms the following operations:
367  - Iterate the following config.psfIterations times, or once if config.doMeasurePsf false:
368  - detect and measure sources and estimate PSF (see detectMeasureAndEstimatePsf for details)
369  - interpolate over cosmic rays
370  - perform final measurement
371 
372  @param[in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF or similar).
373  The following changes are made:
374  - update or set psf
375  - set apCorrMap
376  - update detection and cosmic ray mask planes
377  - subtract background and interpolate over cosmic rays
378  @param[in] exposureIdInfo ID info for exposure (an lsst.obs.base.ExposureIdInfo).
379  If not provided, returned SourceCatalog IDs will not be globally unique.
380  @param[in,out] background initial model of background already subtracted from exposure
381  (an lsst.afw.math.BackgroundList). May be None if no background has been subtracted,
382  which is typical for image characterization.
383 
384  @return pipe_base Struct containing these fields, all from the final iteration
385  of detectMeasureAndEstimatePsf:
386  - exposure: characterized exposure; image is repaired by interpolating over cosmic rays,
387  mask is updated accordingly, and the PSF model is set
388  - sourceCat: detected sources (an lsst.afw.table.SourceCatalog)
389  - background: model of background subtracted from exposure (an lsst.afw.math.BackgroundList)
390  - psfCellSet: spatial cells of PSF candidates (an lsst.afw.math.SpatialCellSet)
391  """
392  self._frame = self._initialFrame # reset debug display frame
393 
394  if not self.config.doMeasurePsf and not exposure.hasPsf():
395  self.log.warn("Source catalog detected and measured with placeholder or default PSF")
396  self.installSimplePsf.run(exposure=exposure)
397 
398  if exposureIdInfo is None:
399  exposureIdInfo = ExposureIdInfo()
400 
401  # subtract an initial estimate of background level
402  background = self.background.run(exposure).background
403 
404  psfIterations = self.config.psfIterations if self.config.doMeasurePsf else 1
405  for i in range(psfIterations):
406  dmeRes = self.detectMeasureAndEstimatePsf(
407  exposure = exposure,
408  exposureIdInfo = exposureIdInfo,
409  background = background,
410  )
411 
412  psf = dmeRes.exposure.getPsf()
413  psfSigma = psf.computeShape().getDeterminantRadius()
414  psfDimensions = psf.computeImage().getDimensions()
415  medBackground = np.median(dmeRes.background.getImage().getArray())
416  self.log.info("iter %s; PSF sigma=%0.2f, dimensions=%s; median background=%0.2f" %
417  (i + 1, psfSigma, psfDimensions, medBackground))
418 
419  self.display("psf", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
420 
421  # perform final repair with final PSF
422  self.repair.run(exposure=dmeRes.exposure)
423  self.display("repair", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
424 
425  # perform final measurement with final PSF, including measuring and applying aperture correction,
426  # if wanted
427  self.measurement.run(measCat=dmeRes.sourceCat, exposure=dmeRes.exposure,
428  exposureId = exposureIdInfo.expId)
429  if self.config.doApCorr:
430  apCorrMap = self.measureApCorr.run(exposure=dmeRes.exposure, catalog=dmeRes.sourceCat).apCorrMap
431  dmeRes.exposure.getInfo().setApCorrMap(apCorrMap)
432  self.applyApCorr.run(catalog=dmeRes.sourceCat, apCorrMap=exposure.getInfo().getApCorrMap())
433  self.catalogCalculation.run(dmeRes.sourceCat)
434 
435  self.display("measure", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
436 
437  return dmeRes
438 
439  @pipeBase.timeMethod
440  def detectMeasureAndEstimatePsf(self, exposure, exposureIdInfo, background):
441  """!Perform one iteration of detect, measure and estimate PSF
442 
443  Performs the following operations:
444  - if config.doMeasurePsf or not exposure.hasPsf():
445  - install a simple PSF model (replacing the existing one, if need be)
446  - interpolate over cosmic rays with keepCRs=True
447  - estimate background and subtract it from the exposure
448  - detect, deblend and measure sources, and subtract a refined background model;
449  - if config.doMeasurePsf:
450  - measure PSF
451 
452  @param[in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF or similar)
453  The following changes are made:
454  - update or set psf
455  - update detection and cosmic ray mask planes
456  - subtract background
457  @param[in] exposureIdInfo ID info for exposure (an lsst.obs_base.ExposureIdInfo)
458  @param[in,out] background initial model of background already subtracted from exposure
459  (an lsst.afw.math.BackgroundList).
460 
461  @return pipe_base Struct containing these fields, all from the final iteration
462  of detect sources, measure sources and estimate PSF:
463  - exposure characterized exposure; image is repaired by interpolating over cosmic rays,
464  mask is updated accordingly, and the PSF model is set
465  - sourceCat detected sources (an lsst.afw.table.SourceCatalog)
466  - background model of background subtracted from exposure (an lsst.afw.math.BackgroundList)
467  - psfCellSet spatial cells of PSF candidates (an lsst.afw.math.SpatialCellSet)
468  """
469  # install a simple PSF model, if needed or wanted
470  if not exposure.hasPsf() or (self.config.doMeasurePsf and self.config.useSimplePsf):
471  self.log.warn("Source catalog detected and measured with placeholder or default PSF")
472  self.installSimplePsf.run(exposure=exposure)
473 
474  # run repair, but do not interpolate over cosmic rays (do that elsewhere, with the final PSF model)
475  self.repair.run(exposure=exposure, keepCRs=True)
476  self.display("repair_iter", exposure=exposure)
477 
478  if background is None:
479  background = BackgroundList()
480 
481  sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId, exposureIdInfo.unusedBits)
482  table = SourceTable.make(self.schema, sourceIdFactory)
483  table.setMetadata(self.algMetadata)
484 
485  detRes = self.detection.run(table=table, exposure=exposure, doSmooth=True)
486  sourceCat = detRes.sources
487  if detRes.fpSets.background:
488  background.append(detRes.fpSets.background)
489 
490  if self.config.doDeblend:
491  self.deblend.run(exposure=exposure, sources=sourceCat)
492 
493  self.measurement.run(measCat=sourceCat, exposure=exposure, exposureId=exposureIdInfo.expId)
494 
495  measPsfRes = pipeBase.Struct(cellSet=None)
496  if self.config.doMeasurePsf:
497  if self.measurePsf.usesMatches:
498  matches = self.ref_match.loadAndMatch(exposure=exposure, sourceCat=sourceCat).matches
499  else:
500  matches = None
501  measPsfRes = self.measurePsf.run(exposure=exposure, sources=sourceCat, matches=matches,
502  expId=exposureIdInfo.expId)
503  self.display("measure_iter", exposure=exposure, sourceCat=sourceCat)
504 
505  return pipeBase.Struct(
506  exposure = exposure,
507  sourceCat = sourceCat,
508  background = background,
509  psfCellSet = measPsfRes.cellSet,
510  )
511 
512  def getSchemaCatalogs(self):
513  """Return a dict of empty catalogs for each catalog dataset produced by this task.
514  """
515  sourceCat = SourceCatalog(self.schema)
516  sourceCat.getTable().setMetadata(self.algMetadata)
517  return {"icSrc": sourceCat}
518 
519  def display(self, itemName, exposure, sourceCat=None):
520  """Display exposure and sources on next frame, if display of itemName has been requested
521 
522  @param[in] itemName name of item in debugInfo
523  @param[in] exposure exposure to display
524  @param[in] sourceCat source catalog to display
525  """
526  val = getDebugFrame(self._display, itemName)
527  if not val:
528  return
529 
530  displayAstrometry(exposure=exposure, sourceCat=sourceCat, frame=self._frame, pause=False)
531  self._frame += 1
def display(self, itemName, exposure, sourceCat=None)
Measure bright sources and use this to estimate background and PSF of an exposure.
def characterize(self, exposure, exposureIdInfo=None, background=None)
Characterize a science image.
def run(self, dataRef, exposure=None, background=None, doUnpersist=True)
Characterize a science image and, if wanted, persist the results.
def __init__(self, butler=None, refObjLoader=None, schema=None, kwargs)
Construct a CharacterizeImageTask.
def detectMeasureAndEstimatePsf(self, exposure, exposureIdInfo, background)
Perform one iteration of detect, measure and estimate PSF.