lsst.pipe.tasks  16.0-13-g1e751bcc+10
characterizeImage.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2015 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import numpy as np
23 
24 from lsstDebug import getDebugFrame
25 import lsst.pex.config as pexConfig
26 import lsst.pipe.base as pipeBase
27 import lsst.daf.base as dafBase
28 from lsst.afw.math import BackgroundList
29 from lsst.afw.table import SourceTable, SourceCatalog, IdFactory
30 from lsst.meas.algorithms import SubtractBackgroundTask, SourceDetectionTask, MeasureApCorrTask
31 from lsst.meas.algorithms.installGaussianPsf import InstallGaussianPsfTask
32 from lsst.meas.astrom import RefMatchTask, displayAstrometry
33 from lsst.meas.extensions.astrometryNet import LoadAstrometryNetObjectsTask
34 from lsst.obs.base import ExposureIdInfo
35 from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
36 from lsst.meas.deblender import SourceDeblendTask
37 from .measurePsf import MeasurePsfTask
38 from .repair import RepairTask
39 
40 __all__ = ["CharacterizeImageConfig", "CharacterizeImageTask"]
41 
42 
43 class CharacterizeImageConfig(pexConfig.Config):
44  """!Config for CharacterizeImageTask"""
45  doMeasurePsf = pexConfig.Field(
46  dtype=bool,
47  default=True,
48  doc="Measure PSF? If False then for all subsequent operations use either existing PSF "
49  "model when present, or install simple PSF model when not (see installSimplePsf "
50  "config options)"
51  )
52  doWrite = pexConfig.Field(
53  dtype=bool,
54  default=True,
55  doc="Persist results?",
56  )
57  doWriteExposure = pexConfig.Field(
58  dtype=bool,
59  default=True,
60  doc="Write icExp and icExpBackground in addition to icSrc? Ignored if doWrite False.",
61  )
62  psfIterations = pexConfig.RangeField(
63  dtype=int,
64  default=2,
65  min=1,
66  doc="Number of iterations of detect sources, measure sources, "
67  "estimate PSF. If useSimplePsf is True then 2 should be plenty; "
68  "otherwise more may be wanted.",
69  )
70  background = pexConfig.ConfigurableField(
71  target=SubtractBackgroundTask,
72  doc="Configuration for initial background estimation",
73  )
74  detection = pexConfig.ConfigurableField(
75  target=SourceDetectionTask,
76  doc="Detect sources"
77  )
78  doDeblend = pexConfig.Field(
79  dtype=bool,
80  default=True,
81  doc="Run deblender input exposure"
82  )
83  deblend = pexConfig.ConfigurableField(
84  target=SourceDeblendTask,
85  doc="Split blended source into their components"
86  )
87  measurement = pexConfig.ConfigurableField(
88  target=SingleFrameMeasurementTask,
89  doc="Measure sources"
90  )
91  doApCorr = pexConfig.Field(
92  dtype=bool,
93  default=True,
94  doc="Run subtasks to measure and apply aperture corrections"
95  )
96  measureApCorr = pexConfig.ConfigurableField(
97  target=MeasureApCorrTask,
98  doc="Subtask to measure aperture corrections"
99  )
100  applyApCorr = pexConfig.ConfigurableField(
101  target=ApplyApCorrTask,
102  doc="Subtask to apply aperture corrections"
103  )
104  # If doApCorr is False, and the exposure does not have apcorrections already applied, the
105  # active plugins in catalogCalculation almost certainly should not contain the characterization plugin
106  catalogCalculation = pexConfig.ConfigurableField(
107  target=CatalogCalculationTask,
108  doc="Subtask to run catalogCalculation plugins on catalog"
109  )
110  useSimplePsf = pexConfig.Field(
111  dtype=bool,
112  default=True,
113  doc="Replace the existing PSF model with a simplified version that has the same sigma "
114  "at the start of each PSF determination iteration? Doing so makes PSF determination "
115  "converge more robustly and quickly.",
116  )
117  installSimplePsf = pexConfig.ConfigurableField(
118  target=InstallGaussianPsfTask,
119  doc="Install a simple PSF model",
120  )
121  refObjLoader = pexConfig.ConfigurableField(
122  target=LoadAstrometryNetObjectsTask,
123  doc="reference object loader",
124  )
125  ref_match = pexConfig.ConfigurableField(
126  target=RefMatchTask,
127  doc="Task to load and match reference objects. Only used if measurePsf can use matches. "
128  "Warning: matching will only work well if the initial WCS is accurate enough "
129  "to give good matches (roughly: good to 3 arcsec across the CCD).",
130  )
131  measurePsf = pexConfig.ConfigurableField(
132  target=MeasurePsfTask,
133  doc="Measure PSF",
134  )
135  repair = pexConfig.ConfigurableField(
136  target=RepairTask,
137  doc="Remove cosmic rays",
138  )
139  checkUnitsParseStrict = pexConfig.Field(
140  doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
141  dtype=str,
142  default="raise",
143  )
144 
145  def setDefaults(self):
146  pexConfig.Config.setDefaults(self)
147  # just detect bright stars; includeThresholdMultipler=10 seems large,
148  # but these are the values we have been using
149  self.detection.thresholdValue = 5.0
150  self.detection.includeThresholdMultiplier = 10.0
151  # do not deblend, as it makes a mess
152  self.doDeblend = False
153  # measure and apply aperture correction; note: measuring and applying aperture
154  # correction are disabled until the final measurement, after PSF is measured
155  self.doApCorr = True
156  # minimal set of measurements needed to determine PSF
157  self.measurement.plugins.names = [
158  "base_PixelFlags",
159  "base_SdssCentroid",
160  "base_SdssShape",
161  "base_GaussianFlux",
162  "base_PsfFlux",
163  "base_CircularApertureFlux",
164  ]
165 
166  def validate(self):
167  if self.doApCorr and not self.measurePsf:
168  raise RuntimeError("Must measure PSF to measure aperture correction, "
169  "because flags determined by PSF measurement are used to identify "
170  "sources used to measure aperture correction")
171 
172 
178 
179 
180 class CharacterizeImageTask(pipeBase.CmdLineTask):
181  """!Measure bright sources and use this to estimate background and PSF of an exposure
182 
183  @anchor CharacterizeImageTask_
184 
185  @section pipe_tasks_characterizeImage_Contents Contents
186 
187  - @ref pipe_tasks_characterizeImage_Purpose
188  - @ref pipe_tasks_characterizeImage_Initialize
189  - @ref pipe_tasks_characterizeImage_IO
190  - @ref pipe_tasks_characterizeImage_Config
191  - @ref pipe_tasks_characterizeImage_Debug
192 
193 
194  @section pipe_tasks_characterizeImage_Purpose Description
195 
196  Given an exposure with defects repaired (masked and interpolated over, e.g. as output by IsrTask):
197  - detect and measure bright sources
198  - repair cosmic rays
199  - measure and subtract background
200  - measure PSF
201 
202  @section pipe_tasks_characterizeImage_Initialize Task initialisation
203 
204  @copydoc \_\_init\_\_
205 
206  @section pipe_tasks_characterizeImage_IO Invoking the Task
207 
208  If you want this task to unpersist inputs or persist outputs, then call
209  the `runDataRef` method (a thin wrapper around the `run` method).
210 
211  If you already have the inputs unpersisted and do not want to persist the output
212  then it is more direct to call the `run` method:
213 
214  @section pipe_tasks_characterizeImage_Config Configuration parameters
215 
216  See @ref CharacterizeImageConfig
217 
218  @section pipe_tasks_characterizeImage_Debug Debug variables
219 
220  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a flag
221  `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug for more about `debug.py`.
222 
223  CharacterizeImageTask has a debug dictionary with the following keys:
224  <dl>
225  <dt>frame
226  <dd>int: if specified, the frame of first debug image displayed (defaults to 1)
227  <dt>repair_iter
228  <dd>bool; if True display image after each repair in the measure PSF loop
229  <dt>background_iter
230  <dd>bool; if True display image after each background subtraction in the measure PSF loop
231  <dt>measure_iter
232  <dd>bool; if True display image and sources at the end of each iteration of the measure PSF loop
233  See @ref lsst.meas.astrom.displayAstrometry for the meaning of the various symbols.
234  <dt>psf
235  <dd>bool; if True display image and sources after PSF is measured;
236  this will be identical to the final image displayed by measure_iter if measure_iter is true
237  <dt>repair
238  <dd>bool; if True display image and sources after final repair
239  <dt>measure
240  <dd>bool; if True display image and sources after final measurement
241  </dl>
242 
243  For example, put something like:
244  @code{.py}
245  import lsstDebug
246  def DebugInfo(name):
247  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
248  if name == "lsst.pipe.tasks.characterizeImage":
249  di.display = dict(
250  repair = True,
251  )
252 
253  return di
254 
255  lsstDebug.Info = DebugInfo
256  @endcode
257  into your `debug.py` file and run `calibrateTask.py` with the `--debug` flag.
258 
259  Some subtasks may have their own debug variables; see individual Task documentation.
260  """
261 
262  # Example description used to live here, removed 2-20-2017 by MSSG
263 
264  ConfigClass = CharacterizeImageConfig
265  _DefaultName = "characterizeImage"
266  RunnerClass = pipeBase.ButlerInitializedTaskRunner
267 
268  def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs):
269  """!Construct a CharacterizeImageTask
270 
271  @param[in] butler A butler object is passed to the refObjLoader constructor in case
272  it is needed to load catalogs. May be None if a catalog-based star selector is
273  not used, if the reference object loader constructor does not require a butler,
274  or if a reference object loader is passed directly via the refObjLoader argument.
275  @param[in] refObjLoader An instance of LoadReferenceObjectsTasks that supplies an
276  external reference catalog to a catalog-based star selector. May be None if a
277  catalog star selector is not used or the loader can be constructed from the
278  butler argument.
279  @param[in,out] schema initial schema (an lsst.afw.table.SourceTable), or None
280  @param[in,out] kwargs other keyword arguments for lsst.pipe.base.CmdLineTask
281  """
282  pipeBase.CmdLineTask.__init__(self, **kwargs)
283  if schema is None:
284  schema = SourceTable.makeMinimalSchema()
285  self.schema = schema
286  self.makeSubtask("background")
287  self.makeSubtask("installSimplePsf")
288  self.makeSubtask("repair")
289  self.makeSubtask("measurePsf", schema=self.schema)
290  if self.config.doMeasurePsf and self.measurePsf.usesMatches:
291  if not refObjLoader:
292  self.makeSubtask('refObjLoader', butler=butler)
293  refObjLoader = self.refObjLoader
294  self.makeSubtask("ref_match", refObjLoader=refObjLoader)
295  self.algMetadata = dafBase.PropertyList()
296  self.makeSubtask('detection', schema=self.schema)
297  if self.config.doDeblend:
298  self.makeSubtask("deblend", schema=self.schema)
299  self.makeSubtask('measurement', schema=self.schema, algMetadata=self.algMetadata)
300  if self.config.doApCorr:
301  self.makeSubtask('measureApCorr', schema=self.schema)
302  self.makeSubtask('applyApCorr', schema=self.schema)
303  self.makeSubtask('catalogCalculation', schema=self.schema)
304  self._initialFrame = getDebugFrame(self._display, "frame") or 1
305  self._frame = self._initialFrame
306  self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
307 
308  @pipeBase.timeMethod
309  def runDataRef(self, dataRef, exposure=None, background=None, doUnpersist=True):
310  """!Characterize a science image and, if wanted, persist the results
311 
312  This simply unpacks the exposure and passes it to the characterize method to do the work.
313 
314  @param[in] dataRef: butler data reference for science exposure
315  @param[in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF or similar).
316  If None then unpersist from "postISRCCD".
317  The following changes are made, depending on the config:
318  - set psf to the measured PSF
319  - set apCorrMap to the measured aperture correction
320  - subtract background
321  - interpolate over cosmic rays
322  - update detection and cosmic ray mask planes
323  @param[in,out] background initial model of background already subtracted from exposure
324  (an lsst.afw.math.BackgroundList). May be None if no background has been subtracted,
325  which is typical for image characterization.
326  A refined background model is output.
327  @param[in] doUnpersist if True the exposure is read from the repository
328  and the exposure and background arguments must be None;
329  if False the exposure must be provided.
330  True is intended for running as a command-line task, False for running as a subtask
331 
332  @return same data as the characterize method
333  """
334  self._frame = self._initialFrame # reset debug display frame
335  self.log.info("Processing %s" % (dataRef.dataId))
336 
337  if doUnpersist:
338  if exposure is not None or background is not None:
339  raise RuntimeError("doUnpersist true; exposure and background must be None")
340  exposure = dataRef.get("postISRCCD", immediate=True)
341  elif exposure is None:
342  raise RuntimeError("doUnpersist false; exposure must be provided")
343 
344  exposureIdInfo = dataRef.get("expIdInfo")
345 
346  charRes = self.run(
347  exposure=exposure,
348  exposureIdInfo=exposureIdInfo,
349  background=background,
350  )
351 
352  if self.config.doWrite:
353  dataRef.put(charRes.sourceCat, "icSrc")
354  if self.config.doWriteExposure:
355  dataRef.put(charRes.exposure, "icExp")
356  dataRef.put(charRes.background, "icExpBackground")
357 
358  return charRes
359 
360  @pipeBase.timeMethod
361  def run(self, exposure, exposureIdInfo=None, background=None):
362  """!Characterize a science image
363 
364  Peforms the following operations:
365  - Iterate the following config.psfIterations times, or once if config.doMeasurePsf false:
366  - detect and measure sources and estimate PSF (see detectMeasureAndEstimatePsf for details)
367  - interpolate over cosmic rays
368  - perform final measurement
369 
370  @param[in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF or similar).
371  The following changes are made:
372  - update or set psf
373  - set apCorrMap
374  - update detection and cosmic ray mask planes
375  - subtract background and interpolate over cosmic rays
376  @param[in] exposureIdInfo ID info for exposure (an lsst.obs.base.ExposureIdInfo).
377  If not provided, returned SourceCatalog IDs will not be globally unique.
378  @param[in,out] background initial model of background already subtracted from exposure
379  (an lsst.afw.math.BackgroundList). May be None if no background has been subtracted,
380  which is typical for image characterization.
381 
382  @return pipe_base Struct containing these fields, all from the final iteration
383  of detectMeasureAndEstimatePsf:
384  - exposure: characterized exposure; image is repaired by interpolating over cosmic rays,
385  mask is updated accordingly, and the PSF model is set
386  - sourceCat: detected sources (an lsst.afw.table.SourceCatalog)
387  - background: model of background subtracted from exposure (an lsst.afw.math.BackgroundList)
388  - psfCellSet: spatial cells of PSF candidates (an lsst.afw.math.SpatialCellSet)
389  """
390  self._frame = self._initialFrame # reset debug display frame
391 
392  if not self.config.doMeasurePsf and not exposure.hasPsf():
393  self.log.warn("Source catalog detected and measured with placeholder or default PSF")
394  self.installSimplePsf.run(exposure=exposure)
395 
396  if exposureIdInfo is None:
397  exposureIdInfo = ExposureIdInfo()
398 
399  # subtract an initial estimate of background level
400  background = self.background.run(exposure).background
401 
402  psfIterations = self.config.psfIterations if self.config.doMeasurePsf else 1
403  for i in range(psfIterations):
404  dmeRes = self.detectMeasureAndEstimatePsf(
405  exposure=exposure,
406  exposureIdInfo=exposureIdInfo,
407  background=background,
408  )
409 
410  psf = dmeRes.exposure.getPsf()
411  psfSigma = psf.computeShape().getDeterminantRadius()
412  psfDimensions = psf.computeImage().getDimensions()
413  medBackground = np.median(dmeRes.background.getImage().getArray())
414  self.log.info("iter %s; PSF sigma=%0.2f, dimensions=%s; median background=%0.2f" %
415  (i + 1, psfSigma, psfDimensions, medBackground))
416 
417  self.display("psf", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
418 
419  # perform final repair with final PSF
420  self.repair.run(exposure=dmeRes.exposure)
421  self.display("repair", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
422 
423  # perform final measurement with final PSF, including measuring and applying aperture correction,
424  # if wanted
425  self.measurement.run(measCat=dmeRes.sourceCat, exposure=dmeRes.exposure,
426  exposureId=exposureIdInfo.expId)
427  if self.config.doApCorr:
428  apCorrMap = self.measureApCorr.run(exposure=dmeRes.exposure, catalog=dmeRes.sourceCat).apCorrMap
429  dmeRes.exposure.getInfo().setApCorrMap(apCorrMap)
430  self.applyApCorr.run(catalog=dmeRes.sourceCat, apCorrMap=exposure.getInfo().getApCorrMap())
431  self.catalogCalculation.run(dmeRes.sourceCat)
432 
433  self.display("measure", exposure=dmeRes.exposure, sourceCat=dmeRes.sourceCat)
434 
435  return dmeRes
436 
437  @pipeBase.timeMethod
438  def detectMeasureAndEstimatePsf(self, exposure, exposureIdInfo, background):
439  """!Perform one iteration of detect, measure and estimate PSF
440 
441  Performs the following operations:
442  - if config.doMeasurePsf or not exposure.hasPsf():
443  - install a simple PSF model (replacing the existing one, if need be)
444  - interpolate over cosmic rays with keepCRs=True
445  - estimate background and subtract it from the exposure
446  - detect, deblend and measure sources, and subtract a refined background model;
447  - if config.doMeasurePsf:
448  - measure PSF
449 
450  @param[in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF or similar)
451  The following changes are made:
452  - update or set psf
453  - update detection and cosmic ray mask planes
454  - subtract background
455  @param[in] exposureIdInfo ID info for exposure (an lsst.obs_base.ExposureIdInfo)
456  @param[in,out] background initial model of background already subtracted from exposure
457  (an lsst.afw.math.BackgroundList).
458 
459  @return pipe_base Struct containing these fields, all from the final iteration
460  of detect sources, measure sources and estimate PSF:
461  - exposure characterized exposure; image is repaired by interpolating over cosmic rays,
462  mask is updated accordingly, and the PSF model is set
463  - sourceCat detected sources (an lsst.afw.table.SourceCatalog)
464  - background model of background subtracted from exposure (an lsst.afw.math.BackgroundList)
465  - psfCellSet spatial cells of PSF candidates (an lsst.afw.math.SpatialCellSet)
466  """
467  # install a simple PSF model, if needed or wanted
468  if not exposure.hasPsf() or (self.config.doMeasurePsf and self.config.useSimplePsf):
469  self.log.warn("Source catalog detected and measured with placeholder or default PSF")
470  self.installSimplePsf.run(exposure=exposure)
471 
472  # run repair, but do not interpolate over cosmic rays (do that elsewhere, with the final PSF model)
473  self.repair.run(exposure=exposure, keepCRs=True)
474  self.display("repair_iter", exposure=exposure)
475 
476  if background is None:
477  background = BackgroundList()
478 
479  sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId, exposureIdInfo.unusedBits)
480  table = SourceTable.make(self.schema, sourceIdFactory)
481  table.setMetadata(self.algMetadata)
482 
483  detRes = self.detection.run(table=table, exposure=exposure, doSmooth=True)
484  sourceCat = detRes.sources
485  if detRes.fpSets.background:
486  for bg in detRes.fpSets.background:
487  background.append(bg)
488 
489  if self.config.doDeblend:
490  self.deblend.run(exposure=exposure, sources=sourceCat)
491 
492  self.measurement.run(measCat=sourceCat, exposure=exposure, exposureId=exposureIdInfo.expId)
493 
494  measPsfRes = pipeBase.Struct(cellSet=None)
495  if self.config.doMeasurePsf:
496  if self.measurePsf.usesMatches:
497  matches = self.ref_match.loadAndMatch(exposure=exposure, sourceCat=sourceCat).matches
498  else:
499  matches = None
500  measPsfRes = self.measurePsf.run(exposure=exposure, sources=sourceCat, matches=matches,
501  expId=exposureIdInfo.expId)
502  self.display("measure_iter", exposure=exposure, sourceCat=sourceCat)
503 
504  return pipeBase.Struct(
505  exposure=exposure,
506  sourceCat=sourceCat,
507  background=background,
508  psfCellSet=measPsfRes.cellSet,
509  )
510 
511  def getSchemaCatalogs(self):
512  """Return a dict of empty catalogs for each catalog dataset produced by this task.
513  """
514  sourceCat = SourceCatalog(self.schema)
515  sourceCat.getTable().setMetadata(self.algMetadata)
516  return {"icSrc": sourceCat}
517 
518  def display(self, itemName, exposure, sourceCat=None):
519  """Display exposure and sources on next frame, if display of itemName has been requested
520 
521  @param[in] itemName name of item in debugInfo
522  @param[in] exposure exposure to display
523  @param[in] sourceCat source catalog to display
524  """
525  val = getDebugFrame(self._display, itemName)
526  if not val:
527  return
528 
529  displayAstrometry(exposure=exposure, sourceCat=sourceCat, frame=self._frame, pause=False)
530  self._frame += 1
def display(self, itemName, exposure, sourceCat=None)
def run(self, exposure, exposureIdInfo=None, background=None)
Characterize a science image.
Measure bright sources and use this to estimate background and PSF of an exposure.
def runDataRef(self, dataRef, exposure=None, background=None, doUnpersist=True)
Characterize a science image and, if wanted, persist the results.
def __init__(self, butler=None, refObjLoader=None, schema=None, kwargs)
Construct a CharacterizeImageTask.
def detectMeasureAndEstimatePsf(self, exposure, exposureIdInfo, background)
Perform one iteration of detect, measure and estimate PSF.