lsst.pipe.tasks  13.0-37-g58c8d4e+3
 All Classes Namespaces Files Functions Variables Groups Pages
snapCombine.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2016 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 from __future__ import absolute_import, division, print_function
23 import numpy as num
24 import lsst.pex.config as pexConfig
25 import lsst.daf.base as dafBase
26 import lsst.afw.image as afwImage
27 import lsst.afw.table as afwTable
28 import lsst.pipe.base as pipeBase
29 from lsstDebug import getDebugFrame
30 from lsst.afw.display import getDisplay
31 from lsst.coadd.utils import addToCoadd, setCoaddEdgeBits
32 from lsst.ip.diffim import SnapPsfMatchTask
33 from lsst.meas.algorithms import SourceDetectionTask
34 from lsst.meas.base import SingleFrameMeasurementTask
35 import lsst.meas.algorithms as measAlg
36 
37 from .repair import RepairTask
38 
39 
40 class InitialPsfConfig(pexConfig.Config):
41  """!Describes the initial PSF used for detection and measurement before we do PSF determination."""
42 
43  model = pexConfig.ChoiceField(
44  dtype=str,
45  doc="PSF model type",
46  default="SingleGaussian",
47  allowed={
48  "SingleGaussian": "Single Gaussian model",
49  "DoubleGaussian": "Double Gaussian model",
50  },
51  )
52  pixelScale = pexConfig.Field(
53  dtype=float,
54  doc="Pixel size (arcsec). Only needed if no Wcs is provided",
55  default=0.25,
56  )
57  fwhm = pexConfig.Field(
58  dtype=float,
59  doc="FWHM of PSF model (arcsec)",
60  default=1.0,
61  )
62  size = pexConfig.Field(
63  dtype=int,
64  doc="Size of PSF model (pixels)",
65  default=15,
66  )
67 
68 
69 class SnapCombineConfig(pexConfig.Config):
70  doRepair = pexConfig.Field(
71  dtype=bool,
72  doc="Repair images (CR reject and interpolate) before combining",
73  default=True,
74  )
75  repairPsfFwhm = pexConfig.Field(
76  dtype=float,
77  doc="Psf FWHM (pixels) used to detect CRs",
78  default=2.5,
79  )
80  doDiffIm = pexConfig.Field(
81  dtype=bool,
82  doc="Perform difference imaging before combining",
83  default=False,
84  )
85  doPsfMatch = pexConfig.Field(
86  dtype=bool,
87  doc="Perform PSF matching for difference imaging (ignored if doDiffIm false)",
88  default=True,
89  )
90  doMeasurement = pexConfig.Field(
91  dtype=bool,
92  doc="Measure difference sources (ignored if doDiffIm false)",
93  default=True,
94  )
95  badMaskPlanes = pexConfig.ListField(
96  dtype=str,
97  doc="Mask planes that, if set, the associated pixels are not included in the combined exposure; "
98  "DETECTED excludes cosmic rays",
99  default=("DETECTED",),
100  )
101  averageKeys = pexConfig.ListField(
102  dtype=str,
103  doc="List of float metadata keys to average when combining snaps, e.g. float positions and dates; "
104  "non-float data must be handled by overriding the fixMetadata method",
105  optional=True,
106 
107  )
108  sumKeys = pexConfig.ListField(
109  dtype=str,
110  doc="List of float or int metadata keys to sum when combining snaps, e.g. exposure time; "
111  "non-float, non-int data must be handled by overriding the fixMetadata method",
112  optional=True,
113  )
114 
115  repair = pexConfig.ConfigurableField(target=RepairTask, doc="")
116  diffim = pexConfig.ConfigurableField(target=SnapPsfMatchTask, doc="")
117  detection = pexConfig.ConfigurableField(target=SourceDetectionTask, doc="")
118  initialPsf = pexConfig.ConfigField(dtype=InitialPsfConfig, doc="")
119  measurement = pexConfig.ConfigurableField(target=SingleFrameMeasurementTask, doc="")
120 
121  def setDefaults(self):
122  self.detection.thresholdPolarity = "both"
123 
124  def validate(self):
125  if self.detection.thresholdPolarity != "both":
126  raise ValueError("detection.thresholdPolarity must be 'both' for SnapCombineTask")
127 
128 ## \addtogroup LSST_task_documentation
129 ## \{
130 ## \page SnapCombineTask
131 ## \ref SnapCombineTask_ "SnapCombineTask"
132 ## \copybrief SnapCombineTask
133 ## \}
134 
135 
136 class SnapCombineTask(pipeBase.Task):
137  """!
138  \anchor SnapCombineTask_
139 
140  \brief Combine snaps.
141 
142  \section pipe_tasks_snapcombine_Contents Contents
143 
144  - \ref pipe_tasks_snapcombine_Debug
145 
146  \section pipe_tasks_snapcombine_Debug Debug variables
147 
148  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
149  flag \c -d to import \b debug.py from your \c PYTHONPATH; see <a
150  href="http://lsst-web.ncsa.illinois.edu/~buildbot/doxygen/x_masterDoxyDoc/base_debug.html">
151  Using lsstDebug to control debugging output</a> for more about \b debug.py files.
152 
153  The available variables in SnapCombineTask are:
154  <DL>
155  <DT> \c display
156  <DD> A dictionary containing debug point names as keys with frame number as value. Valid keys are:
157  <DL>
158  <DT> repair0
159  <DD> Display the first snap after repairing.
160  <DT> repair1
161  <DD> Display the second snap after repairing.
162  </DL>
163  </DD>
164  </DL>
165  """
166  ConfigClass = SnapCombineConfig
167  _DefaultName = "snapCombine"
168 
169  def __init__(self, *args, **kwargs):
170  pipeBase.Task.__init__(self, *args, **kwargs)
171  self.makeSubtask("repair")
172  self.makeSubtask("diffim")
173  self.schema = afwTable.SourceTable.makeMinimalSchema()
174  self.algMetadata = dafBase.PropertyList()
175  self.makeSubtask("detection", schema=self.schema)
176  if self.config.doMeasurement:
177  self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata)
178 
179  @pipeBase.timeMethod
180  def run(self, snap0, snap1, defects=None):
181  """Combine two snaps
182 
183  @param[in] snap0: snapshot exposure 0
184  @param[in] snap1: snapshot exposure 1
185  @defects[in] defect list (for repair task)
186  @return a pipe_base Struct with fields:
187  - exposure: snap-combined exposure
188  - sources: detected sources, or None if detection not performed
189  """
190  # initialize optional outputs
191  sources = None
192 
193  if self.config.doRepair:
194  self.log.info("snapCombine repair")
195  psf = self.makeInitialPsf(snap0, fwhmPix=self.config.repairPsfFwhm)
196  snap0.setPsf(psf)
197  snap1.setPsf(psf)
198  self.repair.run(snap0, defects=defects, keepCRs=False)
199  self.repair.run(snap1, defects=defects, keepCRs=False)
200 
201  repair0frame = getDebugFrame(self._display, "repair0")
202  if repair0frame:
203  getDisplay(repair0frame).mtv(snap0)
204  repair1frame = getDebugFrame(self._display, "repair1")
205  if repair1frame:
206  getDisplay(repair1frame).mtv(snap1)
207 
208  if self.config.doDiffIm:
209  if self.config.doPsfMatch:
210  self.log.info("snapCombine psfMatch")
211  diffRet = self.diffim.run(snap0, snap1, "subtractExposures")
212  diffExp = diffRet.subtractedImage
213 
214  # Measure centroid and width of kernel; dependent on ticket #1980
215  # Useful diagnostic for the degree of astrometric shift between snaps.
216  diffKern = diffRet.psfMatchingKernel
217  width, height = diffKern.getDimensions()
218  # TBD...
219  #psfAttr = measAlg.PsfAttributes(diffKern, width//2, height//2)
220 
221  else:
222  diffExp = afwImage.ExposureF(snap0, True)
223  diffMi = diffExp.getMaskedImage()
224  diffMi -= snap1.getMaskedImage()
225 
226  psf = self.makeInitialPsf(snap0)
227  diffExp.setPsf(psf)
228  table = afwTable.SourceTable.make(self.schema)
229  table.setMetadata(self.algMetadata)
230  detRet = self.detection.makeSourceCatalog(table, diffExp)
231  sources = detRet.sources
232  fpSets = detRet.fpSets
233  if self.config.doMeasurement:
234  self.measurement.measure(diffExp, sources)
235 
236  mask0 = snap0.getMaskedImage().getMask()
237  mask1 = snap1.getMaskedImage().getMask()
238  fpSets.positive.setMask(mask0, "DETECTED")
239  fpSets.negative.setMask(mask1, "DETECTED")
240 
241  maskD = diffExp.getMaskedImage().getMask()
242  fpSets.positive.setMask(maskD, "DETECTED")
243  fpSets.negative.setMask(maskD, "DETECTED_NEGATIVE")
244 
245  combinedExp = self.addSnaps(snap0, snap1)
246 
247  return pipeBase.Struct(
248  exposure=combinedExp,
249  sources=sources,
250  )
251 
252  def addSnaps(self, snap0, snap1):
253  """Add two snap exposures together, returning a new exposure
254 
255  @param[in] snap0 snap exposure 0
256  @param[in] snap1 snap exposure 1
257  @return combined exposure
258  """
259  self.log.info("snapCombine addSnaps")
260 
261  combinedExp = snap0.Factory(snap0, True)
262  combinedMi = combinedExp.getMaskedImage()
263  combinedMi.set(0)
264 
265  weightMap = combinedMi.getImage().Factory(combinedMi.getBBox())
266  weight = 1.0
267  badPixelMask = afwImage.MaskU.getPlaneBitMask(self.config.badMaskPlanes)
268  addToCoadd(combinedMi, weightMap, snap0.getMaskedImage(), badPixelMask, weight)
269  addToCoadd(combinedMi, weightMap, snap1.getMaskedImage(), badPixelMask, weight)
270 
271  # pre-scaling the weight map instead of post-scaling the combinedMi saves a bit of time
272  # because the weight map is a simple Image instead of a MaskedImage
273  weightMap *= 0.5 # so result is sum of both images, instead of average
274  combinedMi /= weightMap
275  setCoaddEdgeBits(combinedMi.getMask(), weightMap)
276 
277  # note: none of the inputs has a valid Calib object, so that is not touched
278  # Filter was already copied
279 
280  combinedMetadata = combinedExp.getMetadata()
281  metadata0 = snap0.getMetadata()
282  metadata1 = snap1.getMetadata()
283  self.fixMetadata(combinedMetadata, metadata0, metadata1)
284 
285  return combinedExp
286 
287  def fixMetadata(self, combinedMetadata, metadata0, metadata1):
288  """Fix the metadata of the combined exposure (in place)
289 
290  This implementation handles items specified by config.averageKeys and config.sumKeys,
291  which have data type restrictions. To handle other data types (such as sexagesimal
292  positions and ISO dates) you must supplement this method with your own code.
293 
294  @param[in,out] combinedMetadata metadata of combined exposure;
295  on input this is a deep copy of metadata0 (a PropertySet)
296  @param[in] metadata0 metadata of snap0 (a PropertySet)
297  @param[in] metadata1 metadata of snap1 (a PropertySet)
298 
299  @note the inputs are presently PropertySets due to ticket #2542. However, in some sense
300  they are just PropertyLists that are missing some methods. In particular: comments and order
301  are preserved if you alter an existing value with set(key, value).
302  """
303  keyDoAvgList = []
304  if self.config.averageKeys:
305  keyDoAvgList += [(key, 1) for key in self.config.averageKeys]
306  if self.config.sumKeys:
307  keyDoAvgList += [(key, 0) for key in self.config.sumKeys]
308  for key, doAvg in keyDoAvgList:
309  opStr = "average" if doAvg else "sum"
310  try:
311  val0 = metadata0.get(key)
312  val1 = metadata1.get(key)
313  except Exception:
314  self.log.warn("Could not %s metadata %r: missing from one or both exposures" % (opStr, key,))
315  continue
316 
317  try:
318  combinedVal = val0 + val1
319  if doAvg:
320  combinedVal /= 2.0
321  except Exception:
322  self.log.warn("Could not %s metadata %r: value %r and/or %r not numeric" %
323  (opStr, key, val0, val1))
324  continue
325 
326  combinedMetadata.set(key, combinedVal)
327 
328  def makeInitialPsf(self, exposure, fwhmPix=None):
329  """Initialise the detection procedure by setting the PSF and WCS
330 
331  @param exposure Exposure to process
332  @return PSF, WCS
333  """
334  assert exposure, "No exposure provided"
335  wcs = exposure.getWcs()
336  assert wcs, "No wcs in exposure"
337 
338  if fwhmPix is None:
339  fwhmPix = self.config.initialPsf.fwhm / wcs.pixelScale().asArcseconds()
340 
341  size = self.config.initialPsf.size
342  model = self.config.initialPsf.model
343  self.log.info("installInitialPsf fwhm=%s pixels; size=%s pixels" % (fwhmPix, size))
344  psfCls = getattr(measAlg, model + "Psf")
345  psf = psfCls(size, size, fwhmPix/(2.0*num.sqrt(2*num.log(2.0))))
346  return psf
Describes the initial PSF used for detection and measurement before we do PSF determination.
Definition: snapCombine.py:40