lsst.ip.isr  14.0-2-g0af5a6c+23
isrFunctions.py
Go to the documentation of this file.
1 from __future__ import division, print_function, absolute_import
2 from builtins import input
3 from builtins import range
4 #
5 # LSST Data Management System
6 # Copyright 2008, 2009, 2010 LSST Corporation.
7 #
8 # This product includes software developed by the
9 # LSST Project (http://www.lsst.org/).
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the LSST License Statement and
22 # the GNU General Public License along with this program. If not,
23 # see <http://www.lsstcorp.org/LegalNotices/>.
24 #
25 import math
26 
27 import numpy
28 
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.image as afwImage
31 import lsst.afw.detection as afwDetection
32 import lsst.afw.math as afwMath
33 import lsst.meas.algorithms as measAlg
34 import lsst.pex.exceptions as pexExcept
35 
36 
37 def createPsf(fwhm):
38  """Make a double Gaussian PSF
39 
40  @param[in] fwhm FWHM of double Gaussian smoothing kernel
41  @return measAlg.DoubleGaussianPsf
42  """
43  ksize = 4*int(fwhm) + 1
44  return measAlg.DoubleGaussianPsf(ksize, ksize, fwhm/(2*math.sqrt(2*math.log(2))))
45 
46 def transposeMaskedImage(maskedImage):
47  """Make a transposed copy of a masked image
48 
49  @param[in] maskedImage afw.image.MaskedImage to process
50  @return transposed masked image
51  """
52  transposed = maskedImage.Factory(afwGeom.Extent2I(maskedImage.getHeight(), maskedImage.getWidth()))
53  transposed.getImage().getArray()[:] = maskedImage.getImage().getArray().T
54  transposed.getMask().getArray()[:] = maskedImage.getMask().getArray().T
55  transposed.getVariance().getArray()[:] = maskedImage.getVariance().getArray().T
56  return transposed
57 
58 
59 def interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=None):
60  """Interpolate over defects specified in a defect list
61 
62  @param[in,out] maskedImage masked image to process
63  @param[in] defectList defect list
64  @param[in] fwhm FWHM of double Gaussian smoothing kernel
65  @param[in] fallbackValue fallback value if an interpolated value cannot be determined;
66  if None then use clipped mean image value
67  """
68  psf = createPsf(fwhm)
69  if fallbackValue is None:
70  fallbackValue = afwMath.makeStatistics(maskedImage.getImage(), afwMath.MEANCLIP).getValue()
71  if 'INTRP' not in maskedImage.getMask().getMaskPlaneDict():
72  maskedImage.getMask.addMaskPlane('INTRP')
73  measAlg.interpolateOverDefects(maskedImage, psf, defectList, fallbackValue, True)
74 
75 
76 def defectListFromFootprintList(fpList, growFootprints=1):
77  """Compute a defect list from a footprint list, optionally growing the footprints
78 
79  @param[in] fpList footprint list
80  @param[in] growFootprints amount by which to grow footprints of detected regions
81  @return a list of defects (meas.algorithms.Defect)
82  """
83  defectList = []
84  for fp in fpList:
85  if growFootprints > 0:
86  # if "True", growing requires a convolution
87  # if "False", its faster
88  tempSpans = fp.spans.dilated(growFootprints,
89  afwGeom.Stencil.MANHATTAN)
90  fpGrow = afwDetection.Footprint(tempSpans, fp.getRegion())
91  else:
92  fpGrow = fp
93  for bbox in afwDetection.footprintToBBoxList(fpGrow):
94  defect = measAlg.Defect(bbox)
95  defectList.append(defect)
96  return defectList
97 
98 
99 def transposeDefectList(defectList):
100  """Make a transposed copy of a defect list
101 
102  @param[in] defectList a list of defects (afw.meas.algorithms.Defect)
103  @return a defect list with transposed defects
104  """
105  retDefectList = []
106  for defect in defectList:
107  bbox = defect.getBBox()
108  nbbox = afwGeom.Box2I(afwGeom.Point2I(bbox.getMinY(), bbox.getMinX()),
109  afwGeom.Extent2I(bbox.getDimensions()[1], bbox.getDimensions()[0]))
110  retDefectList.append(measAlg.Defect(nbbox))
111  return retDefectList
112 
113 
114 def maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD'):
115  """Set mask plane based on a defect list
116 
117  @param[in,out] maskedImage afw.image.MaskedImage to process; mask plane is updated
118  @param[in] defectList a list of defects (afw.meas.algorithms.Defect)
119  @param[in] maskName mask plane name
120  """
121  # mask bad pixels
122  mask = maskedImage.getMask()
123  bitmask = mask.getPlaneBitMask(maskName)
124  for defect in defectList:
125  bbox = defect.getBBox()
126  afwGeom.SpanSet(bbox).clippedTo(mask.getBBox()).setMask(mask, bitmask)
127 
128 
129 def getDefectListFromMask(maskedImage, maskName, growFootprints=1):
130  """Compute a defect list from a specified mask plane
131 
132  @param[in] maskedImage masked image to process
133  @param[in] maskName mask plane name, or list of names
134  @param[in] growFootprints amount by which to grow footprints of detected regions
135  @return a list of defects (each an meas.algrithms.Defect) of regions in mask
136  """
137  mask = maskedImage.getMask()
138  thresh = afwDetection.Threshold(mask.getPlaneBitMask(maskName), afwDetection.Threshold.BITMASK)
139  fpList = afwDetection.FootprintSet(mask, thresh).getFootprints()
140  return defectListFromFootprintList(fpList, growFootprints)
141 
142 
143 def makeThresholdMask(maskedImage, threshold, growFootprints=1, maskName='SAT'):
144  """Mask pixels based on threshold detection
145 
146  @param[in,out] maskedImage afw.image.MaskedImage to process; the mask is altered
147  @param[in] threshold detection threshold
148  @param[in] growFootprints amount by which to grow footprints of detected regions
149  @param[in] maskName mask plane name
150  @return a list of defects (meas.algrithms.Defect) of regions set in the mask.
151  """
152  # find saturated regions
153  thresh = afwDetection.Threshold(threshold)
154  fs = afwDetection.FootprintSet(maskedImage, thresh)
155 
156  if growFootprints > 0:
157  fs = afwDetection.FootprintSet(fs, growFootprints)
158 
159  fpList = fs.getFootprints()
160  # set mask
161  mask = maskedImage.getMask()
162  bitmask = mask.getPlaneBitMask(maskName)
163  afwDetection.setMaskFromFootprintList(mask, fpList, bitmask)
164 
165  return defectListFromFootprintList(fpList, growFootprints=0)
166 
167 
168 def interpolateFromMask(maskedImage, fwhm, growFootprints=1, maskName='SAT', fallbackValue=None):
169  """Interpolate over defects identified by a particular mask plane
170 
171  @param[in,out] maskedImage afw.image.MaskedImage to process
172  @param[in] fwhm FWHM of double Gaussian smoothing kernel
173  @param[in] growFootprints amount by which to grow footprints of detected regions
174  @param[in] maskName mask plane name
175  @param[in] fallbackValue value of last resort for interpolation
176  """
177  defectList = getDefectListFromMask(maskedImage, maskName, growFootprints)
178  interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=fallbackValue)
179 
180 
181 def saturationCorrection(maskedImage, saturation, fwhm, growFootprints=1, interpolate=True, maskName='SAT',
182  fallbackValue=None):
183  """Mark saturated pixels and optionally interpolate over them
184 
185  @param[in,out] maskedImage afw.image.MaskedImage to process
186  @param[in] saturation saturation level (used as a detection threshold)
187  @param[in] fwhm FWHM of double Gaussian smoothing kernel
188  @param[in] growFootprints amount by which to grow footprints of detected regions
189  @param[in] interpolate interpolate over saturated pixels?
190  @param[in] maskName mask plane name
191  @param[in] fallbackValue value of last resort for interpolation
192  """
193  defectList = makeThresholdMask(
194  maskedImage=maskedImage,
195  threshold=saturation,
196  growFootprints=growFootprints,
197  maskName=maskName,
198  )
199  if interpolate:
200  interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=fallbackValue)
201 
202 
203 def biasCorrection(maskedImage, biasMaskedImage):
204  """Apply bias correction in place
205 
206  @param[in,out] maskedImage masked image to correct
207  @param[in] biasMaskedImage bias, as a masked image
208  """
209  if maskedImage.getBBox(afwImage.LOCAL) != biasMaskedImage.getBBox(afwImage.LOCAL):
210  raise RuntimeError("maskedImage bbox %s != biasMaskedImage bbox %s" %
211  (maskedImage.getBBox(afwImage.LOCAL), biasMaskedImage.getBBox(afwImage.LOCAL)))
212  maskedImage -= biasMaskedImage
213 
214 
215 def darkCorrection(maskedImage, darkMaskedImage, expScale, darkScale):
216  """Apply dark correction in place
217 
218  maskedImage -= dark * expScaling / darkScaling
219 
220  @param[in,out] maskedImage afw.image.MaskedImage to correct
221  @param[in] darkMaskedImage dark afw.image.MaskedImage
222  @param[in] expScale exposure scale
223  @param[in] darkScale dark scale
224  """
225  if maskedImage.getBBox(afwImage.LOCAL) != darkMaskedImage.getBBox(afwImage.LOCAL):
226  raise RuntimeError("maskedImage bbox %s != darkMaskedImage bbox %s" %
227  (maskedImage.getBBox(afwImage.LOCAL), darkMaskedImage.getBBox(afwImage.LOCAL)))
228 
229  scale = expScale / darkScale
230  maskedImage.scaledMinus(scale, darkMaskedImage)
231 
232 
233 def updateVariance(maskedImage, gain, readNoise):
234  """Set the variance plane based on the image plane
235 
236  @param[in,out] maskedImage afw.image.MaskedImage; image plane is read and variance plane is written
237  @param[in] gain amplifier gain (e-/ADU)
238  @param[in] readNoise amplifier read noise (ADU/pixel)
239  """
240  var = maskedImage.getVariance()
241  var[:] = maskedImage.getImage()
242  var /= gain
243  var += readNoise**2
244 
245 
246 def flatCorrection(maskedImage, flatMaskedImage, scalingType, userScale=1.0):
247  """Apply flat correction in place
248 
249  @param[in,out] maskedImage afw.image.MaskedImage to correct
250  @param[in] flatMaskedImage flat field afw.image.MaskedImage
251  @param[in] scalingType how to compute flat scale; one of 'MEAN', 'MEDIAN' or 'USER'
252  @param[in] userScale scale to use if scalingType is 'USER', else ignored
253  """
254  if maskedImage.getBBox(afwImage.LOCAL) != flatMaskedImage.getBBox(afwImage.LOCAL):
255  raise RuntimeError("maskedImage bbox %s != flatMaskedImage bbox %s" %
256  (maskedImage.getBBox(afwImage.LOCAL), flatMaskedImage.getBBox(afwImage.LOCAL)))
257 
258  # Figure out scale from the data
259  # Ideally the flats are normalized by the calibration product pipelin, but this allows some flexibility
260  # in the case that the flat is created by some other mechanism.
261  if scalingType == 'MEAN':
262  flatScale = afwMath.makeStatistics(flatMaskedImage.getImage(), afwMath.MEAN).getValue(afwMath.MEAN)
263  elif scalingType == 'MEDIAN':
264  flatScale = afwMath.makeStatistics(flatMaskedImage.getImage(),
265  afwMath.MEDIAN).getValue(afwMath.MEDIAN)
266  elif scalingType == 'USER':
267  flatScale = userScale
268  else:
269  raise pexExcept.Exception('%s : %s not implemented' % ("flatCorrection", scalingType))
270 
271  maskedImage.scaledDivides(1.0/flatScale, flatMaskedImage)
272 
273 
274 def illuminationCorrection(maskedImage, illumMaskedImage, illumScale):
275  """Apply illumination correction in place
276 
277  @param[in,out] maskedImage afw.image.MaskedImage to correct
278  @param[in] illumMaskedImage illumination correction masked image
279  @param[in] illumScale scale value for illumination correction
280  """
281  if maskedImage.getBBox(afwImage.LOCAL) != illumMaskedImage.getBBox(afwImage.LOCAL):
282  raise RuntimeError("maskedImage bbox %s != illumMaskedImage bbox %s" %
283  (maskedImage.getBBox(afwImage.LOCAL), illumMaskedImage.getBBox(afwImage.LOCAL)))
284 
285  maskedImage.scaledDivides(1./illumScale, illumMaskedImage)
286 
287 
288 def overscanCorrection(ampMaskedImage, overscanImage, fitType='MEDIAN', order=1, collapseRej=3.0,
289  statControl=None):
290  """Apply overscan correction in place
291 
292  @param[in,out] ampMaskedImage masked image to correct
293  @param[in] overscanImage overscan data as an afw.image.Image or afw.image.MaskedImage.
294  If a masked image is passed in the mask plane will be used
295  to constrain the fit of the bias level.
296  @param[in] fitType type of fit for overscan correction; one of:
297  - 'MEAN'
298  - 'MEDIAN'
299  - 'POLY' (ordinary polynomial)
300  - 'CHEB' (Chebyshev polynomial)
301  - 'LEG' (Legendre polynomial)
302  - 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE' (splines)
303  @param[in] order polynomial order or spline knots (ignored unless fitType
304  indicates a polynomial or spline)
305  @param[in] collapseRej Rejection threshold (sigma) for collapsing dimension of overscan
306  @param[in] statControl Statistics control object
307  """
308  ampImage = ampMaskedImage.getImage()
309  if statControl is None:
310  statControl = afwMath.StatisticsControl()
311  if fitType == 'MEAN':
312  offImage = afwMath.makeStatistics(overscanImage, afwMath.MEAN, statControl).getValue(afwMath.MEAN)
313  elif fitType == 'MEDIAN':
314  offImage = afwMath.makeStatistics(overscanImage, afwMath.MEDIAN, statControl).getValue(afwMath.MEDIAN)
315  elif fitType in ('POLY', 'CHEB', 'LEG', 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE'):
316  if hasattr(overscanImage, "getImage"):
317  biasArray = overscanImage.getImage().getArray()
318  biasArray = numpy.ma.masked_where(overscanImage.getMask().getArray() & statControl.getAndMask(),
319  biasArray)
320  else:
321  biasArray = overscanImage.getArray()
322  # Fit along the long axis, so collapse along each short row and fit the resulting array
323  shortInd = numpy.argmin(biasArray.shape)
324  if shortInd == 0:
325  # Convert to some 'standard' representation to make things easier
326  biasArray = numpy.transpose(biasArray)
327 
328  # Do a single round of clipping to weed out CR hits and signal leaking into the overscan
329  percentiles = numpy.percentile(biasArray, [25.0, 50.0, 75.0], axis=1)
330  medianBiasArr = percentiles[1]
331  stdevBiasArr = 0.74*(percentiles[2] - percentiles[0]) # robust stdev
332  diff = numpy.abs(biasArray - medianBiasArr[:, numpy.newaxis])
333  biasMaskedArr = numpy.ma.masked_where(diff > collapseRej*stdevBiasArr[:, numpy.newaxis], biasArray)
334  collapsed = numpy.mean(biasMaskedArr, axis=1)
335  if collapsed.mask.sum() > 0:
336  collapsed.data[collapsed.mask] = numpy.mean(biasArray.data[collapsed.mask], axis=1)
337  del biasArray, percentiles, stdevBiasArr, diff, biasMaskedArr
338 
339  if shortInd == 0:
340  collapsed = numpy.transpose(collapsed)
341 
342  num = len(collapsed)
343  indices = 2.0*numpy.arange(num)/float(num) - 1.0
344 
345  if fitType in ('POLY', 'CHEB', 'LEG'):
346  # A numpy polynomial
347  poly = numpy.polynomial
348  fitter, evaler = {"POLY": (poly.polynomial.polyfit, poly.polynomial.polyval),
349  "CHEB": (poly.chebyshev.chebfit, poly.chebyshev.chebval),
350  "LEG": (poly.legendre.legfit, poly.legendre.legval),
351  }[fitType]
352 
353  coeffs = fitter(indices, collapsed, order)
354  fitBiasArr = evaler(indices, coeffs)
355  elif 'SPLINE' in fitType:
356  # An afw interpolation
357  numBins = order
358  #
359  # numpy.histogram needs a real array for the mask, but numpy.ma "optimises" the case
360  # no-values-are-masked by replacing the mask array by a scalar, numpy.ma.nomask
361  #
362  # Issue DM-415
363  #
364  collapsedMask = collapsed.mask
365  try:
366  if collapsedMask == numpy.ma.nomask:
367  collapsedMask = numpy.array(len(collapsed)*[numpy.ma.nomask])
368  except ValueError: # If collapsedMask is an array the test fails [needs .all()]
369  pass
370 
371  numPerBin, binEdges = numpy.histogram(indices, bins=numBins,
372  weights=1-collapsedMask.astype(int))
373  # Binning is just a histogram, with weights equal to the values.
374  # Use a similar trick to get the bin centers (this deals with different numbers per bin).
375  values = numpy.histogram(indices, bins=numBins,
376  weights=collapsed.data*~collapsedMask)[0]/numPerBin
377  binCenters = numpy.histogram(indices, bins=numBins,
378  weights=indices*~collapsedMask)[0]/numPerBin
379  interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0],
380  values.astype(float)[numPerBin > 0],
381  afwMath.stringToInterpStyle(fitType))
382  fitBiasArr = numpy.array([interp.interpolate(i) for i in indices])
383 
384  import lsstDebug
385  if lsstDebug.Info(__name__).display:
386  import matplotlib.pyplot as plot
387  figure = plot.figure(1)
388  figure.clear()
389  axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
390  axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+')
391  if collapsedMask.sum() > 0:
392  axes.plot(indices[collapsedMask], collapsed.data[collapsedMask], 'b+')
393  axes.plot(indices, fitBiasArr, 'r-')
394  figure.show()
395  prompt = "Press Enter or c to continue [chp]... "
396  while True:
397  ans = input(prompt).lower()
398  if ans in ("", "c",):
399  break
400  if ans in ("p",):
401  import pdb
402  pdb.set_trace()
403  elif ans in ("h", ):
404  print("h[elp] c[ontinue] p[db]")
405  figure.close()
406 
407  offImage = ampImage.Factory(ampImage.getDimensions())
408  offArray = offImage.getArray()
409  if shortInd == 1:
410  offArray[:, :] = fitBiasArr[:, numpy.newaxis]
411  else:
412  offArray[:, :] = fitBiasArr[numpy.newaxis, :]
413 
414  # We don't trust any extrapolation: mask those pixels as SUSPECT
415  # This will occur when the top and or bottom edges of the overscan
416  # contain saturated values. The values will be extrapolated from
417  # the surrounding pixels, but we cannot entirely trust the value of
418  # the extrapolation, and will mark the image mask plane to flag the
419  # image as such.
420  mask = ampMaskedImage.getMask()
421  maskArray = mask.getArray() if shortInd == 1 else mask.getArray().transpose()
422  suspect = mask.getPlaneBitMask("SUSPECT")
423  try:
424  if collapsed.mask == numpy.ma.nomask:
425  # There is no mask, so the whole array is fine
426  pass
427  except ValueError: # If collapsed.mask is an array the test fails [needs .all()]
428  for low in range(num):
429  if not collapsed.mask[low]:
430  break
431  if low > 0:
432  maskArray[:low, :] |= suspect
433  for high in range(1, num):
434  if not collapsed.mask[-high]:
435  break
436  if high > 1:
437  maskArray[-high:, :] |= suspect
438 
439  else:
440  raise pexExcept.Exception('%s : %s an invalid overscan type' % \
441  ("overscanCorrection", fitType))
442  ampImage -= offImage
def illuminationCorrection(maskedImage, illumMaskedImage, illumScale)
def saturationCorrection(maskedImage, saturation, fwhm, growFootprints=1, interpolate=True, maskName='SAT', fallbackValue=None)
def transposeDefectList(defectList)
Definition: isrFunctions.py:99
def interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=None)
Definition: isrFunctions.py:59
def transposeMaskedImage(maskedImage)
Definition: isrFunctions.py:46
def defectListFromFootprintList(fpList, growFootprints=1)
Definition: isrFunctions.py:76
def biasCorrection(maskedImage, biasMaskedImage)
def interpolateFromMask(maskedImage, fwhm, growFootprints=1, maskName='SAT', fallbackValue=None)
def getDefectListFromMask(maskedImage, maskName, growFootprints=1)
def makeThresholdMask(maskedImage, threshold, growFootprints=1, maskName='SAT')
def overscanCorrection(ampMaskedImage, overscanImage, fitType='MEDIAN', order=1, collapseRej=3.0, statControl=None)
def flatCorrection(maskedImage, flatMaskedImage, scalingType, userScale=1.0)
def updateVariance(maskedImage, gain, readNoise)
def darkCorrection(maskedImage, darkMaskedImage, expScale, darkScale)
def maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD')