lsst.ip.isr  14.0-3-g27b2198
isrFunctions.py
Go to the documentation of this file.
1 from __future__ import division, print_function, absolute_import
2 from builtins import input
3 from builtins import range
4 #
5 # LSST Data Management System
6 # Copyright 2008, 2009, 2010 LSST Corporation.
7 #
8 # This product includes software developed by the
9 # LSST Project (http://www.lsst.org/).
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the LSST License Statement and
22 # the GNU General Public License along with this program. If not,
23 # see <http://www.lsstcorp.org/LegalNotices/>.
24 #
25 import math
26 
27 import numpy
28 
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.image as afwImage
31 import lsst.afw.detection as afwDetection
32 import lsst.afw.math as afwMath
33 import lsst.meas.algorithms as measAlg
34 import lsst.pex.exceptions as pexExcept
35 
36 
37 def createPsf(fwhm):
38  """Make a double Gaussian PSF
39 
40  @param[in] fwhm FWHM of double Gaussian smoothing kernel
41  @return measAlg.DoubleGaussianPsf
42  """
43  ksize = 4*int(fwhm) + 1
44  return measAlg.DoubleGaussianPsf(ksize, ksize, fwhm/(2*math.sqrt(2*math.log(2))))
45 
46 def transposeMaskedImage(maskedImage):
47  """Make a transposed copy of a masked image
48 
49  @param[in] maskedImage afw.image.MaskedImage to process
50  @return transposed masked image
51  """
52  transposed = maskedImage.Factory(afwGeom.Extent2I(maskedImage.getHeight(), maskedImage.getWidth()))
53  transposed.getImage().getArray()[:] = maskedImage.getImage().getArray().T
54  transposed.getMask().getArray()[:] = maskedImage.getMask().getArray().T
55  transposed.getVariance().getArray()[:] = maskedImage.getVariance().getArray().T
56  return transposed
57 
58 
59 def interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=None):
60  """Interpolate over defects specified in a defect list
61 
62  @param[in,out] maskedImage masked image to process
63  @param[in] defectList defect list
64  @param[in] fwhm FWHM of double Gaussian smoothing kernel
65  @param[in] fallbackValue fallback value if an interpolated value cannot be determined;
66  if None then use clipped mean image value
67  """
68  psf = createPsf(fwhm)
69  if fallbackValue is None:
70  fallbackValue = afwMath.makeStatistics(maskedImage.getImage(), afwMath.MEANCLIP).getValue()
71  if 'INTRP' not in maskedImage.getMask().getMaskPlaneDict():
72  maskedImage.getMask.addMaskPlane('INTRP')
73  measAlg.interpolateOverDefects(maskedImage, psf, defectList, fallbackValue, True)
74 
75 
77  """Compute a defect list from a footprint list, optionally growing the footprints
78 
79  @param[in] fpList footprint list
80  """
81  defectList = []
82  for fp in fpList:
83  for bbox in afwDetection.footprintToBBoxList(fp):
84  defect = measAlg.Defect(bbox)
85  defectList.append(defect)
86  return defectList
87 
88 
89 def transposeDefectList(defectList):
90  """Make a transposed copy of a defect list
91 
92  @param[in] defectList a list of defects (afw.meas.algorithms.Defect)
93  @return a defect list with transposed defects
94  """
95  retDefectList = []
96  for defect in defectList:
97  bbox = defect.getBBox()
98  nbbox = afwGeom.Box2I(afwGeom.Point2I(bbox.getMinY(), bbox.getMinX()),
99  afwGeom.Extent2I(bbox.getDimensions()[1], bbox.getDimensions()[0]))
100  retDefectList.append(measAlg.Defect(nbbox))
101  return retDefectList
102 
103 
104 def maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD'):
105  """Set mask plane based on a defect list
106 
107  @param[in,out] maskedImage afw.image.MaskedImage to process; mask plane is updated
108  @param[in] defectList a list of defects (afw.meas.algorithms.Defect)
109  @param[in] maskName mask plane name
110  """
111  # mask bad pixels
112  mask = maskedImage.getMask()
113  bitmask = mask.getPlaneBitMask(maskName)
114  for defect in defectList:
115  bbox = defect.getBBox()
116  afwGeom.SpanSet(bbox).clippedTo(mask.getBBox()).setMask(mask, bitmask)
117 
118 
119 def getDefectListFromMask(maskedImage, maskName):
120  """Compute a defect list from a specified mask plane
121 
122  @param[in] maskedImage masked image to process
123  @param[in] maskName mask plane name, or list of names
124  """
125  mask = maskedImage.getMask()
126  thresh = afwDetection.Threshold(mask.getPlaneBitMask(maskName), afwDetection.Threshold.BITMASK)
127  fpList = afwDetection.FootprintSet(mask, thresh).getFootprints()
128  return defectListFromFootprintList(fpList)
129 
130 
131 def makeThresholdMask(maskedImage, threshold, growFootprints=1, maskName='SAT'):
132  """Mask pixels based on threshold detection
133 
134  @param[in,out] maskedImage afw.image.MaskedImage to process; the mask is altered
135  @param[in] threshold detection threshold
136  @param[in] growFootprints amount by which to grow footprints of detected regions
137  @param[in] maskName mask plane name
138  @return a list of defects (meas.algrithms.Defect) of regions set in the mask.
139  """
140  # find saturated regions
141  thresh = afwDetection.Threshold(threshold)
142  fs = afwDetection.FootprintSet(maskedImage, thresh)
143 
144  if growFootprints > 0:
145  fs = afwDetection.FootprintSet(fs, growFootprints)
146 
147  fpList = fs.getFootprints()
148  # set mask
149  mask = maskedImage.getMask()
150  bitmask = mask.getPlaneBitMask(maskName)
151  afwDetection.setMaskFromFootprintList(mask, fpList, bitmask)
152 
153  return defectListFromFootprintList(fpList)
154 
155 
156 def interpolateFromMask(maskedImage, fwhm, growFootprints=1, maskName='SAT', fallbackValue=None):
157  """Interpolate over defects identified by a particular mask plane
158 
159  @param[in,out] maskedImage afw.image.MaskedImage to process
160  @param[in] fwhm FWHM of double Gaussian smoothing kernel
161  @param[in] growFootprints amount by which to grow footprints of detected regions
162  @param[in] maskName mask plane name
163  @param[in] fallbackValue value of last resort for interpolation
164  """
165  mask = maskedImage.getMask()
166  thresh = afwDetection.Threshold(mask.getPlaneBitMask(maskName), afwDetection.Threshold.BITMASK)
167  fpSet = afwDetection.FootprintSet(mask, thresh)
168  if growFootprints > 0:
169  fpSet = afwDetection.FootprintSet(fpSet, rGrow=growFootprints, isotropic=False)
170  # If we are interpolating over an area larger than the original masked region, we need
171  # to expand the original mask bit to the full area to explain why we interpolated there.
172  fpSet.setMask(mask, maskName)
173  defectList = defectListFromFootprintList(fpSet.getFootprints())
174  interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=fallbackValue)
175 
176 
177 def saturationCorrection(maskedImage, saturation, fwhm, growFootprints=1, interpolate=True, maskName='SAT',
178  fallbackValue=None):
179  """Mark saturated pixels and optionally interpolate over them
180 
181  @param[in,out] maskedImage afw.image.MaskedImage to process
182  @param[in] saturation saturation level (used as a detection threshold)
183  @param[in] fwhm FWHM of double Gaussian smoothing kernel
184  @param[in] growFootprints amount by which to grow footprints of detected regions
185  @param[in] interpolate interpolate over saturated pixels?
186  @param[in] maskName mask plane name
187  @param[in] fallbackValue value of last resort for interpolation
188  """
189  defectList = makeThresholdMask(
190  maskedImage=maskedImage,
191  threshold=saturation,
192  growFootprints=growFootprints,
193  maskName=maskName,
194  )
195  if interpolate:
196  interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=fallbackValue)
197 
198 
199 def biasCorrection(maskedImage, biasMaskedImage):
200  """Apply bias correction in place
201 
202  @param[in,out] maskedImage masked image to correct
203  @param[in] biasMaskedImage bias, as a masked image
204  """
205  if maskedImage.getBBox(afwImage.LOCAL) != biasMaskedImage.getBBox(afwImage.LOCAL):
206  raise RuntimeError("maskedImage bbox %s != biasMaskedImage bbox %s" %
207  (maskedImage.getBBox(afwImage.LOCAL), biasMaskedImage.getBBox(afwImage.LOCAL)))
208  maskedImage -= biasMaskedImage
209 
210 
211 def darkCorrection(maskedImage, darkMaskedImage, expScale, darkScale, invert=False):
212  """Apply dark correction in place
213 
214  maskedImage -= dark * expScaling / darkScaling
215 
216  @param[in,out] maskedImage afw.image.MaskedImage to correct
217  @param[in] darkMaskedImage dark afw.image.MaskedImage
218  @param[in] expScale exposure scale
219  @param[in] darkScale dark scale
220  @param[in] invert if True, remove the dark from an already-corrected image
221  """
222  if maskedImage.getBBox(afwImage.LOCAL) != darkMaskedImage.getBBox(afwImage.LOCAL):
223  raise RuntimeError("maskedImage bbox %s != darkMaskedImage bbox %s" %
224  (maskedImage.getBBox(afwImage.LOCAL), darkMaskedImage.getBBox(afwImage.LOCAL)))
225 
226  scale = expScale / darkScale
227  if not invert:
228  maskedImage.scaledMinus(scale, darkMaskedImage)
229  else:
230  maskedImage.scaledPlus(scale, darkMaskedImage)
231 
232 
233 def updateVariance(maskedImage, gain, readNoise):
234  """Set the variance plane based on the image plane
235 
236  @param[in,out] maskedImage afw.image.MaskedImage; image plane is read and variance plane is written
237  @param[in] gain amplifier gain (e-/ADU)
238  @param[in] readNoise amplifier read noise (ADU/pixel)
239  """
240  var = maskedImage.getVariance()
241  var[:] = maskedImage.getImage()
242  var /= gain
243  var += readNoise**2
244 
245 
246 def flatCorrection(maskedImage, flatMaskedImage, scalingType, userScale=1.0, invert=False):
247  """Apply flat correction in place
248 
249  @param[in,out] maskedImage afw.image.MaskedImage to correct
250  @param[in] flatMaskedImage flat field afw.image.MaskedImage
251  @param[in] scalingType how to compute flat scale; one of 'MEAN', 'MEDIAN' or 'USER'
252  @param[in] userScale scale to use if scalingType is 'USER', else ignored
253  @param[in] invert if True, unflatten an already-flattened image instead.
254  """
255  if maskedImage.getBBox(afwImage.LOCAL) != flatMaskedImage.getBBox(afwImage.LOCAL):
256  raise RuntimeError("maskedImage bbox %s != flatMaskedImage bbox %s" %
257  (maskedImage.getBBox(afwImage.LOCAL), flatMaskedImage.getBBox(afwImage.LOCAL)))
258 
259  # Figure out scale from the data
260  # Ideally the flats are normalized by the calibration product pipelin, but this allows some flexibility
261  # in the case that the flat is created by some other mechanism.
262  if scalingType == 'MEAN':
263  flatScale = afwMath.makeStatistics(flatMaskedImage.getImage(), afwMath.MEAN).getValue(afwMath.MEAN)
264  elif scalingType == 'MEDIAN':
265  flatScale = afwMath.makeStatistics(flatMaskedImage.getImage(),
266  afwMath.MEDIAN).getValue(afwMath.MEDIAN)
267  elif scalingType == 'USER':
268  flatScale = userScale
269  else:
270  raise pexExcept.Exception('%s : %s not implemented' % ("flatCorrection", scalingType))
271 
272  if not invert:
273  maskedImage.scaledDivides(1.0/flatScale, flatMaskedImage)
274  else:
275  maskedImage.scaledMultiplies(1.0/flatScale, flatMaskedImage)
276 
277 
278 def illuminationCorrection(maskedImage, illumMaskedImage, illumScale):
279  """Apply illumination correction in place
280 
281  @param[in,out] maskedImage afw.image.MaskedImage to correct
282  @param[in] illumMaskedImage illumination correction masked image
283  @param[in] illumScale scale value for illumination correction
284  """
285  if maskedImage.getBBox(afwImage.LOCAL) != illumMaskedImage.getBBox(afwImage.LOCAL):
286  raise RuntimeError("maskedImage bbox %s != illumMaskedImage bbox %s" %
287  (maskedImage.getBBox(afwImage.LOCAL), illumMaskedImage.getBBox(afwImage.LOCAL)))
288 
289  maskedImage.scaledDivides(1./illumScale, illumMaskedImage)
290 
291 
292 def overscanCorrection(ampMaskedImage, overscanImage, fitType='MEDIAN', order=1, collapseRej=3.0,
293  statControl=None):
294  """Apply overscan correction in place
295 
296  @param[in,out] ampMaskedImage masked image to correct
297  @param[in] overscanImage overscan data as an afw.image.Image or afw.image.MaskedImage.
298  If a masked image is passed in the mask plane will be used
299  to constrain the fit of the bias level.
300  @param[in] fitType type of fit for overscan correction; one of:
301  - 'MEAN'
302  - 'MEDIAN'
303  - 'POLY' (ordinary polynomial)
304  - 'CHEB' (Chebyshev polynomial)
305  - 'LEG' (Legendre polynomial)
306  - 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE' (splines)
307  @param[in] order polynomial order or spline knots (ignored unless fitType
308  indicates a polynomial or spline)
309  @param[in] collapseRej Rejection threshold (sigma) for collapsing dimension of overscan
310  @param[in] statControl Statistics control object
311  """
312  ampImage = ampMaskedImage.getImage()
313  if statControl is None:
314  statControl = afwMath.StatisticsControl()
315  if fitType == 'MEAN':
316  offImage = afwMath.makeStatistics(overscanImage, afwMath.MEAN, statControl).getValue(afwMath.MEAN)
317  elif fitType == 'MEDIAN':
318  offImage = afwMath.makeStatistics(overscanImage, afwMath.MEDIAN, statControl).getValue(afwMath.MEDIAN)
319  elif fitType in ('POLY', 'CHEB', 'LEG', 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE'):
320  if hasattr(overscanImage, "getImage"):
321  biasArray = overscanImage.getImage().getArray()
322  biasArray = numpy.ma.masked_where(overscanImage.getMask().getArray() & statControl.getAndMask(),
323  biasArray)
324  else:
325  biasArray = overscanImage.getArray()
326  # Fit along the long axis, so collapse along each short row and fit the resulting array
327  shortInd = numpy.argmin(biasArray.shape)
328  if shortInd == 0:
329  # Convert to some 'standard' representation to make things easier
330  biasArray = numpy.transpose(biasArray)
331 
332  # Do a single round of clipping to weed out CR hits and signal leaking into the overscan
333  percentiles = numpy.percentile(biasArray, [25.0, 50.0, 75.0], axis=1)
334  medianBiasArr = percentiles[1]
335  stdevBiasArr = 0.74*(percentiles[2] - percentiles[0]) # robust stdev
336  diff = numpy.abs(biasArray - medianBiasArr[:, numpy.newaxis])
337  biasMaskedArr = numpy.ma.masked_where(diff > collapseRej*stdevBiasArr[:, numpy.newaxis], biasArray)
338  collapsed = numpy.mean(biasMaskedArr, axis=1)
339  if collapsed.mask.sum() > 0:
340  collapsed.data[collapsed.mask] = numpy.mean(biasArray.data[collapsed.mask], axis=1)
341  del biasArray, percentiles, stdevBiasArr, diff, biasMaskedArr
342 
343  if shortInd == 0:
344  collapsed = numpy.transpose(collapsed)
345 
346  num = len(collapsed)
347  indices = 2.0*numpy.arange(num)/float(num) - 1.0
348 
349  if fitType in ('POLY', 'CHEB', 'LEG'):
350  # A numpy polynomial
351  poly = numpy.polynomial
352  fitter, evaler = {"POLY": (poly.polynomial.polyfit, poly.polynomial.polyval),
353  "CHEB": (poly.chebyshev.chebfit, poly.chebyshev.chebval),
354  "LEG": (poly.legendre.legfit, poly.legendre.legval),
355  }[fitType]
356 
357  coeffs = fitter(indices, collapsed, order)
358  fitBiasArr = evaler(indices, coeffs)
359  elif 'SPLINE' in fitType:
360  # An afw interpolation
361  numBins = order
362  #
363  # numpy.histogram needs a real array for the mask, but numpy.ma "optimises" the case
364  # no-values-are-masked by replacing the mask array by a scalar, numpy.ma.nomask
365  #
366  # Issue DM-415
367  #
368  collapsedMask = collapsed.mask
369  try:
370  if collapsedMask == numpy.ma.nomask:
371  collapsedMask = numpy.array(len(collapsed)*[numpy.ma.nomask])
372  except ValueError: # If collapsedMask is an array the test fails [needs .all()]
373  pass
374 
375  numPerBin, binEdges = numpy.histogram(indices, bins=numBins,
376  weights=1-collapsedMask.astype(int))
377  # Binning is just a histogram, with weights equal to the values.
378  # Use a similar trick to get the bin centers (this deals with different numbers per bin).
379  values = numpy.histogram(indices, bins=numBins,
380  weights=collapsed.data*~collapsedMask)[0]/numPerBin
381  binCenters = numpy.histogram(indices, bins=numBins,
382  weights=indices*~collapsedMask)[0]/numPerBin
383  interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0],
384  values.astype(float)[numPerBin > 0],
385  afwMath.stringToInterpStyle(fitType))
386  fitBiasArr = numpy.array([interp.interpolate(i) for i in indices])
387 
388  import lsstDebug
389  if lsstDebug.Info(__name__).display:
390  import matplotlib.pyplot as plot
391  figure = plot.figure(1)
392  figure.clear()
393  axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
394  axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+')
395  if collapsedMask.sum() > 0:
396  axes.plot(indices[collapsedMask], collapsed.data[collapsedMask], 'b+')
397  axes.plot(indices, fitBiasArr, 'r-')
398  figure.show()
399  prompt = "Press Enter or c to continue [chp]... "
400  while True:
401  ans = input(prompt).lower()
402  if ans in ("", "c",):
403  break
404  if ans in ("p",):
405  import pdb
406  pdb.set_trace()
407  elif ans in ("h", ):
408  print("h[elp] c[ontinue] p[db]")
409  figure.close()
410 
411  offImage = ampImage.Factory(ampImage.getDimensions())
412  offArray = offImage.getArray()
413  if shortInd == 1:
414  offArray[:, :] = fitBiasArr[:, numpy.newaxis]
415  else:
416  offArray[:, :] = fitBiasArr[numpy.newaxis, :]
417 
418  # We don't trust any extrapolation: mask those pixels as SUSPECT
419  # This will occur when the top and or bottom edges of the overscan
420  # contain saturated values. The values will be extrapolated from
421  # the surrounding pixels, but we cannot entirely trust the value of
422  # the extrapolation, and will mark the image mask plane to flag the
423  # image as such.
424  mask = ampMaskedImage.getMask()
425  maskArray = mask.getArray() if shortInd == 1 else mask.getArray().transpose()
426  suspect = mask.getPlaneBitMask("SUSPECT")
427  try:
428  if collapsed.mask == numpy.ma.nomask:
429  # There is no mask, so the whole array is fine
430  pass
431  except ValueError: # If collapsed.mask is an array the test fails [needs .all()]
432  for low in range(num):
433  if not collapsed.mask[low]:
434  break
435  if low > 0:
436  maskArray[:low, :] |= suspect
437  for high in range(1, num):
438  if not collapsed.mask[-high]:
439  break
440  if high > 1:
441  maskArray[-high:, :] |= suspect
442 
443  else:
444  raise pexExcept.Exception('%s : %s an invalid overscan type' % \
445  ("overscanCorrection", fitType))
446  ampImage -= offImage
def darkCorrection(maskedImage, darkMaskedImage, expScale, darkScale, invert=False)
def illuminationCorrection(maskedImage, illumMaskedImage, illumScale)
def saturationCorrection(maskedImage, saturation, fwhm, growFootprints=1, interpolate=True, maskName='SAT', fallbackValue=None)
def transposeDefectList(defectList)
Definition: isrFunctions.py:89
def getDefectListFromMask(maskedImage, maskName)
def interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=None)
Definition: isrFunctions.py:59
def defectListFromFootprintList(fpList)
Definition: isrFunctions.py:76
def transposeMaskedImage(maskedImage)
Definition: isrFunctions.py:46
def biasCorrection(maskedImage, biasMaskedImage)
def interpolateFromMask(maskedImage, fwhm, growFootprints=1, maskName='SAT', fallbackValue=None)
def makeThresholdMask(maskedImage, threshold, growFootprints=1, maskName='SAT')
def overscanCorrection(ampMaskedImage, overscanImage, fitType='MEDIAN', order=1, collapseRej=3.0, statControl=None)
def flatCorrection(maskedImage, flatMaskedImage, scalingType, userScale=1.0, invert=False)
def updateVariance(maskedImage, gain, readNoise)
def maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD')