lsst.ip.isr  14.0-6-g095d685
isrFunctions.py
Go to the documentation of this file.
1 from __future__ import division, print_function, absolute_import
2 from builtins import input
3 from builtins import range
4 #
5 # LSST Data Management System
6 # Copyright 2008, 2009, 2010 LSST Corporation.
7 #
8 # This product includes software developed by the
9 # LSST Project (http://www.lsst.org/).
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the LSST License Statement and
22 # the GNU General Public License along with this program. If not,
23 # see <http://www.lsstcorp.org/LegalNotices/>.
24 #
25 import math
26 
27 import numpy
28 
29 import lsst.afw.geom as afwGeom
30 import lsst.afw.image as afwImage
31 import lsst.afw.detection as afwDetection
32 import lsst.afw.math as afwMath
33 import lsst.meas.algorithms as measAlg
34 import lsst.pex.exceptions as pexExcept
35 import lsst.afw.cameraGeom as camGeom
36 
37 
38 def createPsf(fwhm):
39  """Make a double Gaussian PSF
40 
41  @param[in] fwhm FWHM of double Gaussian smoothing kernel
42  @return measAlg.DoubleGaussianPsf
43  """
44  ksize = 4*int(fwhm) + 1
45  return measAlg.DoubleGaussianPsf(ksize, ksize, fwhm/(2*math.sqrt(2*math.log(2))))
46 
47 def transposeMaskedImage(maskedImage):
48  """Make a transposed copy of a masked image
49 
50  @param[in] maskedImage afw.image.MaskedImage to process
51  @return transposed masked image
52  """
53  transposed = maskedImage.Factory(afwGeom.Extent2I(maskedImage.getHeight(), maskedImage.getWidth()))
54  transposed.getImage().getArray()[:] = maskedImage.getImage().getArray().T
55  transposed.getMask().getArray()[:] = maskedImage.getMask().getArray().T
56  transposed.getVariance().getArray()[:] = maskedImage.getVariance().getArray().T
57  return transposed
58 
59 
60 def interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=None):
61  """Interpolate over defects specified in a defect list
62 
63  @param[in,out] maskedImage masked image to process
64  @param[in] defectList defect list
65  @param[in] fwhm FWHM of double Gaussian smoothing kernel
66  @param[in] fallbackValue fallback value if an interpolated value cannot be determined;
67  if None then use clipped mean image value
68  """
69  psf = createPsf(fwhm)
70  if fallbackValue is None:
71  fallbackValue = afwMath.makeStatistics(maskedImage.getImage(), afwMath.MEANCLIP).getValue()
72  if 'INTRP' not in maskedImage.getMask().getMaskPlaneDict():
73  maskedImage.getMask.addMaskPlane('INTRP')
74  measAlg.interpolateOverDefects(maskedImage, psf, defectList, fallbackValue, True)
75 
76 
78  """Compute a defect list from a footprint list, optionally growing the footprints
79 
80  @param[in] fpList footprint list
81  """
82  defectList = []
83  for fp in fpList:
84  for bbox in afwDetection.footprintToBBoxList(fp):
85  defect = measAlg.Defect(bbox)
86  defectList.append(defect)
87  return defectList
88 
89 
90 def transposeDefectList(defectList):
91  """Make a transposed copy of a defect list
92 
93  @param[in] defectList a list of defects (afw.meas.algorithms.Defect)
94  @return a defect list with transposed defects
95  """
96  retDefectList = []
97  for defect in defectList:
98  bbox = defect.getBBox()
99  nbbox = afwGeom.Box2I(afwGeom.Point2I(bbox.getMinY(), bbox.getMinX()),
100  afwGeom.Extent2I(bbox.getDimensions()[1], bbox.getDimensions()[0]))
101  retDefectList.append(measAlg.Defect(nbbox))
102  return retDefectList
103 
104 
105 def maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD'):
106  """Set mask plane based on a defect list
107 
108  @param[in,out] maskedImage afw.image.MaskedImage to process; mask plane is updated
109  @param[in] defectList a list of defects (afw.meas.algorithms.Defect)
110  @param[in] maskName mask plane name
111  """
112  # mask bad pixels
113  mask = maskedImage.getMask()
114  bitmask = mask.getPlaneBitMask(maskName)
115  for defect in defectList:
116  bbox = defect.getBBox()
117  afwGeom.SpanSet(bbox).clippedTo(mask.getBBox()).setMask(mask, bitmask)
118 
119 
120 def getDefectListFromMask(maskedImage, maskName):
121  """Compute a defect list from a specified mask plane
122 
123  @param[in] maskedImage masked image to process
124  @param[in] maskName mask plane name, or list of names
125  """
126  mask = maskedImage.getMask()
127  thresh = afwDetection.Threshold(mask.getPlaneBitMask(maskName), afwDetection.Threshold.BITMASK)
128  fpList = afwDetection.FootprintSet(mask, thresh).getFootprints()
129  return defectListFromFootprintList(fpList)
130 
131 
132 def makeThresholdMask(maskedImage, threshold, growFootprints=1, maskName='SAT'):
133  """Mask pixels based on threshold detection
134 
135  @param[in,out] maskedImage afw.image.MaskedImage to process; the mask is altered
136  @param[in] threshold detection threshold
137  @param[in] growFootprints amount by which to grow footprints of detected regions
138  @param[in] maskName mask plane name
139  @return a list of defects (meas.algrithms.Defect) of regions set in the mask.
140  """
141  # find saturated regions
142  thresh = afwDetection.Threshold(threshold)
143  fs = afwDetection.FootprintSet(maskedImage, thresh)
144 
145  if growFootprints > 0:
146  fs = afwDetection.FootprintSet(fs, growFootprints)
147 
148  fpList = fs.getFootprints()
149  # set mask
150  mask = maskedImage.getMask()
151  bitmask = mask.getPlaneBitMask(maskName)
152  afwDetection.setMaskFromFootprintList(mask, fpList, bitmask)
153 
154  return defectListFromFootprintList(fpList)
155 
156 
157 def interpolateFromMask(maskedImage, fwhm, growFootprints=1, maskName='SAT', fallbackValue=None):
158  """Interpolate over defects identified by a particular mask plane
159 
160  @param[in,out] maskedImage afw.image.MaskedImage to process
161  @param[in] fwhm FWHM of double Gaussian smoothing kernel
162  @param[in] growFootprints amount by which to grow footprints of detected regions
163  @param[in] maskName mask plane name
164  @param[in] fallbackValue value of last resort for interpolation
165  """
166  mask = maskedImage.getMask()
167  thresh = afwDetection.Threshold(mask.getPlaneBitMask(maskName), afwDetection.Threshold.BITMASK)
168  fpSet = afwDetection.FootprintSet(mask, thresh)
169  if growFootprints > 0:
170  fpSet = afwDetection.FootprintSet(fpSet, rGrow=growFootprints, isotropic=False)
171  # If we are interpolating over an area larger than the original masked region, we need
172  # to expand the original mask bit to the full area to explain why we interpolated there.
173  fpSet.setMask(mask, maskName)
174  defectList = defectListFromFootprintList(fpSet.getFootprints())
175  interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=fallbackValue)
176 
177 
178 def saturationCorrection(maskedImage, saturation, fwhm, growFootprints=1, interpolate=True, maskName='SAT',
179  fallbackValue=None):
180  """Mark saturated pixels and optionally interpolate over them
181 
182  @param[in,out] maskedImage afw.image.MaskedImage to process
183  @param[in] saturation saturation level (used as a detection threshold)
184  @param[in] fwhm FWHM of double Gaussian smoothing kernel
185  @param[in] growFootprints amount by which to grow footprints of detected regions
186  @param[in] interpolate interpolate over saturated pixels?
187  @param[in] maskName mask plane name
188  @param[in] fallbackValue value of last resort for interpolation
189  """
190  defectList = makeThresholdMask(
191  maskedImage=maskedImage,
192  threshold=saturation,
193  growFootprints=growFootprints,
194  maskName=maskName,
195  )
196  if interpolate:
197  interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=fallbackValue)
198 
199 
200 def biasCorrection(maskedImage, biasMaskedImage):
201  """Apply bias correction in place
202 
203  @param[in,out] maskedImage masked image to correct
204  @param[in] biasMaskedImage bias, as a masked image
205  """
206  if maskedImage.getBBox(afwImage.LOCAL) != biasMaskedImage.getBBox(afwImage.LOCAL):
207  raise RuntimeError("maskedImage bbox %s != biasMaskedImage bbox %s" %
208  (maskedImage.getBBox(afwImage.LOCAL), biasMaskedImage.getBBox(afwImage.LOCAL)))
209  maskedImage -= biasMaskedImage
210 
211 
212 def darkCorrection(maskedImage, darkMaskedImage, expScale, darkScale, invert=False):
213  """Apply dark correction in place
214 
215  maskedImage -= dark * expScaling / darkScaling
216 
217  @param[in,out] maskedImage afw.image.MaskedImage to correct
218  @param[in] darkMaskedImage dark afw.image.MaskedImage
219  @param[in] expScale exposure scale
220  @param[in] darkScale dark scale
221  @param[in] invert if True, remove the dark from an already-corrected image
222  """
223  if maskedImage.getBBox(afwImage.LOCAL) != darkMaskedImage.getBBox(afwImage.LOCAL):
224  raise RuntimeError("maskedImage bbox %s != darkMaskedImage bbox %s" %
225  (maskedImage.getBBox(afwImage.LOCAL), darkMaskedImage.getBBox(afwImage.LOCAL)))
226 
227  scale = expScale / darkScale
228  if not invert:
229  maskedImage.scaledMinus(scale, darkMaskedImage)
230  else:
231  maskedImage.scaledPlus(scale, darkMaskedImage)
232 
233 
234 def updateVariance(maskedImage, gain, readNoise):
235  """Set the variance plane based on the image plane
236 
237  @param[in,out] maskedImage afw.image.MaskedImage; image plane is read and variance plane is written
238  @param[in] gain amplifier gain (e-/ADU)
239  @param[in] readNoise amplifier read noise (ADU/pixel)
240  """
241  var = maskedImage.getVariance()
242  var[:] = maskedImage.getImage()
243  var /= gain
244  var += readNoise**2
245 
246 
247 def flatCorrection(maskedImage, flatMaskedImage, scalingType, userScale=1.0, invert=False):
248  """Apply flat correction in place
249 
250  @param[in,out] maskedImage afw.image.MaskedImage to correct
251  @param[in] flatMaskedImage flat field afw.image.MaskedImage
252  @param[in] scalingType how to compute flat scale; one of 'MEAN', 'MEDIAN' or 'USER'
253  @param[in] userScale scale to use if scalingType is 'USER', else ignored
254  @param[in] invert if True, unflatten an already-flattened image instead.
255  """
256  if maskedImage.getBBox(afwImage.LOCAL) != flatMaskedImage.getBBox(afwImage.LOCAL):
257  raise RuntimeError("maskedImage bbox %s != flatMaskedImage bbox %s" %
258  (maskedImage.getBBox(afwImage.LOCAL), flatMaskedImage.getBBox(afwImage.LOCAL)))
259 
260  # Figure out scale from the data
261  # Ideally the flats are normalized by the calibration product pipelin, but this allows some flexibility
262  # in the case that the flat is created by some other mechanism.
263  if scalingType == 'MEAN':
264  flatScale = afwMath.makeStatistics(flatMaskedImage.getImage(), afwMath.MEAN).getValue(afwMath.MEAN)
265  elif scalingType == 'MEDIAN':
266  flatScale = afwMath.makeStatistics(flatMaskedImage.getImage(),
267  afwMath.MEDIAN).getValue(afwMath.MEDIAN)
268  elif scalingType == 'USER':
269  flatScale = userScale
270  else:
271  raise pexExcept.Exception('%s : %s not implemented' % ("flatCorrection", scalingType))
272 
273  if not invert:
274  maskedImage.scaledDivides(1.0/flatScale, flatMaskedImage)
275  else:
276  maskedImage.scaledMultiplies(1.0/flatScale, flatMaskedImage)
277 
278 
279 def illuminationCorrection(maskedImage, illumMaskedImage, illumScale):
280  """Apply illumination correction in place
281 
282  @param[in,out] maskedImage afw.image.MaskedImage to correct
283  @param[in] illumMaskedImage illumination correction masked image
284  @param[in] illumScale scale value for illumination correction
285  """
286  if maskedImage.getBBox(afwImage.LOCAL) != illumMaskedImage.getBBox(afwImage.LOCAL):
287  raise RuntimeError("maskedImage bbox %s != illumMaskedImage bbox %s" %
288  (maskedImage.getBBox(afwImage.LOCAL), illumMaskedImage.getBBox(afwImage.LOCAL)))
289 
290  maskedImage.scaledDivides(1./illumScale, illumMaskedImage)
291 
292 
293 def overscanCorrection(ampMaskedImage, overscanImage, fitType='MEDIAN', order=1, collapseRej=3.0,
294  statControl=None):
295  """Apply overscan correction in place
296 
297  @param[in,out] ampMaskedImage masked image to correct
298  @param[in] overscanImage overscan data as an afw.image.Image or afw.image.MaskedImage.
299  If a masked image is passed in the mask plane will be used
300  to constrain the fit of the bias level.
301  @param[in] fitType type of fit for overscan correction; one of:
302  - 'MEAN'
303  - 'MEDIAN'
304  - 'POLY' (ordinary polynomial)
305  - 'CHEB' (Chebyshev polynomial)
306  - 'LEG' (Legendre polynomial)
307  - 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE' (splines)
308  @param[in] order polynomial order or spline knots (ignored unless fitType
309  indicates a polynomial or spline)
310  @param[in] collapseRej Rejection threshold (sigma) for collapsing dimension of overscan
311  @param[in] statControl Statistics control object
312  """
313  ampImage = ampMaskedImage.getImage()
314  if statControl is None:
315  statControl = afwMath.StatisticsControl()
316  if fitType == 'MEAN':
317  offImage = afwMath.makeStatistics(overscanImage, afwMath.MEAN, statControl).getValue(afwMath.MEAN)
318  elif fitType == 'MEDIAN':
319  offImage = afwMath.makeStatistics(overscanImage, afwMath.MEDIAN, statControl).getValue(afwMath.MEDIAN)
320  elif fitType in ('POLY', 'CHEB', 'LEG', 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE'):
321  if hasattr(overscanImage, "getImage"):
322  biasArray = overscanImage.getImage().getArray()
323  biasArray = numpy.ma.masked_where(overscanImage.getMask().getArray() & statControl.getAndMask(),
324  biasArray)
325  else:
326  biasArray = overscanImage.getArray()
327  # Fit along the long axis, so collapse along each short row and fit the resulting array
328  shortInd = numpy.argmin(biasArray.shape)
329  if shortInd == 0:
330  # Convert to some 'standard' representation to make things easier
331  biasArray = numpy.transpose(biasArray)
332 
333  # Do a single round of clipping to weed out CR hits and signal leaking into the overscan
334  percentiles = numpy.percentile(biasArray, [25.0, 50.0, 75.0], axis=1)
335  medianBiasArr = percentiles[1]
336  stdevBiasArr = 0.74*(percentiles[2] - percentiles[0]) # robust stdev
337  diff = numpy.abs(biasArray - medianBiasArr[:, numpy.newaxis])
338  biasMaskedArr = numpy.ma.masked_where(diff > collapseRej*stdevBiasArr[:, numpy.newaxis], biasArray)
339  collapsed = numpy.mean(biasMaskedArr, axis=1)
340  if collapsed.mask.sum() > 0:
341  collapsed.data[collapsed.mask] = numpy.mean(biasArray.data[collapsed.mask], axis=1)
342  del biasArray, percentiles, stdevBiasArr, diff, biasMaskedArr
343 
344  if shortInd == 0:
345  collapsed = numpy.transpose(collapsed)
346 
347  num = len(collapsed)
348  indices = 2.0*numpy.arange(num)/float(num) - 1.0
349 
350  if fitType in ('POLY', 'CHEB', 'LEG'):
351  # A numpy polynomial
352  poly = numpy.polynomial
353  fitter, evaler = {"POLY": (poly.polynomial.polyfit, poly.polynomial.polyval),
354  "CHEB": (poly.chebyshev.chebfit, poly.chebyshev.chebval),
355  "LEG": (poly.legendre.legfit, poly.legendre.legval),
356  }[fitType]
357 
358  coeffs = fitter(indices, collapsed, order)
359  fitBiasArr = evaler(indices, coeffs)
360  elif 'SPLINE' in fitType:
361  # An afw interpolation
362  numBins = order
363  #
364  # numpy.histogram needs a real array for the mask, but numpy.ma "optimises" the case
365  # no-values-are-masked by replacing the mask array by a scalar, numpy.ma.nomask
366  #
367  # Issue DM-415
368  #
369  collapsedMask = collapsed.mask
370  try:
371  if collapsedMask == numpy.ma.nomask:
372  collapsedMask = numpy.array(len(collapsed)*[numpy.ma.nomask])
373  except ValueError: # If collapsedMask is an array the test fails [needs .all()]
374  pass
375 
376  numPerBin, binEdges = numpy.histogram(indices, bins=numBins,
377  weights=1-collapsedMask.astype(int))
378  # Binning is just a histogram, with weights equal to the values.
379  # Use a similar trick to get the bin centers (this deals with different numbers per bin).
380  values = numpy.histogram(indices, bins=numBins,
381  weights=collapsed.data*~collapsedMask)[0]/numPerBin
382  binCenters = numpy.histogram(indices, bins=numBins,
383  weights=indices*~collapsedMask)[0]/numPerBin
384  interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0],
385  values.astype(float)[numPerBin > 0],
386  afwMath.stringToInterpStyle(fitType))
387  fitBiasArr = numpy.array([interp.interpolate(i) for i in indices])
388 
389  import lsstDebug
390  if lsstDebug.Info(__name__).display:
391  import matplotlib.pyplot as plot
392  figure = plot.figure(1)
393  figure.clear()
394  axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
395  axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+')
396  if collapsedMask.sum() > 0:
397  axes.plot(indices[collapsedMask], collapsed.data[collapsedMask], 'b+')
398  axes.plot(indices, fitBiasArr, 'r-')
399  figure.show()
400  prompt = "Press Enter or c to continue [chp]... "
401  while True:
402  ans = input(prompt).lower()
403  if ans in ("", "c",):
404  break
405  if ans in ("p",):
406  import pdb
407  pdb.set_trace()
408  elif ans in ("h", ):
409  print("h[elp] c[ontinue] p[db]")
410  figure.close()
411 
412  offImage = ampImage.Factory(ampImage.getDimensions())
413  offArray = offImage.getArray()
414  if shortInd == 1:
415  offArray[:, :] = fitBiasArr[:, numpy.newaxis]
416  else:
417  offArray[:, :] = fitBiasArr[numpy.newaxis, :]
418 
419  # We don't trust any extrapolation: mask those pixels as SUSPECT
420  # This will occur when the top and or bottom edges of the overscan
421  # contain saturated values. The values will be extrapolated from
422  # the surrounding pixels, but we cannot entirely trust the value of
423  # the extrapolation, and will mark the image mask plane to flag the
424  # image as such.
425  mask = ampMaskedImage.getMask()
426  maskArray = mask.getArray() if shortInd == 1 else mask.getArray().transpose()
427  suspect = mask.getPlaneBitMask("SUSPECT")
428  try:
429  if collapsed.mask == numpy.ma.nomask:
430  # There is no mask, so the whole array is fine
431  pass
432  except ValueError: # If collapsed.mask is an array the test fails [needs .all()]
433  for low in range(num):
434  if not collapsed.mask[low]:
435  break
436  if low > 0:
437  maskArray[:low, :] |= suspect
438  for high in range(1, num):
439  if not collapsed.mask[-high]:
440  break
441  if high > 1:
442  maskArray[-high:, :] |= suspect
443 
444  else:
445  raise pexExcept.Exception('%s : %s an invalid overscan type' % \
446  ("overscanCorrection", fitType))
447  ampImage -= offImage
448 
449 
450 def attachTransmissionCurve(exposure, opticsTransmission=None, filterTransmission=None,
451  sensorTransmission=None, atmosphereTransmission=None):
452  """Attach a TransmissionCurve to an Exposure, given separate curves for
453  different components.
454 
455  Parameters
456  ----------
457  exposure : `lsst.afw.image.Exposure`
458  Exposure object to modify by attaching the product of all given
459  ``TransmissionCurves`` in post-assembly trimmed detector coordinates.
460  Must have a valid ``Detector`` attached that matches the detector
461  associated with sensorTransmission.
462  opticsTransmission : `lsst.afw.image.TransmissionCurve`
463  A ``TransmissionCurve`` that represents the throughput of the optics,
464  to be evaluated in focal-plane coordinates.
465  filterTransmission : `lsst.afw.image.TransmissionCurve`
466  A ``TransmissionCurve`` that represents the throughput of the filter
467  itself, to be evaluated in focal-plane coordinates.
468  sensorTransmission : `lsst.afw.image.TransmissionCurve`
469  A ``TransmissionCurve`` that represents the throughput of the sensor
470  itself, to be evaluated in post-assembly trimmed detector coordinates.
471  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
472  A ``TransmissionCurve`` that represents the throughput of the
473  atmosphere, assumed to be spatially constant.
474 
475  All ``TransmissionCurve`` arguments are optional; if none are provided, the
476  attached ``TransmissionCurve`` will have unit transmission everywhere.
477 
478  Returns
479  -------
480  combined : ``lsst.afw.image.TransmissionCurve``
481  The TransmissionCurve attached to the exposure.
482  """
483  combined = afwImage.TransmissionCurve.makeIdentity()
484  if atmosphereTransmission is not None:
485  combined *= atmosphereTransmission
486  if opticsTransmission is not None:
487  combined *= opticsTransmission
488  if filterTransmission is not None:
489  combined *= filterTransmission
490  detector = exposure.getDetector()
491  fpToPix = detector.getTransform(fromSys=camGeom.FOCAL_PLANE,
492  toSys=camGeom.PIXELS)
493  combined = combined.transformedBy(fpToPix)
494  if sensorTransmission is not None:
495  combined *= sensorTransmission
496  exposure.getInfo().setTransmissionCurve(combined)
497  return combined
def darkCorrection(maskedImage, darkMaskedImage, expScale, darkScale, invert=False)
def illuminationCorrection(maskedImage, illumMaskedImage, illumScale)
def saturationCorrection(maskedImage, saturation, fwhm, growFootprints=1, interpolate=True, maskName='SAT', fallbackValue=None)
def transposeDefectList(defectList)
Definition: isrFunctions.py:90
def getDefectListFromMask(maskedImage, maskName)
def interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=None)
Definition: isrFunctions.py:60
def defectListFromFootprintList(fpList)
Definition: isrFunctions.py:77
def transposeMaskedImage(maskedImage)
Definition: isrFunctions.py:47
def biasCorrection(maskedImage, biasMaskedImage)
def interpolateFromMask(maskedImage, fwhm, growFootprints=1, maskName='SAT', fallbackValue=None)
def attachTransmissionCurve(exposure, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None)
def makeThresholdMask(maskedImage, threshold, growFootprints=1, maskName='SAT')
def overscanCorrection(ampMaskedImage, overscanImage, fitType='MEDIAN', order=1, collapseRej=3.0, statControl=None)
def flatCorrection(maskedImage, flatMaskedImage, scalingType, userScale=1.0, invert=False)
def updateVariance(maskedImage, gain, readNoise)
def maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD')