lsst.ip.isr  17.0.1-12-g84017aa+2
isrFunctions.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import math
23 import numpy
24 from deprecated.sphinx import deprecated
25 
26 import lsst.geom
27 import lsst.afw.image as afwImage
28 import lsst.afw.detection as afwDetection
29 import lsst.afw.math as afwMath
30 import lsst.meas.algorithms as measAlg
31 import lsst.pex.exceptions as pexExcept
32 import lsst.afw.cameraGeom as camGeom
33 
34 from lsst.afw.geom.wcsUtils import makeDistortedTanWcs
35 from lsst.meas.algorithms.detection import SourceDetectionTask
36 from lsst.pipe.base import Struct
37 
38 from contextlib import contextmanager
39 
40 
41 def createPsf(fwhm):
42  """Make a double Gaussian PSF.
43 
44  Parameters
45  ----------
46  fwhm : scalar
47  FWHM of double Gaussian smoothing kernel.
48 
49  Returns
50  -------
51  psf : `lsst.meas.algorithms.DoubleGaussianPsf`
52  The created smoothing kernel.
53  """
54  ksize = 4*int(fwhm) + 1
55  return measAlg.DoubleGaussianPsf(ksize, ksize, fwhm/(2*math.sqrt(2*math.log(2))))
56 
57 
58 def transposeMaskedImage(maskedImage):
59  """Make a transposed copy of a masked image.
60 
61  Parameters
62  ----------
63  maskedImage : `lsst.afw.image.MaskedImage`
64  Image to process.
65 
66  Returns
67  -------
68  transposed : `lsst.afw.image.MaskedImage`
69  The transposed copy of the input image.
70  """
71  transposed = maskedImage.Factory(lsst.geom.Extent2I(maskedImage.getHeight(), maskedImage.getWidth()))
72  transposed.getImage().getArray()[:] = maskedImage.getImage().getArray().T
73  transposed.getMask().getArray()[:] = maskedImage.getMask().getArray().T
74  transposed.getVariance().getArray()[:] = maskedImage.getVariance().getArray().T
75  return transposed
76 
77 
78 def interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=None):
79  """Interpolate over defects specified in a defect list.
80 
81  Parameters
82  ----------
83  maskedImage : `lsst.afw.image.MaskedImage`
84  Image to process.
85  defectList : `lsst.meas.algorithms.Defects`
86  List of defects to interpolate over.
87  fwhm : scalar
88  FWHM of double Gaussian smoothing kernel.
89  fallbackValue : scalar, optional
90  Fallback value if an interpolated value cannot be determined.
91  If None, then the clipped mean of the image is used.
92  """
93  psf = createPsf(fwhm)
94  if fallbackValue is None:
95  fallbackValue = afwMath.makeStatistics(maskedImage.getImage(), afwMath.MEANCLIP).getValue()
96  if 'INTRP' not in maskedImage.getMask().getMaskPlaneDict():
97  maskedImage.getMask.addMaskPlane('INTRP')
98  measAlg.interpolateOverDefects(maskedImage, psf, defectList, fallbackValue, True)
99 
100 
101 @deprecated(reason="Replaced by Defects.fromFootPrintList() (will be removed after v18)",
102  category=FutureWarning)
104  """Compute a defect list from a footprint list, optionally growing the footprints.
105 
106  Parameters
107  ----------
108  fpList : `list` of `lsst.afw.detection.Footprint`
109  Footprint list to process.
110 
111  Returns
112  -------
113  defectList : `lsst.meas.algorithms.Defects`
114  List of defects.
115  """
116  return measAlg.Defects.fromFootprintList(fpList)
117 
118 @deprecated(reason="Replaced by Defects.transpose() (will be removed after v18)",
119  category=FutureWarning)
120 def transposeDefectList(defectList):
121  """Make a transposed copy of a defect list.
122 
123  Parameters
124  ----------
125  defectList : `lsst.meas.algorithms.Defects`
126  Input list of defects.
127 
128  Returns
129  -------
130  retDefectList : `lsst.meas.algorithms.Defects`
131  Transposed list of defects.
132  """
133  if isinstance(defectList, measAlg.Defects):
134  return defectList.transpose()
135  return measAlg.Defects(defectList).transpose()
136 
137 
138 @deprecated(reason="Replaced by Defects.maskPixels() (will be removed after v18)",
139  category=FutureWarning)
140 def maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD'):
141  """Set mask plane based on a defect list.
142 
143  Parameters
144  ----------
145  maskedImage : `lsst.afw.image.MaskedImage`
146  Image to process. Only the mask plane is updated.
147  defectList : `lsst.meas.algorithms.Defects`
148  Defect list to mask.
149  maskName : str, optional
150  Mask plane name to use.
151  """
152  return defectList.maskPixels(maskedImage, maskName=maskName)
153 
154 
155 @deprecated(reason="Replaced by Defects.fromMask() (will be removed after v18)",
156  category=FutureWarning)
157 def getDefectListFromMask(maskedImage, maskName):
158  """Compute a defect list from a specified mask plane.
159 
160  Parameters
161  ----------
162  maskedImage : `lsst.afw.image.MaskedImage`
163  Image to process.
164  maskName : `str` or `list`
165  Mask plane name, or list of names to convert.
166 
167  Returns
168  -------
169  defectList : `lsst.meas.algorithms.Defects`
170  Defect list constructed from masked pixels.
171  """
172  return measAlg.Defects.fromMask(maskedImage, maskName)
173 
174 
175 def makeThresholdMask(maskedImage, threshold, growFootprints=1, maskName='SAT'):
176  """Mask pixels based on threshold detection.
177 
178  Parameters
179  ----------
180  maskedImage : `lsst.afw.image.MaskedImage`
181  Image to process. Only the mask plane is updated.
182  threshold : scalar
183  Detection threshold.
184  growFootprints : scalar, optional
185  Number of pixels to grow footprints of detected regions.
186  maskName : str, optional
187  Mask plane name, or list of names to convert
188 
189  Returns
190  -------
191  defectList : `lsst.meas.algorithms.Defects`
192  Defect list constructed from pixels above the threshold.
193  """
194  # find saturated regions
195  thresh = afwDetection.Threshold(threshold)
196  fs = afwDetection.FootprintSet(maskedImage, thresh)
197 
198  if growFootprints > 0:
199  fs = afwDetection.FootprintSet(fs, growFootprints)
200  fpList = fs.getFootprints()
201 
202  # set mask
203  mask = maskedImage.getMask()
204  bitmask = mask.getPlaneBitMask(maskName)
205  afwDetection.setMaskFromFootprintList(mask, fpList, bitmask)
206 
207  return measAlg.Defects.fromFootprintList(fpList)
208 
209 
210 def interpolateFromMask(maskedImage, fwhm, growFootprints=1, maskName='SAT', fallbackValue=None):
211  """Interpolate over defects identified by a particular mask plane.
212 
213  Parameters
214  ----------
215  maskedImage : `lsst.afw.image.MaskedImage`
216  Image to process.
217  fwhm : scalar
218  FWHM of double Gaussian smoothing kernel.
219  growFootprints : scalar, optional
220  Number of pixels to grow footprints of detected regions.
221  maskName : str, optional
222  Mask plane name.
223  fallbackValue : scalar, optional
224  Value of last resort for interpolation.
225  """
226  mask = maskedImage.getMask()
227  thresh = afwDetection.Threshold(mask.getPlaneBitMask(maskName), afwDetection.Threshold.BITMASK)
228  fpSet = afwDetection.FootprintSet(mask, thresh)
229  if growFootprints > 0:
230  fpSet = afwDetection.FootprintSet(fpSet, rGrow=growFootprints, isotropic=False)
231  # If we are interpolating over an area larger than the original masked region, we need
232  # to expand the original mask bit to the full area to explain why we interpolated there.
233  fpSet.setMask(mask, maskName)
234  defectList = measAlg.Defects.fromFootprintList(fpSet.getFootprints())
235  interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=fallbackValue)
236 
237 
238 def saturationCorrection(maskedImage, saturation, fwhm, growFootprints=1, interpolate=True, maskName='SAT',
239  fallbackValue=None):
240  """Mark saturated pixels and optionally interpolate over them
241 
242  Parameters
243  ----------
244  maskedImage : `lsst.afw.image.MaskedImage`
245  Image to process.
246  saturation : scalar
247  Saturation level used as the detection threshold.
248  fwhm : scalar
249  FWHM of double Gaussian smoothing kernel.
250  growFootprints : scalar, optional
251  Number of pixels to grow footprints of detected regions.
252  interpolate : Bool, optional
253  If True, saturated pixels are interpolated over.
254  maskName : str, optional
255  Mask plane name.
256  fallbackValue : scalar, optional
257  Value of last resort for interpolation.
258  """
259  defectList = makeThresholdMask(
260  maskedImage=maskedImage,
261  threshold=saturation,
262  growFootprints=growFootprints,
263  maskName=maskName,
264  )
265  if interpolate:
266  interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=fallbackValue)
267 
268 
269 def trimToMatchCalibBBox(rawMaskedImage, calibMaskedImage):
270  """Compute number of edge trim pixels to match the calibration data.
271 
272  Use the dimension difference between the raw exposure and the
273  calibration exposure to compute the edge trim pixels. This trim
274  is applied symmetrically, with the same number of pixels masked on
275  each side.
276 
277  Parameters
278  ----------
279  rawMaskedImage : `lsst.afw.image.MaskedImage`
280  Image to trim.
281  calibMaskedImage : `lsst.afw.image.MaskedImage`
282  Calibration image to draw new bounding box from.
283 
284  Returns
285  -------
286  replacementMaskedImage : `lsst.afw.image.MaskedImage`
287  ``rawMaskedImage`` trimmed to the appropriate size
288  Raises
289  ------
290  RuntimeError
291  Rasied if ``rawMaskedImage`` cannot be symmetrically trimmed to
292  match ``calibMaskedImage``.
293  """
294  nx, ny = rawMaskedImage.getBBox().getDimensions() - calibMaskedImage.getBBox().getDimensions()
295  if nx != ny:
296  raise RuntimeError("Raw and calib maskedImages are trimmed differently in X and Y.")
297  if nx % 2 != 0:
298  raise RuntimeError("Calibration maskedImage is trimmed unevenly in X.")
299  if nx <= 0:
300  raise RuntimeError("Calibration maskedImage is larger than raw data.")
301 
302  nEdge = nx//2
303  if nEdge > 0:
304  replacementMaskedImage = rawMaskedImage[nEdge:-nEdge, nEdge:-nEdge, afwImage.LOCAL]
305  SourceDetectionTask.setEdgeBits(
306  rawMaskedImage,
307  replacementMaskedImage.getBBox(),
308  rawMaskedImage.getMask().getPlaneBitMask("EDGE")
309  )
310  else:
311  replacementMaskedImage = rawMaskedImage
312 
313  return replacementMaskedImage
314 
315 
316 def biasCorrection(maskedImage, biasMaskedImage, trimToFit=False):
317  """Apply bias correction in place.
318 
319  Parameters
320  ----------
321  maskedImage : `lsst.afw.image.MaskedImage`
322  Image to process. The image is modified by this method.
323  biasMaskedImage : `lsst.afw.image.MaskedImage`
324  Bias image of the same size as ``maskedImage``
325  trimToFit : `Bool`, optional
326  If True, raw data is symmetrically trimmed to match
327  calibration size.
328 
329  Raises
330  ------
331  RuntimeError
332  Raised if ``maskedImage`` and ``biasMaskedImage`` do not have
333  the same size.
334 
335  """
336  if trimToFit:
337  maskedImage = trimToMatchCalibBBox(maskedImage, biasMaskedImage)
338 
339  if maskedImage.getBBox(afwImage.LOCAL) != biasMaskedImage.getBBox(afwImage.LOCAL):
340  raise RuntimeError("maskedImage bbox %s != biasMaskedImage bbox %s" %
341  (maskedImage.getBBox(afwImage.LOCAL), biasMaskedImage.getBBox(afwImage.LOCAL)))
342  maskedImage -= biasMaskedImage
343 
344 
345 def darkCorrection(maskedImage, darkMaskedImage, expScale, darkScale, invert=False, trimToFit=False):
346  """Apply dark correction in place.
347 
348  Parameters
349  ----------
350  maskedImage : `lsst.afw.image.MaskedImage`
351  Image to process. The image is modified by this method.
352  darkMaskedImage : `lsst.afw.image.MaskedImage`
353  Dark image of the same size as ``maskedImage``.
354  expScale : scalar
355  Dark exposure time for ``maskedImage``.
356  darkScale : scalar
357  Dark exposure time for ``darkMaskedImage``.
358  invert : `Bool`, optional
359  If True, re-add the dark to an already corrected image.
360  trimToFit : `Bool`, optional
361  If True, raw data is symmetrically trimmed to match
362  calibration size.
363 
364  Raises
365  ------
366  RuntimeError
367  Raised if ``maskedImage`` and ``darkMaskedImage`` do not have
368  the same size.
369 
370  Notes
371  -----
372  The dark correction is applied by calculating:
373  maskedImage -= dark * expScaling / darkScaling
374  """
375  if trimToFit:
376  maskedImage = trimToMatchCalibBBox(maskedImage, darkMaskedImage)
377 
378  if maskedImage.getBBox(afwImage.LOCAL) != darkMaskedImage.getBBox(afwImage.LOCAL):
379  raise RuntimeError("maskedImage bbox %s != darkMaskedImage bbox %s" %
380  (maskedImage.getBBox(afwImage.LOCAL), darkMaskedImage.getBBox(afwImage.LOCAL)))
381 
382  scale = expScale / darkScale
383  if not invert:
384  maskedImage.scaledMinus(scale, darkMaskedImage)
385  else:
386  maskedImage.scaledPlus(scale, darkMaskedImage)
387 
388 
389 def updateVariance(maskedImage, gain, readNoise):
390  """Set the variance plane based on the image plane.
391 
392  Parameters
393  ----------
394  maskedImage : `lsst.afw.image.MaskedImage`
395  Image to process. The variance plane is modified.
396  gain : scalar
397  The amplifier gain in electrons/ADU.
398  readNoise : scalar
399  The amplifier read nmoise in ADU/pixel.
400  """
401  var = maskedImage.getVariance()
402  var[:] = maskedImage.getImage()
403  var /= gain
404  var += readNoise**2
405 
406 
407 def flatCorrection(maskedImage, flatMaskedImage, scalingType, userScale=1.0, invert=False, trimToFit=False):
408  """Apply flat correction in place.
409 
410  Parameters
411  ----------
412  maskedImage : `lsst.afw.image.MaskedImage`
413  Image to process. The image is modified.
414  flatMaskedImage : `lsst.afw.image.MaskedImage`
415  Flat image of the same size as ``maskedImage``
416  scalingType : str
417  Flat scale computation method. Allowed values are 'MEAN',
418  'MEDIAN', or 'USER'.
419  userScale : scalar, optional
420  Scale to use if ``scalingType``='USER'.
421  invert : `Bool`, optional
422  If True, unflatten an already flattened image.
423  trimToFit : `Bool`, optional
424  If True, raw data is symmetrically trimmed to match
425  calibration size.
426 
427  Raises
428  ------
429  RuntimeError
430  Raised if ``maskedImage`` and ``flatMaskedImage`` do not have
431  the same size.
432  pexExcept.Exception
433  Raised if ``scalingType`` is not an allowed value.
434  """
435  if trimToFit:
436  maskedImage = trimToMatchCalibBBox(maskedImage, flatMaskedImage)
437 
438  if maskedImage.getBBox(afwImage.LOCAL) != flatMaskedImage.getBBox(afwImage.LOCAL):
439  raise RuntimeError("maskedImage bbox %s != flatMaskedImage bbox %s" %
440  (maskedImage.getBBox(afwImage.LOCAL), flatMaskedImage.getBBox(afwImage.LOCAL)))
441 
442  # Figure out scale from the data
443  # Ideally the flats are normalized by the calibration product pipeline, but this allows some flexibility
444  # in the case that the flat is created by some other mechanism.
445  if scalingType in ('MEAN', 'MEDIAN'):
446  scalingType = afwMath.stringToStatisticsProperty(scalingType)
447  flatScale = afwMath.makeStatistics(flatMaskedImage.image, scalingType).getValue()
448  elif scalingType == 'USER':
449  flatScale = userScale
450  else:
451  raise pexExcept.Exception('%s : %s not implemented' % ("flatCorrection", scalingType))
452 
453  if not invert:
454  maskedImage.scaledDivides(1.0/flatScale, flatMaskedImage)
455  else:
456  maskedImage.scaledMultiplies(1.0/flatScale, flatMaskedImage)
457 
458 
459 def illuminationCorrection(maskedImage, illumMaskedImage, illumScale):
460  """Apply illumination correction in place.
461 
462  Parameters
463  ----------
464  maskedImage : `lsst.afw.image.MaskedImage`
465  Image to process. The image is modified.
466  illumMaskedImage : `lsst.afw.image.MaskedImage`
467  Illumination correction image of the same size as ``maskedImage``.
468  illumScale : scalar
469  Scale factor for the illumination correction.
470 
471  Raises
472  ------
473  RuntimeError
474  Raised if ``maskedImage`` and ``illumMaskedImage`` do not have
475  the same size.
476  """
477  if maskedImage.getBBox(afwImage.LOCAL) != illumMaskedImage.getBBox(afwImage.LOCAL):
478  raise RuntimeError("maskedImage bbox %s != illumMaskedImage bbox %s" %
479  (maskedImage.getBBox(afwImage.LOCAL), illumMaskedImage.getBBox(afwImage.LOCAL)))
480 
481  maskedImage.scaledDivides(1./illumScale, illumMaskedImage)
482 
483 
484 def overscanCorrection(ampMaskedImage, overscanImage, fitType='MEDIAN', order=1, collapseRej=3.0,
485  statControl=None, overscanIsInt=True):
486  """Apply overscan correction in place.
487 
488  Parameters
489  ----------
490  ampMaskedImage : `lsst.afw.image.MaskedImage`
491  Image of amplifier to correct; modified.
492  overscanImage : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
493  Image of overscan; modified.
494  fitType : `str`
495  Type of fit for overscan correction. May be one of:
496 
497  - ``MEAN``: use mean of overscan.
498  - ``MEANCLIP``: use clipped mean of overscan.
499  - ``MEDIAN``: use median of overscan.
500  - ``POLY``: fit with ordinary polynomial.
501  - ``CHEB``: fit with Chebyshev polynomial.
502  - ``LEG``: fit with Legendre polynomial.
503  - ``NATURAL_SPLINE``: fit with natural spline.
504  - ``CUBIC_SPLINE``: fit with cubic spline.
505  - ``AKIMA_SPLINE``: fit with Akima spline.
506 
507  order : `int`
508  Polynomial order or number of spline knots; ignored unless
509  ``fitType`` indicates a polynomial or spline.
510  statControl : `lsst.afw.math.StatisticsControl`
511  Statistics control object. In particular, we pay attention to numSigmaClip
512  overscanIsInt : `bool`
513  Treat the overscan region as consisting of integers, even if it's been
514  converted to float. E.g. handle ties properly.
515 
516  Returns
517  -------
518  result : `lsst.pipe.base.Struct`
519  Result struct with components:
520 
521  - ``imageFit``: Value(s) removed from image (scalar or
522  `lsst.afw.image.Image`)
523  - ``overscanFit``: Value(s) removed from overscan (scalar or
524  `lsst.afw.image.Image`)
525  - ``overscanImage``: Overscan corrected overscan region
526  (`lsst.afw.image.Image`)
527  Raises
528  ------
529  pexExcept.Exception
530  Raised if ``fitType`` is not an allowed value.
531 
532  Notes
533  -----
534  The ``ampMaskedImage`` and ``overscanImage`` are modified, with the fit
535  subtracted. Note that the ``overscanImage`` should not be a subimage of
536  the ``ampMaskedImage``, to avoid being subtracted twice.
537 
538  Debug plots are available for the SPLINE fitTypes by setting the
539  `debug.display` for `name` == "lsst.ip.isr.isrFunctions". These
540  plots show the scatter plot of the overscan data (collapsed along
541  the perpendicular dimension) as a function of position on the CCD
542  (normalized between +/-1).
543  """
544  ampImage = ampMaskedImage.getImage()
545  if statControl is None:
546  statControl = afwMath.StatisticsControl()
547 
548  numSigmaClip = statControl.getNumSigmaClip()
549 
550  if fitType in ('MEAN', 'MEANCLIP'):
551  fitType = afwMath.stringToStatisticsProperty(fitType)
552  offImage = afwMath.makeStatistics(overscanImage, fitType, statControl).getValue()
553  overscanFit = offImage
554  elif fitType in ('MEDIAN',):
555  if overscanIsInt:
556  # we need an image with integer pixels to handle ties properly
557  if hasattr(overscanImage, "image"):
558  imageI = overscanImage.image.convertI()
559  overscanImageI = afwImage.MaskedImageI(imageI, overscanImage.mask, overscanImage.variance)
560  else:
561  overscanImageI = overscanImage.convertI()
562  else:
563  overscanImageI = overscanImage
564 
565  fitType = afwMath.stringToStatisticsProperty(fitType)
566  offImage = afwMath.makeStatistics(overscanImageI, fitType, statControl).getValue()
567  overscanFit = offImage
568 
569  if overscanIsInt:
570  del overscanImageI
571  elif fitType in ('POLY', 'CHEB', 'LEG', 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE'):
572  if hasattr(overscanImage, "getImage"):
573  biasArray = overscanImage.getImage().getArray()
574  biasArray = numpy.ma.masked_where(overscanImage.getMask().getArray() & statControl.getAndMask(),
575  biasArray)
576  else:
577  biasArray = overscanImage.getArray()
578  # Fit along the long axis, so collapse along each short row and fit the resulting array
579  shortInd = numpy.argmin(biasArray.shape)
580  if shortInd == 0:
581  # Convert to some 'standard' representation to make things easier
582  biasArray = numpy.transpose(biasArray)
583 
584  # Do a single round of clipping to weed out CR hits and signal leaking into the overscan
585  percentiles = numpy.percentile(biasArray, [25.0, 50.0, 75.0], axis=1)
586  medianBiasArr = percentiles[1]
587  stdevBiasArr = 0.74*(percentiles[2] - percentiles[0]) # robust stdev
588  diff = numpy.abs(biasArray - medianBiasArr[:, numpy.newaxis])
589  biasMaskedArr = numpy.ma.masked_where(diff > numSigmaClip*stdevBiasArr[:, numpy.newaxis], biasArray)
590  collapsed = numpy.mean(biasMaskedArr, axis=1)
591  if collapsed.mask.sum() > 0:
592  collapsed.data[collapsed.mask] = numpy.mean(biasArray.data[collapsed.mask], axis=1)
593  del biasArray, percentiles, stdevBiasArr, diff, biasMaskedArr
594 
595  if shortInd == 0:
596  collapsed = numpy.transpose(collapsed)
597 
598  num = len(collapsed)
599  indices = 2.0*numpy.arange(num)/float(num) - 1.0
600 
601  if fitType in ('POLY', 'CHEB', 'LEG'):
602  # A numpy polynomial
603  poly = numpy.polynomial
604  fitter, evaler = {"POLY": (poly.polynomial.polyfit, poly.polynomial.polyval),
605  "CHEB": (poly.chebyshev.chebfit, poly.chebyshev.chebval),
606  "LEG": (poly.legendre.legfit, poly.legendre.legval),
607  }[fitType]
608 
609  coeffs = fitter(indices, collapsed, order)
610  fitBiasArr = evaler(indices, coeffs)
611  elif 'SPLINE' in fitType:
612  # An afw interpolation
613  numBins = order
614  #
615  # numpy.histogram needs a real array for the mask, but numpy.ma "optimises" the case
616  # no-values-are-masked by replacing the mask array by a scalar, numpy.ma.nomask
617  #
618  # Issue DM-415
619  #
620  collapsedMask = collapsed.mask
621  try:
622  if collapsedMask == numpy.ma.nomask:
623  collapsedMask = numpy.array(len(collapsed)*[numpy.ma.nomask])
624  except ValueError: # If collapsedMask is an array the test fails [needs .all()]
625  pass
626 
627  numPerBin, binEdges = numpy.histogram(indices, bins=numBins,
628  weights=1-collapsedMask.astype(int))
629  # Binning is just a histogram, with weights equal to the values.
630  # Use a similar trick to get the bin centers (this deals with different numbers per bin).
631  with numpy.errstate(invalid="ignore"): # suppress NAN warnings
632  values = numpy.histogram(indices, bins=numBins,
633  weights=collapsed.data*~collapsedMask)[0]/numPerBin
634  binCenters = numpy.histogram(indices, bins=numBins,
635  weights=indices*~collapsedMask)[0]/numPerBin
636  interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0],
637  values.astype(float)[numPerBin > 0],
638  afwMath.stringToInterpStyle(fitType))
639  fitBiasArr = numpy.array([interp.interpolate(i) for i in indices])
640 
641  import lsstDebug
642  if lsstDebug.Info(__name__).display:
643  import matplotlib.pyplot as plot
644  figure = plot.figure(1)
645  figure.clear()
646  axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
647  axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+')
648  if collapsedMask.sum() > 0:
649  axes.plot(indices[collapsedMask], collapsed.data[collapsedMask], 'b+')
650  axes.plot(indices, fitBiasArr, 'r-')
651  plot.xlabel("centered/scaled position along overscan region")
652  plot.ylabel("pixel value/fit value")
653  figure.show()
654  prompt = "Press Enter or c to continue [chp]... "
655  while True:
656  ans = input(prompt).lower()
657  if ans in ("", "c",):
658  break
659  if ans in ("p",):
660  import pdb
661  pdb.set_trace()
662  elif ans in ("h", ):
663  print("h[elp] c[ontinue] p[db]")
664  plot.close()
665 
666  offImage = ampImage.Factory(ampImage.getDimensions())
667  offArray = offImage.getArray()
668  overscanFit = afwImage.ImageF(overscanImage.getDimensions())
669  overscanArray = overscanFit.getArray()
670  if shortInd == 1:
671  offArray[:, :] = fitBiasArr[:, numpy.newaxis]
672  overscanArray[:, :] = fitBiasArr[:, numpy.newaxis]
673  else:
674  offArray[:, :] = fitBiasArr[numpy.newaxis, :]
675  overscanArray[:, :] = fitBiasArr[numpy.newaxis, :]
676 
677  # We don't trust any extrapolation: mask those pixels as SUSPECT
678  # This will occur when the top and or bottom edges of the overscan
679  # contain saturated values. The values will be extrapolated from
680  # the surrounding pixels, but we cannot entirely trust the value of
681  # the extrapolation, and will mark the image mask plane to flag the
682  # image as such.
683  mask = ampMaskedImage.getMask()
684  maskArray = mask.getArray() if shortInd == 1 else mask.getArray().transpose()
685  suspect = mask.getPlaneBitMask("SUSPECT")
686  try:
687  if collapsed.mask == numpy.ma.nomask:
688  # There is no mask, so the whole array is fine
689  pass
690  except ValueError: # If collapsed.mask is an array the test fails [needs .all()]
691  for low in range(num):
692  if not collapsed.mask[low]:
693  break
694  if low > 0:
695  maskArray[:low, :] |= suspect
696  for high in range(1, num):
697  if not collapsed.mask[-high]:
698  break
699  if high > 1:
700  maskArray[-high:, :] |= suspect
701 
702  else:
703  raise pexExcept.Exception('%s : %s an invalid overscan type' % ("overscanCorrection", fitType))
704  ampImage -= offImage
705  overscanImage -= overscanFit
706  return Struct(imageFit=offImage, overscanFit=overscanFit, overscanImage=overscanImage)
707 
708 
709 def brighterFatterCorrection(exposure, kernel, maxIter, threshold, applyGain):
710  """Apply brighter fatter correction in place for the image.
711 
712  Parameters
713  ----------
714  exposure : `lsst.afw.image.Exposure`
715  Exposure to have brighter-fatter correction applied. Modified
716  by this method.
717  kernel : `numpy.ndarray`
718  Brighter-fatter kernel to apply.
719  maxIter : scalar
720  Number of correction iterations to run.
721  threshold : scalar
722  Convergence threshold in terms of the sum of absolute
723  deviations between an iteration and the previous one.
724  applyGain : `Bool`
725  If True, then the exposure values are scaled by the gain prior
726  to correction.
727 
728  Notes
729  -----
730  This correction takes a kernel that has been derived from flat
731  field images to redistribute the charge. The gradient of the
732  kernel is the deflection field due to the accumulated charge.
733 
734  Given the original image I(x) and the kernel K(x) we can compute
735  the corrected image Ic(x) using the following equation:
736 
737  Ic(x) = I(x) + 0.5*d/dx(I(x)*d/dx(int( dy*K(x-y)*I(y))))
738 
739  To evaluate the derivative term we expand it as follows:
740 
741  0.5 * ( d/dx(I(x))*d/dx(int(dy*K(x-y)*I(y))) + I(x)*d^2/dx^2(int(dy* K(x-y)*I(y))) )
742 
743  Because we use the measured counts instead of the incident counts
744  we apply the correction iteratively to reconstruct the original
745  counts and the correction. We stop iterating when the summed
746  difference between the current corrected image and the one from
747  the previous iteration is below the threshold. We do not require
748  convergence because the number of iterations is too large a
749  computational cost. How we define the threshold still needs to be
750  evaluated, the current default was shown to work reasonably well
751  on a small set of images. For more information on the method see
752  DocuShare Document-19407.
753 
754  The edges as defined by the kernel are not corrected because they
755  have spurious values due to the convolution.
756  """
757  image = exposure.getMaskedImage().getImage()
758 
759  # The image needs to be units of electrons/holes
760  with gainContext(exposure, image, applyGain):
761 
762  kLx = numpy.shape(kernel)[0]
763  kLy = numpy.shape(kernel)[1]
764  kernelImage = afwImage.ImageD(kLx, kLy)
765  kernelImage.getArray()[:, :] = kernel
766  tempImage = image.clone()
767 
768  nanIndex = numpy.isnan(tempImage.getArray())
769  tempImage.getArray()[nanIndex] = 0.
770 
771  outImage = afwImage.ImageF(image.getDimensions())
772  corr = numpy.zeros_like(image.getArray())
773  prev_image = numpy.zeros_like(image.getArray())
774  convCntrl = afwMath.ConvolutionControl(False, True, 1)
775  fixedKernel = afwMath.FixedKernel(kernelImage)
776 
777  # Define boundary by convolution region. The region that the correction will be
778  # calculated for is one fewer in each dimension because of the second derivative terms.
779  # NOTE: these need to use integer math, as we're using start:end as numpy index ranges.
780  startX = kLx//2
781  endX = -kLx//2
782  startY = kLy//2
783  endY = -kLy//2
784 
785  for iteration in range(maxIter):
786 
787  afwMath.convolve(outImage, tempImage, fixedKernel, convCntrl)
788  tmpArray = tempImage.getArray()
789  outArray = outImage.getArray()
790 
791  with numpy.errstate(invalid="ignore", over="ignore"):
792  # First derivative term
793  gradTmp = numpy.gradient(tmpArray[startY:endY, startX:endX])
794  gradOut = numpy.gradient(outArray[startY:endY, startX:endX])
795  first = (gradTmp[0]*gradOut[0] + gradTmp[1]*gradOut[1])[1:-1, 1:-1]
796 
797  # Second derivative term
798  diffOut20 = numpy.diff(outArray, 2, 0)[startY:endY, startX + 1:endX - 1]
799  diffOut21 = numpy.diff(outArray, 2, 1)[startY + 1:endY - 1, startX:endX]
800  second = tmpArray[startY + 1:endY - 1, startX + 1:endX - 1]*(diffOut20 + diffOut21)
801 
802  corr[startY + 1:endY - 1, startX + 1:endX - 1] = 0.5*(first + second)
803 
804  tmpArray[:, :] = image.getArray()[:, :]
805  tmpArray[nanIndex] = 0.
806  tmpArray[startY:endY, startX:endX] += corr[startY:endY, startX:endX]
807 
808  if iteration > 0:
809  diff = numpy.sum(numpy.abs(prev_image - tmpArray))
810 
811  if diff < threshold:
812  break
813  prev_image[:, :] = tmpArray[:, :]
814 
815  # if iteration == maxIter - 1:
816  # self.log.warn("Brighter fatter correction did not converge,
817  # final difference %f" % diff)
818 
819  # self.log.info("Finished brighter fatter in %d iterations" % (iteration + 1))
820  image.getArray()[startY + 1:endY - 1, startX + 1:endX - 1] += \
821  corr[startY + 1:endY - 1, startX + 1:endX - 1]
822 
823 
824 @contextmanager
825 def gainContext(exp, image, apply):
826  """Context manager that applies and removes gain.
827 
828  Parameters
829  ----------
830  exp : `lsst.afw.image.Exposure`
831  Exposure to apply/remove gain.
832  image : `lsst.afw.image.Image`
833  Image to apply/remove gain.
834  apply : `Bool`
835  If True, apply and remove the amplifier gain.
836 
837  Yields
838  ------
839  exp : `lsst.afw.image.Exposure`
840  Exposure with the gain applied.
841  """
842  if apply:
843  ccd = exp.getDetector()
844  for amp in ccd:
845  sim = image.Factory(image, amp.getBBox())
846  sim *= amp.getGain()
847 
848  try:
849  yield exp
850  finally:
851  if apply:
852  ccd = exp.getDetector()
853  for amp in ccd:
854  sim = image.Factory(image, amp.getBBox())
855  sim /= amp.getGain()
856 
857 
858 def attachTransmissionCurve(exposure, opticsTransmission=None, filterTransmission=None,
859  sensorTransmission=None, atmosphereTransmission=None):
860  """Attach a TransmissionCurve to an Exposure, given separate curves for
861  different components.
862 
863  Parameters
864  ----------
865  exposure : `lsst.afw.image.Exposure`
866  Exposure object to modify by attaching the product of all given
867  ``TransmissionCurves`` in post-assembly trimmed detector coordinates.
868  Must have a valid ``Detector`` attached that matches the detector
869  associated with sensorTransmission.
870  opticsTransmission : `lsst.afw.image.TransmissionCurve`
871  A ``TransmissionCurve`` that represents the throughput of the optics,
872  to be evaluated in focal-plane coordinates.
873  filterTransmission : `lsst.afw.image.TransmissionCurve`
874  A ``TransmissionCurve`` that represents the throughput of the filter
875  itself, to be evaluated in focal-plane coordinates.
876  sensorTransmission : `lsst.afw.image.TransmissionCurve`
877  A ``TransmissionCurve`` that represents the throughput of the sensor
878  itself, to be evaluated in post-assembly trimmed detector coordinates.
879  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
880  A ``TransmissionCurve`` that represents the throughput of the
881  atmosphere, assumed to be spatially constant.
882 
883  Returns
884  -------
885  combined : `lsst.afw.image.TransmissionCurve`
886  The TransmissionCurve attached to the exposure.
887 
888  Notes
889  -----
890  All ``TransmissionCurve`` arguments are optional; if none are provided, the
891  attached ``TransmissionCurve`` will have unit transmission everywhere.
892  """
893  combined = afwImage.TransmissionCurve.makeIdentity()
894  if atmosphereTransmission is not None:
895  combined *= atmosphereTransmission
896  if opticsTransmission is not None:
897  combined *= opticsTransmission
898  if filterTransmission is not None:
899  combined *= filterTransmission
900  detector = exposure.getDetector()
901  fpToPix = detector.getTransform(fromSys=camGeom.FOCAL_PLANE,
902  toSys=camGeom.PIXELS)
903  combined = combined.transformedBy(fpToPix)
904  if sensorTransmission is not None:
905  combined *= sensorTransmission
906  exposure.getInfo().setTransmissionCurve(combined)
907  return combined
908 
909 
910 def addDistortionModel(exposure, camera):
911  """!Update the WCS in exposure with a distortion model based on camera
912  geometry.
913 
914  Parameters
915  ----------
916  exposure : `lsst.afw.image.Exposure`
917  Exposure to process. Must contain a Detector and WCS. The
918  exposure is modified.
919  camera : `lsst.afw.cameraGeom.Camera`
920  Camera geometry.
921 
922  Raises
923  ------
924  RuntimeError
925  Raised if ``exposure`` is lacking a Detector or WCS, or if
926  ``camera`` is None.
927  Notes
928  -----
929  Add a model for optical distortion based on geometry found in ``camera``
930  and the ``exposure``'s detector. The raw input exposure is assumed
931  have a TAN WCS that has no compensation for optical distortion.
932  Two other possibilities are:
933  - The raw input exposure already has a model for optical distortion,
934  as is the case for raw DECam data.
935  In that case you should set config.doAddDistortionModel False.
936  - The raw input exposure has a model for distortion, but it has known
937  deficiencies severe enough to be worth fixing (e.g. because they
938  cause problems for fitting a better WCS). In that case you should
939  override this method with a version suitable for your raw data.
940 
941  """
942  wcs = exposure.getWcs()
943  if wcs is None:
944  raise RuntimeError("exposure has no WCS")
945  if camera is None:
946  raise RuntimeError("camera is None")
947  detector = exposure.getDetector()
948  if detector is None:
949  raise RuntimeError("exposure has no Detector")
950  pixelToFocalPlane = detector.getTransform(camGeom.PIXELS, camGeom.FOCAL_PLANE)
951  focalPlaneToFieldAngle = camera.getTransformMap().getTransform(camGeom.FOCAL_PLANE,
952  camGeom.FIELD_ANGLE)
953  distortedWcs = makeDistortedTanWcs(wcs, pixelToFocalPlane, focalPlaneToFieldAngle)
954  exposure.setWcs(distortedWcs)
955 
956 
957 def applyGains(exposure, normalizeGains=False):
958  """Scale an exposure by the amplifier gains.
959 
960  Parameters
961  ----------
962  exposure : `lsst.afw.image.Exposure`
963  Exposure to process. The image is modified.
964  normalizeGains : `Bool`, optional
965  If True, then amplifiers are scaled to force the median of
966  each amplifier to equal the median of those medians.
967  """
968  ccd = exposure.getDetector()
969  ccdImage = exposure.getMaskedImage()
970 
971  medians = []
972  for amp in ccd:
973  sim = ccdImage.Factory(ccdImage, amp.getBBox())
974  sim *= amp.getGain()
975 
976  if normalizeGains:
977  medians.append(numpy.median(sim.getImage().getArray()))
978 
979  if normalizeGains:
980  median = numpy.median(numpy.array(medians))
981  for index, amp in enumerate(ccd):
982  sim = ccdImage.Factory(ccdImage, amp.getDataSec())
983  sim *= median/medians[index]
984 
985 
987  """Grow the saturation trails by an amount dependent on the width of the trail.
988 
989  Parameters
990  ----------
991  mask : `lsst.afw.image.Mask`
992  Mask which will have the saturated areas grown.
993  """
994 
995  extraGrowDict = {}
996  for i in range(1, 6):
997  extraGrowDict[i] = 0
998  for i in range(6, 8):
999  extraGrowDict[i] = 1
1000  for i in range(8, 10):
1001  extraGrowDict[i] = 3
1002  extraGrowMax = 4
1003 
1004  if extraGrowMax <= 0:
1005  return
1006 
1007  saturatedBit = mask.getPlaneBitMask("SAT")
1008 
1009  xmin, ymin = mask.getBBox().getMin()
1010  width = mask.getWidth()
1011 
1012  thresh = afwDetection.Threshold(saturatedBit, afwDetection.Threshold.BITMASK)
1013  fpList = afwDetection.FootprintSet(mask, thresh).getFootprints()
1014 
1015  for fp in fpList:
1016  for s in fp.getSpans():
1017  x0, x1 = s.getX0(), s.getX1()
1018 
1019  extraGrow = extraGrowDict.get(x1 - x0 + 1, extraGrowMax)
1020  if extraGrow > 0:
1021  y = s.getY() - ymin
1022  x0 -= xmin + extraGrow
1023  x1 -= xmin - extraGrow
1024 
1025  if x0 < 0:
1026  x0 = 0
1027  if x1 >= width - 1:
1028  x1 = width - 1
1029 
1030  mask.array[y, x0:x1+1] |= saturatedBit
1031 
1032 
1033 def setBadRegions(exposure, badStatistic="MEDIAN"):
1034  """Set all BAD areas of the chip to the average of the rest of the exposure
1035 
1036  Parameters
1037  ----------
1038  exposure : `lsst.afw.image.Exposure`
1039  Exposure to mask. The exposure mask is modified.
1040  badStatistic : `str`, optional
1041  Statistic to use to generate the replacement value from the
1042  image data. Allowed values are 'MEDIAN' or 'MEANCLIP'.
1043 
1044  Returns
1045  -------
1046  badPixelCount : scalar
1047  Number of bad pixels masked.
1048  badPixelValue : scalar
1049  Value substituted for bad pixels.
1050 
1051  Raises
1052  ------
1053  RuntimeError
1054  Raised if `badStatistic` is not an allowed value.
1055  """
1056  if badStatistic == "MEDIAN":
1057  statistic = afwMath.MEDIAN
1058  elif badStatistic == "MEANCLIP":
1059  statistic = afwMath.MEANCLIP
1060  else:
1061  raise RuntimeError("Impossible method %s of bad region correction" % badStatistic)
1062 
1063  mi = exposure.getMaskedImage()
1064  mask = mi.getMask()
1065  BAD = mask.getPlaneBitMask("BAD")
1066  INTRP = mask.getPlaneBitMask("INTRP")
1067 
1068  sctrl = afwMath.StatisticsControl()
1069  sctrl.setAndMask(BAD)
1070  value = afwMath.makeStatistics(mi, statistic, sctrl).getValue()
1071 
1072  maskArray = mask.getArray()
1073  imageArray = mi.getImage().getArray()
1074  badPixels = numpy.logical_and((maskArray & BAD) > 0, (maskArray & INTRP) == 0)
1075  imageArray[:] = numpy.where(badPixels, value, imageArray)
1076 
1077  return badPixels.sum(), value
def illuminationCorrection(maskedImage, illumMaskedImage, illumScale)
def addDistortionModel(exposure, camera)
Update the WCS in exposure with a distortion model based on camera geometry.
def saturationCorrection(maskedImage, saturation, fwhm, growFootprints=1, interpolate=True, maskName='SAT', fallbackValue=None)
def setBadRegions(exposure, badStatistic="MEDIAN")
def brighterFatterCorrection(exposure, kernel, maxIter, threshold, applyGain)
def transposeDefectList(defectList)
def getDefectListFromMask(maskedImage, maskName)
def interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=None)
Definition: isrFunctions.py:78
def defectListFromFootprintList(fpList)
def transposeMaskedImage(maskedImage)
Definition: isrFunctions.py:58
def interpolateFromMask(maskedImage, fwhm, growFootprints=1, maskName='SAT', fallbackValue=None)
def trimToMatchCalibBBox(rawMaskedImage, calibMaskedImage)
def attachTransmissionCurve(exposure, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None)
def flatCorrection(maskedImage, flatMaskedImage, scalingType, userScale=1.0, invert=False, trimToFit=False)
def makeThresholdMask(maskedImage, threshold, growFootprints=1, maskName='SAT')
def applyGains(exposure, normalizeGains=False)
def darkCorrection(maskedImage, darkMaskedImage, expScale, darkScale, invert=False, trimToFit=False)
def biasCorrection(maskedImage, biasMaskedImage, trimToFit=False)
def gainContext(exp, image, apply)
def updateVariance(maskedImage, gain, readNoise)
def overscanCorrection(ampMaskedImage, overscanImage, fitType='MEDIAN', order=1, collapseRej=3.0, statControl=None, overscanIsInt=True)
def maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD')