lsst.ip.isr  19.0.0-14-g5673ca6+11
isrFunctions.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import math
23 import numpy
24 from deprecated.sphinx import deprecated
25 
26 import lsst.geom
27 import lsst.afw.image as afwImage
28 import lsst.afw.detection as afwDetection
29 import lsst.afw.math as afwMath
30 import lsst.meas.algorithms as measAlg
31 import lsst.pex.exceptions as pexExcept
32 import lsst.afw.cameraGeom as camGeom
33 
34 from lsst.afw.geom.wcsUtils import makeDistortedTanWcs
35 from lsst.meas.algorithms.detection import SourceDetectionTask
36 from lsst.pipe.base import Struct
37 
38 from contextlib import contextmanager
39 
40 
41 def createPsf(fwhm):
42  """Make a double Gaussian PSF.
43 
44  Parameters
45  ----------
46  fwhm : scalar
47  FWHM of double Gaussian smoothing kernel.
48 
49  Returns
50  -------
51  psf : `lsst.meas.algorithms.DoubleGaussianPsf`
52  The created smoothing kernel.
53  """
54  ksize = 4*int(fwhm) + 1
55  return measAlg.DoubleGaussianPsf(ksize, ksize, fwhm/(2*math.sqrt(2*math.log(2))))
56 
57 
58 def transposeMaskedImage(maskedImage):
59  """Make a transposed copy of a masked image.
60 
61  Parameters
62  ----------
63  maskedImage : `lsst.afw.image.MaskedImage`
64  Image to process.
65 
66  Returns
67  -------
68  transposed : `lsst.afw.image.MaskedImage`
69  The transposed copy of the input image.
70  """
71  transposed = maskedImage.Factory(lsst.geom.Extent2I(maskedImage.getHeight(), maskedImage.getWidth()))
72  transposed.getImage().getArray()[:] = maskedImage.getImage().getArray().T
73  transposed.getMask().getArray()[:] = maskedImage.getMask().getArray().T
74  transposed.getVariance().getArray()[:] = maskedImage.getVariance().getArray().T
75  return transposed
76 
77 
78 def interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=None):
79  """Interpolate over defects specified in a defect list.
80 
81  Parameters
82  ----------
83  maskedImage : `lsst.afw.image.MaskedImage`
84  Image to process.
85  defectList : `lsst.meas.algorithms.Defects`
86  List of defects to interpolate over.
87  fwhm : scalar
88  FWHM of double Gaussian smoothing kernel.
89  fallbackValue : scalar, optional
90  Fallback value if an interpolated value cannot be determined.
91  If None, then the clipped mean of the image is used.
92  """
93  psf = createPsf(fwhm)
94  if fallbackValue is None:
95  fallbackValue = afwMath.makeStatistics(maskedImage.getImage(), afwMath.MEANCLIP).getValue()
96  if 'INTRP' not in maskedImage.getMask().getMaskPlaneDict():
97  maskedImage.getMask().addMaskPlane('INTRP')
98  measAlg.interpolateOverDefects(maskedImage, psf, defectList, fallbackValue, True)
99  return maskedImage
100 
101 
102 def makeThresholdMask(maskedImage, threshold, growFootprints=1, maskName='SAT'):
103  """Mask pixels based on threshold detection.
104 
105  Parameters
106  ----------
107  maskedImage : `lsst.afw.image.MaskedImage`
108  Image to process. Only the mask plane is updated.
109  threshold : scalar
110  Detection threshold.
111  growFootprints : scalar, optional
112  Number of pixels to grow footprints of detected regions.
113  maskName : str, optional
114  Mask plane name, or list of names to convert
115 
116  Returns
117  -------
118  defectList : `lsst.meas.algorithms.Defects`
119  Defect list constructed from pixels above the threshold.
120  """
121  # find saturated regions
122  thresh = afwDetection.Threshold(threshold)
123  fs = afwDetection.FootprintSet(maskedImage, thresh)
124 
125  if growFootprints > 0:
126  fs = afwDetection.FootprintSet(fs, rGrow=growFootprints, isotropic=False)
127  fpList = fs.getFootprints()
128 
129  # set mask
130  mask = maskedImage.getMask()
131  bitmask = mask.getPlaneBitMask(maskName)
132  afwDetection.setMaskFromFootprintList(mask, fpList, bitmask)
133 
134  return measAlg.Defects.fromFootprintList(fpList)
135 
136 
137 def growMasks(mask, radius=0, maskNameList=['BAD'], maskValue="BAD"):
138  """Grow a mask by an amount and add to the requested plane.
139 
140  Parameters
141  ----------
142  mask : `lsst.afw.image.Mask`
143  Mask image to process.
144  radius : scalar
145  Amount to grow the mask.
146  maskNameList : `str` or `list` [`str`]
147  Mask names that should be grown.
148  maskValue : `str`
149  Mask plane to assign the newly masked pixels to.
150  """
151  if radius > 0:
152  thresh = afwDetection.Threshold(mask.getPlaneBitMask(maskNameList), afwDetection.Threshold.BITMASK)
153  fpSet = afwDetection.FootprintSet(mask, thresh)
154  fpSet = afwDetection.FootprintSet(fpSet, rGrow=radius, isotropic=False)
155  fpSet.setMask(mask, maskValue)
156 
157 
158 def interpolateFromMask(maskedImage, fwhm, growSaturatedFootprints=1,
159  maskNameList=['SAT'], fallbackValue=None):
160  """Interpolate over defects identified by a particular set of mask planes.
161 
162  Parameters
163  ----------
164  maskedImage : `lsst.afw.image.MaskedImage`
165  Image to process.
166  fwhm : scalar
167  FWHM of double Gaussian smoothing kernel.
168  growSaturatedFootprints : scalar, optional
169  Number of pixels to grow footprints for saturated pixels.
170  maskNameList : `List` of `str`, optional
171  Mask plane name.
172  fallbackValue : scalar, optional
173  Value of last resort for interpolation.
174  """
175  mask = maskedImage.getMask()
176 
177  if growSaturatedFootprints > 0 and "SAT" in maskNameList:
178  # If we are interpolating over an area larger than the original masked region, we need
179  # to expand the original mask bit to the full area to explain why we interpolated there.
180  growMasks(mask, radius=growSaturatedFootprints, maskNameList=['SAT'], maskValue="SAT")
181 
182  thresh = afwDetection.Threshold(mask.getPlaneBitMask(maskNameList), afwDetection.Threshold.BITMASK)
183  fpSet = afwDetection.FootprintSet(mask, thresh)
184  defectList = measAlg.Defects.fromFootprintList(fpSet.getFootprints())
185 
186  interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=fallbackValue)
187 
188  return maskedImage
189 
190 
191 def saturationCorrection(maskedImage, saturation, fwhm, growFootprints=1, interpolate=True, maskName='SAT',
192  fallbackValue=None):
193  """Mark saturated pixels and optionally interpolate over them
194 
195  Parameters
196  ----------
197  maskedImage : `lsst.afw.image.MaskedImage`
198  Image to process.
199  saturation : scalar
200  Saturation level used as the detection threshold.
201  fwhm : scalar
202  FWHM of double Gaussian smoothing kernel.
203  growFootprints : scalar, optional
204  Number of pixels to grow footprints of detected regions.
205  interpolate : Bool, optional
206  If True, saturated pixels are interpolated over.
207  maskName : str, optional
208  Mask plane name.
209  fallbackValue : scalar, optional
210  Value of last resort for interpolation.
211  """
212  defectList = makeThresholdMask(
213  maskedImage=maskedImage,
214  threshold=saturation,
215  growFootprints=growFootprints,
216  maskName=maskName,
217  )
218  if interpolate:
219  interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=fallbackValue)
220 
221  return maskedImage
222 
223 
224 def trimToMatchCalibBBox(rawMaskedImage, calibMaskedImage):
225  """Compute number of edge trim pixels to match the calibration data.
226 
227  Use the dimension difference between the raw exposure and the
228  calibration exposure to compute the edge trim pixels. This trim
229  is applied symmetrically, with the same number of pixels masked on
230  each side.
231 
232  Parameters
233  ----------
234  rawMaskedImage : `lsst.afw.image.MaskedImage`
235  Image to trim.
236  calibMaskedImage : `lsst.afw.image.MaskedImage`
237  Calibration image to draw new bounding box from.
238 
239  Returns
240  -------
241  replacementMaskedImage : `lsst.afw.image.MaskedImage`
242  ``rawMaskedImage`` trimmed to the appropriate size
243  Raises
244  ------
245  RuntimeError
246  Rasied if ``rawMaskedImage`` cannot be symmetrically trimmed to
247  match ``calibMaskedImage``.
248  """
249  nx, ny = rawMaskedImage.getBBox().getDimensions() - calibMaskedImage.getBBox().getDimensions()
250  if nx != ny:
251  raise RuntimeError("Raw and calib maskedImages are trimmed differently in X and Y.")
252  if nx % 2 != 0:
253  raise RuntimeError("Calibration maskedImage is trimmed unevenly in X.")
254  if nx < 0:
255  raise RuntimeError("Calibration maskedImage is larger than raw data.")
256 
257  nEdge = nx//2
258  if nEdge > 0:
259  replacementMaskedImage = rawMaskedImage[nEdge:-nEdge, nEdge:-nEdge, afwImage.LOCAL]
260  SourceDetectionTask.setEdgeBits(
261  rawMaskedImage,
262  replacementMaskedImage.getBBox(),
263  rawMaskedImage.getMask().getPlaneBitMask("EDGE")
264  )
265  else:
266  replacementMaskedImage = rawMaskedImage
267 
268  return replacementMaskedImage
269 
270 
271 def biasCorrection(maskedImage, biasMaskedImage, trimToFit=False):
272  """Apply bias correction in place.
273 
274  Parameters
275  ----------
276  maskedImage : `lsst.afw.image.MaskedImage`
277  Image to process. The image is modified by this method.
278  biasMaskedImage : `lsst.afw.image.MaskedImage`
279  Bias image of the same size as ``maskedImage``
280  trimToFit : `Bool`, optional
281  If True, raw data is symmetrically trimmed to match
282  calibration size.
283 
284  Raises
285  ------
286  RuntimeError
287  Raised if ``maskedImage`` and ``biasMaskedImage`` do not have
288  the same size.
289 
290  """
291  if trimToFit:
292  maskedImage = trimToMatchCalibBBox(maskedImage, biasMaskedImage)
293 
294  if maskedImage.getBBox(afwImage.LOCAL) != biasMaskedImage.getBBox(afwImage.LOCAL):
295  raise RuntimeError("maskedImage bbox %s != biasMaskedImage bbox %s" %
296  (maskedImage.getBBox(afwImage.LOCAL), biasMaskedImage.getBBox(afwImage.LOCAL)))
297  maskedImage -= biasMaskedImage
298 
299 
300 def darkCorrection(maskedImage, darkMaskedImage, expScale, darkScale, invert=False, trimToFit=False):
301  """Apply dark correction in place.
302 
303  Parameters
304  ----------
305  maskedImage : `lsst.afw.image.MaskedImage`
306  Image to process. The image is modified by this method.
307  darkMaskedImage : `lsst.afw.image.MaskedImage`
308  Dark image of the same size as ``maskedImage``.
309  expScale : scalar
310  Dark exposure time for ``maskedImage``.
311  darkScale : scalar
312  Dark exposure time for ``darkMaskedImage``.
313  invert : `Bool`, optional
314  If True, re-add the dark to an already corrected image.
315  trimToFit : `Bool`, optional
316  If True, raw data is symmetrically trimmed to match
317  calibration size.
318 
319  Raises
320  ------
321  RuntimeError
322  Raised if ``maskedImage`` and ``darkMaskedImage`` do not have
323  the same size.
324 
325  Notes
326  -----
327  The dark correction is applied by calculating:
328  maskedImage -= dark * expScaling / darkScaling
329  """
330  if trimToFit:
331  maskedImage = trimToMatchCalibBBox(maskedImage, darkMaskedImage)
332 
333  if maskedImage.getBBox(afwImage.LOCAL) != darkMaskedImage.getBBox(afwImage.LOCAL):
334  raise RuntimeError("maskedImage bbox %s != darkMaskedImage bbox %s" %
335  (maskedImage.getBBox(afwImage.LOCAL), darkMaskedImage.getBBox(afwImage.LOCAL)))
336 
337  scale = expScale / darkScale
338  if not invert:
339  maskedImage.scaledMinus(scale, darkMaskedImage)
340  else:
341  maskedImage.scaledPlus(scale, darkMaskedImage)
342 
343 
344 def updateVariance(maskedImage, gain, readNoise):
345  """Set the variance plane based on the image plane.
346 
347  Parameters
348  ----------
349  maskedImage : `lsst.afw.image.MaskedImage`
350  Image to process. The variance plane is modified.
351  gain : scalar
352  The amplifier gain in electrons/ADU.
353  readNoise : scalar
354  The amplifier read nmoise in ADU/pixel.
355  """
356  var = maskedImage.getVariance()
357  var[:] = maskedImage.getImage()
358  var /= gain
359  var += readNoise**2
360 
361 
362 def flatCorrection(maskedImage, flatMaskedImage, scalingType, userScale=1.0, invert=False, trimToFit=False):
363  """Apply flat correction in place.
364 
365  Parameters
366  ----------
367  maskedImage : `lsst.afw.image.MaskedImage`
368  Image to process. The image is modified.
369  flatMaskedImage : `lsst.afw.image.MaskedImage`
370  Flat image of the same size as ``maskedImage``
371  scalingType : str
372  Flat scale computation method. Allowed values are 'MEAN',
373  'MEDIAN', or 'USER'.
374  userScale : scalar, optional
375  Scale to use if ``scalingType``='USER'.
376  invert : `Bool`, optional
377  If True, unflatten an already flattened image.
378  trimToFit : `Bool`, optional
379  If True, raw data is symmetrically trimmed to match
380  calibration size.
381 
382  Raises
383  ------
384  RuntimeError
385  Raised if ``maskedImage`` and ``flatMaskedImage`` do not have
386  the same size or if ``scalingType`` is not an allowed value.
387  """
388  if trimToFit:
389  maskedImage = trimToMatchCalibBBox(maskedImage, flatMaskedImage)
390 
391  if maskedImage.getBBox(afwImage.LOCAL) != flatMaskedImage.getBBox(afwImage.LOCAL):
392  raise RuntimeError("maskedImage bbox %s != flatMaskedImage bbox %s" %
393  (maskedImage.getBBox(afwImage.LOCAL), flatMaskedImage.getBBox(afwImage.LOCAL)))
394 
395  # Figure out scale from the data
396  # Ideally the flats are normalized by the calibration product pipeline, but this allows some flexibility
397  # in the case that the flat is created by some other mechanism.
398  if scalingType in ('MEAN', 'MEDIAN'):
399  scalingType = afwMath.stringToStatisticsProperty(scalingType)
400  flatScale = afwMath.makeStatistics(flatMaskedImage.image, scalingType).getValue()
401  elif scalingType == 'USER':
402  flatScale = userScale
403  else:
404  raise RuntimeError('%s : %s not implemented' % ("flatCorrection", scalingType))
405 
406  if not invert:
407  maskedImage.scaledDivides(1.0/flatScale, flatMaskedImage)
408  else:
409  maskedImage.scaledMultiplies(1.0/flatScale, flatMaskedImage)
410 
411 
412 def illuminationCorrection(maskedImage, illumMaskedImage, illumScale, trimToFit=True):
413  """Apply illumination correction in place.
414 
415  Parameters
416  ----------
417  maskedImage : `lsst.afw.image.MaskedImage`
418  Image to process. The image is modified.
419  illumMaskedImage : `lsst.afw.image.MaskedImage`
420  Illumination correction image of the same size as ``maskedImage``.
421  illumScale : scalar
422  Scale factor for the illumination correction.
423  trimToFit : `Bool`, optional
424  If True, raw data is symmetrically trimmed to match
425  calibration size.
426 
427  Raises
428  ------
429  RuntimeError
430  Raised if ``maskedImage`` and ``illumMaskedImage`` do not have
431  the same size.
432  """
433  if trimToFit:
434  maskedImage = trimToMatchCalibBBox(maskedImage, illumMaskedImage)
435 
436  if maskedImage.getBBox(afwImage.LOCAL) != illumMaskedImage.getBBox(afwImage.LOCAL):
437  raise RuntimeError("maskedImage bbox %s != illumMaskedImage bbox %s" %
438  (maskedImage.getBBox(afwImage.LOCAL), illumMaskedImage.getBBox(afwImage.LOCAL)))
439 
440  maskedImage.scaledDivides(1.0/illumScale, illumMaskedImage)
441 
442 
443 def overscanCorrection(ampMaskedImage, overscanImage, fitType='MEDIAN', order=1, collapseRej=3.0,
444  statControl=None, overscanIsInt=True):
445  """Apply overscan correction in place.
446 
447  Parameters
448  ----------
449  ampMaskedImage : `lsst.afw.image.MaskedImage`
450  Image of amplifier to correct; modified.
451  overscanImage : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
452  Image of overscan; modified.
453  fitType : `str`
454  Type of fit for overscan correction. May be one of:
455 
456  - ``MEAN``: use mean of overscan.
457  - ``MEANCLIP``: use clipped mean of overscan.
458  - ``MEDIAN``: use median of overscan.
459  - ``MEDIAN_PER_ROW``: use median per row of overscan.
460  - ``POLY``: fit with ordinary polynomial.
461  - ``CHEB``: fit with Chebyshev polynomial.
462  - ``LEG``: fit with Legendre polynomial.
463  - ``NATURAL_SPLINE``: fit with natural spline.
464  - ``CUBIC_SPLINE``: fit with cubic spline.
465  - ``AKIMA_SPLINE``: fit with Akima spline.
466 
467  order : `int`
468  Polynomial order or number of spline knots; ignored unless
469  ``fitType`` indicates a polynomial or spline.
470  statControl : `lsst.afw.math.StatisticsControl`
471  Statistics control object. In particular, we pay attention to numSigmaClip
472  overscanIsInt : `bool`
473  Treat the overscan region as consisting of integers, even if it's been
474  converted to float. E.g. handle ties properly.
475 
476  Returns
477  -------
478  result : `lsst.pipe.base.Struct`
479  Result struct with components:
480 
481  - ``imageFit``: Value(s) removed from image (scalar or
482  `lsst.afw.image.Image`)
483  - ``overscanFit``: Value(s) removed from overscan (scalar or
484  `lsst.afw.image.Image`)
485  - ``overscanImage``: Overscan corrected overscan region
486  (`lsst.afw.image.Image`)
487  Raises
488  ------
489  pexExcept.Exception
490  Raised if ``fitType`` is not an allowed value.
491 
492  Notes
493  -----
494  The ``ampMaskedImage`` and ``overscanImage`` are modified, with the fit
495  subtracted. Note that the ``overscanImage`` should not be a subimage of
496  the ``ampMaskedImage``, to avoid being subtracted twice.
497 
498  Debug plots are available for the SPLINE fitTypes by setting the
499  `debug.display` for `name` == "lsst.ip.isr.isrFunctions". These
500  plots show the scatter plot of the overscan data (collapsed along
501  the perpendicular dimension) as a function of position on the CCD
502  (normalized between +/-1).
503  """
504  ampImage = ampMaskedImage.getImage()
505  if statControl is None:
506  statControl = afwMath.StatisticsControl()
507 
508  numSigmaClip = statControl.getNumSigmaClip()
509  if fitType in ('MEAN', 'MEANCLIP'):
510  fitType = afwMath.stringToStatisticsProperty(fitType)
511  offImage = afwMath.makeStatistics(overscanImage, fitType, statControl).getValue()
512  overscanFit = offImage
513  elif fitType in ('MEDIAN', 'MEDIAN_PER_ROW',):
514  if overscanIsInt:
515  # we need an image with integer pixels to handle ties properly
516  if hasattr(overscanImage, "image"):
517  imageI = overscanImage.image.convertI()
518  overscanImageI = afwImage.MaskedImageI(imageI, overscanImage.mask, overscanImage.variance)
519  else:
520  overscanImageI = overscanImage.convertI()
521  else:
522  overscanImageI = overscanImage
523  if fitType in ('MEDIAN',):
524  fitTypeStats = afwMath.stringToStatisticsProperty(fitType)
525  offImage = afwMath.makeStatistics(overscanImageI, fitTypeStats, statControl).getValue()
526  overscanFit = offImage
527  elif fitType in ('MEDIAN_PER_ROW',):
528  if hasattr(overscanImageI, "getImage"):
529  biasArray = overscanImageI.getImage().getArray()
530  else:
531  biasArray = overscanImageI.getArray()
532  shortInd = numpy.argmin(biasArray.shape)
533  if shortInd == 0:
534  # Convert to some 'standard' representation to make things easier
535  biasArray = numpy.transpose(biasArray)
536 
537  fitTypeStats = afwMath.stringToStatisticsProperty('MEDIAN')
538  collapsed = []
539  for row in biasArray:
540  rowMedian = afwMath.makeStatistics(row, fitTypeStats, statControl).getValue()
541  collapsed.append(rowMedian)
542  collapsed = numpy.array(collapsed)
543  offImage = ampImage.Factory(ampImage.getDimensions())
544  offArray = offImage.getArray()
545  overscanFit = afwImage.ImageF(overscanImage.getDimensions())
546  overscanArray = overscanFit.getArray()
547 
548  if shortInd == 1:
549  offArray[:, :] = collapsed[:, numpy.newaxis]
550  overscanArray[:, :] = collapsed[:, numpy.newaxis]
551  else:
552  offArray[:, :] = collapsed[numpy.newaxis, :]
553  overscanArray[:, :] = collapsed[numpy.newaxis, :]
554 
555  del collapsed, biasArray
556 
557  if overscanIsInt:
558  del overscanImageI
559  elif fitType in ('POLY', 'CHEB', 'LEG', 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE'):
560  if hasattr(overscanImage, "getImage"):
561  biasArray = overscanImage.getImage().getArray()
562  biasArray = numpy.ma.masked_where(overscanImage.getMask().getArray() & statControl.getAndMask(),
563  biasArray)
564  else:
565  biasArray = overscanImage.getArray()
566  # Fit along the long axis, so collapse along each short row and fit the resulting array
567  shortInd = numpy.argmin(biasArray.shape)
568  if shortInd == 0:
569  # Convert to some 'standard' representation to make things easier
570  biasArray = numpy.transpose(biasArray)
571 
572  # Do a single round of clipping to weed out CR hits and signal leaking into the overscan
573  percentiles = numpy.percentile(biasArray, [25.0, 50.0, 75.0], axis=1)
574  medianBiasArr = percentiles[1]
575  stdevBiasArr = 0.74*(percentiles[2] - percentiles[0]) # robust stdev
576  diff = numpy.abs(biasArray - medianBiasArr[:, numpy.newaxis])
577  biasMaskedArr = numpy.ma.masked_where(diff > numSigmaClip*stdevBiasArr[:, numpy.newaxis], biasArray)
578  collapsed = numpy.mean(biasMaskedArr, axis=1)
579  if collapsed.mask.sum() > 0:
580  collapsed.data[collapsed.mask] = numpy.mean(biasArray.data[collapsed.mask], axis=1)
581 
582  del biasArray, percentiles, stdevBiasArr, diff, biasMaskedArr
583 
584  if shortInd == 0:
585  collapsed = numpy.transpose(collapsed)
586 
587  num = len(collapsed)
588  indices = 2.0*numpy.arange(num)/float(num) - 1.0
589 
590  if fitType in ('POLY', 'CHEB', 'LEG'):
591  # A numpy polynomial
592  poly = numpy.polynomial
593  fitter, evaler = {"POLY": (poly.polynomial.polyfit, poly.polynomial.polyval),
594  "CHEB": (poly.chebyshev.chebfit, poly.chebyshev.chebval),
595  "LEG": (poly.legendre.legfit, poly.legendre.legval),
596  }[fitType]
597 
598  coeffs = fitter(indices, collapsed, order)
599  fitBiasArr = evaler(indices, coeffs)
600  elif 'SPLINE' in fitType:
601  # An afw interpolation
602  numBins = order
603  #
604  # numpy.histogram needs a real array for the mask, but numpy.ma "optimises" the case
605  # no-values-are-masked by replacing the mask array by a scalar, numpy.ma.nomask
606  #
607  # Issue DM-415
608  #
609  collapsedMask = collapsed.mask
610  try:
611  if collapsedMask == numpy.ma.nomask:
612  collapsedMask = numpy.array(len(collapsed)*[numpy.ma.nomask])
613  except ValueError: # If collapsedMask is an array the test fails [needs .all()]
614  pass
615 
616  numPerBin, binEdges = numpy.histogram(indices, bins=numBins,
617  weights=1-collapsedMask.astype(int))
618  # Binning is just a histogram, with weights equal to the values.
619  # Use a similar trick to get the bin centers (this deals with different numbers per bin).
620  with numpy.errstate(invalid="ignore"): # suppress NAN warnings
621  values = numpy.histogram(indices, bins=numBins,
622  weights=collapsed.data*~collapsedMask)[0]/numPerBin
623  binCenters = numpy.histogram(indices, bins=numBins,
624  weights=indices*~collapsedMask)[0]/numPerBin
625  interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0],
626  values.astype(float)[numPerBin > 0],
627  afwMath.stringToInterpStyle(fitType))
628  fitBiasArr = numpy.array([interp.interpolate(i) for i in indices])
629 
630  import lsstDebug
631  if lsstDebug.Info(__name__).display:
632  import matplotlib.pyplot as plot
633  figure = plot.figure(1)
634  figure.clear()
635  axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
636  axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+')
637  if collapsedMask.sum() > 0:
638  axes.plot(indices[collapsedMask], collapsed.data[collapsedMask], 'b+')
639  axes.plot(indices, fitBiasArr, 'r-')
640  plot.xlabel("centered/scaled position along overscan region")
641  plot.ylabel("pixel value/fit value")
642  figure.show()
643  prompt = "Press Enter or c to continue [chp]... "
644  while True:
645  ans = input(prompt).lower()
646  if ans in ("", "c",):
647  break
648  if ans in ("p",):
649  import pdb
650  pdb.set_trace()
651  elif ans in ("h", ):
652  print("h[elp] c[ontinue] p[db]")
653  plot.close()
654 
655  offImage = ampImage.Factory(ampImage.getDimensions())
656  offArray = offImage.getArray()
657  overscanFit = afwImage.ImageF(overscanImage.getDimensions())
658  overscanArray = overscanFit.getArray()
659  if shortInd == 1:
660  offArray[:, :] = fitBiasArr[:, numpy.newaxis]
661  overscanArray[:, :] = fitBiasArr[:, numpy.newaxis]
662  else:
663  offArray[:, :] = fitBiasArr[numpy.newaxis, :]
664  overscanArray[:, :] = fitBiasArr[numpy.newaxis, :]
665 
666  # We don't trust any extrapolation: mask those pixels as SUSPECT
667  # This will occur when the top and or bottom edges of the overscan
668  # contain saturated values. The values will be extrapolated from
669  # the surrounding pixels, but we cannot entirely trust the value of
670  # the extrapolation, and will mark the image mask plane to flag the
671  # image as such.
672  mask = ampMaskedImage.getMask()
673  maskArray = mask.getArray() if shortInd == 1 else mask.getArray().transpose()
674  suspect = mask.getPlaneBitMask("SUSPECT")
675  try:
676  if collapsed.mask == numpy.ma.nomask:
677  # There is no mask, so the whole array is fine
678  pass
679  except ValueError: # If collapsed.mask is an array the test fails [needs .all()]
680  for low in range(num):
681  if not collapsed.mask[low]:
682  break
683  if low > 0:
684  maskArray[:low, :] |= suspect
685  for high in range(1, num):
686  if not collapsed.mask[-high]:
687  break
688  if high > 1:
689  maskArray[-high:, :] |= suspect
690 
691  else:
692  raise pexExcept.Exception('%s : %s an invalid overscan type' % ("overscanCorrection", fitType))
693  ampImage -= offImage
694  overscanImage -= overscanFit
695  return Struct(imageFit=offImage, overscanFit=overscanFit, overscanImage=overscanImage)
696 
697 
698 def brighterFatterCorrection(exposure, kernel, maxIter, threshold, applyGain, gains=None):
699  """Apply brighter fatter correction in place for the image.
700 
701  Parameters
702  ----------
703  exposure : `lsst.afw.image.Exposure`
704  Exposure to have brighter-fatter correction applied. Modified
705  by this method.
706  kernel : `numpy.ndarray`
707  Brighter-fatter kernel to apply.
708  maxIter : scalar
709  Number of correction iterations to run.
710  threshold : scalar
711  Convergence threshold in terms of the sum of absolute
712  deviations between an iteration and the previous one.
713  applyGain : `Bool`
714  If True, then the exposure values are scaled by the gain prior
715  to correction.
716  gains : `dict` [`str`, `float`]
717  A dictionary, keyed by amplifier name, of the gains to use.
718  If gains is None, the nominal gains in the amplifier object are used.
719 
720  Returns
721  -------
722  diff : `float`
723  Final difference between iterations achieved in correction.
724  iteration : `int`
725  Number of iterations used to calculate correction.
726 
727  Notes
728  -----
729  This correction takes a kernel that has been derived from flat
730  field images to redistribute the charge. The gradient of the
731  kernel is the deflection field due to the accumulated charge.
732 
733  Given the original image I(x) and the kernel K(x) we can compute
734  the corrected image Ic(x) using the following equation:
735 
736  Ic(x) = I(x) + 0.5*d/dx(I(x)*d/dx(int( dy*K(x-y)*I(y))))
737 
738  To evaluate the derivative term we expand it as follows:
739 
740  0.5 * ( d/dx(I(x))*d/dx(int(dy*K(x-y)*I(y))) + I(x)*d^2/dx^2(int(dy* K(x-y)*I(y))) )
741 
742  Because we use the measured counts instead of the incident counts
743  we apply the correction iteratively to reconstruct the original
744  counts and the correction. We stop iterating when the summed
745  difference between the current corrected image and the one from
746  the previous iteration is below the threshold. We do not require
747  convergence because the number of iterations is too large a
748  computational cost. How we define the threshold still needs to be
749  evaluated, the current default was shown to work reasonably well
750  on a small set of images. For more information on the method see
751  DocuShare Document-19407.
752 
753  The edges as defined by the kernel are not corrected because they
754  have spurious values due to the convolution.
755  """
756  image = exposure.getMaskedImage().getImage()
757 
758  # The image needs to be units of electrons/holes
759  with gainContext(exposure, image, applyGain, gains):
760 
761  kLx = numpy.shape(kernel)[0]
762  kLy = numpy.shape(kernel)[1]
763  kernelImage = afwImage.ImageD(kLx, kLy)
764  kernelImage.getArray()[:, :] = kernel
765  tempImage = image.clone()
766 
767  nanIndex = numpy.isnan(tempImage.getArray())
768  tempImage.getArray()[nanIndex] = 0.
769 
770  outImage = afwImage.ImageF(image.getDimensions())
771  corr = numpy.zeros_like(image.getArray())
772  prev_image = numpy.zeros_like(image.getArray())
773  convCntrl = afwMath.ConvolutionControl(False, True, 1)
774  fixedKernel = afwMath.FixedKernel(kernelImage)
775 
776  # Define boundary by convolution region. The region that the correction will be
777  # calculated for is one fewer in each dimension because of the second derivative terms.
778  # NOTE: these need to use integer math, as we're using start:end as numpy index ranges.
779  startX = kLx//2
780  endX = -kLx//2
781  startY = kLy//2
782  endY = -kLy//2
783 
784  for iteration in range(maxIter):
785 
786  afwMath.convolve(outImage, tempImage, fixedKernel, convCntrl)
787  tmpArray = tempImage.getArray()
788  outArray = outImage.getArray()
789 
790  with numpy.errstate(invalid="ignore", over="ignore"):
791  # First derivative term
792  gradTmp = numpy.gradient(tmpArray[startY:endY, startX:endX])
793  gradOut = numpy.gradient(outArray[startY:endY, startX:endX])
794  first = (gradTmp[0]*gradOut[0] + gradTmp[1]*gradOut[1])[1:-1, 1:-1]
795 
796  # Second derivative term
797  diffOut20 = numpy.diff(outArray, 2, 0)[startY:endY, startX + 1:endX - 1]
798  diffOut21 = numpy.diff(outArray, 2, 1)[startY + 1:endY - 1, startX:endX]
799  second = tmpArray[startY + 1:endY - 1, startX + 1:endX - 1]*(diffOut20 + diffOut21)
800 
801  corr[startY + 1:endY - 1, startX + 1:endX - 1] = 0.5*(first + second)
802 
803  tmpArray[:, :] = image.getArray()[:, :]
804  tmpArray[nanIndex] = 0.
805  tmpArray[startY:endY, startX:endX] += corr[startY:endY, startX:endX]
806 
807  if iteration > 0:
808  diff = numpy.sum(numpy.abs(prev_image - tmpArray))
809 
810  if diff < threshold:
811  break
812  prev_image[:, :] = tmpArray[:, :]
813 
814  image.getArray()[startY + 1:endY - 1, startX + 1:endX - 1] += \
815  corr[startY + 1:endY - 1, startX + 1:endX - 1]
816 
817  return diff, iteration
818 
819 
820 @contextmanager
821 def gainContext(exp, image, apply, gains=None):
822  """Context manager that applies and removes gain.
823 
824  Parameters
825  ----------
826  exp : `lsst.afw.image.Exposure`
827  Exposure to apply/remove gain.
828  image : `lsst.afw.image.Image`
829  Image to apply/remove gain.
830  apply : `Bool`
831  If True, apply and remove the amplifier gain.
832  gains : `dict` [`str`, `float`]
833  A dictionary, keyed by amplifier name, of the gains to use.
834  If gains is None, the nominal gains in the amplifier object are used.
835 
836  Yields
837  ------
838  exp : `lsst.afw.image.Exposure`
839  Exposure with the gain applied.
840  """
841  # check we have all of them if provided because mixing and matching would
842  # be a real mess
843  if gains and apply is True:
844  ampNames = [amp.getName() for amp in exp.getDetector()]
845  for ampName in ampNames:
846  if ampName not in gains.keys():
847  raise RuntimeError(f"Gains provided to gain context, but no entry found for amp {ampName}")
848 
849  if apply:
850  ccd = exp.getDetector()
851  for amp in ccd:
852  sim = image.Factory(image, amp.getBBox())
853  if gains:
854  gain = gains[amp.getName()]
855  else:
856  gain = amp.getGain()
857  sim *= gain
858 
859  try:
860  yield exp
861  finally:
862  if apply:
863  ccd = exp.getDetector()
864  for amp in ccd:
865  sim = image.Factory(image, amp.getBBox())
866  if gains:
867  gain = gains[amp.getName()]
868  else:
869  gain = amp.getGain()
870  sim /= gain
871 
872 
873 def attachTransmissionCurve(exposure, opticsTransmission=None, filterTransmission=None,
874  sensorTransmission=None, atmosphereTransmission=None):
875  """Attach a TransmissionCurve to an Exposure, given separate curves for
876  different components.
877 
878  Parameters
879  ----------
880  exposure : `lsst.afw.image.Exposure`
881  Exposure object to modify by attaching the product of all given
882  ``TransmissionCurves`` in post-assembly trimmed detector coordinates.
883  Must have a valid ``Detector`` attached that matches the detector
884  associated with sensorTransmission.
885  opticsTransmission : `lsst.afw.image.TransmissionCurve`
886  A ``TransmissionCurve`` that represents the throughput of the optics,
887  to be evaluated in focal-plane coordinates.
888  filterTransmission : `lsst.afw.image.TransmissionCurve`
889  A ``TransmissionCurve`` that represents the throughput of the filter
890  itself, to be evaluated in focal-plane coordinates.
891  sensorTransmission : `lsst.afw.image.TransmissionCurve`
892  A ``TransmissionCurve`` that represents the throughput of the sensor
893  itself, to be evaluated in post-assembly trimmed detector coordinates.
894  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
895  A ``TransmissionCurve`` that represents the throughput of the
896  atmosphere, assumed to be spatially constant.
897 
898  Returns
899  -------
900  combined : `lsst.afw.image.TransmissionCurve`
901  The TransmissionCurve attached to the exposure.
902 
903  Notes
904  -----
905  All ``TransmissionCurve`` arguments are optional; if none are provided, the
906  attached ``TransmissionCurve`` will have unit transmission everywhere.
907  """
908  combined = afwImage.TransmissionCurve.makeIdentity()
909  if atmosphereTransmission is not None:
910  combined *= atmosphereTransmission
911  if opticsTransmission is not None:
912  combined *= opticsTransmission
913  if filterTransmission is not None:
914  combined *= filterTransmission
915  detector = exposure.getDetector()
916  fpToPix = detector.getTransform(fromSys=camGeom.FOCAL_PLANE,
917  toSys=camGeom.PIXELS)
918  combined = combined.transformedBy(fpToPix)
919  if sensorTransmission is not None:
920  combined *= sensorTransmission
921  exposure.getInfo().setTransmissionCurve(combined)
922  return combined
923 
924 
925 @deprecated(reason="Camera geometry-based SkyWcs are now set when reading raws. To be removed after v19.",
926  category=FutureWarning)
927 def addDistortionModel(exposure, camera):
928  """!Update the WCS in exposure with a distortion model based on camera
929  geometry.
930 
931  Parameters
932  ----------
933  exposure : `lsst.afw.image.Exposure`
934  Exposure to process. Must contain a Detector and WCS. The
935  exposure is modified.
936  camera : `lsst.afw.cameraGeom.Camera`
937  Camera geometry.
938 
939  Raises
940  ------
941  RuntimeError
942  Raised if ``exposure`` is lacking a Detector or WCS, or if
943  ``camera`` is None.
944  Notes
945  -----
946  Add a model for optical distortion based on geometry found in ``camera``
947  and the ``exposure``'s detector. The raw input exposure is assumed
948  have a TAN WCS that has no compensation for optical distortion.
949  Two other possibilities are:
950  - The raw input exposure already has a model for optical distortion,
951  as is the case for raw DECam data.
952  In that case you should set config.doAddDistortionModel False.
953  - The raw input exposure has a model for distortion, but it has known
954  deficiencies severe enough to be worth fixing (e.g. because they
955  cause problems for fitting a better WCS). In that case you should
956  override this method with a version suitable for your raw data.
957 
958  """
959  wcs = exposure.getWcs()
960  if wcs is None:
961  raise RuntimeError("exposure has no WCS")
962  if camera is None:
963  raise RuntimeError("camera is None")
964  detector = exposure.getDetector()
965  if detector is None:
966  raise RuntimeError("exposure has no Detector")
967  pixelToFocalPlane = detector.getTransform(camGeom.PIXELS, camGeom.FOCAL_PLANE)
968  focalPlaneToFieldAngle = camera.getTransformMap().getTransform(camGeom.FOCAL_PLANE,
969  camGeom.FIELD_ANGLE)
970  distortedWcs = makeDistortedTanWcs(wcs, pixelToFocalPlane, focalPlaneToFieldAngle)
971  exposure.setWcs(distortedWcs)
972 
973 
974 def applyGains(exposure, normalizeGains=False):
975  """Scale an exposure by the amplifier gains.
976 
977  Parameters
978  ----------
979  exposure : `lsst.afw.image.Exposure`
980  Exposure to process. The image is modified.
981  normalizeGains : `Bool`, optional
982  If True, then amplifiers are scaled to force the median of
983  each amplifier to equal the median of those medians.
984  """
985  ccd = exposure.getDetector()
986  ccdImage = exposure.getMaskedImage()
987 
988  medians = []
989  for amp in ccd:
990  sim = ccdImage.Factory(ccdImage, amp.getBBox())
991  sim *= amp.getGain()
992 
993  if normalizeGains:
994  medians.append(numpy.median(sim.getImage().getArray()))
995 
996  if normalizeGains:
997  median = numpy.median(numpy.array(medians))
998  for index, amp in enumerate(ccd):
999  sim = ccdImage.Factory(ccdImage, amp.getBBox())
1000  if medians[index] != 0.0:
1001  sim *= median/medians[index]
1002 
1003 
1005  """Grow the saturation trails by an amount dependent on the width of the trail.
1006 
1007  Parameters
1008  ----------
1009  mask : `lsst.afw.image.Mask`
1010  Mask which will have the saturated areas grown.
1011  """
1012 
1013  extraGrowDict = {}
1014  for i in range(1, 6):
1015  extraGrowDict[i] = 0
1016  for i in range(6, 8):
1017  extraGrowDict[i] = 1
1018  for i in range(8, 10):
1019  extraGrowDict[i] = 3
1020  extraGrowMax = 4
1021 
1022  if extraGrowMax <= 0:
1023  return
1024 
1025  saturatedBit = mask.getPlaneBitMask("SAT")
1026 
1027  xmin, ymin = mask.getBBox().getMin()
1028  width = mask.getWidth()
1029 
1030  thresh = afwDetection.Threshold(saturatedBit, afwDetection.Threshold.BITMASK)
1031  fpList = afwDetection.FootprintSet(mask, thresh).getFootprints()
1032 
1033  for fp in fpList:
1034  for s in fp.getSpans():
1035  x0, x1 = s.getX0(), s.getX1()
1036 
1037  extraGrow = extraGrowDict.get(x1 - x0 + 1, extraGrowMax)
1038  if extraGrow > 0:
1039  y = s.getY() - ymin
1040  x0 -= xmin + extraGrow
1041  x1 -= xmin - extraGrow
1042 
1043  if x0 < 0:
1044  x0 = 0
1045  if x1 >= width - 1:
1046  x1 = width - 1
1047 
1048  mask.array[y, x0:x1+1] |= saturatedBit
1049 
1050 
1051 def setBadRegions(exposure, badStatistic="MEDIAN"):
1052  """Set all BAD areas of the chip to the average of the rest of the exposure
1053 
1054  Parameters
1055  ----------
1056  exposure : `lsst.afw.image.Exposure`
1057  Exposure to mask. The exposure mask is modified.
1058  badStatistic : `str`, optional
1059  Statistic to use to generate the replacement value from the
1060  image data. Allowed values are 'MEDIAN' or 'MEANCLIP'.
1061 
1062  Returns
1063  -------
1064  badPixelCount : scalar
1065  Number of bad pixels masked.
1066  badPixelValue : scalar
1067  Value substituted for bad pixels.
1068 
1069  Raises
1070  ------
1071  RuntimeError
1072  Raised if `badStatistic` is not an allowed value.
1073  """
1074  if badStatistic == "MEDIAN":
1075  statistic = afwMath.MEDIAN
1076  elif badStatistic == "MEANCLIP":
1077  statistic = afwMath.MEANCLIP
1078  else:
1079  raise RuntimeError("Impossible method %s of bad region correction" % badStatistic)
1080 
1081  mi = exposure.getMaskedImage()
1082  mask = mi.getMask()
1083  BAD = mask.getPlaneBitMask("BAD")
1084  INTRP = mask.getPlaneBitMask("INTRP")
1085 
1086  sctrl = afwMath.StatisticsControl()
1087  sctrl.setAndMask(BAD)
1088  value = afwMath.makeStatistics(mi, statistic, sctrl).getValue()
1089 
1090  maskArray = mask.getArray()
1091  imageArray = mi.getImage().getArray()
1092  badPixels = numpy.logical_and((maskArray & BAD) > 0, (maskArray & INTRP) == 0)
1093  imageArray[:] = numpy.where(badPixels, value, imageArray)
1094 
1095  return badPixels.sum(), value
def addDistortionModel(exposure, camera)
Update the WCS in exposure with a distortion model based on camera geometry.
def saturationCorrection(maskedImage, saturation, fwhm, growFootprints=1, interpolate=True, maskName='SAT', fallbackValue=None)
def setBadRegions(exposure, badStatistic="MEDIAN")
def interpolateFromMask(maskedImage, fwhm, growSaturatedFootprints=1, maskNameList=['SAT'], fallbackValue=None)
def interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=None)
Definition: isrFunctions.py:78
def brighterFatterCorrection(exposure, kernel, maxIter, threshold, applyGain, gains=None)
def transposeMaskedImage(maskedImage)
Definition: isrFunctions.py:58
def trimToMatchCalibBBox(rawMaskedImage, calibMaskedImage)
def attachTransmissionCurve(exposure, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None)
def flatCorrection(maskedImage, flatMaskedImage, scalingType, userScale=1.0, invert=False, trimToFit=False)
def makeThresholdMask(maskedImage, threshold, growFootprints=1, maskName='SAT')
def growMasks(mask, radius=0, maskNameList=['BAD'], maskValue="BAD")
def applyGains(exposure, normalizeGains=False)
def darkCorrection(maskedImage, darkMaskedImage, expScale, darkScale, invert=False, trimToFit=False)
def biasCorrection(maskedImage, biasMaskedImage, trimToFit=False)
def gainContext(exp, image, apply, gains=None)
def illuminationCorrection(maskedImage, illumMaskedImage, illumScale, trimToFit=True)
def updateVariance(maskedImage, gain, readNoise)
def overscanCorrection(ampMaskedImage, overscanImage, fitType='MEDIAN', order=1, collapseRej=3.0, statControl=None, overscanIsInt=True)