lsst.ip.isr  18.1.0-12-gc95f69a+5
isrFunctions.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import math
23 import numpy
24 from deprecated.sphinx import deprecated
25 
26 import lsst.geom
27 import lsst.afw.image as afwImage
28 import lsst.afw.detection as afwDetection
29 import lsst.afw.math as afwMath
30 import lsst.meas.algorithms as measAlg
31 import lsst.pex.exceptions as pexExcept
32 import lsst.afw.cameraGeom as camGeom
33 
34 from lsst.afw.geom.wcsUtils import makeDistortedTanWcs
35 from lsst.meas.algorithms.detection import SourceDetectionTask
36 from lsst.pipe.base import Struct
37 
38 from contextlib import contextmanager
39 
40 
41 def createPsf(fwhm):
42  """Make a double Gaussian PSF.
43 
44  Parameters
45  ----------
46  fwhm : scalar
47  FWHM of double Gaussian smoothing kernel.
48 
49  Returns
50  -------
51  psf : `lsst.meas.algorithms.DoubleGaussianPsf`
52  The created smoothing kernel.
53  """
54  ksize = 4*int(fwhm) + 1
55  return measAlg.DoubleGaussianPsf(ksize, ksize, fwhm/(2*math.sqrt(2*math.log(2))))
56 
57 
58 def transposeMaskedImage(maskedImage):
59  """Make a transposed copy of a masked image.
60 
61  Parameters
62  ----------
63  maskedImage : `lsst.afw.image.MaskedImage`
64  Image to process.
65 
66  Returns
67  -------
68  transposed : `lsst.afw.image.MaskedImage`
69  The transposed copy of the input image.
70  """
71  transposed = maskedImage.Factory(lsst.geom.Extent2I(maskedImage.getHeight(), maskedImage.getWidth()))
72  transposed.getImage().getArray()[:] = maskedImage.getImage().getArray().T
73  transposed.getMask().getArray()[:] = maskedImage.getMask().getArray().T
74  transposed.getVariance().getArray()[:] = maskedImage.getVariance().getArray().T
75  return transposed
76 
77 
78 def interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=None):
79  """Interpolate over defects specified in a defect list.
80 
81  Parameters
82  ----------
83  maskedImage : `lsst.afw.image.MaskedImage`
84  Image to process.
85  defectList : `lsst.meas.algorithms.Defects`
86  List of defects to interpolate over.
87  fwhm : scalar
88  FWHM of double Gaussian smoothing kernel.
89  fallbackValue : scalar, optional
90  Fallback value if an interpolated value cannot be determined.
91  If None, then the clipped mean of the image is used.
92  """
93  psf = createPsf(fwhm)
94  if fallbackValue is None:
95  fallbackValue = afwMath.makeStatistics(maskedImage.getImage(), afwMath.MEANCLIP).getValue()
96  if 'INTRP' not in maskedImage.getMask().getMaskPlaneDict():
97  maskedImage.getMask().addMaskPlane('INTRP')
98  measAlg.interpolateOverDefects(maskedImage, psf, defectList, fallbackValue, True)
99  return maskedImage
100 
101 
102 @deprecated(reason="Replaced by Defects.fromFootPrintList() (will be removed after v18)",
103  category=FutureWarning)
105  """Compute a defect list from a footprint list, optionally growing the footprints.
106 
107  Parameters
108  ----------
109  fpList : `list` of `lsst.afw.detection.Footprint`
110  Footprint list to process.
111 
112  Returns
113  -------
114  defectList : `lsst.meas.algorithms.Defects`
115  List of defects.
116  """
117  return measAlg.Defects.fromFootprintList(fpList)
118 
119 @deprecated(reason="Replaced by Defects.transpose() (will be removed after v18)",
120  category=FutureWarning)
121 def transposeDefectList(defectList):
122  """Make a transposed copy of a defect list.
123 
124  Parameters
125  ----------
126  defectList : `lsst.meas.algorithms.Defects`
127  Input list of defects.
128 
129  Returns
130  -------
131  retDefectList : `lsst.meas.algorithms.Defects`
132  Transposed list of defects.
133  """
134  if isinstance(defectList, measAlg.Defects):
135  return defectList.transpose()
136  return measAlg.Defects(defectList).transpose()
137 
138 
139 @deprecated(reason="Replaced by Defects.maskPixels() (will be removed after v18)",
140  category=FutureWarning)
141 def maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD'):
142  """Set mask plane based on a defect list.
143 
144  Parameters
145  ----------
146  maskedImage : `lsst.afw.image.MaskedImage`
147  Image to process. Only the mask plane is updated.
148  defectList : `lsst.meas.algorithms.Defects`
149  Defect list to mask.
150  maskName : str, optional
151  Mask plane name to use.
152  """
153  return lsst.meas.algorithms.Defects(defectList).maskPixels(maskedImage, maskName=maskName)
154 
155 
156 @deprecated(reason="Replaced by Defects.fromMask() (will be removed after v18)",
157  category=FutureWarning)
158 def getDefectListFromMask(maskedImage, maskName):
159  """Compute a defect list from a specified mask plane.
160 
161  Parameters
162  ----------
163  maskedImage : `lsst.afw.image.MaskedImage`
164  Image to process.
165  maskName : `str` or `list`
166  Mask plane name, or list of names to convert.
167 
168  Returns
169  -------
170  defectList : `lsst.meas.algorithms.Defects`
171  Defect list constructed from masked pixels.
172  """
173  return measAlg.Defects.fromMask(maskedImage, maskName)
174 
175 
176 def makeThresholdMask(maskedImage, threshold, growFootprints=1, maskName='SAT'):
177  """Mask pixels based on threshold detection.
178 
179  Parameters
180  ----------
181  maskedImage : `lsst.afw.image.MaskedImage`
182  Image to process. Only the mask plane is updated.
183  threshold : scalar
184  Detection threshold.
185  growFootprints : scalar, optional
186  Number of pixels to grow footprints of detected regions.
187  maskName : str, optional
188  Mask plane name, or list of names to convert
189 
190  Returns
191  -------
192  defectList : `lsst.meas.algorithms.Defects`
193  Defect list constructed from pixels above the threshold.
194  """
195  # find saturated regions
196  thresh = afwDetection.Threshold(threshold)
197  fs = afwDetection.FootprintSet(maskedImage, thresh)
198 
199  if growFootprints > 0:
200  fs = afwDetection.FootprintSet(fs, rGrow=growFootprints, isotropic=False)
201  fpList = fs.getFootprints()
202 
203  # set mask
204  mask = maskedImage.getMask()
205  bitmask = mask.getPlaneBitMask(maskName)
206  afwDetection.setMaskFromFootprintList(mask, fpList, bitmask)
207 
208  return measAlg.Defects.fromFootprintList(fpList)
209 
210 
211 def interpolateFromMask(maskedImage, fwhm, growSaturatedFootprints=1,
212  maskNameList=['SAT'], fallbackValue=None):
213  """Interpolate over defects identified by a particular set of mask planes.
214 
215  Parameters
216  ----------
217  maskedImage : `lsst.afw.image.MaskedImage`
218  Image to process.
219  fwhm : scalar
220  FWHM of double Gaussian smoothing kernel.
221  growSaturatedFootprints : scalar, optional
222  Number of pixels to grow footprints for saturated pixels.
223  maskNameList : `List` of `str`, optional
224  Mask plane name.
225  fallbackValue : scalar, optional
226  Value of last resort for interpolation.
227  """
228  mask = maskedImage.getMask()
229 
230  if growSaturatedFootprints > 0 and "SAT" in maskNameList:
231  thresh = afwDetection.Threshold(mask.getPlaneBitMask("SAT"), afwDetection.Threshold.BITMASK)
232  fpSet = afwDetection.FootprintSet(mask, thresh)
233  # If we are interpolating over an area larger than the original masked region, we need
234  # to expand the original mask bit to the full area to explain why we interpolated there.
235  fpSet = afwDetection.FootprintSet(fpSet, rGrow=growSaturatedFootprints, isotropic=False)
236  fpSet.setMask(mask, "SAT")
237 
238  thresh = afwDetection.Threshold(mask.getPlaneBitMask(maskNameList), afwDetection.Threshold.BITMASK)
239  fpSet = afwDetection.FootprintSet(mask, thresh)
240  defectList = measAlg.Defects.fromFootprintList(fpSet.getFootprints())
241 
242  interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=fallbackValue)
243 
244  return maskedImage
245 
246 
247 def saturationCorrection(maskedImage, saturation, fwhm, growFootprints=1, interpolate=True, maskName='SAT',
248  fallbackValue=None):
249  """Mark saturated pixels and optionally interpolate over them
250 
251  Parameters
252  ----------
253  maskedImage : `lsst.afw.image.MaskedImage`
254  Image to process.
255  saturation : scalar
256  Saturation level used as the detection threshold.
257  fwhm : scalar
258  FWHM of double Gaussian smoothing kernel.
259  growFootprints : scalar, optional
260  Number of pixels to grow footprints of detected regions.
261  interpolate : Bool, optional
262  If True, saturated pixels are interpolated over.
263  maskName : str, optional
264  Mask plane name.
265  fallbackValue : scalar, optional
266  Value of last resort for interpolation.
267  """
268  defectList = makeThresholdMask(
269  maskedImage=maskedImage,
270  threshold=saturation,
271  growFootprints=growFootprints,
272  maskName=maskName,
273  )
274  if interpolate:
275  interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=fallbackValue)
276 
277  return maskedImage
278 
279 
280 def trimToMatchCalibBBox(rawMaskedImage, calibMaskedImage):
281  """Compute number of edge trim pixels to match the calibration data.
282 
283  Use the dimension difference between the raw exposure and the
284  calibration exposure to compute the edge trim pixels. This trim
285  is applied symmetrically, with the same number of pixels masked on
286  each side.
287 
288  Parameters
289  ----------
290  rawMaskedImage : `lsst.afw.image.MaskedImage`
291  Image to trim.
292  calibMaskedImage : `lsst.afw.image.MaskedImage`
293  Calibration image to draw new bounding box from.
294 
295  Returns
296  -------
297  replacementMaskedImage : `lsst.afw.image.MaskedImage`
298  ``rawMaskedImage`` trimmed to the appropriate size
299  Raises
300  ------
301  RuntimeError
302  Rasied if ``rawMaskedImage`` cannot be symmetrically trimmed to
303  match ``calibMaskedImage``.
304  """
305  nx, ny = rawMaskedImage.getBBox().getDimensions() - calibMaskedImage.getBBox().getDimensions()
306  if nx != ny:
307  raise RuntimeError("Raw and calib maskedImages are trimmed differently in X and Y.")
308  if nx % 2 != 0:
309  raise RuntimeError("Calibration maskedImage is trimmed unevenly in X.")
310  if nx < 0:
311  raise RuntimeError("Calibration maskedImage is larger than raw data.")
312 
313  nEdge = nx//2
314  if nEdge > 0:
315  replacementMaskedImage = rawMaskedImage[nEdge:-nEdge, nEdge:-nEdge, afwImage.LOCAL]
316  SourceDetectionTask.setEdgeBits(
317  rawMaskedImage,
318  replacementMaskedImage.getBBox(),
319  rawMaskedImage.getMask().getPlaneBitMask("EDGE")
320  )
321  else:
322  replacementMaskedImage = rawMaskedImage
323 
324  return replacementMaskedImage
325 
326 
327 def biasCorrection(maskedImage, biasMaskedImage, trimToFit=False):
328  """Apply bias correction in place.
329 
330  Parameters
331  ----------
332  maskedImage : `lsst.afw.image.MaskedImage`
333  Image to process. The image is modified by this method.
334  biasMaskedImage : `lsst.afw.image.MaskedImage`
335  Bias image of the same size as ``maskedImage``
336  trimToFit : `Bool`, optional
337  If True, raw data is symmetrically trimmed to match
338  calibration size.
339 
340  Raises
341  ------
342  RuntimeError
343  Raised if ``maskedImage`` and ``biasMaskedImage`` do not have
344  the same size.
345 
346  """
347  if trimToFit:
348  maskedImage = trimToMatchCalibBBox(maskedImage, biasMaskedImage)
349 
350  if maskedImage.getBBox(afwImage.LOCAL) != biasMaskedImage.getBBox(afwImage.LOCAL):
351  raise RuntimeError("maskedImage bbox %s != biasMaskedImage bbox %s" %
352  (maskedImage.getBBox(afwImage.LOCAL), biasMaskedImage.getBBox(afwImage.LOCAL)))
353  maskedImage -= biasMaskedImage
354 
355 
356 def darkCorrection(maskedImage, darkMaskedImage, expScale, darkScale, invert=False, trimToFit=False):
357  """Apply dark correction in place.
358 
359  Parameters
360  ----------
361  maskedImage : `lsst.afw.image.MaskedImage`
362  Image to process. The image is modified by this method.
363  darkMaskedImage : `lsst.afw.image.MaskedImage`
364  Dark image of the same size as ``maskedImage``.
365  expScale : scalar
366  Dark exposure time for ``maskedImage``.
367  darkScale : scalar
368  Dark exposure time for ``darkMaskedImage``.
369  invert : `Bool`, optional
370  If True, re-add the dark to an already corrected image.
371  trimToFit : `Bool`, optional
372  If True, raw data is symmetrically trimmed to match
373  calibration size.
374 
375  Raises
376  ------
377  RuntimeError
378  Raised if ``maskedImage`` and ``darkMaskedImage`` do not have
379  the same size.
380 
381  Notes
382  -----
383  The dark correction is applied by calculating:
384  maskedImage -= dark * expScaling / darkScaling
385  """
386  if trimToFit:
387  maskedImage = trimToMatchCalibBBox(maskedImage, darkMaskedImage)
388 
389  if maskedImage.getBBox(afwImage.LOCAL) != darkMaskedImage.getBBox(afwImage.LOCAL):
390  raise RuntimeError("maskedImage bbox %s != darkMaskedImage bbox %s" %
391  (maskedImage.getBBox(afwImage.LOCAL), darkMaskedImage.getBBox(afwImage.LOCAL)))
392 
393  scale = expScale / darkScale
394  if not invert:
395  maskedImage.scaledMinus(scale, darkMaskedImage)
396  else:
397  maskedImage.scaledPlus(scale, darkMaskedImage)
398 
399 
400 def updateVariance(maskedImage, gain, readNoise):
401  """Set the variance plane based on the image plane.
402 
403  Parameters
404  ----------
405  maskedImage : `lsst.afw.image.MaskedImage`
406  Image to process. The variance plane is modified.
407  gain : scalar
408  The amplifier gain in electrons/ADU.
409  readNoise : scalar
410  The amplifier read nmoise in ADU/pixel.
411  """
412  var = maskedImage.getVariance()
413  var[:] = maskedImage.getImage()
414  var /= gain
415  var += readNoise**2
416 
417 
418 def flatCorrection(maskedImage, flatMaskedImage, scalingType, userScale=1.0, invert=False, trimToFit=False):
419  """Apply flat correction in place.
420 
421  Parameters
422  ----------
423  maskedImage : `lsst.afw.image.MaskedImage`
424  Image to process. The image is modified.
425  flatMaskedImage : `lsst.afw.image.MaskedImage`
426  Flat image of the same size as ``maskedImage``
427  scalingType : str
428  Flat scale computation method. Allowed values are 'MEAN',
429  'MEDIAN', or 'USER'.
430  userScale : scalar, optional
431  Scale to use if ``scalingType``='USER'.
432  invert : `Bool`, optional
433  If True, unflatten an already flattened image.
434  trimToFit : `Bool`, optional
435  If True, raw data is symmetrically trimmed to match
436  calibration size.
437 
438  Raises
439  ------
440  RuntimeError
441  Raised if ``maskedImage`` and ``flatMaskedImage`` do not have
442  the same size or if ``scalingType`` is not an allowed value.
443  """
444  if trimToFit:
445  maskedImage = trimToMatchCalibBBox(maskedImage, flatMaskedImage)
446 
447  if maskedImage.getBBox(afwImage.LOCAL) != flatMaskedImage.getBBox(afwImage.LOCAL):
448  raise RuntimeError("maskedImage bbox %s != flatMaskedImage bbox %s" %
449  (maskedImage.getBBox(afwImage.LOCAL), flatMaskedImage.getBBox(afwImage.LOCAL)))
450 
451  # Figure out scale from the data
452  # Ideally the flats are normalized by the calibration product pipeline, but this allows some flexibility
453  # in the case that the flat is created by some other mechanism.
454  if scalingType in ('MEAN', 'MEDIAN'):
455  scalingType = afwMath.stringToStatisticsProperty(scalingType)
456  flatScale = afwMath.makeStatistics(flatMaskedImage.image, scalingType).getValue()
457  elif scalingType == 'USER':
458  flatScale = userScale
459  else:
460  raise RuntimeError('%s : %s not implemented' % ("flatCorrection", scalingType))
461 
462  if not invert:
463  maskedImage.scaledDivides(1.0/flatScale, flatMaskedImage)
464  else:
465  maskedImage.scaledMultiplies(1.0/flatScale, flatMaskedImage)
466 
467 
468 def illuminationCorrection(maskedImage, illumMaskedImage, illumScale, trimToFit=True):
469  """Apply illumination correction in place.
470 
471  Parameters
472  ----------
473  maskedImage : `lsst.afw.image.MaskedImage`
474  Image to process. The image is modified.
475  illumMaskedImage : `lsst.afw.image.MaskedImage`
476  Illumination correction image of the same size as ``maskedImage``.
477  illumScale : scalar
478  Scale factor for the illumination correction.
479  trimToFit : `Bool`, optional
480  If True, raw data is symmetrically trimmed to match
481  calibration size.
482 
483  Raises
484  ------
485  RuntimeError
486  Raised if ``maskedImage`` and ``illumMaskedImage`` do not have
487  the same size.
488  """
489  if trimToFit:
490  maskedImage = trimToMatchCalibBBox(maskedImage, illumMaskedImage)
491 
492  if maskedImage.getBBox(afwImage.LOCAL) != illumMaskedImage.getBBox(afwImage.LOCAL):
493  raise RuntimeError("maskedImage bbox %s != illumMaskedImage bbox %s" %
494  (maskedImage.getBBox(afwImage.LOCAL), illumMaskedImage.getBBox(afwImage.LOCAL)))
495 
496  maskedImage.scaledDivides(1.0/illumScale, illumMaskedImage)
497 
498 
499 def overscanCorrection(ampMaskedImage, overscanImage, fitType='MEDIAN', order=1, collapseRej=3.0,
500  statControl=None, overscanIsInt=True):
501  """Apply overscan correction in place.
502 
503  Parameters
504  ----------
505  ampMaskedImage : `lsst.afw.image.MaskedImage`
506  Image of amplifier to correct; modified.
507  overscanImage : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
508  Image of overscan; modified.
509  fitType : `str`
510  Type of fit for overscan correction. May be one of:
511 
512  - ``MEAN``: use mean of overscan.
513  - ``MEANCLIP``: use clipped mean of overscan.
514  - ``MEDIAN``: use median of overscan.
515  - ``POLY``: fit with ordinary polynomial.
516  - ``CHEB``: fit with Chebyshev polynomial.
517  - ``LEG``: fit with Legendre polynomial.
518  - ``NATURAL_SPLINE``: fit with natural spline.
519  - ``CUBIC_SPLINE``: fit with cubic spline.
520  - ``AKIMA_SPLINE``: fit with Akima spline.
521 
522  order : `int`
523  Polynomial order or number of spline knots; ignored unless
524  ``fitType`` indicates a polynomial or spline.
525  statControl : `lsst.afw.math.StatisticsControl`
526  Statistics control object. In particular, we pay attention to numSigmaClip
527  overscanIsInt : `bool`
528  Treat the overscan region as consisting of integers, even if it's been
529  converted to float. E.g. handle ties properly.
530 
531  Returns
532  -------
533  result : `lsst.pipe.base.Struct`
534  Result struct with components:
535 
536  - ``imageFit``: Value(s) removed from image (scalar or
537  `lsst.afw.image.Image`)
538  - ``overscanFit``: Value(s) removed from overscan (scalar or
539  `lsst.afw.image.Image`)
540  - ``overscanImage``: Overscan corrected overscan region
541  (`lsst.afw.image.Image`)
542  Raises
543  ------
544  pexExcept.Exception
545  Raised if ``fitType`` is not an allowed value.
546 
547  Notes
548  -----
549  The ``ampMaskedImage`` and ``overscanImage`` are modified, with the fit
550  subtracted. Note that the ``overscanImage`` should not be a subimage of
551  the ``ampMaskedImage``, to avoid being subtracted twice.
552 
553  Debug plots are available for the SPLINE fitTypes by setting the
554  `debug.display` for `name` == "lsst.ip.isr.isrFunctions". These
555  plots show the scatter plot of the overscan data (collapsed along
556  the perpendicular dimension) as a function of position on the CCD
557  (normalized between +/-1).
558  """
559  ampImage = ampMaskedImage.getImage()
560  if statControl is None:
561  statControl = afwMath.StatisticsControl()
562 
563  numSigmaClip = statControl.getNumSigmaClip()
564 
565  if fitType in ('MEAN', 'MEANCLIP'):
566  fitType = afwMath.stringToStatisticsProperty(fitType)
567  offImage = afwMath.makeStatistics(overscanImage, fitType, statControl).getValue()
568  overscanFit = offImage
569  elif fitType in ('MEDIAN',):
570  if overscanIsInt:
571  # we need an image with integer pixels to handle ties properly
572  if hasattr(overscanImage, "image"):
573  imageI = overscanImage.image.convertI()
574  overscanImageI = afwImage.MaskedImageI(imageI, overscanImage.mask, overscanImage.variance)
575  else:
576  overscanImageI = overscanImage.convertI()
577  else:
578  overscanImageI = overscanImage
579 
580  fitType = afwMath.stringToStatisticsProperty(fitType)
581  offImage = afwMath.makeStatistics(overscanImageI, fitType, statControl).getValue()
582  overscanFit = offImage
583 
584  if overscanIsInt:
585  del overscanImageI
586  elif fitType in ('POLY', 'CHEB', 'LEG', 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE'):
587  if hasattr(overscanImage, "getImage"):
588  biasArray = overscanImage.getImage().getArray()
589  biasArray = numpy.ma.masked_where(overscanImage.getMask().getArray() & statControl.getAndMask(),
590  biasArray)
591  else:
592  biasArray = overscanImage.getArray()
593  # Fit along the long axis, so collapse along each short row and fit the resulting array
594  shortInd = numpy.argmin(biasArray.shape)
595  if shortInd == 0:
596  # Convert to some 'standard' representation to make things easier
597  biasArray = numpy.transpose(biasArray)
598 
599  # Do a single round of clipping to weed out CR hits and signal leaking into the overscan
600  percentiles = numpy.percentile(biasArray, [25.0, 50.0, 75.0], axis=1)
601  medianBiasArr = percentiles[1]
602  stdevBiasArr = 0.74*(percentiles[2] - percentiles[0]) # robust stdev
603  diff = numpy.abs(biasArray - medianBiasArr[:, numpy.newaxis])
604  biasMaskedArr = numpy.ma.masked_where(diff > numSigmaClip*stdevBiasArr[:, numpy.newaxis], biasArray)
605  collapsed = numpy.mean(biasMaskedArr, axis=1)
606  if collapsed.mask.sum() > 0:
607  collapsed.data[collapsed.mask] = numpy.mean(biasArray.data[collapsed.mask], axis=1)
608  del biasArray, percentiles, stdevBiasArr, diff, biasMaskedArr
609 
610  if shortInd == 0:
611  collapsed = numpy.transpose(collapsed)
612 
613  num = len(collapsed)
614  indices = 2.0*numpy.arange(num)/float(num) - 1.0
615 
616  if fitType in ('POLY', 'CHEB', 'LEG'):
617  # A numpy polynomial
618  poly = numpy.polynomial
619  fitter, evaler = {"POLY": (poly.polynomial.polyfit, poly.polynomial.polyval),
620  "CHEB": (poly.chebyshev.chebfit, poly.chebyshev.chebval),
621  "LEG": (poly.legendre.legfit, poly.legendre.legval),
622  }[fitType]
623 
624  coeffs = fitter(indices, collapsed, order)
625  fitBiasArr = evaler(indices, coeffs)
626  elif 'SPLINE' in fitType:
627  # An afw interpolation
628  numBins = order
629  #
630  # numpy.histogram needs a real array for the mask, but numpy.ma "optimises" the case
631  # no-values-are-masked by replacing the mask array by a scalar, numpy.ma.nomask
632  #
633  # Issue DM-415
634  #
635  collapsedMask = collapsed.mask
636  try:
637  if collapsedMask == numpy.ma.nomask:
638  collapsedMask = numpy.array(len(collapsed)*[numpy.ma.nomask])
639  except ValueError: # If collapsedMask is an array the test fails [needs .all()]
640  pass
641 
642  numPerBin, binEdges = numpy.histogram(indices, bins=numBins,
643  weights=1-collapsedMask.astype(int))
644  # Binning is just a histogram, with weights equal to the values.
645  # Use a similar trick to get the bin centers (this deals with different numbers per bin).
646  with numpy.errstate(invalid="ignore"): # suppress NAN warnings
647  values = numpy.histogram(indices, bins=numBins,
648  weights=collapsed.data*~collapsedMask)[0]/numPerBin
649  binCenters = numpy.histogram(indices, bins=numBins,
650  weights=indices*~collapsedMask)[0]/numPerBin
651  interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0],
652  values.astype(float)[numPerBin > 0],
653  afwMath.stringToInterpStyle(fitType))
654  fitBiasArr = numpy.array([interp.interpolate(i) for i in indices])
655 
656  import lsstDebug
657  if lsstDebug.Info(__name__).display:
658  import matplotlib.pyplot as plot
659  figure = plot.figure(1)
660  figure.clear()
661  axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
662  axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+')
663  if collapsedMask.sum() > 0:
664  axes.plot(indices[collapsedMask], collapsed.data[collapsedMask], 'b+')
665  axes.plot(indices, fitBiasArr, 'r-')
666  plot.xlabel("centered/scaled position along overscan region")
667  plot.ylabel("pixel value/fit value")
668  figure.show()
669  prompt = "Press Enter or c to continue [chp]... "
670  while True:
671  ans = input(prompt).lower()
672  if ans in ("", "c",):
673  break
674  if ans in ("p",):
675  import pdb
676  pdb.set_trace()
677  elif ans in ("h", ):
678  print("h[elp] c[ontinue] p[db]")
679  plot.close()
680 
681  offImage = ampImage.Factory(ampImage.getDimensions())
682  offArray = offImage.getArray()
683  overscanFit = afwImage.ImageF(overscanImage.getDimensions())
684  overscanArray = overscanFit.getArray()
685  if shortInd == 1:
686  offArray[:, :] = fitBiasArr[:, numpy.newaxis]
687  overscanArray[:, :] = fitBiasArr[:, numpy.newaxis]
688  else:
689  offArray[:, :] = fitBiasArr[numpy.newaxis, :]
690  overscanArray[:, :] = fitBiasArr[numpy.newaxis, :]
691 
692  # We don't trust any extrapolation: mask those pixels as SUSPECT
693  # This will occur when the top and or bottom edges of the overscan
694  # contain saturated values. The values will be extrapolated from
695  # the surrounding pixels, but we cannot entirely trust the value of
696  # the extrapolation, and will mark the image mask plane to flag the
697  # image as such.
698  mask = ampMaskedImage.getMask()
699  maskArray = mask.getArray() if shortInd == 1 else mask.getArray().transpose()
700  suspect = mask.getPlaneBitMask("SUSPECT")
701  try:
702  if collapsed.mask == numpy.ma.nomask:
703  # There is no mask, so the whole array is fine
704  pass
705  except ValueError: # If collapsed.mask is an array the test fails [needs .all()]
706  for low in range(num):
707  if not collapsed.mask[low]:
708  break
709  if low > 0:
710  maskArray[:low, :] |= suspect
711  for high in range(1, num):
712  if not collapsed.mask[-high]:
713  break
714  if high > 1:
715  maskArray[-high:, :] |= suspect
716 
717  else:
718  raise pexExcept.Exception('%s : %s an invalid overscan type' % ("overscanCorrection", fitType))
719  ampImage -= offImage
720  overscanImage -= overscanFit
721  return Struct(imageFit=offImage, overscanFit=overscanFit, overscanImage=overscanImage)
722 
723 
724 def brighterFatterCorrection(exposure, kernel, maxIter, threshold, applyGain):
725  """Apply brighter fatter correction in place for the image.
726 
727  Parameters
728  ----------
729  exposure : `lsst.afw.image.Exposure`
730  Exposure to have brighter-fatter correction applied. Modified
731  by this method.
732  kernel : `numpy.ndarray`
733  Brighter-fatter kernel to apply.
734  maxIter : scalar
735  Number of correction iterations to run.
736  threshold : scalar
737  Convergence threshold in terms of the sum of absolute
738  deviations between an iteration and the previous one.
739  applyGain : `Bool`
740  If True, then the exposure values are scaled by the gain prior
741  to correction.
742 
743  Returns
744  -------
745  diff : `float`
746  Final difference between iterations achieved in correction.
747  iteration : `int`
748  Number of iterations used to calculate correction.
749 
750  Notes
751  -----
752  This correction takes a kernel that has been derived from flat
753  field images to redistribute the charge. The gradient of the
754  kernel is the deflection field due to the accumulated charge.
755 
756  Given the original image I(x) and the kernel K(x) we can compute
757  the corrected image Ic(x) using the following equation:
758 
759  Ic(x) = I(x) + 0.5*d/dx(I(x)*d/dx(int( dy*K(x-y)*I(y))))
760 
761  To evaluate the derivative term we expand it as follows:
762 
763  0.5 * ( d/dx(I(x))*d/dx(int(dy*K(x-y)*I(y))) + I(x)*d^2/dx^2(int(dy* K(x-y)*I(y))) )
764 
765  Because we use the measured counts instead of the incident counts
766  we apply the correction iteratively to reconstruct the original
767  counts and the correction. We stop iterating when the summed
768  difference between the current corrected image and the one from
769  the previous iteration is below the threshold. We do not require
770  convergence because the number of iterations is too large a
771  computational cost. How we define the threshold still needs to be
772  evaluated, the current default was shown to work reasonably well
773  on a small set of images. For more information on the method see
774  DocuShare Document-19407.
775 
776  The edges as defined by the kernel are not corrected because they
777  have spurious values due to the convolution.
778  """
779  image = exposure.getMaskedImage().getImage()
780 
781  # The image needs to be units of electrons/holes
782  with gainContext(exposure, image, applyGain):
783 
784  kLx = numpy.shape(kernel)[0]
785  kLy = numpy.shape(kernel)[1]
786  kernelImage = afwImage.ImageD(kLx, kLy)
787  kernelImage.getArray()[:, :] = kernel
788  tempImage = image.clone()
789 
790  nanIndex = numpy.isnan(tempImage.getArray())
791  tempImage.getArray()[nanIndex] = 0.
792 
793  outImage = afwImage.ImageF(image.getDimensions())
794  corr = numpy.zeros_like(image.getArray())
795  prev_image = numpy.zeros_like(image.getArray())
796  convCntrl = afwMath.ConvolutionControl(False, True, 1)
797  fixedKernel = afwMath.FixedKernel(kernelImage)
798 
799  # Define boundary by convolution region. The region that the correction will be
800  # calculated for is one fewer in each dimension because of the second derivative terms.
801  # NOTE: these need to use integer math, as we're using start:end as numpy index ranges.
802  startX = kLx//2
803  endX = -kLx//2
804  startY = kLy//2
805  endY = -kLy//2
806 
807  for iteration in range(maxIter):
808 
809  afwMath.convolve(outImage, tempImage, fixedKernel, convCntrl)
810  tmpArray = tempImage.getArray()
811  outArray = outImage.getArray()
812 
813  with numpy.errstate(invalid="ignore", over="ignore"):
814  # First derivative term
815  gradTmp = numpy.gradient(tmpArray[startY:endY, startX:endX])
816  gradOut = numpy.gradient(outArray[startY:endY, startX:endX])
817  first = (gradTmp[0]*gradOut[0] + gradTmp[1]*gradOut[1])[1:-1, 1:-1]
818 
819  # Second derivative term
820  diffOut20 = numpy.diff(outArray, 2, 0)[startY:endY, startX + 1:endX - 1]
821  diffOut21 = numpy.diff(outArray, 2, 1)[startY + 1:endY - 1, startX:endX]
822  second = tmpArray[startY + 1:endY - 1, startX + 1:endX - 1]*(diffOut20 + diffOut21)
823 
824  corr[startY + 1:endY - 1, startX + 1:endX - 1] = 0.5*(first + second)
825 
826  tmpArray[:, :] = image.getArray()[:, :]
827  tmpArray[nanIndex] = 0.
828  tmpArray[startY:endY, startX:endX] += corr[startY:endY, startX:endX]
829 
830  if iteration > 0:
831  diff = numpy.sum(numpy.abs(prev_image - tmpArray))
832 
833  if diff < threshold:
834  break
835  prev_image[:, :] = tmpArray[:, :]
836 
837  image.getArray()[startY + 1:endY - 1, startX + 1:endX - 1] += \
838  corr[startY + 1:endY - 1, startX + 1:endX - 1]
839 
840  return diff, iteration
841 
842 
843 @contextmanager
844 def gainContext(exp, image, apply):
845  """Context manager that applies and removes gain.
846 
847  Parameters
848  ----------
849  exp : `lsst.afw.image.Exposure`
850  Exposure to apply/remove gain.
851  image : `lsst.afw.image.Image`
852  Image to apply/remove gain.
853  apply : `Bool`
854  If True, apply and remove the amplifier gain.
855 
856  Yields
857  ------
858  exp : `lsst.afw.image.Exposure`
859  Exposure with the gain applied.
860  """
861  if apply:
862  ccd = exp.getDetector()
863  for amp in ccd:
864  sim = image.Factory(image, amp.getBBox())
865  sim *= amp.getGain()
866 
867  try:
868  yield exp
869  finally:
870  if apply:
871  ccd = exp.getDetector()
872  for amp in ccd:
873  sim = image.Factory(image, amp.getBBox())
874  sim /= amp.getGain()
875 
876 
877 def attachTransmissionCurve(exposure, opticsTransmission=None, filterTransmission=None,
878  sensorTransmission=None, atmosphereTransmission=None):
879  """Attach a TransmissionCurve to an Exposure, given separate curves for
880  different components.
881 
882  Parameters
883  ----------
884  exposure : `lsst.afw.image.Exposure`
885  Exposure object to modify by attaching the product of all given
886  ``TransmissionCurves`` in post-assembly trimmed detector coordinates.
887  Must have a valid ``Detector`` attached that matches the detector
888  associated with sensorTransmission.
889  opticsTransmission : `lsst.afw.image.TransmissionCurve`
890  A ``TransmissionCurve`` that represents the throughput of the optics,
891  to be evaluated in focal-plane coordinates.
892  filterTransmission : `lsst.afw.image.TransmissionCurve`
893  A ``TransmissionCurve`` that represents the throughput of the filter
894  itself, to be evaluated in focal-plane coordinates.
895  sensorTransmission : `lsst.afw.image.TransmissionCurve`
896  A ``TransmissionCurve`` that represents the throughput of the sensor
897  itself, to be evaluated in post-assembly trimmed detector coordinates.
898  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
899  A ``TransmissionCurve`` that represents the throughput of the
900  atmosphere, assumed to be spatially constant.
901 
902  Returns
903  -------
904  combined : `lsst.afw.image.TransmissionCurve`
905  The TransmissionCurve attached to the exposure.
906 
907  Notes
908  -----
909  All ``TransmissionCurve`` arguments are optional; if none are provided, the
910  attached ``TransmissionCurve`` will have unit transmission everywhere.
911  """
912  combined = afwImage.TransmissionCurve.makeIdentity()
913  if atmosphereTransmission is not None:
914  combined *= atmosphereTransmission
915  if opticsTransmission is not None:
916  combined *= opticsTransmission
917  if filterTransmission is not None:
918  combined *= filterTransmission
919  detector = exposure.getDetector()
920  fpToPix = detector.getTransform(fromSys=camGeom.FOCAL_PLANE,
921  toSys=camGeom.PIXELS)
922  combined = combined.transformedBy(fpToPix)
923  if sensorTransmission is not None:
924  combined *= sensorTransmission
925  exposure.getInfo().setTransmissionCurve(combined)
926  return combined
927 
928 
929 def addDistortionModel(exposure, camera):
930  """!Update the WCS in exposure with a distortion model based on camera
931  geometry.
932 
933  Parameters
934  ----------
935  exposure : `lsst.afw.image.Exposure`
936  Exposure to process. Must contain a Detector and WCS. The
937  exposure is modified.
938  camera : `lsst.afw.cameraGeom.Camera`
939  Camera geometry.
940 
941  Raises
942  ------
943  RuntimeError
944  Raised if ``exposure`` is lacking a Detector or WCS, or if
945  ``camera`` is None.
946  Notes
947  -----
948  Add a model for optical distortion based on geometry found in ``camera``
949  and the ``exposure``'s detector. The raw input exposure is assumed
950  have a TAN WCS that has no compensation for optical distortion.
951  Two other possibilities are:
952  - The raw input exposure already has a model for optical distortion,
953  as is the case for raw DECam data.
954  In that case you should set config.doAddDistortionModel False.
955  - The raw input exposure has a model for distortion, but it has known
956  deficiencies severe enough to be worth fixing (e.g. because they
957  cause problems for fitting a better WCS). In that case you should
958  override this method with a version suitable for your raw data.
959 
960  """
961  wcs = exposure.getWcs()
962  if wcs is None:
963  raise RuntimeError("exposure has no WCS")
964  if camera is None:
965  raise RuntimeError("camera is None")
966  detector = exposure.getDetector()
967  if detector is None:
968  raise RuntimeError("exposure has no Detector")
969  pixelToFocalPlane = detector.getTransform(camGeom.PIXELS, camGeom.FOCAL_PLANE)
970  focalPlaneToFieldAngle = camera.getTransformMap().getTransform(camGeom.FOCAL_PLANE,
971  camGeom.FIELD_ANGLE)
972  distortedWcs = makeDistortedTanWcs(wcs, pixelToFocalPlane, focalPlaneToFieldAngle)
973  exposure.setWcs(distortedWcs)
974 
975 
976 def applyGains(exposure, normalizeGains=False):
977  """Scale an exposure by the amplifier gains.
978 
979  Parameters
980  ----------
981  exposure : `lsst.afw.image.Exposure`
982  Exposure to process. The image is modified.
983  normalizeGains : `Bool`, optional
984  If True, then amplifiers are scaled to force the median of
985  each amplifier to equal the median of those medians.
986  """
987  ccd = exposure.getDetector()
988  ccdImage = exposure.getMaskedImage()
989 
990  medians = []
991  for amp in ccd:
992  sim = ccdImage.Factory(ccdImage, amp.getBBox())
993  sim *= amp.getGain()
994 
995  if normalizeGains:
996  medians.append(numpy.median(sim.getImage().getArray()))
997 
998  if normalizeGains:
999  median = numpy.median(numpy.array(medians))
1000  for index, amp in enumerate(ccd):
1001  sim = ccdImage.Factory(ccdImage, amp.getBBox())
1002  if medians[index] != 0.0:
1003  sim *= median/medians[index]
1004 
1005 
1007  """Grow the saturation trails by an amount dependent on the width of the trail.
1008 
1009  Parameters
1010  ----------
1011  mask : `lsst.afw.image.Mask`
1012  Mask which will have the saturated areas grown.
1013  """
1014 
1015  extraGrowDict = {}
1016  for i in range(1, 6):
1017  extraGrowDict[i] = 0
1018  for i in range(6, 8):
1019  extraGrowDict[i] = 1
1020  for i in range(8, 10):
1021  extraGrowDict[i] = 3
1022  extraGrowMax = 4
1023 
1024  if extraGrowMax <= 0:
1025  return
1026 
1027  saturatedBit = mask.getPlaneBitMask("SAT")
1028 
1029  xmin, ymin = mask.getBBox().getMin()
1030  width = mask.getWidth()
1031 
1032  thresh = afwDetection.Threshold(saturatedBit, afwDetection.Threshold.BITMASK)
1033  fpList = afwDetection.FootprintSet(mask, thresh).getFootprints()
1034 
1035  for fp in fpList:
1036  for s in fp.getSpans():
1037  x0, x1 = s.getX0(), s.getX1()
1038 
1039  extraGrow = extraGrowDict.get(x1 - x0 + 1, extraGrowMax)
1040  if extraGrow > 0:
1041  y = s.getY() - ymin
1042  x0 -= xmin + extraGrow
1043  x1 -= xmin - extraGrow
1044 
1045  if x0 < 0:
1046  x0 = 0
1047  if x1 >= width - 1:
1048  x1 = width - 1
1049 
1050  mask.array[y, x0:x1+1] |= saturatedBit
1051 
1052 
1053 def setBadRegions(exposure, badStatistic="MEDIAN"):
1054  """Set all BAD areas of the chip to the average of the rest of the exposure
1055 
1056  Parameters
1057  ----------
1058  exposure : `lsst.afw.image.Exposure`
1059  Exposure to mask. The exposure mask is modified.
1060  badStatistic : `str`, optional
1061  Statistic to use to generate the replacement value from the
1062  image data. Allowed values are 'MEDIAN' or 'MEANCLIP'.
1063 
1064  Returns
1065  -------
1066  badPixelCount : scalar
1067  Number of bad pixels masked.
1068  badPixelValue : scalar
1069  Value substituted for bad pixels.
1070 
1071  Raises
1072  ------
1073  RuntimeError
1074  Raised if `badStatistic` is not an allowed value.
1075  """
1076  if badStatistic == "MEDIAN":
1077  statistic = afwMath.MEDIAN
1078  elif badStatistic == "MEANCLIP":
1079  statistic = afwMath.MEANCLIP
1080  else:
1081  raise RuntimeError("Impossible method %s of bad region correction" % badStatistic)
1082 
1083  mi = exposure.getMaskedImage()
1084  mask = mi.getMask()
1085  BAD = mask.getPlaneBitMask("BAD")
1086  INTRP = mask.getPlaneBitMask("INTRP")
1087 
1088  sctrl = afwMath.StatisticsControl()
1089  sctrl.setAndMask(BAD)
1090  value = afwMath.makeStatistics(mi, statistic, sctrl).getValue()
1091 
1092  maskArray = mask.getArray()
1093  imageArray = mi.getImage().getArray()
1094  badPixels = numpy.logical_and((maskArray & BAD) > 0, (maskArray & INTRP) == 0)
1095  imageArray[:] = numpy.where(badPixels, value, imageArray)
1096 
1097  return badPixels.sum(), value
def addDistortionModel(exposure, camera)
Update the WCS in exposure with a distortion model based on camera geometry.
def saturationCorrection(maskedImage, saturation, fwhm, growFootprints=1, interpolate=True, maskName='SAT', fallbackValue=None)
def setBadRegions(exposure, badStatistic="MEDIAN")
def brighterFatterCorrection(exposure, kernel, maxIter, threshold, applyGain)
def transposeDefectList(defectList)
def getDefectListFromMask(maskedImage, maskName)
def interpolateFromMask(maskedImage, fwhm, growSaturatedFootprints=1, maskNameList=['SAT'], fallbackValue=None)
def interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=None)
Definition: isrFunctions.py:78
def defectListFromFootprintList(fpList)
def transposeMaskedImage(maskedImage)
Definition: isrFunctions.py:58
def trimToMatchCalibBBox(rawMaskedImage, calibMaskedImage)
def attachTransmissionCurve(exposure, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None)
def flatCorrection(maskedImage, flatMaskedImage, scalingType, userScale=1.0, invert=False, trimToFit=False)
def makeThresholdMask(maskedImage, threshold, growFootprints=1, maskName='SAT')
def applyGains(exposure, normalizeGains=False)
def darkCorrection(maskedImage, darkMaskedImage, expScale, darkScale, invert=False, trimToFit=False)
def biasCorrection(maskedImage, biasMaskedImage, trimToFit=False)
def gainContext(exp, image, apply)
def illuminationCorrection(maskedImage, illumMaskedImage, illumScale, trimToFit=True)
def updateVariance(maskedImage, gain, readNoise)
def overscanCorrection(ampMaskedImage, overscanImage, fitType='MEDIAN', order=1, collapseRej=3.0, statControl=None, overscanIsInt=True)
def maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD')