lsst.ip.isr  17.0.1-16-gdf99185+1
isrFunctions.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import math
23 import numpy
24 from deprecated.sphinx import deprecated
25 
26 import lsst.geom
27 import lsst.afw.image as afwImage
28 import lsst.afw.detection as afwDetection
29 import lsst.afw.math as afwMath
30 import lsst.meas.algorithms as measAlg
31 import lsst.pex.exceptions as pexExcept
32 import lsst.afw.cameraGeom as camGeom
33 
34 from lsst.afw.geom.wcsUtils import makeDistortedTanWcs
35 from lsst.meas.algorithms.detection import SourceDetectionTask
36 from lsst.pipe.base import Struct
37 
38 from contextlib import contextmanager
39 
40 
41 def createPsf(fwhm):
42  """Make a double Gaussian PSF.
43 
44  Parameters
45  ----------
46  fwhm : scalar
47  FWHM of double Gaussian smoothing kernel.
48 
49  Returns
50  -------
51  psf : `lsst.meas.algorithms.DoubleGaussianPsf`
52  The created smoothing kernel.
53  """
54  ksize = 4*int(fwhm) + 1
55  return measAlg.DoubleGaussianPsf(ksize, ksize, fwhm/(2*math.sqrt(2*math.log(2))))
56 
57 
58 def transposeMaskedImage(maskedImage):
59  """Make a transposed copy of a masked image.
60 
61  Parameters
62  ----------
63  maskedImage : `lsst.afw.image.MaskedImage`
64  Image to process.
65 
66  Returns
67  -------
68  transposed : `lsst.afw.image.MaskedImage`
69  The transposed copy of the input image.
70  """
71  transposed = maskedImage.Factory(lsst.geom.Extent2I(maskedImage.getHeight(), maskedImage.getWidth()))
72  transposed.getImage().getArray()[:] = maskedImage.getImage().getArray().T
73  transposed.getMask().getArray()[:] = maskedImage.getMask().getArray().T
74  transposed.getVariance().getArray()[:] = maskedImage.getVariance().getArray().T
75  return transposed
76 
77 
78 def interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=None):
79  """Interpolate over defects specified in a defect list.
80 
81  Parameters
82  ----------
83  maskedImage : `lsst.afw.image.MaskedImage`
84  Image to process.
85  defectList : `lsst.meas.algorithms.Defects`
86  List of defects to interpolate over.
87  fwhm : scalar
88  FWHM of double Gaussian smoothing kernel.
89  fallbackValue : scalar, optional
90  Fallback value if an interpolated value cannot be determined.
91  If None, then the clipped mean of the image is used.
92  """
93  psf = createPsf(fwhm)
94  if fallbackValue is None:
95  fallbackValue = afwMath.makeStatistics(maskedImage.getImage(), afwMath.MEANCLIP).getValue()
96  if 'INTRP' not in maskedImage.getMask().getMaskPlaneDict():
97  maskedImage.getMask().addMaskPlane('INTRP')
98  measAlg.interpolateOverDefects(maskedImage, psf, defectList, fallbackValue, True)
99  return maskedImage
100 
101 
102 @deprecated(reason="Replaced by Defects.fromFootPrintList() (will be removed after v18)",
103  category=FutureWarning)
105  """Compute a defect list from a footprint list, optionally growing the footprints.
106 
107  Parameters
108  ----------
109  fpList : `list` of `lsst.afw.detection.Footprint`
110  Footprint list to process.
111 
112  Returns
113  -------
114  defectList : `lsst.meas.algorithms.Defects`
115  List of defects.
116  """
117  return measAlg.Defects.fromFootprintList(fpList)
118 
119 @deprecated(reason="Replaced by Defects.transpose() (will be removed after v18)",
120  category=FutureWarning)
121 def transposeDefectList(defectList):
122  """Make a transposed copy of a defect list.
123 
124  Parameters
125  ----------
126  defectList : `lsst.meas.algorithms.Defects`
127  Input list of defects.
128 
129  Returns
130  -------
131  retDefectList : `lsst.meas.algorithms.Defects`
132  Transposed list of defects.
133  """
134  if isinstance(defectList, measAlg.Defects):
135  return defectList.transpose()
136  return measAlg.Defects(defectList).transpose()
137 
138 
139 @deprecated(reason="Replaced by Defects.maskPixels() (will be removed after v18)",
140  category=FutureWarning)
141 def maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD'):
142  """Set mask plane based on a defect list.
143 
144  Parameters
145  ----------
146  maskedImage : `lsst.afw.image.MaskedImage`
147  Image to process. Only the mask plane is updated.
148  defectList : `lsst.meas.algorithms.Defects`
149  Defect list to mask.
150  maskName : str, optional
151  Mask plane name to use.
152  """
153  return lsst.meas.algorithms.Defects(defectList).maskPixels(maskedImage, maskName=maskName)
154 
155 
156 @deprecated(reason="Replaced by Defects.fromMask() (will be removed after v18)",
157  category=FutureWarning)
158 def getDefectListFromMask(maskedImage, maskName):
159  """Compute a defect list from a specified mask plane.
160 
161  Parameters
162  ----------
163  maskedImage : `lsst.afw.image.MaskedImage`
164  Image to process.
165  maskName : `str` or `list`
166  Mask plane name, or list of names to convert.
167 
168  Returns
169  -------
170  defectList : `lsst.meas.algorithms.Defects`
171  Defect list constructed from masked pixels.
172  """
173  return measAlg.Defects.fromMask(maskedImage, maskName)
174 
175 
176 def makeThresholdMask(maskedImage, threshold, growFootprints=1, maskName='SAT'):
177  """Mask pixels based on threshold detection.
178 
179  Parameters
180  ----------
181  maskedImage : `lsst.afw.image.MaskedImage`
182  Image to process. Only the mask plane is updated.
183  threshold : scalar
184  Detection threshold.
185  growFootprints : scalar, optional
186  Number of pixels to grow footprints of detected regions.
187  maskName : str, optional
188  Mask plane name, or list of names to convert
189 
190  Returns
191  -------
192  defectList : `lsst.meas.algorithms.Defects`
193  Defect list constructed from pixels above the threshold.
194  """
195  # find saturated regions
196  thresh = afwDetection.Threshold(threshold)
197  fs = afwDetection.FootprintSet(maskedImage, thresh)
198 
199  if growFootprints > 0:
200  fs = afwDetection.FootprintSet(fs, rGrow=growFootprints, isotropic=False)
201  fpList = fs.getFootprints()
202 
203  # set mask
204  mask = maskedImage.getMask()
205  bitmask = mask.getPlaneBitMask(maskName)
206  afwDetection.setMaskFromFootprintList(mask, fpList, bitmask)
207 
208  return measAlg.Defects.fromFootprintList(fpList)
209 
210 
211 def interpolateFromMask(maskedImage, fwhm, growFootprints=1, maskName='SAT', fallbackValue=None):
212  """Interpolate over defects identified by a particular mask plane.
213 
214  Parameters
215  ----------
216  maskedImage : `lsst.afw.image.MaskedImage`
217  Image to process.
218  fwhm : scalar
219  FWHM of double Gaussian smoothing kernel.
220  growFootprints : scalar, optional
221  Number of pixels to grow footprints of detected regions.
222  maskName : str, optional
223  Mask plane name.
224  fallbackValue : scalar, optional
225  Value of last resort for interpolation.
226  """
227  mask = maskedImage.getMask()
228  thresh = afwDetection.Threshold(mask.getPlaneBitMask(maskName), afwDetection.Threshold.BITMASK)
229  fpSet = afwDetection.FootprintSet(mask, thresh)
230  if growFootprints > 0:
231  fpSet = afwDetection.FootprintSet(fpSet, rGrow=growFootprints, isotropic=False)
232  # If we are interpolating over an area larger than the original masked region, we need
233  # to expand the original mask bit to the full area to explain why we interpolated there.
234  fpSet.setMask(mask, maskName)
235  defectList = measAlg.Defects.fromFootprintList(fpSet.getFootprints())
236  interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=fallbackValue)
237 
238  return maskedImage
239 
240 
241 def saturationCorrection(maskedImage, saturation, fwhm, growFootprints=1, interpolate=True, maskName='SAT',
242  fallbackValue=None):
243  """Mark saturated pixels and optionally interpolate over them
244 
245  Parameters
246  ----------
247  maskedImage : `lsst.afw.image.MaskedImage`
248  Image to process.
249  saturation : scalar
250  Saturation level used as the detection threshold.
251  fwhm : scalar
252  FWHM of double Gaussian smoothing kernel.
253  growFootprints : scalar, optional
254  Number of pixels to grow footprints of detected regions.
255  interpolate : Bool, optional
256  If True, saturated pixels are interpolated over.
257  maskName : str, optional
258  Mask plane name.
259  fallbackValue : scalar, optional
260  Value of last resort for interpolation.
261  """
262  defectList = makeThresholdMask(
263  maskedImage=maskedImage,
264  threshold=saturation,
265  growFootprints=growFootprints,
266  maskName=maskName,
267  )
268  if interpolate:
269  interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=fallbackValue)
270 
271  return maskedImage
272 
273 
274 def trimToMatchCalibBBox(rawMaskedImage, calibMaskedImage):
275  """Compute number of edge trim pixels to match the calibration data.
276 
277  Use the dimension difference between the raw exposure and the
278  calibration exposure to compute the edge trim pixels. This trim
279  is applied symmetrically, with the same number of pixels masked on
280  each side.
281 
282  Parameters
283  ----------
284  rawMaskedImage : `lsst.afw.image.MaskedImage`
285  Image to trim.
286  calibMaskedImage : `lsst.afw.image.MaskedImage`
287  Calibration image to draw new bounding box from.
288 
289  Returns
290  -------
291  replacementMaskedImage : `lsst.afw.image.MaskedImage`
292  ``rawMaskedImage`` trimmed to the appropriate size
293  Raises
294  ------
295  RuntimeError
296  Rasied if ``rawMaskedImage`` cannot be symmetrically trimmed to
297  match ``calibMaskedImage``.
298  """
299  nx, ny = rawMaskedImage.getBBox().getDimensions() - calibMaskedImage.getBBox().getDimensions()
300  if nx != ny:
301  raise RuntimeError("Raw and calib maskedImages are trimmed differently in X and Y.")
302  if nx % 2 != 0:
303  raise RuntimeError("Calibration maskedImage is trimmed unevenly in X.")
304  if nx < 0:
305  raise RuntimeError("Calibration maskedImage is larger than raw data.")
306 
307  nEdge = nx//2
308  if nEdge > 0:
309  replacementMaskedImage = rawMaskedImage[nEdge:-nEdge, nEdge:-nEdge, afwImage.LOCAL]
310  SourceDetectionTask.setEdgeBits(
311  rawMaskedImage,
312  replacementMaskedImage.getBBox(),
313  rawMaskedImage.getMask().getPlaneBitMask("EDGE")
314  )
315  else:
316  replacementMaskedImage = rawMaskedImage
317 
318  return replacementMaskedImage
319 
320 
321 def biasCorrection(maskedImage, biasMaskedImage, trimToFit=False):
322  """Apply bias correction in place.
323 
324  Parameters
325  ----------
326  maskedImage : `lsst.afw.image.MaskedImage`
327  Image to process. The image is modified by this method.
328  biasMaskedImage : `lsst.afw.image.MaskedImage`
329  Bias image of the same size as ``maskedImage``
330  trimToFit : `Bool`, optional
331  If True, raw data is symmetrically trimmed to match
332  calibration size.
333 
334  Raises
335  ------
336  RuntimeError
337  Raised if ``maskedImage`` and ``biasMaskedImage`` do not have
338  the same size.
339 
340  """
341  if trimToFit:
342  maskedImage = trimToMatchCalibBBox(maskedImage, biasMaskedImage)
343 
344  if maskedImage.getBBox(afwImage.LOCAL) != biasMaskedImage.getBBox(afwImage.LOCAL):
345  raise RuntimeError("maskedImage bbox %s != biasMaskedImage bbox %s" %
346  (maskedImage.getBBox(afwImage.LOCAL), biasMaskedImage.getBBox(afwImage.LOCAL)))
347  maskedImage -= biasMaskedImage
348 
349 
350 def darkCorrection(maskedImage, darkMaskedImage, expScale, darkScale, invert=False, trimToFit=False):
351  """Apply dark correction in place.
352 
353  Parameters
354  ----------
355  maskedImage : `lsst.afw.image.MaskedImage`
356  Image to process. The image is modified by this method.
357  darkMaskedImage : `lsst.afw.image.MaskedImage`
358  Dark image of the same size as ``maskedImage``.
359  expScale : scalar
360  Dark exposure time for ``maskedImage``.
361  darkScale : scalar
362  Dark exposure time for ``darkMaskedImage``.
363  invert : `Bool`, optional
364  If True, re-add the dark to an already corrected image.
365  trimToFit : `Bool`, optional
366  If True, raw data is symmetrically trimmed to match
367  calibration size.
368 
369  Raises
370  ------
371  RuntimeError
372  Raised if ``maskedImage`` and ``darkMaskedImage`` do not have
373  the same size.
374 
375  Notes
376  -----
377  The dark correction is applied by calculating:
378  maskedImage -= dark * expScaling / darkScaling
379  """
380  if trimToFit:
381  maskedImage = trimToMatchCalibBBox(maskedImage, darkMaskedImage)
382 
383  if maskedImage.getBBox(afwImage.LOCAL) != darkMaskedImage.getBBox(afwImage.LOCAL):
384  raise RuntimeError("maskedImage bbox %s != darkMaskedImage bbox %s" %
385  (maskedImage.getBBox(afwImage.LOCAL), darkMaskedImage.getBBox(afwImage.LOCAL)))
386 
387  scale = expScale / darkScale
388  if not invert:
389  maskedImage.scaledMinus(scale, darkMaskedImage)
390  else:
391  maskedImage.scaledPlus(scale, darkMaskedImage)
392 
393 
394 def updateVariance(maskedImage, gain, readNoise):
395  """Set the variance plane based on the image plane.
396 
397  Parameters
398  ----------
399  maskedImage : `lsst.afw.image.MaskedImage`
400  Image to process. The variance plane is modified.
401  gain : scalar
402  The amplifier gain in electrons/ADU.
403  readNoise : scalar
404  The amplifier read nmoise in ADU/pixel.
405  """
406  var = maskedImage.getVariance()
407  var[:] = maskedImage.getImage()
408  var /= gain
409  var += readNoise**2
410 
411 
412 def flatCorrection(maskedImage, flatMaskedImage, scalingType, userScale=1.0, invert=False, trimToFit=False):
413  """Apply flat correction in place.
414 
415  Parameters
416  ----------
417  maskedImage : `lsst.afw.image.MaskedImage`
418  Image to process. The image is modified.
419  flatMaskedImage : `lsst.afw.image.MaskedImage`
420  Flat image of the same size as ``maskedImage``
421  scalingType : str
422  Flat scale computation method. Allowed values are 'MEAN',
423  'MEDIAN', or 'USER'.
424  userScale : scalar, optional
425  Scale to use if ``scalingType``='USER'.
426  invert : `Bool`, optional
427  If True, unflatten an already flattened image.
428  trimToFit : `Bool`, optional
429  If True, raw data is symmetrically trimmed to match
430  calibration size.
431 
432  Raises
433  ------
434  RuntimeError
435  Raised if ``maskedImage`` and ``flatMaskedImage`` do not have
436  the same size or if ``scalingType`` is not an allowed value.
437  """
438  if trimToFit:
439  maskedImage = trimToMatchCalibBBox(maskedImage, flatMaskedImage)
440 
441  if maskedImage.getBBox(afwImage.LOCAL) != flatMaskedImage.getBBox(afwImage.LOCAL):
442  raise RuntimeError("maskedImage bbox %s != flatMaskedImage bbox %s" %
443  (maskedImage.getBBox(afwImage.LOCAL), flatMaskedImage.getBBox(afwImage.LOCAL)))
444 
445  # Figure out scale from the data
446  # Ideally the flats are normalized by the calibration product pipeline, but this allows some flexibility
447  # in the case that the flat is created by some other mechanism.
448  if scalingType in ('MEAN', 'MEDIAN'):
449  scalingType = afwMath.stringToStatisticsProperty(scalingType)
450  flatScale = afwMath.makeStatistics(flatMaskedImage.image, scalingType).getValue()
451  elif scalingType == 'USER':
452  flatScale = userScale
453  else:
454  raise RuntimeError('%s : %s not implemented' % ("flatCorrection", scalingType))
455 
456  if not invert:
457  maskedImage.scaledDivides(1.0/flatScale, flatMaskedImage)
458  else:
459  maskedImage.scaledMultiplies(1.0/flatScale, flatMaskedImage)
460 
461 
462 def illuminationCorrection(maskedImage, illumMaskedImage, illumScale):
463  """Apply illumination correction in place.
464 
465  Parameters
466  ----------
467  maskedImage : `lsst.afw.image.MaskedImage`
468  Image to process. The image is modified.
469  illumMaskedImage : `lsst.afw.image.MaskedImage`
470  Illumination correction image of the same size as ``maskedImage``.
471  illumScale : scalar
472  Scale factor for the illumination correction.
473 
474  Raises
475  ------
476  RuntimeError
477  Raised if ``maskedImage`` and ``illumMaskedImage`` do not have
478  the same size.
479  """
480  if maskedImage.getBBox(afwImage.LOCAL) != illumMaskedImage.getBBox(afwImage.LOCAL):
481  raise RuntimeError("maskedImage bbox %s != illumMaskedImage bbox %s" %
482  (maskedImage.getBBox(afwImage.LOCAL), illumMaskedImage.getBBox(afwImage.LOCAL)))
483 
484  maskedImage.scaledDivides(1./illumScale, illumMaskedImage)
485 
486 
487 def overscanCorrection(ampMaskedImage, overscanImage, fitType='MEDIAN', order=1, collapseRej=3.0,
488  statControl=None, overscanIsInt=True):
489  """Apply overscan correction in place.
490 
491  Parameters
492  ----------
493  ampMaskedImage : `lsst.afw.image.MaskedImage`
494  Image of amplifier to correct; modified.
495  overscanImage : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
496  Image of overscan; modified.
497  fitType : `str`
498  Type of fit for overscan correction. May be one of:
499 
500  - ``MEAN``: use mean of overscan.
501  - ``MEANCLIP``: use clipped mean of overscan.
502  - ``MEDIAN``: use median of overscan.
503  - ``POLY``: fit with ordinary polynomial.
504  - ``CHEB``: fit with Chebyshev polynomial.
505  - ``LEG``: fit with Legendre polynomial.
506  - ``NATURAL_SPLINE``: fit with natural spline.
507  - ``CUBIC_SPLINE``: fit with cubic spline.
508  - ``AKIMA_SPLINE``: fit with Akima spline.
509 
510  order : `int`
511  Polynomial order or number of spline knots; ignored unless
512  ``fitType`` indicates a polynomial or spline.
513  statControl : `lsst.afw.math.StatisticsControl`
514  Statistics control object. In particular, we pay attention to numSigmaClip
515  overscanIsInt : `bool`
516  Treat the overscan region as consisting of integers, even if it's been
517  converted to float. E.g. handle ties properly.
518 
519  Returns
520  -------
521  result : `lsst.pipe.base.Struct`
522  Result struct with components:
523 
524  - ``imageFit``: Value(s) removed from image (scalar or
525  `lsst.afw.image.Image`)
526  - ``overscanFit``: Value(s) removed from overscan (scalar or
527  `lsst.afw.image.Image`)
528  - ``overscanImage``: Overscan corrected overscan region
529  (`lsst.afw.image.Image`)
530  Raises
531  ------
532  pexExcept.Exception
533  Raised if ``fitType`` is not an allowed value.
534 
535  Notes
536  -----
537  The ``ampMaskedImage`` and ``overscanImage`` are modified, with the fit
538  subtracted. Note that the ``overscanImage`` should not be a subimage of
539  the ``ampMaskedImage``, to avoid being subtracted twice.
540 
541  Debug plots are available for the SPLINE fitTypes by setting the
542  `debug.display` for `name` == "lsst.ip.isr.isrFunctions". These
543  plots show the scatter plot of the overscan data (collapsed along
544  the perpendicular dimension) as a function of position on the CCD
545  (normalized between +/-1).
546  """
547  ampImage = ampMaskedImage.getImage()
548  if statControl is None:
549  statControl = afwMath.StatisticsControl()
550 
551  numSigmaClip = statControl.getNumSigmaClip()
552 
553  if fitType in ('MEAN', 'MEANCLIP'):
554  fitType = afwMath.stringToStatisticsProperty(fitType)
555  offImage = afwMath.makeStatistics(overscanImage, fitType, statControl).getValue()
556  overscanFit = offImage
557  elif fitType in ('MEDIAN',):
558  if overscanIsInt:
559  # we need an image with integer pixels to handle ties properly
560  if hasattr(overscanImage, "image"):
561  imageI = overscanImage.image.convertI()
562  overscanImageI = afwImage.MaskedImageI(imageI, overscanImage.mask, overscanImage.variance)
563  else:
564  overscanImageI = overscanImage.convertI()
565  else:
566  overscanImageI = overscanImage
567 
568  fitType = afwMath.stringToStatisticsProperty(fitType)
569  offImage = afwMath.makeStatistics(overscanImageI, fitType, statControl).getValue()
570  overscanFit = offImage
571 
572  if overscanIsInt:
573  del overscanImageI
574  elif fitType in ('POLY', 'CHEB', 'LEG', 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE'):
575  if hasattr(overscanImage, "getImage"):
576  biasArray = overscanImage.getImage().getArray()
577  biasArray = numpy.ma.masked_where(overscanImage.getMask().getArray() & statControl.getAndMask(),
578  biasArray)
579  else:
580  biasArray = overscanImage.getArray()
581  # Fit along the long axis, so collapse along each short row and fit the resulting array
582  shortInd = numpy.argmin(biasArray.shape)
583  if shortInd == 0:
584  # Convert to some 'standard' representation to make things easier
585  biasArray = numpy.transpose(biasArray)
586 
587  # Do a single round of clipping to weed out CR hits and signal leaking into the overscan
588  percentiles = numpy.percentile(biasArray, [25.0, 50.0, 75.0], axis=1)
589  medianBiasArr = percentiles[1]
590  stdevBiasArr = 0.74*(percentiles[2] - percentiles[0]) # robust stdev
591  diff = numpy.abs(biasArray - medianBiasArr[:, numpy.newaxis])
592  biasMaskedArr = numpy.ma.masked_where(diff > numSigmaClip*stdevBiasArr[:, numpy.newaxis], biasArray)
593  collapsed = numpy.mean(biasMaskedArr, axis=1)
594  if collapsed.mask.sum() > 0:
595  collapsed.data[collapsed.mask] = numpy.mean(biasArray.data[collapsed.mask], axis=1)
596  del biasArray, percentiles, stdevBiasArr, diff, biasMaskedArr
597 
598  if shortInd == 0:
599  collapsed = numpy.transpose(collapsed)
600 
601  num = len(collapsed)
602  indices = 2.0*numpy.arange(num)/float(num) - 1.0
603 
604  if fitType in ('POLY', 'CHEB', 'LEG'):
605  # A numpy polynomial
606  poly = numpy.polynomial
607  fitter, evaler = {"POLY": (poly.polynomial.polyfit, poly.polynomial.polyval),
608  "CHEB": (poly.chebyshev.chebfit, poly.chebyshev.chebval),
609  "LEG": (poly.legendre.legfit, poly.legendre.legval),
610  }[fitType]
611 
612  coeffs = fitter(indices, collapsed, order)
613  fitBiasArr = evaler(indices, coeffs)
614  elif 'SPLINE' in fitType:
615  # An afw interpolation
616  numBins = order
617  #
618  # numpy.histogram needs a real array for the mask, but numpy.ma "optimises" the case
619  # no-values-are-masked by replacing the mask array by a scalar, numpy.ma.nomask
620  #
621  # Issue DM-415
622  #
623  collapsedMask = collapsed.mask
624  try:
625  if collapsedMask == numpy.ma.nomask:
626  collapsedMask = numpy.array(len(collapsed)*[numpy.ma.nomask])
627  except ValueError: # If collapsedMask is an array the test fails [needs .all()]
628  pass
629 
630  numPerBin, binEdges = numpy.histogram(indices, bins=numBins,
631  weights=1-collapsedMask.astype(int))
632  # Binning is just a histogram, with weights equal to the values.
633  # Use a similar trick to get the bin centers (this deals with different numbers per bin).
634  with numpy.errstate(invalid="ignore"): # suppress NAN warnings
635  values = numpy.histogram(indices, bins=numBins,
636  weights=collapsed.data*~collapsedMask)[0]/numPerBin
637  binCenters = numpy.histogram(indices, bins=numBins,
638  weights=indices*~collapsedMask)[0]/numPerBin
639  interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0],
640  values.astype(float)[numPerBin > 0],
641  afwMath.stringToInterpStyle(fitType))
642  fitBiasArr = numpy.array([interp.interpolate(i) for i in indices])
643 
644  import lsstDebug
645  if lsstDebug.Info(__name__).display:
646  import matplotlib.pyplot as plot
647  figure = plot.figure(1)
648  figure.clear()
649  axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
650  axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+')
651  if collapsedMask.sum() > 0:
652  axes.plot(indices[collapsedMask], collapsed.data[collapsedMask], 'b+')
653  axes.plot(indices, fitBiasArr, 'r-')
654  plot.xlabel("centered/scaled position along overscan region")
655  plot.ylabel("pixel value/fit value")
656  figure.show()
657  prompt = "Press Enter or c to continue [chp]... "
658  while True:
659  ans = input(prompt).lower()
660  if ans in ("", "c",):
661  break
662  if ans in ("p",):
663  import pdb
664  pdb.set_trace()
665  elif ans in ("h", ):
666  print("h[elp] c[ontinue] p[db]")
667  plot.close()
668 
669  offImage = ampImage.Factory(ampImage.getDimensions())
670  offArray = offImage.getArray()
671  overscanFit = afwImage.ImageF(overscanImage.getDimensions())
672  overscanArray = overscanFit.getArray()
673  if shortInd == 1:
674  offArray[:, :] = fitBiasArr[:, numpy.newaxis]
675  overscanArray[:, :] = fitBiasArr[:, numpy.newaxis]
676  else:
677  offArray[:, :] = fitBiasArr[numpy.newaxis, :]
678  overscanArray[:, :] = fitBiasArr[numpy.newaxis, :]
679 
680  # We don't trust any extrapolation: mask those pixels as SUSPECT
681  # This will occur when the top and or bottom edges of the overscan
682  # contain saturated values. The values will be extrapolated from
683  # the surrounding pixels, but we cannot entirely trust the value of
684  # the extrapolation, and will mark the image mask plane to flag the
685  # image as such.
686  mask = ampMaskedImage.getMask()
687  maskArray = mask.getArray() if shortInd == 1 else mask.getArray().transpose()
688  suspect = mask.getPlaneBitMask("SUSPECT")
689  try:
690  if collapsed.mask == numpy.ma.nomask:
691  # There is no mask, so the whole array is fine
692  pass
693  except ValueError: # If collapsed.mask is an array the test fails [needs .all()]
694  for low in range(num):
695  if not collapsed.mask[low]:
696  break
697  if low > 0:
698  maskArray[:low, :] |= suspect
699  for high in range(1, num):
700  if not collapsed.mask[-high]:
701  break
702  if high > 1:
703  maskArray[-high:, :] |= suspect
704 
705  else:
706  raise pexExcept.Exception('%s : %s an invalid overscan type' % ("overscanCorrection", fitType))
707  ampImage -= offImage
708  overscanImage -= overscanFit
709  return Struct(imageFit=offImage, overscanFit=overscanFit, overscanImage=overscanImage)
710 
711 
712 def brighterFatterCorrection(exposure, kernel, maxIter, threshold, applyGain):
713  """Apply brighter fatter correction in place for the image.
714 
715  Parameters
716  ----------
717  exposure : `lsst.afw.image.Exposure`
718  Exposure to have brighter-fatter correction applied. Modified
719  by this method.
720  kernel : `numpy.ndarray`
721  Brighter-fatter kernel to apply.
722  maxIter : scalar
723  Number of correction iterations to run.
724  threshold : scalar
725  Convergence threshold in terms of the sum of absolute
726  deviations between an iteration and the previous one.
727  applyGain : `Bool`
728  If True, then the exposure values are scaled by the gain prior
729  to correction.
730 
731  Notes
732  -----
733  This correction takes a kernel that has been derived from flat
734  field images to redistribute the charge. The gradient of the
735  kernel is the deflection field due to the accumulated charge.
736 
737  Given the original image I(x) and the kernel K(x) we can compute
738  the corrected image Ic(x) using the following equation:
739 
740  Ic(x) = I(x) + 0.5*d/dx(I(x)*d/dx(int( dy*K(x-y)*I(y))))
741 
742  To evaluate the derivative term we expand it as follows:
743 
744  0.5 * ( d/dx(I(x))*d/dx(int(dy*K(x-y)*I(y))) + I(x)*d^2/dx^2(int(dy* K(x-y)*I(y))) )
745 
746  Because we use the measured counts instead of the incident counts
747  we apply the correction iteratively to reconstruct the original
748  counts and the correction. We stop iterating when the summed
749  difference between the current corrected image and the one from
750  the previous iteration is below the threshold. We do not require
751  convergence because the number of iterations is too large a
752  computational cost. How we define the threshold still needs to be
753  evaluated, the current default was shown to work reasonably well
754  on a small set of images. For more information on the method see
755  DocuShare Document-19407.
756 
757  The edges as defined by the kernel are not corrected because they
758  have spurious values due to the convolution.
759  """
760  image = exposure.getMaskedImage().getImage()
761 
762  # The image needs to be units of electrons/holes
763  with gainContext(exposure, image, applyGain):
764 
765  kLx = numpy.shape(kernel)[0]
766  kLy = numpy.shape(kernel)[1]
767  kernelImage = afwImage.ImageD(kLx, kLy)
768  kernelImage.getArray()[:, :] = kernel
769  tempImage = image.clone()
770 
771  nanIndex = numpy.isnan(tempImage.getArray())
772  tempImage.getArray()[nanIndex] = 0.
773 
774  outImage = afwImage.ImageF(image.getDimensions())
775  corr = numpy.zeros_like(image.getArray())
776  prev_image = numpy.zeros_like(image.getArray())
777  convCntrl = afwMath.ConvolutionControl(False, True, 1)
778  fixedKernel = afwMath.FixedKernel(kernelImage)
779 
780  # Define boundary by convolution region. The region that the correction will be
781  # calculated for is one fewer in each dimension because of the second derivative terms.
782  # NOTE: these need to use integer math, as we're using start:end as numpy index ranges.
783  startX = kLx//2
784  endX = -kLx//2
785  startY = kLy//2
786  endY = -kLy//2
787 
788  for iteration in range(maxIter):
789 
790  afwMath.convolve(outImage, tempImage, fixedKernel, convCntrl)
791  tmpArray = tempImage.getArray()
792  outArray = outImage.getArray()
793 
794  with numpy.errstate(invalid="ignore", over="ignore"):
795  # First derivative term
796  gradTmp = numpy.gradient(tmpArray[startY:endY, startX:endX])
797  gradOut = numpy.gradient(outArray[startY:endY, startX:endX])
798  first = (gradTmp[0]*gradOut[0] + gradTmp[1]*gradOut[1])[1:-1, 1:-1]
799 
800  # Second derivative term
801  diffOut20 = numpy.diff(outArray, 2, 0)[startY:endY, startX + 1:endX - 1]
802  diffOut21 = numpy.diff(outArray, 2, 1)[startY + 1:endY - 1, startX:endX]
803  second = tmpArray[startY + 1:endY - 1, startX + 1:endX - 1]*(diffOut20 + diffOut21)
804 
805  corr[startY + 1:endY - 1, startX + 1:endX - 1] = 0.5*(first + second)
806 
807  tmpArray[:, :] = image.getArray()[:, :]
808  tmpArray[nanIndex] = 0.
809  tmpArray[startY:endY, startX:endX] += corr[startY:endY, startX:endX]
810 
811  if iteration > 0:
812  diff = numpy.sum(numpy.abs(prev_image - tmpArray))
813 
814  if diff < threshold:
815  break
816  prev_image[:, :] = tmpArray[:, :]
817 
818  # if iteration == maxIter - 1:
819  # self.log.warn("Brighter fatter correction did not converge,
820  # final difference %f" % diff)
821 
822  # self.log.info("Finished brighter fatter in %d iterations" % (iteration + 1))
823  image.getArray()[startY + 1:endY - 1, startX + 1:endX - 1] += \
824  corr[startY + 1:endY - 1, startX + 1:endX - 1]
825 
826 
827 @contextmanager
828 def gainContext(exp, image, apply):
829  """Context manager that applies and removes gain.
830 
831  Parameters
832  ----------
833  exp : `lsst.afw.image.Exposure`
834  Exposure to apply/remove gain.
835  image : `lsst.afw.image.Image`
836  Image to apply/remove gain.
837  apply : `Bool`
838  If True, apply and remove the amplifier gain.
839 
840  Yields
841  ------
842  exp : `lsst.afw.image.Exposure`
843  Exposure with the gain applied.
844  """
845  if apply:
846  ccd = exp.getDetector()
847  for amp in ccd:
848  sim = image.Factory(image, amp.getBBox())
849  sim *= amp.getGain()
850 
851  try:
852  yield exp
853  finally:
854  if apply:
855  ccd = exp.getDetector()
856  for amp in ccd:
857  sim = image.Factory(image, amp.getBBox())
858  sim /= amp.getGain()
859 
860 
861 def attachTransmissionCurve(exposure, opticsTransmission=None, filterTransmission=None,
862  sensorTransmission=None, atmosphereTransmission=None):
863  """Attach a TransmissionCurve to an Exposure, given separate curves for
864  different components.
865 
866  Parameters
867  ----------
868  exposure : `lsst.afw.image.Exposure`
869  Exposure object to modify by attaching the product of all given
870  ``TransmissionCurves`` in post-assembly trimmed detector coordinates.
871  Must have a valid ``Detector`` attached that matches the detector
872  associated with sensorTransmission.
873  opticsTransmission : `lsst.afw.image.TransmissionCurve`
874  A ``TransmissionCurve`` that represents the throughput of the optics,
875  to be evaluated in focal-plane coordinates.
876  filterTransmission : `lsst.afw.image.TransmissionCurve`
877  A ``TransmissionCurve`` that represents the throughput of the filter
878  itself, to be evaluated in focal-plane coordinates.
879  sensorTransmission : `lsst.afw.image.TransmissionCurve`
880  A ``TransmissionCurve`` that represents the throughput of the sensor
881  itself, to be evaluated in post-assembly trimmed detector coordinates.
882  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
883  A ``TransmissionCurve`` that represents the throughput of the
884  atmosphere, assumed to be spatially constant.
885 
886  Returns
887  -------
888  combined : `lsst.afw.image.TransmissionCurve`
889  The TransmissionCurve attached to the exposure.
890 
891  Notes
892  -----
893  All ``TransmissionCurve`` arguments are optional; if none are provided, the
894  attached ``TransmissionCurve`` will have unit transmission everywhere.
895  """
896  combined = afwImage.TransmissionCurve.makeIdentity()
897  if atmosphereTransmission is not None:
898  combined *= atmosphereTransmission
899  if opticsTransmission is not None:
900  combined *= opticsTransmission
901  if filterTransmission is not None:
902  combined *= filterTransmission
903  detector = exposure.getDetector()
904  fpToPix = detector.getTransform(fromSys=camGeom.FOCAL_PLANE,
905  toSys=camGeom.PIXELS)
906  combined = combined.transformedBy(fpToPix)
907  if sensorTransmission is not None:
908  combined *= sensorTransmission
909  exposure.getInfo().setTransmissionCurve(combined)
910  return combined
911 
912 
913 def addDistortionModel(exposure, camera):
914  """!Update the WCS in exposure with a distortion model based on camera
915  geometry.
916 
917  Parameters
918  ----------
919  exposure : `lsst.afw.image.Exposure`
920  Exposure to process. Must contain a Detector and WCS. The
921  exposure is modified.
922  camera : `lsst.afw.cameraGeom.Camera`
923  Camera geometry.
924 
925  Raises
926  ------
927  RuntimeError
928  Raised if ``exposure`` is lacking a Detector or WCS, or if
929  ``camera`` is None.
930  Notes
931  -----
932  Add a model for optical distortion based on geometry found in ``camera``
933  and the ``exposure``'s detector. The raw input exposure is assumed
934  have a TAN WCS that has no compensation for optical distortion.
935  Two other possibilities are:
936  - The raw input exposure already has a model for optical distortion,
937  as is the case for raw DECam data.
938  In that case you should set config.doAddDistortionModel False.
939  - The raw input exposure has a model for distortion, but it has known
940  deficiencies severe enough to be worth fixing (e.g. because they
941  cause problems for fitting a better WCS). In that case you should
942  override this method with a version suitable for your raw data.
943 
944  """
945  wcs = exposure.getWcs()
946  if wcs is None:
947  raise RuntimeError("exposure has no WCS")
948  if camera is None:
949  raise RuntimeError("camera is None")
950  detector = exposure.getDetector()
951  if detector is None:
952  raise RuntimeError("exposure has no Detector")
953  pixelToFocalPlane = detector.getTransform(camGeom.PIXELS, camGeom.FOCAL_PLANE)
954  focalPlaneToFieldAngle = camera.getTransformMap().getTransform(camGeom.FOCAL_PLANE,
955  camGeom.FIELD_ANGLE)
956  distortedWcs = makeDistortedTanWcs(wcs, pixelToFocalPlane, focalPlaneToFieldAngle)
957  exposure.setWcs(distortedWcs)
958 
959 
960 def applyGains(exposure, normalizeGains=False):
961  """Scale an exposure by the amplifier gains.
962 
963  Parameters
964  ----------
965  exposure : `lsst.afw.image.Exposure`
966  Exposure to process. The image is modified.
967  normalizeGains : `Bool`, optional
968  If True, then amplifiers are scaled to force the median of
969  each amplifier to equal the median of those medians.
970  """
971  ccd = exposure.getDetector()
972  ccdImage = exposure.getMaskedImage()
973 
974  medians = []
975  for amp in ccd:
976  sim = ccdImage.Factory(ccdImage, amp.getBBox())
977  sim *= amp.getGain()
978 
979  if normalizeGains:
980  medians.append(numpy.median(sim.getImage().getArray()))
981 
982  if normalizeGains:
983  median = numpy.median(numpy.array(medians))
984  for index, amp in enumerate(ccd):
985  sim = ccdImage.Factory(ccdImage, amp.getBBox())
986  if medians[index] != 0.0:
987  sim *= median/medians[index]
988 
989 
991  """Grow the saturation trails by an amount dependent on the width of the trail.
992 
993  Parameters
994  ----------
995  mask : `lsst.afw.image.Mask`
996  Mask which will have the saturated areas grown.
997  """
998 
999  extraGrowDict = {}
1000  for i in range(1, 6):
1001  extraGrowDict[i] = 0
1002  for i in range(6, 8):
1003  extraGrowDict[i] = 1
1004  for i in range(8, 10):
1005  extraGrowDict[i] = 3
1006  extraGrowMax = 4
1007 
1008  if extraGrowMax <= 0:
1009  return
1010 
1011  saturatedBit = mask.getPlaneBitMask("SAT")
1012 
1013  xmin, ymin = mask.getBBox().getMin()
1014  width = mask.getWidth()
1015 
1016  thresh = afwDetection.Threshold(saturatedBit, afwDetection.Threshold.BITMASK)
1017  fpList = afwDetection.FootprintSet(mask, thresh).getFootprints()
1018 
1019  for fp in fpList:
1020  for s in fp.getSpans():
1021  x0, x1 = s.getX0(), s.getX1()
1022 
1023  extraGrow = extraGrowDict.get(x1 - x0 + 1, extraGrowMax)
1024  if extraGrow > 0:
1025  y = s.getY() - ymin
1026  x0 -= xmin + extraGrow
1027  x1 -= xmin - extraGrow
1028 
1029  if x0 < 0:
1030  x0 = 0
1031  if x1 >= width - 1:
1032  x1 = width - 1
1033 
1034  mask.array[y, x0:x1+1] |= saturatedBit
1035 
1036 
1037 def setBadRegions(exposure, badStatistic="MEDIAN"):
1038  """Set all BAD areas of the chip to the average of the rest of the exposure
1039 
1040  Parameters
1041  ----------
1042  exposure : `lsst.afw.image.Exposure`
1043  Exposure to mask. The exposure mask is modified.
1044  badStatistic : `str`, optional
1045  Statistic to use to generate the replacement value from the
1046  image data. Allowed values are 'MEDIAN' or 'MEANCLIP'.
1047 
1048  Returns
1049  -------
1050  badPixelCount : scalar
1051  Number of bad pixels masked.
1052  badPixelValue : scalar
1053  Value substituted for bad pixels.
1054 
1055  Raises
1056  ------
1057  RuntimeError
1058  Raised if `badStatistic` is not an allowed value.
1059  """
1060  if badStatistic == "MEDIAN":
1061  statistic = afwMath.MEDIAN
1062  elif badStatistic == "MEANCLIP":
1063  statistic = afwMath.MEANCLIP
1064  else:
1065  raise RuntimeError("Impossible method %s of bad region correction" % badStatistic)
1066 
1067  mi = exposure.getMaskedImage()
1068  mask = mi.getMask()
1069  BAD = mask.getPlaneBitMask("BAD")
1070  INTRP = mask.getPlaneBitMask("INTRP")
1071 
1072  sctrl = afwMath.StatisticsControl()
1073  sctrl.setAndMask(BAD)
1074  value = afwMath.makeStatistics(mi, statistic, sctrl).getValue()
1075 
1076  maskArray = mask.getArray()
1077  imageArray = mi.getImage().getArray()
1078  badPixels = numpy.logical_and((maskArray & BAD) > 0, (maskArray & INTRP) == 0)
1079  imageArray[:] = numpy.where(badPixels, value, imageArray)
1080 
1081  return badPixels.sum(), value
def illuminationCorrection(maskedImage, illumMaskedImage, illumScale)
def addDistortionModel(exposure, camera)
Update the WCS in exposure with a distortion model based on camera geometry.
def saturationCorrection(maskedImage, saturation, fwhm, growFootprints=1, interpolate=True, maskName='SAT', fallbackValue=None)
def setBadRegions(exposure, badStatistic="MEDIAN")
def brighterFatterCorrection(exposure, kernel, maxIter, threshold, applyGain)
def transposeDefectList(defectList)
def getDefectListFromMask(maskedImage, maskName)
def interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=None)
Definition: isrFunctions.py:78
def defectListFromFootprintList(fpList)
def transposeMaskedImage(maskedImage)
Definition: isrFunctions.py:58
def interpolateFromMask(maskedImage, fwhm, growFootprints=1, maskName='SAT', fallbackValue=None)
def trimToMatchCalibBBox(rawMaskedImage, calibMaskedImage)
def attachTransmissionCurve(exposure, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None)
def flatCorrection(maskedImage, flatMaskedImage, scalingType, userScale=1.0, invert=False, trimToFit=False)
def makeThresholdMask(maskedImage, threshold, growFootprints=1, maskName='SAT')
def applyGains(exposure, normalizeGains=False)
def darkCorrection(maskedImage, darkMaskedImage, expScale, darkScale, invert=False, trimToFit=False)
def biasCorrection(maskedImage, biasMaskedImage, trimToFit=False)
def gainContext(exp, image, apply)
def updateVariance(maskedImage, gain, readNoise)
def overscanCorrection(ampMaskedImage, overscanImage, fitType='MEDIAN', order=1, collapseRej=3.0, statControl=None, overscanIsInt=True)
def maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD')