lsst.cp.pipe  16.0-2-gc6e0ed0+4
makeBrighterFatterKernel.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 #
4 # Copyright 2008-2017 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 """Calculation of brighter-fatter effect correlations and kernels."""
24 
25 __all__ = ['MakeBrighterFatterKernelTaskConfig',
26  'MakeBrighterFatterKernelTask',
27  'calcBiasCorr']
28 
29 import os
30 from scipy import stats
31 import numpy as np
32 import matplotlib.pyplot as plt
33 # the following import is required for 3d projection
34 from mpl_toolkits.mplot3d import axes3d # noqa: F401
35 
36 import lsstDebug
37 import lsst.afw.image as afwImage
38 import lsst.afw.math as afwMath
39 import lsst.afw.display as afwDisp
40 from lsst.ip.isr import IsrTask
41 import lsst.pex.config as pexConfig
42 import lsst.pipe.base as pipeBase
43 
44 
45 class MakeBrighterFatterKernelTaskConfig(pexConfig.Config):
46  """Config class for bright-fatter effect coefficient calculation."""
47 
48  isr = pexConfig.ConfigurableField(
49  target=IsrTask,
50  doc="""Task to perform instrumental signature removal""",
51  )
52  isrMandatorySteps = pexConfig.ListField(
53  dtype=str,
54  doc="isr operations that must be performed for valid results. Raises if any of these are False",
55  default=['doAssembleCcd']
56  )
57  isrForbiddenSteps = pexConfig.ListField(
58  dtype=str,
59  doc="isr operations that must NOT be performed for valid results. Raises if any of these are True",
60  default=['doFlat', 'doFringe', 'doAddDistortionModel', 'doBrighterFatter', 'doUseOpticsTransmission',
61  'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
62  )
63  isrDesirableSteps = pexConfig.ListField(
64  dtype=str,
65  doc="isr operations that it is advisable to perform, but are not mission-critical." +
66  " WARNs are logged for any of these found to be False.",
67  default=['doBias', 'doDark', 'doCrosstalk', 'doDefect', 'doLinearize']
68  )
69  doCalcGains = pexConfig.Field(
70  dtype=bool,
71  doc="Measure the per-amplifier gains (using the photon transfer curve method)?",
72  default=True,
73  )
74  ccdKey = pexConfig.Field(
75  dtype=str,
76  doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'",
77  default='ccd',
78  )
79  maxIterRegression = pexConfig.Field(
80  dtype=int,
81  doc="Maximum number of iterations for the regression fitter",
82  default=10
83  )
84  nSigmaClipGainCalc = pexConfig.Field(
85  dtype=int,
86  doc="Number of sigma to clip the pixel value distribution to during gain calculation",
87  default=5
88  )
89  nSigmaClipRegression = pexConfig.Field(
90  dtype=int,
91  doc="Number of sigma to clip outliers from the line of best fit to during iterative regression",
92  default=3
93  )
94  xcorrCheckRejectLevel = pexConfig.Field(
95  dtype=float,
96  doc="Sanity check level for the sum of the input cross-correlations. Arrays which " +
97  "sum to greater than this are discarded before the clipped mean is calculated.",
98  default=2.0 # TODO: DM-15401 Restore this to nominal value of 0.2 if appropriate
99  )
100  maxIterSuccessiveOverRelaxation = pexConfig.Field(
101  dtype=int,
102  doc="The maximum number of iterations allowed for the successive over-relaxation method",
103  default=10000
104  )
105  eLevelSuccessiveOverRelaxation = pexConfig.Field(
106  dtype=float,
107  doc="The target residual error for the successive over-relaxation method",
108  default=5.0e-14
109  )
110  nSigmaClipKernelGen = pexConfig.Field(
111  dtype=float,
112  doc="Number of sigma to clip to during pixel-wise clipping when generating the kernel. See " +
113  "the generateKernel docstring for more info.",
114  default=4
115  )
116  nSigmaClipXCorr = pexConfig.Field(
117  dtype=float,
118  doc="Number of sigma to clip when calculating means for the cross-correlation",
119  default=5
120  )
121  maxLag = pexConfig.Field(
122  dtype=int,
123  doc="The maximum lag (in pixels) to use when calculating the cross-correlation/kernel",
124  default=8
125  )
126  nPixBorderGainCalc = pexConfig.Field(
127  dtype=int,
128  doc="The number of border pixels to exclude when calculating the gain",
129  default=10
130  )
131  nPixBorderXCorr = pexConfig.Field(
132  dtype=int,
133  doc="The number of border pixels to exclude when calculating the cross-correlation and kernel",
134  default=10
135  )
136  biasCorr = pexConfig.Field(
137  dtype=float,
138  doc="An empirically determined correction factor, used to correct for the sigma-clipping of" +
139  " a non-Gaussian distribution. Post DM-15277, code will exist here to calulate appropriate values",
140  default=0.9241
141  )
142  backgroundBinSize = pexConfig.Field(
143  dtype=int,
144  doc="Size of the background bins",
145  default=128
146  )
147  fixPtcThroughOrigin = pexConfig.Field(
148  dtype=bool,
149  doc="Constrain the fit of the photon transfer curve to go through the origin when measuring" +
150  "the gain?",
151  default=True
152  )
153  level = pexConfig.ChoiceField(
154  doc="The level at which to calculate the brighter-fatter kernels",
155  dtype=str, default="DETECTOR",
156  allowed={
157  "AMP": "Every amplifier treated separately",
158  "DETECTOR": "One kernel per detector",
159  }
160  )
161  backgroundWarnLevel = pexConfig.Field(
162  dtype=float,
163  doc="Log warnings if the mean of the fitted background is found to be above this level after " +
164  "differencing image pair.",
165  default=0.1
166  )
167 
168 
169 class MakeBrighterFatterKernelTaskRunner(pipeBase.TaskRunner):
170  """Subclass of TaskRunner for the MakeBrighterFatterKernelTask.
171 
172  This transforms the processed arguments generated by the ArgumentParser
173  into the arguments expected by makeBrighterFatterKernelTask.run().
174 
175  makeBrighterFatterKernelTask.run() takes a two arguments,
176  one of which is the dataRef (as usual), and the other is the list
177  of visit-pairs, in the form of a list of tuples.
178  This list is supplied on the command line as documented,
179  and this class parses that, and passes the parsed version
180  to the run() method.
181 
182  See pipeBase.TaskRunner for more information.
183  """
184 
185  @staticmethod
186  def getTargetList(parsedCmd, **kwargs):
187  """Parse the visit list and pass through explicitly."""
188  visitPairs = []
189  for visitStringPair in parsedCmd.visitPairs:
190  visitStrings = visitStringPair.split(",")
191  if len(visitStrings) != 2:
192  raise RuntimeError("Found {} visits in {} instead of 2".format(len(visitStrings),
193  visitStringPair))
194  try:
195  visits = [int(visit) for visit in visitStrings]
196  except Exception:
197  raise RuntimeError("Could not parse {} as two integer visit numbers".format(visitStringPair))
198  visitPairs.append(visits)
199 
200  return pipeBase.TaskRunner.getTargetList(parsedCmd, visitPairs=visitPairs, **kwargs)
201 
202 
203 class BrighterFatterKernelTaskDataIdContainer(pipeBase.DataIdContainer):
204  """A DataIdContainer for the MakeBrighterFatterKernelTask."""
205 
206  def makeDataRefList(self, namespace):
207  """Compute refList based on idList.
208 
209  This method must be defined as the dataset does not exist before this
210  task is run.
211 
212  Parameters
213  ----------
214  namespace
215  Results of parsing the command-line.
216 
217  Notes
218  -----
219  Not called if ``add_id_argument`` called
220  with ``doMakeDataRefList=False``.
221  Note that this is almost a copy-and-paste of the vanilla implementation,
222  but without checking if the datasets already exist,
223  as this task exists to make them.
224  """
225  if self.datasetType is None:
226  raise RuntimeError("Must call setDatasetType first")
227  butler = namespace.butler
228  for dataId in self.idList:
229  refList = list(butler.subset(datasetType=self.datasetType, level=self.level, dataId=dataId))
230  # exclude nonexistent data
231  # this is a recursive test, e.g. for the sake of "raw" data
232  if not refList:
233  namespace.log.warn("No data found for dataId=%s", dataId)
234  continue
235  self.refList += refList
236 
237 
239  """A (very) simple class to hold the kernel(s) generated.
240 
241  The kernel.kernel is a dictionary holding the kernels themselves.
242  One kernel if the level is 'DETECTOR' or,
243  nAmps in length, if level is 'AMP'.
244  The dict is keyed by either the detector ID or the amplifier IDs.
245 
246  The level is the level for which the kernel(s) were generated so that one
247  can know how to access the kernels without having to query the shape of
248  the dictionary holding the kernel(s).
249  """
250 
251  def __init__(self, level, kernelDict):
252  assert type(level) == str
253  assert type(kernelDict) == dict
254  if level == 'DETECTOR':
255  assert len(kernelDict.keys()) == 1
256  if level == 'AMP':
257  assert len(kernelDict.keys()) > 1
258 
259  self.level = level
260  self.kernel = kernelDict
261 
262 
263 class MakeBrighterFatterKernelTask(pipeBase.CmdLineTask):
264  """Brighter-fatter effect correction-kernel calculation task.
265 
266  A command line task for calculating the brighter-fatter correction
267  kernel from pairs of flat-field images (with the same exposure length).
268 
269  The following operations are performed:
270 
271  - The configurable isr task is called, which unpersists and assembles the
272  raw images, and performs the selected instrument signature removal tasks.
273  For the purpose of brighter-fatter coefficient calculation is it
274  essential that certain components of isr are *not* performed, and
275  recommended that certain others are. The task checks the selected isr
276  configuration before it is run, and if forbidden components have been
277  selected task will raise, and if recommended ones have not been selected,
278  warnings are logged.
279 
280  - The gain of the each amplifier in the detector is calculated using
281  the photon transfer curve (PTC) method and used to correct the images
282  so that all calculations are done in units of electrons, and so that the
283  level across amplifier boundaries is continuous.
284  Outliers in the PTC are iteratively rejected
285  before fitting, with the nSigma rejection level set by
286  config.nSigmaClipRegression. Individual pixels are ignored in the input
287  images the image based on config.nSigmaClipGainCalc.
288 
289  - Each image is then cross-correlated with the one it's paired with
290  (with the pairing defined by the --visit-pairs command line argument),
291  which is done either the whole-image to whole-image,
292  or amplifier-by-amplifier, depending on config.level.
293 
294  - Once the cross-correlations have been calculated for each visit pair,
295  these are used to generate the correction kernel.
296  The maximum lag used, in pixels, and hence the size of the half-size
297  of the kernel generated, is given by config.maxLag,
298  i.e. a value of 10 will result in a kernel of size 2n-1 = 19x19 pixels.
299  Outlier values in these cross-correlations are rejected by using a
300  pixel-wise sigma-clipped thresholding to each cross-correlation in
301  the visit-pairs-length stack of cross-correlations.
302  The number of sigma clipped to is set by config.nSigmaClipKernelGen.
303 
304  - Once DM-15277 has been completed, a method will exist to calculate the
305  empirical correction factor, config.biasCorr.
306  TODO: DM-15277 update this part of the docstring once the ticket is done.
307  """
308 
309  RunnerClass = MakeBrighterFatterKernelTaskRunner
310  ConfigClass = MakeBrighterFatterKernelTaskConfig
311  _DefaultName = "makeBrighterFatterKernel"
312 
313  def __init__(self, *args, **kwargs):
314  pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
315  self.makeSubtask("isr")
316 
317  self.debug = lsstDebug.Info(__name__)
318  if self.debug.enabled:
319  self.log.info("Running with debug enabled...")
320  # If we're displaying, test it works and save displays for later.
321  # It's worth testing here as displays are flaky and sometimes
322  # can't be contacted, and given processing takes a while,
323  # it's a shame to fail late due to display issues.
324  if self.debug.display:
325  try:
326  afwDisp.setDefaultBackend(self.debug.displayBackend)
327  afwDisp.Display.delAllDisplays()
328  self.disp1 = afwDisp.Display(0, open=True)
329  self.disp2 = afwDisp.Display(1, open=True)
330 
331  im = afwImage.ImageF(1, 1)
332  im.array[:] = [[1]]
333  self.disp1.mtv(im)
334  self.disp1.erase()
335  except NameError:
336  self.debug.display = False
337  self.log.warn('Failed to setup/connect to display! Debug display has been disabled')
338 
339  plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too
340  self.validateIsrConfig()
341  self.config.validate()
342  self.config.freeze()
343 
344  @classmethod
345  def _makeArgumentParser(cls):
346  """Augment argument parser for the MakeBrighterFatterKernelTask."""
347  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
348  parser.add_argument("--visit-pairs", dest="visitPairs", nargs="*",
349  help="Visit pairs to use. Each pair must be of the form INT,INT e.g. 123,456")
350  parser.add_id_argument("--id", datasetType="brighterFatterKernel",
351  ContainerClass=BrighterFatterKernelTaskDataIdContainer,
352  help="The ccds to use, e.g. --id ccd=0..100")
353  return parser
354 
355  def validateIsrConfig(self):
356  """Check that appropriate ISR settings are being used
357  for brighter-fatter kernel calculation."""
358 
359  # How should we handle saturation/bad regions?
360  # 'doSaturationInterpolation': True
361  # 'doNanInterpAfterFlat': False
362  # 'doSaturation': True
363  # 'doSuspect': True
364  # 'doWidenSaturationTrails': True
365  # 'doSetBadRegions': True
366 
367  configDict = self.config.isr.toDict()
368 
369  for configParam in self.config.isrMandatorySteps:
370  if configDict[configParam] is False:
371  raise RuntimeError('Must set config.isr.%s to True '
372  'for brighter-fatter kernel calulation' % configParam)
373 
374  for configParam in self.config.isrForbiddenSteps:
375  if configDict[configParam] is True:
376  raise RuntimeError('Must set config.isr.%s to False '
377  'for brighter-fatter kernel calulation' % configParam)
378 
379  for configParam in self.config.isrDesirableSteps:
380  if configParam not in configDict:
381  self.log.info('Failed to find key %s in the isr config dict. You probably want ' +
382  'to set the equivalent for your obs_package to True.' % configParam)
383  continue
384  if configDict[configParam] is False:
385  self.log.warn('Found config.isr.%s set to False for brighter-fatter kernel calulation. '
386  'It is probably desirable to have this set to True' % configParam)
387 
388  # subtask settings
389  if not self.config.isr.assembleCcd.doTrim:
390  raise RuntimeError('Must trim when assembling CCDs. Set config.isr.assembleCcd.doTrim to True')
391 
392  @pipeBase.timeMethod
393  def runDataRef(self, dataRef, visitPairs):
394  """Run the brighter-fatter measurement task.
395 
396  For a dataRef (which is each detector here),
397  and given a list of visit pairs, calulate the
398  brighter-fatter kernel for the detector.
399 
400  Parameters
401  ----------
402  dataRef : list of lsst.daf.persistence.ButlerDataRef
403  dataRef for the detector for the visits to be fit.
404  visitPairs : `iterable` of `tuple` of `int`
405  Pairs of visit numbers to be processed together
406  """
407  xcorrs = {} # dict of lists keyed by either amp or detector depending on config.level
408  means = {}
409  kernels = {}
410 
411  # setup necessary objects
412  detNum = dataRef.dataId[self.config.ccdKey]
413  if self.config.level == 'DETECTOR':
414  xcorrs = {detNum: []}
415  means = {detNum: []}
416  elif self.config.level == 'AMP':
417  # NB: don't use dataRef.get('raw_detector')
418  # this currently doesn't work for composites because of the way
419  # composite objects (i.e. LSST images) are handled/constructed
420  # these need to be retrieved from the camera and dereferenced
421  # rather than accessed directly
422  detector = dataRef.get('camera')[dataRef.dataId[self.config.ccdKey]]
423  ampInfoCat = detector.getAmpInfoCatalog()
424  ampNames = [amp.getName() for amp in ampInfoCat]
425  xcorrs = {key: [] for key in ampNames}
426  means = {key: [] for key in ampNames}
427  else:
428  raise RuntimeError("Unsupported level: {}".format(self.config.level))
429 
430  # calculate or retrieve the gains
431  if self.config.doCalcGains:
432  self.log.info('Compute gains for detector %s' % detNum)
433  gains, nomGains = self.estimateGains(dataRef, visitPairs)
434  dataRef.put(gains, datasetType='brighterFatterGain')
435  self.log.debug('Finished gain estimation for detector %s' % detNum)
436  else:
437  gains = dataRef.get('brighterFatterGain')
438  if not gains:
439  raise RuntimeError('Failed to retrieved gains for detector %s' % detNum)
440  self.log.info('Retrieved stored gain for detector %s' % detNum)
441  self.log.debug('Detector %s has gains %s' % (detNum, gains))
442 
443  # Loop over pairs of visits
444  # calculating the cross-correlations at the required level
445  for (v1, v2) in visitPairs:
446 
447  dataRef.dataId['visit'] = v1
448  exp1 = self.isr.runDataRef(dataRef).exposure
449  dataRef.dataId['visit'] = v2
450  exp2 = self.isr.runDataRef(dataRef).exposure
451  del dataRef.dataId['visit']
452  self._checkExpLengthEqual(exp1, exp2, v1, v2)
453 
454  self.log.info('Preparing images for cross-correlation calculation for detector %s' % detNum)
455  # note the shape of these returns depends on level
456  _scaledMaskedIms1, _means1 = self._makeCroppedExposures(exp1, gains, self.config.level)
457  _scaledMaskedIms2, _means2 = self._makeCroppedExposures(exp2, gains, self.config.level)
458 
459  # Compute the cross-correlation and means
460  # at the appropriate config.level:
461  # - "DETECTOR": one key, so compare the two visits to each other
462  # - "AMP": n_amp keys, comparing each amplifier of one visit
463  # to the same amplifier in the visit its paired with
464  for det_object in _scaledMaskedIms1.keys():
465  _xcorr, _ = self._crossCorrelate(_scaledMaskedIms1[det_object],
466  _scaledMaskedIms2[det_object])
467  xcorrs[det_object].append(_xcorr)
468  means[det_object].append([_means1[det_object], _means2[det_object]])
469 
470  # TODO: DM-15305 improve debug functionality here.
471  # This is position 1 for the removed code.
472 
473  # generate the kernel(s)
474  self.log.info('Generating kernel(s) for %s' % detNum)
475  for det_object in xcorrs.keys(): # looping over either detectors or amps
476  if self.config.level == 'DETECTOR':
477  objId = 'detector %s' % det_object
478  elif self.config.level == 'AMP':
479  objId = 'detector %s AMP %s' % (detNum, det_object)
480  kernels[det_object] = self.generateKernel(xcorrs[det_object], means[det_object], objId)
481  dataRef.put(BrighterFatterKernel(self.config.level, kernels))
482 
483  self.log.info('Finished generating kernel(s) for %s' % detNum)
484  return pipeBase.Struct(exitStatus=0)
485 
486  def _makeCroppedExposures(self, exp, gains, level):
487  """Prepare exposure for cross-correlation calculation.
488 
489  For each amp, crop by the border amount, specified by
490  config.nPixBorderXCorr, then rescale by the gain
491  and subtract the sigma-clipped mean.
492  If the level is 'DETECTOR' then this is done
493  to the whole image so that it can be cross-correlated, with a copy
494  being returned.
495  If the level is 'AMP' then this is done per-amplifier,
496  and a copy of each prepared amp-image returned.
497 
498  Parameters:
499  -----------
500  exp : `lsst.afw.image.exposure.ExposureF`
501  The exposure to prepare
502  gains : `dict` of `float`
503  Dictionary of the amplifier gain values, keyed by amplifier name
504  level : `str`
505  Either `AMP` or `DETECTOR`
506 
507  Returns:
508  --------
509  scaledMaskedIms : `dict` of `lsst.afw.image.maskedImage.MaskedImageF`
510  Depending on level, this is either one item, or n_amp items,
511  keyed by detectorId or ampName
512 
513  Notes:
514  ------
515  This function is controlled by the following config parameters:
516  nPixBorderXCorr : `int`
517  The number of border pixels to exclude
518  nSigmaClipXCorr : `float`
519  The number of sigma to be clipped to
520  """
521  assert(isinstance(exp, afwImage.ExposureF))
522 
523  local_exp = exp.clone() # we don't want to modify the image passed in
524  del exp # ensure we don't make mistakes!
525 
526  border = self.config.nPixBorderXCorr
527  sigma = self.config.nSigmaClipXCorr
528 
529  sctrl = afwMath.StatisticsControl()
530  sctrl.setNumSigmaClip(sigma)
531 
532  means = {}
533  returnAreas = {}
534 
535  detector = local_exp.getDetector()
536  ampInfoCat = detector.getAmpInfoCatalog()
537 
538  mi = local_exp.getMaskedImage() # makeStatistics does not seem to take exposures
539  temp = mi.clone()
540 
541  # Rescale each amp by the appropriate gain and subtract the mean.
542  # NB these are views modifying the image in-place
543  for amp in ampInfoCat:
544  ampName = amp.getName()
545  rescaleIm = mi[amp.getBBox()] # the soon-to-be scaled, mean subtractedm, amp image
546  rescaleTemp = temp[amp.getBBox()]
547  mean = afwMath.makeStatistics(rescaleIm, afwMath.MEANCLIP, sctrl).getValue()
548  gain = gains[ampName]
549  rescaleIm *= gain
550  rescaleTemp *= gain
551  self.log.debug("mean*gain = %s, clipped mean = %s" %
552  (mean*gain, afwMath.makeStatistics(rescaleIm, afwMath.MEANCLIP,
553  sctrl).getValue()))
554  rescaleIm -= mean*gain
555 
556  if level == 'AMP': # build the dicts if doing amp-wise
557  means[ampName] = afwMath.makeStatistics(rescaleTemp[border: -border, border: -border,
558  afwImage.LOCAL], afwMath.MEANCLIP, sctrl).getValue()
559  returnAreas[ampName] = rescaleIm
560 
561  if level == 'DETECTOR': # else just average the whole detector
562  detName = local_exp.getDetector().getId()
563  means[detName] = afwMath.makeStatistics(temp[border: -border, border: -border, afwImage.LOCAL],
564  afwMath.MEANCLIP, sctrl).getValue()
565  returnAreas[detName] = rescaleIm
566 
567  return returnAreas, means
568 
569  def _crossCorrelate(self, maskedIm0, maskedIm1, frameId=None, detId=None):
570  """Calculate the cross-correlation of an area.
571 
572  If the area in question contains multiple amplifiers then they must
573  have been gain corrected.
574 
575  Parameters:
576  -----------
577  maskedIm0 : `lsst.afw.image.MaskedImageF`
578  The first image area
579  maskedIm1 : `lsst.afw.image.MaskedImageF`
580  The first image area
581  frameId : `str`, optional
582  The frame identifier for use in the filename
583  if writing debug outputs.
584  detId : `str`, optional
585  The detector identifier (detector, or detector+amp,
586  depending on config.level) for use in the filename
587  if writing debug outputs.
588 
589  Returns:
590  --------
591  xcorr : `np.ndarray`
592  The quarter-image cross-correlation
593  mean : `float`
594  The sum of the means of the input images,
595  sigma-clipped, and with borders applied.
596  This is used when using this function with simulations to calculate
597  the biasCorr parameter.
598 
599  Notes:
600  ------
601  This function is controlled by the following config parameters:
602  maxLag : `int`
603  The maximum lag to use in the cross-correlation calculation
604  nPixBorderXCorr : `int`
605  The number of border pixels to exclude
606  nSigmaClipXCorr : `float`
607  The number of sigma to be clipped to
608  biasCorr : `float`
609  Parameter used to correct from the bias introduced
610  by the sigma cuts.
611  """
612  maxLag = self.config.maxLag
613  border = self.config.nPixBorderXCorr
614  sigma = self.config.nSigmaClipXCorr
615  biasCorr = self.config.biasCorr
616 
617  sctrl = afwMath.StatisticsControl()
618  sctrl.setNumSigmaClip(sigma)
619 
620  mean = afwMath.makeStatistics(maskedIm0.getImage()[border: -border, border: -border, afwImage.LOCAL],
621  afwMath.MEANCLIP, sctrl).getValue()
622  mean += afwMath.makeStatistics(maskedIm1.getImage()[border: -border, border: -border, afwImage.LOCAL],
623  afwMath.MEANCLIP, sctrl).getValue()
624 
625  # Diff the images, and apply border
626  diff = maskedIm0.clone()
627  diff -= maskedIm1.getImage()
628  diff = diff[border: -border, border: -border, afwImage.LOCAL]
629 
630  if self.debug.writeDiffImages:
631  filename = '_'.join(['diff', 'detector', detId, frameId, '.fits'])
632  diff.writeFits(os.path.join(self.debug.debugDataPath, filename))
633 
634  # Subtract background. It should be a constant, but it isn't always
635  binsize = self.config.backgroundBinSize
636  nx = diff.getWidth()//binsize
637  ny = diff.getHeight()//binsize
638  bctrl = afwMath.BackgroundControl(nx, ny, sctrl, afwMath.MEANCLIP)
639  bkgd = afwMath.makeBackground(diff, bctrl)
640  bgImg = bkgd.getImageF(afwMath.Interpolate.CUBIC_SPLINE, afwMath.REDUCE_INTERP_ORDER)
641  bgMean = np.mean(bgImg.getArray())
642  if abs(bgMean) >= self.config.backgroundWarnLevel:
643  self.log.warn('Mean of background = %s > config.maxBackground' % bgMean)
644 
645  diff -= bgImg
646 
647  if self.debug.writeDiffImages:
648  filename = '_'.join(['bgSub', 'diff', 'detector', detId, frameId, '.fits'])
649  diff.writeFits(os.path.join(self.debug.debugDataPath, filename))
650  if self.debug.display:
651  self.disp1.mtv(diff, title=frameId)
652 
653  self.log.debug("Median and variance of diff:")
654  self.log.debug("%s" % afwMath.makeStatistics(diff, afwMath.MEDIAN, sctrl).getValue())
655  self.log.debug("%s" % afwMath.makeStatistics(diff, afwMath.VARIANCECLIP,
656  sctrl).getValue(), np.var(diff.getImage().getArray()))
657 
658  # Measure the correlations
659  dim0 = diff[0: -maxLag, : -maxLag, afwImage.LOCAL]
660  dim0 -= afwMath.makeStatistics(dim0, afwMath.MEANCLIP, sctrl).getValue()
661  width, height = dim0.getDimensions()
662  xcorr = np.zeros((maxLag + 1, maxLag + 1), dtype=np.float64)
663 
664  for xlag in range(maxLag + 1):
665  for ylag in range(maxLag + 1):
666  dim_xy = diff[xlag:xlag + width, ylag: ylag + height, afwImage.LOCAL].clone()
667  dim_xy -= afwMath.makeStatistics(dim_xy, afwMath.MEANCLIP, sctrl).getValue()
668  dim_xy *= dim0
669  xcorr[xlag, ylag] = afwMath.makeStatistics(dim_xy,
670  afwMath.MEANCLIP, sctrl).getValue()/(biasCorr)
671 
672  # TODO: DM-15305 improve debug functionality here.
673  # This is position 2 for the removed code.
674 
675  return xcorr, mean
676 
677  def estimateGains(self, dataRef, visitPairs):
678  """Estimate the amplifier gains using the specified visits.
679 
680  Given a dataRef and list of flats of varying intensity,
681  calculate the gain for each amplifier in the detector
682  using the photon transfer curve (PTC) method.
683 
684  The config.fixPtcThroughOrigin option determines whether the iterative
685  fitting is forced to go through the origin or not.
686  This defaults to True, fitting var=1/gain * mean.
687  If set to False then var=1/g * mean + const is fitted.
688 
689  This is really a photo transfer curve (PTC) gain measurement task.
690  See DM-14063 for results from of a comparison between
691  this task's numbers and the gain values in the HSC camera model,
692  and those measured by the PTC task in eotest.
693 
694  Parameters
695  ----------
696  dataRef : `lsst.daf.persistence.butler.Butler.dataRef`
697  dataRef for the detector for the flats to be used
698  visitPairs : `list` of `tuple`
699  List of visit-pairs to use, as [(v1,v2), (v3,v4)...]
700 
701  Returns
702  -------
703  gains : `dict` of `float`
704  Dict of the as-calculated amplifier gain values,
705  keyed by amplifier name
706  nominalGains : `dict` of `float`
707  Dict of the amplifier gains, as reported by the `detector` object,
708  keyed by amplifier name
709  """
710  # NB: don't use dataRef.get('raw_detector') due to composites
711  detector = dataRef.get('camera')[dataRef.dataId[self.config.ccdKey]]
712  ampInfoCat = detector.getAmpInfoCatalog()
713  ampNames = [amp.getName() for amp in ampInfoCat]
714 
715  ampMeans = {key: [] for key in ampNames} # these get turned into np.arrays later
716  ampCoVariances = {key: [] for key in ampNames}
717  ampVariances = {key: [] for key in ampNames}
718 
719  # Loop over the amps in the detector,
720  # calculating a PTC for each amplifier.
721  # The amplifier iteration is performed in _calcMeansAndVars()
722  # NB: no gain correction is applied
723  for visPairNum, visPair in enumerate(visitPairs):
724  _means, _vars, _covars = self._calcMeansAndVars(dataRef, visPair[0], visPair[1])
725 
726  # Do sanity checks; if these are failed more investigation is needed
727  breaker = 0
728  for amp in detector:
729  ampName = amp.getName()
730  if _means[ampName]*10 < _vars[ampName] or _means[ampName]*10 < _covars[ampName]:
731  msg = 'Sanity check failed; check visit pair %s amp %s' % (visPair, ampName)
732  self.log.warn(msg)
733  breaker += 1
734  if breaker:
735  continue
736 
737  # having made sanity checks
738  # pull the values out into the respective dicts
739  for k in _means.keys(): # keys are necessarily the same
740  if _vars[k]*1.3 < _covars[k] or _vars[k]*0.7 > _covars[k]:
741  self.log.warn('Dropped a value')
742  continue
743  ampMeans[k].append(_means[k])
744  ampVariances[k].append(_vars[k])
745  ampCoVariances[k].append(_covars[k])
746 
747  gains = {}
748  nomGains = {}
749  for amp in detector:
750  ampName = amp.getName()
751  nomGains[ampName] = amp.getGain()
752  slopeRaw, interceptRaw, rVal, pVal, stdErr = \
753  stats.linregress(np.asarray(ampMeans[ampName]), np.asarray(ampCoVariances[ampName]))
754  slopeFix, _ = self._iterativeRegression(np.asarray(ampMeans[ampName]),
755  np.asarray(ampCoVariances[ampName]),
756  fixThroughOrigin=True)
757  slopeUnfix, intercept = self._iterativeRegression(np.asarray(ampMeans[ampName]),
758  np.asarray(ampCoVariances[ampName]),
759  fixThroughOrigin=False)
760  self.log.info("Slope of raw fit: %s, intercept: %s p value: %s" % (slopeRaw,
761  interceptRaw, pVal))
762  self.log.info("slope of fixed fit: %s, difference vs raw:%s" % (slopeFix,
763  slopeFix - slopeRaw))
764  self.log.info("slope of unfixed fit: %s, difference vs fix:%s" % (slopeUnfix,
765  slopeFix - slopeUnfix))
766  if self.config.fixPtcThroughOrigin:
767  slopeToUse = slopeFix
768  else:
769  slopeToUse = slopeUnfix
770 
771  if self.debug.enabled:
772  fig = plt.figure()
773  ax = fig.add_subplot(111)
774  ax.plot(np.asarray(ampMeans[ampName]),
775  np.asarray(ampCoVariances[ampName]), linestyle='None', marker='x', label='data')
776  if self.config.fixPtcThroughOrigin:
777  ax.plot(np.asarray(ampMeans[ampName]),
778  np.asarray(ampMeans[ampName])*slopeToUse, label='Fit through origin')
779  else:
780  ax.plot(np.asarray(ampMeans[ampName]),
781  np.asarray(ampMeans[ampName])*slopeToUse + intercept,
782  label='Fit (intercept unconstrained')
783 
784  dataRef.put(fig, "plotBrighterFatterPtc", amp=ampName)
785  self.log.info('Saved PTC for detector %s amp %s' % (detector.getId(), ampName))
786  gains[ampName] = 1.0/slopeToUse
787  return gains, nomGains
788 
789  @staticmethod
790  def _checkExpLengthEqual(exp1, exp2, v1=None, v2=None):
791  """Check the exposure lengths of two exposures are equal.
792 
793  Parameters:
794  -----------
795  exp1 : `lsst.afw.image.exposure.ExposureF`
796  First exposure to check
797  exp2 : `lsst.afw.image.exposure.ExposureF`
798  Second exposure to check
799  v1 : `int` or `str`, optional
800  First visit of the visit pair
801  v2 : `int` or `str`, optional
802  Second visit of the visit pair
803 
804  Raises:
805  -------
806  RuntimeError
807  Raised if the exposure lengths of the two exposures are not equal
808  """
809  expTime1 = exp1.getInfo().getVisitInfo().getExposureTime()
810  expTime2 = exp2.getInfo().getVisitInfo().getExposureTime()
811  if expTime1 != expTime2:
812  msg = "Exposure lengths for visit pairs must be equal. " + \
813  "Found %s and %s" % (expTime1, expTime2)
814  if v1 and v2:
815  msg += " for visit pair %s, %s" % (v1, v2)
816  raise RuntimeError(msg)
817 
818  def _calcMeansAndVars(self, dataRef, v1, v2):
819  """Calculate the means, vars, covars, and retrieve the nominal gains,
820  for each amp in each detector.
821 
822  This code runs using two visit numbers, and for the detector specified.
823  It calculates the correlations in the individual amps without
824  rescaling any gains. This allows a photon transfer curve
825  to be generated and the gains measured.
826 
827  Images are assembled with use the isrTask, and basic isr is performed.
828 
829  Parameters:
830  -----------
831  dataRef : `lsst.daf.persistence.butler.Butler.dataRef`
832  dataRef for the detector for the repo containg the flats to be used
833  v1 : `int`
834  First visit of the visit pair
835  v2 : `int`
836  Second visit of the visit pair
837 
838  Returns
839  -------
840  means, vars, covars : `tuple` of `dicts`
841  Three dicts, keyed by ampName,
842  containing the sum of the image-means,
843  the variance, and the quarter-image of the xcorr.
844  """
845  sigma = self.config.nSigmaClipGainCalc
846  maxLag = self.config.maxLag
847  border = self.config.nPixBorderGainCalc
848  biasCorr = self.config.biasCorr
849 
850  # NB: don't use dataRef.get('raw_detector') due to composites
851  detector = dataRef.get('camera')[dataRef.dataId[self.config.ccdKey]]
852 
853  ampMeans = {}
854 
855  # manipulate the dataId to get a postISR exposure for each visit
856  # from the detector obj, restoring its original state afterwards
857  originalDataId = dataRef.dataId.copy()
858  dataRef.dataId['visit'] = v1
859  exp1 = self.isr.runDataRef(dataRef).exposure
860  dataRef.dataId['visit'] = v2
861  exp2 = self.isr.runDataRef(dataRef).exposure
862  dataRef.dataId = originalDataId
863  exps = [exp1, exp2]
864  self._checkExpLengthEqual(exp1, exp2, v1, v2)
865 
866  detector = exps[0].getDetector()
867  ims = [self._convertImagelikeToFloatImage(exp) for exp in exps]
868 
869  if self.debug.display:
870  self.disp1.mtv(ims[0], title=str(v1))
871  self.disp2.mtv(ims[1], title=str(v2))
872 
873  sctrl = afwMath.StatisticsControl()
874  sctrl.setNumSigmaClip(sigma)
875  for imNum, im in enumerate(ims):
876 
877  # calculate the sigma-clipped mean, excluding the borders
878  # safest to apply borders to all amps regardless of edges
879  # easier, camera-agnostic, and mitigates potentially dodgy
880  # overscan-biases around edges as well
881  for amp in detector:
882  ampName = amp.getName()
883  ampIm = im[amp.getBBox()]
884  mean = afwMath.makeStatistics(ampIm[border: -border, border: -border, afwImage.LOCAL],
885  afwMath.MEANCLIP, sctrl).getValue()
886  if ampName not in ampMeans.keys():
887  ampMeans[ampName] = []
888  ampMeans[ampName].append(mean)
889  ampIm -= mean
890 
891  diff = ims[0].clone()
892  diff -= ims[1]
893 
894  temp = diff[border: -border, border: -border, afwImage.LOCAL]
895 
896  # Subtract background. It should be a constant,
897  # but it isn't always (e.g. some SuprimeCam flats)
898  # TODO: Check how this looks, and if this is the "right" way to do this
899  binsize = self.config.backgroundBinSize
900  nx = temp.getWidth()//binsize
901  ny = temp.getHeight()//binsize
902  bctrl = afwMath.BackgroundControl(nx, ny, sctrl, afwMath.MEANCLIP)
903  bkgd = afwMath.makeBackground(temp, bctrl)
904 
905  box = diff.getBBox()
906  box.grow(-border)
907  diff[box, afwImage.LOCAL] -= bkgd.getImageF(afwMath.Interpolate.CUBIC_SPLINE,
908  afwMath.REDUCE_INTERP_ORDER)
909 
910  variances = {}
911  coVars = {}
912  for amp in detector:
913  ampName = amp.getName()
914 
915  diffAmpIm = diff[amp.getBBox()].clone()
916  diffAmpImCrop = diffAmpIm[border: -border - maxLag, border: -border - maxLag, afwImage.LOCAL]
917  diffAmpImCrop -= afwMath.makeStatistics(diffAmpImCrop, afwMath.MEANCLIP, sctrl).getValue()
918  w, h = diffAmpImCrop.getDimensions()
919  xcorr = np.zeros((maxLag + 1, maxLag + 1), dtype=np.float64)
920 
921  # calculate the cross-correlation
922  for xlag in range(maxLag + 1):
923  for ylag in range(maxLag + 1):
924  dim_xy = diffAmpIm[border + xlag: border + xlag + w,
925  border + ylag: border + ylag + h,
926  afwImage.LOCAL].clone()
927  dim_xy -= afwMath.makeStatistics(dim_xy, afwMath.MEANCLIP, sctrl).getValue()
928  dim_xy *= diffAmpImCrop
929  xcorr[xlag, ylag] = afwMath.makeStatistics(dim_xy,
930  afwMath.MEANCLIP, sctrl).getValue()/(biasCorr)
931 
932  variances[ampName] = xcorr[0, 0]
933  xcorr_full = self._tileArray(xcorr)
934  coVars[ampName] = np.sum(xcorr_full)
935 
936  msg = "M1: " + str(ampMeans[ampName][0])
937  msg += " M2 " + str(ampMeans[ampName][1])
938  msg += " M_sum: " + str((ampMeans[ampName][0]) + ampMeans[ampName][1])
939  msg += " Var " + str(variances[ampName])
940  msg += " coVar: " + str(coVars[ampName])
941  self.log.debug(msg)
942 
943  means = {}
944  for amp in detector:
945  ampName = amp.getName()
946  means[ampName] = ampMeans[ampName][0] + ampMeans[ampName][1]
947 
948  return means, variances, coVars
949 
950  def _plotXcorr(self, xcorr, mean, zmax=0.05, title=None, fig=None, saveToFileName=None):
951  """Plot the correlation functions."""
952  try:
953  xcorr = xcorr.getArray()
954  except Exception:
955  pass
956 
957  xcorr /= float(mean)
958  # xcorr.getArray()[0,0]=abs(xcorr.getArray()[0,0]-1)
959 
960  if fig is None:
961  fig = plt.figure()
962  else:
963  fig.clf()
964 
965  ax = fig.add_subplot(111, projection='3d')
966  ax.azim = 30
967  ax.elev = 20
968 
969  nx, ny = np.shape(xcorr)
970 
971  xpos, ypos = np.meshgrid(np.arange(nx), np.arange(ny))
972  xpos = xpos.flatten()
973  ypos = ypos.flatten()
974  zpos = np.zeros(nx*ny)
975  dz = xcorr.flatten()
976  dz[dz > zmax] = zmax
977 
978  ax.bar3d(xpos, ypos, zpos, 1, 1, dz, color='b', zsort='max', sort_zpos=100)
979  if xcorr[0, 0] > zmax:
980  ax.bar3d([0], [0], [zmax], 1, 1, 1e-4, color='c')
981 
982  ax.set_xlabel("row")
983  ax.set_ylabel("column")
984  ax.set_zlabel(r"$\langle{(F_i - \bar{F})(F_i - \bar{F})}\rangle/\bar{F}$")
985 
986  if title:
987  fig.suptitle(title)
988  if saveToFileName:
989  fig.savefig(saveToFileName)
990 
991  def _iterativeRegression(self, x, y, fixThroughOrigin=False, nSigmaClip=None, maxIter=None):
992  """Use linear regression to fit a line, iteratively removing outliers.
993 
994  Useful when you have a sufficiently large numbers of points on your PTC.
995  This function iterates until either there are no outliers of
996  config.nSigmaClip magnitude, or until the specified maximum number
997  of iterations has been performed.
998 
999  Parameters:
1000  -----------
1001  x : `numpy.array`
1002  The independent variable. Must be a numpy array, not a list.
1003  y : `numpy.array`
1004  The dependent variable. Must be a numpy array, not a list.
1005  fixThroughOrigin : `bool`, optional
1006  Whether to fix the PTC through the origin or allow an y-intercept.
1007  nSigmaClip : `float`, optional
1008  The number of sigma to clip to.
1009  Taken from the task config if not specified.
1010  maxIter : `int`, optional
1011  The maximum number of iterations allowed.
1012  Taken from the task config if not specified.
1013 
1014  Returns:
1015  --------
1016  slope : `float`
1017  The slope of the line of best fit
1018  intercept : `float`
1019  The y-intercept of the line of best fit
1020  """
1021  if not maxIter:
1022  maxIter = self.config.maxIterRegression
1023  if not nSigmaClip:
1024  nSigmaClip = self.config.nSigmaClipRegression
1025 
1026  nIter = 0
1027  sctrl = afwMath.StatisticsControl()
1028  sctrl.setNumSigmaClip(nSigmaClip)
1029 
1030  if fixThroughOrigin:
1031  while nIter < maxIter:
1032  nIter += 1
1033  self.log.debug("Origin fixed, iteration # %s using %s elements:" % (nIter, np.shape(x)[0]))
1034  TEST = x[:, np.newaxis]
1035  slope, _, _, _ = np.linalg.lstsq(TEST, y)
1036  slope = slope[0]
1037  res = y - slope * x
1038  resMean = afwMath.makeStatistics(res, afwMath.MEANCLIP, sctrl).getValue()
1039  resStd = np.sqrt(afwMath.makeStatistics(res, afwMath.VARIANCECLIP, sctrl).getValue())
1040  index = np.where((res > (resMean + nSigmaClip*resStd)) |
1041  (res < (resMean - nSigmaClip*resStd)))
1042  self.log.debug("%.3f %.3f %.3f %.3f" % (resMean, resStd, np.max(res), nSigmaClip))
1043  if np.shape(np.where(index))[1] == 0 or (nIter >= maxIter): # run out of points or iters
1044  break
1045  x = np.delete(x, index)
1046  y = np.delete(y, index)
1047 
1048  return slope, 0
1049 
1050  while nIter < maxIter:
1051  nIter += 1
1052  self.log.debug("Iteration # %s using %s elements:" % (nIter, np.shape(x)[0]))
1053  xx = np.vstack([x, np.ones(len(x))]).T
1054  ret, _, _, _ = np.linalg.lstsq(xx, y)
1055  slope, intercept = ret
1056  res = y - slope*x - intercept
1057  resMean = afwMath.makeStatistics(res, afwMath.MEANCLIP, sctrl).getValue()
1058  resStd = np.sqrt(afwMath.makeStatistics(res, afwMath.VARIANCECLIP, sctrl).getValue())
1059  index = np.where((res > (resMean + nSigmaClip * resStd)) | (res < resMean - nSigmaClip * resStd))
1060  self.log.debug("%.3f %.3f %.3f %.3f" % (resMean, resStd, np.max(res), nSigmaClip))
1061  if np.shape(np.where(index))[1] == 0 or (nIter >= maxIter): # run out of points, or iterations
1062  break
1063  x = np.delete(x, index)
1064  y = np.delete(y, index)
1065 
1066  return slope, intercept
1067 
1068  def generateKernel(self, corrs, means, objId, rejectLevel=None):
1069  """Generate the full kernel from a list of cross-correlations and means.
1070 
1071  Taking a list of quarter-image, gain-corrected cross-correlations,
1072  do a pixel-wise sigma-clipped mean of each,
1073  and tile into the full-sized kernel image.
1074 
1075  Each corr in corrs is one quarter of the full cross-correlation,
1076  and has been gain-corrected. Each mean in means is a tuple of the means
1077  of the two individual images, corresponding to that corr.
1078 
1079  Parameters:
1080  -----------
1081  corrs : `list` of `numpy.ndarray`, (Ny, Nx)
1082  A list of the quarter-image cross-correlations
1083  means : `dict` of `tuples` of `floats`
1084  The means of the input images for each corr in corrs
1085  rejectLevel : `float`, optional
1086  This is essentially is a sanity check parameter.
1087  If this condition is violated there is something unexpected
1088  going on in the image, and it is discarded from the stack before
1089  the clipped-mean is calculated.
1090  If not provided then config.xcorrCheckRejectLevel is used
1091 
1092  Returns:
1093  --------
1094  kernel : `numpy.ndarray`, (Ny, Nx)
1095  The output kernel
1096  """
1097  if not rejectLevel:
1098  rejectLevel = self.config.xcorrCheckRejectLevel
1099 
1100  # Try to average over a set of possible inputs.
1101  # This generates a simple function of the kernel that
1102  # should be constant across the images, and averages that.
1103  xcorrList = []
1104  sctrl = afwMath.StatisticsControl()
1105  sctrl.setNumSigmaClip(self.config.nSigmaClipKernelGen)
1106 
1107  for corrNum, ((mean1, mean2), corr) in enumerate(zip(means, corrs)):
1108  corr[0, 0] -= (mean1 + mean2)
1109  if corr[0, 0] > 0:
1110  self.log.warn('Skipped item %s due to unexpected value of (variance-mean)' % corrNum)
1111  continue
1112  corr /= -1.0*(mean1**2 + mean2**2)
1113 
1114  fullCorr = self._tileArray(corr)
1115 
1116  xcorrCheck = np.abs(np.sum(fullCorr))/np.sum(np.abs(fullCorr))
1117  if xcorrCheck > rejectLevel:
1118  self.log.warn("Sum of the xcorr is unexpectedly high. Investigate item num %s for %s. \n"
1119  "value = %s" % (corrNum, objId, xcorrCheck))
1120  continue
1121  xcorrList.append(fullCorr)
1122 
1123  if not xcorrList:
1124  raise RuntimeError("Cannot generate kernel because all inputs were discarded. "
1125  "Either the data is bad, or config.xcorrCheckRejectLevel is too low")
1126 
1127  # stack the individual xcorrs and apply a per-pixel clipped-mean
1128  meanXcorr = np.zeros_like(fullCorr)
1129  xcorrList = np.transpose(xcorrList)
1130  for i in range(np.shape(meanXcorr)[0]):
1131  for j in range(np.shape(meanXcorr)[1]):
1132  meanXcorr[i, j] = afwMath.makeStatistics(xcorrList[i, j], afwMath.MEANCLIP, sctrl).getValue()
1133 
1134  return self.successiveOverRelax(meanXcorr)
1135 
1136  def successiveOverRelax(self, source, maxIter=None, eLevel=None):
1137  """An implementation of the successive over relaxation (SOR) method.
1138 
1139  A numerical method for solving a system of linear equations
1140  with faster convergence than the Gauss-Seidel method.
1141 
1142  Parameters:
1143  -----------
1144  source : `numpy.ndarray`
1145  The input array
1146  maxIter : `int`, optional
1147  Maximum number of iterations to attempt before aborting
1148  eLevel : `float`, optional
1149  The target error level at which we deem convergence to have occured
1150 
1151  Returns:
1152  --------
1153  output : `numpy.ndarray`
1154  The solution
1155  """
1156  if not maxIter:
1157  maxIter = self.config.maxIterSuccessiveOverRelaxation
1158  if not eLevel:
1159  eLevel = self.config.eLevelSuccessiveOverRelaxation
1160 
1161  assert source.shape[0] == source.shape[1], "Input array must be square"
1162  # initialise, and set boundary conditions
1163  func = np.zeros([source.shape[0] + 2, source.shape[1] + 2])
1164  resid = np.zeros([source.shape[0] + 2, source.shape[1] + 2])
1165  rhoSpe = np.cos(np.pi/source.shape[0]) # Here a square grid is assummed
1166 
1167  # Calculate the initial error
1168  for i in range(1, func.shape[0] - 1):
1169  for j in range(1, func.shape[1] - 1):
1170  resid[i, j] = (func[i, j - 1] + func[i, j + 1] + func[i - 1, j] +
1171  func[i + 1, j] - 4*func[i, j] - source[i - 1, j - 1])
1172  inError = np.sum(np.abs(resid))
1173 
1174  # Iterate until convergence
1175  # We perform two sweeps per cycle,
1176  # updating 'odd' and 'even' points separately
1177  nIter = 0
1178  omega = 1.0
1179  dx = 1.0
1180  while nIter < maxIter*2:
1181  outError = 0
1182  if nIter%2 == 0:
1183  for i in range(1, func.shape[0] - 1, 2):
1184  for j in range(1, func.shape[1] - 1, 2):
1185  resid[i, j] = float(func[i, j-1] + func[i, j + 1] + func[i - 1, j] +
1186  func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
1187  func[i, j] += omega*resid[i, j]*.25
1188  for i in range(2, func.shape[0] - 1, 2):
1189  for j in range(2, func.shape[1] - 1, 2):
1190  resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j] +
1191  func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
1192  func[i, j] += omega*resid[i, j]*.25
1193  else:
1194  for i in range(1, func.shape[0] - 1, 2):
1195  for j in range(2, func.shape[1] - 1, 2):
1196  resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j] +
1197  func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
1198  func[i, j] += omega*resid[i, j]*.25
1199  for i in range(2, func.shape[0] - 1, 2):
1200  for j in range(1, func.shape[1] - 1, 2):
1201  resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j] +
1202  func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
1203  func[i, j] += omega*resid[i, j]*.25
1204  outError = np.sum(np.abs(resid))
1205  if outError < inError*eLevel:
1206  break
1207  if nIter == 0:
1208  omega = 1.0/(1 - rhoSpe*rhoSpe/2.0)
1209  else:
1210  omega = 1.0/(1 - rhoSpe*rhoSpe*omega/4.0)
1211  nIter += 1
1212 
1213  if nIter >= maxIter*2:
1214  self.log.warn("Failure: SuccessiveOverRelaxation did not converge in %s iterations."
1215  "\noutError: %s, inError: %s," % (nIter//2, outError, inError*eLevel))
1216  else:
1217  self.log.info("Success: SuccessiveOverRelaxation converged in %s iterations."
1218  "\noutError: %s, inError: %s", nIter//2, outError, inError*eLevel)
1219  return func[1: -1, 1: -1]
1220 
1221  @staticmethod
1222  def _tileArray(in_array):
1223  """Given an input quarter-image, tile/mirror it and return full image.
1224 
1225  Given a square input of side-length n, of the form
1226 
1227  input = array([[1, 2, 3],
1228  [4, 5, 6],
1229  [7, 8, 9]])
1230 
1231  return an array of size 2n-1 as
1232 
1233  output = array([[ 9, 8, 7, 8, 9],
1234  [ 6, 5, 4, 5, 6],
1235  [ 3, 2, 1, 2, 3],
1236  [ 6, 5, 4, 5, 6],
1237  [ 9, 8, 7, 8, 9]])
1238 
1239  Parameters:
1240  -----------
1241  input : `np.array`
1242  The square input quarter-array
1243 
1244  Returns:
1245  --------
1246  output : `np.array`
1247  The full, tiled array
1248  """
1249  assert(in_array.shape[0] == in_array.shape[1])
1250  length = in_array.shape[0] - 1
1251  output = np.zeros((2*length + 1, 2*length + 1))
1252 
1253  for i in range(length + 1):
1254  for j in range(length + 1):
1255  output[i + length, j + length] = in_array[i, j]
1256  output[-i + length, j + length] = in_array[i, j]
1257  output[i + length, -j + length] = in_array[i, j]
1258  output[-i + length, -j + length] = in_array[i, j]
1259  return output
1260 
1261  @staticmethod
1262  def _convertImagelikeToFloatImage(imagelikeObject):
1263  """Turn an exposure or masked image of any type into an ImageF."""
1264  for attr in ("getMaskedImage", "getImage"):
1265  if hasattr(imagelikeObject, attr):
1266  imagelikeObject = getattr(imagelikeObject, attr)()
1267  try:
1268  floatImage = imagelikeObject.convertF()
1269  except AttributeError:
1270  raise RuntimeError("Failed to convert image to float")
1271  return floatImage
1272 
1273 
1274 def _crossCorrelateSimulate(im, im2, maxLag, border, nSigmaClip):
1275  """Perform a simple xcorr with two images.
1276 
1277  This sim code is used to estimate the bias correction used in the main task.
1278 
1279  DM-15756 exists to work out why this performs differently,
1280  which version is correct, and whether this function needs to exist at all.
1281 
1282  Parameters:
1283  -----------
1284  maxLag : `int`, optional
1285  The maximum lag to work to in pixels.
1286  nSigmaClip : `float`, optional
1287  Number of sigma to clip to when calculating the sigma-clipped mean.
1288  border : `int`, optional
1289  Number of border pixels to mask.
1290 
1291  Returns:
1292  --------
1293  xcorr : `np.ndarray`
1294  The xcorr image for the image pair.
1295  mean : `dict` of `list` of `float`
1296  The average of the mean flux level in the images after sigma-clipping
1297  and applying the border.
1298  """
1299  sctrl = afwMath.StatisticsControl()
1300  sctrl.setNumSigmaClip(nSigmaClip)
1301 
1302  means = [0, 0]
1303  means[0] = afwMath.makeStatistics(im[border: -border, border: -border, afwImage.LOCAL],
1304  afwMath.MEANCLIP, sctrl).getValue()
1305  means[1] = afwMath.makeStatistics(im2[border: -border, border: -border, afwImage.LOCAL],
1306  afwMath.MEANCLIP, sctrl).getValue()
1307  im -= means[0]
1308  im2 -= means[1]
1309  diff = im2.clone()
1310  diff -= im.clone()
1311  diff = diff[border: -border, border: -border, afwImage.LOCAL]
1312  binsize = 128
1313  nx = diff.getWidth()//binsize
1314  ny = diff.getHeight()//binsize
1315  bctrl = afwMath.BackgroundControl(nx, ny, sctrl, afwMath.MEANCLIP)
1316  bkgd = afwMath.makeBackground(diff, bctrl)
1317  diff -= bkgd.getImageF(afwMath.Interpolate.CUBIC_SPLINE, afwMath.REDUCE_INTERP_ORDER)
1318  dim0 = diff[0: -maxLag, : -maxLag, afwImage.LOCAL].clone()
1319  dim0 -= afwMath.makeStatistics(dim0, afwMath.MEANCLIP, sctrl).getValue()
1320  w, h = dim0.getDimensions()
1321  xcorr = afwImage.ImageD(maxLag + 1, maxLag + 1)
1322  for di in range(maxLag + 1):
1323  for dj in range(maxLag + 1):
1324  dim_ij = diff[di:di + w, dj: dj + h, afwImage.LOCAL].clone()
1325  dim_ij -= afwMath.makeStatistics(dim_ij, afwMath.MEANCLIP, sctrl).getValue()
1326 
1327  dim_ij *= dim0
1328  xcorr[di, dj] = afwMath.makeStatistics(dim_ij, afwMath.MEANCLIP, sctrl).getValue()
1329  L = np.shape(xcorr.getArray())[0] - 1
1330  XCORR = np.zeros([2*L + 1, 2*L + 1])
1331  for i in range(L + 1):
1332  for j in range(L + 1):
1333  XCORR[i + L, j + L] = xcorr.getArray()[i, j]
1334  XCORR[-i + L, j + L] = xcorr.getArray()[i, j]
1335  XCORR[i + L, -j + L] = xcorr.getArray()[i, j]
1336  XCORR[-i + L, -j + L] = xcorr.getArray()[i, j]
1337  return xcorr, np.sum(means)
1338 
1339 
1340 def calcBiasCorr(fluxLevels, imageShape, useTaskCode=True, repeats=1, seed=0, addCorrelations=False,
1341  correlationStrength=0.1, maxLag=10, nSigmaClip=5, border=10):
1342  """Calculate the bias induced when sigma-clipping non-Gassian distributions.
1343 
1344  Fill image-pairs of the specified size with Poisson-distributed values,
1345  adding correlations as necessary. Then calculate the cross correlation,
1346  using the task code (or the separate code, which is to be removed after
1347  DM-15756 is done as these should agree).
1348  Then calculate the bias induced using the cross-correlation image and the
1349  image means.
1350 
1351  Parameters:
1352  -----------
1353  fluxLevels : `list` of `int`
1354  The mean flux levels at which to simiulate.
1355  Nominal values might be something like [70000, 90000, 110000]
1356  imageShape : `tuple` of `int`
1357  The shape of the image array to simulate, nx by ny pixels.
1358  useTaskCode : `bool`, optional
1359  Use the _crossCorrelate() method in the task if True,
1360  else use the _crossCorrelateSimulate() method.
1361  To be removed after DM-15756.
1362  repeats : `int`, optional
1363  Number of repeats to perform so that results
1364  can be averaged to improve SNR.
1365  seed : `int`, optional
1366  The random seed to use for the Poisson points.
1367  addCorrelations : `bool`, optional
1368  Whether to add brighter-fatter-like correlations to the simulated images
1369  If true, a correlation between x_{i,j} and x_{i+1,j+1} is introduced
1370  by adding a*x_{i,j} to x_{i+1,j+1}
1371  correlationStrength : `float`, optional
1372  The strength of the correlations.
1373  This is the value of the coefficient `a` in the above definition.
1374  maxLag : `int`, optional
1375  The maximum lag to work to in pixels
1376  nSigmaClip : `float`, optional
1377  Number of sigma to clip to when calculating the sigma-clipped mean.
1378  border : `int`, optional
1379  Number of border pixels to mask
1380 
1381 
1382  Returns:
1383  --------
1384  biases : `dict` of `list` of `float`
1385  A dictionary, keyed by flux level, containing a list of the biases
1386  for each repeat at that flux level
1387  means : `dict` of `list` of `float`
1388  A dictionary, keyed by flux level, containing a list of the average mean
1389  fluxes (average of the mean of the two images)
1390  for the image pairs at that flux level
1391  xcorrs : `dict` of `list` of `np.ndarray`
1392  A dictionary, keyed by flux level, containing a list of the xcorr
1393  images for the image pairs at that flux level
1394  """
1395  means = {f: [] for f in fluxLevels}
1396  xcorrs = {f: [] for f in fluxLevels}
1397  biases = {f: [] for f in fluxLevels}
1398 
1399  if useTaskCode:
1401  config.isrMandatorySteps = [] # no isr but the validation routine is still run
1402  config.isrForbiddenSteps = []
1403  config.nSigmaClipXCorr = nSigmaClip
1404  config.nPixBorderXCorr = border
1405  config.maxLag = maxLag
1406  task = MakeBrighterFatterKernelTask(config=config)
1407 
1408  im0 = afwImage.maskedImage.MaskedImageF(imageShape[1], imageShape[0])
1409  im1 = afwImage.maskedImage.MaskedImageF(imageShape[1], imageShape[0])
1410 
1411  random = np.random.RandomState(seed)
1412 
1413  for rep in range(repeats):
1414  for flux in fluxLevels:
1415  data0 = random.poisson(flux, (imageShape)).astype(float)
1416  data1 = random.poisson(flux, (imageShape)).astype(float)
1417  if addCorrelations:
1418  data0[1:, 1:] += correlationStrength*data0[: -1, : -1]
1419  data1[1:, 1:] += correlationStrength*data1[: -1, : -1]
1420  im0.image.array[:, :] = data0
1421  im1.image.array[:, :] = data1
1422 
1423  if useTaskCode:
1424  _xcorr, _means = task._crossCorrelate(im0, im1)
1425  else:
1426  _xcorr, _means = _crossCorrelateSimulate(im0, im1, maxLag=maxLag, border=border,
1427  nSigmaClip=nSigmaClip)
1428  _xcorr = _xcorr.getArray() # this code returns and afwImage.ImageD
1429 
1430  means[flux].append(_means)
1431  xcorrs[flux].append(_xcorr)
1432  if addCorrelations:
1433  bias = xcorrs[flux][-1][1, 1]/means[flux][-1]*(1 + correlationStrength)/correlationStrength
1434  print("Simulated/expected avg. flux: %.1f, %.1f" % (flux, means[flux][-1]/2))
1435  print("Bias: %.6f" % bias)
1436  else:
1437  bias = xcorrs[flux][-1][0, 0]/means[flux][-1]
1438  print("Simulated/expected avg. flux: %.1f, %.1f" % (flux, means[flux][-1]/2))
1439  print("Bias: %.6f" % bias)
1440  biases[flux].append(bias)
1441 
1442  return biases, means, xcorrs
def _iterativeRegression(self, x, y, fixThroughOrigin=False, nSigmaClip=None, maxIter=None)
def calcBiasCorr(fluxLevels, imageShape, useTaskCode=True, repeats=1, seed=0, addCorrelations=False, correlationStrength=0.1, maxLag=10, nSigmaClip=5, border=10)
def _crossCorrelate(self, maskedIm0, maskedIm1, frameId=None, detId=None)