Coverage for python/lsst/cp/pipe/makeBrighterFatterKernel.py : 8%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22"""Calculation of brighter-fatter effect correlations and kernels."""
24__all__ = ['MakeBrighterFatterKernelTaskConfig',
25 'MakeBrighterFatterKernelTask',
26 'calcBiasCorr']
28import os
29import copy
30from scipy import stats
31import numpy as np
32import matplotlib.pyplot as plt
33# the following import is required for 3d projection
34from mpl_toolkits.mplot3d import axes3d # noqa: F401
35from matplotlib.backends.backend_pdf import PdfPages
36from dataclasses import dataclass
38import lsstDebug
39import lsst.afw.image as afwImage
40import lsst.afw.math as afwMath
41import lsst.afw.display as afwDisp
42from lsst.ip.isr import IsrTask
43import lsst.log as lsstLog
44import lsst.pex.config as pexConfig
45import lsst.pipe.base as pipeBase
46from .utils import PairedVisitListTaskRunner, checkExpLengthEqual
47import lsst.daf.persistence.butlerExceptions as butlerExceptions
48from lsst.cp.pipe.ptc import (MeasurePhotonTransferCurveTaskConfig, MeasurePhotonTransferCurveTask,
49 PhotonTransferCurveDataset)
52class MakeBrighterFatterKernelTaskConfig(pexConfig.Config):
53 """Config class for bright-fatter effect coefficient calculation."""
55 isr = pexConfig.ConfigurableField(
56 target=IsrTask,
57 doc="""Task to perform instrumental signature removal""",
58 )
59 isrMandatorySteps = pexConfig.ListField(
60 dtype=str,
61 doc="isr operations that must be performed for valid results. Raises if any of these are False",
62 default=['doAssembleCcd']
63 )
64 isrForbiddenSteps = pexConfig.ListField(
65 dtype=str,
66 doc="isr operations that must NOT be performed for valid results. Raises if any of these are True",
67 default=['doFlat', 'doFringe', 'doBrighterFatter', 'doUseOpticsTransmission',
68 'doUseFilterTransmission', 'doUseSensorTransmission', 'doUseAtmosphereTransmission']
69 )
70 isrDesirableSteps = pexConfig.ListField(
71 dtype=str,
72 doc="isr operations that it is advisable to perform, but are not mission-critical." +
73 " WARNs are logged for any of these found to be False.",
74 default=['doBias', 'doDark', 'doCrosstalk', 'doDefect', 'doLinearize']
75 )
76 doCalcGains = pexConfig.Field(
77 dtype=bool,
78 doc="Measure the per-amplifier gains (using the photon transfer curve method)?",
79 default=True,
80 )
81 doPlotPtcs = pexConfig.Field(
82 dtype=bool,
83 doc="Plot the PTCs and butler.put() them as defined by the plotBrighterFatterPtc template",
84 default=False,
85 )
86 forceZeroSum = pexConfig.Field(
87 dtype=bool,
88 doc="Force the correlation matrix to have zero sum by adjusting the (0,0) value?",
89 default=False,
90 )
91 correlationQuadraticFit = pexConfig.Field(
92 dtype=bool,
93 doc="Use a quadratic fit to find the correlations instead of simple averaging?",
94 default=False,
95 )
96 correlationModelRadius = pexConfig.Field(
97 dtype=int,
98 doc="Build a model of the correlation coefficients for radii larger than this value in pixels?",
99 default=100,
100 )
101 correlationModelSlope = pexConfig.Field(
102 dtype=float,
103 doc="Slope of the correlation model for radii larger than correlationModelRadius",
104 default=-1.35,
105 )
106 ccdKey = pexConfig.Field(
107 dtype=str,
108 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'",
109 default='ccd',
110 )
111 minMeanSignal = pexConfig.Field(
112 dtype=float,
113 doc="Minimum value of mean signal (in ADU) to consider.",
114 default=0,
115 )
116 maxMeanSignal = pexConfig.Field(
117 dtype=float,
118 doc="Maximum value to of mean signal (in ADU) to consider.",
119 default=9e6,
120 )
121 maxIterRegression = pexConfig.Field(
122 dtype=int,
123 doc="Maximum number of iterations for the regression fitter",
124 default=2
125 )
126 nSigmaClipGainCalc = pexConfig.Field(
127 dtype=int,
128 doc="Number of sigma to clip the pixel value distribution to during gain calculation",
129 default=5
130 )
131 nSigmaClipRegression = pexConfig.Field(
132 dtype=int,
133 doc="Number of sigma to clip outliers from the line of best fit to during iterative regression",
134 default=4
135 )
136 xcorrCheckRejectLevel = pexConfig.Field(
137 dtype=float,
138 doc="Sanity check level for the sum of the input cross-correlations. Arrays which " +
139 "sum to greater than this are discarded before the clipped mean is calculated.",
140 default=2.0
141 )
142 maxIterSuccessiveOverRelaxation = pexConfig.Field(
143 dtype=int,
144 doc="The maximum number of iterations allowed for the successive over-relaxation method",
145 default=10000
146 )
147 eLevelSuccessiveOverRelaxation = pexConfig.Field(
148 dtype=float,
149 doc="The target residual error for the successive over-relaxation method",
150 default=5.0e-14
151 )
152 nSigmaClipKernelGen = pexConfig.Field(
153 dtype=float,
154 doc="Number of sigma to clip to during pixel-wise clipping when generating the kernel. See " +
155 "the generateKernel docstring for more info.",
156 default=4
157 )
158 nSigmaClipXCorr = pexConfig.Field(
159 dtype=float,
160 doc="Number of sigma to clip when calculating means for the cross-correlation",
161 default=5
162 )
163 maxLag = pexConfig.Field(
164 dtype=int,
165 doc="The maximum lag (in pixels) to use when calculating the cross-correlation/kernel",
166 default=8
167 )
168 nPixBorderGainCalc = pexConfig.Field(
169 dtype=int,
170 doc="The number of border pixels to exclude when calculating the gain",
171 default=10
172 )
173 nPixBorderXCorr = pexConfig.Field(
174 dtype=int,
175 doc="The number of border pixels to exclude when calculating the cross-correlation and kernel",
176 default=10
177 )
178 biasCorr = pexConfig.Field(
179 dtype=float,
180 doc="An empirically determined correction factor, used to correct for the sigma-clipping of" +
181 " a non-Gaussian distribution. Post DM-15277, code will exist here to calculate appropriate values",
182 default=0.9241
183 )
184 backgroundBinSize = pexConfig.Field(
185 dtype=int,
186 doc="Size of the background bins",
187 default=128
188 )
189 fixPtcThroughOrigin = pexConfig.Field(
190 dtype=bool,
191 doc="Constrain the fit of the photon transfer curve to go through the origin when measuring" +
192 "the gain?",
193 default=True
194 )
195 level = pexConfig.ChoiceField(
196 doc="The level at which to calculate the brighter-fatter kernels",
197 dtype=str,
198 default="DETECTOR",
199 allowed={
200 "AMP": "Every amplifier treated separately",
201 "DETECTOR": "One kernel per detector",
202 }
203 )
204 ignoreAmpsForAveraging = pexConfig.ListField(
205 dtype=str,
206 doc="List of amp names to ignore when averaging the amplifier kernels into the detector" +
207 " kernel. Only relevant for level = AMP",
208 default=[]
209 )
210 backgroundWarnLevel = pexConfig.Field(
211 dtype=float,
212 doc="Log warnings if the mean of the fitted background is found to be above this level after " +
213 "differencing image pair.",
214 default=0.1
215 )
218class BrighterFatterKernelTaskDataIdContainer(pipeBase.DataIdContainer):
219 """A DataIdContainer for the MakeBrighterFatterKernelTask."""
221 def makeDataRefList(self, namespace):
222 """Compute refList based on idList.
224 This method must be defined as the dataset does not exist before this
225 task is run.
227 Parameters
228 ----------
229 namespace
230 Results of parsing the command-line.
232 Notes
233 -----
234 Not called if ``add_id_argument`` called
235 with ``doMakeDataRefList=False``.
236 Note that this is almost a copy-and-paste of the vanilla implementation,
237 but without checking if the datasets already exist,
238 as this task exists to make them.
239 """
240 if self.datasetType is None:
241 raise RuntimeError("Must call setDatasetType first")
242 butler = namespace.butler
243 for dataId in self.idList:
244 refList = list(butler.subset(datasetType=self.datasetType, level=self.level, dataId=dataId))
245 # exclude nonexistent data
246 # this is a recursive test, e.g. for the sake of "raw" data
247 if not refList:
248 namespace.log.warn("No data found for dataId=%s", dataId)
249 continue
250 self.refList += refList
253class BrighterFatterKernel:
254 """A simple class to hold the kernel(s) generated and the intermediate
255 data products.
257 kernel.ampwiseKernels are the kernels for each amplifier in the detector,
258 as generated by having LEVEL == 'AMP'
260 kernel.detectorKernel is the kernel generated for the detector as a whole,
261 as generated by having LEVEL == 'DETECTOR'
263 kernel.detectorKernelFromAmpKernels is the kernel for the detector,
264 generated by averaging together the amps in the detector
266 The originalLevel is the level for which the kernel(s) were generated,
267 i.e. the level at which the task was originally run.
268 """
270 def __init__(self, originalLevel, **kwargs):
271 self.__dict__["originalLevel"] = originalLevel
272 self.__dict__["ampwiseKernels"] = {}
273 self.__dict__["detectorKernel"] = {}
274 self.__dict__["detectorKernelFromAmpKernels"] = {}
275 self.__dict__["means"] = []
276 self.__dict__["rawMeans"] = []
277 self.__dict__["rawXcorrs"] = []
278 self.__dict__["xCorrs"] = []
279 self.__dict__["meanXcorrs"] = []
280 self.__dict__["gain"] = None # will be a dict keyed by amp if set
281 self.__dict__["gainErr"] = None # will be a dict keyed by amp if set
282 self.__dict__["noise"] = None # will be a dict keyed by amp if set
283 self.__dict__["noiseErr"] = None # will be a dict keyed by amp if set
285 for key, value in kwargs.items():
286 if hasattr(self, key):
287 setattr(self, key, value)
289 def __setattr__(self, attribute, value):
290 """Protect class attributes"""
291 if attribute not in self.__dict__:
292 print(f"Cannot set {attribute}")
293 else:
294 self.__dict__[attribute] = value
296 def replaceDetectorKernelWithAmpKernel(self, ampName, detectorName):
297 self.detectorKernel[detectorName] = self.ampwiseKernels[ampName]
299 def makeDetectorKernelFromAmpwiseKernels(self, detectorName, ampsToExclude=[], overwrite=False):
300 if detectorName not in self.detectorKernelFromAmpKernels.keys():
301 self.detectorKernelFromAmpKernels[detectorName] = {}
303 if self.detectorKernelFromAmpKernels[detectorName] != {} and overwrite is False:
304 raise RuntimeError('Was told to replace existing detector kernel with overwrite==False')
306 ampNames = self.ampwiseKernels.keys()
307 ampsToAverage = [amp for amp in ampNames if amp not in ampsToExclude]
308 avgKernel = np.zeros_like(self.ampwiseKernels[ampsToAverage[0]])
309 for ampName in ampsToAverage:
310 avgKernel += self.ampwiseKernels[ampName]
311 avgKernel /= len(ampsToAverage)
313 self.detectorKernelFromAmpKernels[detectorName] = avgKernel
316@dataclass
317class BrighterFatterGain:
318 """The gains and the results of the PTC fits."""
319 gains: dict
320 ptcResults: dict
323class MakeBrighterFatterKernelTask(pipeBase.CmdLineTask):
324 """Brighter-fatter effect correction-kernel calculation task.
326 A command line task for calculating the brighter-fatter correction
327 kernel from pairs of flat-field images (with the same exposure length).
329 The following operations are performed:
331 - The configurable isr task is called, which unpersists and assembles the
332 raw images, and performs the selected instrument signature removal tasks.
333 For the purpose of brighter-fatter coefficient calculation is it
334 essential that certain components of isr are *not* performed, and
335 recommended that certain others are. The task checks the selected isr
336 configuration before it is run, and if forbidden components have been
337 selected task will raise, and if recommended ones have not been selected,
338 warnings are logged.
340 - The gain of the each amplifier in the detector is calculated using
341 the photon transfer curve (PTC) method and used to correct the images
342 so that all calculations are done in units of electrons, and so that the
343 level across amplifier boundaries is continuous.
344 Outliers in the PTC are iteratively rejected
345 before fitting, with the nSigma rejection level set by
346 config.nSigmaClipRegression. Individual pixels are ignored in the input
347 images the image based on config.nSigmaClipGainCalc.
349 - Each image is then cross-correlated with the one it's paired with
350 (with the pairing defined by the --visit-pairs command line argument),
351 which is done either the whole-image to whole-image,
352 or amplifier-by-amplifier, depending on config.level.
354 - Once the cross-correlations have been calculated for each visit pair,
355 these are used to generate the correction kernel.
356 The maximum lag used, in pixels, and hence the size of the half-size
357 of the kernel generated, is given by config.maxLag,
358 i.e. a value of 10 will result in a kernel of size 2n-1 = 19x19 pixels.
359 Outlier values in these cross-correlations are rejected by using a
360 pixel-wise sigma-clipped thresholding to each cross-correlation in
361 the visit-pairs-length stack of cross-correlations.
362 The number of sigma clipped to is set by config.nSigmaClipKernelGen.
364 - Once DM-15277 has been completed, a method will exist to calculate the
365 empirical correction factor, config.biasCorr.
366 TODO: DM-15277 update this part of the docstring once the ticket is done.
367 """
369 RunnerClass = PairedVisitListTaskRunner
370 ConfigClass = MakeBrighterFatterKernelTaskConfig
371 _DefaultName = "makeBrighterFatterKernel"
373 def __init__(self, *args, **kwargs):
374 pipeBase.CmdLineTask.__init__(self, *args, **kwargs)
375 self.makeSubtask("isr")
377 self.debug = lsstDebug.Info(__name__)
378 if self.debug.enabled:
379 self.log.info("Running with debug enabled...")
380 # If we're displaying, test it works and save displays for later.
381 # It's worth testing here as displays are flaky and sometimes
382 # can't be contacted, and given processing takes a while,
383 # it's a shame to fail late due to display issues.
384 if self.debug.display:
385 try:
386 afwDisp.setDefaultBackend(self.debug.displayBackend)
387 afwDisp.Display.delAllDisplays()
388 self.disp1 = afwDisp.Display(0, open=True)
389 self.disp2 = afwDisp.Display(1, open=True)
391 im = afwImage.ImageF(1, 1)
392 im.array[:] = [[1]]
393 self.disp1.mtv(im)
394 self.disp1.erase()
395 except NameError:
396 self.debug.display = False
397 self.log.warn('Failed to setup/connect to display! Debug display has been disabled')
399 plt.interactive(False) # stop windows popping up when plotting. When headless, use 'agg' backend too
400 self.validateIsrConfig()
401 self.config.validate()
402 self.config.freeze()
404 @classmethod
405 def _makeArgumentParser(cls):
406 """Augment argument parser for the MakeBrighterFatterKernelTask."""
407 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
408 parser.add_argument("--visit-pairs", dest="visitPairs", nargs="*",
409 help="Visit pairs to use. Each pair must be of the form INT,INT e.g. 123,456")
410 parser.add_id_argument("--id", datasetType="brighterFatterKernel",
411 ContainerClass=BrighterFatterKernelTaskDataIdContainer,
412 help="The ccds to use, e.g. --id ccd=0..100")
413 return parser
415 def validateIsrConfig(self):
416 """Check that appropriate ISR settings are being used
417 for brighter-fatter kernel calculation."""
419 # How should we handle saturation/bad regions?
420 # 'doSaturationInterpolation': True
421 # 'doNanInterpAfterFlat': False
422 # 'doSaturation': True
423 # 'doSuspect': True
424 # 'doWidenSaturationTrails': True
425 # 'doSetBadRegions': True
427 configDict = self.config.isr.toDict()
429 for configParam in self.config.isrMandatorySteps:
430 if configDict[configParam] is False:
431 raise RuntimeError('Must set config.isr.%s to True '
432 'for brighter-fatter kernel calculation' % configParam)
434 for configParam in self.config.isrForbiddenSteps:
435 if configDict[configParam] is True:
436 raise RuntimeError('Must set config.isr.%s to False '
437 'for brighter-fatter kernel calculation' % configParam)
439 for configParam in self.config.isrDesirableSteps:
440 if configParam not in configDict:
441 self.log.info('Failed to find key %s in the isr config dict. You probably want ' +
442 'to set the equivalent for your obs_package to True.' % configParam)
443 continue
444 if configDict[configParam] is False:
445 self.log.warn('Found config.isr.%s set to False for brighter-fatter kernel calculation. '
446 'It is probably desirable to have this set to True' % configParam)
448 # subtask settings
449 if not self.config.isr.assembleCcd.doTrim:
450 raise RuntimeError('Must trim when assembling CCDs. Set config.isr.assembleCcd.doTrim to True')
452 @pipeBase.timeMethod
453 def runDataRef(self, dataRef, visitPairs):
454 """Run the brighter-fatter measurement task.
456 For a dataRef (which is each detector here),
457 and given a list of visit pairs, calculate the
458 brighter-fatter kernel for the detector.
460 Parameters
461 ----------
462 dataRef : `list` of `lsst.daf.persistence.ButlerDataRef`
463 dataRef for the detector for the visits to be fit.
464 visitPairs : `iterable` of `tuple` of `int`
465 Pairs of visit numbers to be processed together
466 """
467 np.random.seed(0) # used in the PTC fit bootstrap
469 # setup necessary objects
470 # NB: don't use dataRef.get('raw_detector')
471 # this currently doesn't work for composites because of the way
472 # composite objects (i.e. LSST images) are handled/constructed
473 # these need to be retrieved from the camera and dereferenced
474 # rather than accessed directly
475 detNum = dataRef.dataId[self.config.ccdKey]
476 detector = dataRef.get('camera')[dataRef.dataId[self.config.ccdKey]]
477 amps = detector.getAmplifiers()
478 ampNames = [amp.getName() for amp in amps]
480 if self.config.level == 'DETECTOR':
481 kernels = {detNum: []}
482 means = {detNum: []}
483 xcorrs = {detNum: []}
484 meanXcorrs = {detNum: []}
485 elif self.config.level == 'AMP':
486 kernels = {key: [] for key in ampNames}
487 means = {key: [] for key in ampNames}
488 xcorrs = {key: [] for key in ampNames}
489 meanXcorrs = {key: [] for key in ampNames}
490 else:
491 raise RuntimeError("Unsupported level: {}".format(self.config.level))
493 # we must be able to get the gains one way or the other, so check early
494 if not self.config.doCalcGains:
495 deleteMe = None
496 try:
497 deleteMe = dataRef.get('photonTransferCurveDataset')
498 except butlerExceptions.NoResults:
499 try:
500 deleteMe = dataRef.get('brighterFatterGain')
501 except butlerExceptions.NoResults:
502 pass
503 if not deleteMe:
504 raise RuntimeError("doCalcGains == False and gains could not be got from butler") from None
505 else:
506 del deleteMe
508 # if the level is DETECTOR we need to have the gains first so that each
509 # amp can be gain corrected in order to treat the detector as a single
510 # imaging area. However, if the level is AMP we can wait, calculate
511 # the correlations and correct for the gains afterwards
512 if self.config.level == 'DETECTOR':
513 if self.config.doCalcGains:
514 self.log.info('Computing gains for detector %s' % detNum)
515 gains, nomGains = self.estimateGains(dataRef, visitPairs)
516 dataRef.put(gains, datasetType='brighterFatterGain')
517 self.log.debug('Finished gain estimation for detector %s' % detNum)
518 else:
519 gains = dataRef.get('brighterFatterGain')
520 if not gains:
521 raise RuntimeError('Failed to retrieved gains for detector %s' % detNum)
522 self.log.info('Retrieved stored gain for detector %s' % detNum)
523 self.log.debug('Detector %s has gains %s' % (detNum, gains))
524 else: # we fake the gains as 1 for now, and correct later
525 gains = BrighterFatterGain({}, {})
526 for ampName in ampNames:
527 gains.gains[ampName] = 1.0
528 # We'll use the ptc.py code to calculate the gains, so we set this up
529 ptcConfig = MeasurePhotonTransferCurveTaskConfig()
530 ptcConfig.isrForbiddenSteps = []
531 ptcConfig.doFitBootstrap = True
532 ptcConfig.ptcFitType = 'POLYNOMIAL' # default Astier doesn't work for gain correction
533 ptcConfig.polynomialFitDegree = 3
534 ptcConfig.minMeanSignal = self.config.minMeanSignal
535 ptcConfig.maxMeanSignal = self.config.maxMeanSignal
536 ptcTask = MeasurePhotonTransferCurveTask(config=ptcConfig)
537 ptcDataset = PhotonTransferCurveDataset(ampNames)
539 # Loop over pairs of visits
540 # calculating the cross-correlations at the required level
541 for (v1, v2) in visitPairs:
542 dataRef.dataId['expId'] = v1
543 exp1 = self.isr.runDataRef(dataRef).exposure
544 dataRef.dataId['expId'] = v2
545 exp2 = self.isr.runDataRef(dataRef).exposure
546 del dataRef.dataId['expId']
547 checkExpLengthEqual(exp1, exp2, v1, v2, raiseWithMessage=True)
549 self.log.info('Preparing images for cross-correlation calculation for detector %s' % detNum)
550 # note the shape of these returns depends on level
551 _scaledMaskedIms1, _means1 = self._makeCroppedExposures(exp1, gains, self.config.level)
552 _scaledMaskedIms2, _means2 = self._makeCroppedExposures(exp2, gains, self.config.level)
554 # Compute the cross-correlation and means
555 # at the appropriate config.level:
556 # - "DETECTOR": one key, so compare the two visits to each other
557 # - "AMP": n_amp keys, comparing each amplifier of one visit
558 # to the same amplifier in the visit its paired with
559 for det_object in _scaledMaskedIms1.keys(): # det_object is ampName or detName depending on level
560 self.log.debug("Calculating correlations for %s" % det_object)
561 _xcorr, _mean = self._crossCorrelate(_scaledMaskedIms1[det_object],
562 _scaledMaskedIms2[det_object])
563 xcorrs[det_object].append(_xcorr)
564 means[det_object].append([_means1[det_object], _means2[det_object]])
565 if self.config.level != 'DETECTOR':
566 # Populate the ptcDataset for running fitting in the PTC task
567 expTime = exp1.getInfo().getVisitInfo().getExposureTime()
568 ptcDataset.rawExpTimes[det_object].append(expTime)
569 ptcDataset.rawMeans[det_object].append((_means1[det_object] + _means2[det_object]) / 2.0)
570 ptcDataset.rawVars[det_object].append(_xcorr[0, 0] / 2.0)
572 # TODO: DM-15305 improve debug functionality here.
573 # This is position 1 for the removed code.
575 # Save the raw means and xcorrs so we can look at them before any modifications
576 rawMeans = copy.deepcopy(means)
577 rawXcorrs = copy.deepcopy(xcorrs)
579 # gains are always and only pre-applied for DETECTOR
580 # so for all other levels we now calculate them from the correlations
581 # and apply them
582 if self.config.level != 'DETECTOR':
583 if self.config.doCalcGains: # Run the PTC task for calculating the gains, put results
584 self.log.info('Calculating gains for detector %s using PTC task' % detNum)
585 ptcDataset = ptcTask.fitPtc(ptcDataset, ptcConfig.ptcFitType)
586 dataRef.put(ptcDataset, datasetType='photonTransferCurveDataset')
587 self.log.debug('Finished gain estimation for detector %s' % detNum)
588 else: # load results - confirmed to work much earlier on, so can be relied upon here
589 ptcDataset = dataRef.get('photonTransferCurveDataset')
591 self._applyGains(means, xcorrs, ptcDataset)
593 if self.config.doPlotPtcs:
594 dirname = dataRef.getUri(datasetType='cpPipePlotRoot', write=True)
595 if not os.path.exists(dirname):
596 os.makedirs(dirname)
597 detNum = dataRef.dataId[self.config.ccdKey]
598 filename = f"PTC_det{detNum}.pdf"
599 filenameFull = os.path.join(dirname, filename)
600 with PdfPages(filenameFull) as pdfPages:
601 ptcTask._plotPtc(ptcDataset, ptcConfig.ptcFitType, pdfPages)
603 # having calculated and applied the gains for all code-paths we can now
604 # generate the kernel(s)
605 self.log.info('Generating kernel(s) for %s' % detNum)
606 for det_object in xcorrs.keys(): # looping over either detectors or amps
607 if self.config.level == 'DETECTOR':
608 objId = 'detector %s' % det_object
609 elif self.config.level == 'AMP':
610 objId = 'detector %s AMP %s' % (detNum, det_object)
612 try:
613 meanXcorr, kernel = self.generateKernel(xcorrs[det_object], means[det_object], objId)
614 kernels[det_object] = kernel
615 meanXcorrs[det_object] = meanXcorr
616 except RuntimeError:
617 # bad amps will cause failures here which we want to ignore
618 self.log.warn('RuntimeError during kernel generation for %s' % objId)
619 continue
621 bfKernel = BrighterFatterKernel(self.config.level)
622 bfKernel.means = means
623 bfKernel.rawMeans = rawMeans
624 bfKernel.rawXcorrs = rawXcorrs
625 bfKernel.xCorrs = xcorrs
626 bfKernel.meanXcorrs = meanXcorrs
627 bfKernel.originalLevel = self.config.level
628 try:
629 bfKernel.gain = ptcDataset.gain
630 bfKernel.gainErr = ptcDataset.gainErr
631 bfKernel.noise = ptcDataset.noise
632 bfKernel.noiseErr = ptcDataset.noiseErr
633 except NameError: # we don't have a ptcDataset to store results from
634 pass
636 if self.config.level == 'AMP':
637 bfKernel.ampwiseKernels = kernels
638 ex = self.config.ignoreAmpsForAveraging
639 bfKernel.detectorKernel = bfKernel.makeDetectorKernelFromAmpwiseKernels(detNum, ampsToExclude=ex)
641 elif self.config.level == 'DETECTOR':
642 bfKernel.detectorKernel = kernels
643 else:
644 raise RuntimeError('Invalid level for kernel calculation; this should not be possible.')
646 dataRef.put(bfKernel)
648 self.log.info('Finished generating kernel(s) for %s' % detNum)
649 return pipeBase.Struct(exitStatus=0)
651 def _applyGains(self, means, xcorrs, ptcData):
652 """Apply the gains calculated by the PtcTask.
654 It also removes datapoints that were thrown out in the PTC algorithm.
656 Parameters
657 ----------
658 means : `dict` [`str`, `list` of `tuple`]
659 Dictionary, keyed by ampName, containing a list of the means for
660 each visit pair.
662 xcorrs : `dict` [`str`, `list` of `np.array`]
663 Dictionary, keyed by ampName, containing a list of the
664 cross-correlations for each visit pair.
666 ptcDataset : `lsst.cp.pipe.ptc.PhotonTransferCurveDataset`
667 The results of running the ptcTask.
668 """
669 ampNames = means.keys()
670 assert set(xcorrs.keys()) == set(ampNames) == set(ptcData.ampNames)
672 for ampName in ampNames:
673 mask = ptcData.expIdMask[ampName]
674 gain = ptcData.gain[ampName]
676 fitType = ptcData.ptcFitType[ampName]
677 if fitType != 'POLYNOMIAL':
678 raise RuntimeError(f"Only polynomial fit types supported currently, found {fitType}")
679 ptcFitPars = ptcData.ptcFitPars[ampName]
680 # polynomial params go in ascending order, so this is safe w.r.t.
681 # the polynomial order, as the constant term is always first,
682 # the linear term second etc
684 # Adjust xcorrs[0,0] to remove the linear gain part, leaving just the second order part
685 for i in range(len(means[ampName])):
686 ampMean = np.mean(means[ampName][i])
687 xcorrs[ampName][i][0, 0] -= 2.0 * (ampMean * ptcFitPars[1] + ptcFitPars[0])
689 # Now adjust the means and xcorrs for the calculated gain and remove the bad indices
690 means[ampName] = [[value*gain for value in pair] for pair in np.array(means[ampName])[mask]]
691 xcorrs[ampName] = [arr*gain*gain for arr in np.array(xcorrs[ampName])[mask]]
692 return
694 def _makeCroppedExposures(self, exp, gains, level):
695 """Prepare exposure for cross-correlation calculation.
697 For each amp, crop by the border amount, specified by
698 config.nPixBorderXCorr, then rescale by the gain
699 and subtract the sigma-clipped mean.
700 If the level is 'DETECTOR' then this is done
701 to the whole image so that it can be cross-correlated, with a copy
702 being returned.
703 If the level is 'AMP' then this is done per-amplifier,
704 and a copy of each prepared amp-image returned.
706 Parameters:
707 -----------
708 exp : `lsst.afw.image.exposure.ExposureF`
709 The exposure to prepare
710 gains : `lsst.cp.pipe.makeBrighterFatterKernel.BrighterFatterGain`
711 The object holding the amplifier gains, essentially a
712 dictionary of the amplifier gain values, keyed by amplifier name
713 level : `str`
714 Either `AMP` or `DETECTOR`
716 Returns:
717 --------
718 scaledMaskedIms : `dict` [`str`, `lsst.afw.image.maskedImage.MaskedImageF`]
719 Depending on level, this is either one item, or n_amp items,
720 keyed by detectorId or ampName
722 Notes:
723 ------
724 This function is controlled by the following config parameters:
725 nPixBorderXCorr : `int`
726 The number of border pixels to exclude
727 nSigmaClipXCorr : `float`
728 The number of sigma to be clipped to
729 """
730 assert(isinstance(exp, afwImage.ExposureF))
732 local_exp = exp.clone() # we don't want to modify the image passed in
733 del exp # ensure we don't make mistakes!
735 border = self.config.nPixBorderXCorr
736 sigma = self.config.nSigmaClipXCorr
738 sctrl = afwMath.StatisticsControl()
739 sctrl.setNumSigmaClip(sigma)
741 means = {}
742 returnAreas = {}
744 detector = local_exp.getDetector()
745 amps = detector.getAmplifiers()
747 mi = local_exp.getMaskedImage() # makeStatistics does not seem to take exposures
748 temp = mi.clone()
750 # Rescale each amp by the appropriate gain and subtract the mean.
751 # NB these are views modifying the image in-place
752 for amp in amps:
753 ampName = amp.getName()
754 rescaleIm = mi[amp.getBBox()] # the soon-to-be scaled, mean subtracted, amp image
755 rescaleTemp = temp[amp.getBBox()]
756 mean = afwMath.makeStatistics(rescaleIm, afwMath.MEANCLIP, sctrl).getValue()
757 gain = gains.gains[ampName]
758 rescaleIm *= gain
759 rescaleTemp *= gain
760 self.log.debug("mean*gain = %s, clipped mean = %s" %
761 (mean*gain, afwMath.makeStatistics(rescaleIm, afwMath.MEANCLIP,
762 sctrl).getValue()))
763 rescaleIm -= mean*gain
765 if level == 'AMP': # build the dicts if doing amp-wise
766 means[ampName] = afwMath.makeStatistics(rescaleTemp[border: -border, border: -border,
767 afwImage.LOCAL], afwMath.MEANCLIP, sctrl).getValue()
768 returnAreas[ampName] = rescaleIm
770 if level == 'DETECTOR': # else just average the whole detector
771 detName = local_exp.getDetector().getId()
772 means[detName] = afwMath.makeStatistics(temp[border: -border, border: -border, afwImage.LOCAL],
773 afwMath.MEANCLIP, sctrl).getValue()
774 returnAreas[detName] = rescaleIm
776 return returnAreas, means
778 def _crossCorrelate(self, maskedIm0, maskedIm1, runningBiasCorrSim=False, frameId=None, detId=None):
779 """Calculate the cross-correlation of an area.
781 If the area in question contains multiple amplifiers then they must
782 have been gain corrected.
784 Parameters:
785 -----------
786 maskedIm0 : `lsst.afw.image.MaskedImageF`
787 The first image area
788 maskedIm1 : `lsst.afw.image.MaskedImageF`
789 The first image area
790 frameId : `str`, optional
791 The frame identifier for use in the filename
792 if writing debug outputs.
793 detId : `str`, optional
794 The detector identifier (detector, or detector+amp,
795 depending on config.level) for use in the filename
796 if writing debug outputs.
797 runningBiasCorrSim : `bool`
798 Set to true when using this function to calculate the amount of bias
799 introduced by the sigma clipping. If False, the biasCorr parameter
800 is divided by to remove the bias, but this is, of course, not
801 appropriate when this is the parameter being measured.
803 Returns:
804 --------
805 xcorr : `np.ndarray`
806 The quarter-image cross-correlation
807 mean : `float`
808 The sum of the means of the input images,
809 sigma-clipped, and with borders applied.
810 This is used when using this function with simulations to calculate
811 the biasCorr parameter.
813 Notes:
814 ------
815 This function is controlled by the following config parameters:
816 maxLag : `int`
817 The maximum lag to use in the cross-correlation calculation
818 nPixBorderXCorr : `int`
819 The number of border pixels to exclude
820 nSigmaClipXCorr : `float`
821 The number of sigma to be clipped to
822 biasCorr : `float`
823 Parameter used to correct from the bias introduced
824 by the sigma cuts.
825 """
826 maxLag = self.config.maxLag
827 border = self.config.nPixBorderXCorr
828 sigma = self.config.nSigmaClipXCorr
829 biasCorr = self.config.biasCorr
831 sctrl = afwMath.StatisticsControl()
832 sctrl.setNumSigmaClip(sigma)
834 mean = afwMath.makeStatistics(maskedIm0.getImage()[border: -border, border: -border, afwImage.LOCAL],
835 afwMath.MEANCLIP, sctrl).getValue()
836 mean += afwMath.makeStatistics(maskedIm1.getImage()[border: -border, border: -border, afwImage.LOCAL],
837 afwMath.MEANCLIP, sctrl).getValue()
839 # Diff the images, and apply border
840 diff = maskedIm0.clone()
841 diff -= maskedIm1.getImage()
842 diff = diff[border: -border, border: -border, afwImage.LOCAL]
844 if self.debug.writeDiffImages:
845 filename = '_'.join(['diff', 'detector', detId, frameId, '.fits'])
846 diff.writeFits(os.path.join(self.debug.debugDataPath, filename))
848 # Subtract background. It should be a constant, but it isn't always
849 binsize = self.config.backgroundBinSize
850 nx = diff.getWidth()//binsize
851 ny = diff.getHeight()//binsize
852 bctrl = afwMath.BackgroundControl(nx, ny, sctrl, afwMath.MEANCLIP)
853 bkgd = afwMath.makeBackground(diff, bctrl)
854 bgImg = bkgd.getImageF(afwMath.Interpolate.CUBIC_SPLINE, afwMath.REDUCE_INTERP_ORDER)
855 bgMean = np.mean(bgImg.getArray())
856 if abs(bgMean) >= self.config.backgroundWarnLevel:
857 self.log.warn('Mean of background = %s > config.maxBackground' % bgMean)
859 diff -= bgImg
861 if self.debug.writeDiffImages:
862 filename = '_'.join(['bgSub', 'diff', 'detector', detId, frameId, '.fits'])
863 diff.writeFits(os.path.join(self.debug.debugDataPath, filename))
864 if self.debug.display:
865 self.disp1.mtv(diff, title=frameId)
867 self.log.debug("Median and variance of diff:")
868 self.log.debug("%s" % afwMath.makeStatistics(diff, afwMath.MEDIAN, sctrl).getValue())
869 self.log.debug("%s, %s" % (afwMath.makeStatistics(diff, afwMath.VARIANCECLIP, sctrl).getValue(),
870 np.var(diff.getImage().getArray())))
872 # Measure the correlations
873 dim0 = diff[0: -maxLag, : -maxLag, afwImage.LOCAL]
874 dim0 -= afwMath.makeStatistics(dim0, afwMath.MEANCLIP, sctrl).getValue()
875 width, height = dim0.getDimensions()
876 xcorr = np.zeros((maxLag + 1, maxLag + 1), dtype=np.float64)
878 for xlag in range(maxLag + 1):
879 for ylag in range(maxLag + 1):
880 dim_xy = diff[xlag:xlag + width, ylag: ylag + height, afwImage.LOCAL].clone()
881 dim_xy -= afwMath.makeStatistics(dim_xy, afwMath.MEANCLIP, sctrl).getValue()
882 dim_xy *= dim0
883 xcorr[xlag, ylag] = afwMath.makeStatistics(dim_xy, afwMath.MEANCLIP, sctrl).getValue()
884 if not runningBiasCorrSim:
885 xcorr[xlag, ylag] /= biasCorr
887 # TODO: DM-15305 improve debug functionality here.
888 # This is position 2 for the removed code.
890 return xcorr, mean
892 def estimateGains(self, dataRef, visitPairs):
893 """Estimate the amplifier gains using the specified visits.
895 Given a dataRef and list of flats of varying intensity,
896 calculate the gain for each amplifier in the detector
897 using the photon transfer curve (PTC) method.
899 The config.fixPtcThroughOrigin option determines whether the iterative
900 fitting is forced to go through the origin or not.
901 This defaults to True, fitting var=1/gain * mean.
902 If set to False then var=1/g * mean + const is fitted.
904 This is really a photo transfer curve (PTC) gain measurement task.
905 See DM-14063 for results from of a comparison between
906 this task's numbers and the gain values in the HSC camera model,
907 and those measured by the PTC task in eotest.
909 Parameters
910 ----------
911 dataRef : `lsst.daf.persistence.butler.Butler.dataRef`
912 dataRef for the detector for the flats to be used
913 visitPairs : `list` of `tuple`
914 List of visit-pairs to use, as [(v1,v2), (v3,v4)...]
916 Returns
917 -------
918 gains : `lsst.cp.pipe.makeBrighterFatterKernel.BrighterFatterGain`
919 Object holding the per-amplifier gains, essentially a
920 dict of the as-calculated amplifier gain values, keyed by amp name
921 nominalGains : `dict` [`str`, `float`]
922 Dict of the amplifier gains, as reported by the `detector` object,
923 keyed by amplifier name
924 """
925 # NB: don't use dataRef.get('raw_detector') due to composites
926 detector = dataRef.get('camera')[dataRef.dataId[self.config.ccdKey]]
927 amps = detector.getAmplifiers()
928 ampNames = [amp.getName() for amp in amps]
930 ampMeans = {key: [] for key in ampNames} # these get turned into np.arrays later
931 ampCoVariances = {key: [] for key in ampNames}
932 ampVariances = {key: [] for key in ampNames}
934 # Loop over the amps in the detector,
935 # calculating a PTC for each amplifier.
936 # The amplifier iteration is performed in _calcMeansAndVars()
937 # NB: no gain correction is applied
938 for visPairNum, visPair in enumerate(visitPairs):
939 _means, _vars, _covars = self._calcMeansAndVars(dataRef, visPair[0], visPair[1])
941 # Do sanity checks; if these are failed more investigation is needed
942 breaker = 0
943 for amp in detector:
944 ampName = amp.getName()
945 if _means[ampName]*10 < _vars[ampName] or _means[ampName]*10 < _covars[ampName]:
946 msg = 'Sanity check failed; check visit pair %s amp %s' % (visPair, ampName)
947 self.log.warn(msg)
948 breaker += 1
949 if breaker:
950 continue
952 # having made sanity checks
953 # pull the values out into the respective dicts
954 for k in _means.keys(): # keys are necessarily the same
955 if _vars[k]*1.3 < _covars[k] or _vars[k]*0.7 > _covars[k]:
956 self.log.warn('Dropped a value')
957 continue
958 ampMeans[k].append(_means[k])
959 ampVariances[k].append(_vars[k])
960 ampCoVariances[k].append(_covars[k])
962 gains = {}
963 nomGains = {}
964 ptcResults = {}
965 for amp in detector:
966 ampName = amp.getName()
967 if ampMeans[ampName] == []: # all the data was dropped, amp is presumed bad
968 gains[ampName] = 1.0
969 ptcResults[ampName] = (0, 0, 1, 0)
970 continue
972 nomGains[ampName] = amp.getGain()
973 slopeRaw, interceptRaw, rVal, pVal, stdErr = \
974 stats.linregress(np.asarray(ampMeans[ampName]), np.asarray(ampCoVariances[ampName]))
975 slopeFix, _ = self._iterativeRegression(np.asarray(ampMeans[ampName]),
976 np.asarray(ampCoVariances[ampName]),
977 fixThroughOrigin=True)
978 slopeUnfix, intercept = self._iterativeRegression(np.asarray(ampMeans[ampName]),
979 np.asarray(ampCoVariances[ampName]),
980 fixThroughOrigin=False)
981 self.log.info("Slope of raw fit: %s, intercept: %s p value: %s" % (slopeRaw,
982 interceptRaw, pVal))
983 self.log.info("slope of fixed fit: %s, difference vs raw:%s" % (slopeFix,
984 slopeFix - slopeRaw))
985 self.log.info("slope of unfixed fit: %s, difference vs fix:%s" % (slopeUnfix,
986 slopeFix - slopeUnfix))
987 if self.config.fixPtcThroughOrigin:
988 slopeToUse = slopeFix
989 else:
990 slopeToUse = slopeUnfix
992 if self.debug.enabled:
993 fig = plt.figure()
994 ax = fig.add_subplot(111)
995 ax.plot(np.asarray(ampMeans[ampName]),
996 np.asarray(ampCoVariances[ampName]), linestyle='None', marker='x', label='data')
997 if self.config.fixPtcThroughOrigin:
998 ax.plot(np.asarray(ampMeans[ampName]),
999 np.asarray(ampMeans[ampName])*slopeToUse, label='Fit through origin')
1000 else:
1001 ax.plot(np.asarray(ampMeans[ampName]),
1002 np.asarray(ampMeans[ampName])*slopeToUse + intercept,
1003 label='Fit (intercept unconstrained')
1005 dataRef.put(fig, "plotBrighterFatterPtc", amp=ampName)
1006 self.log.info('Saved PTC for detector %s amp %s' % (detector.getId(), ampName))
1007 gains[ampName] = 1.0/slopeToUse
1008 # change the fit to use a cubic and match parameters with Lage method
1009 # or better, use the PTC task here too
1010 ptcResults[ampName] = (0, 0, 1, 0)
1012 return BrighterFatterGain(gains, ptcResults), nomGains
1014 def _calcMeansAndVars(self, dataRef, v1, v2):
1015 """Calculate the means, vars, covars, and retrieve the nominal gains,
1016 for each amp in each detector.
1018 This code runs using two visit numbers, and for the detector specified.
1019 It calculates the correlations in the individual amps without
1020 rescaling any gains. This allows a photon transfer curve
1021 to be generated and the gains measured.
1023 Images are assembled with use the isrTask, and basic isr is performed.
1025 Parameters:
1026 -----------
1027 dataRef : `lsst.daf.persistence.butler.Butler.dataRef`
1028 dataRef for the detector for the repo containing the flats to be used
1029 v1 : `int`
1030 First visit of the visit pair
1031 v2 : `int`
1032 Second visit of the visit pair
1034 Returns
1035 -------
1036 means, vars, covars : `tuple` of `dict`
1037 Three dicts, keyed by ampName,
1038 containing the sum of the image-means,
1039 the variance, and the quarter-image of the xcorr.
1040 """
1041 sigma = self.config.nSigmaClipGainCalc
1042 maxLag = self.config.maxLag
1043 border = self.config.nPixBorderGainCalc
1044 biasCorr = self.config.biasCorr
1046 # NB: don't use dataRef.get('raw_detector') due to composites
1047 detector = dataRef.get('camera')[dataRef.dataId[self.config.ccdKey]]
1049 ampMeans = {}
1051 # manipulate the dataId to get a postISR exposure for each visit
1052 # from the detector obj, restoring its original state afterwards
1053 originalDataId = dataRef.dataId.copy()
1054 dataRef.dataId['expId'] = v1
1055 exp1 = self.isr.runDataRef(dataRef).exposure
1056 dataRef.dataId['expId'] = v2
1057 exp2 = self.isr.runDataRef(dataRef).exposure
1058 dataRef.dataId = originalDataId
1059 exps = [exp1, exp2]
1060 checkExpLengthEqual(exp1, exp2, v1, v2, raiseWithMessage=True)
1062 detector = exps[0].getDetector()
1063 ims = [self._convertImagelikeToFloatImage(exp) for exp in exps]
1065 if self.debug.display:
1066 self.disp1.mtv(ims[0], title=str(v1))
1067 self.disp2.mtv(ims[1], title=str(v2))
1069 sctrl = afwMath.StatisticsControl()
1070 sctrl.setNumSigmaClip(sigma)
1071 for imNum, im in enumerate(ims):
1073 # calculate the sigma-clipped mean, excluding the borders
1074 # safest to apply borders to all amps regardless of edges
1075 # easier, camera-agnostic, and mitigates potentially dodgy
1076 # overscan-biases around edges as well
1077 for amp in detector:
1078 ampName = amp.getName()
1079 ampIm = im[amp.getBBox()]
1080 mean = afwMath.makeStatistics(ampIm[border: -border, border: -border, afwImage.LOCAL],
1081 afwMath.MEANCLIP, sctrl).getValue()
1082 if ampName not in ampMeans.keys():
1083 ampMeans[ampName] = []
1084 ampMeans[ampName].append(mean)
1085 ampIm -= mean
1087 diff = ims[0].clone()
1088 diff -= ims[1]
1090 temp = diff[border: -border, border: -border, afwImage.LOCAL]
1092 # Subtract background. It should be a constant,
1093 # but it isn't always (e.g. some SuprimeCam flats)
1094 # TODO: Check how this looks, and if this is the "right" way to do this
1095 binsize = self.config.backgroundBinSize
1096 nx = temp.getWidth()//binsize
1097 ny = temp.getHeight()//binsize
1098 bctrl = afwMath.BackgroundControl(nx, ny, sctrl, afwMath.MEANCLIP)
1099 bkgd = afwMath.makeBackground(temp, bctrl)
1101 box = diff.getBBox()
1102 box.grow(-border)
1103 diff[box, afwImage.LOCAL] -= bkgd.getImageF(afwMath.Interpolate.CUBIC_SPLINE,
1104 afwMath.REDUCE_INTERP_ORDER)
1106 variances = {}
1107 coVars = {}
1108 for amp in detector:
1109 ampName = amp.getName()
1110 diffAmpIm = diff[amp.getBBox()].clone()
1111 diffAmpImCrop = diffAmpIm[border: -border - maxLag, border: -border - maxLag, afwImage.LOCAL]
1112 diffAmpImCrop -= afwMath.makeStatistics(diffAmpImCrop, afwMath.MEANCLIP, sctrl).getValue()
1113 w, h = diffAmpImCrop.getDimensions()
1114 xcorr = np.zeros((maxLag + 1, maxLag + 1), dtype=np.float64)
1116 # calculate the cross-correlation
1117 for xlag in range(maxLag + 1):
1118 for ylag in range(maxLag + 1):
1119 dim_xy = diffAmpIm[border + xlag: border + xlag + w,
1120 border + ylag: border + ylag + h,
1121 afwImage.LOCAL].clone()
1122 dim_xy -= afwMath.makeStatistics(dim_xy, afwMath.MEANCLIP, sctrl).getValue()
1123 dim_xy *= diffAmpImCrop
1124 xcorr[xlag, ylag] = afwMath.makeStatistics(dim_xy,
1125 afwMath.MEANCLIP, sctrl).getValue()/(biasCorr)
1127 variances[ampName] = xcorr[0, 0]
1128 xcorr_full = self._tileArray(xcorr)
1129 coVars[ampName] = np.sum(xcorr_full)
1131 msg = "M1: " + str(ampMeans[ampName][0])
1132 msg += " M2 " + str(ampMeans[ampName][1])
1133 msg += " M_sum: " + str((ampMeans[ampName][0]) + ampMeans[ampName][1])
1134 msg += " Var " + str(variances[ampName])
1135 msg += " coVar: " + str(coVars[ampName])
1136 self.log.debug(msg)
1138 means = {}
1139 for amp in detector:
1140 ampName = amp.getName()
1141 means[ampName] = ampMeans[ampName][0] + ampMeans[ampName][1]
1143 return means, variances, coVars
1145 def _plotXcorr(self, xcorr, mean, zmax=0.05, title=None, fig=None, saveToFileName=None):
1146 """Plot the correlation functions."""
1147 try:
1148 xcorr = xcorr.getArray()
1149 except Exception:
1150 pass
1152 xcorr /= float(mean)
1153 # xcorr.getArray()[0,0]=abs(xcorr.getArray()[0,0]-1)
1155 if fig is None:
1156 fig = plt.figure()
1157 else:
1158 fig.clf()
1160 ax = fig.add_subplot(111, projection='3d')
1161 ax.azim = 30
1162 ax.elev = 20
1164 nx, ny = np.shape(xcorr)
1166 xpos, ypos = np.meshgrid(np.arange(nx), np.arange(ny))
1167 xpos = xpos.flatten()
1168 ypos = ypos.flatten()
1169 zpos = np.zeros(nx*ny)
1170 dz = xcorr.flatten()
1171 dz[dz > zmax] = zmax
1173 ax.bar3d(xpos, ypos, zpos, 1, 1, dz, color='b', zsort='max', sort_zpos=100)
1174 if xcorr[0, 0] > zmax:
1175 ax.bar3d([0], [0], [zmax], 1, 1, 1e-4, color='c')
1177 ax.set_xlabel("row")
1178 ax.set_ylabel("column")
1179 ax.set_zlabel(r"$\langle{(F_i - \bar{F})(F_i - \bar{F})}\rangle/\bar{F}$")
1181 if title:
1182 fig.suptitle(title)
1183 if saveToFileName:
1184 fig.savefig(saveToFileName)
1186 def _iterativeRegression(self, x, y, fixThroughOrigin=False, nSigmaClip=None, maxIter=None):
1187 """Use linear regression to fit a line, iteratively removing outliers.
1189 Useful when you have a sufficiently large numbers of points on your PTC.
1190 This function iterates until either there are no outliers of
1191 config.nSigmaClip magnitude, or until the specified maximum number
1192 of iterations has been performed.
1194 Parameters:
1195 -----------
1196 x : `numpy.array`
1197 The independent variable. Must be a numpy array, not a list.
1198 y : `numpy.array`
1199 The dependent variable. Must be a numpy array, not a list.
1200 fixThroughOrigin : `bool`, optional
1201 Whether to fix the PTC through the origin or allow an y-intercept.
1202 nSigmaClip : `float`, optional
1203 The number of sigma to clip to.
1204 Taken from the task config if not specified.
1205 maxIter : `int`, optional
1206 The maximum number of iterations allowed.
1207 Taken from the task config if not specified.
1209 Returns:
1210 --------
1211 slope : `float`
1212 The slope of the line of best fit
1213 intercept : `float`
1214 The y-intercept of the line of best fit
1215 """
1216 if not maxIter:
1217 maxIter = self.config.maxIterRegression
1218 if not nSigmaClip:
1219 nSigmaClip = self.config.nSigmaClipRegression
1221 nIter = 0
1222 sctrl = afwMath.StatisticsControl()
1223 sctrl.setNumSigmaClip(nSigmaClip)
1225 if fixThroughOrigin:
1226 while nIter < maxIter:
1227 nIter += 1
1228 self.log.debug("Origin fixed, iteration # %s using %s elements:" % (nIter, np.shape(x)[0]))
1229 TEST = x[:, np.newaxis]
1230 slope, _, _, _ = np.linalg.lstsq(TEST, y)
1231 slope = slope[0]
1232 res = (y - slope * x) / x
1233 resMean = afwMath.makeStatistics(res, afwMath.MEANCLIP, sctrl).getValue()
1234 resStd = np.sqrt(afwMath.makeStatistics(res, afwMath.VARIANCECLIP, sctrl).getValue())
1235 index = np.where((res > (resMean + nSigmaClip*resStd)) |
1236 (res < (resMean - nSigmaClip*resStd)))
1237 self.log.debug("%.3f %.3f %.3f %.3f" % (resMean, resStd, np.max(res), nSigmaClip))
1238 if np.shape(np.where(index))[1] == 0 or (nIter >= maxIter): # run out of points or iters
1239 break
1240 x = np.delete(x, index)
1241 y = np.delete(y, index)
1243 return slope, 0
1245 while nIter < maxIter:
1246 nIter += 1
1247 self.log.debug("Iteration # %s using %s elements:" % (nIter, np.shape(x)[0]))
1248 xx = np.vstack([x, np.ones(len(x))]).T
1249 ret, _, _, _ = np.linalg.lstsq(xx, y)
1250 slope, intercept = ret
1251 res = y - slope*x - intercept
1252 resMean = afwMath.makeStatistics(res, afwMath.MEANCLIP, sctrl).getValue()
1253 resStd = np.sqrt(afwMath.makeStatistics(res, afwMath.VARIANCECLIP, sctrl).getValue())
1254 index = np.where((res > (resMean + nSigmaClip * resStd)) | (res < resMean - nSigmaClip * resStd))
1255 self.log.debug("%.3f %.3f %.3f %.3f" % (resMean, resStd, np.max(res), nSigmaClip))
1256 if np.shape(np.where(index))[1] == 0 or (nIter >= maxIter): # run out of points, or iterations
1257 break
1258 x = np.delete(x, index)
1259 y = np.delete(y, index)
1261 return slope, intercept
1263 def generateKernel(self, corrs, means, objId, rejectLevel=None):
1264 """Generate the full kernel from a list of cross-correlations and means.
1266 Taking a list of quarter-image, gain-corrected cross-correlations,
1267 do a pixel-wise sigma-clipped mean of each,
1268 and tile into the full-sized kernel image.
1270 Each corr in corrs is one quarter of the full cross-correlation,
1271 and has been gain-corrected. Each mean in means is a tuple of the means
1272 of the two individual images, corresponding to that corr.
1274 Parameters:
1275 -----------
1276 corrs : `list` of `numpy.ndarray`, (Ny, Nx)
1277 A list of the quarter-image cross-correlations
1278 means : `list` of `tuples` of `floats`
1279 The means of the input images for each corr in corrs
1280 rejectLevel : `float`, optional
1281 This is essentially is a sanity check parameter.
1282 If this condition is violated there is something unexpected
1283 going on in the image, and it is discarded from the stack before
1284 the clipped-mean is calculated.
1285 If not provided then config.xcorrCheckRejectLevel is used
1287 Returns:
1288 --------
1289 kernel : `numpy.ndarray`, (Ny, Nx)
1290 The output kernel
1291 """
1292 self.log.info('Calculating kernel for %s'%objId)
1294 if not rejectLevel:
1295 rejectLevel = self.config.xcorrCheckRejectLevel
1297 if self.config.correlationQuadraticFit:
1298 xcorrList = []
1299 fluxList = []
1301 for corrNum, ((mean1, mean2), corr) in enumerate(zip(means, corrs)):
1302 msg = 'For item %s, initial corr[0,0] = %g, corr[1,0] = %g'%(corrNum, corr[0, 0], corr[1, 0])
1303 self.log.info(msg)
1304 if self.config.level == 'DETECTOR':
1305 # This is now done in _applyGains() but only if level is not DETECTOR
1306 corr[0, 0] -= (mean1 + mean2)
1307 fullCorr = self._tileArray(corr)
1309 # Craig Lage says he doesn't understand the negative sign, but it needs to be there
1310 xcorrList.append(-fullCorr / 2.0)
1311 flux = (mean1 + mean2) / 2.0
1312 fluxList.append(flux * flux)
1313 # We're using the linear fit algorithm to find a quadratic fit,
1314 # so we square the x-axis.
1315 # The step below does not need to be done, but is included
1316 # so that correlations can be compared
1317 # directly to existing code. We might want to take it out.
1318 corr /= -1.0*(mean1**2 + mean2**2)
1320 if not xcorrList:
1321 raise RuntimeError("Cannot generate kernel because all inputs were discarded. "
1322 "Either the data is bad, or config.xcorrCheckRejectLevel is too low")
1324 # This method fits a quadratic vs flux and keeps only the quadratic term.
1325 meanXcorr = np.zeros_like(fullCorr)
1326 xcorrList = np.asarray(xcorrList)
1328 for i in range(np.shape(meanXcorr)[0]):
1329 for j in range(np.shape(meanXcorr)[1]):
1330 # Note the i,j inversion. This serves the same function as the transpose step in
1331 # the base code. I don't understand why it is there, but I put it in to be consistent.
1332 slopeRaw, interceptRaw, rVal, pVal, stdErr = stats.linregress(np.asarray(fluxList),
1333 xcorrList[:, j, i])
1334 try:
1335 slope, intercept = self._iterativeRegression(np.asarray(fluxList),
1336 xcorrList[:, j, i],
1337 fixThroughOrigin=True)
1338 msg = "(%s,%s):Slope of raw fit: %s, intercept: %s p value: %s" % (i, j, slopeRaw,
1339 interceptRaw, pVal)
1340 self.log.debug(msg)
1341 self.log.debug("(%s,%s):Slope of fixed fit: %s" % (i, j, slope))
1343 meanXcorr[i, j] = slope
1344 except ValueError:
1345 meanXcorr[i, j] = slopeRaw
1347 msg = f"i={i}, j={j}, slope = {slope:.6g}, slopeRaw = {slopeRaw:.6g}"
1348 self.log.debug(msg)
1349 self.log.info('Quad Fit meanXcorr[0,0] = %g, meanXcorr[1,0] = %g'%(meanXcorr[8, 8],
1350 meanXcorr[9, 8]))
1352 else:
1353 # Try to average over a set of possible inputs.
1354 # This generates a simple function of the kernel that
1355 # should be constant across the images, and averages that.
1356 xcorrList = []
1357 sctrl = afwMath.StatisticsControl()
1358 sctrl.setNumSigmaClip(self.config.nSigmaClipKernelGen)
1360 for corrNum, ((mean1, mean2), corr) in enumerate(zip(means, corrs)):
1361 corr[0, 0] -= (mean1 + mean2)
1362 if corr[0, 0] > 0:
1363 self.log.warn('Skipped item %s due to unexpected value of (variance-mean)' % corrNum)
1364 continue
1365 corr /= -1.0*(mean1**2 + mean2**2)
1367 fullCorr = self._tileArray(corr)
1369 xcorrCheck = np.abs(np.sum(fullCorr))/np.sum(np.abs(fullCorr))
1370 if xcorrCheck > rejectLevel:
1371 self.log.warn("Sum of the xcorr is unexpectedly high. Investigate item num %s for %s. \n"
1372 "value = %s" % (corrNum, objId, xcorrCheck))
1373 continue
1374 xcorrList.append(fullCorr)
1376 if not xcorrList:
1377 raise RuntimeError("Cannot generate kernel because all inputs were discarded. "
1378 "Either the data is bad, or config.xcorrCheckRejectLevel is too low")
1380 # stack the individual xcorrs and apply a per-pixel clipped-mean
1381 meanXcorr = np.zeros_like(fullCorr)
1382 xcorrList = np.transpose(xcorrList)
1383 for i in range(np.shape(meanXcorr)[0]):
1384 for j in range(np.shape(meanXcorr)[1]):
1385 meanXcorr[i, j] = afwMath.makeStatistics(xcorrList[i, j],
1386 afwMath.MEANCLIP, sctrl).getValue()
1388 if self.config.correlationModelRadius < (meanXcorr.shape[0] - 1) / 2:
1389 sumToInfinity = self._buildCorrelationModel(meanXcorr, self.config.correlationModelRadius,
1390 self.config.correlationModelSlope)
1391 self.log.info("SumToInfinity = %s" % sumToInfinity)
1392 else:
1393 sumToInfinity = 0.0
1394 if self.config.forceZeroSum:
1395 self.log.info("Forcing sum of correlation matrix to zero")
1396 meanXcorr = self._forceZeroSum(meanXcorr, sumToInfinity)
1398 return meanXcorr, self.successiveOverRelax(meanXcorr)
1400 def successiveOverRelax(self, source, maxIter=None, eLevel=None):
1401 """An implementation of the successive over relaxation (SOR) method.
1403 A numerical method for solving a system of linear equations
1404 with faster convergence than the Gauss-Seidel method.
1406 Parameters:
1407 -----------
1408 source : `numpy.ndarray`
1409 The input array.
1410 maxIter : `int`, optional
1411 Maximum number of iterations to attempt before aborting.
1412 eLevel : `float`, optional
1413 The target error level at which we deem convergence to have
1414 occurred.
1416 Returns:
1417 --------
1418 output : `numpy.ndarray`
1419 The solution.
1420 """
1421 if not maxIter:
1422 maxIter = self.config.maxIterSuccessiveOverRelaxation
1423 if not eLevel:
1424 eLevel = self.config.eLevelSuccessiveOverRelaxation
1426 assert source.shape[0] == source.shape[1], "Input array must be square"
1427 # initialize, and set boundary conditions
1428 func = np.zeros([source.shape[0] + 2, source.shape[1] + 2])
1429 resid = np.zeros([source.shape[0] + 2, source.shape[1] + 2])
1430 rhoSpe = np.cos(np.pi/source.shape[0]) # Here a square grid is assumed
1432 # Calculate the initial error
1433 for i in range(1, func.shape[0] - 1):
1434 for j in range(1, func.shape[1] - 1):
1435 resid[i, j] = (func[i, j - 1] + func[i, j + 1] + func[i - 1, j] +
1436 func[i + 1, j] - 4*func[i, j] - source[i - 1, j - 1])
1437 inError = np.sum(np.abs(resid))
1439 # Iterate until convergence
1440 # We perform two sweeps per cycle,
1441 # updating 'odd' and 'even' points separately
1442 nIter = 0
1443 omega = 1.0
1444 dx = 1.0
1445 while nIter < maxIter*2:
1446 outError = 0
1447 if nIter%2 == 0:
1448 for i in range(1, func.shape[0] - 1, 2):
1449 for j in range(1, func.shape[1] - 1, 2):
1450 resid[i, j] = float(func[i, j-1] + func[i, j + 1] + func[i - 1, j] +
1451 func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
1452 func[i, j] += omega*resid[i, j]*.25
1453 for i in range(2, func.shape[0] - 1, 2):
1454 for j in range(2, func.shape[1] - 1, 2):
1455 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j] +
1456 func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
1457 func[i, j] += omega*resid[i, j]*.25
1458 else:
1459 for i in range(1, func.shape[0] - 1, 2):
1460 for j in range(2, func.shape[1] - 1, 2):
1461 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j] +
1462 func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
1463 func[i, j] += omega*resid[i, j]*.25
1464 for i in range(2, func.shape[0] - 1, 2):
1465 for j in range(1, func.shape[1] - 1, 2):
1466 resid[i, j] = float(func[i, j - 1] + func[i, j + 1] + func[i - 1, j] +
1467 func[i + 1, j] - 4.0*func[i, j] - dx*dx*source[i - 1, j - 1])
1468 func[i, j] += omega*resid[i, j]*.25
1469 outError = np.sum(np.abs(resid))
1470 if outError < inError*eLevel:
1471 break
1472 if nIter == 0:
1473 omega = 1.0/(1 - rhoSpe*rhoSpe/2.0)
1474 else:
1475 omega = 1.0/(1 - rhoSpe*rhoSpe*omega/4.0)
1476 nIter += 1
1478 if nIter >= maxIter*2:
1479 self.log.warn("Failure: SuccessiveOverRelaxation did not converge in %s iterations."
1480 "\noutError: %s, inError: %s," % (nIter//2, outError, inError*eLevel))
1481 else:
1482 self.log.info("Success: SuccessiveOverRelaxation converged in %s iterations."
1483 "\noutError: %s, inError: %s", nIter//2, outError, inError*eLevel)
1484 return func[1: -1, 1: -1]
1486 @staticmethod
1487 def _tileArray(in_array):
1488 """Given an input quarter-image, tile/mirror it and return full image.
1490 Given a square input of side-length n, of the form
1492 input = array([[1, 2, 3],
1493 [4, 5, 6],
1494 [7, 8, 9]])
1496 return an array of size 2n-1 as
1498 output = array([[ 9, 8, 7, 8, 9],
1499 [ 6, 5, 4, 5, 6],
1500 [ 3, 2, 1, 2, 3],
1501 [ 6, 5, 4, 5, 6],
1502 [ 9, 8, 7, 8, 9]])
1504 Parameters:
1505 -----------
1506 input : `np.array`
1507 The square input quarter-array
1509 Returns:
1510 --------
1511 output : `np.array`
1512 The full, tiled array
1513 """
1514 assert(in_array.shape[0] == in_array.shape[1])
1515 length = in_array.shape[0] - 1
1516 output = np.zeros((2*length + 1, 2*length + 1))
1518 for i in range(length + 1):
1519 for j in range(length + 1):
1520 output[i + length, j + length] = in_array[i, j]
1521 output[-i + length, j + length] = in_array[i, j]
1522 output[i + length, -j + length] = in_array[i, j]
1523 output[-i + length, -j + length] = in_array[i, j]
1524 return output
1526 @staticmethod
1527 def _forceZeroSum(inputArray, sumToInfinity):
1528 """Given an array of correlations, adjust the
1529 central value to force the sum of the array to be zero.
1531 Parameters:
1532 -----------
1533 input : `np.array`
1534 The square input array, assumed square and with
1535 shape (2n+1) x (2n+1)
1537 Returns:
1538 --------
1539 output : `np.array`
1540 The same array, with the value of the central value
1541 inputArray[n,n] adjusted to force the array sum to be zero.
1542 """
1543 assert(inputArray.shape[0] == inputArray.shape[1])
1544 assert(inputArray.shape[0] % 2 == 1)
1545 center = int((inputArray.shape[0] - 1) / 2)
1546 outputArray = np.copy(inputArray)
1547 outputArray[center, center] -= inputArray.sum() - sumToInfinity
1548 return outputArray
1550 @staticmethod
1551 def _buildCorrelationModel(array, replacementRadius, slope):
1552 """Given an array of correlations, build a model
1553 for correlations beyond replacementRadius pixels from the center
1554 and replace the measured values with the model.
1556 Parameters:
1557 -----------
1558 input : `np.array`
1559 The square input array, assumed square and with
1560 shape (2n+1) x (2n+1)
1562 Returns:
1563 --------
1564 output : `np.array`
1565 The same array, with the outer values
1566 replaced with a smoothed model.
1567 """
1568 assert(array.shape[0] == array.shape[1])
1569 assert(array.shape[0] % 2 == 1)
1570 assert(replacementRadius > 1)
1571 center = int((array.shape[0] - 1) / 2)
1572 # First we check if either the [0,1] or [1,0] correlation is positive.
1573 # If so, the data is seriously messed up. This has happened in some bad amplifiers.
1574 # In this case, we just return the input array unchanged.
1575 if (array[center, center + 1] >= 0.0) or (array[center + 1, center] >= 0.0):
1576 return 0.0
1578 intercept = (np.log10(-array[center, center + 1]) + np.log10(-array[center + 1, center])) / 2.0
1579 preFactor = 10**intercept
1580 slopeFactor = 2.0*abs(slope) - 2.0
1581 sumToInfinity = 2.0*np.pi*preFactor / (slopeFactor*(float(center)+0.5)**slopeFactor)
1582 # sum_to_ininity is the integral of the model beyond what is measured.
1583 # It is used to adjust C00 in the case of forcing zero sum
1585 # Now replace the pixels beyond replacementRadius with the model values
1586 for i in range(array.shape[0]):
1587 for j in range(array.shape[1]):
1588 r2 = float((i-center)*(i-center) + (j-center)*(j-center))
1589 if abs(i-center) < replacementRadius and abs(j-center) < replacementRadius:
1590 continue
1591 else:
1592 newCvalue = -preFactor * r2**slope
1593 array[i, j] = newCvalue
1594 return sumToInfinity
1596 @staticmethod
1597 def _convertImagelikeToFloatImage(imagelikeObject):
1598 """Turn an exposure or masked image of any type into an ImageF."""
1599 for attr in ("getMaskedImage", "getImage"):
1600 if hasattr(imagelikeObject, attr):
1601 imagelikeObject = getattr(imagelikeObject, attr)()
1602 try:
1603 floatImage = imagelikeObject.convertF()
1604 except AttributeError:
1605 raise RuntimeError("Failed to convert image to float")
1606 return floatImage
1609def calcBiasCorr(fluxLevels, imageShape, repeats=1, seed=0, addCorrelations=False,
1610 correlationStrength=0.1, maxLag=10, nSigmaClip=5, border=10, logger=None):
1611 """Calculate the bias induced when sigma-clipping non-Gaussian distributions.
1613 Fill image-pairs of the specified size with Poisson-distributed values,
1614 adding correlations as necessary. Then calculate the cross correlation,
1615 and calculate the bias induced using the cross-correlation image
1616 and the image means.
1618 Parameters:
1619 -----------
1620 fluxLevels : `list` of `int`
1621 The mean flux levels at which to simulate.
1622 Nominal values might be something like [70000, 90000, 110000]
1623 imageShape : `tuple` of `int`
1624 The shape of the image array to simulate, nx by ny pixels.
1625 repeats : `int`, optional
1626 Number of repeats to perform so that results
1627 can be averaged to improve SNR.
1628 seed : `int`, optional
1629 The random seed to use for the Poisson points.
1630 addCorrelations : `bool`, optional
1631 Whether to add brighter-fatter-like correlations to the simulated images
1632 If true, a correlation between x_{i,j} and x_{i+1,j+1} is introduced
1633 by adding a*x_{i,j} to x_{i+1,j+1}
1634 correlationStrength : `float`, optional
1635 The strength of the correlations.
1636 This is the value of the coefficient `a` in the above definition.
1637 maxLag : `int`, optional
1638 The maximum lag to work to in pixels
1639 nSigmaClip : `float`, optional
1640 Number of sigma to clip to when calculating the sigma-clipped mean.
1641 border : `int`, optional
1642 Number of border pixels to mask
1643 logger : `lsst.log.Log`, optional
1644 Logger to use. Instantiated anew if not provided.
1646 Returns:
1647 --------
1648 biases : `dict` [`float`, `list` of `float`]
1649 A dictionary, keyed by flux level, containing a list of the biases
1650 for each repeat at that flux level
1651 means : `dict` [`float`, `list` of `float`]
1652 A dictionary, keyed by flux level, containing a list of the average
1653 mean fluxes (average of the mean of the two images)
1654 for the image pairs at that flux level
1655 xcorrs : `dict` [`float`, `list` of `np.ndarray`]
1656 A dictionary, keyed by flux level, containing a list of the xcorr
1657 images for the image pairs at that flux level
1658 """
1659 if logger is None:
1660 logger = lsstLog.Log.getDefaultLogger()
1662 means = {f: [] for f in fluxLevels}
1663 xcorrs = {f: [] for f in fluxLevels}
1664 biases = {f: [] for f in fluxLevels}
1666 config = MakeBrighterFatterKernelTaskConfig()
1667 config.isrMandatorySteps = [] # no isr but the validation routine is still run
1668 config.isrForbiddenSteps = []
1669 config.nSigmaClipXCorr = nSigmaClip
1670 config.nPixBorderXCorr = border
1671 config.maxLag = maxLag
1672 task = MakeBrighterFatterKernelTask(config=config)
1674 im0 = afwImage.maskedImage.MaskedImageF(imageShape[1], imageShape[0])
1675 im1 = afwImage.maskedImage.MaskedImageF(imageShape[1], imageShape[0])
1677 random = np.random.RandomState(seed)
1679 for rep in range(repeats):
1680 for flux in fluxLevels:
1681 data0 = random.poisson(flux, (imageShape)).astype(float)
1682 data1 = random.poisson(flux, (imageShape)).astype(float)
1683 if addCorrelations:
1684 data0[1:, 1:] += correlationStrength*data0[: -1, : -1]
1685 data1[1:, 1:] += correlationStrength*data1[: -1, : -1]
1686 im0.image.array[:, :] = data0
1687 im1.image.array[:, :] = data1
1689 _xcorr, _means = task._crossCorrelate(im0, im1, runningBiasCorrSim=True)
1691 means[flux].append(_means)
1692 xcorrs[flux].append(_xcorr)
1693 if addCorrelations:
1694 bias = xcorrs[flux][-1][1, 1]/means[flux][-1]*(1 + correlationStrength)/correlationStrength
1695 msg = f"Simulated/expected avg. flux: {flux:.1f}, {(means[flux][-1]/2):.1f}"
1696 logger.info(msg)
1697 logger.info(f"Bias: {bias:.6f}")
1698 else:
1699 bias = xcorrs[flux][-1][0, 0]/means[flux][-1]
1700 msg = f"Simulated/expected avg. flux: {flux:.1f}, {(means[flux][-1]/2):.1f}"
1701 logger.info(msg)
1702 logger.info(f"Bias: {bias:.6f}")
1703 biases[flux].append(bias)
1705 return biases, means, xcorrs