Coverage for python/lsst/cp/pipe/deferredCharge.py: 14%
333 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-27 10:15 +0000
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22__all__ = ('CpCtiSolveConnections',
23 'CpCtiSolveConfig',
24 'CpCtiSolveTask',
25 'OverscanModel',
26 'SimpleModel',
27 'SimulatedModel',
28 'SegmentSimulator',
29 'FloatingOutputAmplifier',
30 )
32import copy
33import numpy as np
35import lsst.pipe.base as pipeBase
36import lsst.pipe.base.connectionTypes as cT
37import lsst.pex.config as pexConfig
39from lsst.ip.isr import DeferredChargeCalib, SerialTrap
40from lmfit import Minimizer, Parameters
42from ._lookupStaticCalibration import lookupStaticCalibration
45class CpCtiSolveConnections(pipeBase.PipelineTaskConnections,
46 dimensions=("instrument", "detector")):
47 inputMeasurements = cT.Input(
48 name="cpCtiMeas",
49 doc="Input overscan measurements to fit.",
50 storageClass='StructuredDataDict',
51 dimensions=("instrument", "exposure", "detector"),
52 multiple=True,
53 )
54 camera = cT.PrerequisiteInput(
55 name="camera",
56 doc="Camera geometry to use.",
57 storageClass="Camera",
58 dimensions=("instrument", ),
59 lookupFunction=lookupStaticCalibration,
60 isCalibration=True,
61 )
63 outputCalib = cT.Output(
64 name="cpCtiCalib",
65 doc="Output CTI calibration.",
66 storageClass="IsrCalib",
67 dimensions=("instrument", "detector"),
68 isCalibration=True,
69 )
72class CpCtiSolveConfig(pipeBase.PipelineTaskConfig,
73 pipelineConnections=CpCtiSolveConnections):
74 """Configuration for the CTI combination.
75 """
76 maxImageMean = pexConfig.Field(
77 dtype=float,
78 default=150000.0,
79 doc="Upper limit on acceptable image flux mean.",
80 )
81 localOffsetColumnRange = pexConfig.ListField(
82 dtype=int,
83 default=[3, 13],
84 doc="First and last overscan column to use for local offset effect.",
85 )
87 maxSignalForCti = pexConfig.Field(
88 dtype=float,
89 default=10000.0,
90 doc="Upper flux limit to use for CTI fit.",
91 )
92 globalCtiColumnRange = pexConfig.ListField(
93 dtype=int,
94 default=[1, 2],
95 doc="First and last overscan column to use for global CTI fit.",
96 )
98 trapColumnRange = pexConfig.ListField(
99 dtype=int,
100 default=[1, 20],
101 doc="First and last overscan column to use for serial trap fit.",
102 )
104 fitError = pexConfig.Field(
105 # This gives the error on the mean in a given column, and so
106 # is expected to be $RN / sqrt(N_rows)$.
107 dtype=float,
108 default=7.0/np.sqrt(2000),
109 doc="Error to use during parameter fitting.",
110 )
113class CpCtiSolveTask(pipeBase.PipelineTask):
114 """Combine CTI measurements to a final calibration.
116 This task uses the extended pixel edge response (EPER) method as
117 described by Snyder et al. 2021, Journal of Astronimcal
118 Telescopes, Instruments, and Systems, 7,
119 048002. doi:10.1117/1.JATIS.7.4.048002
120 """
122 ConfigClass = CpCtiSolveConfig
123 _DefaultName = 'cpCtiSolve'
125 def __init__(self, **kwargs):
126 super().__init__(**kwargs)
127 self.allowDebug = True
129 def runQuantum(self, butlerQC, inputRefs, outputRefs):
130 inputs = butlerQC.get(inputRefs)
132 dimensions = [exp.dataId.byName() for exp in inputRefs.inputMeasurements]
133 inputs['inputDims'] = dimensions
135 outputs = self.run(**inputs)
136 butlerQC.put(outputs, outputRefs)
138 def run(self, inputMeasurements, camera, inputDims):
139 """Solve for charge transfer inefficiency from overscan measurements.
141 Parameters
142 ----------
143 inputMeasurements : `list` [`dict`]
144 List of overscan measurements from each input exposure.
145 Each dictionary is nested within a top level 'CTI' key,
146 with measurements organized by amplifier name, containing
147 keys:
149 ``"FIRST_MEAN"``
150 Mean value of first image column (`float`).
151 ``"LAST_MEAN"``
152 Mean value of last image column (`float`).
153 ``"IMAGE_MEAN"``
154 Mean value of the entire image region (`float`).
155 ``"OVERSCAN_COLUMNS"``
156 List of overscan column indicies (`list` [`int`]).
157 ``"OVERSCAN_VALUES"``
158 List of overscan column means (`list` [`float`]).
159 camera : `lsst.afw.cameraGeom.Camera`
160 Camera geometry to use to find detectors.
161 inputDims : `list` [`dict`]
162 List of input dimensions from each input exposure.
164 Returns
165 -------
166 results : `lsst.pipe.base.Struct`
167 Result struct containing:
169 ``outputCalib``
170 Final CTI calibration data
171 (`lsst.ip.isr.DeferredChargeCalib`).
173 Raises
174 ------
175 RuntimeError
176 Raised if data from multiple detectors are passed in.
177 """
178 detectorSet = set([d['detector'] for d in inputDims])
179 if len(detectorSet) != 1:
180 raise RuntimeError("Inputs for too many detectors passed.")
181 detectorId = detectorSet.pop()
182 detector = camera[detectorId]
184 # Initialize with detector.
185 calib = DeferredChargeCalib(camera=camera, detector=detector)
187 localCalib = self.solveLocalOffsets(inputMeasurements, calib, detector)
189 globalCalib = self.solveGlobalCti(inputMeasurements, localCalib, detector)
191 finalCalib = self.findTraps(inputMeasurements, globalCalib, detector)
193 return pipeBase.Struct(
194 outputCalib=finalCalib,
195 )
197 def solveLocalOffsets(self, inputMeasurements, calib, detector):
198 """Solve for local (pixel-to-pixel) electronic offsets.
200 This method fits for \tau_L, the local electronic offset decay
201 time constant, and A_L, the local electronic offset constant
202 of proportionality.
204 Parameters
205 ----------
206 inputMeasurements : `list` [`dict`]
207 List of overscan measurements from each input exposure.
208 Each dictionary is nested within a top level 'CTI' key,
209 with measurements organized by amplifier name, containing
210 keys:
212 ``"FIRST_MEAN"``
213 Mean value of first image column (`float`).
214 ``"LAST_MEAN"``
215 Mean value of last image column (`float`).
216 ``"IMAGE_MEAN"``
217 Mean value of the entire image region (`float`).
218 ``"OVERSCAN_COLUMNS"``
219 List of overscan column indicies (`list` [`int`]).
220 ``"OVERSCAN_VALUES"``
221 List of overscan column means (`list` [`float`]).
222 calib : `lsst.ip.isr.DeferredChargeCalib`
223 Calibration to populate with values.
224 detector : `lsst.afw.cameraGeom.Detector`
225 Detector object containing the geometry information for
226 the amplifiers.
228 Returns
229 -------
230 calib : `lsst.ip.isr.DeferredChargeCalib`
231 Populated calibration.
233 Raises
234 ------
235 RuntimeError
236 Raised if no data remains after flux filtering.
238 Notes
239 -----
240 The original CTISIM code (https://github.com/Snyder005/ctisim)
241 uses a data model in which the "overscan" consists of the
242 standard serial overscan bbox with the values for the last
243 imaging data column prepended to that list. This version of
244 the code keeps the overscan and imaging sections separate, and
245 so a -1 offset is needed to ensure that the same columns are
246 used for fitting between this code and CTISIM. This offset
247 removes that last imaging data column from the count.
248 """
249 # Range to fit. These are in "camera" coordinates, and so
250 # need to have the count for last image column removed.
251 start, stop = self.config.localOffsetColumnRange
252 start -= 1
253 stop -= 1
255 # Loop over amps/inputs, fitting those columns from
256 # "non-saturated" inputs.
257 for amp in detector.getAmplifiers():
258 ampName = amp.getName()
260 # Number of serial shifts.
261 nCols = amp.getRawDataBBox().getWidth() + amp.getRawSerialPrescanBBox().getWidth()
263 # The signal is the mean intensity of each input, and the
264 # data are the overscan columns to fit. For detectors
265 # with non-zero CTI, the charge from the imaging region
266 # leaks into the overscan region.
267 signal = []
268 data = []
269 Nskipped = 0
270 for exposureEntry in inputMeasurements:
271 exposureDict = exposureEntry['CTI']
272 if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxImageMean:
273 signal.append(exposureDict[ampName]['IMAGE_MEAN'])
274 data.append(exposureDict[ampName]['OVERSCAN_VALUES'][start:stop+1])
275 else:
276 Nskipped += 1
277 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxImageMean}.")
278 if len(signal) == 0 or len(data) == 0:
279 raise RuntimeError("All exposures brighter than config.maxImageMean and excluded.")
281 signal = np.array(signal)
282 data = np.array(data)
284 ind = signal.argsort()
285 signal = signal[ind]
286 data = data[ind]
288 params = Parameters()
289 params.add('ctiexp', value=-6, min=-7, max=-5, vary=False)
290 params.add('trapsize', value=0.0, min=0.0, max=10., vary=False)
291 params.add('scaling', value=0.08, min=0.0, max=1.0, vary=False)
292 params.add('emissiontime', value=0.4, min=0.1, max=1.0, vary=False)
293 params.add('driftscale', value=0.00022, min=0., max=0.001, vary=True)
294 params.add('decaytime', value=2.4, min=0.1, max=4.0, vary=True)
296 model = SimpleModel()
297 minner = Minimizer(model.difference, params,
298 fcn_args=(signal, data, self.config.fitError, nCols),
299 fcn_kws={'start': start, 'stop': stop})
300 result = minner.minimize()
302 # Save results for the drift scale and decay time.
303 if not result.success:
304 self.log.warning("Electronics fitting failure for amplifier %s.", ampName)
306 calib.globalCti[ampName] = 10**result.params['ctiexp']
307 calib.driftScale[ampName] = result.params['driftscale'].value if result.success else 0.0
308 calib.decayTime[ampName] = result.params['decaytime'].value if result.success else 2.4
309 self.log.info("CTI Local Fit %s: cti: %g decayTime: %g driftScale %g",
310 ampName, calib.globalCti[ampName], calib.decayTime[ampName],
311 calib.driftScale[ampName])
312 return calib
314 def solveGlobalCti(self, inputMeasurements, calib, detector):
315 """Solve for global CTI constant.
317 This method solves for the mean global CTI, b.
319 Parameters
320 ----------
321 inputMeasurements : `list` [`dict`]
322 List of overscan measurements from each input exposure.
323 Each dictionary is nested within a top level 'CTI' key,
324 with measurements organized by amplifier name, containing
325 keys:
327 ``"FIRST_MEAN"``
328 Mean value of first image column (`float`).
329 ``"LAST_MEAN"``
330 Mean value of last image column (`float`).
331 ``"IMAGE_MEAN"``
332 Mean value of the entire image region (`float`).
333 ``"OVERSCAN_COLUMNS"``
334 List of overscan column indicies (`list` [`int`]).
335 ``"OVERSCAN_VALUES"``
336 List of overscan column means (`list` [`float`]).
337 calib : `lsst.ip.isr.DeferredChargeCalib`
338 Calibration to populate with values.
339 detector : `lsst.afw.cameraGeom.Detector`
340 Detector object containing the geometry information for
341 the amplifiers.
343 Returns
344 -------
345 calib : `lsst.ip.isr.DeferredChargeCalib`
346 Populated calibration.
348 Raises
349 ------
350 RuntimeError
351 Raised if no data remains after flux filtering.
353 Notes
354 -----
355 The original CTISIM code uses a data model in which the
356 "overscan" consists of the standard serial overscan bbox with
357 the values for the last imaging data column prepended to that
358 list. This version of the code keeps the overscan and imaging
359 sections separate, and so a -1 offset is needed to ensure that
360 the same columns are used for fitting between this code and
361 CTISIM. This offset removes that last imaging data column
362 from the count.
363 """
364 # Range to fit. These are in "camera" coordinates, and so
365 # need to have the count for last image column removed.
366 start, stop = self.config.globalCtiColumnRange
367 start -= 1
368 stop -= 1
370 # Loop over amps/inputs, fitting those columns from
371 # "non-saturated" inputs.
372 for amp in detector.getAmplifiers():
373 ampName = amp.getName()
375 # Number of serial shifts.
376 nCols = amp.getRawDataBBox().getWidth() + amp.getRawSerialPrescanBBox().getWidth()
378 # The signal is the mean intensity of each input, and the
379 # data are the overscan columns to fit. For detectors
380 # with non-zero CTI, the charge from the imaging region
381 # leaks into the overscan region.
382 signal = []
383 data = []
384 Nskipped = 0
385 for exposureEntry in inputMeasurements:
386 exposureDict = exposureEntry['CTI']
387 if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxSignalForCti:
388 signal.append(exposureDict[ampName]['IMAGE_MEAN'])
389 data.append(exposureDict[ampName]['OVERSCAN_VALUES'][start:stop+1])
390 else:
391 Nskipped += 1
392 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxSignalForCti}.")
393 if len(signal) == 0 or len(data) == 0:
394 raise RuntimeError("All exposures brighter than config.maxSignalForCti and excluded.")
396 signal = np.array(signal)
397 data = np.array(data)
399 ind = signal.argsort()
400 signal = signal[ind]
401 data = data[ind]
403 # CTI test. This looks at the charge that has leaked into
404 # the first few columns of the overscan.
405 overscan1 = data[:, 0]
406 overscan2 = data[:, 1]
407 test = (np.array(overscan1) + np.array(overscan2))/(nCols*np.array(signal))
408 testResult = np.median(test) > 5.E-6
409 self.log.info("Estimate of CTI test is %f for amp %s, %s.", np.median(test), ampName,
410 "full fitting will be performed" if testResult else
411 "only global CTI fitting will be performed")
413 self.debugView(ampName, signal, test)
415 params = Parameters()
416 params.add('ctiexp', value=-6, min=-7, max=-5, vary=True)
417 params.add('trapsize', value=5.0 if testResult else 0.0, min=0.0, max=30.,
418 vary=True if testResult else False)
419 params.add('scaling', value=0.08, min=0.0, max=1.0,
420 vary=True if testResult else False)
421 params.add('emissiontime', value=0.35, min=0.1, max=1.0,
422 vary=True if testResult else False)
423 params.add('driftscale', value=calib.driftScale[ampName], min=0., max=0.001, vary=False)
424 params.add('decaytime', value=calib.decayTime[ampName], min=0.1, max=4.0, vary=False)
426 model = SimulatedModel()
427 minner = Minimizer(model.difference, params,
428 fcn_args=(signal, data, self.config.fitError, nCols, amp),
429 fcn_kws={'start': start, 'stop': stop, 'trap_type': 'linear'})
430 result = minner.minimize()
432 # Only the global CTI term is retained from this fit.
433 calib.globalCti[ampName] = 10**result.params['ctiexp'].value
434 self.log.info("CTI Global Cti %s: cti: %g decayTime: %g driftScale %g",
435 ampName, calib.globalCti[ampName], calib.decayTime[ampName],
436 calib.driftScale[ampName])
438 return calib
440 def debugView(self, ampName, signal, test):
441 """Debug method for global CTI test value.
443 Parameters
444 ----------
445 ampName : `str`
446 Name of the amp for plot title.
447 signal : `list` [`float`]
448 Image means for the input exposures.
449 test : `list` [`float`]
450 CTI test value to plot.
451 """
452 import lsstDebug
453 if not lsstDebug.Info(__name__).display:
454 return
455 if not self.allowDebug:
456 return
458 import matplotlib.pyplot as plot
459 figure = plot.figure(1)
460 figure.clear()
461 plot.xscale('log', base=10.0)
462 plot.yscale('log', base=10.0)
463 plot.xlabel('Flat Field Signal [e-?]')
464 plot.ylabel('Serial CTI')
465 plot.title(ampName)
466 plot.plot(signal, test)
468 figure.show()
469 prompt = "Press Enter or c to continue [chp]..."
470 while True:
471 ans = input(prompt).lower()
472 if ans in ("", " ", "c",):
473 break
474 elif ans in ("p", ):
475 import pdb
476 pdb.set_trace()
477 elif ans in ('x', ):
478 self.allowDebug = False
479 break
480 elif ans in ("h", ):
481 print("[h]elp [c]ontinue [p]db e[x]itDebug")
482 plot.close()
484 def findTraps(self, inputMeasurements, calib, detector):
485 """Solve for serial trap parameters.
487 Parameters
488 ----------
489 inputMeasurements : `list` [`dict`]
490 List of overscan measurements from each input exposure.
491 Each dictionary is nested within a top level 'CTI' key,
492 with measurements organized by amplifier name, containing
493 keys:
495 ``"FIRST_MEAN"``
496 Mean value of first image column (`float`).
497 ``"LAST_MEAN"``
498 Mean value of last image column (`float`).
499 ``"IMAGE_MEAN"``
500 Mean value of the entire image region (`float`).
501 ``"OVERSCAN_COLUMNS"``
502 List of overscan column indicies (`list` [`int`]).
503 ``"OVERSCAN_VALUES"``
504 List of overscan column means (`list` [`float`]).
505 calib : `lsst.ip.isr.DeferredChargeCalib`
506 Calibration to populate with values.
507 detector : `lsst.afw.cameraGeom.Detector`
508 Detector object containing the geometry information for
509 the amplifiers.
511 Returns
512 -------
513 calib : `lsst.ip.isr.DeferredChargeCalib`
514 Populated calibration.
516 Raises
517 ------
518 RuntimeError
519 Raised if no data remains after flux filtering.
521 Notes
522 -----
523 The original CTISIM code uses a data model in which the
524 "overscan" consists of the standard serial overscan bbox with
525 the values for the last imaging data column prepended to that
526 list. This version of the code keeps the overscan and imaging
527 sections separate, and so a -1 offset is needed to ensure that
528 the same columns are used for fitting between this code and
529 CTISIM. This offset removes that last imaging data column
530 from the count.
531 """
532 # Range to fit. These are in "camera" coordinates, and so
533 # need to have the count for last image column removed.
534 start, stop = self.config.trapColumnRange
535 start -= 1
536 stop -= 1
538 # Loop over amps/inputs, fitting those columns from
539 # "non-saturated" inputs.
540 for amp in detector.getAmplifiers():
541 ampName = amp.getName()
543 # Number of serial shifts.
544 nCols = amp.getRawDataBBox().getWidth() + amp.getRawSerialPrescanBBox().getWidth()
546 # The signal is the mean intensity of each input, and the
547 # data are the overscan columns to fit. The new_signal is
548 # the mean in the last image column. Any serial trap will
549 # take charge from this column, and deposit it into the
550 # overscan columns.
551 signal = []
552 data = []
553 new_signal = []
554 Nskipped = 0
555 for exposureEntry in inputMeasurements:
556 exposureDict = exposureEntry['CTI']
557 if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxImageMean:
558 signal.append(exposureDict[ampName]['IMAGE_MEAN'])
559 data.append(exposureDict[ampName]['OVERSCAN_VALUES'][start:stop+1])
560 new_signal.append(exposureDict[ampName]['LAST_MEAN'])
561 else:
562 Nskipped += 1
563 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxImageMean}.")
564 if len(signal) == 0 or len(data) == 0:
565 raise RuntimeError("All exposures brighter than config.maxImageMean and excluded.")
567 signal = np.array(signal)
568 data = np.array(data)
569 new_signal = np.array(new_signal)
571 ind = signal.argsort()
572 signal = signal[ind]
573 data = data[ind]
574 new_signal = new_signal[ind]
576 # In the absense of any trap, the model results using the
577 # parameters already determined will match the observed
578 # overscan results.
579 params = Parameters()
580 params.add('ctiexp', value=np.log10(calib.globalCti[ampName]),
581 min=-7, max=-5, vary=False)
582 params.add('trapsize', value=0.0, min=0.0, max=10., vary=False)
583 params.add('scaling', value=0.08, min=0.0, max=1.0, vary=False)
584 params.add('emissiontime', value=0.35, min=0.1, max=1.0, vary=False)
585 params.add('driftscale', value=calib.driftScale[ampName],
586 min=0.0, max=0.001, vary=False)
587 params.add('decaytime', value=calib.decayTime[ampName],
588 min=0.1, max=4.0, vary=False)
590 model = SimpleModel.model_results(params, signal, nCols,
591 start=start, stop=stop)
593 # Evaluating trap: the difference between the model and
594 # observed data.
595 res = np.sum((data-model)[:, :3], axis=1)
597 # Create spline model for the trap, using the residual
598 # between data and model as a function of the last image
599 # column mean (new_signal) scaled by (1 - A_L).
600 new_signal = np.asarray((1 - calib.driftScale[ampName])*new_signal, dtype=np.float64)
601 x = new_signal
602 y = np.maximum(0, res)
604 # Pad left with ramp
605 y = np.pad(y, (10, 0), 'linear_ramp', end_values=(0, 0))
606 x = np.pad(x, (10, 0), 'linear_ramp', end_values=(0, 0))
608 # Pad right with constant
609 y = np.pad(y, (1, 1), 'constant', constant_values=(0, y[-1]))
610 x = np.pad(x, (1, 1), 'constant', constant_values=(-1, 200000.))
612 trap = SerialTrap(20000.0, 0.4, 1, 'spline', np.concatenate((x, y)).tolist())
613 calib.serialTraps[ampName] = trap
615 return calib
618class OverscanModel:
619 """Base class for handling model/data fit comparisons.
621 This handles all of the methods needed for the lmfit Minimizer to
622 run.
623 """
625 @staticmethod
626 def model_results(params, signal, num_transfers, start=1, stop=10):
627 """Generate a realization of the overscan model, using the specified
628 fit parameters and input signal.
630 Parameters
631 ----------
632 params : `lmfit.Parameters`
633 Object containing the model parameters.
634 signal : `np.ndarray`, (nMeasurements)
635 Array of image means.
636 num_transfers : `int`
637 Number of serial transfers that the charge undergoes.
638 start : `int`, optional
639 First overscan column to fit. This number includes the
640 last imaging column, and needs to be adjusted by one when
641 using the overscan bounding box.
642 stop : `int`, optional
643 Last overscan column to fit. This number includes the
644 last imaging column, and needs to be adjusted by one when
645 using the overscan bounding box.
647 Returns
648 -------
649 results : `np.ndarray`, (nMeasurements, nCols)
650 Model results.
651 """
652 raise NotImplementedError("Subclasses must implement the model calculation.")
654 def loglikelihood(self, params, signal, data, error, *args, **kwargs):
655 """Calculate log likelihood of the model.
657 Parameters
658 ----------
659 params : `lmfit.Parameters`
660 Object containing the model parameters.
661 signal : `np.ndarray`, (nMeasurements)
662 Array of image means.
663 data : `np.ndarray`, (nMeasurements, nCols)
664 Array of overscan column means from each measurement.
665 error : `float`
666 Fixed error value.
667 *args :
668 Additional position arguments.
669 **kwargs :
670 Additional keyword arguments.
672 Returns
673 -------
674 logL : `float`
675 The log-likelihood of the observed data given the model
676 parameters.
677 """
678 model_results = self.model_results(params, signal, *args, **kwargs)
680 inv_sigma2 = 1.0/(error**2.0)
681 diff = model_results - data
683 return -0.5*(np.sum(inv_sigma2*(diff)**2.))
685 def negative_loglikelihood(self, params, signal, data, error, *args, **kwargs):
686 """Calculate negative log likelihood of the model.
688 Parameters
689 ----------
690 params : `lmfit.Parameters`
691 Object containing the model parameters.
692 signal : `np.ndarray`, (nMeasurements)
693 Array of image means.
694 data : `np.ndarray`, (nMeasurements, nCols)
695 Array of overscan column means from each measurement.
696 error : `float`
697 Fixed error value.
698 *args :
699 Additional position arguments.
700 **kwargs :
701 Additional keyword arguments.
703 Returns
704 -------
705 negativelogL : `float`
706 The negative log-likelihood of the observed data given the
707 model parameters.
708 """
709 ll = self.loglikelihood(params, signal, data, error, *args, **kwargs)
711 return -ll
713 def rms_error(self, params, signal, data, error, *args, **kwargs):
714 """Calculate RMS error between model and data.
716 Parameters
717 ----------
718 params : `lmfit.Parameters`
719 Object containing the model parameters.
720 signal : `np.ndarray`, (nMeasurements)
721 Array of image means.
722 data : `np.ndarray`, (nMeasurements, nCols)
723 Array of overscan column means from each measurement.
724 error : `float`
725 Fixed error value.
726 *args :
727 Additional position arguments.
728 **kwargs :
729 Additional keyword arguments.
731 Returns
732 -------
733 rms : `float`
734 The rms error between the model and input data.
735 """
736 model_results = self.model_results(params, signal, *args, **kwargs)
738 diff = model_results - data
739 rms = np.sqrt(np.mean(np.square(diff)))
741 return rms
743 def difference(self, params, signal, data, error, *args, **kwargs):
744 """Calculate the flattened difference array between model and data.
746 Parameters
747 ----------
748 params : `lmfit.Parameters`
749 Object containing the model parameters.
750 signal : `np.ndarray`, (nMeasurements)
751 Array of image means.
752 data : `np.ndarray`, (nMeasurements, nCols)
753 Array of overscan column means from each measurement.
754 error : `float`
755 Fixed error value.
756 *args :
757 Additional position arguments.
758 **kwargs :
759 Additional keyword arguments.
761 Returns
762 -------
763 difference : `np.ndarray`, (nMeasurements*nCols)
764 The rms error between the model and input data.
765 """
766 model_results = self.model_results(params, signal, *args, **kwargs)
767 diff = (model_results-data).flatten()
769 return diff
772class SimpleModel(OverscanModel):
773 """Simple analytic overscan model."""
775 @staticmethod
776 def model_results(params, signal, num_transfers, start=1, stop=10):
777 """Generate a realization of the overscan model, using the specified
778 fit parameters and input signal.
780 Parameters
781 ----------
782 params : `lmfit.Parameters`
783 Object containing the model parameters.
784 signal : `np.ndarray`, (nMeasurements)
785 Array of image means.
786 num_transfers : `int`
787 Number of serial transfers that the charge undergoes.
788 start : `int`, optional
789 First overscan column to fit. This number includes the
790 last imaging column, and needs to be adjusted by one when
791 using the overscan bounding box.
792 stop : `int`, optional
793 Last overscan column to fit. This number includes the
794 last imaging column, and needs to be adjusted by one when
795 using the overscan bounding box.
797 Returns
798 -------
799 res : `np.ndarray`, (nMeasurements, nCols)
800 Model results.
801 """
802 v = params.valuesdict()
803 v['cti'] = 10**v['ctiexp']
805 # Adjust column numbering to match DM overscan bbox.
806 start += 1
807 stop += 1
809 x = np.arange(start, stop+1)
810 res = np.zeros((signal.shape[0], x.shape[0]))
812 for i, s in enumerate(signal):
813 # This is largely equivalent to equation 2. The minimum
814 # indicates that a trap cannot emit more charge than is
815 # available, nor can it emit more charge than it can hold.
816 # This scales the exponential release of charge from the
817 # trap. The next term defines the contribution from the
818 # global CTI at each pixel transfer, and the final term
819 # includes the contribution from local CTI effects.
820 res[i, :] = (np.minimum(v['trapsize'], s*v['scaling'])
821 * (np.exp(1/v['emissiontime']) - 1.0)
822 * np.exp(-x/v['emissiontime'])
823 + s*num_transfers*v['cti']**x
824 + v['driftscale']*s*np.exp(-x/float(v['decaytime'])))
826 return res
829class SimulatedModel(OverscanModel):
830 """Simulated overscan model."""
832 @staticmethod
833 def model_results(params, signal, num_transfers, amp, start=1, stop=10, trap_type=None):
834 """Generate a realization of the overscan model, using the specified
835 fit parameters and input signal.
837 Parameters
838 ----------
839 params : `lmfit.Parameters`
840 Object containing the model parameters.
841 signal : `np.ndarray`, (nMeasurements)
842 Array of image means.
843 num_transfers : `int`
844 Number of serial transfers that the charge undergoes.
845 amp : `lsst.afw.cameraGeom.Amplifier`
846 Amplifier to use for geometry information.
847 start : `int`, optional
848 First overscan column to fit. This number includes the
849 last imaging column, and needs to be adjusted by one when
850 using the overscan bounding box.
851 stop : `int`, optional
852 Last overscan column to fit. This number includes the
853 last imaging column, and needs to be adjusted by one when
854 using the overscan bounding box.
855 trap_type : `str`, optional
856 Type of trap model to use.
858 Returns
859 -------
860 results : `np.ndarray`, (nMeasurements, nCols)
861 Model results.
862 """
863 v = params.valuesdict()
865 # Adjust column numbering to match DM overscan bbox.
866 start += 1
867 stop += 1
869 # Electronics effect optimization
870 output_amplifier = FloatingOutputAmplifier(1.0, v['driftscale'], v['decaytime'])
872 # CTI optimization
873 v['cti'] = 10**v['ctiexp']
875 # Trap type for optimization
876 if trap_type is None:
877 trap = None
878 elif trap_type == 'linear':
879 trap = SerialTrap(v['trapsize'], v['emissiontime'], 1, 'linear',
880 [v['scaling']])
881 elif trap_type == 'logistic':
882 trap = SerialTrap(v['trapsize'], v['emissiontime'], 1, 'logistic',
883 [v['f0'], v['k']])
884 else:
885 raise ValueError('Trap type must be linear or logistic or None')
887 # Simulate ramp readout
888 imarr = np.zeros((signal.shape[0], amp.getRawDataBBox().getWidth()))
889 ramp = SegmentSimulator(imarr, amp.getRawSerialPrescanBBox().getWidth(), output_amplifier,
890 cti=v['cti'], traps=trap)
891 ramp.ramp_exp(signal)
892 model_results = ramp.readout(serial_overscan_width=amp.getRawSerialOverscanBBox().getWidth(),
893 parallel_overscan_width=0)
895 ncols = amp.getRawSerialPrescanBBox().getWidth() + amp.getRawDataBBox().getWidth()
897 return model_results[:, ncols+start-1:ncols+stop]
900class SegmentSimulator:
901 """Controls the creation of simulated segment images.
903 Parameters
904 ----------
905 imarr : `np.ndarray` (nx, ny)
906 Image data array.
907 prescan_width : `int`
908 Number of serial prescan columns.
909 output_amplifier : `lsst.cp.pipe.FloatingOutputAmplifier`
910 An object holding the gain, read noise, and global_offset.
911 cti : `float`
912 Global CTI value.
913 traps : `list` [`lsst.ip.isr.SerialTrap`]
914 Serial traps to simulate.
915 """
917 def __init__(self, imarr, prescan_width, output_amplifier, cti=0.0, traps=None):
918 # Image array geometry
919 self.prescan_width = prescan_width
920 self.ny, self.nx = imarr.shape
922 self.segarr = np.zeros((self.ny, self.nx+prescan_width))
923 self.segarr[:, prescan_width:] = imarr
925 # Serial readout information
926 self.output_amplifier = output_amplifier
927 if isinstance(cti, np.ndarray):
928 raise ValueError("cti must be single value, not an array.")
929 self.cti = cti
931 self.serial_traps = None
932 self.do_trapping = False
933 if traps is not None:
934 if not isinstance(traps, list):
935 traps = [traps]
936 for trap in traps:
937 self.add_trap(trap)
939 def add_trap(self, serial_trap):
940 """Add a trap to the serial register.
942 Parameters
943 ----------
944 serial_trap : `lsst.ip.isr.SerialTrap`
945 The trap to add.
946 """
947 try:
948 self.serial_traps.append(serial_trap)
949 except AttributeError:
950 self.serial_traps = [serial_trap]
951 self.do_trapping = True
953 def ramp_exp(self, signal_list):
954 """Simulate an image with varying flux illumination per row.
956 This method simulates a segment image where the signal level
957 increases along the horizontal direction, according to the
958 provided list of signal levels.
960 Parameters
961 ----------
962 signal_list : `list` [`float`]
963 List of signal levels.
965 Raises
966 ------
967 ValueError
968 Raised if the length of the signal list does not equal the
969 number of rows.
970 """
971 if len(signal_list) != self.ny:
972 raise ValueError("Signal list does not match row count.")
974 ramp = np.tile(signal_list, (self.nx, 1)).T
975 self.segarr[:, self.prescan_width:] += ramp
977 def readout(self, serial_overscan_width=10, parallel_overscan_width=0):
978 """Simulate serial readout of the segment image.
980 This method performs the serial readout of a segment image
981 given the appropriate SerialRegister object and the properties
982 of the ReadoutAmplifier. Additional arguments can be provided
983 to account for the number of desired overscan transfers. The
984 result is a simulated final segment image, in ADU.
986 Parameters
987 ----------
988 serial_overscan_width : `int`, optional
989 Number of serial overscan columns.
990 parallel_overscan_width : `int`, optional
991 Number of parallel overscan rows.
993 Returns
994 -------
995 result : `np.ndarray` (nx, ny)
996 Simulated image, including serial prescan, serial
997 overscan, and parallel overscan regions.
998 """
999 # Create output array
1000 iy = int(self.ny + parallel_overscan_width)
1001 ix = int(self.nx + self.prescan_width + serial_overscan_width)
1002 image = np.random.normal(loc=self.output_amplifier.global_offset,
1003 scale=self.output_amplifier.noise,
1004 size=(iy, ix))
1005 free_charge = copy.deepcopy(self.segarr)
1007 # Set flow control parameters
1008 do_trapping = self.do_trapping
1009 cti = self.cti
1011 offset = np.zeros(self.ny)
1012 cte = 1 - cti
1013 if do_trapping:
1014 for trap in self.serial_traps:
1015 trap.initialize(self.ny, self.nx, self.prescan_width)
1017 for i in range(ix):
1018 # Trap capture
1019 if do_trapping:
1020 for trap in self.serial_traps:
1021 captured_charge = trap.trap_charge(free_charge)
1022 free_charge -= captured_charge
1024 # Pixel-to-pixel proportional loss
1025 transferred_charge = free_charge*cte
1026 deferred_charge = free_charge*cti
1028 # Pixel transfer and readout
1029 offset = self.output_amplifier.local_offset(offset,
1030 transferred_charge[:, 0])
1031 image[:iy-parallel_overscan_width, i] += transferred_charge[:, 0] + offset
1033 free_charge = np.pad(transferred_charge, ((0, 0), (0, 1)),
1034 mode='constant')[:, 1:] + deferred_charge
1036 # Trap emission
1037 if do_trapping:
1038 for trap in self.serial_traps:
1039 released_charge = trap.release_charge()
1040 free_charge += released_charge
1042 return image/float(self.output_amplifier.gain)
1045class FloatingOutputAmplifier:
1046 """Object representing the readout amplifier of a single channel.
1048 Parameters
1049 ----------
1050 gain : `float`
1051 Amplifier gain.
1052 scale : `float`
1053 Drift scale for the amplifier.
1054 decay_time : `float`
1055 Decay time for the bias drift.
1056 noise : `float`, optional
1057 Amplifier read noise.
1058 offset : `float`, optional
1059 Global CTI offset.
1060 """
1062 def __init__(self, gain, scale, decay_time, noise=0.0, offset=0.0):
1064 self.gain = gain
1065 self.noise = noise
1066 self.global_offset = offset
1068 self.update_parameters(scale, decay_time)
1070 def local_offset(self, old, signal):
1071 """Calculate local offset hysteresis.
1073 Parameters
1074 ----------
1075 old : `np.ndarray`, (,)
1076 Previous iteration.
1077 signal : `np.ndarray`, (,)
1078 Current column measurements.
1080 Returns
1081 -------
1082 offset : `np.ndarray`
1083 Local offset.
1084 """
1085 new = self.scale*signal
1087 return np.maximum(new, old*np.exp(-1/self.decay_time))
1089 def update_parameters(self, scale, decay_time):
1090 """Update parameter values, if within acceptable values.
1092 Parameters
1093 ----------
1094 scale : `float`
1095 Drift scale for the amplifier.
1096 decay_time : `float`
1097 Decay time for the bias drift.
1099 Raises
1100 ------
1101 ValueError
1102 Raised if the input parameters are out of range.
1103 """
1104 if scale < 0.0:
1105 raise ValueError("Scale must be greater than or equal to 0.")
1106 if np.isnan(scale):
1107 raise ValueError("Scale must be real-valued number, not NaN.")
1108 self.scale = scale
1109 if decay_time <= 0.0:
1110 raise ValueError("Decay time must be greater than 0.")
1111 if np.isnan(decay_time):
1112 raise ValueError("Decay time must be real-valued number, not NaN.")
1113 self.decay_time = decay_time