Coverage for python/lsst/cp/pipe/deferredCharge.py: 14%
332 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-28 09:20 +0000
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22__all__ = ('CpCtiSolveConnections',
23 'CpCtiSolveConfig',
24 'CpCtiSolveTask',
25 'OverscanModel',
26 'SimpleModel',
27 'SimulatedModel',
28 'SegmentSimulator',
29 'FloatingOutputAmplifier',
30 )
32import copy
33import numpy as np
35import lsst.pipe.base as pipeBase
36import lsst.pipe.base.connectionTypes as cT
37import lsst.pex.config as pexConfig
39from lsst.ip.isr import DeferredChargeCalib, SerialTrap
40from lmfit import Minimizer, Parameters
43class CpCtiSolveConnections(pipeBase.PipelineTaskConnections,
44 dimensions=("instrument", "detector")):
45 inputMeasurements = cT.Input(
46 name="cpCtiMeas",
47 doc="Input overscan measurements to fit.",
48 storageClass='StructuredDataDict',
49 dimensions=("instrument", "exposure", "detector"),
50 multiple=True,
51 )
52 camera = cT.PrerequisiteInput(
53 name="camera",
54 doc="Camera geometry to use.",
55 storageClass="Camera",
56 dimensions=("instrument", ),
57 isCalibration=True,
58 )
60 outputCalib = cT.Output(
61 name="cpCtiCalib",
62 doc="Output CTI calibration.",
63 storageClass="IsrCalib",
64 dimensions=("instrument", "detector"),
65 isCalibration=True,
66 )
69class CpCtiSolveConfig(pipeBase.PipelineTaskConfig,
70 pipelineConnections=CpCtiSolveConnections):
71 """Configuration for the CTI combination.
72 """
73 maxImageMean = pexConfig.Field(
74 dtype=float,
75 default=150000.0,
76 doc="Upper limit on acceptable image flux mean.",
77 )
78 localOffsetColumnRange = pexConfig.ListField(
79 dtype=int,
80 default=[3, 13],
81 doc="First and last overscan column to use for local offset effect.",
82 )
84 maxSignalForCti = pexConfig.Field(
85 dtype=float,
86 default=10000.0,
87 doc="Upper flux limit to use for CTI fit.",
88 )
89 globalCtiColumnRange = pexConfig.ListField(
90 dtype=int,
91 default=[1, 2],
92 doc="First and last overscan column to use for global CTI fit.",
93 )
95 trapColumnRange = pexConfig.ListField(
96 dtype=int,
97 default=[1, 20],
98 doc="First and last overscan column to use for serial trap fit.",
99 )
101 fitError = pexConfig.Field(
102 # This gives the error on the mean in a given column, and so
103 # is expected to be $RN / sqrt(N_rows)$.
104 dtype=float,
105 default=7.0/np.sqrt(2000),
106 doc="Error to use during parameter fitting.",
107 )
110class CpCtiSolveTask(pipeBase.PipelineTask):
111 """Combine CTI measurements to a final calibration.
113 This task uses the extended pixel edge response (EPER) method as
114 described by Snyder et al. 2021, Journal of Astronimcal
115 Telescopes, Instruments, and Systems, 7,
116 048002. doi:10.1117/1.JATIS.7.4.048002
117 """
119 ConfigClass = CpCtiSolveConfig
120 _DefaultName = 'cpCtiSolve'
122 def __init__(self, **kwargs):
123 super().__init__(**kwargs)
124 self.allowDebug = True
126 def runQuantum(self, butlerQC, inputRefs, outputRefs):
127 inputs = butlerQC.get(inputRefs)
129 dimensions = [exp.dataId.byName() for exp in inputRefs.inputMeasurements]
130 inputs['inputDims'] = dimensions
132 outputs = self.run(**inputs)
133 butlerQC.put(outputs, outputRefs)
135 def run(self, inputMeasurements, camera, inputDims):
136 """Solve for charge transfer inefficiency from overscan measurements.
138 Parameters
139 ----------
140 inputMeasurements : `list` [`dict`]
141 List of overscan measurements from each input exposure.
142 Each dictionary is nested within a top level 'CTI' key,
143 with measurements organized by amplifier name, containing
144 keys:
146 ``"FIRST_MEAN"``
147 Mean value of first image column (`float`).
148 ``"LAST_MEAN"``
149 Mean value of last image column (`float`).
150 ``"IMAGE_MEAN"``
151 Mean value of the entire image region (`float`).
152 ``"OVERSCAN_COLUMNS"``
153 List of overscan column indicies (`list` [`int`]).
154 ``"OVERSCAN_VALUES"``
155 List of overscan column means (`list` [`float`]).
156 camera : `lsst.afw.cameraGeom.Camera`
157 Camera geometry to use to find detectors.
158 inputDims : `list` [`dict`]
159 List of input dimensions from each input exposure.
161 Returns
162 -------
163 results : `lsst.pipe.base.Struct`
164 Result struct containing:
166 ``outputCalib``
167 Final CTI calibration data
168 (`lsst.ip.isr.DeferredChargeCalib`).
170 Raises
171 ------
172 RuntimeError
173 Raised if data from multiple detectors are passed in.
174 """
175 detectorSet = set([d['detector'] for d in inputDims])
176 if len(detectorSet) != 1:
177 raise RuntimeError("Inputs for too many detectors passed.")
178 detectorId = detectorSet.pop()
179 detector = camera[detectorId]
181 # Initialize with detector.
182 calib = DeferredChargeCalib(camera=camera, detector=detector)
184 localCalib = self.solveLocalOffsets(inputMeasurements, calib, detector)
186 globalCalib = self.solveGlobalCti(inputMeasurements, localCalib, detector)
188 finalCalib = self.findTraps(inputMeasurements, globalCalib, detector)
190 return pipeBase.Struct(
191 outputCalib=finalCalib,
192 )
194 def solveLocalOffsets(self, inputMeasurements, calib, detector):
195 """Solve for local (pixel-to-pixel) electronic offsets.
197 This method fits for \tau_L, the local electronic offset decay
198 time constant, and A_L, the local electronic offset constant
199 of proportionality.
201 Parameters
202 ----------
203 inputMeasurements : `list` [`dict`]
204 List of overscan measurements from each input exposure.
205 Each dictionary is nested within a top level 'CTI' key,
206 with measurements organized by amplifier name, containing
207 keys:
209 ``"FIRST_MEAN"``
210 Mean value of first image column (`float`).
211 ``"LAST_MEAN"``
212 Mean value of last image column (`float`).
213 ``"IMAGE_MEAN"``
214 Mean value of the entire image region (`float`).
215 ``"OVERSCAN_COLUMNS"``
216 List of overscan column indicies (`list` [`int`]).
217 ``"OVERSCAN_VALUES"``
218 List of overscan column means (`list` [`float`]).
219 calib : `lsst.ip.isr.DeferredChargeCalib`
220 Calibration to populate with values.
221 detector : `lsst.afw.cameraGeom.Detector`
222 Detector object containing the geometry information for
223 the amplifiers.
225 Returns
226 -------
227 calib : `lsst.ip.isr.DeferredChargeCalib`
228 Populated calibration.
230 Raises
231 ------
232 RuntimeError
233 Raised if no data remains after flux filtering.
235 Notes
236 -----
237 The original CTISIM code (https://github.com/Snyder005/ctisim)
238 uses a data model in which the "overscan" consists of the
239 standard serial overscan bbox with the values for the last
240 imaging data column prepended to that list. This version of
241 the code keeps the overscan and imaging sections separate, and
242 so a -1 offset is needed to ensure that the same columns are
243 used for fitting between this code and CTISIM. This offset
244 removes that last imaging data column from the count.
245 """
246 # Range to fit. These are in "camera" coordinates, and so
247 # need to have the count for last image column removed.
248 start, stop = self.config.localOffsetColumnRange
249 start -= 1
250 stop -= 1
252 # Loop over amps/inputs, fitting those columns from
253 # "non-saturated" inputs.
254 for amp in detector.getAmplifiers():
255 ampName = amp.getName()
257 # Number of serial shifts.
258 nCols = amp.getRawDataBBox().getWidth() + amp.getRawSerialPrescanBBox().getWidth()
260 # The signal is the mean intensity of each input, and the
261 # data are the overscan columns to fit. For detectors
262 # with non-zero CTI, the charge from the imaging region
263 # leaks into the overscan region.
264 signal = []
265 data = []
266 Nskipped = 0
267 for exposureEntry in inputMeasurements:
268 exposureDict = exposureEntry['CTI']
269 if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxImageMean:
270 signal.append(exposureDict[ampName]['IMAGE_MEAN'])
271 data.append(exposureDict[ampName]['OVERSCAN_VALUES'][start:stop+1])
272 else:
273 Nskipped += 1
274 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxImageMean}.")
275 if len(signal) == 0 or len(data) == 0:
276 raise RuntimeError("All exposures brighter than config.maxImageMean and excluded.")
278 signal = np.array(signal)
279 data = np.array(data)
281 ind = signal.argsort()
282 signal = signal[ind]
283 data = data[ind]
285 params = Parameters()
286 params.add('ctiexp', value=-6, min=-7, max=-5, vary=False)
287 params.add('trapsize', value=0.0, min=0.0, max=10., vary=False)
288 params.add('scaling', value=0.08, min=0.0, max=1.0, vary=False)
289 params.add('emissiontime', value=0.4, min=0.1, max=1.0, vary=False)
290 params.add('driftscale', value=0.00022, min=0., max=0.001, vary=True)
291 params.add('decaytime', value=2.4, min=0.1, max=4.0, vary=True)
293 model = SimpleModel()
294 minner = Minimizer(model.difference, params,
295 fcn_args=(signal, data, self.config.fitError, nCols),
296 fcn_kws={'start': start, 'stop': stop})
297 result = minner.minimize()
299 # Save results for the drift scale and decay time.
300 if not result.success:
301 self.log.warning("Electronics fitting failure for amplifier %s.", ampName)
303 calib.globalCti[ampName] = 10**result.params['ctiexp']
304 calib.driftScale[ampName] = result.params['driftscale'].value if result.success else 0.0
305 calib.decayTime[ampName] = result.params['decaytime'].value if result.success else 2.4
306 self.log.info("CTI Local Fit %s: cti: %g decayTime: %g driftScale %g",
307 ampName, calib.globalCti[ampName], calib.decayTime[ampName],
308 calib.driftScale[ampName])
309 return calib
311 def solveGlobalCti(self, inputMeasurements, calib, detector):
312 """Solve for global CTI constant.
314 This method solves for the mean global CTI, b.
316 Parameters
317 ----------
318 inputMeasurements : `list` [`dict`]
319 List of overscan measurements from each input exposure.
320 Each dictionary is nested within a top level 'CTI' key,
321 with measurements organized by amplifier name, containing
322 keys:
324 ``"FIRST_MEAN"``
325 Mean value of first image column (`float`).
326 ``"LAST_MEAN"``
327 Mean value of last image column (`float`).
328 ``"IMAGE_MEAN"``
329 Mean value of the entire image region (`float`).
330 ``"OVERSCAN_COLUMNS"``
331 List of overscan column indicies (`list` [`int`]).
332 ``"OVERSCAN_VALUES"``
333 List of overscan column means (`list` [`float`]).
334 calib : `lsst.ip.isr.DeferredChargeCalib`
335 Calibration to populate with values.
336 detector : `lsst.afw.cameraGeom.Detector`
337 Detector object containing the geometry information for
338 the amplifiers.
340 Returns
341 -------
342 calib : `lsst.ip.isr.DeferredChargeCalib`
343 Populated calibration.
345 Raises
346 ------
347 RuntimeError
348 Raised if no data remains after flux filtering.
350 Notes
351 -----
352 The original CTISIM code uses a data model in which the
353 "overscan" consists of the standard serial overscan bbox with
354 the values for the last imaging data column prepended to that
355 list. This version of the code keeps the overscan and imaging
356 sections separate, and so a -1 offset is needed to ensure that
357 the same columns are used for fitting between this code and
358 CTISIM. This offset removes that last imaging data column
359 from the count.
360 """
361 # Range to fit. These are in "camera" coordinates, and so
362 # need to have the count for last image column removed.
363 start, stop = self.config.globalCtiColumnRange
364 start -= 1
365 stop -= 1
367 # Loop over amps/inputs, fitting those columns from
368 # "non-saturated" inputs.
369 for amp in detector.getAmplifiers():
370 ampName = amp.getName()
372 # Number of serial shifts.
373 nCols = amp.getRawDataBBox().getWidth() + amp.getRawSerialPrescanBBox().getWidth()
375 # The signal is the mean intensity of each input, and the
376 # data are the overscan columns to fit. For detectors
377 # with non-zero CTI, the charge from the imaging region
378 # leaks into the overscan region.
379 signal = []
380 data = []
381 Nskipped = 0
382 for exposureEntry in inputMeasurements:
383 exposureDict = exposureEntry['CTI']
384 if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxSignalForCti:
385 signal.append(exposureDict[ampName]['IMAGE_MEAN'])
386 data.append(exposureDict[ampName]['OVERSCAN_VALUES'][start:stop+1])
387 else:
388 Nskipped += 1
389 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxSignalForCti}.")
390 if len(signal) == 0 or len(data) == 0:
391 raise RuntimeError("All exposures brighter than config.maxSignalForCti and excluded.")
393 signal = np.array(signal)
394 data = np.array(data)
396 ind = signal.argsort()
397 signal = signal[ind]
398 data = data[ind]
400 # CTI test. This looks at the charge that has leaked into
401 # the first few columns of the overscan.
402 overscan1 = data[:, 0]
403 overscan2 = data[:, 1]
404 test = (np.array(overscan1) + np.array(overscan2))/(nCols*np.array(signal))
405 testResult = np.median(test) > 5.E-6
406 self.log.info("Estimate of CTI test is %f for amp %s, %s.", np.median(test), ampName,
407 "full fitting will be performed" if testResult else
408 "only global CTI fitting will be performed")
410 self.debugView(ampName, signal, test)
412 params = Parameters()
413 params.add('ctiexp', value=-6, min=-7, max=-5, vary=True)
414 params.add('trapsize', value=5.0 if testResult else 0.0, min=0.0, max=30.,
415 vary=True if testResult else False)
416 params.add('scaling', value=0.08, min=0.0, max=1.0,
417 vary=True if testResult else False)
418 params.add('emissiontime', value=0.35, min=0.1, max=1.0,
419 vary=True if testResult else False)
420 params.add('driftscale', value=calib.driftScale[ampName], min=0., max=0.001, vary=False)
421 params.add('decaytime', value=calib.decayTime[ampName], min=0.1, max=4.0, vary=False)
423 model = SimulatedModel()
424 minner = Minimizer(model.difference, params,
425 fcn_args=(signal, data, self.config.fitError, nCols, amp),
426 fcn_kws={'start': start, 'stop': stop, 'trap_type': 'linear'})
427 result = minner.minimize()
429 # Only the global CTI term is retained from this fit.
430 calib.globalCti[ampName] = 10**result.params['ctiexp'].value
431 self.log.info("CTI Global Cti %s: cti: %g decayTime: %g driftScale %g",
432 ampName, calib.globalCti[ampName], calib.decayTime[ampName],
433 calib.driftScale[ampName])
435 return calib
437 def debugView(self, ampName, signal, test):
438 """Debug method for global CTI test value.
440 Parameters
441 ----------
442 ampName : `str`
443 Name of the amp for plot title.
444 signal : `list` [`float`]
445 Image means for the input exposures.
446 test : `list` [`float`]
447 CTI test value to plot.
448 """
449 import lsstDebug
450 if not lsstDebug.Info(__name__).display:
451 return
452 if not self.allowDebug:
453 return
455 import matplotlib.pyplot as plot
456 figure = plot.figure(1)
457 figure.clear()
458 plot.xscale('log', base=10.0)
459 plot.yscale('log', base=10.0)
460 plot.xlabel('Flat Field Signal [e-?]')
461 plot.ylabel('Serial CTI')
462 plot.title(ampName)
463 plot.plot(signal, test)
465 figure.show()
466 prompt = "Press Enter or c to continue [chp]..."
467 while True:
468 ans = input(prompt).lower()
469 if ans in ("", " ", "c",):
470 break
471 elif ans in ("p", ):
472 import pdb
473 pdb.set_trace()
474 elif ans in ('x', ):
475 self.allowDebug = False
476 break
477 elif ans in ("h", ):
478 print("[h]elp [c]ontinue [p]db e[x]itDebug")
479 plot.close()
481 def findTraps(self, inputMeasurements, calib, detector):
482 """Solve for serial trap parameters.
484 Parameters
485 ----------
486 inputMeasurements : `list` [`dict`]
487 List of overscan measurements from each input exposure.
488 Each dictionary is nested within a top level 'CTI' key,
489 with measurements organized by amplifier name, containing
490 keys:
492 ``"FIRST_MEAN"``
493 Mean value of first image column (`float`).
494 ``"LAST_MEAN"``
495 Mean value of last image column (`float`).
496 ``"IMAGE_MEAN"``
497 Mean value of the entire image region (`float`).
498 ``"OVERSCAN_COLUMNS"``
499 List of overscan column indicies (`list` [`int`]).
500 ``"OVERSCAN_VALUES"``
501 List of overscan column means (`list` [`float`]).
502 calib : `lsst.ip.isr.DeferredChargeCalib`
503 Calibration to populate with values.
504 detector : `lsst.afw.cameraGeom.Detector`
505 Detector object containing the geometry information for
506 the amplifiers.
508 Returns
509 -------
510 calib : `lsst.ip.isr.DeferredChargeCalib`
511 Populated calibration.
513 Raises
514 ------
515 RuntimeError
516 Raised if no data remains after flux filtering.
518 Notes
519 -----
520 The original CTISIM code uses a data model in which the
521 "overscan" consists of the standard serial overscan bbox with
522 the values for the last imaging data column prepended to that
523 list. This version of the code keeps the overscan and imaging
524 sections separate, and so a -1 offset is needed to ensure that
525 the same columns are used for fitting between this code and
526 CTISIM. This offset removes that last imaging data column
527 from the count.
528 """
529 # Range to fit. These are in "camera" coordinates, and so
530 # need to have the count for last image column removed.
531 start, stop = self.config.trapColumnRange
532 start -= 1
533 stop -= 1
535 # Loop over amps/inputs, fitting those columns from
536 # "non-saturated" inputs.
537 for amp in detector.getAmplifiers():
538 ampName = amp.getName()
540 # Number of serial shifts.
541 nCols = amp.getRawDataBBox().getWidth() + amp.getRawSerialPrescanBBox().getWidth()
543 # The signal is the mean intensity of each input, and the
544 # data are the overscan columns to fit. The new_signal is
545 # the mean in the last image column. Any serial trap will
546 # take charge from this column, and deposit it into the
547 # overscan columns.
548 signal = []
549 data = []
550 new_signal = []
551 Nskipped = 0
552 for exposureEntry in inputMeasurements:
553 exposureDict = exposureEntry['CTI']
554 if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxImageMean:
555 signal.append(exposureDict[ampName]['IMAGE_MEAN'])
556 data.append(exposureDict[ampName]['OVERSCAN_VALUES'][start:stop+1])
557 new_signal.append(exposureDict[ampName]['LAST_MEAN'])
558 else:
559 Nskipped += 1
560 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxImageMean}.")
561 if len(signal) == 0 or len(data) == 0:
562 raise RuntimeError("All exposures brighter than config.maxImageMean and excluded.")
564 signal = np.array(signal)
565 data = np.array(data)
566 new_signal = np.array(new_signal)
568 ind = signal.argsort()
569 signal = signal[ind]
570 data = data[ind]
571 new_signal = new_signal[ind]
573 # In the absense of any trap, the model results using the
574 # parameters already determined will match the observed
575 # overscan results.
576 params = Parameters()
577 params.add('ctiexp', value=np.log10(calib.globalCti[ampName]),
578 min=-7, max=-5, vary=False)
579 params.add('trapsize', value=0.0, min=0.0, max=10., vary=False)
580 params.add('scaling', value=0.08, min=0.0, max=1.0, vary=False)
581 params.add('emissiontime', value=0.35, min=0.1, max=1.0, vary=False)
582 params.add('driftscale', value=calib.driftScale[ampName],
583 min=0.0, max=0.001, vary=False)
584 params.add('decaytime', value=calib.decayTime[ampName],
585 min=0.1, max=4.0, vary=False)
587 model = SimpleModel.model_results(params, signal, nCols,
588 start=start, stop=stop)
590 # Evaluating trap: the difference between the model and
591 # observed data.
592 res = np.sum((data-model)[:, :3], axis=1)
594 # Create spline model for the trap, using the residual
595 # between data and model as a function of the last image
596 # column mean (new_signal) scaled by (1 - A_L).
597 new_signal = np.asarray((1 - calib.driftScale[ampName])*new_signal, dtype=np.float64)
598 x = new_signal
599 y = np.maximum(0, res)
601 # Pad left with ramp
602 y = np.pad(y, (10, 0), 'linear_ramp', end_values=(0, 0))
603 x = np.pad(x, (10, 0), 'linear_ramp', end_values=(0, 0))
605 # Pad right with constant
606 y = np.pad(y, (1, 1), 'constant', constant_values=(0, y[-1]))
607 x = np.pad(x, (1, 1), 'constant', constant_values=(-1, 200000.))
609 trap = SerialTrap(20000.0, 0.4, 1, 'spline', np.concatenate((x, y)).tolist())
610 calib.serialTraps[ampName] = trap
612 return calib
615class OverscanModel:
616 """Base class for handling model/data fit comparisons.
618 This handles all of the methods needed for the lmfit Minimizer to
619 run.
620 """
622 @staticmethod
623 def model_results(params, signal, num_transfers, start=1, stop=10):
624 """Generate a realization of the overscan model, using the specified
625 fit parameters and input signal.
627 Parameters
628 ----------
629 params : `lmfit.Parameters`
630 Object containing the model parameters.
631 signal : `np.ndarray`, (nMeasurements)
632 Array of image means.
633 num_transfers : `int`
634 Number of serial transfers that the charge undergoes.
635 start : `int`, optional
636 First overscan column to fit. This number includes the
637 last imaging column, and needs to be adjusted by one when
638 using the overscan bounding box.
639 stop : `int`, optional
640 Last overscan column to fit. This number includes the
641 last imaging column, and needs to be adjusted by one when
642 using the overscan bounding box.
644 Returns
645 -------
646 results : `np.ndarray`, (nMeasurements, nCols)
647 Model results.
648 """
649 raise NotImplementedError("Subclasses must implement the model calculation.")
651 def loglikelihood(self, params, signal, data, error, *args, **kwargs):
652 """Calculate log likelihood of the model.
654 Parameters
655 ----------
656 params : `lmfit.Parameters`
657 Object containing the model parameters.
658 signal : `np.ndarray`, (nMeasurements)
659 Array of image means.
660 data : `np.ndarray`, (nMeasurements, nCols)
661 Array of overscan column means from each measurement.
662 error : `float`
663 Fixed error value.
664 *args :
665 Additional position arguments.
666 **kwargs :
667 Additional keyword arguments.
669 Returns
670 -------
671 logL : `float`
672 The log-likelihood of the observed data given the model
673 parameters.
674 """
675 model_results = self.model_results(params, signal, *args, **kwargs)
677 inv_sigma2 = 1.0/(error**2.0)
678 diff = model_results - data
680 return -0.5*(np.sum(inv_sigma2*(diff)**2.))
682 def negative_loglikelihood(self, params, signal, data, error, *args, **kwargs):
683 """Calculate negative log likelihood of the model.
685 Parameters
686 ----------
687 params : `lmfit.Parameters`
688 Object containing the model parameters.
689 signal : `np.ndarray`, (nMeasurements)
690 Array of image means.
691 data : `np.ndarray`, (nMeasurements, nCols)
692 Array of overscan column means from each measurement.
693 error : `float`
694 Fixed error value.
695 *args :
696 Additional position arguments.
697 **kwargs :
698 Additional keyword arguments.
700 Returns
701 -------
702 negativelogL : `float`
703 The negative log-likelihood of the observed data given the
704 model parameters.
705 """
706 ll = self.loglikelihood(params, signal, data, error, *args, **kwargs)
708 return -ll
710 def rms_error(self, params, signal, data, error, *args, **kwargs):
711 """Calculate RMS error between model and data.
713 Parameters
714 ----------
715 params : `lmfit.Parameters`
716 Object containing the model parameters.
717 signal : `np.ndarray`, (nMeasurements)
718 Array of image means.
719 data : `np.ndarray`, (nMeasurements, nCols)
720 Array of overscan column means from each measurement.
721 error : `float`
722 Fixed error value.
723 *args :
724 Additional position arguments.
725 **kwargs :
726 Additional keyword arguments.
728 Returns
729 -------
730 rms : `float`
731 The rms error between the model and input data.
732 """
733 model_results = self.model_results(params, signal, *args, **kwargs)
735 diff = model_results - data
736 rms = np.sqrt(np.mean(np.square(diff)))
738 return rms
740 def difference(self, params, signal, data, error, *args, **kwargs):
741 """Calculate the flattened difference array between model and data.
743 Parameters
744 ----------
745 params : `lmfit.Parameters`
746 Object containing the model parameters.
747 signal : `np.ndarray`, (nMeasurements)
748 Array of image means.
749 data : `np.ndarray`, (nMeasurements, nCols)
750 Array of overscan column means from each measurement.
751 error : `float`
752 Fixed error value.
753 *args :
754 Additional position arguments.
755 **kwargs :
756 Additional keyword arguments.
758 Returns
759 -------
760 difference : `np.ndarray`, (nMeasurements*nCols)
761 The rms error between the model and input data.
762 """
763 model_results = self.model_results(params, signal, *args, **kwargs)
764 diff = (model_results-data).flatten()
766 return diff
769class SimpleModel(OverscanModel):
770 """Simple analytic overscan model."""
772 @staticmethod
773 def model_results(params, signal, num_transfers, start=1, stop=10):
774 """Generate a realization of the overscan model, using the specified
775 fit parameters and input signal.
777 Parameters
778 ----------
779 params : `lmfit.Parameters`
780 Object containing the model parameters.
781 signal : `np.ndarray`, (nMeasurements)
782 Array of image means.
783 num_transfers : `int`
784 Number of serial transfers that the charge undergoes.
785 start : `int`, optional
786 First overscan column to fit. This number includes the
787 last imaging column, and needs to be adjusted by one when
788 using the overscan bounding box.
789 stop : `int`, optional
790 Last overscan column to fit. This number includes the
791 last imaging column, and needs to be adjusted by one when
792 using the overscan bounding box.
794 Returns
795 -------
796 res : `np.ndarray`, (nMeasurements, nCols)
797 Model results.
798 """
799 v = params.valuesdict()
800 v['cti'] = 10**v['ctiexp']
802 # Adjust column numbering to match DM overscan bbox.
803 start += 1
804 stop += 1
806 x = np.arange(start, stop+1)
807 res = np.zeros((signal.shape[0], x.shape[0]))
809 for i, s in enumerate(signal):
810 # This is largely equivalent to equation 2. The minimum
811 # indicates that a trap cannot emit more charge than is
812 # available, nor can it emit more charge than it can hold.
813 # This scales the exponential release of charge from the
814 # trap. The next term defines the contribution from the
815 # global CTI at each pixel transfer, and the final term
816 # includes the contribution from local CTI effects.
817 res[i, :] = (np.minimum(v['trapsize'], s*v['scaling'])
818 * (np.exp(1/v['emissiontime']) - 1.0)
819 * np.exp(-x/v['emissiontime'])
820 + s*num_transfers*v['cti']**x
821 + v['driftscale']*s*np.exp(-x/float(v['decaytime'])))
823 return res
826class SimulatedModel(OverscanModel):
827 """Simulated overscan model."""
829 @staticmethod
830 def model_results(params, signal, num_transfers, amp, start=1, stop=10, trap_type=None):
831 """Generate a realization of the overscan model, using the specified
832 fit parameters and input signal.
834 Parameters
835 ----------
836 params : `lmfit.Parameters`
837 Object containing the model parameters.
838 signal : `np.ndarray`, (nMeasurements)
839 Array of image means.
840 num_transfers : `int`
841 Number of serial transfers that the charge undergoes.
842 amp : `lsst.afw.cameraGeom.Amplifier`
843 Amplifier to use for geometry information.
844 start : `int`, optional
845 First overscan column to fit. This number includes the
846 last imaging column, and needs to be adjusted by one when
847 using the overscan bounding box.
848 stop : `int`, optional
849 Last overscan column to fit. This number includes the
850 last imaging column, and needs to be adjusted by one when
851 using the overscan bounding box.
852 trap_type : `str`, optional
853 Type of trap model to use.
855 Returns
856 -------
857 results : `np.ndarray`, (nMeasurements, nCols)
858 Model results.
859 """
860 v = params.valuesdict()
862 # Adjust column numbering to match DM overscan bbox.
863 start += 1
864 stop += 1
866 # Electronics effect optimization
867 output_amplifier = FloatingOutputAmplifier(1.0, v['driftscale'], v['decaytime'])
869 # CTI optimization
870 v['cti'] = 10**v['ctiexp']
872 # Trap type for optimization
873 if trap_type is None:
874 trap = None
875 elif trap_type == 'linear':
876 trap = SerialTrap(v['trapsize'], v['emissiontime'], 1, 'linear',
877 [v['scaling']])
878 elif trap_type == 'logistic':
879 trap = SerialTrap(v['trapsize'], v['emissiontime'], 1, 'logistic',
880 [v['f0'], v['k']])
881 else:
882 raise ValueError('Trap type must be linear or logistic or None')
884 # Simulate ramp readout
885 imarr = np.zeros((signal.shape[0], amp.getRawDataBBox().getWidth()))
886 ramp = SegmentSimulator(imarr, amp.getRawSerialPrescanBBox().getWidth(), output_amplifier,
887 cti=v['cti'], traps=trap)
888 ramp.ramp_exp(signal)
889 model_results = ramp.readout(serial_overscan_width=amp.getRawSerialOverscanBBox().getWidth(),
890 parallel_overscan_width=0)
892 ncols = amp.getRawSerialPrescanBBox().getWidth() + amp.getRawDataBBox().getWidth()
894 return model_results[:, ncols+start-1:ncols+stop]
897class SegmentSimulator:
898 """Controls the creation of simulated segment images.
900 Parameters
901 ----------
902 imarr : `np.ndarray` (nx, ny)
903 Image data array.
904 prescan_width : `int`
905 Number of serial prescan columns.
906 output_amplifier : `lsst.cp.pipe.FloatingOutputAmplifier`
907 An object holding the gain, read noise, and global_offset.
908 cti : `float`
909 Global CTI value.
910 traps : `list` [`lsst.ip.isr.SerialTrap`]
911 Serial traps to simulate.
912 """
914 def __init__(self, imarr, prescan_width, output_amplifier, cti=0.0, traps=None):
915 # Image array geometry
916 self.prescan_width = prescan_width
917 self.ny, self.nx = imarr.shape
919 self.segarr = np.zeros((self.ny, self.nx+prescan_width))
920 self.segarr[:, prescan_width:] = imarr
922 # Serial readout information
923 self.output_amplifier = output_amplifier
924 if isinstance(cti, np.ndarray):
925 raise ValueError("cti must be single value, not an array.")
926 self.cti = cti
928 self.serial_traps = None
929 self.do_trapping = False
930 if traps is not None:
931 if not isinstance(traps, list):
932 traps = [traps]
933 for trap in traps:
934 self.add_trap(trap)
936 def add_trap(self, serial_trap):
937 """Add a trap to the serial register.
939 Parameters
940 ----------
941 serial_trap : `lsst.ip.isr.SerialTrap`
942 The trap to add.
943 """
944 try:
945 self.serial_traps.append(serial_trap)
946 except AttributeError:
947 self.serial_traps = [serial_trap]
948 self.do_trapping = True
950 def ramp_exp(self, signal_list):
951 """Simulate an image with varying flux illumination per row.
953 This method simulates a segment image where the signal level
954 increases along the horizontal direction, according to the
955 provided list of signal levels.
957 Parameters
958 ----------
959 signal_list : `list` [`float`]
960 List of signal levels.
962 Raises
963 ------
964 ValueError
965 Raised if the length of the signal list does not equal the
966 number of rows.
967 """
968 if len(signal_list) != self.ny:
969 raise ValueError("Signal list does not match row count.")
971 ramp = np.tile(signal_list, (self.nx, 1)).T
972 self.segarr[:, self.prescan_width:] += ramp
974 def readout(self, serial_overscan_width=10, parallel_overscan_width=0):
975 """Simulate serial readout of the segment image.
977 This method performs the serial readout of a segment image
978 given the appropriate SerialRegister object and the properties
979 of the ReadoutAmplifier. Additional arguments can be provided
980 to account for the number of desired overscan transfers. The
981 result is a simulated final segment image, in ADU.
983 Parameters
984 ----------
985 serial_overscan_width : `int`, optional
986 Number of serial overscan columns.
987 parallel_overscan_width : `int`, optional
988 Number of parallel overscan rows.
990 Returns
991 -------
992 result : `np.ndarray` (nx, ny)
993 Simulated image, including serial prescan, serial
994 overscan, and parallel overscan regions.
995 """
996 # Create output array
997 iy = int(self.ny + parallel_overscan_width)
998 ix = int(self.nx + self.prescan_width + serial_overscan_width)
999 image = np.random.normal(loc=self.output_amplifier.global_offset,
1000 scale=self.output_amplifier.noise,
1001 size=(iy, ix))
1002 free_charge = copy.deepcopy(self.segarr)
1004 # Set flow control parameters
1005 do_trapping = self.do_trapping
1006 cti = self.cti
1008 offset = np.zeros(self.ny)
1009 cte = 1 - cti
1010 if do_trapping:
1011 for trap in self.serial_traps:
1012 trap.initialize(self.ny, self.nx, self.prescan_width)
1014 for i in range(ix):
1015 # Trap capture
1016 if do_trapping:
1017 for trap in self.serial_traps:
1018 captured_charge = trap.trap_charge(free_charge)
1019 free_charge -= captured_charge
1021 # Pixel-to-pixel proportional loss
1022 transferred_charge = free_charge*cte
1023 deferred_charge = free_charge*cti
1025 # Pixel transfer and readout
1026 offset = self.output_amplifier.local_offset(offset,
1027 transferred_charge[:, 0])
1028 image[:iy-parallel_overscan_width, i] += transferred_charge[:, 0] + offset
1030 free_charge = np.pad(transferred_charge, ((0, 0), (0, 1)),
1031 mode='constant')[:, 1:] + deferred_charge
1033 # Trap emission
1034 if do_trapping:
1035 for trap in self.serial_traps:
1036 released_charge = trap.release_charge()
1037 free_charge += released_charge
1039 return image/float(self.output_amplifier.gain)
1042class FloatingOutputAmplifier:
1043 """Object representing the readout amplifier of a single channel.
1045 Parameters
1046 ----------
1047 gain : `float`
1048 Amplifier gain.
1049 scale : `float`
1050 Drift scale for the amplifier.
1051 decay_time : `float`
1052 Decay time for the bias drift.
1053 noise : `float`, optional
1054 Amplifier read noise.
1055 offset : `float`, optional
1056 Global CTI offset.
1057 """
1059 def __init__(self, gain, scale, decay_time, noise=0.0, offset=0.0):
1061 self.gain = gain
1062 self.noise = noise
1063 self.global_offset = offset
1065 self.update_parameters(scale, decay_time)
1067 def local_offset(self, old, signal):
1068 """Calculate local offset hysteresis.
1070 Parameters
1071 ----------
1072 old : `np.ndarray`, (,)
1073 Previous iteration.
1074 signal : `np.ndarray`, (,)
1075 Current column measurements.
1077 Returns
1078 -------
1079 offset : `np.ndarray`
1080 Local offset.
1081 """
1082 new = self.scale*signal
1084 return np.maximum(new, old*np.exp(-1/self.decay_time))
1086 def update_parameters(self, scale, decay_time):
1087 """Update parameter values, if within acceptable values.
1089 Parameters
1090 ----------
1091 scale : `float`
1092 Drift scale for the amplifier.
1093 decay_time : `float`
1094 Decay time for the bias drift.
1096 Raises
1097 ------
1098 ValueError
1099 Raised if the input parameters are out of range.
1100 """
1101 if scale < 0.0:
1102 raise ValueError("Scale must be greater than or equal to 0.")
1103 if np.isnan(scale):
1104 raise ValueError("Scale must be real-valued number, not NaN.")
1105 self.scale = scale
1106 if decay_time <= 0.0:
1107 raise ValueError("Decay time must be greater than 0.")
1108 if np.isnan(decay_time):
1109 raise ValueError("Decay time must be real-valued number, not NaN.")
1110 self.decay_time = decay_time