Coverage for python/lsst/cp/pipe/deferredCharge.py: 16%
327 statements
« prev ^ index » next coverage.py v6.4.3, created at 2022-08-09 00:47 -0700
« prev ^ index » next coverage.py v6.4.3, created at 2022-08-09 00:47 -0700
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22import copy
23import numpy as np
25import lsst.pipe.base as pipeBase
26import lsst.pipe.base.connectionTypes as cT
27import lsst.pex.config as pexConfig
29from lsst.ip.isr import DeferredChargeCalib, SerialTrap
30from lmfit import Minimizer, Parameters
32from ._lookupStaticCalibration import lookupStaticCalibration
34__all__ = ('CpCtiSolveConnections',
35 'CpCtiSolveConfig',
36 'CpCtiSolveTask',
37 'OverscanModel',
38 'SimpleModel',
39 'SimulatedModel',
40 'SegmentSimulator',
41 'FloatingOutputAmplifier',
42 )
45class CpCtiSolveConnections(pipeBase.PipelineTaskConnections,
46 dimensions=("instrument", "detector")):
47 inputMeasurements = cT.Input(
48 name="cpCtiMeas",
49 doc="Input overscan measurements to fit.",
50 storageClass='StructuredDataDict',
51 dimensions=("instrument", "exposure", "detector"),
52 multiple=True,
53 )
54 camera = cT.PrerequisiteInput(
55 name="camera",
56 doc="Camera geometry to use.",
57 storageClass="Camera",
58 dimensions=("instrument", ),
59 lookupFunction=lookupStaticCalibration,
60 isCalibration=True,
61 )
63 outputCalib = cT.Output(
64 name="cpCtiCalib",
65 doc="Output CTI calibration.",
66 storageClass="IsrCalib",
67 dimensions=("instrument", "detector"),
68 )
71class CpCtiSolveConfig(pipeBase.PipelineTaskConfig,
72 pipelineConnections=CpCtiSolveConnections):
73 """Configuration for the CTI combination.
74 """
75 maxImageMean = pexConfig.Field(
76 dtype=float,
77 default=150000.0,
78 doc="Upper limit on acceptable image flux mean.",
79 )
80 localOffsetColumnRange = pexConfig.ListField(
81 dtype=int,
82 default=[3, 13],
83 doc="First and last overscan column to use for local offset effect.",
84 )
86 maxSignalForCti = pexConfig.Field(
87 dtype=float,
88 default=10000.0,
89 doc="Upper flux limit to use for CTI fit.",
90 )
91 globalCtiColumnRange = pexConfig.ListField(
92 dtype=int,
93 default=[1, 2],
94 doc="First and last overscan column to use for global CTI fit.",
95 )
97 trapColumnRange = pexConfig.ListField(
98 dtype=int,
99 default=[1, 20],
100 doc="First and last overscan column to use for serial trap fit.",
101 )
103 fitError = pexConfig.Field(
104 # This gives the error on the mean in a given column, and so
105 # is expected to be $RN / sqrt(N_rows)$.
106 dtype=float,
107 default=7.0/np.sqrt(2000),
108 doc="Error to use during parameter fitting.",
109 )
112class CpCtiSolveTask(pipeBase.PipelineTask):
113 """Combine CTI measurements to a final calibration.
115 This task uses the extended pixel edge response (EPER) method as
116 described by Snyder et al. 2021, Journal of Astronimcal
117 Telescopes, Instruments, and Systems, 7,
118 048002. doi:10.1117/1.JATIS.7.4.048002
119 """
121 ConfigClass = CpCtiSolveConfig
122 _DefaultName = 'cpCtiSolve'
124 def __init__(self, **kwargs):
125 super().__init__(**kwargs)
126 self.allowDebug = True
128 def runQuantum(self, butlerQC, inputRefs, outputRefs):
129 inputs = butlerQC.get(inputRefs)
131 dimensions = [exp.dataId.byName() for exp in inputRefs.inputMeasurements]
132 inputs['inputDims'] = dimensions
134 outputs = self.run(**inputs)
135 butlerQC.put(outputs, outputRefs)
137 def run(self, inputMeasurements, camera, inputDims):
138 """Solve for charge transfer inefficiency from overscan measurements.
140 Parameters
141 ----------
142 inputMeasurements : `list` [`dict`]
143 List of overscan measurements from each input exposure.
144 Each dictionary is nested within a top level 'CTI' key,
145 with measurements organized by amplifier name, containing
146 keys:
148 ``"FIRST_MEAN"``
149 Mean value of first image column (`float`).
150 ``"LAST_MEAN"``
151 Mean value of last image column (`float`).
152 ``"IMAGE_MEAN"``
153 Mean value of the entire image region (`float`).
154 ``"OVERSCAN_COLUMNS"``
155 List of overscan column indicies (`list` [`int`]).
156 ``"OVERSCAN_VALUES"``
157 List of overscan column means (`list` [`float`]).
158 camera : `lsst.afw.cameraGeom.Camera`
159 Camera geometry to use to find detectors.
160 inputDims : `list` [`dict`]
161 List of input dimensions from each input exposure.
163 Returns
164 -------
165 results : `lsst.pipe.base.Struct`
166 Result struct containing:
168 ``outputCalib``
169 Final CTI calibration data
170 (`lsst.ip.isr.DeferredChargeCalib`).
172 Raises
173 ------
174 RuntimeError
175 Raised if data from multiple detectors are passed in.
176 """
177 detectorSet = set([d['detector'] for d in inputDims])
178 if len(detectorSet) != 1:
179 raise RuntimeError("Inputs for too many detectors passed.")
180 detectorId = detectorSet.pop()
181 detector = camera[detectorId]
183 # Initialize with detector.
184 calib = DeferredChargeCalib(camera=camera, detector=detector)
186 localCalib = self.solveLocalOffsets(inputMeasurements, calib, detector)
188 globalCalib = self.solveGlobalCti(inputMeasurements, localCalib, detector)
190 finalCalib = self.findTraps(inputMeasurements, globalCalib, detector)
192 return pipeBase.Struct(
193 outputCalib=finalCalib,
194 )
196 def solveLocalOffsets(self, inputMeasurements, calib, detector):
197 """Solve for local (pixel-to-pixel) electronic offsets.
199 This method fits for \tau_L, the local electronic offset decay
200 time constant, and A_L, the local electronic offset constant
201 of proportionality.
203 Parameters
204 ----------
205 inputMeasurements : `list` [`dict`]
206 List of overscan measurements from each input exposure.
207 Each dictionary is nested within a top level 'CTI' key,
208 with measurements organized by amplifier name, containing
209 keys:
211 ``"FIRST_MEAN"``
212 Mean value of first image column (`float`).
213 ``"LAST_MEAN"``
214 Mean value of last image column (`float`).
215 ``"IMAGE_MEAN"``
216 Mean value of the entire image region (`float`).
217 ``"OVERSCAN_COLUMNS"``
218 List of overscan column indicies (`list` [`int`]).
219 ``"OVERSCAN_VALUES"``
220 List of overscan column means (`list` [`float`]).
221 calib : `lsst.ip.isr.DeferredChargeCalib`
222 Calibration to populate with values.
223 detector : `lsst.afw.cameraGeom.Detector`
224 Detector object containing the geometry information for
225 the amplifiers.
227 Returns
228 -------
229 calib : `lsst.ip.isr.DeferredChargeCalib`
230 Populated calibration.
232 Notes
233 -----
234 The original CTISIM code uses a data model in which the
235 "overscan" consists of the standard serial overscan bbox with
236 the values for the last imaging data column prepended to that
237 list. This version of the code keeps the overscan and imaging
238 sections separate, and so a -1 offset is needed to ensure that
239 the same columns are used for fitting between this code and
240 CTISIM. This offset removes that last imaging data column
241 from the count.
242 """
243 # Range to fit. These are in "camera" coordinates, and so
244 # need to have the count for last image column removed.
245 start, stop = self.config.localOffsetColumnRange
246 start -= 1
247 stop -= 1
249 # Loop over amps/inputs, fitting those columns from
250 # "non-saturated" inputs.
251 for amp in detector.getAmplifiers():
252 ampName = amp.getName()
254 # Number of serial shifts.
255 nCols = amp.getRawDataBBox().getWidth() + amp.getRawSerialPrescanBBox().getWidth()
257 # The signal is the mean intensity of each input, and the
258 # data are the overscan columns to fit. For detectors
259 # with non-zero CTI, the charge from the imaging region
260 # leaks into the overscan region.
261 signal = []
262 data = []
263 Nskipped = 0
264 for exposureEntry in inputMeasurements:
265 exposureDict = exposureEntry['CTI']
266 if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxImageMean:
267 signal.append(exposureDict[ampName]['IMAGE_MEAN'])
268 data.append(exposureDict[ampName]['OVERSCAN_VALUES'][start:stop+1])
269 else:
270 Nskipped += 1
271 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxImageMean}.")
273 signal = np.array(signal)
274 data = np.array(data)
276 ind = signal.argsort()
277 signal = signal[ind]
278 data = data[ind]
280 params = Parameters()
281 params.add('ctiexp', value=-6, min=-7, max=-5, vary=False)
282 params.add('trapsize', value=0.0, min=0.0, max=10., vary=False)
283 params.add('scaling', value=0.08, min=0.0, max=1.0, vary=False)
284 params.add('emissiontime', value=0.4, min=0.1, max=1.0, vary=False)
285 params.add('driftscale', value=0.00022, min=0., max=0.001, vary=True)
286 params.add('decaytime', value=2.4, min=0.1, max=4.0, vary=True)
288 model = SimpleModel()
289 minner = Minimizer(model.difference, params,
290 fcn_args=(signal, data, self.config.fitError, nCols),
291 fcn_kws={'start': start, 'stop': stop})
292 result = minner.minimize()
294 # Save results for the drift scale and decay time.
295 if not result.success:
296 self.log.warning("Electronics fitting failure for amplifier %s.", ampName)
298 calib.globalCti[ampName] = 10**result.params['ctiexp']
299 calib.driftScale[ampName] = result.params['driftscale'].value if result.success else 0.0
300 calib.decayTime[ampName] = result.params['decaytime'].value if result.success else 2.4
301 self.log.info("CTI Local Fit %s: cti: %g decayTime: %g driftScale %g",
302 ampName, calib.globalCti[ampName], calib.decayTime[ampName],
303 calib.driftScale[ampName])
304 return calib
306 def solveGlobalCti(self, inputMeasurements, calib, detector):
307 """Solve for global CTI constant.
309 This method solves for the mean global CTI, b.
311 Parameters
312 ----------
313 inputMeasurements : `list` [`dict`]
314 List of overscan measurements from each input exposure.
315 Each dictionary is nested within a top level 'CTI' key,
316 with measurements organized by amplifier name, containing
317 keys:
319 ``"FIRST_MEAN"``
320 Mean value of first image column (`float`).
321 ``"LAST_MEAN"``
322 Mean value of last image column (`float`).
323 ``"IMAGE_MEAN"``
324 Mean value of the entire image region (`float`).
325 ``"OVERSCAN_COLUMNS"``
326 List of overscan column indicies (`list` [`int`]).
327 ``"OVERSCAN_VALUES"``
328 List of overscan column means (`list` [`float`]).
329 calib : `lsst.ip.isr.DeferredChargeCalib`
330 Calibration to populate with values.
331 detector : `lsst.afw.cameraGeom.Detector`
332 Detector object containing the geometry information for
333 the amplifiers.
335 Returns
336 -------
337 calib : `lsst.ip.isr.DeferredChargeCalib`
338 Populated calibration.
340 Notes
341 -----
342 The original CTISIM code uses a data model in which the
343 "overscan" consists of the standard serial overscan bbox with
344 the values for the last imaging data column prepended to that
345 list. This version of the code keeps the overscan and imaging
346 sections separate, and so a -1 offset is needed to ensure that
347 the same columns are used for fitting between this code and
348 CTISIM. This offset removes that last imaging data column
349 from the count.
350 """
351 # Range to fit. These are in "camera" coordinates, and so
352 # need to have the count for last image column removed.
353 start, stop = self.config.globalCtiColumnRange
354 start -= 1
355 stop -= 1
357 # Loop over amps/inputs, fitting those columns from
358 # "non-saturated" inputs.
359 for amp in detector.getAmplifiers():
360 ampName = amp.getName()
362 # Number of serial shifts.
363 nCols = amp.getRawDataBBox().getWidth() + amp.getRawSerialPrescanBBox().getWidth()
365 # The signal is the mean intensity of each input, and the
366 # data are the overscan columns to fit. For detectors
367 # with non-zero CTI, the charge from the imaging region
368 # leaks into the overscan region.
369 signal = []
370 data = []
371 Nskipped = 0
372 for exposureEntry in inputMeasurements:
373 exposureDict = exposureEntry['CTI']
374 if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxSignalForCti:
375 signal.append(exposureDict[ampName]['IMAGE_MEAN'])
376 data.append(exposureDict[ampName]['OVERSCAN_VALUES'][start:stop+1])
377 else:
378 Nskipped += 1
379 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxSignalForCti}.")
381 signal = np.array(signal)
382 data = np.array(data)
384 ind = signal.argsort()
385 signal = signal[ind]
386 data = data[ind]
388 # CTI test. This looks at the charge that has leaked into
389 # the first few columns of the overscan.
390 overscan1 = data[:, 0]
391 overscan2 = data[:, 1]
392 test = (np.array(overscan1) + np.array(overscan2))/(nCols*np.array(signal))
393 testResult = np.median(test) > 5.E-6
394 self.log.info("Estimate of CTI test is %f for amp %s, %s.", np.median(test), ampName,
395 "full fitting will be performed" if testResult else
396 "only global CTI fitting will be performed")
398 self.debugView(ampName, signal, test)
400 params = Parameters()
401 params.add('ctiexp', value=-6, min=-7, max=-5, vary=True)
402 params.add('trapsize', value=5.0 if testResult else 0.0, min=0.0, max=30.,
403 vary=True if testResult else False)
404 params.add('scaling', value=0.08, min=0.0, max=1.0,
405 vary=True if testResult else False)
406 params.add('emissiontime', value=0.35, min=0.1, max=1.0,
407 vary=True if testResult else False)
408 params.add('driftscale', value=calib.driftScale[ampName], min=0., max=0.001, vary=False)
409 params.add('decaytime', value=calib.decayTime[ampName], min=0.1, max=4.0, vary=False)
411 model = SimulatedModel()
412 minner = Minimizer(model.difference, params,
413 fcn_args=(signal, data, self.config.fitError, nCols, amp),
414 fcn_kws={'start': start, 'stop': stop, 'trap_type': 'linear'})
415 result = minner.minimize()
417 # Only the global CTI term is retained from this fit.
418 calib.globalCti[ampName] = 10**result.params['ctiexp'].value
419 self.log.info("CTI Global Cti %s: cti: %g decayTime: %g driftScale %g",
420 ampName, calib.globalCti[ampName], calib.decayTime[ampName],
421 calib.driftScale[ampName])
423 return calib
425 def debugView(self, ampName, signal, test):
426 """Debug method for global CTI test value.
428 Parameters
429 ----------
430 ampName : `str`
431 Name of the amp for plot title.
432 signal : `list` [`float`]
433 Image means for the input exposures.
434 test : `list` [`float`]
435 CTI test value to plot.
436 """
437 import lsstDebug
438 if not lsstDebug.Info(__name__).display:
439 return
440 if not self.allowDebug:
441 return
443 import matplotlib.pyplot as plot
444 figure = plot.figure(1)
445 figure.clear()
446 plot.xscale('log', base=10.0)
447 plot.yscale('log', base=10.0)
448 plot.xlabel('Flat Field Signal [e-?]')
449 plot.ylabel('Serial CTI')
450 plot.title(ampName)
451 plot.plot(signal, test)
453 figure.show()
454 prompt = "Press Enter or c to continue [chp]..."
455 while True:
456 ans = input(prompt).lower()
457 if ans in ("", " ", "c",):
458 break
459 elif ans in ("p", ):
460 import pdb
461 pdb.set_trace()
462 elif ans in ('x', ):
463 self.allowDebug = False
464 break
465 elif ans in ("h", ):
466 print("[h]elp [c]ontinue [p]db e[x]itDebug")
467 plot.close()
469 def findTraps(self, inputMeasurements, calib, detector):
470 """Solve for serial trap parameters.
472 Parameters
473 ----------
474 inputMeasurements : `list` [`dict`]
475 List of overscan measurements from each input exposure.
476 Each dictionary is nested within a top level 'CTI' key,
477 with measurements organized by amplifier name, containing
478 keys:
480 ``"FIRST_MEAN"``
481 Mean value of first image column (`float`).
482 ``"LAST_MEAN"``
483 Mean value of last image column (`float`).
484 ``"IMAGE_MEAN"``
485 Mean value of the entire image region (`float`).
486 ``"OVERSCAN_COLUMNS"``
487 List of overscan column indicies (`list` [`int`]).
488 ``"OVERSCAN_VALUES"``
489 List of overscan column means (`list` [`float`]).
490 calib : `lsst.ip.isr.DeferredChargeCalib`
491 Calibration to populate with values.
492 detector : `lsst.afw.cameraGeom.Detector`
493 Detector object containing the geometry information for
494 the amplifiers.
496 Returns
497 -------
498 calib : `lsst.ip.isr.DeferredChargeCalib`
499 Populated calibration.
501 Notes
502 -----
503 The original CTISIM code uses a data model in which the
504 "overscan" consists of the standard serial overscan bbox with
505 the values for the last imaging data column prepended to that
506 list. This version of the code keeps the overscan and imaging
507 sections separate, and so a -1 offset is needed to ensure that
508 the same columns are used for fitting between this code and
509 CTISIM. This offset removes that last imaging data column
510 from the count.
511 """
512 # Range to fit. These are in "camera" coordinates, and so
513 # need to have the count for last image column removed.
514 start, stop = self.config.trapColumnRange
515 start -= 1
516 stop -= 1
518 # Loop over amps/inputs, fitting those columns from
519 # "non-saturated" inputs.
520 for amp in detector.getAmplifiers():
521 ampName = amp.getName()
523 # Number of serial shifts.
524 nCols = amp.getRawDataBBox().getWidth() + amp.getRawSerialPrescanBBox().getWidth()
526 # The signal is the mean intensity of each input, and the
527 # data are the overscan columns to fit. The new_signal is
528 # the mean in the last image column. Any serial trap will
529 # take charge from this column, and deposit it into the
530 # overscan columns.
531 signal = []
532 data = []
533 new_signal = []
534 Nskipped = 0
535 for exposureEntry in inputMeasurements:
536 exposureDict = exposureEntry['CTI']
537 if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxImageMean:
538 signal.append(exposureDict[ampName]['IMAGE_MEAN'])
539 data.append(exposureDict[ampName]['OVERSCAN_VALUES'][start:stop+1])
540 new_signal.append(exposureDict[ampName]['LAST_MEAN'])
541 else:
542 Nskipped += 1
543 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxSignalForCti}.")
545 signal = np.array(signal)
546 data = np.array(data)
547 new_signal = np.array(new_signal)
549 ind = signal.argsort()
550 signal = signal[ind]
551 data = data[ind]
552 new_signal = new_signal[ind]
554 # In the absense of any trap, the model results using the
555 # parameters already determined will match the observed
556 # overscan results.
557 params = Parameters()
558 params.add('ctiexp', value=np.log10(calib.globalCti[ampName]),
559 min=-7, max=-5, vary=False)
560 params.add('trapsize', value=0.0, min=0.0, max=10., vary=False)
561 params.add('scaling', value=0.08, min=0.0, max=1.0, vary=False)
562 params.add('emissiontime', value=0.35, min=0.1, max=1.0, vary=False)
563 params.add('driftscale', value=calib.driftScale[ampName],
564 min=0.0, max=0.001, vary=False)
565 params.add('decaytime', value=calib.decayTime[ampName],
566 min=0.1, max=4.0, vary=False)
568 model = SimpleModel.model_results(params, signal, nCols,
569 start=start, stop=stop)
571 # Evaluating trap: the difference between the model and
572 # observed data.
573 res = np.sum((data-model)[:, :3], axis=1)
575 # Create spline model for the trap, using the residual
576 # between data and model as a function of the last image
577 # column mean (new_signal) scaled by (1 - A_L).
578 new_signal = np.asarray((1 - calib.driftScale[ampName])*new_signal, dtype=np.float64)
579 x = new_signal
580 y = np.maximum(0, res)
582 # Pad left with ramp
583 y = np.pad(y, (10, 0), 'linear_ramp', end_values=(0, 0))
584 x = np.pad(x, (10, 0), 'linear_ramp', end_values=(0, 0))
586 # Pad right with constant
587 y = np.pad(y, (1, 1), 'constant', constant_values=(0, y[-1]))
588 x = np.pad(x, (1, 1), 'constant', constant_values=(-1, 200000.))
590 trap = SerialTrap(20000.0, 0.4, 1, 'spline', np.concatenate((x, y)).tolist())
591 calib.serialTraps[ampName] = trap
593 return calib
596class OverscanModel:
597 """Base class for handling model/data fit comparisons.
599 This handles all of the methods needed for the lmfit Minimizer to
600 run.
601 """
603 @staticmethod
604 def model_results(params, signal, num_transfers, start=1, stop=10):
605 """Generate a realization of the overscan model, using the specified
606 fit parameters and input signal.
608 Parameters
609 ----------
610 params : `lmfit.Parameters`
611 Object containing the model parameters.
612 signal : `np.ndarray`, (nMeasurements)
613 Array of image means.
614 num_transfers : `int`
615 Number of serial transfers that the charge undergoes.
616 start : `int`, optional
617 First overscan column to fit. This number includes the
618 last imaging column, and needs to be adjusted by one when
619 using the overscan bounding box.
620 stop : `int`, optional
621 Last overscan column to fit. This number includes the
622 last imaging column, and needs to be adjusted by one when
623 using the overscan bounding box.
625 Returns
626 -------
627 results : `np.ndarray`, (nMeasurements, nCols)
628 Model results.
629 """
630 raise NotImplementedError("Subclasses must implement the model calculation.")
632 def loglikelihood(self, params, signal, data, error, *args, **kwargs):
633 """Calculate log likelihood of the model.
635 Parameters
636 ----------
637 params : `lmfit.Parameters`
638 Object containing the model parameters.
639 signal : `np.ndarray`, (nMeasurements)
640 Array of image means.
641 data : `np.ndarray`, (nMeasurements, nCols)
642 Array of overscan column means from each measurement.
643 error : `float`
644 Fixed error value.
645 *args :
646 Additional position arguments.
647 **kwargs :
648 Additional keyword arguments.
650 Returns
651 -------
652 logL : `float`
653 The log-likelihood of the observed data given the model
654 parameters.
655 """
656 model_results = self.model_results(params, signal, *args, **kwargs)
658 inv_sigma2 = 1.0/(error**2.0)
659 diff = model_results - data
661 return -0.5*(np.sum(inv_sigma2*(diff)**2.))
663 def negative_loglikelihood(self, params, signal, data, error, *args, **kwargs):
664 """Calculate negative log likelihood of the model.
666 Parameters
667 ----------
668 params : `lmfit.Parameters`
669 Object containing the model parameters.
670 signal : `np.ndarray`, (nMeasurements)
671 Array of image means.
672 data : `np.ndarray`, (nMeasurements, nCols)
673 Array of overscan column means from each measurement.
674 error : `float`
675 Fixed error value.
676 *args :
677 Additional position arguments.
678 **kwargs :
679 Additional keyword arguments.
681 Returns
682 -------
683 negativelogL : `float`
684 The negative log-likelihood of the observed data given the
685 model parameters.
686 """
687 ll = self.loglikelihood(params, signal, data, error, *args, **kwargs)
689 return -ll
691 def rms_error(self, params, signal, data, error, *args, **kwargs):
692 """Calculate RMS error between model and data.
694 Parameters
695 ----------
696 params : `lmfit.Parameters`
697 Object containing the model parameters.
698 signal : `np.ndarray`, (nMeasurements)
699 Array of image means.
700 data : `np.ndarray`, (nMeasurements, nCols)
701 Array of overscan column means from each measurement.
702 error : `float`
703 Fixed error value.
704 *args :
705 Additional position arguments.
706 **kwargs :
707 Additional keyword arguments.
709 Returns
710 -------
711 rms : `float`
712 The rms error between the model and input data.
713 """
714 model_results = self.model_results(params, signal, *args, **kwargs)
716 diff = model_results - data
717 rms = np.sqrt(np.mean(np.square(diff)))
719 return rms
721 def difference(self, params, signal, data, error, *args, **kwargs):
722 """Calculate the flattened difference array between model and data.
724 Parameters
725 ----------
726 params : `lmfit.Parameters`
727 Object containing the model parameters.
728 signal : `np.ndarray`, (nMeasurements)
729 Array of image means.
730 data : `np.ndarray`, (nMeasurements, nCols)
731 Array of overscan column means from each measurement.
732 error : `float`
733 Fixed error value.
734 *args :
735 Additional position arguments.
736 **kwargs :
737 Additional keyword arguments.
739 Returns
740 -------
741 difference : `np.ndarray`, (nMeasurements*nCols)
742 The rms error between the model and input data.
743 """
744 model_results = self.model_results(params, signal, *args, **kwargs)
745 diff = (model_results-data).flatten()
747 return diff
750class SimpleModel(OverscanModel):
751 """Simple analytic overscan model."""
753 @staticmethod
754 def model_results(params, signal, num_transfers, start=1, stop=10):
755 """Generate a realization of the overscan model, using the specified
756 fit parameters and input signal.
758 Parameters
759 ----------
760 params : `lmfit.Parameters`
761 Object containing the model parameters.
762 signal : `np.ndarray`, (nMeasurements)
763 Array of image means.
764 num_transfers : `int`
765 Number of serial transfers that the charge undergoes.
766 start : `int`, optional
767 First overscan column to fit. This number includes the
768 last imaging column, and needs to be adjusted by one when
769 using the overscan bounding box.
770 stop : `int`, optional
771 Last overscan column to fit. This number includes the
772 last imaging column, and needs to be adjusted by one when
773 using the overscan bounding box.
775 Returns
776 -------
777 res : `np.ndarray`, (nMeasurements, nCols)
778 Model results.
779 """
780 v = params.valuesdict()
781 v['cti'] = 10**v['ctiexp']
783 # Adjust column numbering to match DM overscan bbox.
784 start += 1
785 stop += 1
787 x = np.arange(start, stop+1)
788 res = np.zeros((signal.shape[0], x.shape[0]))
790 for i, s in enumerate(signal):
791 # This is largely equivalent to equation 2. The minimum
792 # indicates that a trap cannot emit more charge than is
793 # available, nor can it emit more charge than it can hold.
794 # This scales the exponential release of charge from the
795 # trap. The next term defines the contribution from the
796 # global CTI at each pixel transfer, and the final term
797 # includes the contribution from local CTI effects.
798 res[i, :] = (np.minimum(v['trapsize'], s*v['scaling'])
799 * (np.exp(1/v['emissiontime']) - 1.0)
800 * np.exp(-x/v['emissiontime'])
801 + s*num_transfers*v['cti']**x
802 + v['driftscale']*s*np.exp(-x/float(v['decaytime'])))
804 return res
807class SimulatedModel(OverscanModel):
808 """Simulated overscan model."""
810 @staticmethod
811 def model_results(params, signal, num_transfers, amp, start=1, stop=10, trap_type=None):
812 """Generate a realization of the overscan model, using the specified
813 fit parameters and input signal.
815 Parameters
816 ----------
817 params : `lmfit.Parameters`
818 Object containing the model parameters.
819 signal : `np.ndarray`, (nMeasurements)
820 Array of image means.
821 num_transfers : `int`
822 Number of serial transfers that the charge undergoes.
823 amp : `lsst.afw.cameraGeom.Amplifier`
824 Amplifier to use for geometry information.
825 start : `int`, optional
826 First overscan column to fit. This number includes the
827 last imaging column, and needs to be adjusted by one when
828 using the overscan bounding box.
829 stop : `int`, optional
830 Last overscan column to fit. This number includes the
831 last imaging column, and needs to be adjusted by one when
832 using the overscan bounding box.
833 trap_type : `str`, optional
834 Type of trap model to use.
836 Returns
837 -------
838 results : `np.ndarray`, (nMeasurements, nCols)
839 Model results.
840 """
841 v = params.valuesdict()
843 # Adjust column numbering to match DM overscan bbox.
844 start += 1
845 stop += 1
847 # Electronics effect optimization
848 output_amplifier = FloatingOutputAmplifier(1.0, v['driftscale'], v['decaytime'])
850 # CTI optimization
851 v['cti'] = 10**v['ctiexp']
853 # Trap type for optimization
854 if trap_type is None:
855 trap = None
856 elif trap_type == 'linear':
857 trap = SerialTrap(v['trapsize'], v['emissiontime'], 1, 'linear',
858 [v['scaling']])
859 elif trap_type == 'logistic':
860 trap = SerialTrap(v['trapsize'], v['emissiontime'], 1, 'logistic',
861 [v['f0'], v['k']])
862 else:
863 raise ValueError('Trap type must be linear or logistic or None')
865 # Simulate ramp readout
866 imarr = np.zeros((signal.shape[0], amp.getRawDataBBox().getWidth()))
867 ramp = SegmentSimulator(imarr, amp.getRawSerialPrescanBBox().getWidth(), output_amplifier,
868 cti=v['cti'], traps=trap)
869 ramp.ramp_exp(signal)
870 model_results = ramp.readout(serial_overscan_width=amp.getRawSerialOverscanBBox().getWidth(),
871 parallel_overscan_width=0)
873 ncols = amp.getRawSerialPrescanBBox().getWidth() + amp.getRawDataBBox().getWidth()
875 return model_results[:, ncols+start-1:ncols+stop]
878class SegmentSimulator:
879 """Controls the creation of simulated segment images.
881 Parameters
882 ----------
883 imarr : `np.ndarray` (nx, ny)
884 Image data array.
885 prescan_width : `int`
886 Number of serial prescan columns.
887 output_amplifier : `lsst.cp.pipe.FloatingOutputAmplifier`
888 An object holding the gain, read noise, and global_offset.
889 cti : `float`
890 Global CTI value.
891 traps : `list` [`lsst.ip.isr.SerialTrap`]
892 Serial traps to simulate.
893 """
895 def __init__(self, imarr, prescan_width, output_amplifier, cti=0.0, traps=None):
896 # Image array geometry
897 self.prescan_width = prescan_width
898 self.ny, self.nx = imarr.shape
900 self.segarr = np.zeros((self.ny, self.nx+prescan_width))
901 self.segarr[:, prescan_width:] = imarr
903 # Serial readout information
904 self.output_amplifier = output_amplifier
905 if isinstance(cti, np.ndarray):
906 raise ValueError("cti must be single value, not an array.")
907 self.cti = cti
909 self.serial_traps = None
910 self.do_trapping = False
911 if traps is not None:
912 if not isinstance(traps, list):
913 traps = [traps]
914 for trap in traps:
915 self.add_trap(trap)
917 def add_trap(self, serial_trap):
918 """Add a trap to the serial register.
920 Parameters
921 ----------
922 serial_trap : `lsst.ip.isr.SerialTrap`
923 The trap to add.
924 """
925 try:
926 self.serial_traps.append(serial_trap)
927 except AttributeError:
928 self.serial_traps = [serial_trap]
929 self.do_trapping = True
931 def ramp_exp(self, signal_list):
932 """Simulate an image with varying flux illumination per row.
934 This method simulates a segment image where the signal level
935 increases along the horizontal direction, according to the
936 provided list of signal levels.
938 Parameters
939 ----------
940 signal_list : `list` [`float`]
941 List of signal levels.
943 Raises
944 ------
945 ValueError
946 Raised if the length of the signal list does not equal the
947 number of rows.
948 """
949 if len(signal_list) != self.ny:
950 raise ValueError("Signal list does not match row count.")
952 ramp = np.tile(signal_list, (self.nx, 1)).T
953 self.segarr[:, self.prescan_width:] += ramp
955 def readout(self, serial_overscan_width=10, parallel_overscan_width=0):
956 """Simulate serial readout of the segment image.
958 This method performs the serial readout of a segment image
959 given the appropriate SerialRegister object and the properties
960 of the ReadoutAmplifier. Additional arguments can be provided
961 to account for the number of desired overscan transfers. The
962 result is a simulated final segment image, in ADU.
964 Parameters
965 ----------
966 serial_overscan_width : `int`, optional
967 Number of serial overscan columns.
968 parallel_overscan_width : `int`, optional
969 Number of parallel overscan rows.
971 Returns
972 -------
973 result : `np.ndarray` (nx, ny)
974 Simulated image, including serial prescan, serial
975 overscan, and parallel overscan regions.
976 """
977 # Create output array
978 iy = int(self.ny + parallel_overscan_width)
979 ix = int(self.nx + self.prescan_width + serial_overscan_width)
980 image = np.random.normal(loc=self.output_amplifier.global_offset,
981 scale=self.output_amplifier.noise,
982 size=(iy, ix))
983 free_charge = copy.deepcopy(self.segarr)
985 # Set flow control parameters
986 do_trapping = self.do_trapping
987 cti = self.cti
989 offset = np.zeros(self.ny)
990 cte = 1 - cti
991 if do_trapping:
992 for trap in self.serial_traps:
993 trap.initialize(self.ny, self.nx, self.prescan_width)
995 for i in range(ix):
996 # Trap capture
997 if do_trapping:
998 for trap in self.serial_traps:
999 captured_charge = trap.trap_charge(free_charge)
1000 free_charge -= captured_charge
1002 # Pixel-to-pixel proportional loss
1003 transferred_charge = free_charge*cte
1004 deferred_charge = free_charge*cti
1006 # Pixel transfer and readout
1007 offset = self.output_amplifier.local_offset(offset,
1008 transferred_charge[:, 0])
1009 image[:iy-parallel_overscan_width, i] += transferred_charge[:, 0] + offset
1011 free_charge = np.pad(transferred_charge, ((0, 0), (0, 1)),
1012 mode='constant')[:, 1:] + deferred_charge
1014 # Trap emission
1015 if do_trapping:
1016 for trap in self.serial_traps:
1017 released_charge = trap.release_charge()
1018 free_charge += released_charge
1020 return image/float(self.output_amplifier.gain)
1023class FloatingOutputAmplifier:
1024 """Object representing the readout amplifier of a single channel.
1026 Parameters
1027 ----------
1028 gain : `float`
1029 Amplifier gain.
1030 scale : `float`
1031 Drift scale for the amplifier.
1032 decay_time : `float`
1033 Decay time for the bias drift.
1034 noise : `float`, optional
1035 Amplifier read noise.
1036 offset : `float`, optional
1037 Global CTI offset.
1038 """
1040 def __init__(self, gain, scale, decay_time, noise=0.0, offset=0.0):
1042 self.gain = gain
1043 self.noise = noise
1044 self.global_offset = offset
1046 self.update_parameters(scale, decay_time)
1048 def local_offset(self, old, signal):
1049 """Calculate local offset hysteresis.
1051 Parameters
1052 ----------
1053 old : `np.ndarray`, (,)
1054 Previous iteration.
1055 signal : `np.ndarray`, (,)
1056 Current column measurements.
1058 Returns
1059 -------
1060 offset : `np.ndarray`
1061 Local offset.
1062 """
1063 new = self.scale*signal
1065 return np.maximum(new, old*np.exp(-1/self.decay_time))
1067 def update_parameters(self, scale, decay_time):
1068 """Update parameter values, if within acceptable values.
1070 Parameters
1071 ----------
1072 scale : `float`
1073 Drift scale for the amplifier.
1074 decay_time : `float`
1075 Decay time for the bias drift.
1077 Raises
1078 ------
1079 ValueError
1080 Raised if the input parameters are out of range.
1081 """
1082 if scale < 0.0:
1083 raise ValueError("Scale must be greater than or equal to 0.")
1084 if np.isnan(scale):
1085 raise ValueError("Scale must be real-valued number, not NaN.")
1086 self.scale = scale
1087 if decay_time <= 0.0:
1088 raise ValueError("Decay time must be greater than 0.")
1089 if np.isnan(decay_time):
1090 raise ValueError("Decay time must be real-valued number, not NaN.")
1091 self.decay_time = decay_time