Coverage for python/lsst/cp/pipe/deferredCharge.py: 14%
331 statements
« prev ^ index » next coverage.py v7.4.3, created at 2024-02-28 12:47 +0000
« prev ^ index » next coverage.py v7.4.3, created at 2024-02-28 12:47 +0000
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21#
22__all__ = ('CpCtiSolveConnections',
23 'CpCtiSolveConfig',
24 'CpCtiSolveTask',
25 'OverscanModel',
26 'SimpleModel',
27 'SimulatedModel',
28 'SegmentSimulator',
29 'FloatingOutputAmplifier',
30 )
32import copy
33import numpy as np
35import lsst.pipe.base as pipeBase
36import lsst.pipe.base.connectionTypes as cT
37import lsst.pex.config as pexConfig
39from lsst.ip.isr import DeferredChargeCalib, SerialTrap
40from lmfit import Minimizer, Parameters
43class CpCtiSolveConnections(pipeBase.PipelineTaskConnections,
44 dimensions=("instrument", "detector")):
45 inputMeasurements = cT.Input(
46 name="cpCtiMeas",
47 doc="Input overscan measurements to fit.",
48 storageClass='StructuredDataDict',
49 dimensions=("instrument", "exposure", "detector"),
50 multiple=True,
51 )
52 camera = cT.PrerequisiteInput(
53 name="camera",
54 doc="Camera geometry to use.",
55 storageClass="Camera",
56 dimensions=("instrument", ),
57 isCalibration=True,
58 )
60 outputCalib = cT.Output(
61 name="cpCtiCalib",
62 doc="Output CTI calibration.",
63 storageClass="IsrCalib",
64 dimensions=("instrument", "detector"),
65 isCalibration=True,
66 )
69class CpCtiSolveConfig(pipeBase.PipelineTaskConfig,
70 pipelineConnections=CpCtiSolveConnections):
71 """Configuration for the CTI combination.
72 """
73 maxImageMean = pexConfig.Field(
74 dtype=float,
75 default=150000.0,
76 doc="Upper limit on acceptable image flux mean.",
77 )
78 localOffsetColumnRange = pexConfig.ListField(
79 dtype=int,
80 default=[3, 13],
81 doc="First and last overscan column to use for local offset effect.",
82 )
84 useGains = pexConfig.Field(
85 dtype=bool,
86 default=True,
87 doc="Use gains in calculation.",
88 )
90 maxSignalForCti = pexConfig.Field(
91 dtype=float,
92 default=10000.0,
93 doc="Upper flux limit to use for CTI fit.",
94 )
95 globalCtiColumnRange = pexConfig.ListField(
96 dtype=int,
97 default=[1, 2],
98 doc="First and last overscan column to use for global CTI fit.",
99 )
101 trapColumnRange = pexConfig.ListField(
102 dtype=int,
103 default=[1, 20],
104 doc="First and last overscan column to use for serial trap fit.",
105 )
107 fitError = pexConfig.Field(
108 # This gives the error on the mean in a given column, and so
109 # is expected to be $RN / sqrt(N_rows)$.
110 dtype=float,
111 default=7.0/np.sqrt(2000),
112 doc="Error to use during parameter fitting.",
113 )
116class CpCtiSolveTask(pipeBase.PipelineTask):
117 """Combine CTI measurements to a final calibration.
119 This task uses the extended pixel edge response (EPER) method as
120 described by Snyder et al. 2021, Journal of Astronimcal
121 Telescopes, Instruments, and Systems, 7,
122 048002. doi:10.1117/1.JATIS.7.4.048002
123 """
125 ConfigClass = CpCtiSolveConfig
126 _DefaultName = 'cpCtiSolve'
128 def __init__(self, **kwargs):
129 super().__init__(**kwargs)
130 self.allowDebug = True
132 def runQuantum(self, butlerQC, inputRefs, outputRefs):
133 inputs = butlerQC.get(inputRefs)
135 dimensions = [dict(exp.dataId.required) for exp in inputRefs.inputMeasurements]
136 inputs['inputDims'] = dimensions
138 outputs = self.run(**inputs)
139 butlerQC.put(outputs, outputRefs)
141 def run(self, inputMeasurements, camera, inputDims):
142 """Solve for charge transfer inefficiency from overscan measurements.
144 Parameters
145 ----------
146 inputMeasurements : `list` [`dict`]
147 List of overscan measurements from each input exposure.
148 Each dictionary is nested within a top level 'CTI' key,
149 with measurements organized by amplifier name, containing
150 keys:
152 ``"FIRST_MEAN"``
153 Mean value of first image column (`float`).
154 ``"LAST_MEAN"``
155 Mean value of last image column (`float`).
156 ``"IMAGE_MEAN"``
157 Mean value of the entire image region (`float`).
158 ``"OVERSCAN_COLUMNS"``
159 List of overscan column indicies (`list` [`int`]).
160 ``"OVERSCAN_VALUES"``
161 List of overscan column means (`list` [`float`]).
162 camera : `lsst.afw.cameraGeom.Camera`
163 Camera geometry to use to find detectors.
164 inputDims : `list` [`dict`]
165 List of input dimensions from each input exposure.
167 Returns
168 -------
169 results : `lsst.pipe.base.Struct`
170 Result struct containing:
172 ``outputCalib``
173 Final CTI calibration data
174 (`lsst.ip.isr.DeferredChargeCalib`).
176 Raises
177 ------
178 RuntimeError
179 Raised if data from multiple detectors are passed in.
180 """
181 detectorSet = set([d['detector'] for d in inputDims])
182 if len(detectorSet) != 1:
183 raise RuntimeError("Inputs for too many detectors passed.")
184 detectorId = detectorSet.pop()
185 detector = camera[detectorId]
187 # Initialize with detector.
188 calib = DeferredChargeCalib(camera=camera, detector=detector, useGains=self.config.useGains)
190 localCalib = self.solveLocalOffsets(inputMeasurements, calib, detector)
192 globalCalib = self.solveGlobalCti(inputMeasurements, localCalib, detector)
194 finalCalib = self.findTraps(inputMeasurements, globalCalib, detector)
196 return pipeBase.Struct(
197 outputCalib=finalCalib,
198 )
200 def solveLocalOffsets(self, inputMeasurements, calib, detector):
201 """Solve for local (pixel-to-pixel) electronic offsets.
203 This method fits for \tau_L, the local electronic offset decay
204 time constant, and A_L, the local electronic offset constant
205 of proportionality.
207 Parameters
208 ----------
209 inputMeasurements : `list` [`dict`]
210 List of overscan measurements from each input exposure.
211 Each dictionary is nested within a top level 'CTI' key,
212 with measurements organized by amplifier name, containing
213 keys:
215 ``"FIRST_MEAN"``
216 Mean value of first image column (`float`).
217 ``"LAST_MEAN"``
218 Mean value of last image column (`float`).
219 ``"IMAGE_MEAN"``
220 Mean value of the entire image region (`float`).
221 ``"OVERSCAN_COLUMNS"``
222 List of overscan column indicies (`list` [`int`]).
223 ``"OVERSCAN_VALUES"``
224 List of overscan column means (`list` [`float`]).
225 calib : `lsst.ip.isr.DeferredChargeCalib`
226 Calibration to populate with values.
227 detector : `lsst.afw.cameraGeom.Detector`
228 Detector object containing the geometry information for
229 the amplifiers.
231 Returns
232 -------
233 calib : `lsst.ip.isr.DeferredChargeCalib`
234 Populated calibration.
236 Raises
237 ------
238 RuntimeError
239 Raised if no data remains after flux filtering.
241 Notes
242 -----
243 The original CTISIM code (https://github.com/Snyder005/ctisim)
244 uses a data model in which the "overscan" consists of the
245 standard serial overscan bbox with the values for the last
246 imaging data column prepended to that list. This version of
247 the code keeps the overscan and imaging sections separate, and
248 so a -1 offset is needed to ensure that the same columns are
249 used for fitting between this code and CTISIM. This offset
250 removes that last imaging data column from the count.
251 """
252 # Range to fit. These are in "camera" coordinates, and so
253 # need to have the count for last image column removed.
254 start, stop = self.config.localOffsetColumnRange
255 start -= 1
256 stop -= 1
258 # Loop over amps/inputs, fitting those columns from
259 # "non-saturated" inputs.
260 for amp in detector.getAmplifiers():
261 ampName = amp.getName()
263 # Number of serial shifts.
264 nCols = amp.getRawDataBBox().getWidth() + amp.getRawSerialPrescanBBox().getWidth()
266 # The signal is the mean intensity of each input, and the
267 # data are the overscan columns to fit. For detectors
268 # with non-zero CTI, the charge from the imaging region
269 # leaks into the overscan region.
270 signal = []
271 data = []
272 Nskipped = 0
273 for exposureEntry in inputMeasurements:
274 exposureDict = exposureEntry['CTI']
275 if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxImageMean:
276 signal.append(exposureDict[ampName]['IMAGE_MEAN'])
277 data.append(exposureDict[ampName]['OVERSCAN_VALUES'][start:stop+1])
278 else:
279 Nskipped += 1
280 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxImageMean}.")
281 if len(signal) == 0 or len(data) == 0:
282 raise RuntimeError("All exposures brighter than config.maxImageMean and excluded.")
284 signal = np.array(signal)
285 data = np.array(data)
287 ind = signal.argsort()
288 signal = signal[ind]
289 data = data[ind]
291 params = Parameters()
292 params.add('ctiexp', value=-6, min=-7, max=-5, vary=False)
293 params.add('trapsize', value=0.0, min=0.0, max=10., vary=False)
294 params.add('scaling', value=0.08, min=0.0, max=1.0, vary=False)
295 params.add('emissiontime', value=0.4, min=0.1, max=1.0, vary=False)
296 params.add('driftscale', value=0.00022, min=0., max=0.001, vary=True)
297 params.add('decaytime', value=2.4, min=0.1, max=4.0, vary=True)
299 model = SimpleModel()
300 minner = Minimizer(model.difference, params,
301 fcn_args=(signal, data, self.config.fitError, nCols),
302 fcn_kws={'start': start, 'stop': stop})
303 result = minner.minimize()
305 # Save results for the drift scale and decay time.
306 if not result.success:
307 self.log.warning("Electronics fitting failure for amplifier %s.", ampName)
309 calib.globalCti[ampName] = 10**result.params['ctiexp']
310 calib.driftScale[ampName] = result.params['driftscale'].value if result.success else 0.0
311 calib.decayTime[ampName] = result.params['decaytime'].value if result.success else 2.4
312 self.log.info("CTI Local Fit %s: cti: %g decayTime: %g driftScale %g",
313 ampName, calib.globalCti[ampName], calib.decayTime[ampName],
314 calib.driftScale[ampName])
315 return calib
317 def solveGlobalCti(self, inputMeasurements, calib, detector):
318 """Solve for global CTI constant.
320 This method solves for the mean global CTI, b.
322 Parameters
323 ----------
324 inputMeasurements : `list` [`dict`]
325 List of overscan measurements from each input exposure.
326 Each dictionary is nested within a top level 'CTI' key,
327 with measurements organized by amplifier name, containing
328 keys:
330 ``"FIRST_MEAN"``
331 Mean value of first image column (`float`).
332 ``"LAST_MEAN"``
333 Mean value of last image column (`float`).
334 ``"IMAGE_MEAN"``
335 Mean value of the entire image region (`float`).
336 ``"OVERSCAN_COLUMNS"``
337 List of overscan column indicies (`list` [`int`]).
338 ``"OVERSCAN_VALUES"``
339 List of overscan column means (`list` [`float`]).
340 calib : `lsst.ip.isr.DeferredChargeCalib`
341 Calibration to populate with values.
342 detector : `lsst.afw.cameraGeom.Detector`
343 Detector object containing the geometry information for
344 the amplifiers.
346 Returns
347 -------
348 calib : `lsst.ip.isr.DeferredChargeCalib`
349 Populated calibration.
351 Raises
352 ------
353 RuntimeError
354 Raised if no data remains after flux filtering.
356 Notes
357 -----
358 The original CTISIM code uses a data model in which the
359 "overscan" consists of the standard serial overscan bbox with
360 the values for the last imaging data column prepended to that
361 list. This version of the code keeps the overscan and imaging
362 sections separate, and so a -1 offset is needed to ensure that
363 the same columns are used for fitting between this code and
364 CTISIM. This offset removes that last imaging data column
365 from the count.
366 """
367 # Range to fit. These are in "camera" coordinates, and so
368 # need to have the count for last image column removed.
369 start, stop = self.config.globalCtiColumnRange
370 start -= 1
371 stop -= 1
373 # Loop over amps/inputs, fitting those columns from
374 # "non-saturated" inputs.
375 for amp in detector.getAmplifiers():
376 ampName = amp.getName()
378 # Number of serial shifts.
379 nCols = amp.getRawDataBBox().getWidth() + amp.getRawSerialPrescanBBox().getWidth()
381 # The signal is the mean intensity of each input, and the
382 # data are the overscan columns to fit. For detectors
383 # with non-zero CTI, the charge from the imaging region
384 # leaks into the overscan region.
385 signal = []
386 data = []
387 Nskipped = 0
388 for exposureEntry in inputMeasurements:
389 exposureDict = exposureEntry['CTI']
390 if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxSignalForCti:
391 signal.append(exposureDict[ampName]['IMAGE_MEAN'])
392 data.append(exposureDict[ampName]['OVERSCAN_VALUES'][start:stop+1])
393 else:
394 Nskipped += 1
395 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxSignalForCti}.")
396 if len(signal) == 0 or len(data) == 0:
397 raise RuntimeError("All exposures brighter than config.maxSignalForCti and excluded.")
399 signal = np.array(signal)
400 data = np.array(data)
402 ind = signal.argsort()
403 signal = signal[ind]
404 data = data[ind]
406 # CTI test. This looks at the charge that has leaked into
407 # the first few columns of the overscan.
408 overscan1 = data[:, 0]
409 overscan2 = data[:, 1]
410 test = (np.array(overscan1) + np.array(overscan2))/(nCols*np.array(signal))
411 testResult = np.median(test) > 5.E-6
412 self.log.info("Estimate of CTI test is %f for amp %s, %s.", np.median(test), ampName,
413 "full fitting will be performed" if testResult else
414 "only global CTI fitting will be performed")
416 self.debugView(ampName, signal, test)
418 params = Parameters()
419 params.add('ctiexp', value=-6, min=-7, max=-5, vary=True)
420 params.add('trapsize', value=5.0 if testResult else 0.0, min=0.0, max=30.,
421 vary=True if testResult else False)
422 params.add('scaling', value=0.08, min=0.0, max=1.0,
423 vary=True if testResult else False)
424 params.add('emissiontime', value=0.35, min=0.1, max=1.0,
425 vary=True if testResult else False)
426 params.add('driftscale', value=calib.driftScale[ampName], min=0., max=0.001, vary=False)
427 params.add('decaytime', value=calib.decayTime[ampName], min=0.1, max=4.0, vary=False)
429 model = SimulatedModel()
430 minner = Minimizer(model.difference, params,
431 fcn_args=(signal, data, self.config.fitError, nCols, amp),
432 fcn_kws={'start': start, 'stop': stop, 'trap_type': 'linear'})
433 result = minner.minimize()
435 # Only the global CTI term is retained from this fit.
436 calib.globalCti[ampName] = 10**result.params['ctiexp'].value
437 self.log.info("CTI Global Cti %s: cti: %g decayTime: %g driftScale %g",
438 ampName, calib.globalCti[ampName], calib.decayTime[ampName],
439 calib.driftScale[ampName])
441 return calib
443 def debugView(self, ampName, signal, test):
444 """Debug method for global CTI test value.
446 Parameters
447 ----------
448 ampName : `str`
449 Name of the amp for plot title.
450 signal : `list` [`float`]
451 Image means for the input exposures.
452 test : `list` [`float`]
453 CTI test value to plot.
454 """
455 import lsstDebug
456 if not lsstDebug.Info(__name__).display:
457 return
458 if not self.allowDebug:
459 return
461 import matplotlib.pyplot as plot
462 figure = plot.figure(1)
463 figure.clear()
464 plot.xscale('log', base=10.0)
465 plot.yscale('log', base=10.0)
466 plot.xlabel('Flat Field Signal [e-?]')
467 plot.ylabel('Serial CTI')
468 plot.title(ampName)
469 plot.plot(signal, test)
471 figure.show()
472 prompt = "Press Enter or c to continue [chp]..."
473 while True:
474 ans = input(prompt).lower()
475 if ans in ("", " ", "c",):
476 break
477 elif ans in ("p", ):
478 import pdb
479 pdb.set_trace()
480 elif ans in ('x', ):
481 self.allowDebug = False
482 break
483 elif ans in ("h", ):
484 print("[h]elp [c]ontinue [p]db e[x]itDebug")
485 plot.close()
487 def findTraps(self, inputMeasurements, calib, detector):
488 """Solve for serial trap parameters.
490 Parameters
491 ----------
492 inputMeasurements : `list` [`dict`]
493 List of overscan measurements from each input exposure.
494 Each dictionary is nested within a top level 'CTI' key,
495 with measurements organized by amplifier name, containing
496 keys:
498 ``"FIRST_MEAN"``
499 Mean value of first image column (`float`).
500 ``"LAST_MEAN"``
501 Mean value of last image column (`float`).
502 ``"IMAGE_MEAN"``
503 Mean value of the entire image region (`float`).
504 ``"OVERSCAN_COLUMNS"``
505 List of overscan column indicies (`list` [`int`]).
506 ``"OVERSCAN_VALUES"``
507 List of overscan column means (`list` [`float`]).
508 calib : `lsst.ip.isr.DeferredChargeCalib`
509 Calibration to populate with values.
510 detector : `lsst.afw.cameraGeom.Detector`
511 Detector object containing the geometry information for
512 the amplifiers.
514 Returns
515 -------
516 calib : `lsst.ip.isr.DeferredChargeCalib`
517 Populated calibration.
519 Raises
520 ------
521 RuntimeError
522 Raised if no data remains after flux filtering.
524 Notes
525 -----
526 The original CTISIM code uses a data model in which the
527 "overscan" consists of the standard serial overscan bbox with
528 the values for the last imaging data column prepended to that
529 list. This version of the code keeps the overscan and imaging
530 sections separate, and so a -1 offset is needed to ensure that
531 the same columns are used for fitting between this code and
532 CTISIM. This offset removes that last imaging data column
533 from the count.
534 """
535 # Range to fit. These are in "camera" coordinates, and so
536 # need to have the count for last image column removed.
537 start, stop = self.config.trapColumnRange
538 start -= 1
539 stop -= 1
541 # Loop over amps/inputs, fitting those columns from
542 # "non-saturated" inputs.
543 for amp in detector.getAmplifiers():
544 ampName = amp.getName()
546 # Number of serial shifts.
547 nCols = amp.getRawDataBBox().getWidth() + amp.getRawSerialPrescanBBox().getWidth()
549 # The signal is the mean intensity of each input, and the
550 # data are the overscan columns to fit. The new_signal is
551 # the mean in the last image column. Any serial trap will
552 # take charge from this column, and deposit it into the
553 # overscan columns.
554 signal = []
555 data = []
556 new_signal = []
557 Nskipped = 0
558 for exposureEntry in inputMeasurements:
559 exposureDict = exposureEntry['CTI']
560 if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxImageMean:
561 signal.append(exposureDict[ampName]['IMAGE_MEAN'])
562 data.append(exposureDict[ampName]['OVERSCAN_VALUES'][start:stop+1])
563 new_signal.append(exposureDict[ampName]['LAST_MEAN'])
564 else:
565 Nskipped += 1
566 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxImageMean}.")
567 if len(signal) == 0 or len(data) == 0:
568 raise RuntimeError("All exposures brighter than config.maxImageMean and excluded.")
570 signal = np.array(signal)
571 data = np.array(data)
572 new_signal = np.array(new_signal)
574 ind = signal.argsort()
575 signal = signal[ind]
576 data = data[ind]
577 new_signal = new_signal[ind]
579 # In the absense of any trap, the model results using the
580 # parameters already determined will match the observed
581 # overscan results.
582 params = Parameters()
583 params.add('ctiexp', value=np.log10(calib.globalCti[ampName]),
584 min=-7, max=-5, vary=False)
585 params.add('trapsize', value=0.0, min=0.0, max=10., vary=False)
586 params.add('scaling', value=0.08, min=0.0, max=1.0, vary=False)
587 params.add('emissiontime', value=0.35, min=0.1, max=1.0, vary=False)
588 params.add('driftscale', value=calib.driftScale[ampName],
589 min=0.0, max=0.001, vary=False)
590 params.add('decaytime', value=calib.decayTime[ampName],
591 min=0.1, max=4.0, vary=False)
593 model = SimpleModel.model_results(params, signal, nCols,
594 start=start, stop=stop)
596 # Evaluating trap: the difference between the model and
597 # observed data.
598 res = np.sum((data-model)[:, :3], axis=1)
600 # Create spline model for the trap, using the residual
601 # between data and model as a function of the last image
602 # column mean (new_signal) scaled by (1 - A_L).
603 # Note that this ``spline`` model is actually a piecewise
604 # linear interpolation and not a true spline.
605 new_signal = np.asarray((1 - calib.driftScale[ampName])*new_signal, dtype=np.float64)
606 x = new_signal
607 y = np.maximum(0, res)
609 # Pad left with ramp
610 y = np.pad(y, (10, 0), 'linear_ramp', end_values=(0, 0))
611 x = np.pad(x, (10, 0), 'linear_ramp', end_values=(0, 0))
613 trap = SerialTrap(20000.0, 0.4, 1, 'spline', np.concatenate((x, y)).tolist())
614 calib.serialTraps[ampName] = trap
616 return calib
619class OverscanModel:
620 """Base class for handling model/data fit comparisons.
622 This handles all of the methods needed for the lmfit Minimizer to
623 run.
624 """
626 @staticmethod
627 def model_results(params, signal, num_transfers, start=1, stop=10):
628 """Generate a realization of the overscan model, using the specified
629 fit parameters and input signal.
631 Parameters
632 ----------
633 params : `lmfit.Parameters`
634 Object containing the model parameters.
635 signal : `np.ndarray`, (nMeasurements)
636 Array of image means.
637 num_transfers : `int`
638 Number of serial transfers that the charge undergoes.
639 start : `int`, optional
640 First overscan column to fit. This number includes the
641 last imaging column, and needs to be adjusted by one when
642 using the overscan bounding box.
643 stop : `int`, optional
644 Last overscan column to fit. This number includes the
645 last imaging column, and needs to be adjusted by one when
646 using the overscan bounding box.
648 Returns
649 -------
650 results : `np.ndarray`, (nMeasurements, nCols)
651 Model results.
652 """
653 raise NotImplementedError("Subclasses must implement the model calculation.")
655 def loglikelihood(self, params, signal, data, error, *args, **kwargs):
656 """Calculate log likelihood of the model.
658 Parameters
659 ----------
660 params : `lmfit.Parameters`
661 Object containing the model parameters.
662 signal : `np.ndarray`, (nMeasurements)
663 Array of image means.
664 data : `np.ndarray`, (nMeasurements, nCols)
665 Array of overscan column means from each measurement.
666 error : `float`
667 Fixed error value.
668 *args :
669 Additional position arguments.
670 **kwargs :
671 Additional keyword arguments.
673 Returns
674 -------
675 logL : `float`
676 The log-likelihood of the observed data given the model
677 parameters.
678 """
679 model_results = self.model_results(params, signal, *args, **kwargs)
681 inv_sigma2 = 1.0/(error**2.0)
682 diff = model_results - data
684 return -0.5*(np.sum(inv_sigma2*(diff)**2.))
686 def negative_loglikelihood(self, params, signal, data, error, *args, **kwargs):
687 """Calculate negative log likelihood of the model.
689 Parameters
690 ----------
691 params : `lmfit.Parameters`
692 Object containing the model parameters.
693 signal : `np.ndarray`, (nMeasurements)
694 Array of image means.
695 data : `np.ndarray`, (nMeasurements, nCols)
696 Array of overscan column means from each measurement.
697 error : `float`
698 Fixed error value.
699 *args :
700 Additional position arguments.
701 **kwargs :
702 Additional keyword arguments.
704 Returns
705 -------
706 negativelogL : `float`
707 The negative log-likelihood of the observed data given the
708 model parameters.
709 """
710 ll = self.loglikelihood(params, signal, data, error, *args, **kwargs)
712 return -ll
714 def rms_error(self, params, signal, data, error, *args, **kwargs):
715 """Calculate RMS error between model and data.
717 Parameters
718 ----------
719 params : `lmfit.Parameters`
720 Object containing the model parameters.
721 signal : `np.ndarray`, (nMeasurements)
722 Array of image means.
723 data : `np.ndarray`, (nMeasurements, nCols)
724 Array of overscan column means from each measurement.
725 error : `float`
726 Fixed error value.
727 *args :
728 Additional position arguments.
729 **kwargs :
730 Additional keyword arguments.
732 Returns
733 -------
734 rms : `float`
735 The rms error between the model and input data.
736 """
737 model_results = self.model_results(params, signal, *args, **kwargs)
739 diff = model_results - data
740 rms = np.sqrt(np.mean(np.square(diff)))
742 return rms
744 def difference(self, params, signal, data, error, *args, **kwargs):
745 """Calculate the flattened difference array between model and data.
747 Parameters
748 ----------
749 params : `lmfit.Parameters`
750 Object containing the model parameters.
751 signal : `np.ndarray`, (nMeasurements)
752 Array of image means.
753 data : `np.ndarray`, (nMeasurements, nCols)
754 Array of overscan column means from each measurement.
755 error : `float`
756 Fixed error value.
757 *args :
758 Additional position arguments.
759 **kwargs :
760 Additional keyword arguments.
762 Returns
763 -------
764 difference : `np.ndarray`, (nMeasurements*nCols)
765 The rms error between the model and input data.
766 """
767 model_results = self.model_results(params, signal, *args, **kwargs)
768 diff = (model_results-data).flatten()
770 return diff
773class SimpleModel(OverscanModel):
774 """Simple analytic overscan model."""
776 @staticmethod
777 def model_results(params, signal, num_transfers, start=1, stop=10):
778 """Generate a realization of the overscan model, using the specified
779 fit parameters and input signal.
781 Parameters
782 ----------
783 params : `lmfit.Parameters`
784 Object containing the model parameters.
785 signal : `np.ndarray`, (nMeasurements)
786 Array of image means.
787 num_transfers : `int`
788 Number of serial transfers that the charge undergoes.
789 start : `int`, optional
790 First overscan column to fit. This number includes the
791 last imaging column, and needs to be adjusted by one when
792 using the overscan bounding box.
793 stop : `int`, optional
794 Last overscan column to fit. This number includes the
795 last imaging column, and needs to be adjusted by one when
796 using the overscan bounding box.
798 Returns
799 -------
800 res : `np.ndarray`, (nMeasurements, nCols)
801 Model results.
802 """
803 v = params.valuesdict()
804 v['cti'] = 10**v['ctiexp']
806 # Adjust column numbering to match DM overscan bbox.
807 start += 1
808 stop += 1
810 x = np.arange(start, stop+1)
811 res = np.zeros((signal.shape[0], x.shape[0]))
813 for i, s in enumerate(signal):
814 # This is largely equivalent to equation 2. The minimum
815 # indicates that a trap cannot emit more charge than is
816 # available, nor can it emit more charge than it can hold.
817 # This scales the exponential release of charge from the
818 # trap. The next term defines the contribution from the
819 # global CTI at each pixel transfer, and the final term
820 # includes the contribution from local CTI effects.
821 res[i, :] = (np.minimum(v['trapsize'], s*v['scaling'])
822 * (np.exp(1/v['emissiontime']) - 1.0)
823 * np.exp(-x/v['emissiontime'])
824 + s*num_transfers*v['cti']**x
825 + v['driftscale']*s*np.exp(-x/float(v['decaytime'])))
827 return res
830class SimulatedModel(OverscanModel):
831 """Simulated overscan model."""
833 @staticmethod
834 def model_results(params, signal, num_transfers, amp, start=1, stop=10, trap_type=None):
835 """Generate a realization of the overscan model, using the specified
836 fit parameters and input signal.
838 Parameters
839 ----------
840 params : `lmfit.Parameters`
841 Object containing the model parameters.
842 signal : `np.ndarray`, (nMeasurements)
843 Array of image means.
844 num_transfers : `int`
845 Number of serial transfers that the charge undergoes.
846 amp : `lsst.afw.cameraGeom.Amplifier`
847 Amplifier to use for geometry information.
848 start : `int`, optional
849 First overscan column to fit. This number includes the
850 last imaging column, and needs to be adjusted by one when
851 using the overscan bounding box.
852 stop : `int`, optional
853 Last overscan column to fit. This number includes the
854 last imaging column, and needs to be adjusted by one when
855 using the overscan bounding box.
856 trap_type : `str`, optional
857 Type of trap model to use.
859 Returns
860 -------
861 results : `np.ndarray`, (nMeasurements, nCols)
862 Model results.
863 """
864 v = params.valuesdict()
866 # Adjust column numbering to match DM overscan bbox.
867 start += 1
868 stop += 1
870 # Electronics effect optimization
871 output_amplifier = FloatingOutputAmplifier(1.0, v['driftscale'], v['decaytime'])
873 # CTI optimization
874 v['cti'] = 10**v['ctiexp']
876 # Trap type for optimization
877 if trap_type is None:
878 trap = None
879 elif trap_type == 'linear':
880 trap = SerialTrap(v['trapsize'], v['emissiontime'], 1, 'linear',
881 [v['scaling']])
882 elif trap_type == 'logistic':
883 trap = SerialTrap(v['trapsize'], v['emissiontime'], 1, 'logistic',
884 [v['f0'], v['k']])
885 else:
886 raise ValueError('Trap type must be linear or logistic or None')
888 # Simulate ramp readout
889 imarr = np.zeros((signal.shape[0], amp.getRawDataBBox().getWidth()))
890 ramp = SegmentSimulator(imarr, amp.getRawSerialPrescanBBox().getWidth(), output_amplifier,
891 cti=v['cti'], traps=trap)
892 ramp.ramp_exp(signal)
893 model_results = ramp.readout(serial_overscan_width=amp.getRawSerialOverscanBBox().getWidth(),
894 parallel_overscan_width=0)
896 ncols = amp.getRawSerialPrescanBBox().getWidth() + amp.getRawDataBBox().getWidth()
898 return model_results[:, ncols+start-1:ncols+stop]
901class SegmentSimulator:
902 """Controls the creation of simulated segment images.
904 Parameters
905 ----------
906 imarr : `np.ndarray` (nx, ny)
907 Image data array.
908 prescan_width : `int`
909 Number of serial prescan columns.
910 output_amplifier : `lsst.cp.pipe.FloatingOutputAmplifier`
911 An object holding the gain, read noise, and global_offset.
912 cti : `float`
913 Global CTI value.
914 traps : `list` [`lsst.ip.isr.SerialTrap`]
915 Serial traps to simulate.
916 """
918 def __init__(self, imarr, prescan_width, output_amplifier, cti=0.0, traps=None):
919 # Image array geometry
920 self.prescan_width = prescan_width
921 self.ny, self.nx = imarr.shape
923 self.segarr = np.zeros((self.ny, self.nx+prescan_width))
924 self.segarr[:, prescan_width:] = imarr
926 # Serial readout information
927 self.output_amplifier = output_amplifier
928 if isinstance(cti, np.ndarray):
929 raise ValueError("cti must be single value, not an array.")
930 self.cti = cti
932 self.serial_traps = None
933 self.do_trapping = False
934 if traps is not None:
935 if not isinstance(traps, list):
936 traps = [traps]
937 for trap in traps:
938 self.add_trap(trap)
940 def add_trap(self, serial_trap):
941 """Add a trap to the serial register.
943 Parameters
944 ----------
945 serial_trap : `lsst.ip.isr.SerialTrap`
946 The trap to add.
947 """
948 try:
949 self.serial_traps.append(serial_trap)
950 except AttributeError:
951 self.serial_traps = [serial_trap]
952 self.do_trapping = True
954 def ramp_exp(self, signal_list):
955 """Simulate an image with varying flux illumination per row.
957 This method simulates a segment image where the signal level
958 increases along the horizontal direction, according to the
959 provided list of signal levels.
961 Parameters
962 ----------
963 signal_list : `list` [`float`]
964 List of signal levels.
966 Raises
967 ------
968 ValueError
969 Raised if the length of the signal list does not equal the
970 number of rows.
971 """
972 if len(signal_list) != self.ny:
973 raise ValueError("Signal list does not match row count.")
975 ramp = np.tile(signal_list, (self.nx, 1)).T
976 self.segarr[:, self.prescan_width:] += ramp
978 def readout(self, serial_overscan_width=10, parallel_overscan_width=0):
979 """Simulate serial readout of the segment image.
981 This method performs the serial readout of a segment image
982 given the appropriate SerialRegister object and the properties
983 of the ReadoutAmplifier. Additional arguments can be provided
984 to account for the number of desired overscan transfers. The
985 result is a simulated final segment image, in ADU.
987 Parameters
988 ----------
989 serial_overscan_width : `int`, optional
990 Number of serial overscan columns.
991 parallel_overscan_width : `int`, optional
992 Number of parallel overscan rows.
994 Returns
995 -------
996 result : `np.ndarray` (nx, ny)
997 Simulated image, including serial prescan, serial
998 overscan, and parallel overscan regions.
999 """
1000 # Create output array
1001 iy = int(self.ny + parallel_overscan_width)
1002 ix = int(self.nx + self.prescan_width + serial_overscan_width)
1003 image = np.random.normal(loc=self.output_amplifier.global_offset,
1004 scale=self.output_amplifier.noise,
1005 size=(iy, ix))
1006 free_charge = copy.deepcopy(self.segarr)
1008 # Set flow control parameters
1009 do_trapping = self.do_trapping
1010 cti = self.cti
1012 offset = np.zeros(self.ny)
1013 cte = 1 - cti
1014 if do_trapping:
1015 for trap in self.serial_traps:
1016 trap.initialize(self.ny, self.nx, self.prescan_width)
1018 for i in range(ix):
1019 # Trap capture
1020 if do_trapping:
1021 for trap in self.serial_traps:
1022 captured_charge = trap.trap_charge(free_charge)
1023 free_charge -= captured_charge
1025 # Pixel-to-pixel proportional loss
1026 transferred_charge = free_charge*cte
1027 deferred_charge = free_charge*cti
1029 # Pixel transfer and readout
1030 offset = self.output_amplifier.local_offset(offset,
1031 transferred_charge[:, 0])
1032 image[:iy-parallel_overscan_width, i] += transferred_charge[:, 0] + offset
1034 free_charge = np.pad(transferred_charge, ((0, 0), (0, 1)),
1035 mode='constant')[:, 1:] + deferred_charge
1037 # Trap emission
1038 if do_trapping:
1039 for trap in self.serial_traps:
1040 released_charge = trap.release_charge()
1041 free_charge += released_charge
1043 return image/float(self.output_amplifier.gain)
1046class FloatingOutputAmplifier:
1047 """Object representing the readout amplifier of a single channel.
1049 Parameters
1050 ----------
1051 gain : `float`
1052 Amplifier gain.
1053 scale : `float`
1054 Drift scale for the amplifier.
1055 decay_time : `float`
1056 Decay time for the bias drift.
1057 noise : `float`, optional
1058 Amplifier read noise.
1059 offset : `float`, optional
1060 Global CTI offset.
1061 """
1063 def __init__(self, gain, scale, decay_time, noise=0.0, offset=0.0):
1065 self.gain = gain
1066 self.noise = noise
1067 self.global_offset = offset
1069 self.update_parameters(scale, decay_time)
1071 def local_offset(self, old, signal):
1072 """Calculate local offset hysteresis.
1074 Parameters
1075 ----------
1076 old : `np.ndarray`, (,)
1077 Previous iteration.
1078 signal : `np.ndarray`, (,)
1079 Current column measurements.
1081 Returns
1082 -------
1083 offset : `np.ndarray`
1084 Local offset.
1085 """
1086 new = self.scale*signal
1088 return np.maximum(new, old*np.exp(-1/self.decay_time))
1090 def update_parameters(self, scale, decay_time):
1091 """Update parameter values, if within acceptable values.
1093 Parameters
1094 ----------
1095 scale : `float`
1096 Drift scale for the amplifier.
1097 decay_time : `float`
1098 Decay time for the bias drift.
1100 Raises
1101 ------
1102 ValueError
1103 Raised if the input parameters are out of range.
1104 """
1105 if scale < 0.0:
1106 raise ValueError("Scale must be greater than or equal to 0.")
1107 if np.isnan(scale):
1108 raise ValueError("Scale must be real-valued number, not NaN.")
1109 self.scale = scale
1110 if decay_time <= 0.0:
1111 raise ValueError("Decay time must be greater than 0.")
1112 if np.isnan(decay_time):
1113 raise ValueError("Decay time must be real-valued number, not NaN.")
1114 self.decay_time = decay_time