Coverage for python/lsst/cp/pipe/deferredCharge.py: 14%

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22__all__ = ('CpCtiSolveConnections', 

23 'CpCtiSolveConfig', 

24 'CpCtiSolveTask', 

25 'OverscanModel', 

26 'SimpleModel', 

27 'SimulatedModel', 

28 'SegmentSimulator', 

29 'FloatingOutputAmplifier', 

30 ) 

31 

32import copy 

33import numpy as np 

34 

35import lsst.pipe.base as pipeBase 

36import lsst.pipe.base.connectionTypes as cT 

37import lsst.pex.config as pexConfig 

38 

39from lsst.ip.isr import DeferredChargeCalib, SerialTrap 

40from lmfit import Minimizer, Parameters 

41 

42 

43class CpCtiSolveConnections(pipeBase.PipelineTaskConnections, 

44 dimensions=("instrument", "detector")): 

45 inputMeasurements = cT.Input( 

46 name="cpCtiMeas", 

47 doc="Input overscan measurements to fit.", 

48 storageClass='StructuredDataDict', 

49 dimensions=("instrument", "exposure", "detector"), 

50 multiple=True, 

51 ) 

52 camera = cT.PrerequisiteInput( 

53 name="camera", 

54 doc="Camera geometry to use.", 

55 storageClass="Camera", 

56 dimensions=("instrument", ), 

57 isCalibration=True, 

58 ) 

59 

60 outputCalib = cT.Output( 

61 name="cpCtiCalib", 

62 doc="Output CTI calibration.", 

63 storageClass="IsrCalib", 

64 dimensions=("instrument", "detector"), 

65 isCalibration=True, 

66 ) 

67 

68 

69class CpCtiSolveConfig(pipeBase.PipelineTaskConfig, 

70 pipelineConnections=CpCtiSolveConnections): 

71 """Configuration for the CTI combination. 

72 """ 

73 maxImageMean = pexConfig.Field( 

74 dtype=float, 

75 default=150000.0, 

76 doc="Upper limit on acceptable image flux mean.", 

77 ) 

78 localOffsetColumnRange = pexConfig.ListField( 

79 dtype=int, 

80 default=[3, 13], 

81 doc="First and last overscan column to use for local offset effect.", 

82 ) 

83 

84 maxSignalForCti = pexConfig.Field( 

85 dtype=float, 

86 default=10000.0, 

87 doc="Upper flux limit to use for CTI fit.", 

88 ) 

89 globalCtiColumnRange = pexConfig.ListField( 

90 dtype=int, 

91 default=[1, 2], 

92 doc="First and last overscan column to use for global CTI fit.", 

93 ) 

94 

95 trapColumnRange = pexConfig.ListField( 

96 dtype=int, 

97 default=[1, 20], 

98 doc="First and last overscan column to use for serial trap fit.", 

99 ) 

100 

101 fitError = pexConfig.Field( 

102 # This gives the error on the mean in a given column, and so 

103 # is expected to be $RN / sqrt(N_rows)$. 

104 dtype=float, 

105 default=7.0/np.sqrt(2000), 

106 doc="Error to use during parameter fitting.", 

107 ) 

108 

109 

110class CpCtiSolveTask(pipeBase.PipelineTask): 

111 """Combine CTI measurements to a final calibration. 

112 

113 This task uses the extended pixel edge response (EPER) method as 

114 described by Snyder et al. 2021, Journal of Astronimcal 

115 Telescopes, Instruments, and Systems, 7, 

116 048002. doi:10.1117/1.JATIS.7.4.048002 

117 """ 

118 

119 ConfigClass = CpCtiSolveConfig 

120 _DefaultName = 'cpCtiSolve' 

121 

122 def __init__(self, **kwargs): 

123 super().__init__(**kwargs) 

124 self.allowDebug = True 

125 

126 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

127 inputs = butlerQC.get(inputRefs) 

128 

129 dimensions = [dict(exp.dataId.required) for exp in inputRefs.inputMeasurements] 

130 inputs['inputDims'] = dimensions 

131 

132 outputs = self.run(**inputs) 

133 butlerQC.put(outputs, outputRefs) 

134 

135 def run(self, inputMeasurements, camera, inputDims): 

136 """Solve for charge transfer inefficiency from overscan measurements. 

137 

138 Parameters 

139 ---------- 

140 inputMeasurements : `list` [`dict`] 

141 List of overscan measurements from each input exposure. 

142 Each dictionary is nested within a top level 'CTI' key, 

143 with measurements organized by amplifier name, containing 

144 keys: 

145 

146 ``"FIRST_MEAN"`` 

147 Mean value of first image column (`float`). 

148 ``"LAST_MEAN"`` 

149 Mean value of last image column (`float`). 

150 ``"IMAGE_MEAN"`` 

151 Mean value of the entire image region (`float`). 

152 ``"OVERSCAN_COLUMNS"`` 

153 List of overscan column indicies (`list` [`int`]). 

154 ``"OVERSCAN_VALUES"`` 

155 List of overscan column means (`list` [`float`]). 

156 camera : `lsst.afw.cameraGeom.Camera` 

157 Camera geometry to use to find detectors. 

158 inputDims : `list` [`dict`] 

159 List of input dimensions from each input exposure. 

160 

161 Returns 

162 ------- 

163 results : `lsst.pipe.base.Struct` 

164 Result struct containing: 

165 

166 ``outputCalib`` 

167 Final CTI calibration data 

168 (`lsst.ip.isr.DeferredChargeCalib`). 

169 

170 Raises 

171 ------ 

172 RuntimeError 

173 Raised if data from multiple detectors are passed in. 

174 """ 

175 detectorSet = set([d['detector'] for d in inputDims]) 

176 if len(detectorSet) != 1: 

177 raise RuntimeError("Inputs for too many detectors passed.") 

178 detectorId = detectorSet.pop() 

179 detector = camera[detectorId] 

180 

181 # Initialize with detector. 

182 calib = DeferredChargeCalib(camera=camera, detector=detector) 

183 

184 localCalib = self.solveLocalOffsets(inputMeasurements, calib, detector) 

185 

186 globalCalib = self.solveGlobalCti(inputMeasurements, localCalib, detector) 

187 

188 finalCalib = self.findTraps(inputMeasurements, globalCalib, detector) 

189 

190 return pipeBase.Struct( 

191 outputCalib=finalCalib, 

192 ) 

193 

194 def solveLocalOffsets(self, inputMeasurements, calib, detector): 

195 """Solve for local (pixel-to-pixel) electronic offsets. 

196 

197 This method fits for \tau_L, the local electronic offset decay 

198 time constant, and A_L, the local electronic offset constant 

199 of proportionality. 

200 

201 Parameters 

202 ---------- 

203 inputMeasurements : `list` [`dict`] 

204 List of overscan measurements from each input exposure. 

205 Each dictionary is nested within a top level 'CTI' key, 

206 with measurements organized by amplifier name, containing 

207 keys: 

208 

209 ``"FIRST_MEAN"`` 

210 Mean value of first image column (`float`). 

211 ``"LAST_MEAN"`` 

212 Mean value of last image column (`float`). 

213 ``"IMAGE_MEAN"`` 

214 Mean value of the entire image region (`float`). 

215 ``"OVERSCAN_COLUMNS"`` 

216 List of overscan column indicies (`list` [`int`]). 

217 ``"OVERSCAN_VALUES"`` 

218 List of overscan column means (`list` [`float`]). 

219 calib : `lsst.ip.isr.DeferredChargeCalib` 

220 Calibration to populate with values. 

221 detector : `lsst.afw.cameraGeom.Detector` 

222 Detector object containing the geometry information for 

223 the amplifiers. 

224 

225 Returns 

226 ------- 

227 calib : `lsst.ip.isr.DeferredChargeCalib` 

228 Populated calibration. 

229 

230 Raises 

231 ------ 

232 RuntimeError 

233 Raised if no data remains after flux filtering. 

234 

235 Notes 

236 ----- 

237 The original CTISIM code (https://github.com/Snyder005/ctisim) 

238 uses a data model in which the "overscan" consists of the 

239 standard serial overscan bbox with the values for the last 

240 imaging data column prepended to that list. This version of 

241 the code keeps the overscan and imaging sections separate, and 

242 so a -1 offset is needed to ensure that the same columns are 

243 used for fitting between this code and CTISIM. This offset 

244 removes that last imaging data column from the count. 

245 """ 

246 # Range to fit. These are in "camera" coordinates, and so 

247 # need to have the count for last image column removed. 

248 start, stop = self.config.localOffsetColumnRange 

249 start -= 1 

250 stop -= 1 

251 

252 # Loop over amps/inputs, fitting those columns from 

253 # "non-saturated" inputs. 

254 for amp in detector.getAmplifiers(): 

255 ampName = amp.getName() 

256 

257 # Number of serial shifts. 

258 nCols = amp.getRawDataBBox().getWidth() + amp.getRawSerialPrescanBBox().getWidth() 

259 

260 # The signal is the mean intensity of each input, and the 

261 # data are the overscan columns to fit. For detectors 

262 # with non-zero CTI, the charge from the imaging region 

263 # leaks into the overscan region. 

264 signal = [] 

265 data = [] 

266 Nskipped = 0 

267 for exposureEntry in inputMeasurements: 

268 exposureDict = exposureEntry['CTI'] 

269 if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxImageMean: 

270 signal.append(exposureDict[ampName]['IMAGE_MEAN']) 

271 data.append(exposureDict[ampName]['OVERSCAN_VALUES'][start:stop+1]) 

272 else: 

273 Nskipped += 1 

274 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxImageMean}.") 

275 if len(signal) == 0 or len(data) == 0: 

276 raise RuntimeError("All exposures brighter than config.maxImageMean and excluded.") 

277 

278 signal = np.array(signal) 

279 data = np.array(data) 

280 

281 ind = signal.argsort() 

282 signal = signal[ind] 

283 data = data[ind] 

284 

285 params = Parameters() 

286 params.add('ctiexp', value=-6, min=-7, max=-5, vary=False) 

287 params.add('trapsize', value=0.0, min=0.0, max=10., vary=False) 

288 params.add('scaling', value=0.08, min=0.0, max=1.0, vary=False) 

289 params.add('emissiontime', value=0.4, min=0.1, max=1.0, vary=False) 

290 params.add('driftscale', value=0.00022, min=0., max=0.001, vary=True) 

291 params.add('decaytime', value=2.4, min=0.1, max=4.0, vary=True) 

292 

293 model = SimpleModel() 

294 minner = Minimizer(model.difference, params, 

295 fcn_args=(signal, data, self.config.fitError, nCols), 

296 fcn_kws={'start': start, 'stop': stop}) 

297 result = minner.minimize() 

298 

299 # Save results for the drift scale and decay time. 

300 if not result.success: 

301 self.log.warning("Electronics fitting failure for amplifier %s.", ampName) 

302 

303 calib.globalCti[ampName] = 10**result.params['ctiexp'] 

304 calib.driftScale[ampName] = result.params['driftscale'].value if result.success else 0.0 

305 calib.decayTime[ampName] = result.params['decaytime'].value if result.success else 2.4 

306 self.log.info("CTI Local Fit %s: cti: %g decayTime: %g driftScale %g", 

307 ampName, calib.globalCti[ampName], calib.decayTime[ampName], 

308 calib.driftScale[ampName]) 

309 return calib 

310 

311 def solveGlobalCti(self, inputMeasurements, calib, detector): 

312 """Solve for global CTI constant. 

313 

314 This method solves for the mean global CTI, b. 

315 

316 Parameters 

317 ---------- 

318 inputMeasurements : `list` [`dict`] 

319 List of overscan measurements from each input exposure. 

320 Each dictionary is nested within a top level 'CTI' key, 

321 with measurements organized by amplifier name, containing 

322 keys: 

323 

324 ``"FIRST_MEAN"`` 

325 Mean value of first image column (`float`). 

326 ``"LAST_MEAN"`` 

327 Mean value of last image column (`float`). 

328 ``"IMAGE_MEAN"`` 

329 Mean value of the entire image region (`float`). 

330 ``"OVERSCAN_COLUMNS"`` 

331 List of overscan column indicies (`list` [`int`]). 

332 ``"OVERSCAN_VALUES"`` 

333 List of overscan column means (`list` [`float`]). 

334 calib : `lsst.ip.isr.DeferredChargeCalib` 

335 Calibration to populate with values. 

336 detector : `lsst.afw.cameraGeom.Detector` 

337 Detector object containing the geometry information for 

338 the amplifiers. 

339 

340 Returns 

341 ------- 

342 calib : `lsst.ip.isr.DeferredChargeCalib` 

343 Populated calibration. 

344 

345 Raises 

346 ------ 

347 RuntimeError 

348 Raised if no data remains after flux filtering. 

349 

350 Notes 

351 ----- 

352 The original CTISIM code uses a data model in which the 

353 "overscan" consists of the standard serial overscan bbox with 

354 the values for the last imaging data column prepended to that 

355 list. This version of the code keeps the overscan and imaging 

356 sections separate, and so a -1 offset is needed to ensure that 

357 the same columns are used for fitting between this code and 

358 CTISIM. This offset removes that last imaging data column 

359 from the count. 

360 """ 

361 # Range to fit. These are in "camera" coordinates, and so 

362 # need to have the count for last image column removed. 

363 start, stop = self.config.globalCtiColumnRange 

364 start -= 1 

365 stop -= 1 

366 

367 # Loop over amps/inputs, fitting those columns from 

368 # "non-saturated" inputs. 

369 for amp in detector.getAmplifiers(): 

370 ampName = amp.getName() 

371 

372 # Number of serial shifts. 

373 nCols = amp.getRawDataBBox().getWidth() + amp.getRawSerialPrescanBBox().getWidth() 

374 

375 # The signal is the mean intensity of each input, and the 

376 # data are the overscan columns to fit. For detectors 

377 # with non-zero CTI, the charge from the imaging region 

378 # leaks into the overscan region. 

379 signal = [] 

380 data = [] 

381 Nskipped = 0 

382 for exposureEntry in inputMeasurements: 

383 exposureDict = exposureEntry['CTI'] 

384 if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxSignalForCti: 

385 signal.append(exposureDict[ampName]['IMAGE_MEAN']) 

386 data.append(exposureDict[ampName]['OVERSCAN_VALUES'][start:stop+1]) 

387 else: 

388 Nskipped += 1 

389 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxSignalForCti}.") 

390 if len(signal) == 0 or len(data) == 0: 

391 raise RuntimeError("All exposures brighter than config.maxSignalForCti and excluded.") 

392 

393 signal = np.array(signal) 

394 data = np.array(data) 

395 

396 ind = signal.argsort() 

397 signal = signal[ind] 

398 data = data[ind] 

399 

400 # CTI test. This looks at the charge that has leaked into 

401 # the first few columns of the overscan. 

402 overscan1 = data[:, 0] 

403 overscan2 = data[:, 1] 

404 test = (np.array(overscan1) + np.array(overscan2))/(nCols*np.array(signal)) 

405 testResult = np.median(test) > 5.E-6 

406 self.log.info("Estimate of CTI test is %f for amp %s, %s.", np.median(test), ampName, 

407 "full fitting will be performed" if testResult else 

408 "only global CTI fitting will be performed") 

409 

410 self.debugView(ampName, signal, test) 

411 

412 params = Parameters() 

413 params.add('ctiexp', value=-6, min=-7, max=-5, vary=True) 

414 params.add('trapsize', value=5.0 if testResult else 0.0, min=0.0, max=30., 

415 vary=True if testResult else False) 

416 params.add('scaling', value=0.08, min=0.0, max=1.0, 

417 vary=True if testResult else False) 

418 params.add('emissiontime', value=0.35, min=0.1, max=1.0, 

419 vary=True if testResult else False) 

420 params.add('driftscale', value=calib.driftScale[ampName], min=0., max=0.001, vary=False) 

421 params.add('decaytime', value=calib.decayTime[ampName], min=0.1, max=4.0, vary=False) 

422 

423 model = SimulatedModel() 

424 minner = Minimizer(model.difference, params, 

425 fcn_args=(signal, data, self.config.fitError, nCols, amp), 

426 fcn_kws={'start': start, 'stop': stop, 'trap_type': 'linear'}) 

427 result = minner.minimize() 

428 

429 # Only the global CTI term is retained from this fit. 

430 calib.globalCti[ampName] = 10**result.params['ctiexp'].value 

431 self.log.info("CTI Global Cti %s: cti: %g decayTime: %g driftScale %g", 

432 ampName, calib.globalCti[ampName], calib.decayTime[ampName], 

433 calib.driftScale[ampName]) 

434 

435 return calib 

436 

437 def debugView(self, ampName, signal, test): 

438 """Debug method for global CTI test value. 

439 

440 Parameters 

441 ---------- 

442 ampName : `str` 

443 Name of the amp for plot title. 

444 signal : `list` [`float`] 

445 Image means for the input exposures. 

446 test : `list` [`float`] 

447 CTI test value to plot. 

448 """ 

449 import lsstDebug 

450 if not lsstDebug.Info(__name__).display: 

451 return 

452 if not self.allowDebug: 

453 return 

454 

455 import matplotlib.pyplot as plot 

456 figure = plot.figure(1) 

457 figure.clear() 

458 plot.xscale('log', base=10.0) 

459 plot.yscale('log', base=10.0) 

460 plot.xlabel('Flat Field Signal [e-?]') 

461 plot.ylabel('Serial CTI') 

462 plot.title(ampName) 

463 plot.plot(signal, test) 

464 

465 figure.show() 

466 prompt = "Press Enter or c to continue [chp]..." 

467 while True: 

468 ans = input(prompt).lower() 

469 if ans in ("", " ", "c",): 

470 break 

471 elif ans in ("p", ): 

472 import pdb 

473 pdb.set_trace() 

474 elif ans in ('x', ): 

475 self.allowDebug = False 

476 break 

477 elif ans in ("h", ): 

478 print("[h]elp [c]ontinue [p]db e[x]itDebug") 

479 plot.close() 

480 

481 def findTraps(self, inputMeasurements, calib, detector): 

482 """Solve for serial trap parameters. 

483 

484 Parameters 

485 ---------- 

486 inputMeasurements : `list` [`dict`] 

487 List of overscan measurements from each input exposure. 

488 Each dictionary is nested within a top level 'CTI' key, 

489 with measurements organized by amplifier name, containing 

490 keys: 

491 

492 ``"FIRST_MEAN"`` 

493 Mean value of first image column (`float`). 

494 ``"LAST_MEAN"`` 

495 Mean value of last image column (`float`). 

496 ``"IMAGE_MEAN"`` 

497 Mean value of the entire image region (`float`). 

498 ``"OVERSCAN_COLUMNS"`` 

499 List of overscan column indicies (`list` [`int`]). 

500 ``"OVERSCAN_VALUES"`` 

501 List of overscan column means (`list` [`float`]). 

502 calib : `lsst.ip.isr.DeferredChargeCalib` 

503 Calibration to populate with values. 

504 detector : `lsst.afw.cameraGeom.Detector` 

505 Detector object containing the geometry information for 

506 the amplifiers. 

507 

508 Returns 

509 ------- 

510 calib : `lsst.ip.isr.DeferredChargeCalib` 

511 Populated calibration. 

512 

513 Raises 

514 ------ 

515 RuntimeError 

516 Raised if no data remains after flux filtering. 

517 

518 Notes 

519 ----- 

520 The original CTISIM code uses a data model in which the 

521 "overscan" consists of the standard serial overscan bbox with 

522 the values for the last imaging data column prepended to that 

523 list. This version of the code keeps the overscan and imaging 

524 sections separate, and so a -1 offset is needed to ensure that 

525 the same columns are used for fitting between this code and 

526 CTISIM. This offset removes that last imaging data column 

527 from the count. 

528 """ 

529 # Range to fit. These are in "camera" coordinates, and so 

530 # need to have the count for last image column removed. 

531 start, stop = self.config.trapColumnRange 

532 start -= 1 

533 stop -= 1 

534 

535 # Loop over amps/inputs, fitting those columns from 

536 # "non-saturated" inputs. 

537 for amp in detector.getAmplifiers(): 

538 ampName = amp.getName() 

539 

540 # Number of serial shifts. 

541 nCols = amp.getRawDataBBox().getWidth() + amp.getRawSerialPrescanBBox().getWidth() 

542 

543 # The signal is the mean intensity of each input, and the 

544 # data are the overscan columns to fit. The new_signal is 

545 # the mean in the last image column. Any serial trap will 

546 # take charge from this column, and deposit it into the 

547 # overscan columns. 

548 signal = [] 

549 data = [] 

550 new_signal = [] 

551 Nskipped = 0 

552 for exposureEntry in inputMeasurements: 

553 exposureDict = exposureEntry['CTI'] 

554 if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxImageMean: 

555 signal.append(exposureDict[ampName]['IMAGE_MEAN']) 

556 data.append(exposureDict[ampName]['OVERSCAN_VALUES'][start:stop+1]) 

557 new_signal.append(exposureDict[ampName]['LAST_MEAN']) 

558 else: 

559 Nskipped += 1 

560 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxImageMean}.") 

561 if len(signal) == 0 or len(data) == 0: 

562 raise RuntimeError("All exposures brighter than config.maxImageMean and excluded.") 

563 

564 signal = np.array(signal) 

565 data = np.array(data) 

566 new_signal = np.array(new_signal) 

567 

568 ind = signal.argsort() 

569 signal = signal[ind] 

570 data = data[ind] 

571 new_signal = new_signal[ind] 

572 

573 # In the absense of any trap, the model results using the 

574 # parameters already determined will match the observed 

575 # overscan results. 

576 params = Parameters() 

577 params.add('ctiexp', value=np.log10(calib.globalCti[ampName]), 

578 min=-7, max=-5, vary=False) 

579 params.add('trapsize', value=0.0, min=0.0, max=10., vary=False) 

580 params.add('scaling', value=0.08, min=0.0, max=1.0, vary=False) 

581 params.add('emissiontime', value=0.35, min=0.1, max=1.0, vary=False) 

582 params.add('driftscale', value=calib.driftScale[ampName], 

583 min=0.0, max=0.001, vary=False) 

584 params.add('decaytime', value=calib.decayTime[ampName], 

585 min=0.1, max=4.0, vary=False) 

586 

587 model = SimpleModel.model_results(params, signal, nCols, 

588 start=start, stop=stop) 

589 

590 # Evaluating trap: the difference between the model and 

591 # observed data. 

592 res = np.sum((data-model)[:, :3], axis=1) 

593 

594 # Create spline model for the trap, using the residual 

595 # between data and model as a function of the last image 

596 # column mean (new_signal) scaled by (1 - A_L). 

597 new_signal = np.asarray((1 - calib.driftScale[ampName])*new_signal, dtype=np.float64) 

598 x = new_signal 

599 y = np.maximum(0, res) 

600 

601 # Pad left with ramp 

602 y = np.pad(y, (10, 0), 'linear_ramp', end_values=(0, 0)) 

603 x = np.pad(x, (10, 0), 'linear_ramp', end_values=(0, 0)) 

604 

605 # Pad right with constant 

606 y = np.pad(y, (1, 1), 'constant', constant_values=(0, y[-1])) 

607 x = np.pad(x, (1, 1), 'constant', constant_values=(-1, 200000.)) 

608 

609 trap = SerialTrap(20000.0, 0.4, 1, 'spline', np.concatenate((x, y)).tolist()) 

610 calib.serialTraps[ampName] = trap 

611 

612 return calib 

613 

614 

615class OverscanModel: 

616 """Base class for handling model/data fit comparisons. 

617 

618 This handles all of the methods needed for the lmfit Minimizer to 

619 run. 

620 """ 

621 

622 @staticmethod 

623 def model_results(params, signal, num_transfers, start=1, stop=10): 

624 """Generate a realization of the overscan model, using the specified 

625 fit parameters and input signal. 

626 

627 Parameters 

628 ---------- 

629 params : `lmfit.Parameters` 

630 Object containing the model parameters. 

631 signal : `np.ndarray`, (nMeasurements) 

632 Array of image means. 

633 num_transfers : `int` 

634 Number of serial transfers that the charge undergoes. 

635 start : `int`, optional 

636 First overscan column to fit. This number includes the 

637 last imaging column, and needs to be adjusted by one when 

638 using the overscan bounding box. 

639 stop : `int`, optional 

640 Last overscan column to fit. This number includes the 

641 last imaging column, and needs to be adjusted by one when 

642 using the overscan bounding box. 

643 

644 Returns 

645 ------- 

646 results : `np.ndarray`, (nMeasurements, nCols) 

647 Model results. 

648 """ 

649 raise NotImplementedError("Subclasses must implement the model calculation.") 

650 

651 def loglikelihood(self, params, signal, data, error, *args, **kwargs): 

652 """Calculate log likelihood of the model. 

653 

654 Parameters 

655 ---------- 

656 params : `lmfit.Parameters` 

657 Object containing the model parameters. 

658 signal : `np.ndarray`, (nMeasurements) 

659 Array of image means. 

660 data : `np.ndarray`, (nMeasurements, nCols) 

661 Array of overscan column means from each measurement. 

662 error : `float` 

663 Fixed error value. 

664 *args : 

665 Additional position arguments. 

666 **kwargs : 

667 Additional keyword arguments. 

668 

669 Returns 

670 ------- 

671 logL : `float` 

672 The log-likelihood of the observed data given the model 

673 parameters. 

674 """ 

675 model_results = self.model_results(params, signal, *args, **kwargs) 

676 

677 inv_sigma2 = 1.0/(error**2.0) 

678 diff = model_results - data 

679 

680 return -0.5*(np.sum(inv_sigma2*(diff)**2.)) 

681 

682 def negative_loglikelihood(self, params, signal, data, error, *args, **kwargs): 

683 """Calculate negative log likelihood of the model. 

684 

685 Parameters 

686 ---------- 

687 params : `lmfit.Parameters` 

688 Object containing the model parameters. 

689 signal : `np.ndarray`, (nMeasurements) 

690 Array of image means. 

691 data : `np.ndarray`, (nMeasurements, nCols) 

692 Array of overscan column means from each measurement. 

693 error : `float` 

694 Fixed error value. 

695 *args : 

696 Additional position arguments. 

697 **kwargs : 

698 Additional keyword arguments. 

699 

700 Returns 

701 ------- 

702 negativelogL : `float` 

703 The negative log-likelihood of the observed data given the 

704 model parameters. 

705 """ 

706 ll = self.loglikelihood(params, signal, data, error, *args, **kwargs) 

707 

708 return -ll 

709 

710 def rms_error(self, params, signal, data, error, *args, **kwargs): 

711 """Calculate RMS error between model and data. 

712 

713 Parameters 

714 ---------- 

715 params : `lmfit.Parameters` 

716 Object containing the model parameters. 

717 signal : `np.ndarray`, (nMeasurements) 

718 Array of image means. 

719 data : `np.ndarray`, (nMeasurements, nCols) 

720 Array of overscan column means from each measurement. 

721 error : `float` 

722 Fixed error value. 

723 *args : 

724 Additional position arguments. 

725 **kwargs : 

726 Additional keyword arguments. 

727 

728 Returns 

729 ------- 

730 rms : `float` 

731 The rms error between the model and input data. 

732 """ 

733 model_results = self.model_results(params, signal, *args, **kwargs) 

734 

735 diff = model_results - data 

736 rms = np.sqrt(np.mean(np.square(diff))) 

737 

738 return rms 

739 

740 def difference(self, params, signal, data, error, *args, **kwargs): 

741 """Calculate the flattened difference array between model and data. 

742 

743 Parameters 

744 ---------- 

745 params : `lmfit.Parameters` 

746 Object containing the model parameters. 

747 signal : `np.ndarray`, (nMeasurements) 

748 Array of image means. 

749 data : `np.ndarray`, (nMeasurements, nCols) 

750 Array of overscan column means from each measurement. 

751 error : `float` 

752 Fixed error value. 

753 *args : 

754 Additional position arguments. 

755 **kwargs : 

756 Additional keyword arguments. 

757 

758 Returns 

759 ------- 

760 difference : `np.ndarray`, (nMeasurements*nCols) 

761 The rms error between the model and input data. 

762 """ 

763 model_results = self.model_results(params, signal, *args, **kwargs) 

764 diff = (model_results-data).flatten() 

765 

766 return diff 

767 

768 

769class SimpleModel(OverscanModel): 

770 """Simple analytic overscan model.""" 

771 

772 @staticmethod 

773 def model_results(params, signal, num_transfers, start=1, stop=10): 

774 """Generate a realization of the overscan model, using the specified 

775 fit parameters and input signal. 

776 

777 Parameters 

778 ---------- 

779 params : `lmfit.Parameters` 

780 Object containing the model parameters. 

781 signal : `np.ndarray`, (nMeasurements) 

782 Array of image means. 

783 num_transfers : `int` 

784 Number of serial transfers that the charge undergoes. 

785 start : `int`, optional 

786 First overscan column to fit. This number includes the 

787 last imaging column, and needs to be adjusted by one when 

788 using the overscan bounding box. 

789 stop : `int`, optional 

790 Last overscan column to fit. This number includes the 

791 last imaging column, and needs to be adjusted by one when 

792 using the overscan bounding box. 

793 

794 Returns 

795 ------- 

796 res : `np.ndarray`, (nMeasurements, nCols) 

797 Model results. 

798 """ 

799 v = params.valuesdict() 

800 v['cti'] = 10**v['ctiexp'] 

801 

802 # Adjust column numbering to match DM overscan bbox. 

803 start += 1 

804 stop += 1 

805 

806 x = np.arange(start, stop+1) 

807 res = np.zeros((signal.shape[0], x.shape[0])) 

808 

809 for i, s in enumerate(signal): 

810 # This is largely equivalent to equation 2. The minimum 

811 # indicates that a trap cannot emit more charge than is 

812 # available, nor can it emit more charge than it can hold. 

813 # This scales the exponential release of charge from the 

814 # trap. The next term defines the contribution from the 

815 # global CTI at each pixel transfer, and the final term 

816 # includes the contribution from local CTI effects. 

817 res[i, :] = (np.minimum(v['trapsize'], s*v['scaling']) 

818 * (np.exp(1/v['emissiontime']) - 1.0) 

819 * np.exp(-x/v['emissiontime']) 

820 + s*num_transfers*v['cti']**x 

821 + v['driftscale']*s*np.exp(-x/float(v['decaytime']))) 

822 

823 return res 

824 

825 

826class SimulatedModel(OverscanModel): 

827 """Simulated overscan model.""" 

828 

829 @staticmethod 

830 def model_results(params, signal, num_transfers, amp, start=1, stop=10, trap_type=None): 

831 """Generate a realization of the overscan model, using the specified 

832 fit parameters and input signal. 

833 

834 Parameters 

835 ---------- 

836 params : `lmfit.Parameters` 

837 Object containing the model parameters. 

838 signal : `np.ndarray`, (nMeasurements) 

839 Array of image means. 

840 num_transfers : `int` 

841 Number of serial transfers that the charge undergoes. 

842 amp : `lsst.afw.cameraGeom.Amplifier` 

843 Amplifier to use for geometry information. 

844 start : `int`, optional 

845 First overscan column to fit. This number includes the 

846 last imaging column, and needs to be adjusted by one when 

847 using the overscan bounding box. 

848 stop : `int`, optional 

849 Last overscan column to fit. This number includes the 

850 last imaging column, and needs to be adjusted by one when 

851 using the overscan bounding box. 

852 trap_type : `str`, optional 

853 Type of trap model to use. 

854 

855 Returns 

856 ------- 

857 results : `np.ndarray`, (nMeasurements, nCols) 

858 Model results. 

859 """ 

860 v = params.valuesdict() 

861 

862 # Adjust column numbering to match DM overscan bbox. 

863 start += 1 

864 stop += 1 

865 

866 # Electronics effect optimization 

867 output_amplifier = FloatingOutputAmplifier(1.0, v['driftscale'], v['decaytime']) 

868 

869 # CTI optimization 

870 v['cti'] = 10**v['ctiexp'] 

871 

872 # Trap type for optimization 

873 if trap_type is None: 

874 trap = None 

875 elif trap_type == 'linear': 

876 trap = SerialTrap(v['trapsize'], v['emissiontime'], 1, 'linear', 

877 [v['scaling']]) 

878 elif trap_type == 'logistic': 

879 trap = SerialTrap(v['trapsize'], v['emissiontime'], 1, 'logistic', 

880 [v['f0'], v['k']]) 

881 else: 

882 raise ValueError('Trap type must be linear or logistic or None') 

883 

884 # Simulate ramp readout 

885 imarr = np.zeros((signal.shape[0], amp.getRawDataBBox().getWidth())) 

886 ramp = SegmentSimulator(imarr, amp.getRawSerialPrescanBBox().getWidth(), output_amplifier, 

887 cti=v['cti'], traps=trap) 

888 ramp.ramp_exp(signal) 

889 model_results = ramp.readout(serial_overscan_width=amp.getRawSerialOverscanBBox().getWidth(), 

890 parallel_overscan_width=0) 

891 

892 ncols = amp.getRawSerialPrescanBBox().getWidth() + amp.getRawDataBBox().getWidth() 

893 

894 return model_results[:, ncols+start-1:ncols+stop] 

895 

896 

897class SegmentSimulator: 

898 """Controls the creation of simulated segment images. 

899 

900 Parameters