Coverage for python/lsst/cp/pipe/deferredCharge.py: 12%

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22__all__ = ('CpCtiSolveConnections', 

23 'CpCtiSolveConfig', 

24 'CpCtiSolveTask', 

25 'OverscanModel', 

26 'SimpleModel', 

27 'SimulatedModel', 

28 'SegmentSimulator', 

29 'FloatingOutputAmplifier', 

30 ) 

31 

32import copy 

33import numpy as np 

34 

35import lsst.pipe.base as pipeBase 

36import lsst.pipe.base.connectionTypes as cT 

37import lsst.pex.config as pexConfig 

38 

39from lsst.ip.isr import DeferredChargeCalib, SerialTrap 

40from lmfit import Minimizer, Parameters 

41 

42from ._lookupStaticCalibration import lookupStaticCalibration 

43 

44 

45class CpCtiSolveConnections(pipeBase.PipelineTaskConnections, 

46 dimensions=("instrument", "detector")): 

47 inputMeasurements = cT.Input( 

48 name="cpCtiMeas", 

49 doc="Input overscan measurements to fit.", 

50 storageClass='StructuredDataDict', 

51 dimensions=("instrument", "exposure", "detector"), 

52 multiple=True, 

53 ) 

54 camera = cT.PrerequisiteInput( 

55 name="camera", 

56 doc="Camera geometry to use.", 

57 storageClass="Camera", 

58 dimensions=("instrument", ), 

59 lookupFunction=lookupStaticCalibration, 

60 isCalibration=True, 

61 ) 

62 

63 outputCalib = cT.Output( 

64 name="cpCtiCalib", 

65 doc="Output CTI calibration.", 

66 storageClass="IsrCalib", 

67 dimensions=("instrument", "detector"), 

68 ) 

69 

70 

71class CpCtiSolveConfig(pipeBase.PipelineTaskConfig, 

72 pipelineConnections=CpCtiSolveConnections): 

73 """Configuration for the CTI combination. 

74 """ 

75 maxImageMean = pexConfig.Field( 

76 dtype=float, 

77 default=150000.0, 

78 doc="Upper limit on acceptable image flux mean.", 

79 ) 

80 localOffsetColumnRange = pexConfig.ListField( 

81 dtype=int, 

82 default=[3, 13], 

83 doc="First and last overscan column to use for local offset effect.", 

84 ) 

85 

86 maxSignalForCti = pexConfig.Field( 

87 dtype=float, 

88 default=10000.0, 

89 doc="Upper flux limit to use for CTI fit.", 

90 ) 

91 globalCtiColumnRange = pexConfig.ListField( 

92 dtype=int, 

93 default=[1, 2], 

94 doc="First and last overscan column to use for global CTI fit.", 

95 ) 

96 

97 trapColumnRange = pexConfig.ListField( 

98 dtype=int, 

99 default=[1, 20], 

100 doc="First and last overscan column to use for serial trap fit.", 

101 ) 

102 

103 fitError = pexConfig.Field( 

104 # This gives the error on the mean in a given column, and so 

105 # is expected to be $RN / sqrt(N_rows)$. 

106 dtype=float, 

107 default=7.0/np.sqrt(2000), 

108 doc="Error to use during parameter fitting.", 

109 ) 

110 

111 

112class CpCtiSolveTask(pipeBase.PipelineTask): 

113 """Combine CTI measurements to a final calibration. 

114 

115 This task uses the extended pixel edge response (EPER) method as 

116 described by Snyder et al. 2021, Journal of Astronimcal 

117 Telescopes, Instruments, and Systems, 7, 

118 048002. doi:10.1117/1.JATIS.7.4.048002 

119 """ 

120 

121 ConfigClass = CpCtiSolveConfig 

122 _DefaultName = 'cpCtiSolve' 

123 

124 def __init__(self, **kwargs): 

125 super().__init__(**kwargs) 

126 self.allowDebug = True 

127 

128 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

129 inputs = butlerQC.get(inputRefs) 

130 

131 dimensions = [exp.dataId.byName() for exp in inputRefs.inputMeasurements] 

132 inputs['inputDims'] = dimensions 

133 

134 outputs = self.run(**inputs) 

135 butlerQC.put(outputs, outputRefs) 

136 

137 def run(self, inputMeasurements, camera, inputDims): 

138 """Solve for charge transfer inefficiency from overscan measurements. 

139 

140 Parameters 

141 ---------- 

142 inputMeasurements : `list` [`dict`] 

143 List of overscan measurements from each input exposure. 

144 Each dictionary is nested within a top level 'CTI' key, 

145 with measurements organized by amplifier name, containing 

146 keys: 

147 

148 ``"FIRST_MEAN"`` 

149 Mean value of first image column (`float`). 

150 ``"LAST_MEAN"`` 

151 Mean value of last image column (`float`). 

152 ``"IMAGE_MEAN"`` 

153 Mean value of the entire image region (`float`). 

154 ``"OVERSCAN_COLUMNS"`` 

155 List of overscan column indicies (`list` [`int`]). 

156 ``"OVERSCAN_VALUES"`` 

157 List of overscan column means (`list` [`float`]). 

158 camera : `lsst.afw.cameraGeom.Camera` 

159 Camera geometry to use to find detectors. 

160 inputDims : `list` [`dict`] 

161 List of input dimensions from each input exposure. 

162 

163 Returns 

164 ------- 

165 results : `lsst.pipe.base.Struct` 

166 Result struct containing: 

167 

168 ``outputCalib`` 

169 Final CTI calibration data 

170 (`lsst.ip.isr.DeferredChargeCalib`). 

171 

172 Raises 

173 ------ 

174 RuntimeError 

175 Raised if data from multiple detectors are passed in. 

176 """ 

177 detectorSet = set([d['detector'] for d in inputDims]) 

178 if len(detectorSet) != 1: 

179 raise RuntimeError("Inputs for too many detectors passed.") 

180 detectorId = detectorSet.pop() 

181 detector = camera[detectorId] 

182 

183 # Initialize with detector. 

184 calib = DeferredChargeCalib(camera=camera, detector=detector) 

185 

186 localCalib = self.solveLocalOffsets(inputMeasurements, calib, detector) 

187 

188 globalCalib = self.solveGlobalCti(inputMeasurements, localCalib, detector) 

189 

190 finalCalib = self.findTraps(inputMeasurements, globalCalib, detector) 

191 

192 return pipeBase.Struct( 

193 outputCalib=finalCalib, 

194 ) 

195 

196 def solveLocalOffsets(self, inputMeasurements, calib, detector): 

197 """Solve for local (pixel-to-pixel) electronic offsets. 

198 

199 This method fits for \tau_L, the local electronic offset decay 

200 time constant, and A_L, the local electronic offset constant 

201 of proportionality. 

202 

203 Parameters 

204 ---------- 

205 inputMeasurements : `list` [`dict`] 

206 List of overscan measurements from each input exposure. 

207 Each dictionary is nested within a top level 'CTI' key, 

208 with measurements organized by amplifier name, containing 

209 keys: 

210 

211 ``"FIRST_MEAN"`` 

212 Mean value of first image column (`float`). 

213 ``"LAST_MEAN"`` 

214 Mean value of last image column (`float`). 

215 ``"IMAGE_MEAN"`` 

216 Mean value of the entire image region (`float`). 

217 ``"OVERSCAN_COLUMNS"`` 

218 List of overscan column indicies (`list` [`int`]). 

219 ``"OVERSCAN_VALUES"`` 

220 List of overscan column means (`list` [`float`]). 

221 calib : `lsst.ip.isr.DeferredChargeCalib` 

222 Calibration to populate with values. 

223 detector : `lsst.afw.cameraGeom.Detector` 

224 Detector object containing the geometry information for 

225 the amplifiers. 

226 

227 Returns 

228 ------- 

229 calib : `lsst.ip.isr.DeferredChargeCalib` 

230 Populated calibration. 

231 

232 Raises 

233 ------ 

234 RuntimeError 

235 Raised if no data remains after flux filtering. 

236 

237 Notes 

238 ----- 

239 The original CTISIM code (https://github.com/Snyder005/ctisim) 

240 uses a data model in which the "overscan" consists of the 

241 standard serial overscan bbox with the values for the last 

242 imaging data column prepended to that list. This version of 

243 the code keeps the overscan and imaging sections separate, and 

244 so a -1 offset is needed to ensure that the same columns are 

245 used for fitting between this code and CTISIM. This offset 

246 removes that last imaging data column from the count. 

247 """ 

248 # Range to fit. These are in "camera" coordinates, and so 

249 # need to have the count for last image column removed. 

250 start, stop = self.config.localOffsetColumnRange 

251 start -= 1 

252 stop -= 1 

253 

254 # Loop over amps/inputs, fitting those columns from 

255 # "non-saturated" inputs. 

256 for amp in detector.getAmplifiers(): 

257 ampName = amp.getName() 

258 

259 # Number of serial shifts. 

260 nCols = amp.getRawDataBBox().getWidth() + amp.getRawSerialPrescanBBox().getWidth() 

261 

262 # The signal is the mean intensity of each input, and the 

263 # data are the overscan columns to fit. For detectors 

264 # with non-zero CTI, the charge from the imaging region 

265 # leaks into the overscan region. 

266 signal = [] 

267 data = [] 

268 Nskipped = 0 

269 for exposureEntry in inputMeasurements: 

270 exposureDict = exposureEntry['CTI'] 

271 if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxImageMean: 

272 signal.append(exposureDict[ampName]['IMAGE_MEAN']) 

273 data.append(exposureDict[ampName]['OVERSCAN_VALUES'][start:stop+1]) 

274 else: 

275 Nskipped += 1 

276 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxImageMean}.") 

277 if len(signal) == 0 or len(data) == 0: 

278 raise RuntimeError("All exposures brighter than config.maxImageMean and excluded.") 

279 

280 signal = np.array(signal) 

281 data = np.array(data) 

282 

283 ind = signal.argsort() 

284 signal = signal[ind] 

285 data = data[ind] 

286 

287 params = Parameters() 

288 params.add('ctiexp', value=-6, min=-7, max=-5, vary=False) 

289 params.add('trapsize', value=0.0, min=0.0, max=10., vary=False) 

290 params.add('scaling', value=0.08, min=0.0, max=1.0, vary=False) 

291 params.add('emissiontime', value=0.4, min=0.1, max=1.0, vary=False) 

292 params.add('driftscale', value=0.00022, min=0., max=0.001, vary=True) 

293 params.add('decaytime', value=2.4, min=0.1, max=4.0, vary=True) 

294 

295 model = SimpleModel() 

296 minner = Minimizer(model.difference, params, 

297 fcn_args=(signal, data, self.config.fitError, nCols), 

298 fcn_kws={'start': start, 'stop': stop}) 

299 result = minner.minimize() 

300 

301 # Save results for the drift scale and decay time. 

302 if not result.success: 

303 self.log.warning("Electronics fitting failure for amplifier %s.", ampName) 

304 

305 calib.globalCti[ampName] = 10**result.params['ctiexp'] 

306 calib.driftScale[ampName] = result.params['driftscale'].value if result.success else 0.0 

307 calib.decayTime[ampName] = result.params['decaytime'].value if result.success else 2.4 

308 self.log.info("CTI Local Fit %s: cti: %g decayTime: %g driftScale %g", 

309 ampName, calib.globalCti[ampName], calib.decayTime[ampName], 

310 calib.driftScale[ampName]) 

311 return calib 

312 

313 def solveGlobalCti(self, inputMeasurements, calib, detector): 

314 """Solve for global CTI constant. 

315 

316 This method solves for the mean global CTI, b. 

317 

318 Parameters 

319 ---------- 

320 inputMeasurements : `list` [`dict`] 

321 List of overscan measurements from each input exposure. 

322 Each dictionary is nested within a top level 'CTI' key, 

323 with measurements organized by amplifier name, containing 

324 keys: 

325 

326 ``"FIRST_MEAN"`` 

327 Mean value of first image column (`float`). 

328 ``"LAST_MEAN"`` 

329 Mean value of last image column (`float`). 

330 ``"IMAGE_MEAN"`` 

331 Mean value of the entire image region (`float`). 

332 ``"OVERSCAN_COLUMNS"`` 

333 List of overscan column indicies (`list` [`int`]). 

334 ``"OVERSCAN_VALUES"`` 

335 List of overscan column means (`list` [`float`]). 

336 calib : `lsst.ip.isr.DeferredChargeCalib` 

337 Calibration to populate with values. 

338 detector : `lsst.afw.cameraGeom.Detector` 

339 Detector object containing the geometry information for 

340 the amplifiers. 

341 

342 Returns 

343 ------- 

344 calib : `lsst.ip.isr.DeferredChargeCalib` 

345 Populated calibration. 

346 

347 Raises 

348 ------ 

349 RuntimeError 

350 Raised if no data remains after flux filtering. 

351 

352 Notes 

353 ----- 

354 The original CTISIM code uses a data model in which the 

355 "overscan" consists of the standard serial overscan bbox with 

356 the values for the last imaging data column prepended to that 

357 list. This version of the code keeps the overscan and imaging 

358 sections separate, and so a -1 offset is needed to ensure that 

359 the same columns are used for fitting between this code and 

360 CTISIM. This offset removes that last imaging data column 

361 from the count. 

362 """ 

363 # Range to fit. These are in "camera" coordinates, and so 

364 # need to have the count for last image column removed. 

365 start, stop = self.config.globalCtiColumnRange 

366 start -= 1 

367 stop -= 1 

368 

369 # Loop over amps/inputs, fitting those columns from 

370 # "non-saturated" inputs. 

371 for amp in detector.getAmplifiers(): 

372 ampName = amp.getName() 

373 

374 # Number of serial shifts. 

375 nCols = amp.getRawDataBBox().getWidth() + amp.getRawSerialPrescanBBox().getWidth() 

376 

377 # The signal is the mean intensity of each input, and the 

378 # data are the overscan columns to fit. For detectors 

379 # with non-zero CTI, the charge from the imaging region 

380 # leaks into the overscan region. 

381 signal = [] 

382 data = [] 

383 Nskipped = 0 

384 for exposureEntry in inputMeasurements: 

385 exposureDict = exposureEntry['CTI'] 

386 if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxSignalForCti: 

387 signal.append(exposureDict[ampName]['IMAGE_MEAN']) 

388 data.append(exposureDict[ampName]['OVERSCAN_VALUES'][start:stop+1]) 

389 else: 

390 Nskipped += 1 

391 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxSignalForCti}.") 

392 if len(signal) == 0 or len(data) == 0: 

393 raise RuntimeError("All exposures brighter than config.maxSignalForCti and excluded.") 

394 

395 signal = np.array(signal) 

396 data = np.array(data) 

397 

398 ind = signal.argsort() 

399 signal = signal[ind] 

400 data = data[ind] 

401 

402 # CTI test. This looks at the charge that has leaked into 

403 # the first few columns of the overscan. 

404 overscan1 = data[:, 0] 

405 overscan2 = data[:, 1] 

406 test = (np.array(overscan1) + np.array(overscan2))/(nCols*np.array(signal)) 

407 testResult = np.median(test) > 5.E-6 

408 self.log.info("Estimate of CTI test is %f for amp %s, %s.", np.median(test), ampName, 

409 "full fitting will be performed" if testResult else 

410 "only global CTI fitting will be performed") 

411 

412 self.debugView(ampName, signal, test) 

413 

414 params = Parameters() 

415 params.add('ctiexp', value=-6, min=-7, max=-5, vary=True) 

416 params.add('trapsize', value=5.0 if testResult else 0.0, min=0.0, max=30., 

417 vary=True if testResult else False) 

418 params.add('scaling', value=0.08, min=0.0, max=1.0, 

419 vary=True if testResult else False) 

420 params.add('emissiontime', value=0.35, min=0.1, max=1.0, 

421 vary=True if testResult else False) 

422 params.add('driftscale', value=calib.driftScale[ampName], min=0., max=0.001, vary=False) 

423 params.add('decaytime', value=calib.decayTime[ampName], min=0.1, max=4.0, vary=False) 

424 

425 model = SimulatedModel() 

426 minner = Minimizer(model.difference, params, 

427 fcn_args=(signal, data, self.config.fitError, nCols, amp), 

428 fcn_kws={'start': start, 'stop': stop, 'trap_type': 'linear'}) 

429 result = minner.minimize() 

430 

431 # Only the global CTI term is retained from this fit. 

432 calib.globalCti[ampName] = 10**result.params['ctiexp'].value 

433 self.log.info("CTI Global Cti %s: cti: %g decayTime: %g driftScale %g", 

434 ampName, calib.globalCti[ampName], calib.decayTime[ampName], 

435 calib.driftScale[ampName]) 

436 

437 return calib 

438 

439 def debugView(self, ampName, signal, test): 

440 """Debug method for global CTI test value. 

441 

442 Parameters 

443 ---------- 

444 ampName : `str` 

445 Name of the amp for plot title. 

446 signal : `list` [`float`] 

447 Image means for the input exposures. 

448 test : `list` [`float`] 

449 CTI test value to plot. 

450 """ 

451 import lsstDebug 

452 if not lsstDebug.Info(__name__).display: 

453 return 

454 if not self.allowDebug: 

455 return 

456 

457 import matplotlib.pyplot as plot 

458 figure = plot.figure(1) 

459 figure.clear() 

460 plot.xscale('log', base=10.0) 

461 plot.yscale('log', base=10.0) 

462 plot.xlabel('Flat Field Signal [e-?]') 

463 plot.ylabel('Serial CTI') 

464 plot.title(ampName) 

465 plot.plot(signal, test) 

466 

467 figure.show() 

468 prompt = "Press Enter or c to continue [chp]..." 

469 while True: 

470 ans = input(prompt).lower() 

471 if ans in ("", " ", "c",): 

472 break 

473 elif ans in ("p", ): 

474 import pdb 

475 pdb.set_trace() 

476 elif ans in ('x', ): 

477 self.allowDebug = False 

478 break 

479 elif ans in ("h", ): 

480 print("[h]elp [c]ontinue [p]db e[x]itDebug") 

481 plot.close() 

482 

483 def findTraps(self, inputMeasurements, calib, detector): 

484 """Solve for serial trap parameters. 

485 

486 Parameters 

487 ---------- 

488 inputMeasurements : `list` [`dict`] 

489 List of overscan measurements from each input exposure. 

490 Each dictionary is nested within a top level 'CTI' key, 

491 with measurements organized by amplifier name, containing 

492 keys: 

493 

494 ``"FIRST_MEAN"`` 

495 Mean value of first image column (`float`). 

496 ``"LAST_MEAN"`` 

497 Mean value of last image column (`float`). 

498 ``"IMAGE_MEAN"`` 

499 Mean value of the entire image region (`float`). 

500 ``"OVERSCAN_COLUMNS"`` 

501 List of overscan column indicies (`list` [`int`]). 

502 ``"OVERSCAN_VALUES"`` 

503 List of overscan column means (`list` [`float`]). 

504 calib : `lsst.ip.isr.DeferredChargeCalib` 

505 Calibration to populate with values. 

506 detector : `lsst.afw.cameraGeom.Detector` 

507 Detector object containing the geometry information for 

508 the amplifiers. 

509 

510 Returns 

511 ------- 

512 calib : `lsst.ip.isr.DeferredChargeCalib` 

513 Populated calibration. 

514 

515 Raises 

516 ------ 

517 RuntimeError 

518 Raised if no data remains after flux filtering. 

519 

520 Notes 

521 ----- 

522 The original CTISIM code uses a data model in which the 

523 "overscan" consists of the standard serial overscan bbox with 

524 the values for the last imaging data column prepended to that 

525 list. This version of the code keeps the overscan and imaging 

526 sections separate, and so a -1 offset is needed to ensure that 

527 the same columns are used for fitting between this code and 

528 CTISIM. This offset removes that last imaging data column 

529 from the count. 

530 """ 

531 # Range to fit. These are in "camera" coordinates, and so 

532 # need to have the count for last image column removed. 

533 start, stop = self.config.trapColumnRange 

534 start -= 1 

535 stop -= 1 

536 

537 # Loop over amps/inputs, fitting those columns from 

538 # "non-saturated" inputs. 

539 for amp in detector.getAmplifiers(): 

540 ampName = amp.getName() 

541 

542 # Number of serial shifts. 

543 nCols = amp.getRawDataBBox().getWidth() + amp.getRawSerialPrescanBBox().getWidth() 

544 

545 # The signal is the mean intensity of each input, and the 

546 # data are the overscan columns to fit. The new_signal is 

547 # the mean in the last image column. Any serial trap will 

548 # take charge from this column, and deposit it into the 

549 # overscan columns. 

550 signal = [] 

551 data = [] 

552 new_signal = [] 

553 Nskipped = 0 

554 for exposureEntry in inputMeasurements: 

555 exposureDict = exposureEntry['CTI'] 

556 if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxImageMean: 

557 signal.append(exposureDict[ampName]['IMAGE_MEAN']) 

558 data.append(exposureDict[ampName]['OVERSCAN_VALUES'][start:stop+1]) 

559 new_signal.append(exposureDict[ampName]['LAST_MEAN']) 

560 else: 

561 Nskipped += 1 

562 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxImageMean}.") 

563 if len(signal) == 0 or len(data) == 0: 

564 raise RuntimeError("All exposures brighter than config.maxImageMean and excluded.") 

565 

566 signal = np.array(signal) 

567 data = np.array(data) 

568 new_signal = np.array(new_signal) 

569 

570 ind = signal.argsort() 

571 signal = signal[ind] 

572 data = data[ind] 

573 new_signal = new_signal[ind] 

574 

575 # In the absense of any trap, the model results using the 

576 # parameters already determined will match the observed 

577 # overscan results. 

578 params = Parameters() 

579 params.add('ctiexp', value=np.log10(calib.globalCti[ampName]), 

580 min=-7, max=-5, vary=False) 

581 params.add('trapsize', value=0.0, min=0.0, max=10., vary=False) 

582 params.add('scaling', value=0.08, min=0.0, max=1.0, vary=False) 

583 params.add('emissiontime', value=0.35, min=0.1, max=1.0, vary=False) 

584 params.add('driftscale', value=calib.driftScale[ampName], 

585 min=0.0, max=0.001, vary=False) 

586 params.add('decaytime', value=calib.decayTime[ampName], 

587 min=0.1, max=4.0, vary=False) 

588 

589 model = SimpleModel.model_results(params, signal, nCols, 

590 start=start, stop=stop) 

591 

592 # Evaluating trap: the difference between the model and 

593 # observed data. 

594 res = np.sum((data-model)[:, :3], axis=1) 

595 

596 # Create spline model for the trap, using the residual 

597 # between data and model as a function of the last image 

598 # column mean (new_signal) scaled by (1 - A_L). 

599 new_signal = np.asarray((1 - calib.driftScale[ampName])*new_signal, dtype=np.float64) 

600 x = new_signal 

601 y = np.maximum(0, res) 

602 

603 # Pad left with ramp 

604 y = np.pad(y, (10, 0), 'linear_ramp', end_values=(0, 0)) 

605 x = np.pad(x, (10, 0), 'linear_ramp', end_values=(0, 0)) 

606 

607 # Pad right with constant 

608 y = np.pad(y, (1, 1), 'constant', constant_values=(0, y[-1])) 

609 x = np.pad(x, (1, 1), 'constant', constant_values=(-1, 200000.)) 

610 

611 trap = SerialTrap(20000.0, 0.4, 1, 'spline', np.concatenate((x, y)).tolist()) 

612 calib.serialTraps[ampName] = trap 

613 

614 return calib 

615 

616 

617class OverscanModel: 

618 """Base class for handling model/data fit comparisons. 

619 

620 This handles all of the methods needed for the lmfit Minimizer to 

621 run. 

622 """ 

623 

624 @staticmethod 

625 def model_results(params, signal, num_transfers, start=1, stop=10): 

626 """Generate a realization of the overscan model, using the specified 

627 fit parameters and input signal. 

628 

629 Parameters 

630 ---------- 

631 params : `lmfit.Parameters` 

632 Object containing the model parameters. 

633 signal : `np.ndarray`, (nMeasurements) 

634 Array of image means. 

635 num_transfers : `int` 

636 Number of serial transfers that the charge undergoes. 

637 start : `int`, optional 

638 First overscan column to fit. This number includes the 

639 last imaging column, and needs to be adjusted by one when 

640 using the overscan bounding box. 

641 stop : `int`, optional 

642 Last overscan column to fit. This number includes the 

643 last imaging column, and needs to be adjusted by one when 

644 using the overscan bounding box. 

645 

646 Returns 

647 ------- 

648 results : `np.ndarray`, (nMeasurements, nCols) 

649 Model results. 

650 """ 

651 raise NotImplementedError("Subclasses must implement the model calculation.") 

652 

653 def loglikelihood(self, params, signal, data, error, *args, **kwargs): 

654 """Calculate log likelihood of the model. 

655 

656 Parameters 

657 ---------- 

658 params : `lmfit.Parameters` 

659 Object containing the model parameters. 

660 signal : `np.ndarray`, (nMeasurements) 

661 Array of image means. 

662 data : `np.ndarray`, (nMeasurements, nCols) 

663 Array of overscan column means from each measurement. 

664 error : `float` 

665 Fixed error value. 

666 *args : 

667 Additional position arguments. 

668 **kwargs : 

669 Additional keyword arguments. 

670 

671 Returns 

672 ------- 

673 logL : `float` 

674 The log-likelihood of the observed data given the model 

675 parameters. 

676 """ 

677 model_results = self.model_results(params, signal, *args, **kwargs) 

678 

679 inv_sigma2 = 1.0/(error**2.0) 

680 diff = model_results - data 

681 

682 return -0.5*(np.sum(inv_sigma2*(diff)**2.)) 

683 

684 def negative_loglikelihood(self, params, signal, data, error, *args, **kwargs): 

685 """Calculate negative log likelihood of the model. 

686 

687 Parameters 

688 ---------- 

689 params : `lmfit.Parameters` 

690 Object containing the model parameters. 

691 signal : `np.ndarray`, (nMeasurements) 

692 Array of image means. 

693 data : `np.ndarray`, (nMeasurements, nCols) 

694 Array of overscan column means from each measurement. 

695 error : `float` 

696 Fixed error value. 

697 *args : 

698 Additional position arguments. 

699 **kwargs : 

700 Additional keyword arguments. 

701 

702 Returns 

703 ------- 

704 negativelogL : `float` 

705 The negative log-likelihood of the observed data given the 

706 model parameters. 

707 """ 

708 ll = self.loglikelihood(params, signal, data, error, *args, **kwargs) 

709 

710 return -ll 

711 

712 def rms_error(self, params, signal, data, error, *args, **kwargs): 

713 """Calculate RMS error between model and data. 

714 

715 Parameters 

716 ---------- 

717 params : `lmfit.Parameters` 

718 Object containing the model parameters. 

719 signal : `np.ndarray`, (nMeasurements) 

720 Array of image means. 

721 data : `np.ndarray`, (nMeasurements, nCols) 

722 Array of overscan column means from each measurement. 

723 error : `float` 

724 Fixed error value. 

725 *args : 

726 Additional position arguments. 

727 **kwargs : 

728 Additional keyword arguments. 

729 

730 Returns 

731 ------- 

732 rms : `float` 

733 The rms error between the model and input data. 

734 """ 

735 model_results = self.model_results(params, signal, *args, **kwargs) 

736 

737 diff = model_results - data 

738 rms = np.sqrt(np.mean(np.square(diff))) 

739 

740 return rms 

741 

742 def difference(self, params, signal, data, error, *args, **kwargs): 

743 """Calculate the flattened difference array between model and data. 

744 

745 Parameters 

746 ---------- 

747 params : `lmfit.Parameters` 

748 Object containing the model parameters. 

749 signal : `np.ndarray`, (nMeasurements) 

750 Array of image means. 

751 data : `np.ndarray`, (nMeasurements, nCols) 

752 Array of overscan column means from each measurement. 

753 error : `float` 

754 Fixed error value. 

755 *args : 

756 Additional position arguments. 

757 **kwargs : 

758 Additional keyword arguments. 

759 

760 Returns 

761 ------- 

762 difference : `np.ndarray`, (nMeasurements*nCols) 

763 The rms error between the model and input data. 

764 """ 

765 model_results = self.model_results(params, signal, *args, **kwargs) 

766 diff = (model_results-data).flatten() 

767 

768 return diff 

769 

770 

771class SimpleModel(OverscanModel): 

772 """Simple analytic overscan model.""" 

773 

774 @staticmethod 

775 def model_results(params, signal, num_transfers, start=1, stop=10): 

776 """Generate a realization of the overscan model, using the specified 

777 fit parameters and input signal. 

778 

779 Parameters 

780 ---------- 

781 params : `lmfit.Parameters` 

782 Object containing the model parameters. 

783 signal : `np.ndarray`, (nMeasurements) 

784 Array of image means. 

785 num_transfers : `int` 

786 Number of serial transfers that the charge undergoes. 

787 start : `int`, optional 

788 First overscan column to fit. This number includes the 

789 last imaging column, and needs to be adjusted by one when 

790 using the overscan bounding box. 

791 stop : `int`, optional 

792 Last overscan column to fit. This number includes the 

793 last imaging column, and needs to be adjusted by one when 

794 using the overscan bounding box. 

795 

796 Returns 

797 ------- 

798 res : `np.ndarray`, (nMeasurements, nCols) 

799 Model results. 

800 """ 

801 v = params.valuesdict() 

802 v['cti'] = 10**v['ctiexp'] 

803 

804 # Adjust column numbering to match DM overscan bbox. 

805 start += 1 

806 stop += 1 

807 

808 x = np.arange(start, stop+1) 

809 res = np.zeros((signal.shape[0], x.shape[0])) 

810 

811 for i, s in enumerate(signal): 

812 # This is largely equivalent to equation 2. The minimum 

813 # indicates that a trap cannot emit more charge than is 

814 # available, nor can it emit more charge than it can hold. 

815 # This scales the exponential release of charge from the 

816 # trap. The next term defines the contribution from the 

817 # global CTI at each pixel transfer, and the final term 

818 # includes the contribution from local CTI effects. 

819 res[i, :] = (np.minimum(v['trapsize'], s*v['scaling']) 

820 * (np.exp(1/v['emissiontime']) - 1.0) 

821 * np.exp(-x/v['emissiontime']) 

822 + s*num_transfers*v['cti']**x 

823 + v['driftscale']*s*np.exp(-x/float(v['decaytime']))) 

824 

825 return res 

826 

827 

828class SimulatedModel(OverscanModel): 

829 """Simulated overscan model.""" 

830 

831 @staticmethod 

832 def model_results(params, signal, num_transfers, amp, start=1, stop=10, trap_type=None): 

833 """Generate a realization of the overscan model, using the specified 

834 fit parameters and input signal. 

835 

836 Parameters 

837 ---------- 

838 params : `lmfit.Parameters` 

839 Object containing the model parameters. 

840 signal : `np.ndarray`, (nMeasurements) 

841 Array of image means. 

842 num_transfers : `int` 

843 Number of serial transfers that the charge undergoes. 

844 amp : `lsst.afw.cameraGeom.Amplifier` 

845 Amplifier to use for geometry information. 

846 start : `int`, optional 

847 First overscan column to fit. This number includes the 

848 last imaging column, and needs to be adjusted by one when 

849 using the overscan bounding box. 

850 stop : `int`, optional 

851 Last overscan column to fit. This number includes the 

852 last imaging column, and needs to be adjusted by one when 

853 using the overscan bounding box. 

854 trap_type : `str`, optional 

855 Type of trap model to use. 

856 

857 Returns 

858 ------- 

859 results : `np.ndarray`, (nMeasurements, nCols) 

860 Model results. 

861 """ 

862 v = params.valuesdict() 

863 

864 # Adjust column numbering to match DM overscan bbox. 

865 start += 1 

866 stop += 1 

867 

868 # Electronics effect optimization 

869 output_amplifier = FloatingOutputAmplifier(1.0, v['driftscale'], v['decaytime']) 

870 

871 # CTI optimization 

872 v['cti'] = 10**v['ctiexp'] 

873 

874 # Trap type for optimization 

875 if trap_type is None: 

876 trap = None 

877 elif trap_type == 'linear': 

878 trap = SerialTrap(v['trapsize'], v['emissiontime'], 1, 'linear', 

879 [v['scaling']]) 

880 elif trap_type == 'logistic': 

881 trap = SerialTrap(v['trapsize'], v['emissiontime'], 1, 'logistic', 

882 [v['f0'], v['k']]) 

883 else: 

884 raise ValueError('Trap type must be linear or logistic or None') 

885 

886 # Simulate ramp readout 

887 imarr = np.zeros((signal.shape[0], amp.getRawDataBBox().getWidth())) 

888 ramp = SegmentSimulator(imarr, amp.getRawSerialPrescanBBox().getWidth(), output_amplifier, 

889 cti=v['cti'], traps=trap) 

890 ramp.ramp_exp(signal) 

891 model_results = ramp.readout(serial_overscan_width=amp.getRawSerialOverscanBBox().getWidth(), 

892 parallel_overscan_width=0) 

893 

894 ncols = amp.getRawSerialPrescanBBox().getWidth() + amp.getRawDataBBox().getWidth() 

895 

896 return model_results[:, ncols+start-1:ncols+stop] 

897 

898 

899class SegmentSimulator: