Coverage for python/lsst/cp/pipe/deferredCharge.py: 14%

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22__all__ = ('CpCtiSolveConnections', 

23 'CpCtiSolveConfig', 

24 'CpCtiSolveTask', 

25 'OverscanModel', 

26 'SimpleModel', 

27 'SimulatedModel', 

28 'SegmentSimulator', 

29 'FloatingOutputAmplifier', 

30 ) 

31 

32import copy 

33import numpy as np 

34 

35import lsst.pipe.base as pipeBase 

36import lsst.pipe.base.connectionTypes as cT 

37import lsst.pex.config as pexConfig 

38 

39from lsst.ip.isr import DeferredChargeCalib, SerialTrap 

40from lmfit import Minimizer, Parameters 

41 

42 

43class CpCtiSolveConnections(pipeBase.PipelineTaskConnections, 

44 dimensions=("instrument", "detector")): 

45 inputMeasurements = cT.Input( 

46 name="cpCtiMeas", 

47 doc="Input overscan measurements to fit.", 

48 storageClass='StructuredDataDict', 

49 dimensions=("instrument", "exposure", "detector"), 

50 multiple=True, 

51 ) 

52 camera = cT.PrerequisiteInput( 

53 name="camera", 

54 doc="Camera geometry to use.", 

55 storageClass="Camera", 

56 dimensions=("instrument", ), 

57 isCalibration=True, 

58 ) 

59 

60 outputCalib = cT.Output( 

61 name="cpCtiCalib", 

62 doc="Output CTI calibration.", 

63 storageClass="IsrCalib", 

64 dimensions=("instrument", "detector"), 

65 isCalibration=True, 

66 ) 

67 

68 

69class CpCtiSolveConfig(pipeBase.PipelineTaskConfig, 

70 pipelineConnections=CpCtiSolveConnections): 

71 """Configuration for the CTI combination. 

72 """ 

73 maxImageMean = pexConfig.Field( 

74 dtype=float, 

75 default=150000.0, 

76 doc="Upper limit on acceptable image flux mean.", 

77 ) 

78 localOffsetColumnRange = pexConfig.ListField( 

79 dtype=int, 

80 default=[3, 13], 

81 doc="First and last overscan column to use for local offset effect.", 

82 ) 

83 

84 useGains = pexConfig.Field( 

85 dtype=bool, 

86 default=True, 

87 doc="Use gains in calculation.", 

88 ) 

89 

90 maxSignalForCti = pexConfig.Field( 

91 dtype=float, 

92 default=10000.0, 

93 doc="Upper flux limit to use for CTI fit.", 

94 ) 

95 globalCtiColumnRange = pexConfig.ListField( 

96 dtype=int, 

97 default=[1, 2], 

98 doc="First and last overscan column to use for global CTI fit.", 

99 ) 

100 

101 trapColumnRange = pexConfig.ListField( 

102 dtype=int, 

103 default=[1, 20], 

104 doc="First and last overscan column to use for serial trap fit.", 

105 ) 

106 

107 fitError = pexConfig.Field( 

108 # This gives the error on the mean in a given column, and so 

109 # is expected to be $RN / sqrt(N_rows)$. 

110 dtype=float, 

111 default=7.0/np.sqrt(2000), 

112 doc="Error to use during parameter fitting.", 

113 ) 

114 

115 

116class CpCtiSolveTask(pipeBase.PipelineTask): 

117 """Combine CTI measurements to a final calibration. 

118 

119 This task uses the extended pixel edge response (EPER) method as 

120 described by Snyder et al. 2021, Journal of Astronimcal 

121 Telescopes, Instruments, and Systems, 7, 

122 048002. doi:10.1117/1.JATIS.7.4.048002 

123 """ 

124 

125 ConfigClass = CpCtiSolveConfig 

126 _DefaultName = 'cpCtiSolve' 

127 

128 def __init__(self, **kwargs): 

129 super().__init__(**kwargs) 

130 self.allowDebug = True 

131 

132 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

133 inputs = butlerQC.get(inputRefs) 

134 

135 dimensions = [dict(exp.dataId.required) for exp in inputRefs.inputMeasurements] 

136 inputs['inputDims'] = dimensions 

137 

138 outputs = self.run(**inputs) 

139 butlerQC.put(outputs, outputRefs) 

140 

141 def run(self, inputMeasurements, camera, inputDims): 

142 """Solve for charge transfer inefficiency from overscan measurements. 

143 

144 Parameters 

145 ---------- 

146 inputMeasurements : `list` [`dict`] 

147 List of overscan measurements from each input exposure. 

148 Each dictionary is nested within a top level 'CTI' key, 

149 with measurements organized by amplifier name, containing 

150 keys: 

151 

152 ``"FIRST_MEAN"`` 

153 Mean value of first image column (`float`). 

154 ``"LAST_MEAN"`` 

155 Mean value of last image column (`float`). 

156 ``"IMAGE_MEAN"`` 

157 Mean value of the entire image region (`float`). 

158 ``"OVERSCAN_COLUMNS"`` 

159 List of overscan column indicies (`list` [`int`]). 

160 ``"OVERSCAN_VALUES"`` 

161 List of overscan column means (`list` [`float`]). 

162 camera : `lsst.afw.cameraGeom.Camera` 

163 Camera geometry to use to find detectors. 

164 inputDims : `list` [`dict`] 

165 List of input dimensions from each input exposure. 

166 

167 Returns 

168 ------- 

169 results : `lsst.pipe.base.Struct` 

170 Result struct containing: 

171 

172 ``outputCalib`` 

173 Final CTI calibration data 

174 (`lsst.ip.isr.DeferredChargeCalib`). 

175 

176 Raises 

177 ------ 

178 RuntimeError 

179 Raised if data from multiple detectors are passed in. 

180 """ 

181 detectorSet = set([d['detector'] for d in inputDims]) 

182 if len(detectorSet) != 1: 

183 raise RuntimeError("Inputs for too many detectors passed.") 

184 detectorId = detectorSet.pop() 

185 detector = camera[detectorId] 

186 

187 # Initialize with detector. 

188 calib = DeferredChargeCalib(camera=camera, detector=detector, useGains=self.config.useGains) 

189 

190 localCalib = self.solveLocalOffsets(inputMeasurements, calib, detector) 

191 

192 globalCalib = self.solveGlobalCti(inputMeasurements, localCalib, detector) 

193 

194 finalCalib = self.findTraps(inputMeasurements, globalCalib, detector) 

195 

196 return pipeBase.Struct( 

197 outputCalib=finalCalib, 

198 ) 

199 

200 def solveLocalOffsets(self, inputMeasurements, calib, detector): 

201 """Solve for local (pixel-to-pixel) electronic offsets. 

202 

203 This method fits for \tau_L, the local electronic offset decay 

204 time constant, and A_L, the local electronic offset constant 

205 of proportionality. 

206 

207 Parameters 

208 ---------- 

209 inputMeasurements : `list` [`dict`] 

210 List of overscan measurements from each input exposure. 

211 Each dictionary is nested within a top level 'CTI' key, 

212 with measurements organized by amplifier name, containing 

213 keys: 

214 

215 ``"FIRST_MEAN"`` 

216 Mean value of first image column (`float`). 

217 ``"LAST_MEAN"`` 

218 Mean value of last image column (`float`). 

219 ``"IMAGE_MEAN"`` 

220 Mean value of the entire image region (`float`). 

221 ``"OVERSCAN_COLUMNS"`` 

222 List of overscan column indicies (`list` [`int`]). 

223 ``"OVERSCAN_VALUES"`` 

224 List of overscan column means (`list` [`float`]). 

225 calib : `lsst.ip.isr.DeferredChargeCalib` 

226 Calibration to populate with values. 

227 detector : `lsst.afw.cameraGeom.Detector` 

228 Detector object containing the geometry information for 

229 the amplifiers. 

230 

231 Returns 

232 ------- 

233 calib : `lsst.ip.isr.DeferredChargeCalib` 

234 Populated calibration. 

235 

236 Raises 

237 ------ 

238 RuntimeError 

239 Raised if no data remains after flux filtering. 

240 

241 Notes 

242 ----- 

243 The original CTISIM code (https://github.com/Snyder005/ctisim) 

244 uses a data model in which the "overscan" consists of the 

245 standard serial overscan bbox with the values for the last 

246 imaging data column prepended to that list. This version of 

247 the code keeps the overscan and imaging sections separate, and 

248 so a -1 offset is needed to ensure that the same columns are 

249 used for fitting between this code and CTISIM. This offset 

250 removes that last imaging data column from the count. 

251 """ 

252 # Range to fit. These are in "camera" coordinates, and so 

253 # need to have the count for last image column removed. 

254 start, stop = self.config.localOffsetColumnRange 

255 start -= 1 

256 stop -= 1 

257 

258 # Loop over amps/inputs, fitting those columns from 

259 # "non-saturated" inputs. 

260 for amp in detector.getAmplifiers(): 

261 ampName = amp.getName() 

262 

263 # Number of serial shifts. 

264 nCols = amp.getRawDataBBox().getWidth() + amp.getRawSerialPrescanBBox().getWidth() 

265 

266 # The signal is the mean intensity of each input, and the 

267 # data are the overscan columns to fit. For detectors 

268 # with non-zero CTI, the charge from the imaging region 

269 # leaks into the overscan region. 

270 signal = [] 

271 data = [] 

272 Nskipped = 0 

273 for exposureEntry in inputMeasurements: 

274 exposureDict = exposureEntry['CTI'] 

275 if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxImageMean: 

276 signal.append(exposureDict[ampName]['IMAGE_MEAN']) 

277 data.append(exposureDict[ampName]['OVERSCAN_VALUES'][start:stop+1]) 

278 else: 

279 Nskipped += 1 

280 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxImageMean}.") 

281 if len(signal) == 0 or len(data) == 0: 

282 raise RuntimeError("All exposures brighter than config.maxImageMean and excluded.") 

283 

284 signal = np.array(signal) 

285 data = np.array(data) 

286 

287 ind = signal.argsort() 

288 signal = signal[ind] 

289 data = data[ind] 

290 

291 params = Parameters() 

292 params.add('ctiexp', value=-6, min=-7, max=-5, vary=False) 

293 params.add('trapsize', value=0.0, min=0.0, max=10., vary=False) 

294 params.add('scaling', value=0.08, min=0.0, max=1.0, vary=False) 

295 params.add('emissiontime', value=0.4, min=0.1, max=1.0, vary=False) 

296 params.add('driftscale', value=0.00022, min=0., max=0.001, vary=True) 

297 params.add('decaytime', value=2.4, min=0.1, max=4.0, vary=True) 

298 

299 model = SimpleModel() 

300 minner = Minimizer(model.difference, params, 

301 fcn_args=(signal, data, self.config.fitError, nCols), 

302 fcn_kws={'start': start, 'stop': stop}) 

303 result = minner.minimize() 

304 

305 # Save results for the drift scale and decay time. 

306 if not result.success: 

307 self.log.warning("Electronics fitting failure for amplifier %s.", ampName) 

308 

309 calib.globalCti[ampName] = 10**result.params['ctiexp'] 

310 calib.driftScale[ampName] = result.params['driftscale'].value if result.success else 0.0 

311 calib.decayTime[ampName] = result.params['decaytime'].value if result.success else 2.4 

312 self.log.info("CTI Local Fit %s: cti: %g decayTime: %g driftScale %g", 

313 ampName, calib.globalCti[ampName], calib.decayTime[ampName], 

314 calib.driftScale[ampName]) 

315 return calib 

316 

317 def solveGlobalCti(self, inputMeasurements, calib, detector): 

318 """Solve for global CTI constant. 

319 

320 This method solves for the mean global CTI, b. 

321 

322 Parameters 

323 ---------- 

324 inputMeasurements : `list` [`dict`] 

325 List of overscan measurements from each input exposure. 

326 Each dictionary is nested within a top level 'CTI' key, 

327 with measurements organized by amplifier name, containing 

328 keys: 

329 

330 ``"FIRST_MEAN"`` 

331 Mean value of first image column (`float`). 

332 ``"LAST_MEAN"`` 

333 Mean value of last image column (`float`). 

334 ``"IMAGE_MEAN"`` 

335 Mean value of the entire image region (`float`). 

336 ``"OVERSCAN_COLUMNS"`` 

337 List of overscan column indicies (`list` [`int`]). 

338 ``"OVERSCAN_VALUES"`` 

339 List of overscan column means (`list` [`float`]). 

340 calib : `lsst.ip.isr.DeferredChargeCalib` 

341 Calibration to populate with values. 

342 detector : `lsst.afw.cameraGeom.Detector` 

343 Detector object containing the geometry information for 

344 the amplifiers. 

345 

346 Returns 

347 ------- 

348 calib : `lsst.ip.isr.DeferredChargeCalib` 

349 Populated calibration. 

350 

351 Raises 

352 ------ 

353 RuntimeError 

354 Raised if no data remains after flux filtering. 

355 

356 Notes 

357 ----- 

358 The original CTISIM code uses a data model in which the 

359 "overscan" consists of the standard serial overscan bbox with 

360 the values for the last imaging data column prepended to that 

361 list. This version of the code keeps the overscan and imaging 

362 sections separate, and so a -1 offset is needed to ensure that 

363 the same columns are used for fitting between this code and 

364 CTISIM. This offset removes that last imaging data column 

365 from the count. 

366 """ 

367 # Range to fit. These are in "camera" coordinates, and so 

368 # need to have the count for last image column removed. 

369 start, stop = self.config.globalCtiColumnRange 

370 start -= 1 

371 stop -= 1 

372 

373 # Loop over amps/inputs, fitting those columns from 

374 # "non-saturated" inputs. 

375 for amp in detector.getAmplifiers(): 

376 ampName = amp.getName() 

377 

378 # Number of serial shifts. 

379 nCols = amp.getRawDataBBox().getWidth() + amp.getRawSerialPrescanBBox().getWidth() 

380 

381 # The signal is the mean intensity of each input, and the 

382 # data are the overscan columns to fit. For detectors 

383 # with non-zero CTI, the charge from the imaging region 

384 # leaks into the overscan region. 

385 signal = [] 

386 data = [] 

387 Nskipped = 0 

388 for exposureEntry in inputMeasurements: 

389 exposureDict = exposureEntry['CTI'] 

390 if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxSignalForCti: 

391 signal.append(exposureDict[ampName]['IMAGE_MEAN']) 

392 data.append(exposureDict[ampName]['OVERSCAN_VALUES'][start:stop+1]) 

393 else: 

394 Nskipped += 1 

395 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxSignalForCti}.") 

396 if len(signal) == 0 or len(data) == 0: 

397 raise RuntimeError("All exposures brighter than config.maxSignalForCti and excluded.") 

398 

399 signal = np.array(signal) 

400 data = np.array(data) 

401 

402 ind = signal.argsort() 

403 signal = signal[ind] 

404 data = data[ind] 

405 

406 # CTI test. This looks at the charge that has leaked into 

407 # the first few columns of the overscan. 

408 overscan1 = data[:, 0] 

409 overscan2 = data[:, 1] 

410 test = (np.array(overscan1) + np.array(overscan2))/(nCols*np.array(signal)) 

411 testResult = np.median(test) > 5.E-6 

412 self.log.info("Estimate of CTI test is %f for amp %s, %s.", np.median(test), ampName, 

413 "full fitting will be performed" if testResult else 

414 "only global CTI fitting will be performed") 

415 

416 self.debugView(ampName, signal, test) 

417 

418 params = Parameters() 

419 params.add('ctiexp', value=-6, min=-7, max=-5, vary=True) 

420 params.add('trapsize', value=5.0 if testResult else 0.0, min=0.0, max=30., 

421 vary=True if testResult else False) 

422 params.add('scaling', value=0.08, min=0.0, max=1.0, 

423 vary=True if testResult else False) 

424 params.add('emissiontime', value=0.35, min=0.1, max=1.0, 

425 vary=True if testResult else False) 

426 params.add('driftscale', value=calib.driftScale[ampName], min=0., max=0.001, vary=False) 

427 params.add('decaytime', value=calib.decayTime[ampName], min=0.1, max=4.0, vary=False) 

428 

429 model = SimulatedModel() 

430 minner = Minimizer(model.difference, params, 

431 fcn_args=(signal, data, self.config.fitError, nCols, amp), 

432 fcn_kws={'start': start, 'stop': stop, 'trap_type': 'linear'}) 

433 result = minner.minimize() 

434 

435 # Only the global CTI term is retained from this fit. 

436 calib.globalCti[ampName] = 10**result.params['ctiexp'].value 

437 self.log.info("CTI Global Cti %s: cti: %g decayTime: %g driftScale %g", 

438 ampName, calib.globalCti[ampName], calib.decayTime[ampName], 

439 calib.driftScale[ampName]) 

440 

441 return calib 

442 

443 def debugView(self, ampName, signal, test): 

444 """Debug method for global CTI test value. 

445 

446 Parameters 

447 ---------- 

448 ampName : `str` 

449 Name of the amp for plot title. 

450 signal : `list` [`float`] 

451 Image means for the input exposures. 

452 test : `list` [`float`] 

453 CTI test value to plot. 

454 """ 

455 import lsstDebug 

456 if not lsstDebug.Info(__name__).display: 

457 return 

458 if not self.allowDebug: 

459 return 

460 

461 import matplotlib.pyplot as plot 

462 figure = plot.figure(1) 

463 figure.clear() 

464 plot.xscale('log', base=10.0) 

465 plot.yscale('log', base=10.0) 

466 plot.xlabel('Flat Field Signal [e-?]') 

467 plot.ylabel('Serial CTI') 

468 plot.title(ampName) 

469 plot.plot(signal, test) 

470 

471 figure.show() 

472 prompt = "Press Enter or c to continue [chp]..." 

473 while True: 

474 ans = input(prompt).lower() 

475 if ans in ("", " ", "c",): 

476 break 

477 elif ans in ("p", ): 

478 import pdb 

479 pdb.set_trace() 

480 elif ans in ('x', ): 

481 self.allowDebug = False 

482 break 

483 elif ans in ("h", ): 

484 print("[h]elp [c]ontinue [p]db e[x]itDebug") 

485 plot.close() 

486 

487 def findTraps(self, inputMeasurements, calib, detector): 

488 """Solve for serial trap parameters. 

489 

490 Parameters 

491 ---------- 

492 inputMeasurements : `list` [`dict`] 

493 List of overscan measurements from each input exposure. 

494 Each dictionary is nested within a top level 'CTI' key, 

495 with measurements organized by amplifier name, containing 

496 keys: 

497 

498 ``"FIRST_MEAN"`` 

499 Mean value of first image column (`float`). 

500 ``"LAST_MEAN"`` 

501 Mean value of last image column (`float`). 

502 ``"IMAGE_MEAN"`` 

503 Mean value of the entire image region (`float`). 

504 ``"OVERSCAN_COLUMNS"`` 

505 List of overscan column indicies (`list` [`int`]). 

506 ``"OVERSCAN_VALUES"`` 

507 List of overscan column means (`list` [`float`]). 

508 calib : `lsst.ip.isr.DeferredChargeCalib` 

509 Calibration to populate with values. 

510 detector : `lsst.afw.cameraGeom.Detector` 

511 Detector object containing the geometry information for 

512 the amplifiers. 

513 

514 Returns 

515 ------- 

516 calib : `lsst.ip.isr.DeferredChargeCalib` 

517 Populated calibration. 

518 

519 Raises 

520 ------ 

521 RuntimeError 

522 Raised if no data remains after flux filtering. 

523 

524 Notes 

525 ----- 

526 The original CTISIM code uses a data model in which the 

527 "overscan" consists of the standard serial overscan bbox with 

528 the values for the last imaging data column prepended to that 

529 list. This version of the code keeps the overscan and imaging 

530 sections separate, and so a -1 offset is needed to ensure that 

531 the same columns are used for fitting between this code and 

532 CTISIM. This offset removes that last imaging data column 

533 from the count. 

534 """ 

535 # Range to fit. These are in "camera" coordinates, and so 

536 # need to have the count for last image column removed. 

537 start, stop = self.config.trapColumnRange 

538 start -= 1 

539 stop -= 1 

540 

541 # Loop over amps/inputs, fitting those columns from 

542 # "non-saturated" inputs. 

543 for amp in detector.getAmplifiers(): 

544 ampName = amp.getName() 

545 

546 # Number of serial shifts. 

547 nCols = amp.getRawDataBBox().getWidth() + amp.getRawSerialPrescanBBox().getWidth() 

548 

549 # The signal is the mean intensity of each input, and the 

550 # data are the overscan columns to fit. The new_signal is 

551 # the mean in the last image column. Any serial trap will 

552 # take charge from this column, and deposit it into the 

553 # overscan columns. 

554 signal = [] 

555 data = [] 

556 new_signal = [] 

557 Nskipped = 0 

558 for exposureEntry in inputMeasurements: 

559 exposureDict = exposureEntry['CTI'] 

560 if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxImageMean: 

561 signal.append(exposureDict[ampName]['IMAGE_MEAN']) 

562 data.append(exposureDict[ampName]['OVERSCAN_VALUES'][start:stop+1]) 

563 new_signal.append(exposureDict[ampName]['LAST_MEAN']) 

564 else: 

565 Nskipped += 1 

566 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxImageMean}.") 

567 if len(signal) == 0 or len(data) == 0: 

568 raise RuntimeError("All exposures brighter than config.maxImageMean and excluded.") 

569 

570 signal = np.array(signal) 

571 data = np.array(data) 

572 new_signal = np.array(new_signal) 

573 

574 ind = signal.argsort() 

575 signal = signal[ind] 

576 data = data[ind] 

577 new_signal = new_signal[ind] 

578 

579 # In the absense of any trap, the model results using the 

580 # parameters already determined will match the observed 

581 # overscan results. 

582 params = Parameters() 

583 params.add('ctiexp', value=np.log10(calib.globalCti[ampName]), 

584 min=-7, max=-5, vary=False) 

585 params.add('trapsize', value=0.0, min=0.0, max=10., vary=False) 

586 params.add('scaling', value=0.08, min=0.0, max=1.0, vary=False) 

587 params.add('emissiontime', value=0.35, min=0.1, max=1.0, vary=False) 

588 params.add('driftscale', value=calib.driftScale[ampName], 

589 min=0.0, max=0.001, vary=False) 

590 params.add('decaytime', value=calib.decayTime[ampName], 

591 min=0.1, max=4.0, vary=False) 

592 

593 model = SimpleModel.model_results(params, signal, nCols, 

594 start=start, stop=stop) 

595 

596 # Evaluating trap: the difference between the model and 

597 # observed data. 

598 res = np.sum((data-model)[:, :3], axis=1) 

599 

600 # Create spline model for the trap, using the residual 

601 # between data and model as a function of the last image 

602 # column mean (new_signal) scaled by (1 - A_L). 

603 # Note that this ``spline`` model is actually a piecewise 

604 # linear interpolation and not a true spline. 

605 new_signal = np.asarray((1 - calib.driftScale[ampName])*new_signal, dtype=np.float64) 

606 x = new_signal 

607 y = np.maximum(0, res) 

608 

609 # Pad left with ramp 

610 y = np.pad(y, (10, 0), 'linear_ramp', end_values=(0, 0)) 

611 x = np.pad(x, (10, 0), 'linear_ramp', end_values=(0, 0)) 

612 

613 trap = SerialTrap(20000.0, 0.4, 1, 'spline', np.concatenate((x, y)).tolist()) 

614 calib.serialTraps[ampName] = trap 

615 

616 return calib 

617 

618 

619class OverscanModel: 

620 """Base class for handling model/data fit comparisons. 

621 

622 This handles all of the methods needed for the lmfit Minimizer to 

623 run. 

624 """ 

625 

626 @staticmethod 

627 def model_results(params, signal, num_transfers, start=1, stop=10): 

628 """Generate a realization of the overscan model, using the specified 

629 fit parameters and input signal. 

630 

631 Parameters 

632 ---------- 

633 params : `lmfit.Parameters` 

634 Object containing the model parameters. 

635 signal : `np.ndarray`, (nMeasurements) 

636 Array of image means. 

637 num_transfers : `int` 

638 Number of serial transfers that the charge undergoes. 

639 start : `int`, optional 

640 First overscan column to fit. This number includes the 

641 last imaging column, and needs to be adjusted by one when 

642 using the overscan bounding box. 

643 stop : `int`, optional 

644 Last overscan column to fit. This number includes the 

645 last imaging column, and needs to be adjusted by one when 

646 using the overscan bounding box. 

647 

648 Returns 

649 ------- 

650 results : `np.ndarray`, (nMeasurements, nCols) 

651 Model results. 

652 """ 

653 raise NotImplementedError("Subclasses must implement the model calculation.") 

654 

655 def loglikelihood(self, params, signal, data, error, *args, **kwargs): 

656 """Calculate log likelihood of the model. 

657 

658 Parameters 

659 ---------- 

660 params : `lmfit.Parameters` 

661 Object containing the model parameters. 

662 signal : `np.ndarray`, (nMeasurements) 

663 Array of image means. 

664 data : `np.ndarray`, (nMeasurements, nCols) 

665 Array of overscan column means from each measurement. 

666 error : `float` 

667 Fixed error value. 

668 *args : 

669 Additional position arguments. 

670 **kwargs : 

671 Additional keyword arguments. 

672 

673 Returns 

674 ------- 

675 logL : `float` 

676 The log-likelihood of the observed data given the model 

677 parameters. 

678 """ 

679 model_results = self.model_results(params, signal, *args, **kwargs) 

680 

681 inv_sigma2 = 1.0/(error**2.0) 

682 diff = model_results - data 

683 

684 return -0.5*(np.sum(inv_sigma2*(diff)**2.)) 

685 

686 def negative_loglikelihood(self, params, signal, data, error, *args, **kwargs): 

687 """Calculate negative log likelihood of the model. 

688 

689 Parameters 

690 ---------- 

691 params : `lmfit.Parameters` 

692 Object containing the model parameters. 

693 signal : `np.ndarray`, (nMeasurements) 

694 Array of image means. 

695 data : `np.ndarray`, (nMeasurements, nCols) 

696 Array of overscan column means from each measurement. 

697 error : `float` 

698 Fixed error value. 

699 *args : 

700 Additional position arguments. 

701 **kwargs : 

702 Additional keyword arguments. 

703 

704 Returns 

705 ------- 

706 negativelogL : `float` 

707 The negative log-likelihood of the observed data given the 

708 model parameters. 

709 """ 

710 ll = self.loglikelihood(params, signal, data, error, *args, **kwargs) 

711 

712 return -ll 

713 

714 def rms_error(self, params, signal, data, error, *args, **kwargs): 

715 """Calculate RMS error between model and data. 

716 

717 Parameters 

718 ---------- 

719 params : `lmfit.Parameters` 

720 Object containing the model parameters. 

721 signal : `np.ndarray`, (nMeasurements) 

722 Array of image means. 

723 data : `np.ndarray`, (nMeasurements, nCols) 

724 Array of overscan column means from each measurement. 

725 error : `float` 

726 Fixed error value. 

727 *args : 

728 Additional position arguments. 

729 **kwargs : 

730 Additional keyword arguments. 

731 

732 Returns 

733 ------- 

734 rms : `float` 

735 The rms error between the model and input data. 

736 """ 

737 model_results = self.model_results(params, signal, *args, **kwargs) 

738 

739 diff = model_results - data 

740 rms = np.sqrt(np.mean(np.square(diff))) 

741 

742 return rms 

743 

744 def difference(self, params, signal, data, error, *args, **kwargs): 

745 """Calculate the flattened difference array between model and data. 

746 

747 Parameters 

748 ---------- 

749 params : `lmfit.Parameters` 

750 Object containing the model parameters. 

751 signal : `np.ndarray`, (nMeasurements) 

752 Array of image means. 

753 data : `np.ndarray`, (nMeasurements, nCols) 

754 Array of overscan column means from each measurement. 

755 error : `float` 

756 Fixed error value. 

757 *args : 

758 Additional position arguments. 

759 **kwargs : 

760 Additional keyword arguments. 

761 

762 Returns 

763 ------- 

764 difference : `np.ndarray`, (nMeasurements*nCols) 

765 The rms error between the model and input data. 

766 """ 

767 model_results = self.model_results(params, signal, *args, **kwargs) 

768 diff = (model_results-data).flatten() 

769 

770 return diff 

771 

772 

773class SimpleModel(OverscanModel): 

774 """Simple analytic overscan model.""" 

775 

776 @staticmethod 

777 def model_results(params, signal, num_transfers, start=1, stop=10): 

778 """Generate a realization of the overscan model, using the specified 

779 fit parameters and input signal. 

780 

781 Parameters 

782 ---------- 

783 params : `lmfit.Parameters` 

784 Object containing the model parameters. 

785 signal : `np.ndarray`, (nMeasurements) 

786 Array of image means. 

787 num_transfers : `int` 

788 Number of serial transfers that the charge undergoes. 

789 start : `int`, optional 

790 First overscan column to fit. This number includes the 

791 last imaging column, and needs to be adjusted by one when 

792 using the overscan bounding box. 

793 stop : `int`, optional 

794 Last overscan column to fit. This number includes the 

795 last imaging column, and needs to be adjusted by one when 

796 using the overscan bounding box. 

797 

798 Returns 

799 ------- 

800 res : `np.ndarray`, (nMeasurements, nCols) 

801 Model results. 

802 """ 

803 v = params.valuesdict() 

804 v['cti'] = 10**v['ctiexp'] 

805 

806 # Adjust column numbering to match DM overscan bbox. 

807 start += 1 

808 stop += 1 

809 

810 x = np.arange(start, stop+1) 

811 res = np.zeros((signal.shape[0], x.shape[0])) 

812 

813 for i, s in enumerate(signal): 

814 # This is largely equivalent to equation 2. The minimum 

815 # indicates that a trap cannot emit more charge than is 

816 # available, nor can it emit more charge than it can hold. 

817 # This scales the exponential release of charge from the 

818 # trap. The next term defines the contribution from the 

819 # global CTI at each pixel transfer, and the final term 

820 # includes the contribution from local CTI effects. 

821 res[i, :] = (np.minimum(v['trapsize'], s*v['scaling']) 

822 * (np.exp(1/v['emissiontime']) - 1.0) 

823 * np.exp(-x/v['emissiontime']) 

824 + s*num_transfers*v['cti']**x 

825 + v['driftscale']*s*np.exp(-x/float(v['decaytime']))) 

826 

827 return res 

828 

829 

830class SimulatedModel(OverscanModel): 

831 """Simulated overscan model.""" 

832 

833 @staticmethod 

834 def model_results(params, signal, num_transfers, amp, start=1, stop=10, trap_type=None): 

835 """Generate a realization of the overscan model, using the specified 

836 fit parameters and input signal. 

837 

838 Parameters 

839 ---------- 

840 params : `lmfit.Parameters` 

841 Object containing the model parameters. 

842 signal : `np.ndarray`, (nMeasurements) 

843 Array of image means. 

844 num_transfers : `int` 

845 Number of serial transfers that the charge undergoes. 

846 amp : `lsst.afw.cameraGeom.Amplifier` 

847 Amplifier to use for geometry information. 

848 start : `int`, optional 

849 First overscan column to fit. This number includes the 

850 last imaging column, and needs to be adjusted by one when 

851 using the overscan bounding box. 

852 stop : `int`, optional 

853 Last overscan column to fit. This number includes the 

854 last imaging column, and needs to be adjusted by one when 

855 using the overscan bounding box. 

856 trap_type : `str`, optional 

857 Type of trap model to use. 

858 

859 Returns 

860 ------- 

861 results : `np.ndarray`, (nMeasurements, nCols) 

862 Model results. 

863 """ 

864 v = params.valuesdict() 

865 

866 # Adjust column numbering to match DM overscan bbox. 

867 start += 1 

868 stop += 1 

869 

870 # Electronics effect optimization 

871 output_amplifier = FloatingOutputAmplifier(1.0, v['driftscale'], v['decaytime']) 

872 

873 # CTI optimization 

874 v['cti'] = 10**v['ctiexp'] 

875 

876 # Trap type for optimization 

877 if trap_type is None: 

878 trap = None 

879 elif trap_type == 'linear': 

880 trap = SerialTrap(v['trapsize'], v['emissiontime'], 1, 'linear', 

881 [v['scaling']]) 

882 elif trap_type == 'logistic': 

883 trap = SerialTrap(v['trapsize'], v['emissiontime'], 1, 'logistic', 

884 [v['f0'], v['k']]) 

885 else: 

886 raise ValueError('Trap type must be linear or logistic or None') 

887 

888 # Simulate ramp readout 

889 imarr = np.zeros((signal.shape[0], amp.getRawDataBBox().getWidth())) 

890 ramp = SegmentSimulator(imarr, amp.getRawSerialPrescanBBox().getWidth(), output_amplifier, 

891 cti=v['cti'], traps=trap) 

892 ramp.ramp_exp(signal) 

893 model_results = ramp.readout(serial_overscan_width=amp.getRawSerialOverscanBBox().getWidth(), 

894 parallel_overscan_width=0) 

895 

896 ncols = amp.getRawSerialPrescanBBox().getWidth() + amp.getRawDataBBox().getWidth() 

897 

898 return model_results[:, ncols+start-1:ncols+stop] 

899 

900 

901class SegmentSimulator: 

902 """Controls the creation of simulated segment images. 

903 

904 Parameters 

905 ----------