Coverage for python/lsst/cp/pipe/deferredCharge.py: 14%

330 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-11-30 13:56 +0000

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22__all__ = ('CpCtiSolveConnections', 

23 'CpCtiSolveConfig', 

24 'CpCtiSolveTask', 

25 'OverscanModel', 

26 'SimpleModel', 

27 'SimulatedModel', 

28 'SegmentSimulator', 

29 'FloatingOutputAmplifier', 

30 ) 

31 

32import copy 

33import numpy as np 

34 

35import lsst.pipe.base as pipeBase 

36import lsst.pipe.base.connectionTypes as cT 

37import lsst.pex.config as pexConfig 

38 

39from lsst.ip.isr import DeferredChargeCalib, SerialTrap 

40from lmfit import Minimizer, Parameters 

41 

42 

43class CpCtiSolveConnections(pipeBase.PipelineTaskConnections, 

44 dimensions=("instrument", "detector")): 

45 inputMeasurements = cT.Input( 

46 name="cpCtiMeas", 

47 doc="Input overscan measurements to fit.", 

48 storageClass='StructuredDataDict', 

49 dimensions=("instrument", "exposure", "detector"), 

50 multiple=True, 

51 ) 

52 camera = cT.PrerequisiteInput( 

53 name="camera", 

54 doc="Camera geometry to use.", 

55 storageClass="Camera", 

56 dimensions=("instrument", ), 

57 isCalibration=True, 

58 ) 

59 

60 outputCalib = cT.Output( 

61 name="cpCtiCalib", 

62 doc="Output CTI calibration.", 

63 storageClass="IsrCalib", 

64 dimensions=("instrument", "detector"), 

65 isCalibration=True, 

66 ) 

67 

68 

69class CpCtiSolveConfig(pipeBase.PipelineTaskConfig, 

70 pipelineConnections=CpCtiSolveConnections): 

71 """Configuration for the CTI combination. 

72 """ 

73 maxImageMean = pexConfig.Field( 

74 dtype=float, 

75 default=150000.0, 

76 doc="Upper limit on acceptable image flux mean.", 

77 ) 

78 localOffsetColumnRange = pexConfig.ListField( 

79 dtype=int, 

80 default=[3, 13], 

81 doc="First and last overscan column to use for local offset effect.", 

82 ) 

83 

84 maxSignalForCti = pexConfig.Field( 

85 dtype=float, 

86 default=10000.0, 

87 doc="Upper flux limit to use for CTI fit.", 

88 ) 

89 globalCtiColumnRange = pexConfig.ListField( 

90 dtype=int, 

91 default=[1, 2], 

92 doc="First and last overscan column to use for global CTI fit.", 

93 ) 

94 

95 trapColumnRange = pexConfig.ListField( 

96 dtype=int, 

97 default=[1, 20], 

98 doc="First and last overscan column to use for serial trap fit.", 

99 ) 

100 

101 fitError = pexConfig.Field( 

102 # This gives the error on the mean in a given column, and so 

103 # is expected to be $RN / sqrt(N_rows)$. 

104 dtype=float, 

105 default=7.0/np.sqrt(2000), 

106 doc="Error to use during parameter fitting.", 

107 ) 

108 

109 

110class CpCtiSolveTask(pipeBase.PipelineTask): 

111 """Combine CTI measurements to a final calibration. 

112 

113 This task uses the extended pixel edge response (EPER) method as 

114 described by Snyder et al. 2021, Journal of Astronimcal 

115 Telescopes, Instruments, and Systems, 7, 

116 048002. doi:10.1117/1.JATIS.7.4.048002 

117 """ 

118 

119 ConfigClass = CpCtiSolveConfig 

120 _DefaultName = 'cpCtiSolve' 

121 

122 def __init__(self, **kwargs): 

123 super().__init__(**kwargs) 

124 self.allowDebug = True 

125 

126 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

127 inputs = butlerQC.get(inputRefs) 

128 

129 dimensions = [dict(exp.dataId.required) for exp in inputRefs.inputMeasurements] 

130 inputs['inputDims'] = dimensions 

131 

132 outputs = self.run(**inputs) 

133 butlerQC.put(outputs, outputRefs) 

134 

135 def run(self, inputMeasurements, camera, inputDims): 

136 """Solve for charge transfer inefficiency from overscan measurements. 

137 

138 Parameters 

139 ---------- 

140 inputMeasurements : `list` [`dict`] 

141 List of overscan measurements from each input exposure. 

142 Each dictionary is nested within a top level 'CTI' key, 

143 with measurements organized by amplifier name, containing 

144 keys: 

145 

146 ``"FIRST_MEAN"`` 

147 Mean value of first image column (`float`). 

148 ``"LAST_MEAN"`` 

149 Mean value of last image column (`float`). 

150 ``"IMAGE_MEAN"`` 

151 Mean value of the entire image region (`float`). 

152 ``"OVERSCAN_COLUMNS"`` 

153 List of overscan column indicies (`list` [`int`]). 

154 ``"OVERSCAN_VALUES"`` 

155 List of overscan column means (`list` [`float`]). 

156 camera : `lsst.afw.cameraGeom.Camera` 

157 Camera geometry to use to find detectors. 

158 inputDims : `list` [`dict`] 

159 List of input dimensions from each input exposure. 

160 

161 Returns 

162 ------- 

163 results : `lsst.pipe.base.Struct` 

164 Result struct containing: 

165 

166 ``outputCalib`` 

167 Final CTI calibration data 

168 (`lsst.ip.isr.DeferredChargeCalib`). 

169 

170 Raises 

171 ------ 

172 RuntimeError 

173 Raised if data from multiple detectors are passed in. 

174 """ 

175 detectorSet = set([d['detector'] for d in inputDims]) 

176 if len(detectorSet) != 1: 

177 raise RuntimeError("Inputs for too many detectors passed.") 

178 detectorId = detectorSet.pop() 

179 detector = camera[detectorId] 

180 

181 # Initialize with detector. 

182 calib = DeferredChargeCalib(camera=camera, detector=detector) 

183 

184 localCalib = self.solveLocalOffsets(inputMeasurements, calib, detector) 

185 

186 globalCalib = self.solveGlobalCti(inputMeasurements, localCalib, detector) 

187 

188 finalCalib = self.findTraps(inputMeasurements, globalCalib, detector) 

189 

190 return pipeBase.Struct( 

191 outputCalib=finalCalib, 

192 ) 

193 

194 def solveLocalOffsets(self, inputMeasurements, calib, detector): 

195 """Solve for local (pixel-to-pixel) electronic offsets. 

196 

197 This method fits for \tau_L, the local electronic offset decay 

198 time constant, and A_L, the local electronic offset constant 

199 of proportionality. 

200 

201 Parameters 

202 ---------- 

203 inputMeasurements : `list` [`dict`] 

204 List of overscan measurements from each input exposure. 

205 Each dictionary is nested within a top level 'CTI' key, 

206 with measurements organized by amplifier name, containing 

207 keys: 

208 

209 ``"FIRST_MEAN"`` 

210 Mean value of first image column (`float`). 

211 ``"LAST_MEAN"`` 

212 Mean value of last image column (`float`). 

213 ``"IMAGE_MEAN"`` 

214 Mean value of the entire image region (`float`). 

215 ``"OVERSCAN_COLUMNS"`` 

216 List of overscan column indicies (`list` [`int`]). 

217 ``"OVERSCAN_VALUES"`` 

218 List of overscan column means (`list` [`float`]). 

219 calib : `lsst.ip.isr.DeferredChargeCalib` 

220 Calibration to populate with values. 

221 detector : `lsst.afw.cameraGeom.Detector` 

222 Detector object containing the geometry information for 

223 the amplifiers. 

224 

225 Returns 

226 ------- 

227 calib : `lsst.ip.isr.DeferredChargeCalib` 

228 Populated calibration. 

229 

230 Raises 

231 ------ 

232 RuntimeError 

233 Raised if no data remains after flux filtering. 

234 

235 Notes 

236 ----- 

237 The original CTISIM code (https://github.com/Snyder005/ctisim) 

238 uses a data model in which the "overscan" consists of the 

239 standard serial overscan bbox with the values for the last 

240 imaging data column prepended to that list. This version of 

241 the code keeps the overscan and imaging sections separate, and 

242 so a -1 offset is needed to ensure that the same columns are 

243 used for fitting between this code and CTISIM. This offset 

244 removes that last imaging data column from the count. 

245 """ 

246 # Range to fit. These are in "camera" coordinates, and so 

247 # need to have the count for last image column removed. 

248 start, stop = self.config.localOffsetColumnRange 

249 start -= 1 

250 stop -= 1 

251 

252 # Loop over amps/inputs, fitting those columns from 

253 # "non-saturated" inputs. 

254 for amp in detector.getAmplifiers(): 

255 ampName = amp.getName() 

256 

257 # Number of serial shifts. 

258 nCols = amp.getRawDataBBox().getWidth() + amp.getRawSerialPrescanBBox().getWidth() 

259 

260 # The signal is the mean intensity of each input, and the 

261 # data are the overscan columns to fit. For detectors 

262 # with non-zero CTI, the charge from the imaging region 

263 # leaks into the overscan region. 

264 signal = [] 

265 data = [] 

266 Nskipped = 0 

267 for exposureEntry in inputMeasurements: 

268 exposureDict = exposureEntry['CTI'] 

269 if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxImageMean: 

270 signal.append(exposureDict[ampName]['IMAGE_MEAN']) 

271 data.append(exposureDict[ampName]['OVERSCAN_VALUES'][start:stop+1]) 

272 else: 

273 Nskipped += 1 

274 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxImageMean}.") 

275 if len(signal) == 0 or len(data) == 0: 

276 raise RuntimeError("All exposures brighter than config.maxImageMean and excluded.") 

277 

278 signal = np.array(signal) 

279 data = np.array(data) 

280 

281 ind = signal.argsort() 

282 signal = signal[ind] 

283 data = data[ind] 

284 

285 params = Parameters() 

286 params.add('ctiexp', value=-6, min=-7, max=-5, vary=False) 

287 params.add('trapsize', value=0.0, min=0.0, max=10., vary=False) 

288 params.add('scaling', value=0.08, min=0.0, max=1.0, vary=False) 

289 params.add('emissiontime', value=0.4, min=0.1, max=1.0, vary=False) 

290 params.add('driftscale', value=0.00022, min=0., max=0.001, vary=True) 

291 params.add('decaytime', value=2.4, min=0.1, max=4.0, vary=True) 

292 

293 model = SimpleModel() 

294 minner = Minimizer(model.difference, params, 

295 fcn_args=(signal, data, self.config.fitError, nCols), 

296 fcn_kws={'start': start, 'stop': stop}) 

297 result = minner.minimize() 

298 

299 # Save results for the drift scale and decay time. 

300 if not result.success: 

301 self.log.warning("Electronics fitting failure for amplifier %s.", ampName) 

302 

303 calib.globalCti[ampName] = 10**result.params['ctiexp'] 

304 calib.driftScale[ampName] = result.params['driftscale'].value if result.success else 0.0 

305 calib.decayTime[ampName] = result.params['decaytime'].value if result.success else 2.4 

306 self.log.info("CTI Local Fit %s: cti: %g decayTime: %g driftScale %g", 

307 ampName, calib.globalCti[ampName], calib.decayTime[ampName], 

308 calib.driftScale[ampName]) 

309 return calib 

310 

311 def solveGlobalCti(self, inputMeasurements, calib, detector): 

312 """Solve for global CTI constant. 

313 

314 This method solves for the mean global CTI, b. 

315 

316 Parameters 

317 ---------- 

318 inputMeasurements : `list` [`dict`] 

319 List of overscan measurements from each input exposure. 

320 Each dictionary is nested within a top level 'CTI' key, 

321 with measurements organized by amplifier name, containing 

322 keys: 

323 

324 ``"FIRST_MEAN"`` 

325 Mean value of first image column (`float`). 

326 ``"LAST_MEAN"`` 

327 Mean value of last image column (`float`). 

328 ``"IMAGE_MEAN"`` 

329 Mean value of the entire image region (`float`). 

330 ``"OVERSCAN_COLUMNS"`` 

331 List of overscan column indicies (`list` [`int`]). 

332 ``"OVERSCAN_VALUES"`` 

333 List of overscan column means (`list` [`float`]). 

334 calib : `lsst.ip.isr.DeferredChargeCalib` 

335 Calibration to populate with values. 

336 detector : `lsst.afw.cameraGeom.Detector` 

337 Detector object containing the geometry information for 

338 the amplifiers. 

339 

340 Returns 

341 ------- 

342 calib : `lsst.ip.isr.DeferredChargeCalib` 

343 Populated calibration. 

344 

345 Raises 

346 ------ 

347 RuntimeError 

348 Raised if no data remains after flux filtering. 

349 

350 Notes 

351 ----- 

352 The original CTISIM code uses a data model in which the 

353 "overscan" consists of the standard serial overscan bbox with 

354 the values for the last imaging data column prepended to that 

355 list. This version of the code keeps the overscan and imaging 

356 sections separate, and so a -1 offset is needed to ensure that 

357 the same columns are used for fitting between this code and 

358 CTISIM. This offset removes that last imaging data column 

359 from the count. 

360 """ 

361 # Range to fit. These are in "camera" coordinates, and so 

362 # need to have the count for last image column removed. 

363 start, stop = self.config.globalCtiColumnRange 

364 start -= 1 

365 stop -= 1 

366 

367 # Loop over amps/inputs, fitting those columns from 

368 # "non-saturated" inputs. 

369 for amp in detector.getAmplifiers(): 

370 ampName = amp.getName() 

371 

372 # Number of serial shifts. 

373 nCols = amp.getRawDataBBox().getWidth() + amp.getRawSerialPrescanBBox().getWidth() 

374 

375 # The signal is the mean intensity of each input, and the 

376 # data are the overscan columns to fit. For detectors 

377 # with non-zero CTI, the charge from the imaging region 

378 # leaks into the overscan region. 

379 signal = [] 

380 data = [] 

381 Nskipped = 0 

382 for exposureEntry in inputMeasurements: 

383 exposureDict = exposureEntry['CTI'] 

384 if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxSignalForCti: 

385 signal.append(exposureDict[ampName]['IMAGE_MEAN']) 

386 data.append(exposureDict[ampName]['OVERSCAN_VALUES'][start:stop+1]) 

387 else: 

388 Nskipped += 1 

389 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxSignalForCti}.") 

390 if len(signal) == 0 or len(data) == 0: 

391 raise RuntimeError("All exposures brighter than config.maxSignalForCti and excluded.") 

392 

393 signal = np.array(signal) 

394 data = np.array(data) 

395 

396 ind = signal.argsort() 

397 signal = signal[ind] 

398 data = data[ind] 

399 

400 # CTI test. This looks at the charge that has leaked into 

401 # the first few columns of the overscan. 

402 overscan1 = data[:, 0] 

403 overscan2 = data[:, 1] 

404 test = (np.array(overscan1) + np.array(overscan2))/(nCols*np.array(signal)) 

405 testResult = np.median(test) > 5.E-6 

406 self.log.info("Estimate of CTI test is %f for amp %s, %s.", np.median(test), ampName, 

407 "full fitting will be performed" if testResult else 

408 "only global CTI fitting will be performed") 

409 

410 self.debugView(ampName, signal, test) 

411 

412 params = Parameters() 

413 params.add('ctiexp', value=-6, min=-7, max=-5, vary=True) 

414 params.add('trapsize', value=5.0 if testResult else 0.0, min=0.0, max=30., 

415 vary=True if testResult else False) 

416 params.add('scaling', value=0.08, min=0.0, max=1.0, 

417 vary=True if testResult else False) 

418 params.add('emissiontime', value=0.35, min=0.1, max=1.0, 

419 vary=True if testResult else False) 

420 params.add('driftscale', value=calib.driftScale[ampName], min=0., max=0.001, vary=False) 

421 params.add('decaytime', value=calib.decayTime[ampName], min=0.1, max=4.0, vary=False) 

422 

423 model = SimulatedModel() 

424 minner = Minimizer(model.difference, params, 

425 fcn_args=(signal, data, self.config.fitError, nCols, amp), 

426 fcn_kws={'start': start, 'stop': stop, 'trap_type': 'linear'}) 

427 result = minner.minimize() 

428 

429 # Only the global CTI term is retained from this fit. 

430 calib.globalCti[ampName] = 10**result.params['ctiexp'].value 

431 self.log.info("CTI Global Cti %s: cti: %g decayTime: %g driftScale %g", 

432 ampName, calib.globalCti[ampName], calib.decayTime[ampName], 

433 calib.driftScale[ampName]) 

434 

435 return calib 

436 

437 def debugView(self, ampName, signal, test): 

438 """Debug method for global CTI test value. 

439 

440 Parameters 

441 ---------- 

442 ampName : `str` 

443 Name of the amp for plot title. 

444 signal : `list` [`float`] 

445 Image means for the input exposures. 

446 test : `list` [`float`] 

447 CTI test value to plot. 

448 """ 

449 import lsstDebug 

450 if not lsstDebug.Info(__name__).display: 

451 return 

452 if not self.allowDebug: 

453 return 

454 

455 import matplotlib.pyplot as plot 

456 figure = plot.figure(1) 

457 figure.clear() 

458 plot.xscale('log', base=10.0) 

459 plot.yscale('log', base=10.0) 

460 plot.xlabel('Flat Field Signal [e-?]') 

461 plot.ylabel('Serial CTI') 

462 plot.title(ampName) 

463 plot.plot(signal, test) 

464 

465 figure.show() 

466 prompt = "Press Enter or c to continue [chp]..." 

467 while True: 

468 ans = input(prompt).lower() 

469 if ans in ("", " ", "c",): 

470 break 

471 elif ans in ("p", ): 

472 import pdb 

473 pdb.set_trace() 

474 elif ans in ('x', ): 

475 self.allowDebug = False 

476 break 

477 elif ans in ("h", ): 

478 print("[h]elp [c]ontinue [p]db e[x]itDebug") 

479 plot.close() 

480 

481 def findTraps(self, inputMeasurements, calib, detector): 

482 """Solve for serial trap parameters. 

483 

484 Parameters 

485 ---------- 

486 inputMeasurements : `list` [`dict`] 

487 List of overscan measurements from each input exposure. 

488 Each dictionary is nested within a top level 'CTI' key, 

489 with measurements organized by amplifier name, containing 

490 keys: 

491 

492 ``"FIRST_MEAN"`` 

493 Mean value of first image column (`float`). 

494 ``"LAST_MEAN"`` 

495 Mean value of last image column (`float`). 

496 ``"IMAGE_MEAN"`` 

497 Mean value of the entire image region (`float`). 

498 ``"OVERSCAN_COLUMNS"`` 

499 List of overscan column indicies (`list` [`int`]). 

500 ``"OVERSCAN_VALUES"`` 

501 List of overscan column means (`list` [`float`]). 

502 calib : `lsst.ip.isr.DeferredChargeCalib` 

503 Calibration to populate with values. 

504 detector : `lsst.afw.cameraGeom.Detector` 

505 Detector object containing the geometry information for 

506 the amplifiers. 

507 

508 Returns 

509 ------- 

510 calib : `lsst.ip.isr.DeferredChargeCalib` 

511 Populated calibration. 

512 

513 Raises 

514 ------ 

515 RuntimeError 

516 Raised if no data remains after flux filtering. 

517 

518 Notes 

519 ----- 

520 The original CTISIM code uses a data model in which the 

521 "overscan" consists of the standard serial overscan bbox with 

522 the values for the last imaging data column prepended to that 

523 list. This version of the code keeps the overscan and imaging 

524 sections separate, and so a -1 offset is needed to ensure that 

525 the same columns are used for fitting between this code and 

526 CTISIM. This offset removes that last imaging data column 

527 from the count. 

528 """ 

529 # Range to fit. These are in "camera" coordinates, and so 

530 # need to have the count for last image column removed. 

531 start, stop = self.config.trapColumnRange 

532 start -= 1 

533 stop -= 1 

534 

535 # Loop over amps/inputs, fitting those columns from 

536 # "non-saturated" inputs. 

537 for amp in detector.getAmplifiers(): 

538 ampName = amp.getName() 

539 

540 # Number of serial shifts. 

541 nCols = amp.getRawDataBBox().getWidth() + amp.getRawSerialPrescanBBox().getWidth() 

542 

543 # The signal is the mean intensity of each input, and the 

544 # data are the overscan columns to fit. The new_signal is 

545 # the mean in the last image column. Any serial trap will 

546 # take charge from this column, and deposit it into the 

547 # overscan columns. 

548 signal = [] 

549 data = [] 

550 new_signal = [] 

551 Nskipped = 0 

552 for exposureEntry in inputMeasurements: 

553 exposureDict = exposureEntry['CTI'] 

554 if exposureDict[ampName]['IMAGE_MEAN'] < self.config.maxImageMean: 

555 signal.append(exposureDict[ampName]['IMAGE_MEAN']) 

556 data.append(exposureDict[ampName]['OVERSCAN_VALUES'][start:stop+1]) 

557 new_signal.append(exposureDict[ampName]['LAST_MEAN']) 

558 else: 

559 Nskipped += 1 

560 self.log.info(f"Skipped {Nskipped} exposures brighter than {self.config.maxImageMean}.") 

561 if len(signal) == 0 or len(data) == 0: 

562 raise RuntimeError("All exposures brighter than config.maxImageMean and excluded.") 

563 

564 signal = np.array(signal) 

565 data = np.array(data) 

566 new_signal = np.array(new_signal) 

567 

568 ind = signal.argsort() 

569 signal = signal[ind] 

570 data = data[ind] 

571 new_signal = new_signal[ind] 

572 

573 # In the absense of any trap, the model results using the 

574 # parameters already determined will match the observed 

575 # overscan results. 

576 params = Parameters() 

577 params.add('ctiexp', value=np.log10(calib.globalCti[ampName]), 

578 min=-7, max=-5, vary=False) 

579 params.add('trapsize', value=0.0, min=0.0, max=10., vary=False) 

580 params.add('scaling', value=0.08, min=0.0, max=1.0, vary=False) 

581 params.add('emissiontime', value=0.35, min=0.1, max=1.0, vary=False) 

582 params.add('driftscale', value=calib.driftScale[ampName], 

583 min=0.0, max=0.001, vary=False) 

584 params.add('decaytime', value=calib.decayTime[ampName], 

585 min=0.1, max=4.0, vary=False) 

586 

587 model = SimpleModel.model_results(params, signal, nCols, 

588 start=start, stop=stop) 

589 

590 # Evaluating trap: the difference between the model and 

591 # observed data. 

592 res = np.sum((data-model)[:, :3], axis=1) 

593 

594 # Create spline model for the trap, using the residual 

595 # between data and model as a function of the last image 

596 # column mean (new_signal) scaled by (1 - A_L). 

597 # Note that this ``spline`` model is actually a piecewise 

598 # linear interpolation and not a true spline. 

599 new_signal = np.asarray((1 - calib.driftScale[ampName])*new_signal, dtype=np.float64) 

600 x = new_signal 

601 y = np.maximum(0, res) 

602 

603 # Pad left with ramp 

604 y = np.pad(y, (10, 0), 'linear_ramp', end_values=(0, 0)) 

605 x = np.pad(x, (10, 0), 'linear_ramp', end_values=(0, 0)) 

606 

607 trap = SerialTrap(20000.0, 0.4, 1, 'spline', np.concatenate((x, y)).tolist()) 

608 calib.serialTraps[ampName] = trap 

609 

610 return calib 

611 

612 

613class OverscanModel: 

614 """Base class for handling model/data fit comparisons. 

615 

616 This handles all of the methods needed for the lmfit Minimizer to 

617 run. 

618 """ 

619 

620 @staticmethod 

621 def model_results(params, signal, num_transfers, start=1, stop=10): 

622 """Generate a realization of the overscan model, using the specified 

623 fit parameters and input signal. 

624 

625 Parameters 

626 ---------- 

627 params : `lmfit.Parameters` 

628 Object containing the model parameters. 

629 signal : `np.ndarray`, (nMeasurements) 

630 Array of image means. 

631 num_transfers : `int` 

632 Number of serial transfers that the charge undergoes. 

633 start : `int`, optional 

634 First overscan column to fit. This number includes the 

635 last imaging column, and needs to be adjusted by one when 

636 using the overscan bounding box. 

637 stop : `int`, optional 

638 Last overscan column to fit. This number includes the 

639 last imaging column, and needs to be adjusted by one when 

640 using the overscan bounding box. 

641 

642 Returns 

643 ------- 

644 results : `np.ndarray`, (nMeasurements, nCols) 

645 Model results. 

646 """ 

647 raise NotImplementedError("Subclasses must implement the model calculation.") 

648 

649 def loglikelihood(self, params, signal, data, error, *args, **kwargs): 

650 """Calculate log likelihood of the model. 

651 

652 Parameters 

653 ---------- 

654 params : `lmfit.Parameters` 

655 Object containing the model parameters. 

656 signal : `np.ndarray`, (nMeasurements) 

657 Array of image means. 

658 data : `np.ndarray`, (nMeasurements, nCols) 

659 Array of overscan column means from each measurement. 

660 error : `float` 

661 Fixed error value. 

662 *args : 

663 Additional position arguments. 

664 **kwargs : 

665 Additional keyword arguments. 

666 

667 Returns 

668 ------- 

669 logL : `float` 

670 The log-likelihood of the observed data given the model 

671 parameters. 

672 """ 

673 model_results = self.model_results(params, signal, *args, **kwargs) 

674 

675 inv_sigma2 = 1.0/(error**2.0) 

676 diff = model_results - data 

677 

678 return -0.5*(np.sum(inv_sigma2*(diff)**2.)) 

679 

680 def negative_loglikelihood(self, params, signal, data, error, *args, **kwargs): 

681 """Calculate negative log likelihood of the model. 

682 

683 Parameters 

684 ---------- 

685 params : `lmfit.Parameters` 

686 Object containing the model parameters. 

687 signal : `np.ndarray`, (nMeasurements) 

688 Array of image means. 

689 data : `np.ndarray`, (nMeasurements, nCols) 

690 Array of overscan column means from each measurement. 

691 error : `float` 

692 Fixed error value. 

693 *args : 

694 Additional position arguments. 

695 **kwargs : 

696 Additional keyword arguments. 

697 

698 Returns 

699 ------- 

700 negativelogL : `float` 

701 The negative log-likelihood of the observed data given the 

702 model parameters. 

703 """ 

704 ll = self.loglikelihood(params, signal, data, error, *args, **kwargs) 

705 

706 return -ll 

707 

708 def rms_error(self, params, signal, data, error, *args, **kwargs): 

709 """Calculate RMS error between model and data. 

710 

711 Parameters 

712 ---------- 

713 params : `lmfit.Parameters` 

714 Object containing the model parameters. 

715 signal : `np.ndarray`, (nMeasurements) 

716 Array of image means. 

717 data : `np.ndarray`, (nMeasurements, nCols) 

718 Array of overscan column means from each measurement. 

719 error : `float` 

720 Fixed error value. 

721 *args : 

722 Additional position arguments. 

723 **kwargs : 

724 Additional keyword arguments. 

725 

726 Returns 

727 ------- 

728 rms : `float` 

729 The rms error between the model and input data. 

730 """ 

731 model_results = self.model_results(params, signal, *args, **kwargs) 

732 

733 diff = model_results - data 

734 rms = np.sqrt(np.mean(np.square(diff))) 

735 

736 return rms 

737 

738 def difference(self, params, signal, data, error, *args, **kwargs): 

739 """Calculate the flattened difference array between model and data. 

740 

741 Parameters 

742 ---------- 

743 params : `lmfit.Parameters` 

744 Object containing the model parameters. 

745 signal : `np.ndarray`, (nMeasurements) 

746 Array of image means. 

747 data : `np.ndarray`, (nMeasurements, nCols) 

748 Array of overscan column means from each measurement. 

749 error : `float` 

750 Fixed error value. 

751 *args : 

752 Additional position arguments. 

753 **kwargs : 

754 Additional keyword arguments. 

755 

756 Returns 

757 ------- 

758 difference : `np.ndarray`, (nMeasurements*nCols) 

759 The rms error between the model and input data. 

760 """ 

761 model_results = self.model_results(params, signal, *args, **kwargs) 

762 diff = (model_results-data).flatten() 

763 

764 return diff 

765 

766 

767class SimpleModel(OverscanModel): 

768 """Simple analytic overscan model.""" 

769 

770 @staticmethod 

771 def model_results(params, signal, num_transfers, start=1, stop=10): 

772 """Generate a realization of the overscan model, using the specified 

773 fit parameters and input signal. 

774 

775 Parameters 

776 ---------- 

777 params : `lmfit.Parameters` 

778 Object containing the model parameters. 

779 signal : `np.ndarray`, (nMeasurements) 

780 Array of image means. 

781 num_transfers : `int` 

782 Number of serial transfers that the charge undergoes. 

783 start : `int`, optional 

784 First overscan column to fit. This number includes the 

785 last imaging column, and needs to be adjusted by one when 

786 using the overscan bounding box. 

787 stop : `int`, optional 

788 Last overscan column to fit. This number includes the 

789 last imaging column, and needs to be adjusted by one when 

790 using the overscan bounding box. 

791 

792 Returns 

793 ------- 

794 res : `np.ndarray`, (nMeasurements, nCols) 

795 Model results. 

796 """ 

797 v = params.valuesdict() 

798 v['cti'] = 10**v['ctiexp'] 

799 

800 # Adjust column numbering to match DM overscan bbox. 

801 start += 1 

802 stop += 1 

803 

804 x = np.arange(start, stop+1) 

805 res = np.zeros((signal.shape[0], x.shape[0])) 

806 

807 for i, s in enumerate(signal): 

808 # This is largely equivalent to equation 2. The minimum 

809 # indicates that a trap cannot emit more charge than is 

810 # available, nor can it emit more charge than it can hold. 

811 # This scales the exponential release of charge from the 

812 # trap. The next term defines the contribution from the 

813 # global CTI at each pixel transfer, and the final term 

814 # includes the contribution from local CTI effects. 

815 res[i, :] = (np.minimum(v['trapsize'], s*v['scaling']) 

816 * (np.exp(1/v['emissiontime']) - 1.0) 

817 * np.exp(-x/v['emissiontime']) 

818 + s*num_transfers*v['cti']**x 

819 + v['driftscale']*s*np.exp(-x/float(v['decaytime']))) 

820 

821 return res 

822 

823 

824class SimulatedModel(OverscanModel): 

825 """Simulated overscan model.""" 

826 

827 @staticmethod 

828 def model_results(params, signal, num_transfers, amp, start=1, stop=10, trap_type=None): 

829 """Generate a realization of the overscan model, using the specified 

830 fit parameters and input signal. 

831 

832 Parameters 

833 ---------- 

834 params : `lmfit.Parameters` 

835 Object containing the model parameters. 

836 signal : `np.ndarray`, (nMeasurements) 

837 Array of image means. 

838 num_transfers : `int` 

839 Number of serial transfers that the charge undergoes. 

840 amp : `lsst.afw.cameraGeom.Amplifier` 

841 Amplifier to use for geometry information. 

842 start : `int`, optional 

843 First overscan column to fit. This number includes the 

844 last imaging column, and needs to be adjusted by one when 

845 using the overscan bounding box. 

846 stop : `int`, optional 

847 Last overscan column to fit. This number includes the 

848 last imaging column, and needs to be adjusted by one when 

849 using the overscan bounding box. 

850 trap_type : `str`, optional 

851 Type of trap model to use. 

852 

853 Returns 

854 ------- 

855 results : `np.ndarray`, (nMeasurements, nCols) 

856 Model results. 

857 """ 

858 v = params.valuesdict() 

859 

860 # Adjust column numbering to match DM overscan bbox. 

861 start += 1 

862 stop += 1 

863 

864 # Electronics effect optimization 

865 output_amplifier = FloatingOutputAmplifier(1.0, v['driftscale'], v['decaytime']) 

866 

867 # CTI optimization 

868 v['cti'] = 10**v['ctiexp'] 

869 

870 # Trap type for optimization 

871 if trap_type is None: 

872 trap = None 

873 elif trap_type == 'linear': 

874 trap = SerialTrap(v['trapsize'], v['emissiontime'], 1, 'linear', 

875 [v['scaling']]) 

876 elif trap_type == 'logistic': 

877 trap = SerialTrap(v['trapsize'], v['emissiontime'], 1, 'logistic', 

878 [v['f0'], v['k']]) 

879 else: 

880 raise ValueError('Trap type must be linear or logistic or None') 

881 

882 # Simulate ramp readout 

883 imarr = np.zeros((signal.shape[0], amp.getRawDataBBox().getWidth())) 

884 ramp = SegmentSimulator(imarr, amp.getRawSerialPrescanBBox().getWidth(), output_amplifier, 

885 cti=v['cti'], traps=trap) 

886 ramp.ramp_exp(signal) 

887 model_results = ramp.readout(serial_overscan_width=amp.getRawSerialOverscanBBox().getWidth(), 

888 parallel_overscan_width=0) 

889 

890 ncols = amp.getRawSerialPrescanBBox().getWidth() + amp.getRawDataBBox().getWidth() 

891 

892 return model_results[:, ncols+start-1:ncols+stop] 

893 

894 

895class SegmentSimulator: 

896 """Controls the creation of simulated segment images. 

897 

898 Parameters 

899 ---------- 

900 imarr : `np.ndarray` (nx, ny) 

901 Image data array. 

902 prescan_width : `int` 

903 Number of serial prescan columns. 

904 output_amplifier : `lsst.cp.pipe.FloatingOutputAmplifier` 

905 An object holding the gain, read noise, and global_offset. 

906 cti : `float` 

907 Global CTI value. 

908 traps : `list` [`lsst.ip.isr.SerialTrap`] 

909 Serial traps to simulate. 

910 """ 

911 

912 def __init__(self, imarr, prescan_width, output_amplifier, cti=0.0, traps=None): 

913 # Image array geometry 

914 self.prescan_width = prescan_width 

915 self.ny, self.nx = imarr.shape 

916 

917 self.segarr = np.zeros((self.ny, self.nx+prescan_width)) 

918 self.segarr[:, prescan_width:] = imarr 

919 

920 # Serial readout information 

921 self.output_amplifier = output_amplifier 

922 if isinstance(cti, np.ndarray): 

923 raise ValueError("cti must be single value, not an array.") 

924 self.cti = cti 

925 

926 self.serial_traps = None 

927 self.do_trapping = False 

928 if traps is not None: 

929 if not isinstance(traps, list): 

930 traps = [traps] 

931 for trap in traps: 

932 self.add_trap(trap) 

933 

934 def add_trap(self, serial_trap): 

935 """Add a trap to the serial register. 

936 

937 Parameters 

938 ---------- 

939 serial_trap : `lsst.ip.isr.SerialTrap` 

940 The trap to add. 

941 """ 

942 try: 

943 self.serial_traps.append(serial_trap) 

944 except AttributeError: 

945 self.serial_traps = [serial_trap] 

946 self.do_trapping = True 

947 

948 def ramp_exp(self, signal_list): 

949 """Simulate an image with varying flux illumination per row. 

950 

951 This method simulates a segment image where the signal level 

952 increases along the horizontal direction, according to the 

953 provided list of signal levels. 

954 

955 Parameters 

956 ---------- 

957 signal_list : `list` [`float`] 

958 List of signal levels. 

959 

960 Raises 

961 ------ 

962 ValueError 

963 Raised if the length of the signal list does not equal the 

964 number of rows. 

965 """ 

966 if len(signal_list) != self.ny: 

967 raise ValueError("Signal list does not match row count.") 

968 

969 ramp = np.tile(signal_list, (self.nx, 1)).T 

970 self.segarr[:, self.prescan_width:] += ramp 

971 

972 def readout(self, serial_overscan_width=10, parallel_overscan_width=0): 

973 """Simulate serial readout of the segment image. 

974 

975 This method performs the serial readout of a segment image 

976 given the appropriate SerialRegister object and the properties 

977 of the ReadoutAmplifier. Additional arguments can be provided 

978 to account for the number of desired overscan transfers. The 

979 result is a simulated final segment image, in ADU. 

980 

981 Parameters 

982 ---------- 

983 serial_overscan_width : `int`, optional 

984 Number of serial overscan columns. 

985 parallel_overscan_width : `int`, optional 

986 Number of parallel overscan rows. 

987 

988 Returns 

989 ------- 

990 result : `np.ndarray` (nx, ny) 

991 Simulated image, including serial prescan, serial 

992 overscan, and parallel overscan regions. 

993 """ 

994 # Create output array 

995 iy = int(self.ny + parallel_overscan_width) 

996 ix = int(self.nx + self.prescan_width + serial_overscan_width) 

997 image = np.random.normal(loc=self.output_amplifier.global_offset, 

998 scale=self.output_amplifier.noise, 

999 size=(iy, ix)) 

1000 free_charge = copy.deepcopy(self.segarr) 

1001 

1002 # Set flow control parameters 

1003 do_trapping = self.do_trapping 

1004 cti = self.cti 

1005 

1006 offset = np.zeros(self.ny) 

1007 cte = 1 - cti 

1008 if do_trapping: 

1009 for trap in self.serial_traps: 

1010 trap.initialize(self.ny, self.nx, self.prescan_width) 

1011 

1012 for i in range(ix): 

1013 # Trap capture 

1014 if do_trapping: 

1015 for trap in self.serial_traps: 

1016 captured_charge = trap.trap_charge(free_charge) 

1017 free_charge -= captured_charge 

1018 

1019 # Pixel-to-pixel proportional loss 

1020 transferred_charge = free_charge*cte 

1021 deferred_charge = free_charge*cti 

1022 

1023 # Pixel transfer and readout 

1024 offset = self.output_amplifier.local_offset(offset, 

1025 transferred_charge[:, 0]) 

1026 image[:iy-parallel_overscan_width, i] += transferred_charge[:, 0] + offset 

1027 

1028 free_charge = np.pad(transferred_charge, ((0, 0), (0, 1)), 

1029 mode='constant')[:, 1:] + deferred_charge 

1030 

1031 # Trap emission 

1032 if do_trapping: 

1033 for trap in self.serial_traps: 

1034 released_charge = trap.release_charge() 

1035 free_charge += released_charge 

1036 

1037 return image/float(self.output_amplifier.gain) 

1038 

1039 

1040class FloatingOutputAmplifier: 

1041 """Object representing the readout amplifier of a single channel. 

1042 

1043 Parameters 

1044 ---------- 

1045 gain : `float` 

1046 Amplifier gain. 

1047 scale : `float` 

1048 Drift scale for the amplifier. 

1049 decay_time : `float` 

1050 Decay time for the bias drift. 

1051 noise : `float`, optional 

1052 Amplifier read noise. 

1053 offset : `float`, optional 

1054 Global CTI offset. 

1055 """ 

1056 

1057 def __init__(self, gain, scale, decay_time, noise=0.0, offset=0.0): 

1058 

1059 self.gain = gain 

1060 self.noise = noise 

1061 self.global_offset = offset 

1062 

1063 self.update_parameters(scale, decay_time) 

1064 

1065 def local_offset(self, old, signal): 

1066 """Calculate local offset hysteresis. 

1067 

1068 Parameters 

1069 ---------- 

1070 old : `np.ndarray`, (,) 

1071 Previous iteration. 

1072 signal : `np.ndarray`, (,) 

1073 Current column measurements. 

1074 

1075 Returns 

1076 ------- 

1077 offset : `np.ndarray` 

1078 Local offset. 

1079 """ 

1080 new = self.scale*signal 

1081 

1082 return np.maximum(new, old*np.exp(-1/self.decay_time)) 

1083 

1084 def update_parameters(self, scale, decay_time): 

1085 """Update parameter values, if within acceptable values. 

1086 

1087 Parameters 

1088 ---------- 

1089 scale : `float` 

1090 Drift scale for the amplifier. 

1091 decay_time : `float` 

1092 Decay time for the bias drift. 

1093 

1094 Raises 

1095 ------ 

1096 ValueError 

1097 Raised if the input parameters are out of range. 

1098 """ 

1099 if scale < 0.0: 

1100 raise ValueError("Scale must be greater than or equal to 0.") 

1101 if np.isnan(scale): 

1102 raise ValueError("Scale must be real-valued number, not NaN.") 

1103 self.scale = scale 

1104 if decay_time <= 0.0: 

1105 raise ValueError("Decay time must be greater than 0.") 

1106 if np.isnan(decay_time): 

1107 raise ValueError("Decay time must be real-valued number, not NaN.") 

1108 self.decay_time = decay_time