Coverage for python/lsst/cp/pipe/ptc/cpExtractPtcTask.py: 11%

346 statements  

« prev     ^ index     » next       coverage.py v7.3.0, created at 2023-08-19 12:04 +0000

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22import numpy as np 

23from lmfit.models import GaussianModel 

24import scipy.stats 

25import warnings 

26 

27import lsst.afw.math as afwMath 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30from lsst.cp.pipe.utils import (arrangeFlatsByExpTime, arrangeFlatsByExpId, 

31 arrangeFlatsByExpFlux, sigmaClipCorrection, 

32 CovFastFourierTransform) 

33 

34import lsst.pipe.base.connectionTypes as cT 

35 

36from lsst.ip.isr import PhotonTransferCurveDataset 

37from lsst.ip.isr import IsrTask 

38 

39__all__ = ['PhotonTransferCurveExtractConfig', 'PhotonTransferCurveExtractTask'] 

40 

41 

42class PhotonTransferCurveExtractConnections(pipeBase.PipelineTaskConnections, 

43 dimensions=("instrument", "detector")): 

44 

45 inputExp = cT.Input( 

46 name="ptcInputExposurePairs", 

47 doc="Input post-ISR processed exposure pairs (flats) to" 

48 "measure covariances from.", 

49 storageClass="Exposure", 

50 dimensions=("instrument", "exposure", "detector"), 

51 multiple=True, 

52 deferLoad=True, 

53 ) 

54 inputPhotodiodeData = cT.Input( 

55 name="photodiode", 

56 doc="Photodiode readings data.", 

57 storageClass="IsrCalib", 

58 dimensions=("instrument", "exposure"), 

59 multiple=True, 

60 deferLoad=True, 

61 ) 

62 taskMetadata = cT.Input( 

63 name="isr_metadata", 

64 doc="Input task metadata to extract statistics from.", 

65 storageClass="TaskMetadata", 

66 dimensions=("instrument", "exposure", "detector"), 

67 multiple=True, 

68 ) 

69 outputCovariances = cT.Output( 

70 name="ptcCovariances", 

71 doc="Extracted flat (co)variances.", 

72 storageClass="PhotonTransferCurveDataset", 

73 dimensions=("instrument", "exposure", "detector"), 

74 isCalibration=True, 

75 multiple=True, 

76 ) 

77 

78 def __init__(self, *, config=None): 

79 if not config.doExtractPhotodiodeData: 

80 del self.inputPhotodiodeData 

81 

82 

83class PhotonTransferCurveExtractConfig(pipeBase.PipelineTaskConfig, 

84 pipelineConnections=PhotonTransferCurveExtractConnections): 

85 """Configuration for the measurement of covariances from flats. 

86 """ 

87 matchExposuresType = pexConfig.ChoiceField( 

88 dtype=str, 

89 doc="Match input exposures by time, flux, or expId", 

90 default='TIME', 

91 allowed={ 

92 "TIME": "Match exposures by exposure time.", 

93 "FLUX": "Match exposures by target flux. Use header keyword" 

94 " in matchExposuresByFluxKeyword to find the flux.", 

95 "EXPID": "Match exposures by exposure ID." 

96 } 

97 ) 

98 matchExposuresByFluxKeyword = pexConfig.Field( 

99 dtype=str, 

100 doc="Header keyword for flux if matchExposuresType is FLUX.", 

101 default='CCOBFLUX', 

102 ) 

103 maximumRangeCovariancesAstier = pexConfig.Field( 

104 dtype=int, 

105 doc="Maximum range of covariances as in Astier+19", 

106 default=8, 

107 ) 

108 binSize = pexConfig.Field( 

109 dtype=int, 

110 doc="Bin the image by this factor in both dimensions.", 

111 default=1, 

112 ) 

113 minMeanSignal = pexConfig.DictField( 

114 keytype=str, 

115 itemtype=float, 

116 doc="Minimum values (inclusive) of mean signal (in ADU) per amp to use." 

117 " The same cut is applied to all amps if this parameter [`dict`] is passed as " 

118 " {'ALL_AMPS': value}", 

119 default={'ALL_AMPS': 0.0}, 

120 deprecated="This config has been moved to cpSolvePtcTask, and will be removed after v26.", 

121 ) 

122 maxMeanSignal = pexConfig.DictField( 

123 keytype=str, 

124 itemtype=float, 

125 doc="Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp." 

126 " The same cut is applied to all amps if this dictionary is of the form" 

127 " {'ALL_AMPS': value}", 

128 default={'ALL_AMPS': 1e6}, 

129 deprecated="This config has been moved to cpSolvePtcTask, and will be removed after v26.", 

130 ) 

131 maskNameList = pexConfig.ListField( 

132 dtype=str, 

133 doc="Mask list to exclude from statistics calculations.", 

134 default=['SUSPECT', 'BAD', 'NO_DATA', 'SAT'], 

135 ) 

136 nSigmaClipPtc = pexConfig.Field( 

137 dtype=float, 

138 doc="Sigma cut for afwMath.StatisticsControl()", 

139 default=5.5, 

140 ) 

141 nIterSigmaClipPtc = pexConfig.Field( 

142 dtype=int, 

143 doc="Number of sigma-clipping iterations for afwMath.StatisticsControl()", 

144 default=3, 

145 ) 

146 minNumberGoodPixelsForCovariance = pexConfig.Field( 

147 dtype=int, 

148 doc="Minimum number of acceptable good pixels per amp to calculate the covariances (via FFT or" 

149 " direclty).", 

150 default=10000, 

151 ) 

152 thresholdDiffAfwVarVsCov00 = pexConfig.Field( 

153 dtype=float, 

154 doc="If the absolute fractional differece between afwMath.VARIANCECLIP and Cov00 " 

155 "for a region of a difference image is greater than this threshold (percentage), " 

156 "a warning will be issued.", 

157 default=1., 

158 ) 

159 detectorMeasurementRegion = pexConfig.ChoiceField( 

160 dtype=str, 

161 doc="Region of each exposure where to perform the calculations (amplifier or full image).", 

162 default='AMP', 

163 allowed={ 

164 "AMP": "Amplifier of the detector.", 

165 "FULL": "Full image." 

166 } 

167 ) 

168 numEdgeSuspect = pexConfig.Field( 

169 dtype=int, 

170 doc="Number of edge pixels to be flagged as untrustworthy.", 

171 default=0, 

172 ) 

173 edgeMaskLevel = pexConfig.ChoiceField( 

174 dtype=str, 

175 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?", 

176 default="DETECTOR", 

177 allowed={ 

178 'DETECTOR': 'Mask only the edges of the full detector.', 

179 'AMP': 'Mask edges of each amplifier.', 

180 }, 

181 ) 

182 doGain = pexConfig.Field( 

183 dtype=bool, 

184 doc="Calculate a gain per input flat pair.", 

185 default=True, 

186 ) 

187 gainCorrectionType = pexConfig.ChoiceField( 

188 dtype=str, 

189 doc="Correction type for the gain.", 

190 default='FULL', 

191 allowed={ 

192 'NONE': 'No correction.', 

193 'SIMPLE': 'First order correction.', 

194 'FULL': 'Second order correction.' 

195 } 

196 ) 

197 ksHistNBins = pexConfig.Field( 

198 dtype=int, 

199 doc="Number of bins for the KS test histogram.", 

200 default=100, 

201 ) 

202 ksHistLimitMultiplier = pexConfig.Field( 

203 dtype=float, 

204 doc="Number of sigma (as predicted from the mean value) to compute KS test histogram.", 

205 default=8.0, 

206 ) 

207 ksHistMinDataValues = pexConfig.Field( 

208 dtype=int, 

209 doc="Minimum number of good data values to compute KS test histogram.", 

210 default=100, 

211 ) 

212 auxiliaryHeaderKeys = pexConfig.ListField( 

213 dtype=str, 

214 doc="Auxiliary header keys to store with the PTC dataset.", 

215 default=[], 

216 ) 

217 doExtractPhotodiodeData = pexConfig.Field( 

218 dtype=bool, 

219 doc="Extract photodiode data?", 

220 default=False, 

221 ) 

222 photodiodeIntegrationMethod = pexConfig.ChoiceField( 

223 dtype=str, 

224 doc="Integration method for photodiode monitoring data.", 

225 default="CHARGE_SUM", 

226 allowed={ 

227 "DIRECT_SUM": ("Use numpy's trapz integrator on all photodiode " 

228 "readout entries"), 

229 "TRIMMED_SUM": ("Use numpy's trapz integrator, clipping the " 

230 "leading and trailing entries, which are " 

231 "nominally at zero baseline level."), 

232 "CHARGE_SUM": ("Treat the current values as integrated charge " 

233 "over the sampling interval and simply sum " 

234 "the values, after subtracting a baseline level."), 

235 }, 

236 ) 

237 photodiodeCurrentScale = pexConfig.Field( 

238 dtype=float, 

239 doc="Scale factor to apply to photodiode current values for the " 

240 "``CHARGE_SUM`` integration method.", 

241 default=-1.0, 

242 ) 

243 

244 

245class PhotonTransferCurveExtractTask(pipeBase.PipelineTask): 

246 """Task to measure covariances from flat fields. 

247 

248 This task receives as input a list of flat-field images 

249 (flats), and sorts these flats in pairs taken at the 

250 same time (the task will raise if there is one one flat 

251 at a given exposure time, and it will discard extra flats if 

252 there are more than two per exposure time). This task measures 

253 the mean, variance, and covariances from a region (e.g., 

254 an amplifier) of the difference image of the two flats with 

255 the same exposure time (alternatively, all input images could have 

256 the same exposure time but their flux changed). 

257 

258 The variance is calculated via afwMath, and the covariance 

259 via the methods in Astier+19 (appendix A). In theory, 

260 var = covariance[0,0]. This should be validated, and in the 

261 future, we may decide to just keep one (covariance). 

262 At this moment, if the two values differ by more than the value 

263 of `thresholdDiffAfwVarVsCov00` (default: 1%), a warning will 

264 be issued. 

265 

266 The measured covariances at a given exposure time (along with 

267 other quantities such as the mean) are stored in a PTC dataset 

268 object (`~lsst.ip.isr.PhotonTransferCurveDataset`), which gets 

269 partially filled at this stage (the remainder of the attributes 

270 of the dataset will be filled after running the second task of 

271 the PTC-measurement pipeline, `~PhotonTransferCurveSolveTask`). 

272 

273 The number of partially-filled 

274 `~lsst.ip.isr.PhotonTransferCurveDataset` objects will be less 

275 than the number of input exposures because the task combines 

276 input flats in pairs. However, it is required at this moment 

277 that the number of input dimensions matches 

278 bijectively the number of output dimensions. Therefore, a number 

279 of "dummy" PTC datasets are inserted in the output list. This 

280 output list will then be used as input of the next task in the 

281 PTC-measurement pipeline, `PhotonTransferCurveSolveTask`, 

282 which will assemble the multiple `PhotonTransferCurveDataset` 

283 objects into a single one in order to fit the measured covariances 

284 as a function of flux to one of three models 

285 (see `PhotonTransferCurveSolveTask` for details). 

286 

287 Reference: Astier+19: "The Shape of the Photon Transfer Curve of CCD 

288 sensors", arXiv:1905.08677. 

289 """ 

290 

291 ConfigClass = PhotonTransferCurveExtractConfig 

292 _DefaultName = 'cpPtcExtract' 

293 

294 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

295 """Ensure that the input and output dimensions are passed along. 

296 

297 Parameters 

298 ---------- 

299 butlerQC : `~lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext` 

300 Butler to operate on. 

301 inputRefs : `~lsst.pipe.base.connections.InputQuantizedConnection` 

302 Input data refs to load. 

303 ouptutRefs : `~lsst.pipe.base.connections.OutputQuantizedConnection` 

304 Output data refs to persist. 

305 """ 

306 inputs = butlerQC.get(inputRefs) 

307 # Ids of input list of exposure references 

308 # (deferLoad=True in the input connections) 

309 inputs['inputDims'] = [expRef.datasetRef.dataId['exposure'] for expRef in inputRefs.inputExp] 

310 

311 # Dictionary, keyed by expTime (or expFlux or expId), with tuples 

312 # containing flat exposures and their IDs. 

313 matchType = self.config.matchExposuresType 

314 if matchType == 'TIME': 

315 inputs['inputExp'] = arrangeFlatsByExpTime(inputs['inputExp'], inputs['inputDims']) 

316 elif matchType == 'FLUX': 

317 inputs['inputExp'] = arrangeFlatsByExpFlux(inputs['inputExp'], inputs['inputDims'], 

318 self.config.matchExposuresByFluxKeyword) 

319 else: 

320 inputs['inputExp'] = arrangeFlatsByExpId(inputs['inputExp'], inputs['inputDims']) 

321 

322 outputs = self.run(**inputs) 

323 outputs = self._guaranteeOutputs(inputs['inputDims'], outputs, outputRefs) 

324 butlerQC.put(outputs, outputRefs) 

325 

326 def _guaranteeOutputs(self, inputDims, outputs, outputRefs): 

327 """Ensure that all outputRefs have a matching output, and if they do 

328 not, fill the output with dummy PTC datasets. 

329 

330 Parameters 

331 ---------- 

332 inputDims : `dict` [`str`, `int`] 

333 Input exposure dimensions. 

334 outputs : `lsst.pipe.base.Struct` 

335 Outputs from the ``run`` method. Contains the entry: 

336 

337 ``outputCovariances`` 

338 Output PTC datasets (`list` [`lsst.ip.isr.IsrCalib`]) 

339 outputRefs : `~lsst.pipe.base.connections.OutputQuantizedConnection` 

340 Container with all of the outputs expected to be generated. 

341 

342 Returns 

343 ------- 

344 outputs : `lsst.pipe.base.Struct` 

345 Dummy dataset padded version of the input ``outputs`` with 

346 the same entries. 

347 """ 

348 newCovariances = [] 

349 for ref in outputRefs.outputCovariances: 

350 outputExpId = ref.dataId['exposure'] 

351 if outputExpId in inputDims: 

352 entry = inputDims.index(outputExpId) 

353 newCovariances.append(outputs.outputCovariances[entry]) 

354 else: 

355 newPtc = PhotonTransferCurveDataset(['no amp'], 'DUMMY', 1) 

356 newPtc.setAmpValuesPartialDataset('no amp') 

357 newCovariances.append(newPtc) 

358 return pipeBase.Struct(outputCovariances=newCovariances) 

359 

360 def run(self, inputExp, inputDims, taskMetadata, inputPhotodiodeData=None): 

361 

362 """Measure covariances from difference of flat pairs 

363 

364 Parameters 

365 ---------- 

366 inputExp : `dict` [`float`, `list` 

367 [`~lsst.pipe.base.connections.DeferredDatasetRef`]] 

368 Dictionary that groups references to flat-field exposures that 

369 have the same exposure time (seconds), or that groups them 

370 sequentially by their exposure id. 

371 inputDims : `list` 

372 List of exposure IDs. 

373 taskMetadata : `list` [`lsst.pipe.base.TaskMetadata`] 

374 List of exposures metadata from ISR. 

375 inputPhotodiodeData : `dict` [`str`, `lsst.ip.isr.PhotodiodeCalib`] 

376 Photodiode readings data (optional). 

377 

378 Returns 

379 ------- 

380 results : `lsst.pipe.base.Struct` 

381 The resulting Struct contains: 

382 

383 ``outputCovariances`` 

384 A list containing the per-pair PTC measurements (`list` 

385 [`lsst.ip.isr.PhotonTransferCurveDataset`]) 

386 """ 

387 # inputExp.values() returns a view, which we turn into a list. We then 

388 # access the first exposure-ID tuple to get the detector. 

389 # The first "get()" retrieves the exposure from the exposure reference. 

390 detector = list(inputExp.values())[0][0][0].get(component='detector') 

391 detNum = detector.getId() 

392 amps = detector.getAmplifiers() 

393 ampNames = [amp.getName() for amp in amps] 

394 

395 # Each amp may have a different min and max ADU signal 

396 # specified in the config. 

397 maxMeanSignalDict = {ampName: 1e6 for ampName in ampNames} 

398 minMeanSignalDict = {ampName: 0.0 for ampName in ampNames} 

399 for ampName in ampNames: 

400 if 'ALL_AMPS' in self.config.maxMeanSignal: 

401 maxMeanSignalDict[ampName] = self.config.maxMeanSignal['ALL_AMPS'] 

402 elif ampName in self.config.maxMeanSignal: 

403 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName] 

404 

405 if 'ALL_AMPS' in self.config.minMeanSignal: 

406 minMeanSignalDict[ampName] = self.config.minMeanSignal['ALL_AMPS'] 

407 elif ampName in self.config.minMeanSignal: 

408 minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName] 

409 # These are the column names for `tupleRows` below. 

410 tags = [('mu', '<f8'), ('afwVar', '<f8'), ('i', '<i8'), ('j', '<i8'), ('var', '<f8'), 

411 ('cov', '<f8'), ('npix', '<i8'), ('ext', '<i8'), ('expTime', '<f8'), ('ampName', '<U3')] 

412 # Create a dummy ptcDataset. Dummy datasets will be 

413 # used to ensure that the number of output and input 

414 # dimensions match. 

415 dummyPtcDataset = PhotonTransferCurveDataset(ampNames, 'DUMMY', 

416 self.config.maximumRangeCovariancesAstier) 

417 for ampName in ampNames: 

418 dummyPtcDataset.setAmpValuesPartialDataset(ampName) 

419 

420 # Extract the photodiode data if requested. 

421 if self.config.doExtractPhotodiodeData: 

422 # Compute the photodiode integrals once, at the start. 

423 monitorDiodeCharge = {} 

424 for handle in inputPhotodiodeData: 

425 expId = handle.dataId['exposure'] 

426 pdCalib = handle.get() 

427 pdCalib.integrationMethod = self.config.photodiodeIntegrationMethod 

428 pdCalib.currentScale = self.config.photodiodeCurrentScale 

429 monitorDiodeCharge[expId] = pdCalib.integrate() 

430 

431 # Get read noise. Try from the exposure, then try 

432 # taskMetadata. This adds a get() for the exposures. 

433 readNoiseLists = {} 

434 for pairIndex, expRefs in inputExp.items(): 

435 # This yields an index (exposure_time, seq_num, or flux) 

436 # and a pair of references at that index. 

437 for expRef, expId in expRefs: 

438 # This yields an exposure ref and an exposureId. 

439 exposureMetadata = expRef.get(component="metadata") 

440 metadataIndex = inputDims.index(expId) 

441 thisTaskMetadata = taskMetadata[metadataIndex] 

442 

443 for ampName in ampNames: 

444 if ampName not in readNoiseLists: 

445 readNoiseLists[ampName] = [self.getReadNoise(exposureMetadata, 

446 thisTaskMetadata, ampName)] 

447 else: 

448 readNoiseLists[ampName].append(self.getReadNoise(exposureMetadata, 

449 thisTaskMetadata, ampName)) 

450 

451 readNoiseDict = {ampName: 0.0 for ampName in ampNames} 

452 for ampName in ampNames: 

453 # Take median read noise value 

454 readNoiseDict[ampName] = np.nanmedian(readNoiseLists[ampName]) 

455 

456 # Output list with PTC datasets. 

457 partialPtcDatasetList = [] 

458 # The number of output references needs to match that of input 

459 # references: initialize outputlist with dummy PTC datasets. 

460 for i in range(len(inputDims)): 

461 partialPtcDatasetList.append(dummyPtcDataset) 

462 

463 if self.config.numEdgeSuspect > 0: 

464 isrTask = IsrTask() 

465 self.log.info("Masking %d pixels from the edges of all %ss as SUSPECT.", 

466 self.config.numEdgeSuspect, self.config.edgeMaskLevel) 

467 

468 # Depending on the value of config.matchExposuresType 

469 # 'expTime' can stand for exposure time, flux, or ID. 

470 for expTime in inputExp: 

471 exposures = inputExp[expTime] 

472 if len(exposures) == 1: 

473 self.log.warning("Only one exposure found at %s %f. Dropping exposure %d.", 

474 self.config.matchExposuresType, expTime, exposures[0][1]) 

475 continue 

476 else: 

477 # Only use the first two exposures at expTime. Each 

478 # element is a tuple (exposure, expId) 

479 expRef1, expId1 = exposures[0] 

480 expRef2, expId2 = exposures[1] 

481 # use get() to obtain `lsst.afw.image.Exposure` 

482 exp1, exp2 = expRef1.get(), expRef2.get() 

483 

484 if len(exposures) > 2: 

485 self.log.warning("Already found 2 exposures at %s %f. Ignoring exposures: %s", 

486 self.config.matchExposuresType, expTime, 

487 ", ".join(str(i[1]) for i in exposures[2:])) 

488 # Mask pixels at the edge of the detector or of each amp 

489 if self.config.numEdgeSuspect > 0: 

490 isrTask.maskEdges(exp1, numEdgePixels=self.config.numEdgeSuspect, 

491 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

492 isrTask.maskEdges(exp2, numEdgePixels=self.config.numEdgeSuspect, 

493 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

494 

495 # Extract any metadata keys from the headers. 

496 auxDict = {} 

497 metadata = exp1.getMetadata() 

498 for key in self.config.auxiliaryHeaderKeys: 

499 if key not in metadata: 

500 self.log.warning( 

501 "Requested auxiliary keyword %s not found in exposure metadata for %d", 

502 key, 

503 expId1, 

504 ) 

505 value = np.nan 

506 else: 

507 value = metadata[key] 

508 

509 auxDict[key] = value 

510 

511 nAmpsNan = 0 

512 partialPtcDataset = PhotonTransferCurveDataset(ampNames, 'PARTIAL', 

513 self.config.maximumRangeCovariancesAstier) 

514 for ampNumber, amp in enumerate(detector): 

515 ampName = amp.getName() 

516 if self.config.detectorMeasurementRegion == 'AMP': 

517 region = amp.getBBox() 

518 elif self.config.detectorMeasurementRegion == 'FULL': 

519 region = None 

520 

521 # Get masked image regions, masking planes, statistic control 

522 # objects, and clipped means. Calculate once to reuse in 

523 # `measureMeanVarCov` and `getGainFromFlatPair`. 

524 im1Area, im2Area, imStatsCtrl, mu1, mu2 = self.getImageAreasMasksStats(exp1, exp2, 

525 region=region) 

526 

527 # We demand that both mu1 and mu2 be finite and greater than 0. 

528 if not np.isfinite(mu1) or not np.isfinite(mu2) \ 

529 or ((np.nan_to_num(mu1) + np.nan_to_num(mu2)/2.) <= 0.0): 

530 self.log.warning( 

531 "Illegal mean value(s) detected for amp %s on exposure pair %d/%d", 

532 ampName, 

533 expId1, 

534 expId2, 

535 ) 

536 partialPtcDataset.setAmpValuesPartialDataset( 

537 ampName, 

538 inputExpIdPair=(expId1, expId2), 

539 rawExpTime=expTime, 

540 expIdMask=False, 

541 ) 

542 continue 

543 

544 # `measureMeanVarCov` is the function that measures 

545 # the variance and covariances from a region of 

546 # the difference image of two flats at the same 

547 # exposure time. The variable `covAstier` that is 

548 # returned is of the form: 

549 # [(i, j, var (cov[0,0]), cov, npix) for (i,j) in 

550 # {maxLag, maxLag}^2]. 

551 muDiff, varDiff, covAstier = self.measureMeanVarCov(im1Area, im2Area, imStatsCtrl, mu1, mu2) 

552 # Estimate the gain from the flat pair 

553 if self.config.doGain: 

554 gain = self.getGainFromFlatPair(im1Area, im2Area, imStatsCtrl, mu1, mu2, 

555 correctionType=self.config.gainCorrectionType, 

556 readNoise=readNoiseDict[ampName]) 

557 else: 

558 gain = np.nan 

559 

560 # Correction factor for bias introduced by sigma 

561 # clipping. 

562 # Function returns 1/sqrt(varFactor), so it needs 

563 # to be squared. varDiff is calculated via 

564 # afwMath.VARIANCECLIP. 

565 varFactor = sigmaClipCorrection(self.config.nSigmaClipPtc)**2 

566 varDiff *= varFactor 

567 

568 expIdMask = True 

569 # Mask data point at this mean signal level if 

570 # the signal, variance, or covariance calculations 

571 # from `measureMeanVarCov` resulted in NaNs. 

572 if np.isnan(muDiff) or np.isnan(varDiff) or (covAstier is None): 

573 self.log.warning("NaN mean or var, or None cov in amp %s in exposure pair %d, %d of " 

574 "detector %d.", ampName, expId1, expId2, detNum) 

575 nAmpsNan += 1 

576 expIdMask = False 

577 covArray = np.full((1, self.config.maximumRangeCovariancesAstier, 

578 self.config.maximumRangeCovariancesAstier), np.nan) 

579 covSqrtWeights = np.full_like(covArray, np.nan) 

580 

581 # Mask data point if it is outside of the 

582 # specified mean signal range. 

583 if (muDiff <= minMeanSignalDict[ampName]) or (muDiff >= maxMeanSignalDict[ampName]): 

584 expIdMask = False 

585 

586 if covAstier is not None: 

587 # Turn the tuples with the measured information 

588 # into covariance arrays. 

589 # covrow: (i, j, var (cov[0,0]), cov, npix) 

590 tupleRows = [(muDiff, varDiff) + covRow + (ampNumber, expTime, 

591 ampName) for covRow in covAstier] 

592 tempStructArray = np.array(tupleRows, dtype=tags) 

593 

594 covArray, vcov, _ = self.makeCovArray(tempStructArray, 

595 self.config.maximumRangeCovariancesAstier) 

596 

597 # The returned covArray should only have 1 entry; 

598 # raise if this is not the case. 

599 if covArray.shape[0] != 1: 

600 raise RuntimeError("Serious programming error in covArray shape.") 

601 

602 covSqrtWeights = np.nan_to_num(1./np.sqrt(vcov)) 

603 

604 # Correct covArray for sigma clipping: 

605 # 1) Apply varFactor twice for the whole covariance matrix 

606 covArray *= varFactor**2 

607 # 2) But, only once for the variance element of the 

608 # matrix, covArray[0, 0, 0] (so divide one factor out). 

609 # (the first 0 is because this is a 3D array for insertion into 

610 # the combined dataset). 

611 covArray[0, 0, 0] /= varFactor 

612 

613 if expIdMask: 

614 # Run the Gaussian histogram only if this is a legal 

615 # amplifier. 

616 histVar, histChi2Dof, kspValue = self.computeGaussianHistogramParameters( 

617 im1Area, 

618 im2Area, 

619 imStatsCtrl, 

620 mu1, 

621 mu2, 

622 ) 

623 else: 

624 histVar = np.nan 

625 histChi2Dof = np.nan 

626 kspValue = 0.0 

627 

628 if self.config.doExtractPhotodiodeData: 

629 nExps = 0 

630 photoCharge = 0.0 

631 for expId in [expId1, expId2]: 

632 if expId in monitorDiodeCharge: 

633 photoCharge += monitorDiodeCharge[expId] 

634 nExps += 1 

635 if nExps > 0: 

636 photoCharge /= nExps 

637 else: 

638 photoCharge = np.nan 

639 else: 

640 photoCharge = np.nan 

641 

642 partialPtcDataset.setAmpValuesPartialDataset( 

643 ampName, 

644 inputExpIdPair=(expId1, expId2), 

645 rawExpTime=expTime, 

646 rawMean=muDiff, 

647 rawVar=varDiff, 

648 photoCharge=photoCharge, 

649 expIdMask=expIdMask, 

650 covariance=covArray[0, :, :], 

651 covSqrtWeights=covSqrtWeights[0, :, :], 

652 gain=gain, 

653 noise=readNoiseDict[ampName], 

654 histVar=histVar, 

655 histChi2Dof=histChi2Dof, 

656 kspValue=kspValue, 

657 ) 

658 

659 partialPtcDataset.setAuxValuesPartialDataset(auxDict) 

660 

661 # Use location of exp1 to save PTC dataset from (exp1, exp2) pair. 

662 # Below, np.where(expId1 == np.array(inputDims)) returns a tuple 

663 # with a single-element array, so [0][0] 

664 # is necessary to extract the required index. 

665 datasetIndex = np.where(expId1 == np.array(inputDims))[0][0] 

666 # `partialPtcDatasetList` is a list of 

667 # `PhotonTransferCurveDataset` objects. Some of them 

668 # will be dummy datasets (to match length of input 

669 # and output references), and the rest will have 

670 # datasets with the mean signal, variance, and 

671 # covariance measurements at a given exposure 

672 # time. The next ppart of the PTC-measurement 

673 # pipeline, `solve`, will take this list as input, 

674 # and assemble the measurements in the datasets 

675 # in an addecuate manner for fitting a PTC 

676 # model. 

677 partialPtcDataset.updateMetadataFromExposures([exp1, exp2]) 

678 partialPtcDataset.updateMetadata(setDate=True, detector=detector) 

679 partialPtcDatasetList[datasetIndex] = partialPtcDataset 

680 

681 if nAmpsNan == len(ampNames): 

682 msg = f"NaN mean in all amps of exposure pair {expId1}, {expId2} of detector {detNum}." 

683 self.log.warning(msg) 

684 

685 return pipeBase.Struct( 

686 outputCovariances=partialPtcDatasetList, 

687 ) 

688 

689 def makeCovArray(self, inputTuple, maxRangeFromTuple): 

690 """Make covariances array from tuple. 

691 

692 Parameters 

693 ---------- 

694 inputTuple : `numpy.ndarray` 

695 Structured array with rows with at least 

696 (mu, afwVar, cov, var, i, j, npix), where: 

697 mu : `float` 

698 0.5*(m1 + m2), where mu1 is the mean value of flat1 

699 and mu2 is the mean value of flat2. 

700 afwVar : `float` 

701 Variance of difference flat, calculated with afw. 

702 cov : `float` 

703 Covariance value at lag(i, j) 

704 var : `float` 

705 Variance(covariance value at lag(0, 0)) 

706 i : `int` 

707 Lag in dimension "x". 

708 j : `int` 

709 Lag in dimension "y". 

710 npix : `int` 

711 Number of pixels used for covariance calculation. 

712 maxRangeFromTuple : `int` 

713 Maximum range to select from tuple. 

714 

715 Returns 

716 ------- 

717 cov : `numpy.array` 

718 Covariance arrays, indexed by mean signal mu. 

719 vCov : `numpy.array` 

720 Variance of the [co]variance arrays, indexed by mean signal mu. 

721 muVals : `numpy.array` 

722 List of mean signal values. 

723 """ 

724 if maxRangeFromTuple is not None: 

725 cut = (inputTuple['i'] < maxRangeFromTuple) & (inputTuple['j'] < maxRangeFromTuple) 

726 cutTuple = inputTuple[cut] 

727 else: 

728 cutTuple = inputTuple 

729 # increasing mu order, so that we can group measurements with the 

730 # same mu 

731 muTemp = cutTuple['mu'] 

732 ind = np.argsort(muTemp) 

733 

734 cutTuple = cutTuple[ind] 

735 # should group measurements on the same image pairs(same average) 

736 mu = cutTuple['mu'] 

737 xx = np.hstack(([mu[0]], mu)) 

738 delta = xx[1:] - xx[:-1] 

739 steps, = np.where(delta > 0) 

740 ind = np.zeros_like(mu, dtype=int) 

741 ind[steps] = 1 

742 ind = np.cumsum(ind) # this acts as an image pair index. 

743 # now fill the 3-d cov array(and variance) 

744 muVals = np.array(np.unique(mu)) 

745 i = cutTuple['i'].astype(int) 

746 j = cutTuple['j'].astype(int) 

747 c = 0.5*cutTuple['cov'] 

748 n = cutTuple['npix'] 

749 v = 0.5*cutTuple['var'] 

750 # book and fill 

751 cov = np.ndarray((len(muVals), np.max(i)+1, np.max(j)+1)) 

752 var = np.zeros_like(cov) 

753 cov[ind, i, j] = c 

754 var[ind, i, j] = v**2/n 

755 var[:, 0, 0] *= 2 # var(v) = 2*v**2/N 

756 

757 return cov, var, muVals 

758 

759 def measureMeanVarCov(self, im1Area, im2Area, imStatsCtrl, mu1, mu2): 

760 """Calculate the mean of each of two exposures and the variance 

761 and covariance of their difference. The variance is calculated 

762 via afwMath, and the covariance via the methods in Astier+19 

763 (appendix A). In theory, var = covariance[0,0]. This should 

764 be validated, and in the future, we may decide to just keep 

765 one (covariance). 

766 

767 Parameters 

768 ---------- 

769 im1Area : `lsst.afw.image.maskedImage.MaskedImageF` 

770 Masked image from exposure 1. 

771 im2Area : `lsst.afw.image.maskedImage.MaskedImageF` 

772 Masked image from exposure 2. 

773 imStatsCtrl : `lsst.afw.math.StatisticsControl` 

774 Statistics control object. 

775 mu1: `float` 

776 Clipped mean of im1Area (ADU). 

777 mu2: `float` 

778 Clipped mean of im2Area (ADU). 

779 

780 Returns 

781 ------- 

782 mu : `float` or `NaN` 

783 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means 

784 of the regions in both exposures. If either mu1 or m2 are 

785 NaN's, the returned value is NaN. 

786 varDiff : `float` or `NaN` 

787 Half of the clipped variance of the difference of the 

788 regions inthe two input exposures. If either mu1 or m2 are 

789 NaN's, the returned value is NaN. 

790 covDiffAstier : `list` or `NaN` 

791 List with tuples of the form (dx, dy, var, cov, npix), where: 

792 dx : `int` 

793 Lag in x 

794 dy : `int` 

795 Lag in y 

796 var : `float` 

797 Variance at (dx, dy). 

798 cov : `float` 

799 Covariance at (dx, dy). 

800 nPix : `int` 

801 Number of pixel pairs used to evaluate var and cov. 

802 

803 If either mu1 or m2 are NaN's, the returned value is NaN. 

804 """ 

805 if np.isnan(mu1) or np.isnan(mu2): 

806 self.log.warning("Mean of amp in image 1 or 2 is NaN: %f, %f.", mu1, mu2) 

807 return np.nan, np.nan, None 

808 mu = 0.5*(mu1 + mu2) 

809 

810 # Take difference of pairs 

811 # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2)) 

812 temp = im2Area.clone() 

813 temp *= mu1 

814 diffIm = im1Area.clone() 

815 diffIm *= mu2 

816 diffIm -= temp 

817 diffIm /= mu 

818 

819 if self.config.binSize > 1: 

820 diffIm = afwMath.binImage(diffIm, self.config.binSize) 

821 

822 # Variance calculation via afwMath 

823 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, imStatsCtrl).getValue()) 

824 

825 # Covariances calculations 

826 # Get the pixels that were not clipped 

827 varClip = afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, imStatsCtrl).getValue() 

828 meanClip = afwMath.makeStatistics(diffIm, afwMath.MEANCLIP, imStatsCtrl).getValue() 

829 cut = meanClip + self.config.nSigmaClipPtc*np.sqrt(varClip) 

830 unmasked = np.where(np.fabs(diffIm.image.array) <= cut, 1, 0) 

831 

832 # Get the pixels in the mask planes of the difference image 

833 # that were ignored by the clipping algorithm 

834 wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0) 

835 # Combine the two sets of pixels ('1': use; '0': don't use) 

836 # into a final weight matrix to be used in the covariance 

837 # calculations below. 

838 w = unmasked*wDiff 

839 

840 if np.sum(w) < self.config.minNumberGoodPixelsForCovariance/(self.config.binSize**2): 

841 self.log.warning("Number of good points for covariance calculation (%s) is less " 

842 "(than threshold %s)", np.sum(w), 

843 self.config.minNumberGoodPixelsForCovariance/(self.config.binSize**2)) 

844 return np.nan, np.nan, None 

845 

846 maxRangeCov = self.config.maximumRangeCovariancesAstier 

847 

848 # Calculate covariances via FFT. 

849 shapeDiff = np.array(diffIm.image.array.shape) 

850 # Calculate the sizes of FFT dimensions. 

851 s = shapeDiff + maxRangeCov 

852 tempSize = np.array(np.log(s)/np.log(2.)).astype(int) 

853 fftSize = np.array(2**(tempSize+1)).astype(int) 

854 fftShape = (fftSize[0], fftSize[1]) 

855 c = CovFastFourierTransform(diffIm.image.array, w, fftShape, maxRangeCov) 

856 # np.sum(w) is the same as npix[0][0] returned in covDiffAstier 

857 try: 

858 covDiffAstier = c.reportCovFastFourierTransform(maxRangeCov) 

859 except ValueError: 

860 # This is raised if there are not enough pixels. 

861 self.log.warning("Not enough pixels covering the requested covariance range in x/y (%d)", 

862 self.config.maximumRangeCovariancesAstier) 

863 return np.nan, np.nan, None 

864 

865 # Compare Cov[0,0] and afwMath.VARIANCECLIP covDiffAstier[0] 

866 # is the Cov[0,0] element, [3] is the variance, and there's a 

867 # factor of 0.5 difference with afwMath.VARIANCECLIP. 

868 thresholdPercentage = self.config.thresholdDiffAfwVarVsCov00 

869 fractionalDiff = 100*np.fabs(1 - varDiff/(covDiffAstier[0][3]*0.5)) 

870 if fractionalDiff >= thresholdPercentage: 

871 self.log.warning("Absolute fractional difference between afwMatch.VARIANCECLIP and Cov[0,0] " 

872 "is more than %f%%: %f", thresholdPercentage, fractionalDiff) 

873 

874 return mu, varDiff, covDiffAstier 

875 

876 def getImageAreasMasksStats(self, exposure1, exposure2, region=None): 

877 """Get image areas in a region as well as masks and statistic objects. 

878 

879 Parameters 

880 ---------- 

881 exposure1 : `lsst.afw.image.ExposureF` 

882 First exposure of flat field pair. 

883 exposure2 : `lsst.afw.image.ExposureF` 

884 Second exposure of flat field pair. 

885 region : `lsst.geom.Box2I`, optional 

886 Region of each exposure where to perform the calculations 

887 (e.g, an amplifier). 

888 

889 Returns 

890 ------- 

891 im1Area : `lsst.afw.image.MaskedImageF` 

892 Masked image from exposure 1. 

893 im2Area : `lsst.afw.image.MaskedImageF` 

894 Masked image from exposure 2. 

895 imStatsCtrl : `lsst.afw.math.StatisticsControl` 

896 Statistics control object. 

897 mu1 : `float` 

898 Clipped mean of im1Area (ADU). 

899 mu2 : `float` 

900 Clipped mean of im2Area (ADU). 

901 """ 

902 if region is not None: 

903 im1Area = exposure1.maskedImage[region] 

904 im2Area = exposure2.maskedImage[region] 

905 else: 

906 im1Area = exposure1.maskedImage 

907 im2Area = exposure2.maskedImage 

908 

909 # Get mask planes and construct statistics control object from one 

910 # of the exposures 

911 imMaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList) 

912 imStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc, 

913 self.config.nIterSigmaClipPtc, 

914 imMaskVal) 

915 imStatsCtrl.setNanSafe(True) 

916 imStatsCtrl.setAndMask(imMaskVal) 

917 

918 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, imStatsCtrl).getValue() 

919 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, imStatsCtrl).getValue() 

920 

921 return (im1Area, im2Area, imStatsCtrl, mu1, mu2) 

922 

923 def getGainFromFlatPair(self, im1Area, im2Area, imStatsCtrl, mu1, mu2, 

924 correctionType='NONE', readNoise=None): 

925 """Estimate the gain from a single pair of flats. 

926 

927 The basic premise is 1/g = <(I1 - I2)^2/(I1 + I2)> = 1/const, 

928 where I1 and I2 correspond to flats 1 and 2, respectively. 

929 Corrections for the variable QE and the read-noise are then 

930 made following the derivation in Robert Lupton's forthcoming 

931 book, which gets 

932 

933 1/g = <(I1 - I2)^2/(I1 + I2)> - 1/mu(sigma^2 - 1/2g^2). 

934 

935 This is a quadratic equation, whose solutions are given by: 

936 

937 g = mu +/- sqrt(2*sigma^2 - 2*const*mu + mu^2)/(2*const*mu*2 

938 - 2*sigma^2) 

939 

940 where 'mu' is the average signal level and 'sigma' is the 

941 amplifier's readnoise. The positive solution will be used. 

942 The way the correction is applied depends on the value 

943 supplied for correctionType. 

944 

945 correctionType is one of ['NONE', 'SIMPLE' or 'FULL'] 

946 'NONE' : uses the 1/g = <(I1 - I2)^2/(I1 + I2)> formula. 

947 'SIMPLE' : uses the gain from the 'NONE' method for the 

948 1/2g^2 term. 

949 'FULL' : solves the full equation for g, discarding the 

950 non-physical solution to the resulting quadratic. 

951 

952 Parameters 

953 ---------- 

954 im1Area : `lsst.afw.image.maskedImage.MaskedImageF` 

955 Masked image from exposure 1. 

956 im2Area : `lsst.afw.image.maskedImage.MaskedImageF` 

957 Masked image from exposure 2. 

958 imStatsCtrl : `lsst.afw.math.StatisticsControl` 

959 Statistics control object. 

960 mu1: `float` 

961 Clipped mean of im1Area (ADU). 

962 mu2: `float` 

963 Clipped mean of im2Area (ADU). 

964 correctionType : `str`, optional 

965 The correction applied, one of ['NONE', 'SIMPLE', 'FULL'] 

966 readNoise : `float`, optional 

967 Amplifier readout noise (ADU). 

968 

969 Returns 

970 ------- 

971 gain : `float` 

972 Gain, in e/ADU. 

973 

974 Raises 

975 ------ 

976 RuntimeError 

977 Raise if `correctionType` is not one of 'NONE', 

978 'SIMPLE', or 'FULL'. 

979 """ 

980 if correctionType not in ['NONE', 'SIMPLE', 'FULL']: 

981 raise RuntimeError("Unknown correction type: %s" % correctionType) 

982 

983 if correctionType != 'NONE' and not np.isfinite(readNoise): 

984 self.log.warning("'correctionType' in 'getGainFromFlatPair' is %s, " 

985 "but 'readNoise' is NaN. Setting 'correctionType' " 

986 "to 'NONE', so a gain value will be estimated without " 

987 "corrections." % correctionType) 

988 correctionType = 'NONE' 

989 

990 mu = 0.5*(mu1 + mu2) 

991 

992 # ratioIm = (I1 - I2)^2 / (I1 + I2) 

993 temp = im2Area.clone() 

994 ratioIm = im1Area.clone() 

995 ratioIm -= temp 

996 ratioIm *= ratioIm 

997 

998 # Sum of pairs 

999 sumIm = im1Area.clone() 

1000 sumIm += temp 

1001 

1002 ratioIm /= sumIm 

1003 

1004 const = afwMath.makeStatistics(ratioIm, afwMath.MEAN, imStatsCtrl).getValue() 

1005 gain = 1. / const 

1006 

1007 if correctionType == 'SIMPLE': 

1008 gain = 1/(const - (1/mu)*(readNoise**2 - (1/2*gain**2))) 

1009 elif correctionType == 'FULL': 

1010 root = np.sqrt(mu**2 - 2*mu*const + 2*readNoise**2) 

1011 denom = (2*const*mu - 2*readNoise**2) 

1012 positiveSolution = (root + mu)/denom 

1013 gain = positiveSolution 

1014 

1015 return gain 

1016 

1017 def getReadNoise(self, exposureMetadata, taskMetadata, ampName): 

1018 """Gets readout noise for an amp from ISR metadata. 

1019 

1020 If possible, this attempts to get the now-standard headers 

1021 added to the exposure itself. If not found there, the ISR 

1022 TaskMetadata is searched. If neither of these has the value, 

1023 warn and set the read noise to NaN. 

1024 

1025 Parameters 

1026 ---------- 

1027 exposureMetadata : `lsst.daf.base.PropertySet` 

1028 Metadata to check for read noise first. 

1029 taskMetadata : `lsst.pipe.base.TaskMetadata` 

1030 List of exposures metadata from ISR for this exposure. 

1031 ampName : `str` 

1032 Amplifier name. 

1033 

1034 Returns 

1035 ------- 

1036 readNoise : `float` 

1037 The read noise for this set of exposure/amplifier. 

1038 """ 

1039 # Try from the exposure first. 

1040 expectedKey = f"LSST ISR OVERSCAN RESIDUAL SERIAL STDEV {ampName}" 

1041 if expectedKey in exposureMetadata: 

1042 return exposureMetadata[expectedKey] 

1043 

1044 # If not, try getting it from the task metadata. 

1045 expectedKey = f"RESIDUAL STDEV {ampName}" 

1046 if "isr" in taskMetadata: 

1047 if expectedKey in taskMetadata["isr"]: 

1048 return taskMetadata["isr"][expectedKey] 

1049 

1050 self.log.warning("Median readout noise from ISR metadata for amp %s " 

1051 "could not be calculated." % ampName) 

1052 return np.nan 

1053 

1054 def computeGaussianHistogramParameters(self, im1Area, im2Area, imStatsCtrl, mu1, mu2): 

1055 """Compute KS test for a Gaussian model fit to a histogram of the 

1056 difference image. 

1057 

1058 Parameters 

1059 ---------- 

1060 im1Area : `lsst.afw.image.MaskedImageF` 

1061 Masked image from exposure 1. 

1062 im2Area : `lsst.afw.image.MaskedImageF` 

1063 Masked image from exposure 2. 

1064 imStatsCtrl : `lsst.afw.math.StatisticsControl` 

1065 Statistics control object. 

1066 mu1 : `float` 

1067 Clipped mean of im1Area (ADU). 

1068 mu2 : `float` 

1069 Clipped mean of im2Area (ADU). 

1070 

1071 Returns 

1072 ------- 

1073 varFit : `float` 

1074 Variance from the Gaussian fit. 

1075 chi2Dof : `float` 

1076 Chi-squared per degree of freedom of Gaussian fit. 

1077 kspValue : `float` 

1078 The KS test p-value for the Gaussian fit. 

1079 

1080 Notes 

1081 ----- 

1082 The algorithm here was originally developed by Aaron Roodman. 

1083 Tests on the full focal plane of LSSTCam during testing has shown 

1084 that a KS test p-value cut of 0.01 is a good discriminant for 

1085 well-behaved flat pairs (p>0.01) and poorly behaved non-Gaussian 

1086 flat pairs (p<0.01). 

1087 """ 

1088 diffExp = im1Area.clone() 

1089 diffExp -= im2Area 

1090 

1091 sel = (((diffExp.mask.array & imStatsCtrl.getAndMask()) == 0) 

1092 & np.isfinite(diffExp.mask.array)) 

1093 diffArr = diffExp.image.array[sel] 

1094 

1095 numOk = len(diffArr) 

1096 

1097 if numOk >= self.config.ksHistMinDataValues and np.isfinite(mu1) and np.isfinite(mu2): 

1098 # Create a histogram symmetric around zero, with a bin size 

1099 # determined from the expected variance given by the average of 

1100 # the input signal levels. 

1101 lim = self.config.ksHistLimitMultiplier * np.sqrt((mu1 + mu2)/2.) 

1102 yVals, binEdges = np.histogram(diffArr, bins=self.config.ksHistNBins, range=[-lim, lim]) 

1103 

1104 # Fit the histogram with a Gaussian model. 

1105 model = GaussianModel() 

1106 yVals = yVals.astype(np.float64) 

1107 xVals = ((binEdges[0: -1] + binEdges[1:])/2.).astype(np.float64) 

1108 errVals = np.sqrt(yVals) 

1109 errVals[(errVals == 0.0)] = 1.0 

1110 pars = model.guess(yVals, x=xVals) 

1111 with warnings.catch_warnings(): 

1112 warnings.simplefilter("ignore") 

1113 # The least-squares fitter sometimes spouts (spurious) warnings 

1114 # when the model is very bad. Swallow these warnings now and 

1115 # let the KS test check the model below. 

1116 out = model.fit( 

1117 yVals, 

1118 pars, 

1119 x=xVals, 

1120 weights=1./errVals, 

1121 calc_covar=True, 

1122 method="least_squares", 

1123 ) 

1124 

1125 # Calculate chi2. 

1126 chiArr = out.residual 

1127 nDof = len(yVals) - 3 

1128 chi2Dof = np.sum(chiArr**2.)/nDof 

1129 sigmaFit = out.params["sigma"].value 

1130 

1131 # Calculate KS test p-value for the fit. 

1132 ksResult = scipy.stats.ks_1samp( 

1133 diffArr, 

1134 scipy.stats.norm.cdf, 

1135 (out.params["center"].value, sigmaFit), 

1136 ) 

1137 

1138 kspValue = ksResult.pvalue 

1139 if kspValue < 1e-15: 

1140 kspValue = 0.0 

1141 

1142 varFit = sigmaFit**2. 

1143 

1144 else: 

1145 varFit = np.nan 

1146 chi2Dof = np.nan 

1147 kspValue = 0.0 

1148 

1149 return varFit, chi2Dof, kspValue