Coverage for python/lsst/cp/pipe/ptc/cpExtractPtcTask.py: 11%

356 statements  

« prev     ^ index     » next       coverage.py v7.4.1, created at 2024-01-31 12:22 +0000

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22import numpy as np 

23from lmfit.models import GaussianModel 

24import scipy.stats 

25import warnings 

26 

27import lsst.afw.math as afwMath 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30from lsst.geom import (Box2I, Point2I, Extent2I) 

31from lsst.cp.pipe.utils import (arrangeFlatsByExpTime, arrangeFlatsByExpId, 

32 arrangeFlatsByExpFlux, sigmaClipCorrection, 

33 CovFastFourierTransform) 

34 

35import lsst.pipe.base.connectionTypes as cT 

36 

37from lsst.ip.isr import PhotonTransferCurveDataset 

38from lsst.ip.isr import IsrTask 

39 

40__all__ = ['PhotonTransferCurveExtractConfig', 'PhotonTransferCurveExtractTask'] 

41 

42 

43class PhotonTransferCurveExtractConnections(pipeBase.PipelineTaskConnections, 

44 dimensions=("instrument", "detector")): 

45 

46 inputExp = cT.Input( 

47 name="ptcInputExposurePairs", 

48 doc="Input post-ISR processed exposure pairs (flats) to" 

49 "measure covariances from.", 

50 storageClass="Exposure", 

51 dimensions=("instrument", "exposure", "detector"), 

52 multiple=True, 

53 deferLoad=True, 

54 ) 

55 inputPhotodiodeData = cT.Input( 

56 name="photodiode", 

57 doc="Photodiode readings data.", 

58 storageClass="IsrCalib", 

59 dimensions=("instrument", "exposure"), 

60 multiple=True, 

61 deferLoad=True, 

62 ) 

63 taskMetadata = cT.Input( 

64 name="isr_metadata", 

65 doc="Input task metadata to extract statistics from.", 

66 storageClass="TaskMetadata", 

67 dimensions=("instrument", "exposure", "detector"), 

68 multiple=True, 

69 ) 

70 outputCovariances = cT.Output( 

71 name="ptcCovariances", 

72 doc="Extracted flat (co)variances.", 

73 storageClass="PhotonTransferCurveDataset", 

74 dimensions=("instrument", "exposure", "detector"), 

75 isCalibration=True, 

76 multiple=True, 

77 ) 

78 

79 def __init__(self, *, config=None): 

80 if not config.doExtractPhotodiodeData: 

81 del self.inputPhotodiodeData 

82 

83 

84class PhotonTransferCurveExtractConfig(pipeBase.PipelineTaskConfig, 

85 pipelineConnections=PhotonTransferCurveExtractConnections): 

86 """Configuration for the measurement of covariances from flats. 

87 """ 

88 matchExposuresType = pexConfig.ChoiceField( 

89 dtype=str, 

90 doc="Match input exposures by time, flux, or expId", 

91 default='TIME', 

92 allowed={ 

93 "TIME": "Match exposures by exposure time.", 

94 "FLUX": "Match exposures by target flux. Use header keyword" 

95 " in matchExposuresByFluxKeyword to find the flux.", 

96 "EXPID": "Match exposures by exposure ID." 

97 } 

98 ) 

99 matchExposuresByFluxKeyword = pexConfig.Field( 

100 dtype=str, 

101 doc="Header keyword for flux if matchExposuresType is FLUX.", 

102 default='CCOBFLUX', 

103 ) 

104 maximumRangeCovariancesAstier = pexConfig.Field( 

105 dtype=int, 

106 doc="Maximum range of covariances as in Astier+19", 

107 default=8, 

108 ) 

109 binSize = pexConfig.Field( 

110 dtype=int, 

111 doc="Bin the image by this factor in both dimensions.", 

112 default=1, 

113 ) 

114 minMeanSignal = pexConfig.DictField( 

115 keytype=str, 

116 itemtype=float, 

117 doc="Minimum values (inclusive) of mean signal (in ADU) per amp to use." 

118 " The same cut is applied to all amps if this parameter [`dict`] is passed as " 

119 " {'ALL_AMPS': value}", 

120 default={'ALL_AMPS': 0.0}, 

121 deprecated="This config has been moved to cpSolvePtcTask, and will be removed after v26.", 

122 ) 

123 maxMeanSignal = pexConfig.DictField( 

124 keytype=str, 

125 itemtype=float, 

126 doc="Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp." 

127 " The same cut is applied to all amps if this dictionary is of the form" 

128 " {'ALL_AMPS': value}", 

129 default={'ALL_AMPS': 1e6}, 

130 deprecated="This config has been moved to cpSolvePtcTask, and will be removed after v26.", 

131 ) 

132 maskNameList = pexConfig.ListField( 

133 dtype=str, 

134 doc="Mask list to exclude from statistics calculations.", 

135 default=['SUSPECT', 'BAD', 'NO_DATA', 'SAT'], 

136 ) 

137 nSigmaClipPtc = pexConfig.Field( 

138 dtype=float, 

139 doc="Sigma cut for afwMath.StatisticsControl()", 

140 default=5.5, 

141 ) 

142 nIterSigmaClipPtc = pexConfig.Field( 

143 dtype=int, 

144 doc="Number of sigma-clipping iterations for afwMath.StatisticsControl()", 

145 default=3, 

146 ) 

147 minNumberGoodPixelsForCovariance = pexConfig.Field( 

148 dtype=int, 

149 doc="Minimum number of acceptable good pixels per amp to calculate the covariances (via FFT or" 

150 " direclty).", 

151 default=10000, 

152 ) 

153 thresholdDiffAfwVarVsCov00 = pexConfig.Field( 

154 dtype=float, 

155 doc="If the absolute fractional differece between afwMath.VARIANCECLIP and Cov00 " 

156 "for a region of a difference image is greater than this threshold (percentage), " 

157 "a warning will be issued.", 

158 default=1., 

159 ) 

160 detectorMeasurementRegion = pexConfig.ChoiceField( 

161 dtype=str, 

162 doc="Region of each exposure where to perform the calculations (amplifier or full image).", 

163 default='AMP', 

164 allowed={ 

165 "AMP": "Amplifier of the detector.", 

166 "FULL": "Full image." 

167 } 

168 ) 

169 numEdgeSuspect = pexConfig.Field( 

170 dtype=int, 

171 doc="Number of edge pixels to be flagged as untrustworthy.", 

172 default=0, 

173 ) 

174 edgeMaskLevel = pexConfig.ChoiceField( 

175 dtype=str, 

176 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?", 

177 default="DETECTOR", 

178 allowed={ 

179 'DETECTOR': 'Mask only the edges of the full detector.', 

180 'AMP': 'Mask edges of each amplifier.', 

181 }, 

182 ) 

183 doGain = pexConfig.Field( 

184 dtype=bool, 

185 doc="Calculate a gain per input flat pair.", 

186 default=True, 

187 ) 

188 gainCorrectionType = pexConfig.ChoiceField( 

189 dtype=str, 

190 doc="Correction type for the gain.", 

191 default='FULL', 

192 allowed={ 

193 'NONE': 'No correction.', 

194 'SIMPLE': 'First order correction.', 

195 'FULL': 'Second order correction.' 

196 } 

197 ) 

198 ksHistNBins = pexConfig.Field( 

199 dtype=int, 

200 doc="Number of bins for the KS test histogram.", 

201 default=100, 

202 ) 

203 ksHistLimitMultiplier = pexConfig.Field( 

204 dtype=float, 

205 doc="Number of sigma (as predicted from the mean value) to compute KS test histogram.", 

206 default=8.0, 

207 ) 

208 ksHistMinDataValues = pexConfig.Field( 

209 dtype=int, 

210 doc="Minimum number of good data values to compute KS test histogram.", 

211 default=100, 

212 ) 

213 auxiliaryHeaderKeys = pexConfig.ListField( 

214 dtype=str, 

215 doc="Auxiliary header keys to store with the PTC dataset.", 

216 default=[], 

217 ) 

218 doExtractPhotodiodeData = pexConfig.Field( 

219 dtype=bool, 

220 doc="Extract photodiode data?", 

221 default=False, 

222 ) 

223 photodiodeIntegrationMethod = pexConfig.ChoiceField( 

224 dtype=str, 

225 doc="Integration method for photodiode monitoring data.", 

226 default="CHARGE_SUM", 

227 allowed={ 

228 "DIRECT_SUM": ("Use numpy's trapz integrator on all photodiode " 

229 "readout entries"), 

230 "TRIMMED_SUM": ("Use numpy's trapz integrator, clipping the " 

231 "leading and trailing entries, which are " 

232 "nominally at zero baseline level."), 

233 "CHARGE_SUM": ("Treat the current values as integrated charge " 

234 "over the sampling interval and simply sum " 

235 "the values, after subtracting a baseline level."), 

236 }, 

237 ) 

238 photodiodeCurrentScale = pexConfig.Field( 

239 dtype=float, 

240 doc="Scale factor to apply to photodiode current values for the " 

241 "``CHARGE_SUM`` integration method.", 

242 default=-1.0, 

243 ) 

244 

245 

246class PhotonTransferCurveExtractTask(pipeBase.PipelineTask): 

247 """Task to measure covariances from flat fields. 

248 

249 This task receives as input a list of flat-field images 

250 (flats), and sorts these flats in pairs taken at the 

251 same time (the task will raise if there is one one flat 

252 at a given exposure time, and it will discard extra flats if 

253 there are more than two per exposure time). This task measures 

254 the mean, variance, and covariances from a region (e.g., 

255 an amplifier) of the difference image of the two flats with 

256 the same exposure time (alternatively, all input images could have 

257 the same exposure time but their flux changed). 

258 

259 The variance is calculated via afwMath, and the covariance 

260 via the methods in Astier+19 (appendix A). In theory, 

261 var = covariance[0,0]. This should be validated, and in the 

262 future, we may decide to just keep one (covariance). 

263 At this moment, if the two values differ by more than the value 

264 of `thresholdDiffAfwVarVsCov00` (default: 1%), a warning will 

265 be issued. 

266 

267 The measured covariances at a given exposure time (along with 

268 other quantities such as the mean) are stored in a PTC dataset 

269 object (`~lsst.ip.isr.PhotonTransferCurveDataset`), which gets 

270 partially filled at this stage (the remainder of the attributes 

271 of the dataset will be filled after running the second task of 

272 the PTC-measurement pipeline, `~PhotonTransferCurveSolveTask`). 

273 

274 The number of partially-filled 

275 `~lsst.ip.isr.PhotonTransferCurveDataset` objects will be less 

276 than the number of input exposures because the task combines 

277 input flats in pairs. However, it is required at this moment 

278 that the number of input dimensions matches 

279 bijectively the number of output dimensions. Therefore, a number 

280 of "dummy" PTC datasets are inserted in the output list. This 

281 output list will then be used as input of the next task in the 

282 PTC-measurement pipeline, `PhotonTransferCurveSolveTask`, 

283 which will assemble the multiple `PhotonTransferCurveDataset` 

284 objects into a single one in order to fit the measured covariances 

285 as a function of flux to one of three models 

286 (see `PhotonTransferCurveSolveTask` for details). 

287 

288 Reference: Astier+19: "The Shape of the Photon Transfer Curve of CCD 

289 sensors", arXiv:1905.08677. 

290 """ 

291 

292 ConfigClass = PhotonTransferCurveExtractConfig 

293 _DefaultName = 'cpPtcExtract' 

294 

295 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

296 """Ensure that the input and output dimensions are passed along. 

297 

298 Parameters 

299 ---------- 

300 butlerQC : `~lsst.daf.butler.QuantumContext` 

301 Butler to operate on. 

302 inputRefs : `~lsst.pipe.base.InputQuantizedConnection` 

303 Input data refs to load. 

304 ouptutRefs : `~lsst.pipe.base.OutputQuantizedConnection` 

305 Output data refs to persist. 

306 """ 

307 inputs = butlerQC.get(inputRefs) 

308 # Ids of input list of exposure references 

309 # (deferLoad=True in the input connections) 

310 inputs['inputDims'] = [expRef.datasetRef.dataId['exposure'] for expRef in inputRefs.inputExp] 

311 

312 # Dictionary, keyed by expTime (or expFlux or expId), with tuples 

313 # containing flat exposures and their IDs. 

314 matchType = self.config.matchExposuresType 

315 if matchType == 'TIME': 

316 inputs['inputExp'] = arrangeFlatsByExpTime(inputs['inputExp'], inputs['inputDims'], log=self.log) 

317 elif matchType == 'FLUX': 

318 inputs['inputExp'] = arrangeFlatsByExpFlux( 

319 inputs['inputExp'], 

320 inputs['inputDims'], 

321 self.config.matchExposuresByFluxKeyword, 

322 log=self.log, 

323 ) 

324 else: 

325 inputs['inputExp'] = arrangeFlatsByExpId(inputs['inputExp'], inputs['inputDims']) 

326 

327 outputs = self.run(**inputs) 

328 outputs = self._guaranteeOutputs(inputs['inputDims'], outputs, outputRefs) 

329 butlerQC.put(outputs, outputRefs) 

330 

331 def _guaranteeOutputs(self, inputDims, outputs, outputRefs): 

332 """Ensure that all outputRefs have a matching output, and if they do 

333 not, fill the output with dummy PTC datasets. 

334 

335 Parameters 

336 ---------- 

337 inputDims : `dict` [`str`, `int`] 

338 Input exposure dimensions. 

339 outputs : `lsst.pipe.base.Struct` 

340 Outputs from the ``run`` method. Contains the entry: 

341 

342 ``outputCovariances`` 

343 Output PTC datasets (`list` [`lsst.ip.isr.IsrCalib`]) 

344 outputRefs : `~lsst.pipe.base.OutputQuantizedConnection` 

345 Container with all of the outputs expected to be generated. 

346 

347 Returns 

348 ------- 

349 outputs : `lsst.pipe.base.Struct` 

350 Dummy dataset padded version of the input ``outputs`` with 

351 the same entries. 

352 """ 

353 newCovariances = [] 

354 for ref in outputRefs.outputCovariances: 

355 outputExpId = ref.dataId['exposure'] 

356 if outputExpId in inputDims: 

357 entry = inputDims.index(outputExpId) 

358 newCovariances.append(outputs.outputCovariances[entry]) 

359 else: 

360 newPtc = PhotonTransferCurveDataset(['no amp'], 'DUMMY', covMatrixSide=1) 

361 newPtc.setAmpValuesPartialDataset('no amp') 

362 newCovariances.append(newPtc) 

363 return pipeBase.Struct(outputCovariances=newCovariances) 

364 

365 def run(self, inputExp, inputDims, taskMetadata, inputPhotodiodeData=None): 

366 

367 """Measure covariances from difference of flat pairs 

368 

369 Parameters 

370 ---------- 

371 inputExp : `dict` [`float`, `list` 

372 [`~lsst.pipe.base.connections.DeferredDatasetRef`]] 

373 Dictionary that groups references to flat-field exposures that 

374 have the same exposure time (seconds), or that groups them 

375 sequentially by their exposure id. 

376 inputDims : `list` 

377 List of exposure IDs. 

378 taskMetadata : `list` [`lsst.pipe.base.TaskMetadata`] 

379 List of exposures metadata from ISR. 

380 inputPhotodiodeData : `dict` [`str`, `lsst.ip.isr.PhotodiodeCalib`] 

381 Photodiode readings data (optional). 

382 

383 Returns 

384 ------- 

385 results : `lsst.pipe.base.Struct` 

386 The resulting Struct contains: 

387 

388 ``outputCovariances`` 

389 A list containing the per-pair PTC measurements (`list` 

390 [`lsst.ip.isr.PhotonTransferCurveDataset`]) 

391 """ 

392 # inputExp.values() returns a view, which we turn into a list. We then 

393 # access the first exposure-ID tuple to get the detector. 

394 # The first "get()" retrieves the exposure from the exposure reference. 

395 detector = list(inputExp.values())[0][0][0].get(component='detector') 

396 detNum = detector.getId() 

397 amps = detector.getAmplifiers() 

398 ampNames = [amp.getName() for amp in amps] 

399 

400 # Each amp may have a different min and max ADU signal 

401 # specified in the config. 

402 maxMeanSignalDict = {ampName: 1e6 for ampName in ampNames} 

403 minMeanSignalDict = {ampName: 0.0 for ampName in ampNames} 

404 for ampName in ampNames: 

405 if 'ALL_AMPS' in self.config.maxMeanSignal: 

406 maxMeanSignalDict[ampName] = self.config.maxMeanSignal['ALL_AMPS'] 

407 elif ampName in self.config.maxMeanSignal: 

408 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName] 

409 

410 if 'ALL_AMPS' in self.config.minMeanSignal: 

411 minMeanSignalDict[ampName] = self.config.minMeanSignal['ALL_AMPS'] 

412 elif ampName in self.config.minMeanSignal: 

413 minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName] 

414 # These are the column names for `tupleRows` below. 

415 tags = [('mu', '<f8'), ('afwVar', '<f8'), ('i', '<i8'), ('j', '<i8'), ('var', '<f8'), 

416 ('cov', '<f8'), ('npix', '<i8'), ('ext', '<i8'), ('expTime', '<f8'), ('ampName', '<U3')] 

417 # Create a dummy ptcDataset. Dummy datasets will be 

418 # used to ensure that the number of output and input 

419 # dimensions match. 

420 dummyPtcDataset = PhotonTransferCurveDataset( 

421 ampNames, 'DUMMY', 

422 covMatrixSide=self.config.maximumRangeCovariancesAstier) 

423 for ampName in ampNames: 

424 dummyPtcDataset.setAmpValuesPartialDataset(ampName) 

425 

426 # Extract the photodiode data if requested. 

427 if self.config.doExtractPhotodiodeData: 

428 # Compute the photodiode integrals once, at the start. 

429 monitorDiodeCharge = {} 

430 for handle in inputPhotodiodeData: 

431 expId = handle.dataId['exposure'] 

432 pdCalib = handle.get() 

433 pdCalib.integrationMethod = self.config.photodiodeIntegrationMethod 

434 pdCalib.currentScale = self.config.photodiodeCurrentScale 

435 monitorDiodeCharge[expId] = pdCalib.integrate() 

436 

437 # Get read noise. Try from the exposure, then try 

438 # taskMetadata. This adds a get() for the exposures. 

439 readNoiseLists = {} 

440 for pairIndex, expRefs in inputExp.items(): 

441 # This yields an index (exposure_time, seq_num, or flux) 

442 # and a pair of references at that index. 

443 for expRef, expId in expRefs: 

444 # This yields an exposure ref and an exposureId. 

445 exposureMetadata = expRef.get(component="metadata") 

446 metadataIndex = inputDims.index(expId) 

447 thisTaskMetadata = taskMetadata[metadataIndex] 

448 

449 for ampName in ampNames: 

450 if ampName not in readNoiseLists: 

451 readNoiseLists[ampName] = [self.getReadNoise(exposureMetadata, 

452 thisTaskMetadata, ampName)] 

453 else: 

454 readNoiseLists[ampName].append(self.getReadNoise(exposureMetadata, 

455 thisTaskMetadata, ampName)) 

456 

457 readNoiseDict = {ampName: 0.0 for ampName in ampNames} 

458 for ampName in ampNames: 

459 # Take median read noise value 

460 readNoiseDict[ampName] = np.nanmedian(readNoiseLists[ampName]) 

461 

462 # Output list with PTC datasets. 

463 partialPtcDatasetList = [] 

464 # The number of output references needs to match that of input 

465 # references: initialize outputlist with dummy PTC datasets. 

466 for i in range(len(inputDims)): 

467 partialPtcDatasetList.append(dummyPtcDataset) 

468 

469 if self.config.numEdgeSuspect > 0: 

470 isrTask = IsrTask() 

471 self.log.info("Masking %d pixels from the edges of all %ss as SUSPECT.", 

472 self.config.numEdgeSuspect, self.config.edgeMaskLevel) 

473 

474 # Depending on the value of config.matchExposuresType 

475 # 'expTime' can stand for exposure time, flux, or ID. 

476 for expTime in inputExp: 

477 exposures = inputExp[expTime] 

478 if not np.isfinite(expTime): 

479 self.log.warning("Illegal/missing %s found (%s). Dropping exposure %d", 

480 self.config.matchExposuresType, expTime, exposures[0][1]) 

481 continue 

482 elif len(exposures) == 1: 

483 self.log.warning("Only one exposure found at %s %f. Dropping exposure %d.", 

484 self.config.matchExposuresType, expTime, exposures[0][1]) 

485 continue 

486 else: 

487 # Only use the first two exposures at expTime. Each 

488 # element is a tuple (exposure, expId) 

489 expRef1, expId1 = exposures[0] 

490 expRef2, expId2 = exposures[1] 

491 # use get() to obtain `lsst.afw.image.Exposure` 

492 exp1, exp2 = expRef1.get(), expRef2.get() 

493 

494 if len(exposures) > 2: 

495 self.log.warning("Already found 2 exposures at %s %f. Ignoring exposures: %s", 

496 self.config.matchExposuresType, expTime, 

497 ", ".join(str(i[1]) for i in exposures[2:])) 

498 # Mask pixels at the edge of the detector or of each amp 

499 if self.config.numEdgeSuspect > 0: 

500 isrTask.maskEdges(exp1, numEdgePixels=self.config.numEdgeSuspect, 

501 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

502 isrTask.maskEdges(exp2, numEdgePixels=self.config.numEdgeSuspect, 

503 maskPlane="SUSPECT", level=self.config.edgeMaskLevel) 

504 

505 # Extract any metadata keys from the headers. 

506 auxDict = {} 

507 metadata = exp1.getMetadata() 

508 for key in self.config.auxiliaryHeaderKeys: 

509 if key not in metadata: 

510 self.log.warning( 

511 "Requested auxiliary keyword %s not found in exposure metadata for %d", 

512 key, 

513 expId1, 

514 ) 

515 value = np.nan 

516 else: 

517 value = metadata[key] 

518 

519 auxDict[key] = value 

520 

521 nAmpsNan = 0 

522 partialPtcDataset = PhotonTransferCurveDataset( 

523 ampNames, 'PARTIAL', 

524 covMatrixSide=self.config.maximumRangeCovariancesAstier) 

525 for ampNumber, amp in enumerate(detector): 

526 ampName = amp.getName() 

527 if self.config.detectorMeasurementRegion == 'AMP': 

528 region = amp.getBBox() 

529 elif self.config.detectorMeasurementRegion == 'FULL': 

530 region = None 

531 

532 # Get masked image regions, masking planes, statistic control 

533 # objects, and clipped means. Calculate once to reuse in 

534 # `measureMeanVarCov` and `getGainFromFlatPair`. 

535 im1Area, im2Area, imStatsCtrl, mu1, mu2 = self.getImageAreasMasksStats(exp1, exp2, 

536 region=region) 

537 

538 # We demand that both mu1 and mu2 be finite and greater than 0. 

539 if not np.isfinite(mu1) or not np.isfinite(mu2) \ 

540 or ((np.nan_to_num(mu1) + np.nan_to_num(mu2)/2.) <= 0.0): 

541 self.log.warning( 

542 "Illegal mean value(s) detected for amp %s on exposure pair %d/%d", 

543 ampName, 

544 expId1, 

545 expId2, 

546 ) 

547 partialPtcDataset.setAmpValuesPartialDataset( 

548 ampName, 

549 inputExpIdPair=(expId1, expId2), 

550 rawExpTime=expTime, 

551 expIdMask=False, 

552 ) 

553 continue 

554 

555 # `measureMeanVarCov` is the function that measures 

556 # the variance and covariances from a region of 

557 # the difference image of two flats at the same 

558 # exposure time. The variable `covAstier` that is 

559 # returned is of the form: 

560 # [(i, j, var (cov[0,0]), cov, npix) for (i,j) in 

561 # {maxLag, maxLag}^2]. 

562 muDiff, varDiff, covAstier, rowMeanVariance = self.measureMeanVarCov(im1Area, 

563 im2Area, 

564 imStatsCtrl, 

565 mu1, 

566 mu2) 

567 # Estimate the gain from the flat pair 

568 if self.config.doGain: 

569 gain = self.getGainFromFlatPair(im1Area, im2Area, imStatsCtrl, mu1, mu2, 

570 correctionType=self.config.gainCorrectionType, 

571 readNoise=readNoiseDict[ampName]) 

572 else: 

573 gain = np.nan 

574 

575 # Correction factor for bias introduced by sigma 

576 # clipping. 

577 # Function returns 1/sqrt(varFactor), so it needs 

578 # to be squared. varDiff is calculated via 

579 # afwMath.VARIANCECLIP. 

580 varFactor = sigmaClipCorrection(self.config.nSigmaClipPtc)**2 

581 varDiff *= varFactor 

582 

583 expIdMask = True 

584 # Mask data point at this mean signal level if 

585 # the signal, variance, or covariance calculations 

586 # from `measureMeanVarCov` resulted in NaNs. 

587 if (np.isnan(muDiff) or np.isnan(varDiff) or (covAstier is None) 

588 or (rowMeanVariance is np.nan)): 

589 self.log.warning("NaN mean, var or rowmeanVariance, or None cov in amp %s " 

590 "in exposure pair %d, %d of " 

591 "detector %d.", ampName, expId1, expId2, detNum) 

592 nAmpsNan += 1 

593 expIdMask = False 

594 covArray = np.full((1, self.config.maximumRangeCovariancesAstier, 

595 self.config.maximumRangeCovariancesAstier), np.nan) 

596 covSqrtWeights = np.full_like(covArray, np.nan) 

597 

598 # Mask data point if it is outside of the 

599 # specified mean signal range. 

600 if (muDiff <= minMeanSignalDict[ampName]) or (muDiff >= maxMeanSignalDict[ampName]): 

601 expIdMask = False 

602 

603 if covAstier is not None: 

604 # Turn the tuples with the measured information 

605 # into covariance arrays. 

606 # covrow: (i, j, var (cov[0,0]), cov, npix) 

607 tupleRows = [(muDiff, varDiff) + covRow + (ampNumber, expTime, 

608 ampName) for covRow in covAstier] 

609 tempStructArray = np.array(tupleRows, dtype=tags) 

610 

611 covArray, vcov, _ = self.makeCovArray(tempStructArray, 

612 self.config.maximumRangeCovariancesAstier) 

613 

614 # The returned covArray should only have 1 entry; 

615 # raise if this is not the case. 

616 if covArray.shape[0] != 1: 

617 raise RuntimeError("Serious programming error in covArray shape.") 

618 

619 covSqrtWeights = np.nan_to_num(1./np.sqrt(vcov)) 

620 

621 # Correct covArray for sigma clipping: 

622 # 1) Apply varFactor twice for the whole covariance matrix 

623 covArray *= varFactor**2 

624 # 2) But, only once for the variance element of the 

625 # matrix, covArray[0, 0, 0] (so divide one factor out). 

626 # (the first 0 is because this is a 3D array for insertion into 

627 # the combined dataset). 

628 covArray[0, 0, 0] /= varFactor 

629 

630 if expIdMask: 

631 # Run the Gaussian histogram only if this is a legal 

632 # amplifier. 

633 histVar, histChi2Dof, kspValue = self.computeGaussianHistogramParameters( 

634 im1Area, 

635 im2Area, 

636 imStatsCtrl, 

637 mu1, 

638 mu2, 

639 ) 

640 else: 

641 histVar = np.nan 

642 histChi2Dof = np.nan 

643 kspValue = 0.0 

644 

645 if self.config.doExtractPhotodiodeData: 

646 nExps = 0 

647 photoCharge = 0.0 

648 for expId in [expId1, expId2]: 

649 if expId in monitorDiodeCharge: 

650 photoCharge += monitorDiodeCharge[expId] 

651 nExps += 1 

652 if nExps > 0: 

653 photoCharge /= nExps 

654 else: 

655 photoCharge = np.nan 

656 else: 

657 photoCharge = np.nan 

658 

659 partialPtcDataset.setAmpValuesPartialDataset( 

660 ampName, 

661 inputExpIdPair=(expId1, expId2), 

662 rawExpTime=expTime, 

663 rawMean=muDiff, 

664 rawVar=varDiff, 

665 photoCharge=photoCharge, 

666 expIdMask=expIdMask, 

667 covariance=covArray[0, :, :], 

668 covSqrtWeights=covSqrtWeights[0, :, :], 

669 gain=gain, 

670 noise=readNoiseDict[ampName], 

671 histVar=histVar, 

672 histChi2Dof=histChi2Dof, 

673 kspValue=kspValue, 

674 rowMeanVariance=rowMeanVariance, 

675 ) 

676 

677 partialPtcDataset.setAuxValuesPartialDataset(auxDict) 

678 

679 # Use location of exp1 to save PTC dataset from (exp1, exp2) pair. 

680 # Below, np.where(expId1 == np.array(inputDims)) returns a tuple 

681 # with a single-element array, so [0][0] 

682 # is necessary to extract the required index. 

683 datasetIndex = np.where(expId1 == np.array(inputDims))[0][0] 

684 # `partialPtcDatasetList` is a list of 

685 # `PhotonTransferCurveDataset` objects. Some of them 

686 # will be dummy datasets (to match length of input 

687 # and output references), and the rest will have 

688 # datasets with the mean signal, variance, and 

689 # covariance measurements at a given exposure 

690 # time. The next ppart of the PTC-measurement 

691 # pipeline, `solve`, will take this list as input, 

692 # and assemble the measurements in the datasets 

693 # in an addecuate manner for fitting a PTC 

694 # model. 

695 partialPtcDataset.updateMetadataFromExposures([exp1, exp2]) 

696 partialPtcDataset.updateMetadata(setDate=True, detector=detector) 

697 partialPtcDatasetList[datasetIndex] = partialPtcDataset 

698 

699 if nAmpsNan == len(ampNames): 

700 msg = f"NaN mean in all amps of exposure pair {expId1}, {expId2} of detector {detNum}." 

701 self.log.warning(msg) 

702 

703 return pipeBase.Struct( 

704 outputCovariances=partialPtcDatasetList, 

705 ) 

706 

707 def makeCovArray(self, inputTuple, maxRangeFromTuple): 

708 """Make covariances array from tuple. 

709 

710 Parameters 

711 ---------- 

712 inputTuple : `numpy.ndarray` 

713 Structured array with rows with at least 

714 (mu, afwVar, cov, var, i, j, npix), where: 

715 mu : `float` 

716 0.5*(m1 + m2), where mu1 is the mean value of flat1 

717 and mu2 is the mean value of flat2. 

718 afwVar : `float` 

719 Variance of difference flat, calculated with afw. 

720 cov : `float` 

721 Covariance value at lag(i, j) 

722 var : `float` 

723 Variance(covariance value at lag(0, 0)) 

724 i : `int` 

725 Lag in dimension "x". 

726 j : `int` 

727 Lag in dimension "y". 

728 npix : `int` 

729 Number of pixels used for covariance calculation. 

730 maxRangeFromTuple : `int` 

731 Maximum range to select from tuple. 

732 

733 Returns 

734 ------- 

735 cov : `numpy.array` 

736 Covariance arrays, indexed by mean signal mu. 

737 vCov : `numpy.array` 

738 Variance of the [co]variance arrays, indexed by mean signal mu. 

739 muVals : `numpy.array` 

740 List of mean signal values. 

741 """ 

742 if maxRangeFromTuple is not None: 

743 cut = (inputTuple['i'] < maxRangeFromTuple) & (inputTuple['j'] < maxRangeFromTuple) 

744 cutTuple = inputTuple[cut] 

745 else: 

746 cutTuple = inputTuple 

747 # increasing mu order, so that we can group measurements with the 

748 # same mu 

749 muTemp = cutTuple['mu'] 

750 ind = np.argsort(muTemp) 

751 

752 cutTuple = cutTuple[ind] 

753 # should group measurements on the same image pairs(same average) 

754 mu = cutTuple['mu'] 

755 xx = np.hstack(([mu[0]], mu)) 

756 delta = xx[1:] - xx[:-1] 

757 steps, = np.where(delta > 0) 

758 ind = np.zeros_like(mu, dtype=int) 

759 ind[steps] = 1 

760 ind = np.cumsum(ind) # this acts as an image pair index. 

761 # now fill the 3-d cov array(and variance) 

762 muVals = np.array(np.unique(mu)) 

763 i = cutTuple['i'].astype(int) 

764 j = cutTuple['j'].astype(int) 

765 c = 0.5*cutTuple['cov'] 

766 n = cutTuple['npix'] 

767 v = 0.5*cutTuple['var'] 

768 # book and fill 

769 cov = np.ndarray((len(muVals), np.max(i)+1, np.max(j)+1)) 

770 var = np.zeros_like(cov) 

771 cov[ind, i, j] = c 

772 var[ind, i, j] = v**2/n 

773 var[:, 0, 0] *= 2 # var(v) = 2*v**2/N 

774 

775 return cov, var, muVals 

776 

777 def measureMeanVarCov(self, im1Area, im2Area, imStatsCtrl, mu1, mu2): 

778 """Calculate the mean of each of two exposures and the variance 

779 and covariance of their difference. The variance is calculated 

780 via afwMath, and the covariance via the methods in Astier+19 

781 (appendix A). In theory, var = covariance[0,0]. This should 

782 be validated, and in the future, we may decide to just keep 

783 one (covariance). 

784 

785 Parameters 

786 ---------- 

787 im1Area : `lsst.afw.image.maskedImage.MaskedImageF` 

788 Masked image from exposure 1. 

789 im2Area : `lsst.afw.image.maskedImage.MaskedImageF` 

790 Masked image from exposure 2. 

791 imStatsCtrl : `lsst.afw.math.StatisticsControl` 

792 Statistics control object. 

793 mu1: `float` 

794 Clipped mean of im1Area (ADU). 

795 mu2: `float` 

796 Clipped mean of im2Area (ADU). 

797 

798 Returns 

799 ------- 

800 mu : `float` 

801 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means 

802 of the regions in both exposures. If either mu1 or m2 are 

803 NaN's, the returned value is NaN. 

804 varDiff : `float` 

805 Half of the clipped variance of the difference of the 

806 regions inthe two input exposures. If either mu1 or m2 are 

807 NaN's, the returned value is NaN. 

808 covDiffAstier : `list` or `NaN` 

809 List with tuples of the form (dx, dy, var, cov, npix), where: 

810 dx : `int` 

811 Lag in x 

812 dy : `int` 

813 Lag in y 

814 var : `float` 

815 Variance at (dx, dy). 

816 cov : `float` 

817 Covariance at (dx, dy). 

818 nPix : `int` 

819 Number of pixel pairs used to evaluate var and cov. 

820 rowMeanVariance : `float` 

821 Variance of the means of each row in the difference image. 

822 Taken from `github.com/lsst-camera-dh/eo_pipe`. 

823 

824 If either mu1 or m2 are NaN's, the returned value is NaN. 

825 """ 

826 if np.isnan(mu1) or np.isnan(mu2): 

827 self.log.warning("Mean of amp in image 1 or 2 is NaN: %f, %f.", mu1, mu2) 

828 return np.nan, np.nan, None, np.nan 

829 mu = 0.5*(mu1 + mu2) 

830 

831 # Take difference of pairs 

832 # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2)) 

833 temp = im2Area.clone() 

834 temp *= mu1 

835 diffIm = im1Area.clone() 

836 diffIm *= mu2 

837 diffIm -= temp 

838 diffIm /= mu 

839 

840 if self.config.binSize > 1: 

841 diffIm = afwMath.binImage(diffIm, self.config.binSize) 

842 

843 # Calculate the variance (in ADU^2) of the means of rows for diffIm. 

844 # Taken from eo_pipe 

845 region = diffIm.getBBox() 

846 rowMeans = [] 

847 for row in range(region.minY, region.maxY): 

848 regionRow = Box2I(Point2I(region.minX, row), 

849 Extent2I(region.width, 1)) 

850 rowMeans.append(afwMath.makeStatistics(diffIm[regionRow], 

851 afwMath.MEAN, 

852 imStatsCtrl).getValue()) 

853 rowMeanVariance = afwMath.makeStatistics( 

854 np.array(rowMeans), afwMath.VARIANCECLIP, 

855 imStatsCtrl).getValue() 

856 

857 # Variance calculation via afwMath 

858 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, imStatsCtrl).getValue()) 

859 

860 # Covariances calculations 

861 # Get the pixels that were not clipped 

862 varClip = afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, imStatsCtrl).getValue() 

863 meanClip = afwMath.makeStatistics(diffIm, afwMath.MEANCLIP, imStatsCtrl).getValue() 

864 cut = meanClip + self.config.nSigmaClipPtc*np.sqrt(varClip) 

865 unmasked = np.where(np.fabs(diffIm.image.array) <= cut, 1, 0) 

866 

867 # Get the pixels in the mask planes of the difference image 

868 # that were ignored by the clipping algorithm 

869 wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0) 

870 # Combine the two sets of pixels ('1': use; '0': don't use) 

871 # into a final weight matrix to be used in the covariance 

872 # calculations below. 

873 w = unmasked*wDiff 

874 

875 if np.sum(w) < self.config.minNumberGoodPixelsForCovariance/(self.config.binSize**2): 

876 self.log.warning("Number of good points for covariance calculation (%s) is less " 

877 "(than threshold %s)", np.sum(w), 

878 self.config.minNumberGoodPixelsForCovariance/(self.config.binSize**2)) 

879 return np.nan, np.nan, None, np.nan 

880 

881 maxRangeCov = self.config.maximumRangeCovariancesAstier 

882 

883 # Calculate covariances via FFT. 

884 shapeDiff = np.array(diffIm.image.array.shape) 

885 # Calculate the sizes of FFT dimensions. 

886 s = shapeDiff + maxRangeCov 

887 tempSize = np.array(np.log(s)/np.log(2.)).astype(int) 

888 fftSize = np.array(2**(tempSize+1)).astype(int) 

889 fftShape = (fftSize[0], fftSize[1]) 

890 c = CovFastFourierTransform(diffIm.image.array, w, fftShape, maxRangeCov) 

891 # np.sum(w) is the same as npix[0][0] returned in covDiffAstier 

892 try: 

893 covDiffAstier = c.reportCovFastFourierTransform(maxRangeCov) 

894 except ValueError: 

895 # This is raised if there are not enough pixels. 

896 self.log.warning("Not enough pixels covering the requested covariance range in x/y (%d)", 

897 self.config.maximumRangeCovariancesAstier) 

898 return np.nan, np.nan, None, np.nan 

899 

900 # Compare Cov[0,0] and afwMath.VARIANCECLIP covDiffAstier[0] 

901 # is the Cov[0,0] element, [3] is the variance, and there's a 

902 # factor of 0.5 difference with afwMath.VARIANCECLIP. 

903 thresholdPercentage = self.config.thresholdDiffAfwVarVsCov00 

904 fractionalDiff = 100*np.fabs(1 - varDiff/(covDiffAstier[0][3]*0.5)) 

905 if fractionalDiff >= thresholdPercentage: 

906 self.log.warning("Absolute fractional difference between afwMatch.VARIANCECLIP and Cov[0,0] " 

907 "is more than %f%%: %f", thresholdPercentage, fractionalDiff) 

908 

909 return mu, varDiff, covDiffAstier, rowMeanVariance 

910 

911 def getImageAreasMasksStats(self, exposure1, exposure2, region=None): 

912 """Get image areas in a region as well as masks and statistic objects. 

913 

914 Parameters 

915 ---------- 

916 exposure1 : `lsst.afw.image.ExposureF` 

917 First exposure of flat field pair. 

918 exposure2 : `lsst.afw.image.ExposureF` 

919 Second exposure of flat field pair. 

920 region : `lsst.geom.Box2I`, optional 

921 Region of each exposure where to perform the calculations 

922 (e.g, an amplifier). 

923 

924 Returns 

925 ------- 

926 im1Area : `lsst.afw.image.MaskedImageF` 

927 Masked image from exposure 1. 

928 im2Area : `lsst.afw.image.MaskedImageF` 

929 Masked image from exposure 2. 

930 imStatsCtrl : `lsst.afw.math.StatisticsControl` 

931 Statistics control object. 

932 mu1 : `float` 

933 Clipped mean of im1Area (ADU). 

934 mu2 : `float` 

935 Clipped mean of im2Area (ADU). 

936 """ 

937 if region is not None: 

938 im1Area = exposure1.maskedImage[region] 

939 im2Area = exposure2.maskedImage[region] 

940 else: 

941 im1Area = exposure1.maskedImage 

942 im2Area = exposure2.maskedImage 

943 

944 # Get mask planes and construct statistics control object from one 

945 # of the exposures 

946 imMaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList) 

947 imStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc, 

948 self.config.nIterSigmaClipPtc, 

949 imMaskVal) 

950 imStatsCtrl.setNanSafe(True) 

951 imStatsCtrl.setAndMask(imMaskVal) 

952 

953 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, imStatsCtrl).getValue() 

954 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, imStatsCtrl).getValue() 

955 

956 return (im1Area, im2Area, imStatsCtrl, mu1, mu2) 

957 

958 def getGainFromFlatPair(self, im1Area, im2Area, imStatsCtrl, mu1, mu2, 

959 correctionType='NONE', readNoise=None): 

960 """Estimate the gain from a single pair of flats. 

961 

962 The basic premise is 1/g = <(I1 - I2)^2/(I1 + I2)> = 1/const, 

963 where I1 and I2 correspond to flats 1 and 2, respectively. 

964 Corrections for the variable QE and the read-noise are then 

965 made following the derivation in Robert Lupton's forthcoming 

966 book, which gets 

967 

968 1/g = <(I1 - I2)^2/(I1 + I2)> - 1/mu(sigma^2 - 1/2g^2). 

969 

970 This is a quadratic equation, whose solutions are given by: 

971 

972 g = mu +/- sqrt(2*sigma^2 - 2*const*mu + mu^2)/(2*const*mu*2 

973 - 2*sigma^2) 

974 

975 where 'mu' is the average signal level and 'sigma' is the 

976 amplifier's readnoise. The positive solution will be used. 

977 The way the correction is applied depends on the value 

978 supplied for correctionType. 

979 

980 correctionType is one of ['NONE', 'SIMPLE' or 'FULL'] 

981 'NONE' : uses the 1/g = <(I1 - I2)^2/(I1 + I2)> formula. 

982 'SIMPLE' : uses the gain from the 'NONE' method for the 

983 1/2g^2 term. 

984 'FULL' : solves the full equation for g, discarding the 

985 non-physical solution to the resulting quadratic. 

986 

987 Parameters 

988 ---------- 

989 im1Area : `lsst.afw.image.maskedImage.MaskedImageF` 

990 Masked image from exposure 1. 

991 im2Area : `lsst.afw.image.maskedImage.MaskedImageF` 

992 Masked image from exposure 2. 

993 imStatsCtrl : `lsst.afw.math.StatisticsControl` 

994 Statistics control object. 

995 mu1: `float` 

996 Clipped mean of im1Area (ADU). 

997 mu2: `float` 

998 Clipped mean of im2Area (ADU). 

999 correctionType : `str`, optional 

1000 The correction applied, one of ['NONE', 'SIMPLE', 'FULL'] 

1001 readNoise : `float`, optional 

1002 Amplifier readout noise (ADU). 

1003 

1004 Returns 

1005 ------- 

1006 gain : `float` 

1007 Gain, in e/ADU. 

1008 

1009 Raises 

1010 ------ 

1011 RuntimeError 

1012 Raise if `correctionType` is not one of 'NONE', 

1013 'SIMPLE', or 'FULL'. 

1014 """ 

1015 if correctionType not in ['NONE', 'SIMPLE', 'FULL']: 

1016 raise RuntimeError("Unknown correction type: %s" % correctionType) 

1017 

1018 if correctionType != 'NONE' and not np.isfinite(readNoise): 

1019 self.log.warning("'correctionType' in 'getGainFromFlatPair' is %s, " 

1020 "but 'readNoise' is NaN. Setting 'correctionType' " 

1021 "to 'NONE', so a gain value will be estimated without " 

1022 "corrections." % correctionType) 

1023 correctionType = 'NONE' 

1024 

1025 mu = 0.5*(mu1 + mu2) 

1026 

1027 # ratioIm = (I1 - I2)^2 / (I1 + I2) 

1028 temp = im2Area.clone() 

1029 ratioIm = im1Area.clone() 

1030 ratioIm -= temp 

1031 ratioIm *= ratioIm 

1032 

1033 # Sum of pairs 

1034 sumIm = im1Area.clone() 

1035 sumIm += temp 

1036 

1037 ratioIm /= sumIm 

1038 

1039 const = afwMath.makeStatistics(ratioIm, afwMath.MEAN, imStatsCtrl).getValue() 

1040 gain = 1. / const 

1041 

1042 if correctionType == 'SIMPLE': 

1043 gain = 1/(const - (1/mu)*(readNoise**2 - (1/2*gain**2))) 

1044 elif correctionType == 'FULL': 

1045 root = np.sqrt(mu**2 - 2*mu*const + 2*readNoise**2) 

1046 denom = (2*const*mu - 2*readNoise**2) 

1047 positiveSolution = (root + mu)/denom 

1048 gain = positiveSolution 

1049 

1050 return gain 

1051 

1052 def getReadNoise(self, exposureMetadata, taskMetadata, ampName): 

1053 """Gets readout noise for an amp from ISR metadata. 

1054 

1055 If possible, this attempts to get the now-standard headers 

1056 added to the exposure itself. If not found there, the ISR 

1057 TaskMetadata is searched. If neither of these has the value, 

1058 warn and set the read noise to NaN. 

1059 

1060 Parameters 

1061 ---------- 

1062 exposureMetadata : `lsst.daf.base.PropertySet` 

1063 Metadata to check for read noise first. 

1064 taskMetadata : `lsst.pipe.base.TaskMetadata` 

1065 List of exposures metadata from ISR for this exposure. 

1066 ampName : `str` 

1067 Amplifier name. 

1068 

1069 Returns 

1070 ------- 

1071 readNoise : `float` 

1072 The read noise for this set of exposure/amplifier. 

1073 """ 

1074 # Try from the exposure first. 

1075 expectedKey = f"LSST ISR OVERSCAN RESIDUAL SERIAL STDEV {ampName}" 

1076 if expectedKey in exposureMetadata: 

1077 return exposureMetadata[expectedKey] 

1078 

1079 # If not, try getting it from the task metadata. 

1080 expectedKey = f"RESIDUAL STDEV {ampName}" 

1081 if "isr" in taskMetadata: 

1082 if expectedKey in taskMetadata["isr"]: 

1083 return taskMetadata["isr"][expectedKey] 

1084 

1085 self.log.warning("Median readout noise from ISR metadata for amp %s " 

1086 "could not be calculated." % ampName) 

1087 return np.nan 

1088 

1089 def computeGaussianHistogramParameters(self, im1Area, im2Area, imStatsCtrl, mu1, mu2): 

1090 """Compute KS test for a Gaussian model fit to a histogram of the 

1091 difference image. 

1092 

1093 Parameters 

1094 ---------- 

1095 im1Area : `lsst.afw.image.MaskedImageF` 

1096 Masked image from exposure 1. 

1097 im2Area : `lsst.afw.image.MaskedImageF` 

1098 Masked image from exposure 2. 

1099 imStatsCtrl : `lsst.afw.math.StatisticsControl` 

1100 Statistics control object. 

1101 mu1 : `float` 

1102 Clipped mean of im1Area (ADU). 

1103 mu2 : `float` 

1104 Clipped mean of im2Area (ADU). 

1105 

1106 Returns 

1107 ------- 

1108 varFit : `float` 

1109 Variance from the Gaussian fit. 

1110 chi2Dof : `float` 

1111 Chi-squared per degree of freedom of Gaussian fit. 

1112 kspValue : `float` 

1113 The KS test p-value for the Gaussian fit. 

1114 

1115 Notes 

1116 ----- 

1117 The algorithm here was originally developed by Aaron Roodman. 

1118 Tests on the full focal plane of LSSTCam during testing has shown 

1119 that a KS test p-value cut of 0.01 is a good discriminant for 

1120 well-behaved flat pairs (p>0.01) and poorly behaved non-Gaussian 

1121 flat pairs (p<0.01). 

1122 """ 

1123 diffExp = im1Area.clone() 

1124 diffExp -= im2Area 

1125 

1126 sel = (((diffExp.mask.array & imStatsCtrl.getAndMask()) == 0) 

1127 & np.isfinite(diffExp.mask.array)) 

1128 diffArr = diffExp.image.array[sel] 

1129 

1130 numOk = len(diffArr) 

1131 

1132 if numOk >= self.config.ksHistMinDataValues and np.isfinite(mu1) and np.isfinite(mu2): 

1133 # Create a histogram symmetric around zero, with a bin size 

1134 # determined from the expected variance given by the average of 

1135 # the input signal levels. 

1136 lim = self.config.ksHistLimitMultiplier * np.sqrt((mu1 + mu2)/2.) 

1137 yVals, binEdges = np.histogram(diffArr, bins=self.config.ksHistNBins, range=[-lim, lim]) 

1138 

1139 # Fit the histogram with a Gaussian model. 

1140 model = GaussianModel() 

1141 yVals = yVals.astype(np.float64) 

1142 xVals = ((binEdges[0: -1] + binEdges[1:])/2.).astype(np.float64) 

1143 errVals = np.sqrt(yVals) 

1144 errVals[(errVals == 0.0)] = 1.0 

1145 pars = model.guess(yVals, x=xVals) 

1146 with warnings.catch_warnings(): 

1147 warnings.simplefilter("ignore") 

1148 # The least-squares fitter sometimes spouts (spurious) warnings 

1149 # when the model is very bad. Swallow these warnings now and 

1150 # let the KS test check the model below. 

1151 out = model.fit( 

1152 yVals, 

1153 pars, 

1154 x=xVals, 

1155 weights=1./errVals, 

1156 calc_covar=True, 

1157 method="least_squares", 

1158 ) 

1159 

1160 # Calculate chi2. 

1161 chiArr = out.residual 

1162 nDof = len(yVals) - 3 

1163 chi2Dof = np.sum(chiArr**2.)/nDof 

1164 sigmaFit = out.params["sigma"].value 

1165 

1166 # Calculate KS test p-value for the fit. 

1167 ksResult = scipy.stats.ks_1samp( 

1168 diffArr, 

1169 scipy.stats.norm.cdf, 

1170 (out.params["center"].value, sigmaFit), 

1171 ) 

1172 

1173 kspValue = ksResult.pvalue 

1174 if kspValue < 1e-15: 

1175 kspValue = 0.0 

1176 

1177 varFit = sigmaFit**2. 

1178 

1179 else: 

1180 varFit = np.nan 

1181 chi2Dof = np.nan 

1182 kspValue = 0.0 

1183 

1184 return varFit, chi2Dof, kspValue