Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22import numpy as np 

23 

24import lsst.afw.math as afwMath 

25import lsst.pex.config as pexConfig 

26import lsst.pipe.base as pipeBase 

27from lsst.cp.pipe.utils import arrangeFlatsByExpTime 

28 

29import lsst.pipe.base.connectionTypes as cT 

30 

31from .astierCovPtcUtils import (fftSize, CovFft, computeCovDirect) 

32from .astierCovPtcFit import makeCovArray 

33 

34from lsst.ip.isr import PhotonTransferCurveDataset 

35 

36 

37__all__ = ['PhotonTransferCurveExtractConfig', 'PhotonTransferCurveExtractTask'] 

38 

39 

40class PhotonTransferCurveExtractConnections(pipeBase.PipelineTaskConnections, 

41 dimensions=("instrument", "detector")): 

42 

43 inputExp = cT.Input( 

44 name="ptcInputExposurePairs", 

45 doc="Input post-ISR processed exposure pairs (flats) to" 

46 "measure covariances from.", 

47 storageClass="Exposure", 

48 dimensions=("instrument", "exposure", "detector"), 

49 multiple=True, 

50 deferLoad=False, 

51 ) 

52 

53 outputCovariances = cT.Output( 

54 name="ptcCovariances", 

55 doc="Extracted flat (co)variances.", 

56 storageClass="PhotonTransferCurveDataset", 

57 dimensions=("instrument", "exposure", "detector"), 

58 multiple=True, 

59 ) 

60 

61 

62class PhotonTransferCurveExtractConfig(pipeBase.PipelineTaskConfig, 

63 pipelineConnections=PhotonTransferCurveExtractConnections): 

64 """Configuration for the measurement of covariances from flats. 

65 """ 

66 maximumRangeCovariancesAstier = pexConfig.Field( 

67 dtype=int, 

68 doc="Maximum range of covariances as in Astier+19", 

69 default=8, 

70 ) 

71 covAstierRealSpace = pexConfig.Field( 

72 dtype=bool, 

73 doc="Calculate covariances in real space or via FFT? (see appendix A of Astier+19).", 

74 default=False, 

75 ) 

76 binSize = pexConfig.Field( 

77 dtype=int, 

78 doc="Bin the image by this factor in both dimensions.", 

79 default=1, 

80 ) 

81 minMeanSignal = pexConfig.DictField( 

82 keytype=str, 

83 itemtype=float, 

84 doc="Minimum values (inclusive) of mean signal (in ADU) above which to consider, per amp." 

85 " The same cut is applied to all amps if this dictionary is of the form" 

86 " {'ALL_AMPS': value}", 

87 default={'ALL_AMPS': 0.0}, 

88 ) 

89 maxMeanSignal = pexConfig.DictField( 

90 keytype=str, 

91 itemtype=float, 

92 doc="Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp." 

93 " The same cut is applied to all amps if this dictionary is of the form" 

94 " {'ALL_AMPS': value}", 

95 default={'ALL_AMPS': 1e6}, 

96 ) 

97 maskNameList = pexConfig.ListField( 

98 dtype=str, 

99 doc="Mask list to exclude from statistics calculations.", 

100 default=['SUSPECT', 'BAD', 'NO_DATA'], 

101 ) 

102 nSigmaClipPtc = pexConfig.Field( 

103 dtype=float, 

104 doc="Sigma cut for afwMath.StatisticsControl()", 

105 default=5.5, 

106 ) 

107 nIterSigmaClipPtc = pexConfig.Field( 

108 dtype=int, 

109 doc="Number of sigma-clipping iterations for afwMath.StatisticsControl()", 

110 default=1, 

111 ) 

112 minNumberGoodPixelsForFft = pexConfig.Field( 

113 dtype=int, 

114 doc="Minimum number of acceptable good pixels per amp to calculate the covariances via FFT.", 

115 default=10000, 

116 ) 

117 detectorMeasurementRegion = pexConfig.ChoiceField( 

118 dtype=str, 

119 doc="Region of each exposure where to perform the calculations (amplifier or full image).", 

120 default='AMP', 

121 allowed={ 

122 "AMP": "Amplifier of the detector.", 

123 "FULL": "Full image." 

124 } 

125 ) 

126 

127 

128class PhotonTransferCurveExtractTask(pipeBase.PipelineTask, 

129 pipeBase.CmdLineTask): 

130 """Task to measure covariances from flat fields. 

131 This task receives as input a list of flat-field images 

132 (flats), and sorts these flats in pairs taken at the 

133 same time (if there's a different number of flats, 

134 those flats are discarded). The mean, variance, and 

135 covariances are measured from the difference of the flat 

136 pairs at a given time. The variance is calculated 

137 via afwMath, and the covariance via the methods in Astier+19 

138 (appendix A). In theory, var = covariance[0,0]. This should 

139 be validated, and in the future, we may decide to just keep 

140 one (covariance). 

141 

142 The measured covariances at a particular time (along with 

143 other quantities such as the mean) are stored in a PTC dataset 

144 object (`PhotonTransferCurveDataset`), which gets partially 

145 filled. The number of partially-filled PTC dataset objects 

146 will be less than the number of input exposures, but gen3 

147 requires/assumes that the number of input dimensions matches 

148 bijectively the number of output dimensions. Therefore, a 

149 number of "dummy" PTC dataset are inserted in the output list 

150 that has the partially-filled PTC datasets with the covariances. 

151 This output list will be used as input of 

152 `PhotonTransferCurveSolveTask`, which will assemble the multiple 

153 `PhotonTransferCurveDataset`s into a single one in order to fit 

154 the measured covariances as a function of flux to a particular 

155 model. 

156 

157 Astier+19: "The Shape of the Photon Transfer Curve of CCD 

158 sensors", arXiv:1905.08677. 

159 """ 

160 ConfigClass = PhotonTransferCurveExtractConfig 

161 _DefaultName = 'cpPtcExtract' 

162 

163 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

164 """Ensure that the input and output dimensions are passed along. 

165 

166 Parameters 

167 ---------- 

168 butlerQC : `~lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext` 

169 Butler to operate on. 

170 inputRefs : `~lsst.pipe.base.connections.InputQuantizedConnection` 

171 Input data refs to load. 

172 ouptutRefs : `~lsst.pipe.base.connections.OutputQuantizedConnection` 

173 Output data refs to persist. 

174 """ 

175 inputs = butlerQC.get(inputRefs) 

176 # Dictionary, keyed by expTime, with flat exposures 

177 inputs['inputExp'] = arrangeFlatsByExpTime(inputs['inputExp']) 

178 # Ids of input list of exposures 

179 inputs['inputDims'] = [expId.dataId['exposure'] for expId in inputRefs.inputExp] 

180 outputs = self.run(**inputs) 

181 butlerQC.put(outputs, outputRefs) 

182 

183 def run(self, inputExp, inputDims): 

184 """Measure covariances from difference of flat pairs 

185 

186 Parameters 

187 ---------- 

188 inputExp : `dict` [`float`, 

189 (`~lsst.afw.image.exposure.exposure.ExposureF`, 

190 `~lsst.afw.image.exposure.exposure.ExposureF`, ..., 

191 `~lsst.afw.image.exposure.exposure.ExposureF`)] 

192 Dictionary that groups flat-field exposures that have the same 

193 exposure time (seconds). 

194 

195 inputDims : `list` 

196 List of exposure IDs. 

197 """ 

198 # inputExp.values() returns a view, which we turn into a list. We then 

199 # access the first exposure to get teh detector. 

200 detector = list(inputExp.values())[0][0].getDetector() 

201 detNum = detector.getId() 

202 amps = detector.getAmplifiers() 

203 ampNames = [amp.getName() for amp in amps] 

204 maxMeanSignalDict = {ampName: 1e6 for ampName in ampNames} 

205 minMeanSignalDict = {ampName: 0.0 for ampName in ampNames} 

206 for ampName in ampNames: 

207 if 'ALL_AMPS' in self.config.maxMeanSignal: 

208 maxMeanSignalDict[ampName] = self.config.maxMeanSignal['ALL_AMPS'] 

209 elif ampName in self.config.maxMeanSignal: 

210 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName] 

211 

212 if 'ALL_AMPS' in self.config.minMeanSignal: 

213 minMeanSignalDict[ampName] = self.config.minMeanSignal['ALL_AMPS'] 

214 elif ampName in self.config.minMeanSignal: 

215 minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName] 

216 tags = [('mu', '<f8'), ('afwVar', '<f8'), ('i', '<i8'), ('j', '<i8'), ('var', '<f8'), 

217 ('cov', '<f8'), ('npix', '<i8'), ('ext', '<i8'), ('expTime', '<f8'), ('ampName', '<U3')] 

218 dummyPtcDataset = PhotonTransferCurveDataset(ampNames, 'DUMMY', 

219 self.config.maximumRangeCovariancesAstier) 

220 covArray = [np.full((self.config.maximumRangeCovariancesAstier, 

221 self.config.maximumRangeCovariancesAstier), np.nan)] 

222 for ampName in ampNames: 

223 dummyPtcDataset.rawExpTimes[ampName] = [np.nan] 

224 dummyPtcDataset.rawMeans[ampName] = [np.nan] 

225 dummyPtcDataset.rawVars[ampName] = [np.nan] 

226 dummyPtcDataset.inputExpIdPairs[ampName] = [(np.nan, np.nan)] 

227 dummyPtcDataset.expIdMask[ampName] = [np.nan] 

228 dummyPtcDataset.covariances[ampName] = covArray 

229 dummyPtcDataset.covariancesModel[ampName] = np.full_like(covArray, np.nan) 

230 dummyPtcDataset.covariancesSqrtWeights[ampName] = np.full_like(covArray, np.nan) 

231 dummyPtcDataset.covariancesModelNoB[ampName] = np.full_like(covArray, np.nan) 

232 dummyPtcDataset.aMatrix[ampName] = np.full_like(covArray[0], np.nan) 

233 dummyPtcDataset.bMatrix[ampName] = np.full_like(covArray[0], np.nan) 

234 dummyPtcDataset.aMatrixNoB[ampName] = np.full_like(covArray[0], np.nan) 

235 dummyPtcDataset.ptcFitPars[ampName] = [np.nan] 

236 dummyPtcDataset.ptcFitParsError[ampName] = [np.nan] 

237 dummyPtcDataset.ptcFitChiSq[ampName] = np.nan 

238 dummyPtcDataset.finalVars[ampName] = [np.nan] 

239 dummyPtcDataset.finalModelVars[ampName] = [np.nan] 

240 dummyPtcDataset.finalMeans[ampName] = [np.nan] 

241 # Output list with PTC datasets. 

242 partialDatasetPtcList = [] 

243 # The number of output references needs to match that of input references: 

244 # initialize outputlist with dummy PTC datasets. 

245 for i in range(len(inputDims)): 

246 partialDatasetPtcList.append(dummyPtcDataset) 

247 

248 for expTime in inputExp: 

249 exposures = inputExp[expTime] 

250 if len(exposures) == 1: 

251 self.log.warn(f"Only one exposure found at expTime {expTime}. Dropping exposure " 

252 f"{exposures[0].getInfo().getVisitInfo().getExposureId()}.") 

253 continue 

254 else: 

255 # Only use the first two exposures at expTime 

256 exp1, exp2 = exposures[0], exposures[1] 

257 if len(exposures) > 2: 

258 self.log.warn(f"Already found 2 exposures at expTime {expTime}. " 

259 "Ignoring exposures: " 

260 f"{i.getInfo().getVisitInfo().getExposureId() for i in exposures[2:]}") 

261 expId1 = exp1.getInfo().getVisitInfo().getExposureId() 

262 expId2 = exp2.getInfo().getVisitInfo().getExposureId() 

263 nAmpsNan = 0 

264 partialDatasetPtc = PhotonTransferCurveDataset(ampNames, '', 

265 self.config.maximumRangeCovariancesAstier) 

266 for ampNumber, amp in enumerate(detector): 

267 ampName = amp.getName() 

268 # covAstier: [(i, j, var (cov[0,0]), cov, npix) for (i,j) in {maxLag, maxLag}^2] 

269 doRealSpace = self.config.covAstierRealSpace 

270 if self.config.detectorMeasurementRegion == 'AMP': 

271 region = amp.getBBox() 

272 elif self.config.detectorMeasurementRegion == 'FULL': 

273 region = None 

274 # The variable `covAstier` is of the form: [(i, j, var (cov[0,0]), cov, npix) for (i,j) 

275 # in {maxLag, maxLag}^2] 

276 muDiff, varDiff, covAstier = self.measureMeanVarCov(exp1, exp2, region=region, 

277 covAstierRealSpace=doRealSpace) 

278 expIdMask = True 

279 if np.isnan(muDiff) or np.isnan(varDiff) or (covAstier is None): 

280 msg = (f"NaN mean or var, or None cov in amp {ampName} in exposure pair {expId1}," 

281 f" {expId2} of detector {detNum}.") 

282 self.log.warn(msg) 

283 nAmpsNan += 1 

284 expIdMask = False 

285 covArray = np.full((1, self.config.maximumRangeCovariancesAstier, 

286 self.config.maximumRangeCovariancesAstier), np.nan) 

287 covSqrtWeights = np.full_like(covArray, np.nan) 

288 

289 if (muDiff <= minMeanSignalDict[ampName]) or (muDiff >= maxMeanSignalDict[ampName]): 

290 expIdMask = False 

291 

292 partialDatasetPtc.rawExpTimes[ampName] = [expTime] 

293 partialDatasetPtc.rawMeans[ampName] = [muDiff] 

294 partialDatasetPtc.rawVars[ampName] = [varDiff] 

295 

296 if covAstier is not None: 

297 tupleRows = [(muDiff, varDiff) + covRow + (ampNumber, expTime, 

298 ampName) for covRow in covAstier] 

299 tempStructArray = np.array(tupleRows, dtype=tags) 

300 covArray, vcov, _ = makeCovArray(tempStructArray, 

301 self.config.maximumRangeCovariancesAstier) 

302 covSqrtWeights = np.nan_to_num(1./np.sqrt(vcov)) 

303 partialDatasetPtc.inputExpIdPairs[ampName] = [(expId1, expId2)] 

304 partialDatasetPtc.expIdMask[ampName] = [expIdMask] 

305 partialDatasetPtc.covariances[ampName] = covArray 

306 partialDatasetPtc.covariancesSqrtWeights[ampName] = covSqrtWeights 

307 partialDatasetPtc.covariancesModel[ampName] = np.full_like(covArray, np.nan) 

308 partialDatasetPtc.covariancesModelNoB[ampName] = np.full_like(covArray, np.nan) 

309 partialDatasetPtc.aMatrix[ampName] = np.full_like(covArray[0], np.nan) 

310 partialDatasetPtc.bMatrix[ampName] = np.full_like(covArray[0], np.nan) 

311 partialDatasetPtc.aMatrixNoB[ampName] = np.full_like(covArray[0], np.nan) 

312 partialDatasetPtc.ptcFitPars[ampName] = [np.nan] 

313 partialDatasetPtc.ptcFitParsError[ampName] = [np.nan] 

314 partialDatasetPtc.ptcFitChiSq[ampName] = np.nan 

315 partialDatasetPtc.finalVars[ampName] = [np.nan] 

316 partialDatasetPtc.finalModelVars[ampName] = [np.nan] 

317 partialDatasetPtc.finalMeans[ampName] = [np.nan] 

318 # Use location of exp1 to save PTC dataset from (exp1, exp2) pair. 

319 # expId1 and expId2, as returned by getInfo().getVisitInfo().getExposureId(), 

320 # and the exposure IDs stured in inoutDims, 

321 # may have the zero-padded detector number appended at 

322 # the end (in gen3). A temporary fix is to consider expId//1000 and/or 

323 # inputDims//1000. 

324 # Below, np.where(expId1 == np.array(inputDims)) (and the other analogous 

325 # comparisons) returns a tuple with a single-element array, so [0][0] 

326 # is necessary to extract the required index. 

327 try: 

328 datasetIndex = np.where(expId1 == np.array(inputDims))[0][0] 

329 except IndexError: 

330 try: 

331 datasetIndex = np.where(expId1//1000 == np.array(inputDims))[0][0] 

332 except IndexError: 

333 datasetIndex = np.where(expId1//1000 == np.array(inputDims)//1000)[0][0] 

334 partialDatasetPtcList[datasetIndex] = partialDatasetPtc 

335 if nAmpsNan == len(ampNames): 

336 msg = f"NaN mean in all amps of exposure pair {expId1}, {expId2} of detector {detNum}." 

337 self.log.warn(msg) 

338 return pipeBase.Struct( 

339 outputCovariances=partialDatasetPtcList, 

340 ) 

341 

342 def measureMeanVarCov(self, exposure1, exposure2, region=None, covAstierRealSpace=False): 

343 """Calculate the mean of each of two exposures and the variance 

344 and covariance of their difference. The variance is calculated 

345 via afwMath, and the covariance via the methods in Astier+19 

346 (appendix A). In theory, var = covariance[0,0]. This should 

347 be validated, and in the future, we may decide to just keep 

348 one (covariance). 

349 

350 Parameters 

351 ---------- 

352 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF` 

353 First exposure of flat field pair. 

354 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF` 

355 Second exposure of flat field pair. 

356 region : `lsst.geom.Box2I`, optional 

357 Region of each exposure where to perform the calculations (e.g, an amplifier). 

358 covAstierRealSpace : `bool`, optional 

359 Should the covariannces in Astier+19 be calculated in real space or via FFT? 

360 See Appendix A of Astier+19. 

361 

362 Returns 

363 ------- 

364 mu : `float` or `NaN` 

365 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in 

366 both exposures. If either mu1 or m2 are NaN's, the returned value is NaN. 

367 varDiff : `float` or `NaN` 

368 Half of the clipped variance of the difference of the regions inthe two input 

369 exposures. If either mu1 or m2 are NaN's, the returned value is NaN. 

370 covDiffAstier : `list` or `NaN` 

371 List with tuples of the form (dx, dy, var, cov, npix), where: 

372 dx : `int` 

373 Lag in x 

374 dy : `int` 

375 Lag in y 

376 var : `float` 

377 Variance at (dx, dy). 

378 cov : `float` 

379 Covariance at (dx, dy). 

380 nPix : `int` 

381 Number of pixel pairs used to evaluate var and cov. 

382 If either mu1 or m2 are NaN's, the returned value is NaN. 

383 """ 

384 

385 if region is not None: 

386 im1Area = exposure1.maskedImage[region] 

387 im2Area = exposure2.maskedImage[region] 

388 else: 

389 im1Area = exposure1.maskedImage 

390 im2Area = exposure2.maskedImage 

391 

392 if self.config.binSize > 1: 

393 im1Area = afwMath.binImage(im1Area, self.config.binSize) 

394 im2Area = afwMath.binImage(im2Area, self.config.binSize) 

395 

396 im1MaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList) 

397 im1StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc, 

398 self.config.nIterSigmaClipPtc, 

399 im1MaskVal) 

400 im1StatsCtrl.setNanSafe(True) 

401 im1StatsCtrl.setAndMask(im1MaskVal) 

402 

403 im2MaskVal = exposure2.getMask().getPlaneBitMask(self.config.maskNameList) 

404 im2StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc, 

405 self.config.nIterSigmaClipPtc, 

406 im2MaskVal) 

407 im2StatsCtrl.setNanSafe(True) 

408 im2StatsCtrl.setAndMask(im2MaskVal) 

409 

410 # Clipped mean of images; then average of mean. 

411 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, im1StatsCtrl).getValue() 

412 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, im2StatsCtrl).getValue() 

413 if np.isnan(mu1) or np.isnan(mu2): 

414 self.log.warn(f"Mean of amp in image 1 or 2 is NaN: {mu1}, {mu2}.") 

415 return np.nan, np.nan, None 

416 mu = 0.5*(mu1 + mu2) 

417 

418 # Take difference of pairs 

419 # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2)) 

420 temp = im2Area.clone() 

421 temp *= mu1 

422 diffIm = im1Area.clone() 

423 diffIm *= mu2 

424 diffIm -= temp 

425 diffIm /= mu 

426 

427 diffImMaskVal = diffIm.getMask().getPlaneBitMask(self.config.maskNameList) 

428 diffImStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc, 

429 self.config.nIterSigmaClipPtc, 

430 diffImMaskVal) 

431 diffImStatsCtrl.setNanSafe(True) 

432 diffImStatsCtrl.setAndMask(diffImMaskVal) 

433 

434 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue()) 

435 

436 # Get the mask and identify good pixels as '1', and the rest as '0'. 

437 w1 = np.where(im1Area.getMask().getArray() == 0, 1, 0) 

438 w2 = np.where(im2Area.getMask().getArray() == 0, 1, 0) 

439 

440 w12 = w1*w2 

441 wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0) 

442 w = w12*wDiff 

443 

444 if np.sum(w) < self.config.minNumberGoodPixelsForFft: 

445 self.log.warn(f"Number of good points for FFT ({np.sum(w)}) is less than threshold " 

446 f"({self.config.minNumberGoodPixelsForFft})") 

447 return np.nan, np.nan, None 

448 

449 maxRangeCov = self.config.maximumRangeCovariancesAstier 

450 if covAstierRealSpace: 

451 covDiffAstier = computeCovDirect(diffIm.image.array, w, maxRangeCov) 

452 else: 

453 shapeDiff = diffIm.image.array.shape 

454 fftShape = (fftSize(shapeDiff[0] + maxRangeCov), fftSize(shapeDiff[1]+maxRangeCov)) 

455 c = CovFft(diffIm.image.array, w, fftShape, maxRangeCov) 

456 covDiffAstier = c.reportCovFft(maxRangeCov) 

457 

458 return mu, varDiff, covDiffAstier