Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22import numpy as np 

23 

24import lsst.afw.math as afwMath 

25import lsst.pex.config as pexConfig 

26import lsst.pipe.base as pipeBase 

27from lsst.cp.pipe.utils import arrangeFlatsByExpTime, arrangeFlatsByExpId 

28 

29import lsst.pipe.base.connectionTypes as cT 

30 

31from .astierCovPtcUtils import (CovFastFourierTransform, computeCovDirect) 

32from .astierCovPtcFit import makeCovArray 

33 

34from lsst.ip.isr import PhotonTransferCurveDataset 

35 

36 

37__all__ = ['PhotonTransferCurveExtractConfig', 'PhotonTransferCurveExtractTask'] 

38 

39 

40class PhotonTransferCurveExtractConnections(pipeBase.PipelineTaskConnections, 

41 dimensions=("instrument", "detector")): 

42 

43 inputExp = cT.Input( 

44 name="ptcInputExposurePairs", 

45 doc="Input post-ISR processed exposure pairs (flats) to" 

46 "measure covariances from.", 

47 storageClass="Exposure", 

48 dimensions=("instrument", "exposure", "detector"), 

49 multiple=True, 

50 deferLoad=False, 

51 ) 

52 

53 outputCovariances = cT.Output( 

54 name="ptcCovariances", 

55 doc="Extracted flat (co)variances.", 

56 storageClass="PhotonTransferCurveDataset", 

57 dimensions=("instrument", "exposure", "detector"), 

58 multiple=True, 

59 ) 

60 

61 

62class PhotonTransferCurveExtractConfig(pipeBase.PipelineTaskConfig, 

63 pipelineConnections=PhotonTransferCurveExtractConnections): 

64 """Configuration for the measurement of covariances from flats. 

65 """ 

66 matchByExposureId = pexConfig.Field( 

67 dtype=bool, 

68 doc="Should exposures by matched by ID rather than exposure time?", 

69 default=False, 

70 ) 

71 maximumRangeCovariancesAstier = pexConfig.Field( 

72 dtype=int, 

73 doc="Maximum range of covariances as in Astier+19", 

74 default=8, 

75 ) 

76 covAstierRealSpace = pexConfig.Field( 

77 dtype=bool, 

78 doc="Calculate covariances in real space or via FFT? (see appendix A of Astier+19).", 

79 default=False, 

80 ) 

81 binSize = pexConfig.Field( 

82 dtype=int, 

83 doc="Bin the image by this factor in both dimensions.", 

84 default=1, 

85 ) 

86 minMeanSignal = pexConfig.DictField( 

87 keytype=str, 

88 itemtype=float, 

89 doc="Minimum values (inclusive) of mean signal (in ADU) above which to consider, per amp." 

90 " The same cut is applied to all amps if this dictionary is of the form" 

91 " {'ALL_AMPS': value}", 

92 default={'ALL_AMPS': 0.0}, 

93 ) 

94 maxMeanSignal = pexConfig.DictField( 

95 keytype=str, 

96 itemtype=float, 

97 doc="Maximum values (inclusive) of mean signal (in ADU) below which to consider, per amp." 

98 " The same cut is applied to all amps if this dictionary is of the form" 

99 " {'ALL_AMPS': value}", 

100 default={'ALL_AMPS': 1e6}, 

101 ) 

102 maskNameList = pexConfig.ListField( 

103 dtype=str, 

104 doc="Mask list to exclude from statistics calculations.", 

105 default=['SUSPECT', 'BAD', 'NO_DATA'], 

106 ) 

107 nSigmaClipPtc = pexConfig.Field( 

108 dtype=float, 

109 doc="Sigma cut for afwMath.StatisticsControl()", 

110 default=5.5, 

111 ) 

112 nIterSigmaClipPtc = pexConfig.Field( 

113 dtype=int, 

114 doc="Number of sigma-clipping iterations for afwMath.StatisticsControl()", 

115 default=1, 

116 ) 

117 minNumberGoodPixelsForCovariance = pexConfig.Field( 

118 dtype=int, 

119 doc="Minimum number of acceptable good pixels per amp to calculate the covariances (via FFT or" 

120 " direclty).", 

121 default=10000, 

122 ) 

123 detectorMeasurementRegion = pexConfig.ChoiceField( 

124 dtype=str, 

125 doc="Region of each exposure where to perform the calculations (amplifier or full image).", 

126 default='AMP', 

127 allowed={ 

128 "AMP": "Amplifier of the detector.", 

129 "FULL": "Full image." 

130 } 

131 ) 

132 

133 

134class PhotonTransferCurveExtractTask(pipeBase.PipelineTask, 

135 pipeBase.CmdLineTask): 

136 """Task to measure covariances from flat fields. 

137 This task receives as input a list of flat-field images 

138 (flats), and sorts these flats in pairs taken at the 

139 same time (if there's a different number of flats, 

140 those flats are discarded). The mean, variance, and 

141 covariances are measured from the difference of the flat 

142 pairs at a given time. The variance is calculated 

143 via afwMath, and the covariance via the methods in Astier+19 

144 (appendix A). In theory, var = covariance[0,0]. This should 

145 be validated, and in the future, we may decide to just keep 

146 one (covariance). 

147 

148 The measured covariances at a particular time (along with 

149 other quantities such as the mean) are stored in a PTC dataset 

150 object (`PhotonTransferCurveDataset`), which gets partially 

151 filled. The number of partially-filled PTC dataset objects 

152 will be less than the number of input exposures, but gen3 

153 requires/assumes that the number of input dimensions matches 

154 bijectively the number of output dimensions. Therefore, a 

155 number of "dummy" PTC dataset are inserted in the output list 

156 that has the partially-filled PTC datasets with the covariances. 

157 This output list will be used as input of 

158 `PhotonTransferCurveSolveTask`, which will assemble the multiple 

159 `PhotonTransferCurveDataset`s into a single one in order to fit 

160 the measured covariances as a function of flux to a particular 

161 model. 

162 

163 Astier+19: "The Shape of the Photon Transfer Curve of CCD 

164 sensors", arXiv:1905.08677. 

165 """ 

166 ConfigClass = PhotonTransferCurveExtractConfig 

167 _DefaultName = 'cpPtcExtract' 

168 

169 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

170 """Ensure that the input and output dimensions are passed along. 

171 

172 Parameters 

173 ---------- 

174 butlerQC : `~lsst.daf.butler.butlerQuantumContext.ButlerQuantumContext` 

175 Butler to operate on. 

176 inputRefs : `~lsst.pipe.base.connections.InputQuantizedConnection` 

177 Input data refs to load. 

178 ouptutRefs : `~lsst.pipe.base.connections.OutputQuantizedConnection` 

179 Output data refs to persist. 

180 """ 

181 inputs = butlerQC.get(inputRefs) 

182 # Dictionary, keyed by expTime, with flat exposures 

183 if self.config.matchByExposureId: 

184 inputs['inputExp'] = arrangeFlatsByExpId(inputs['inputExp']) 

185 else: 

186 inputs['inputExp'] = arrangeFlatsByExpTime(inputs['inputExp']) 

187 # Ids of input list of exposures 

188 inputs['inputDims'] = [expId.dataId['exposure'] for expId in inputRefs.inputExp] 

189 outputs = self.run(**inputs) 

190 butlerQC.put(outputs, outputRefs) 

191 

192 def run(self, inputExp, inputDims): 

193 """Measure covariances from difference of flat pairs 

194 

195 Parameters 

196 ---------- 

197 inputExp : `dict` [`float`, 

198 (`~lsst.afw.image.exposure.exposure.ExposureF`, 

199 `~lsst.afw.image.exposure.exposure.ExposureF`, ..., 

200 `~lsst.afw.image.exposure.exposure.ExposureF`)] 

201 Dictionary that groups flat-field exposures that have the same 

202 exposure time (seconds). 

203 

204 inputDims : `list` 

205 List of exposure IDs. 

206 """ 

207 # inputExp.values() returns a view, which we turn into a list. We then 

208 # access the first exposure to get teh detector. 

209 detector = list(inputExp.values())[0][0].getDetector() 

210 detNum = detector.getId() 

211 amps = detector.getAmplifiers() 

212 ampNames = [amp.getName() for amp in amps] 

213 

214 # Each amp may have a different min and max ADU signal specified in the config. 

215 maxMeanSignalDict = {ampName: 1e6 for ampName in ampNames} 

216 minMeanSignalDict = {ampName: 0.0 for ampName in ampNames} 

217 for ampName in ampNames: 

218 if 'ALL_AMPS' in self.config.maxMeanSignal: 

219 maxMeanSignalDict[ampName] = self.config.maxMeanSignal['ALL_AMPS'] 

220 elif ampName in self.config.maxMeanSignal: 

221 maxMeanSignalDict[ampName] = self.config.maxMeanSignal[ampName] 

222 

223 if 'ALL_AMPS' in self.config.minMeanSignal: 

224 minMeanSignalDict[ampName] = self.config.minMeanSignal['ALL_AMPS'] 

225 elif ampName in self.config.minMeanSignal: 

226 minMeanSignalDict[ampName] = self.config.minMeanSignal[ampName] 

227 # These are the column names for `tupleRows` below. 

228 tags = [('mu', '<f8'), ('afwVar', '<f8'), ('i', '<i8'), ('j', '<i8'), ('var', '<f8'), 

229 ('cov', '<f8'), ('npix', '<i8'), ('ext', '<i8'), ('expTime', '<f8'), ('ampName', '<U3')] 

230 # Create a dummy ptcDataset 

231 dummyPtcDataset = PhotonTransferCurveDataset(ampNames, 'DUMMY', 

232 self.config.maximumRangeCovariancesAstier) 

233 # Initialize amps of `dummyPtcDatset`. 

234 for ampName in ampNames: 

235 dummyPtcDataset.setAmpValues(ampName) 

236 # Output list with PTC datasets. 

237 partialPtcDatasetList = [] 

238 # The number of output references needs to match that of input references: 

239 # initialize outputlist with dummy PTC datasets. 

240 for i in range(len(inputDims)): 

241 partialPtcDatasetList.append(dummyPtcDataset) 

242 

243 for expTime in inputExp: 

244 exposures = inputExp[expTime] 

245 if len(exposures) == 1: 

246 self.log.warn(f"Only one exposure found at expTime {expTime}. Dropping exposure " 

247 f"{exposures[0].getInfo().getVisitInfo().getExposureId()}.") 

248 continue 

249 else: 

250 # Only use the first two exposures at expTime 

251 exp1, exp2 = exposures[0], exposures[1] 

252 if len(exposures) > 2: 

253 self.log.warn(f"Already found 2 exposures at expTime {expTime}. " 

254 "Ignoring exposures: " 

255 f"{i.getInfo().getVisitInfo().getExposureId() for i in exposures[2:]}") 

256 expId1 = exp1.getInfo().getVisitInfo().getExposureId() 

257 expId2 = exp2.getInfo().getVisitInfo().getExposureId() 

258 nAmpsNan = 0 

259 partialPtcDataset = PhotonTransferCurveDataset(ampNames, '', 

260 self.config.maximumRangeCovariancesAstier) 

261 for ampNumber, amp in enumerate(detector): 

262 ampName = amp.getName() 

263 # covAstier: [(i, j, var (cov[0,0]), cov, npix) for (i,j) in {maxLag, maxLag}^2] 

264 doRealSpace = self.config.covAstierRealSpace 

265 if self.config.detectorMeasurementRegion == 'AMP': 

266 region = amp.getBBox() 

267 elif self.config.detectorMeasurementRegion == 'FULL': 

268 region = None 

269 # `measureMeanVarCov` is the function that measures the variance and covariances from 

270 # the difference image of two flats at the same exposure time. 

271 # The variable `covAstier` is of the form: [(i, j, var (cov[0,0]), cov, npix) for (i,j) 

272 # in {maxLag, maxLag}^2] 

273 muDiff, varDiff, covAstier = self.measureMeanVarCov(exp1, exp2, region=region, 

274 covAstierRealSpace=doRealSpace) 

275 expIdMask = True 

276 if np.isnan(muDiff) or np.isnan(varDiff) or (covAstier is None): 

277 msg = (f"NaN mean or var, or None cov in amp {ampName} in exposure pair {expId1}," 

278 f" {expId2} of detector {detNum}.") 

279 self.log.warn(msg) 

280 nAmpsNan += 1 

281 expIdMask = False 

282 covArray = np.full((1, self.config.maximumRangeCovariancesAstier, 

283 self.config.maximumRangeCovariancesAstier), np.nan) 

284 covSqrtWeights = np.full_like(covArray, np.nan) 

285 

286 if (muDiff <= minMeanSignalDict[ampName]) or (muDiff >= maxMeanSignalDict[ampName]): 

287 expIdMask = False 

288 

289 if covAstier is not None: 

290 tupleRows = [(muDiff, varDiff) + covRow + (ampNumber, expTime, 

291 ampName) for covRow in covAstier] 

292 tempStructArray = np.array(tupleRows, dtype=tags) 

293 covArray, vcov, _ = makeCovArray(tempStructArray, 

294 self.config.maximumRangeCovariancesAstier) 

295 covSqrtWeights = np.nan_to_num(1./np.sqrt(vcov)) 

296 

297 partialPtcDataset.setAmpValues(ampName, rawExpTime=[expTime], rawMean=[muDiff], 

298 rawVar=[varDiff], inputExpIdPair=[(expId1, expId2)], 

299 expIdMask=[expIdMask], covArray=covArray, 

300 covSqrtWeights=covSqrtWeights) 

301 # Use location of exp1 to save PTC dataset from (exp1, exp2) pair. 

302 # expId1 and expId2, as returned by getInfo().getVisitInfo().getExposureId(), 

303 # and the exposure IDs stured in inoutDims, 

304 # may have the zero-padded detector number appended at 

305 # the end (in gen3). A temporary fix is to consider expId//1000 and/or 

306 # inputDims//1000. 

307 # Below, np.where(expId1 == np.array(inputDims)) (and the other analogous 

308 # comparisons) returns a tuple with a single-element array, so [0][0] 

309 # is necessary to extract the required index. 

310 try: 

311 datasetIndex = np.where(expId1 == np.array(inputDims))[0][0] 

312 except IndexError: 

313 try: 

314 datasetIndex = np.where(expId1//1000 == np.array(inputDims))[0][0] 

315 except IndexError: 

316 datasetIndex = np.where(expId1//1000 == np.array(inputDims)//1000)[0][0] 

317 partialPtcDatasetList[datasetIndex] = partialPtcDataset 

318 if nAmpsNan == len(ampNames): 

319 msg = f"NaN mean in all amps of exposure pair {expId1}, {expId2} of detector {detNum}." 

320 self.log.warn(msg) 

321 return pipeBase.Struct( 

322 outputCovariances=partialPtcDatasetList, 

323 ) 

324 

325 def measureMeanVarCov(self, exposure1, exposure2, region=None, covAstierRealSpace=False): 

326 """Calculate the mean of each of two exposures and the variance 

327 and covariance of their difference. The variance is calculated 

328 via afwMath, and the covariance via the methods in Astier+19 

329 (appendix A). In theory, var = covariance[0,0]. This should 

330 be validated, and in the future, we may decide to just keep 

331 one (covariance). 

332 

333 Parameters 

334 ---------- 

335 exposure1 : `lsst.afw.image.exposure.exposure.ExposureF` 

336 First exposure of flat field pair. 

337 exposure2 : `lsst.afw.image.exposure.exposure.ExposureF` 

338 Second exposure of flat field pair. 

339 region : `lsst.geom.Box2I`, optional 

340 Region of each exposure where to perform the calculations (e.g, an amplifier). 

341 covAstierRealSpace : `bool`, optional 

342 Should the covariannces in Astier+19 be calculated in real space or via FFT? 

343 See Appendix A of Astier+19. 

344 

345 Returns 

346 ------- 

347 mu : `float` or `NaN` 

348 0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in 

349 both exposures. If either mu1 or m2 are NaN's, the returned value is NaN. 

350 varDiff : `float` or `NaN` 

351 Half of the clipped variance of the difference of the regions inthe two input 

352 exposures. If either mu1 or m2 are NaN's, the returned value is NaN. 

353 covDiffAstier : `list` or `NaN` 

354 List with tuples of the form (dx, dy, var, cov, npix), where: 

355 dx : `int` 

356 Lag in x 

357 dy : `int` 

358 Lag in y 

359 var : `float` 

360 Variance at (dx, dy). 

361 cov : `float` 

362 Covariance at (dx, dy). 

363 nPix : `int` 

364 Number of pixel pairs used to evaluate var and cov. 

365 If either mu1 or m2 are NaN's, the returned value is NaN. 

366 """ 

367 

368 if region is not None: 

369 im1Area = exposure1.maskedImage[region] 

370 im2Area = exposure2.maskedImage[region] 

371 else: 

372 im1Area = exposure1.maskedImage 

373 im2Area = exposure2.maskedImage 

374 

375 if self.config.binSize > 1: 

376 im1Area = afwMath.binImage(im1Area, self.config.binSize) 

377 im2Area = afwMath.binImage(im2Area, self.config.binSize) 

378 

379 im1MaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList) 

380 im1StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc, 

381 self.config.nIterSigmaClipPtc, 

382 im1MaskVal) 

383 im1StatsCtrl.setNanSafe(True) 

384 im1StatsCtrl.setAndMask(im1MaskVal) 

385 

386 im2MaskVal = exposure2.getMask().getPlaneBitMask(self.config.maskNameList) 

387 im2StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc, 

388 self.config.nIterSigmaClipPtc, 

389 im2MaskVal) 

390 im2StatsCtrl.setNanSafe(True) 

391 im2StatsCtrl.setAndMask(im2MaskVal) 

392 

393 # Clipped mean of images; then average of mean. 

394 mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, im1StatsCtrl).getValue() 

395 mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, im2StatsCtrl).getValue() 

396 if np.isnan(mu1) or np.isnan(mu2): 

397 self.log.warn(f"Mean of amp in image 1 or 2 is NaN: {mu1}, {mu2}.") 

398 return np.nan, np.nan, None 

399 mu = 0.5*(mu1 + mu2) 

400 

401 # Take difference of pairs 

402 # symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2)) 

403 temp = im2Area.clone() 

404 temp *= mu1 

405 diffIm = im1Area.clone() 

406 diffIm *= mu2 

407 diffIm -= temp 

408 diffIm /= mu 

409 

410 diffImMaskVal = diffIm.getMask().getPlaneBitMask(self.config.maskNameList) 

411 diffImStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc, 

412 self.config.nIterSigmaClipPtc, 

413 diffImMaskVal) 

414 diffImStatsCtrl.setNanSafe(True) 

415 diffImStatsCtrl.setAndMask(diffImMaskVal) 

416 

417 # Variance calculation via afwMath 

418 varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue()) 

419 

420 # Covariances calculations 

421 # Get the mask and identify good pixels as '1', and the rest as '0'. 

422 w1 = np.where(im1Area.getMask().getArray() == 0, 1, 0) 

423 w2 = np.where(im2Area.getMask().getArray() == 0, 1, 0) 

424 

425 w12 = w1*w2 

426 wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0) 

427 w = w12*wDiff 

428 

429 if np.sum(w) < self.config.minNumberGoodPixelsForCovariance: 

430 self.log.warn(f"Number of good points for covariance calculation ({np.sum(w)}) is less " 

431 f"(than threshold {self.config.minNumberGoodPixelsForCovariance})") 

432 return np.nan, np.nan, None 

433 

434 maxRangeCov = self.config.maximumRangeCovariancesAstier 

435 if covAstierRealSpace: 

436 # Calculate covariances in real space. 

437 covDiffAstier = computeCovDirect(diffIm.image.array, w, maxRangeCov) 

438 else: 

439 # Calculate covariances via FFT (default). 

440 shapeDiff = np.array(diffIm.image.array.shape) 

441 # Calculate the sizes of FFT dimensions. 

442 s = shapeDiff + maxRangeCov 

443 tempSize = np.array(np.log(s)/np.log(2.)).astype(int) 

444 fftSize = np.array(2**(tempSize+1)).astype(int) 

445 fftShape = (fftSize[0], fftSize[1]) 

446 

447 c = CovFastFourierTransform(diffIm.image.array, w, fftShape, maxRangeCov) 

448 covDiffAstier = c.reportCovFastFourierTransform(maxRangeCov) 

449 

450 return mu, varDiff, covDiffAstier