Coverage for python/lsst/cp/pipe/utils.py: 11%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

256 statements  

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22 

23__all__ = ['PairedVisitListTaskRunner', 'SingleVisitListTaskRunner', 

24 'NonexistentDatasetTaskDataIdContainer', 'parseCmdlineNumberString', 

25 'countMaskedPixels', 'checkExpLengthEqual', 'ddict2dict'] 

26 

27import re 

28import numpy as np 

29from scipy.optimize import leastsq 

30import numpy.polynomial.polynomial as poly 

31from scipy.stats import norm 

32import logging 

33 

34import lsst.pipe.base as pipeBase 

35import lsst.ip.isr as ipIsr 

36from lsst.ip.isr import isrMock 

37import lsst.afw.image 

38 

39import galsim 

40 

41 

42def sigmaClipCorrection(nSigClip): 

43 """Correct measured sigma to account for clipping. 

44 

45 If we clip our input data and then measure sigma, then the 

46 measured sigma is smaller than the true value because real 

47 points beyond the clip threshold have been removed. This is a 

48 small (1.5% at nSigClip=3) effect when nSigClip >~ 3, but the 

49 default parameters for measure crosstalk use nSigClip=2.0. 

50 This causes the measured sigma to be about 15% smaller than 

51 real. This formula corrects the issue, for the symmetric case 

52 (upper clip threshold equal to lower clip threshold). 

53 

54 Parameters 

55 ---------- 

56 nSigClip : `float` 

57 Number of sigma the measurement was clipped by. 

58 

59 Returns 

60 ------- 

61 scaleFactor : `float` 

62 Scale factor to increase the measured sigma by. 

63 """ 

64 varFactor = 1.0 - (2 * nSigClip * norm.pdf(nSigClip)) / (norm.cdf(nSigClip) - norm.cdf(-nSigClip)) 

65 return 1.0 / np.sqrt(varFactor) 

66 

67 

68def calculateWeightedReducedChi2(measured, model, weightsMeasured, nData, nParsModel): 

69 """Calculate weighted reduced chi2. 

70 

71 Parameters 

72 ---------- 

73 

74 measured : `list` 

75 List with measured data. 

76 

77 model : `list` 

78 List with modeled data. 

79 

80 weightsMeasured : `list` 

81 List with weights for the measured data. 

82 

83 nData : `int` 

84 Number of data points. 

85 

86 nParsModel : `int` 

87 Number of parameters in the model. 

88 

89 Returns 

90 ------- 

91 

92 redWeightedChi2 : `float` 

93 Reduced weighted chi2. 

94 """ 

95 wRes = (measured - model)*weightsMeasured 

96 return ((wRes*wRes).sum())/(nData-nParsModel) 

97 

98 

99def makeMockFlats(expTime, gain=1.0, readNoiseElectrons=5, fluxElectrons=1000, 

100 randomSeedFlat1=1984, randomSeedFlat2=666, powerLawBfParams=[], 

101 expId1=0, expId2=1): 

102 """Create a pair or mock flats with isrMock. 

103 

104 Parameters 

105 ---------- 

106 expTime : `float` 

107 Exposure time of the flats. 

108 

109 gain : `float`, optional 

110 Gain, in e/ADU. 

111 

112 readNoiseElectrons : `float`, optional 

113 Read noise rms, in electrons. 

114 

115 fluxElectrons : `float`, optional 

116 Flux of flats, in electrons per second. 

117 

118 randomSeedFlat1 : `int`, optional 

119 Random seed for the normal distrubutions for the mean signal 

120 and noise (flat1). 

121 

122 randomSeedFlat2 : `int`, optional 

123 Random seed for the normal distrubutions for the mean signal 

124 and noise (flat2). 

125 

126 powerLawBfParams : `list`, optional 

127 Parameters for `galsim.cdmodel.PowerLawCD` to simulate the 

128 brightter-fatter effect. 

129 

130 expId1 : `int`, optional 

131 Exposure ID for first flat. 

132 

133 expId2 : `int`, optional 

134 Exposure ID for second flat. 

135 

136 Returns 

137 ------- 

138 

139 flatExp1 : `lsst.afw.image.exposure.ExposureF` 

140 First exposure of flat field pair. 

141 

142 flatExp2 : `lsst.afw.image.exposure.ExposureF` 

143 Second exposure of flat field pair. 

144 

145 Notes 

146 ----- 

147 The parameters of `galsim.cdmodel.PowerLawCD` are `n, r0, t0, rx, 

148 tx, r, t, alpha`. For more information about their meaning, see 

149 the Galsim documentation 

150 https://galsim-developers.github.io/GalSim/_build/html/_modules/galsim/cdmodel.html # noqa: W505 

151 and Gruen+15 (1501.02802). 

152 

153 Example: galsim.cdmodel.PowerLawCD(8, 1.1e-7, 1.1e-7, 1.0e-8, 

154 1.0e-8, 1.0e-9, 1.0e-9, 2.0) 

155 """ 

156 flatFlux = fluxElectrons # e/s 

157 flatMean = flatFlux*expTime # e 

158 readNoise = readNoiseElectrons # e 

159 

160 mockImageConfig = isrMock.IsrMock.ConfigClass() 

161 

162 mockImageConfig.flatDrop = 0.99999 

163 mockImageConfig.isTrimmed = True 

164 

165 flatExp1 = isrMock.FlatMock(config=mockImageConfig).run() 

166 flatExp2 = flatExp1.clone() 

167 (shapeY, shapeX) = flatExp1.getDimensions() 

168 flatWidth = np.sqrt(flatMean) 

169 

170 rng1 = np.random.RandomState(randomSeedFlat1) 

171 flatData1 = rng1.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng1.normal(0.0, readNoise, 

172 (shapeX, shapeY)) 

173 rng2 = np.random.RandomState(randomSeedFlat2) 

174 flatData2 = rng2.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng2.normal(0.0, readNoise, 

175 (shapeX, shapeY)) 

176 # Simulate BF with power law model in galsim 

177 if len(powerLawBfParams): 

178 if not len(powerLawBfParams) == 8: 

179 raise RuntimeError("Wrong number of parameters for `galsim.cdmodel.PowerLawCD`. " 

180 f"Expected 8; passed {len(powerLawBfParams)}.") 

181 cd = galsim.cdmodel.PowerLawCD(*powerLawBfParams) 

182 tempFlatData1 = galsim.Image(flatData1) 

183 temp2FlatData1 = cd.applyForward(tempFlatData1) 

184 

185 tempFlatData2 = galsim.Image(flatData2) 

186 temp2FlatData2 = cd.applyForward(tempFlatData2) 

187 

188 flatExp1.image.array[:] = temp2FlatData1.array/gain # ADU 

189 flatExp2.image.array[:] = temp2FlatData2.array/gain # ADU 

190 else: 

191 flatExp1.image.array[:] = flatData1/gain # ADU 

192 flatExp2.image.array[:] = flatData2/gain # ADU 

193 

194 visitInfoExp1 = lsst.afw.image.VisitInfo(exposureId=expId1, exposureTime=expTime) 

195 visitInfoExp2 = lsst.afw.image.VisitInfo(exposureId=expId2, exposureTime=expTime) 

196 

197 flatExp1.info.id = expId1 

198 flatExp1.getInfo().setVisitInfo(visitInfoExp1) 

199 flatExp2.info.id = expId2 

200 flatExp2.getInfo().setVisitInfo(visitInfoExp2) 

201 

202 return flatExp1, flatExp2 

203 

204 

205def countMaskedPixels(maskedIm, maskPlane): 

206 """Count the number of pixels in a given mask plane. 

207 

208 Parameters 

209 ---------- 

210 maskedIm : `~lsst.afw.image.MaskedImage` 

211 Masked image to examine. 

212 maskPlane : `str` 

213 Name of the mask plane to examine. 

214 

215 Returns 

216 ------- 

217 nPix : `int` 

218 Number of pixels in the requested mask plane. 

219 """ 

220 maskBit = maskedIm.mask.getPlaneBitMask(maskPlane) 

221 nPix = np.where(np.bitwise_and(maskedIm.mask.array, maskBit))[0].flatten().size 

222 return nPix 

223 

224 

225class PairedVisitListTaskRunner(pipeBase.TaskRunner): 

226 """Subclass of TaskRunner for handling intrinsically paired visits. 

227 

228 This transforms the processed arguments generated by the ArgumentParser 

229 into the arguments expected by tasks which take visit pairs for their 

230 run() methods. 

231 

232 Such tasks' run() methods tend to take two arguments, 

233 one of which is the dataRef (as usual), and the other is the list 

234 of visit-pairs, in the form of a list of tuples. 

235 This list is supplied on the command line as documented, 

236 and this class parses that, and passes the parsed version 

237 to the run() method. 

238 

239 See pipeBase.TaskRunner for more information. 

240 """ 

241 

242 @staticmethod 

243 def getTargetList(parsedCmd, **kwargs): 

244 """Parse the visit list and pass through explicitly.""" 

245 visitPairs = [] 

246 for visitStringPair in parsedCmd.visitPairs: 

247 visitStrings = visitStringPair.split(",") 

248 if len(visitStrings) != 2: 

249 raise RuntimeError("Found {} visits in {} instead of 2".format(len(visitStrings), 

250 visitStringPair)) 

251 try: 

252 visits = [int(visit) for visit in visitStrings] 

253 except Exception: 

254 raise RuntimeError("Could not parse {} as two integer visit numbers".format(visitStringPair)) 

255 visitPairs.append(visits) 

256 

257 return pipeBase.TaskRunner.getTargetList(parsedCmd, visitPairs=visitPairs, **kwargs) 

258 

259 

260def parseCmdlineNumberString(inputString): 

261 """Parse command line numerical expression sytax and return as list of int 

262 

263 Take an input of the form "'1..5:2^123..126'" as a string, and return 

264 a list of ints as [1, 3, 5, 123, 124, 125, 126] 

265 

266 Parameters 

267 ---------- 

268 inputString : `str` 

269 String to be parsed. 

270 

271 Returns 

272 ------- 

273 outList : `list` [`int`] 

274 List of integers identified in the string. 

275 """ 

276 outList = [] 

277 for subString in inputString.split("^"): 

278 mat = re.search(r"^(\d+)\.\.(\d+)(?::(\d+))?$", subString) 

279 if mat: 

280 v1 = int(mat.group(1)) 

281 v2 = int(mat.group(2)) 

282 v3 = mat.group(3) 

283 v3 = int(v3) if v3 else 1 

284 for v in range(v1, v2 + 1, v3): 

285 outList.append(int(v)) 

286 else: 

287 outList.append(int(subString)) 

288 return outList 

289 

290 

291class SingleVisitListTaskRunner(pipeBase.TaskRunner): 

292 """Subclass of TaskRunner for tasks requiring a list of visits per dataRef. 

293 

294 This transforms the processed arguments generated by the ArgumentParser 

295 into the arguments expected by tasks which require a list of visits 

296 to be supplied for each dataRef, as is common in `lsst.cp.pipe` code. 

297 

298 Such tasks' run() methods tend to take two arguments, 

299 one of which is the dataRef (as usual), and the other is the list 

300 of visits. 

301 This list is supplied on the command line as documented, 

302 and this class parses that, and passes the parsed version 

303 to the run() method. 

304 

305 See `lsst.pipe.base.TaskRunner` for more information. 

306 """ 

307 

308 @staticmethod 

309 def getTargetList(parsedCmd, **kwargs): 

310 """Parse the visit list and pass through explicitly.""" 

311 # if this has been pre-parsed and therefore doesn't have length of one 

312 # then something has gone wrong, so execution should stop here. 

313 assert len(parsedCmd.visitList) == 1, 'visitList parsing assumptions violated' 

314 visits = parseCmdlineNumberString(parsedCmd.visitList[0]) 

315 

316 return pipeBase.TaskRunner.getTargetList(parsedCmd, visitList=visits, **kwargs) 

317 

318 

319class NonexistentDatasetTaskDataIdContainer(pipeBase.DataIdContainer): 

320 """A DataIdContainer for the tasks for which the output does 

321 not yet exist.""" 

322 

323 def makeDataRefList(self, namespace): 

324 """Compute refList based on idList. 

325 

326 This method must be defined as the dataset does not exist before this 

327 task is run. 

328 

329 Parameters 

330 ---------- 

331 namespace 

332 Results of parsing the command-line. 

333 

334 Notes 

335 ----- 

336 Not called if ``add_id_argument`` called 

337 with ``doMakeDataRefList=False``. 

338 Note that this is almost a copy-and-paste of the vanilla 

339 implementation, but without checking if the datasets already exist, 

340 as this task exists to make them. 

341 """ 

342 if self.datasetType is None: 

343 raise RuntimeError("Must call setDatasetType first") 

344 butler = namespace.butler 

345 for dataId in self.idList: 

346 refList = list(butler.subset(datasetType=self.datasetType, level=self.level, dataId=dataId)) 

347 # exclude nonexistent data 

348 # this is a recursive test, e.g. for the sake of "raw" data 

349 if not refList: 

350 namespace.log.warning("No data found for dataId=%s", dataId) 

351 continue 

352 self.refList += refList 

353 

354 

355def irlsFit(initialParams, dataX, dataY, function, weightsY=None, weightType='Cauchy', scaleResidual=True): 

356 """Iteratively reweighted least squares fit. 

357 

358 This uses the `lsst.cp.pipe.utils.fitLeastSq`, but applies weights 

359 based on the Cauchy distribution by default. Other weight options 

360 are implemented. See e.g. Holland and Welsch, 1977, 

361 doi:10.1080/03610927708827533 

362 

363 Parameters 

364 ---------- 

365 initialParams : `list` [`float`] 

366 Starting parameters. 

367 dataX : `numpy.array`, (N,) 

368 Abscissa data. 

369 dataY : `numpy.array`, (N,) 

370 Ordinate data. 

371 function : callable 

372 Function to fit. 

373 weightsY : `numpy.array`, (N,) 

374 Weights to apply to the data. 

375 weightType : `str`, optional 

376 Type of weighting to use. One of Cauchy, Anderson, bisquare, 

377 box, Welsch, Huber, logistic, or Fair. 

378 scaleResidual : `bool`, optional 

379 If true, the residual is scaled by the sqrt of the Y values. 

380 

381 Returns 

382 ------- 

383 polyFit : `list` [`float`] 

384 Final best fit parameters. 

385 polyFitErr : `list` [`float`] 

386 Final errors on fit parameters. 

387 chiSq : `float` 

388 Reduced chi squared. 

389 weightsY : `list` [`float`] 

390 Final weights used for each point. 

391 

392 Raises 

393 ------ 

394 RuntimeError : 

395 Raised if an unknown weightType string is passed. 

396 """ 

397 if not weightsY: 

398 weightsY = np.ones_like(dataX) 

399 

400 polyFit, polyFitErr, chiSq = fitLeastSq(initialParams, dataX, dataY, function, weightsY=weightsY) 

401 for iteration in range(10): 

402 resid = np.abs(dataY - function(polyFit, dataX)) 

403 if scaleResidual: 

404 resid = resid / np.sqrt(dataY) 

405 if weightType == 'Cauchy': 

406 # Use Cauchy weighting. This is a soft weight. 

407 # At [2, 3, 5, 10] sigma, weights are [.59, .39, .19, .05]. 

408 Z = resid / 2.385 

409 weightsY = 1.0 / (1.0 + np.square(Z)) 

410 elif weightType == 'Anderson': 

411 # Anderson+1972 weighting. This is a hard weight. 

412 # At [2, 3, 5, 10] sigma, weights are [.67, .35, 0.0, 0.0]. 

413 Z = resid / (1.339 * np.pi) 

414 weightsY = np.where(Z < 1.0, np.sinc(Z), 0.0) 

415 elif weightType == 'bisquare': 

416 # Beaton and Tukey (1974) biweight. This is a hard weight. 

417 # At [2, 3, 5, 10] sigma, weights are [.81, .59, 0.0, 0.0]. 

418 Z = resid / 4.685 

419 weightsY = np.where(Z < 1.0, 1.0 - np.square(Z), 0.0) 

420 elif weightType == 'box': 

421 # Hinich and Talwar (1975). This is a hard weight. 

422 # At [2, 3, 5, 10] sigma, weights are [1.0, 0.0, 0.0, 0.0]. 

423 weightsY = np.where(resid < 2.795, 1.0, 0.0) 

424 elif weightType == 'Welsch': 

425 # Dennis and Welsch (1976). This is a hard weight. 

426 # At [2, 3, 5, 10] sigma, weights are [.64, .36, .06, 1e-5]. 

427 Z = resid / 2.985 

428 weightsY = np.exp(-1.0 * np.square(Z)) 

429 elif weightType == 'Huber': 

430 # Huber (1964) weighting. This is a soft weight. 

431 # At [2, 3, 5, 10] sigma, weights are [.67, .45, .27, .13]. 

432 Z = resid / 1.345 

433 weightsY = np.where(Z < 1.0, 1.0, 1 / Z) 

434 elif weightType == 'logistic': 

435 # Logistic weighting. This is a soft weight. 

436 # At [2, 3, 5, 10] sigma, weights are [.56, .40, .24, .12]. 

437 Z = resid / 1.205 

438 weightsY = np.tanh(Z) / Z 

439 elif weightType == 'Fair': 

440 # Fair (1974) weighting. This is a soft weight. 

441 # At [2, 3, 5, 10] sigma, weights are [.41, .32, .22, .12]. 

442 Z = resid / 1.4 

443 weightsY = (1.0 / (1.0 + (Z))) 

444 else: 

445 raise RuntimeError(f"Unknown weighting type: {weightType}") 

446 polyFit, polyFitErr, chiSq = fitLeastSq(initialParams, dataX, dataY, function, weightsY=weightsY) 

447 

448 return polyFit, polyFitErr, chiSq, weightsY 

449 

450 

451def fitLeastSq(initialParams, dataX, dataY, function, weightsY=None): 

452 """Do a fit and estimate the parameter errors using using 

453 scipy.optimize.leastq. 

454 

455 optimize.leastsq returns the fractional covariance matrix. To 

456 estimate the standard deviation of the fit parameters, multiply 

457 the entries of this matrix by the unweighted reduced chi squared 

458 and take the square root of the diagonal elements. 

459 

460 Parameters 

461 ---------- 

462 initialParams : `list` [`float`] 

463 initial values for fit parameters. For ptcFitType=POLYNOMIAL, 

464 its length determines the degree of the polynomial. 

465 

466 dataX : `numpy.array`, (N,) 

467 Data in the abscissa axis. 

468 

469 dataY : `numpy.array`, (N,) 

470 Data in the ordinate axis. 

471 

472 function : callable object (function) 

473 Function to fit the data with. 

474 

475 weightsY : `numpy.array`, (N,) 

476 Weights of the data in the ordinate axis. 

477 

478 Return 

479 ------ 

480 pFitSingleLeastSquares : `list` [`float`] 

481 List with fitted parameters. 

482 

483 pErrSingleLeastSquares : `list` [`float`] 

484 List with errors for fitted parameters. 

485 

486 reducedChiSqSingleLeastSquares : `float` 

487 Reduced chi squared, unweighted if weightsY is not provided. 

488 """ 

489 if weightsY is None: 

490 weightsY = np.ones(len(dataX)) 

491 

492 def errFunc(p, x, y, weightsY=None): 

493 if weightsY is None: 

494 weightsY = np.ones(len(x)) 

495 return (function(p, x) - y)*weightsY 

496 

497 pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams, 

498 args=(dataX, dataY, weightsY), full_output=1, 

499 epsfcn=0.0001) 

500 

501 if (len(dataY) > len(initialParams)) and pCov is not None: 

502 reducedChiSq = calculateWeightedReducedChi2(dataY, function(pFit, dataX), weightsY, len(dataY), 

503 len(initialParams)) 

504 pCov *= reducedChiSq 

505 else: 

506 pCov = np.zeros((len(initialParams), len(initialParams))) 

507 pCov[:, :] = np.nan 

508 reducedChiSq = np.nan 

509 

510 errorVec = [] 

511 for i in range(len(pFit)): 

512 errorVec.append(np.fabs(pCov[i][i])**0.5) 

513 

514 pFitSingleLeastSquares = pFit 

515 pErrSingleLeastSquares = np.array(errorVec) 

516 

517 return pFitSingleLeastSquares, pErrSingleLeastSquares, reducedChiSq 

518 

519 

520def fitBootstrap(initialParams, dataX, dataY, function, weightsY=None, confidenceSigma=1.): 

521 """Do a fit using least squares and bootstrap to estimate parameter errors. 

522 

523 The bootstrap error bars are calculated by fitting 100 random data sets. 

524 

525 Parameters 

526 ---------- 

527 initialParams : `list` [`float`] 

528 initial values for fit parameters. For ptcFitType=POLYNOMIAL, 

529 its length determines the degree of the polynomial. 

530 

531 dataX : `numpy.array`, (N,) 

532 Data in the abscissa axis. 

533 

534 dataY : `numpy.array`, (N,) 

535 Data in the ordinate axis. 

536 

537 function : callable object (function) 

538 Function to fit the data with. 

539 

540 weightsY : `numpy.array`, (N,), optional. 

541 Weights of the data in the ordinate axis. 

542 

543 confidenceSigma : `float`, optional. 

544 Number of sigmas that determine confidence interval for the 

545 bootstrap errors. 

546 

547 Return 

548 ------ 

549 pFitBootstrap : `list` [`float`] 

550 List with fitted parameters. 

551 

552 pErrBootstrap : `list` [`float`] 

553 List with errors for fitted parameters. 

554 

555 reducedChiSqBootstrap : `float` 

556 Reduced chi squared, unweighted if weightsY is not provided. 

557 """ 

558 if weightsY is None: 

559 weightsY = np.ones(len(dataX)) 

560 

561 def errFunc(p, x, y, weightsY): 

562 if weightsY is None: 

563 weightsY = np.ones(len(x)) 

564 return (function(p, x) - y)*weightsY 

565 

566 # Fit first time 

567 pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY, weightsY), full_output=0) 

568 

569 # Get the stdev of the residuals 

570 residuals = errFunc(pFit, dataX, dataY, weightsY) 

571 # 100 random data sets are generated and fitted 

572 pars = [] 

573 for i in range(100): 

574 randomDelta = np.random.normal(0., np.fabs(residuals), len(dataY)) 

575 randomDataY = dataY + randomDelta 

576 randomFit, _ = leastsq(errFunc, initialParams, 

577 args=(dataX, randomDataY, weightsY), full_output=0) 

578 pars.append(randomFit) 

579 pars = np.array(pars) 

580 meanPfit = np.mean(pars, 0) 

581 

582 # confidence interval for parameter estimates 

583 errPfit = confidenceSigma*np.std(pars, 0) 

584 pFitBootstrap = meanPfit 

585 pErrBootstrap = errPfit 

586 

587 reducedChiSq = calculateWeightedReducedChi2(dataY, function(pFitBootstrap, dataX), weightsY, len(dataY), 

588 len(initialParams)) 

589 return pFitBootstrap, pErrBootstrap, reducedChiSq 

590 

591 

592def funcPolynomial(pars, x): 

593 """Polynomial function definition 

594 Parameters 

595 ---------- 

596 params : `list` 

597 Polynomial coefficients. Its length determines the polynomial order. 

598 

599 x : `numpy.array`, (N,) 

600 Abscisa array. 

601 

602 Returns 

603 ------- 

604 y : `numpy.array`, (N,) 

605 Ordinate array after evaluating polynomial of order 

606 len(pars)-1 at `x`. 

607 """ 

608 return poly.polyval(x, [*pars]) 

609 

610 

611def funcAstier(pars, x): 

612 """Single brighter-fatter parameter model for PTC; Equation 16 of 

613 Astier+19. 

614 

615 Parameters 

616 ---------- 

617 params : `list` 

618 Parameters of the model: a00 (brightter-fatter), gain (e/ADU), 

619 and noise (e^2). 

620 

621 x : `numpy.array`, (N,) 

622 Signal mu (ADU). 

623 

624 Returns 

625 ------- 

626 y : `numpy.array`, (N,) 

627 C_00 (variance) in ADU^2. 

628 """ 

629 a00, gain, noise = pars 

630 return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain) # C_00 

631 

632 

633def arrangeFlatsByExpTime(exposureList, exposureIdList): 

634 """Arrange exposures by exposure time. 

635 

636 Parameters 

637 ---------- 

638 exposureList : `list` [`lsst.pipe.base.connections.DeferredDatasetRef`] 

639 Input list of exposure references. 

640 

641 exposureIdList : `list` [`int`] 

642 List of exposure ids as obtained by dataId[`exposure`]. 

643 

644 Returns 

645 ------ 

646 flatsAtExpTime : `dict` [`float`, 

647 `list`[(`lsst.pipe.base.connections.DeferredDatasetRef`, 

648 `int`)]] 

649 Dictionary that groups references to flat-field exposures 

650 (and their IDs) that have the same exposure time (seconds). 

651 """ 

652 flatsAtExpTime = {} 

653 assert len(exposureList) == len(exposureIdList), "Different lengths for exp. list and exp. ID lists" 

654 for expRef, expId in zip(exposureList, exposureIdList): 

655 expTime = expRef.get(component='visitInfo').exposureTime 

656 listAtExpTime = flatsAtExpTime.setdefault(expTime, []) 

657 listAtExpTime.append((expRef, expId)) 

658 

659 return flatsAtExpTime 

660 

661 

662def arrangeFlatsByExpId(exposureList, exposureIdList): 

663 """Arrange exposures by exposure ID. 

664 

665 There is no guarantee that this will properly group exposures, but 

666 allows a sequence of flats that have different illumination 

667 (despite having the same exposure time) to be processed. 

668 

669 Parameters 

670 ---------- 

671 exposureList : `list`[`lsst.pipe.base.connections.DeferredDatasetRef`] 

672 Input list of exposure references. 

673 

674 exposureIdList : `list`[`int`] 

675 List of exposure ids as obtained by dataId[`exposure`]. 

676 

677 Returns 

678 ------ 

679 flatsAtExpId : `dict` [`float`, 

680 `list`[(`lsst.pipe.base.connections.DeferredDatasetRef`, 

681 `int`)]] 

682 Dictionary that groups references to flat-field exposures (and their 

683 IDs) sequentially by their exposure id. 

684 

685 Notes 

686 ----- 

687 

688 This algorithm sorts the input exposure references by their exposure 

689 id, and then assigns each pair of exposure references (exp_j, exp_{j+1}) 

690 to pair k, such that 2*k = j, where j is the python index of one of the 

691 exposure references (starting from zero). By checking for the IndexError 

692 while appending, we can ensure that there will only ever be fully 

693 populated pairs. 

694 """ 

695 flatsAtExpId = {} 

696 assert len(exposureList) == len(exposureIdList), "Different lengths for exp. list and exp. ID lists" 

697 # Sort exposures by expIds, which are in the second list `exposureIdList`. 

698 sortedExposures = sorted(zip(exposureList, exposureIdList), key=lambda pair: pair[1]) 

699 

700 for jPair, expTuple in enumerate(sortedExposures): 

701 if (jPair + 1) % 2: 

702 kPair = jPair // 2 

703 listAtExpId = flatsAtExpId.setdefault(kPair, []) 

704 try: 

705 listAtExpId.append(expTuple) 

706 listAtExpId.append(sortedExposures[jPair + 1]) 

707 except IndexError: 

708 pass 

709 

710 return flatsAtExpId 

711 

712 

713def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False): 

714 """Check the exposure lengths of two exposures are equal. 

715 

716 Parameters 

717 ---------- 

718 exp1 : `lsst.afw.image.Exposure` 

719 First exposure to check 

720 exp2 : `lsst.afw.image.Exposure` 

721 Second exposure to check 

722 v1 : `int` or `str`, optional 

723 First visit of the visit pair 

724 v2 : `int` or `str`, optional 

725 Second visit of the visit pair 

726 raiseWithMessage : `bool` 

727 If True, instead of returning a bool, raise a RuntimeError if 

728 exposure times are not equal, with a message about which 

729 visits mismatch if the information is available. 

730 

731 Returns 

732 ------- 

733 success : `bool` 

734 This is true if the exposures have equal exposure times. 

735 

736 Raises 

737 ------ 

738 RuntimeError 

739 Raised if the exposure lengths of the two exposures are not equal 

740 """ 

741 expTime1 = exp1.getInfo().getVisitInfo().getExposureTime() 

742 expTime2 = exp2.getInfo().getVisitInfo().getExposureTime() 

743 if expTime1 != expTime2: 

744 if raiseWithMessage: 

745 msg = "Exposure lengths for visit pairs must be equal. " + \ 

746 "Found %s and %s" % (expTime1, expTime2) 

747 if v1 and v2: 

748 msg += " for visit pair %s, %s" % (v1, v2) 

749 raise RuntimeError(msg) 

750 else: 

751 return False 

752 return True 

753 

754 

755def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None, 

756 checkTrim=True, logName=None): 

757 """Check that appropriate ISR settings have been selected for the task. 

758 

759 Note that this checks that the task itself is configured correctly rather 

760 than checking a config. 

761 

762 Parameters 

763 ---------- 

764 isrTask : `lsst.ip.isr.IsrTask` 

765 The task whose config is to be validated 

766 

767 mandatory : `iterable` [`str`] 

768 isr steps that must be set to True. Raises if False or missing 

769 

770 forbidden : `iterable` [`str`] 

771 isr steps that must be set to False. Raises if True, warns if missing 

772 

773 desirable : `iterable` [`str`] 

774 isr steps that should probably be set to True. Warns is False, 

775 info if missing 

776 

777 undesirable : `iterable` [`str`] 

778 isr steps that should probably be set to False. Warns is True, 

779 info if missing 

780 

781 checkTrim : `bool` 

782 Check to ensure the isrTask's assembly subtask is trimming the 

783 images. This is a separate config as it is very ugly to do 

784 this within the normal configuration lists as it is an option 

785 of a sub task. 

786 

787 Raises 

788 ------ 

789 RuntimeError 

790 Raised if ``mandatory`` config parameters are False, 

791 or if ``forbidden`` parameters are True. 

792 

793 TypeError 

794 Raised if parameter ``isrTask`` is an invalid type. 

795 

796 Notes 

797 ----- 

798 Logs warnings using an isrValidation logger for desirable/undesirable 

799 options that are of the wrong polarity or if keys are missing. 

800 """ 

801 if not isinstance(isrTask, ipIsr.IsrTask): 

802 raise TypeError(f'Must supply an instance of lsst.ip.isr.IsrTask not {type(isrTask)}') 

803 

804 configDict = isrTask.config.toDict() 

805 

806 if logName and isinstance(logName, str): 

807 log = logging.getLogger(logName) 

808 else: 

809 log = logging.getLogger("lsst.isrValidation") 

810 

811 if mandatory: 

812 for configParam in mandatory: 

813 if configParam not in configDict: 

814 raise RuntimeError(f"Mandatory parameter {configParam} not found in the isr configuration.") 

815 if configDict[configParam] is False: 

816 raise RuntimeError(f"Must set config.isr.{configParam} to True for this task.") 

817 

818 if forbidden: 

819 for configParam in forbidden: 

820 if configParam not in configDict: 

821 log.warning("Failed to find forbidden key %s in the isr config. The keys in the" 

822 " forbidden list should each have an associated Field in IsrConfig:" 

823 " check that there is not a typo in this case.", configParam) 

824 continue 

825 if configDict[configParam] is True: 

826 raise RuntimeError(f"Must set config.isr.{configParam} to False for this task.") 

827 

828 if desirable: 

829 for configParam in desirable: 

830 if configParam not in configDict: 

831 log.info("Failed to find key %s in the isr config. You probably want" 

832 " to set the equivalent for your obs_package to True.", configParam) 

833 continue 

834 if configDict[configParam] is False: 

835 log.warning("Found config.isr.%s set to False for this task." 

836 " The cp_pipe Config recommends setting this to True.", configParam) 

837 if undesirable: 

838 for configParam in undesirable: 

839 if configParam not in configDict: 

840 log.info("Failed to find key %s in the isr config. You probably want" 

841 " to set the equivalent for your obs_package to False.", configParam) 

842 continue 

843 if configDict[configParam] is True: 

844 log.warning("Found config.isr.%s set to True for this task." 

845 " The cp_pipe Config recommends setting this to False.", configParam) 

846 

847 if checkTrim: # subtask setting, seems non-trivial to combine with above lists 

848 if not isrTask.assembleCcd.config.doTrim: 

849 raise RuntimeError("Must trim when assembling CCDs. Set config.isr.assembleCcd.doTrim to True") 

850 

851 

852def ddict2dict(d): 

853 """Convert nested default dictionaries to regular dictionaries. 

854 

855 This is needed to prevent yaml persistence issues. 

856 

857 Parameters 

858 ---------- 

859 d : `defaultdict` 

860 A possibly nested set of `defaultdict`. 

861 

862 Returns 

863 ------- 

864 dict : `dict` 

865 A possibly nested set of `dict`. 

866 """ 

867 for k, v in d.items(): 

868 if isinstance(v, dict): 

869 d[k] = ddict2dict(v) 

870 return dict(d)