Coverage for python/lsst/cp/pipe/utils.py: 10%

252 statements  

« prev     ^ index     » next       coverage.py v7.2.1, created at 2023-03-12 11:11 +0000

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22 

23__all__ = ['PairedVisitListTaskRunner', 'SingleVisitListTaskRunner', 

24 'NonexistentDatasetTaskDataIdContainer', 'parseCmdlineNumberString', 

25 'countMaskedPixels', 'checkExpLengthEqual', 'ddict2dict'] 

26 

27import re 

28import numpy as np 

29from scipy.optimize import leastsq 

30import numpy.polynomial.polynomial as poly 

31from scipy.stats import norm 

32 

33import lsst.pipe.base as pipeBase 

34import lsst.ip.isr as ipIsr 

35from lsst.ip.isr import isrMock 

36import lsst.log 

37import lsst.afw.image 

38 

39import galsim 

40 

41 

42def sigmaClipCorrection(nSigClip): 

43 """Correct measured sigma to account for clipping. 

44 

45 If we clip our input data and then measure sigma, then the 

46 measured sigma is smaller than the true value because real 

47 points beyond the clip threshold have been removed. This is a 

48 small (1.5% at nSigClip=3) effect when nSigClip >~ 3, but the 

49 default parameters for measure crosstalk use nSigClip=2.0. 

50 This causes the measured sigma to be about 15% smaller than 

51 real. This formula corrects the issue, for the symmetric case 

52 (upper clip threshold equal to lower clip threshold). 

53 

54 Parameters 

55 ---------- 

56 nSigClip : `float` 

57 Number of sigma the measurement was clipped by. 

58 

59 Returns 

60 ------- 

61 scaleFactor : `float` 

62 Scale factor to increase the measured sigma by. 

63 """ 

64 varFactor = 1.0 - (2 * nSigClip * norm.pdf(nSigClip)) / (norm.cdf(nSigClip) - norm.cdf(-nSigClip)) 

65 return 1.0 / np.sqrt(varFactor) 

66 

67 

68def calculateWeightedReducedChi2(measured, model, weightsMeasured, nData, nParsModel): 

69 """Calculate weighted reduced chi2. 

70 

71 Parameters 

72 ---------- 

73 

74 measured : `list` 

75 List with measured data. 

76 

77 model : `list` 

78 List with modeled data. 

79 

80 weightsMeasured : `list` 

81 List with weights for the measured data. 

82 

83 nData : `int` 

84 Number of data points. 

85 

86 nParsModel : `int` 

87 Number of parameters in the model. 

88 

89 Returns 

90 ------- 

91 

92 redWeightedChi2 : `float` 

93 Reduced weighted chi2. 

94 """ 

95 wRes = (measured - model)*weightsMeasured 

96 return ((wRes*wRes).sum())/(nData-nParsModel) 

97 

98 

99def makeMockFlats(expTime, gain=1.0, readNoiseElectrons=5, fluxElectrons=1000, 

100 randomSeedFlat1=1984, randomSeedFlat2=666, powerLawBfParams=[], 

101 expId1=0, expId2=1): 

102 """Create a pair or mock flats with isrMock. 

103 

104 Parameters 

105 ---------- 

106 expTime : `float` 

107 Exposure time of the flats. 

108 

109 gain : `float`, optional 

110 Gain, in e/ADU. 

111 

112 readNoiseElectrons : `float`, optional 

113 Read noise rms, in electrons. 

114 

115 fluxElectrons : `float`, optional 

116 Flux of flats, in electrons per second. 

117 

118 randomSeedFlat1 : `int`, optional 

119 Random seed for the normal distrubutions for the mean signal 

120 and noise (flat1). 

121 

122 randomSeedFlat2 : `int`, optional 

123 Random seed for the normal distrubutions for the mean signal 

124 and noise (flat2). 

125 

126 powerLawBfParams : `list`, optional 

127 Parameters for `galsim.cdmodel.PowerLawCD` to simulate the 

128 brightter-fatter effect. 

129 

130 expId1 : `int`, optional 

131 Exposure ID for first flat. 

132 

133 expId2 : `int`, optional 

134 Exposure ID for second flat. 

135 

136 Returns 

137 ------- 

138 

139 flatExp1 : `lsst.afw.image.exposure.ExposureF` 

140 First exposure of flat field pair. 

141 

142 flatExp2 : `lsst.afw.image.exposure.ExposureF` 

143 Second exposure of flat field pair. 

144 

145 Notes 

146 ----- 

147 The parameters of `galsim.cdmodel.PowerLawCD` are `n, r0, t0, rx, 

148 tx, r, t, alpha`. For more information about their meaning, see 

149 the Galsim documentation 

150 https://galsim-developers.github.io/GalSim/_build/html/_modules/galsim/cdmodel.html # noqa: W505 

151 and Gruen+15 (1501.02802). 

152 

153 Example: galsim.cdmodel.PowerLawCD(8, 1.1e-7, 1.1e-7, 1.0e-8, 

154 1.0e-8, 1.0e-9, 1.0e-9, 2.0) 

155 """ 

156 flatFlux = fluxElectrons # e/s 

157 flatMean = flatFlux*expTime # e 

158 readNoise = readNoiseElectrons # e 

159 

160 mockImageConfig = isrMock.IsrMock.ConfigClass() 

161 

162 mockImageConfig.flatDrop = 0.99999 

163 mockImageConfig.isTrimmed = True 

164 

165 flatExp1 = isrMock.FlatMock(config=mockImageConfig).run() 

166 flatExp2 = flatExp1.clone() 

167 (shapeY, shapeX) = flatExp1.getDimensions() 

168 flatWidth = np.sqrt(flatMean) 

169 

170 rng1 = np.random.RandomState(randomSeedFlat1) 

171 flatData1 = rng1.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng1.normal(0.0, readNoise, 

172 (shapeX, shapeY)) 

173 rng2 = np.random.RandomState(randomSeedFlat2) 

174 flatData2 = rng2.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng2.normal(0.0, readNoise, 

175 (shapeX, shapeY)) 

176 # Simulate BF with power law model in galsim 

177 if len(powerLawBfParams): 

178 if not len(powerLawBfParams) == 8: 

179 raise RuntimeError("Wrong number of parameters for `galsim.cdmodel.PowerLawCD`. " 

180 f"Expected 8; passed {len(powerLawBfParams)}.") 

181 cd = galsim.cdmodel.PowerLawCD(*powerLawBfParams) 

182 tempFlatData1 = galsim.Image(flatData1) 

183 temp2FlatData1 = cd.applyForward(tempFlatData1) 

184 

185 tempFlatData2 = galsim.Image(flatData2) 

186 temp2FlatData2 = cd.applyForward(tempFlatData2) 

187 

188 flatExp1.image.array[:] = temp2FlatData1.array/gain # ADU 

189 flatExp2.image.array[:] = temp2FlatData2.array/gain # ADU 

190 else: 

191 flatExp1.image.array[:] = flatData1/gain # ADU 

192 flatExp2.image.array[:] = flatData2/gain # ADU 

193 

194 visitInfoExp1 = lsst.afw.image.VisitInfo(exposureId=expId1, exposureTime=expTime) 

195 visitInfoExp2 = lsst.afw.image.VisitInfo(exposureId=expId2, exposureTime=expTime) 

196 

197 flatExp1.getInfo().setVisitInfo(visitInfoExp1) 

198 flatExp2.getInfo().setVisitInfo(visitInfoExp2) 

199 

200 return flatExp1, flatExp2 

201 

202 

203def countMaskedPixels(maskedIm, maskPlane): 

204 """Count the number of pixels in a given mask plane. 

205 

206 Parameters 

207 ---------- 

208 maskedIm : `~lsst.afw.image.MaskedImage` 

209 Masked image to examine. 

210 maskPlane : `str` 

211 Name of the mask plane to examine. 

212 

213 Returns 

214 ------- 

215 nPix : `int` 

216 Number of pixels in the requested mask plane. 

217 """ 

218 maskBit = maskedIm.mask.getPlaneBitMask(maskPlane) 

219 nPix = np.where(np.bitwise_and(maskedIm.mask.array, maskBit))[0].flatten().size 

220 return nPix 

221 

222 

223class PairedVisitListTaskRunner(pipeBase.TaskRunner): 

224 """Subclass of TaskRunner for handling intrinsically paired visits. 

225 

226 This transforms the processed arguments generated by the ArgumentParser 

227 into the arguments expected by tasks which take visit pairs for their 

228 run() methods. 

229 

230 Such tasks' run() methods tend to take two arguments, 

231 one of which is the dataRef (as usual), and the other is the list 

232 of visit-pairs, in the form of a list of tuples. 

233 This list is supplied on the command line as documented, 

234 and this class parses that, and passes the parsed version 

235 to the run() method. 

236 

237 See pipeBase.TaskRunner for more information. 

238 """ 

239 

240 @staticmethod 

241 def getTargetList(parsedCmd, **kwargs): 

242 """Parse the visit list and pass through explicitly.""" 

243 visitPairs = [] 

244 for visitStringPair in parsedCmd.visitPairs: 

245 visitStrings = visitStringPair.split(",") 

246 if len(visitStrings) != 2: 

247 raise RuntimeError("Found {} visits in {} instead of 2".format(len(visitStrings), 

248 visitStringPair)) 

249 try: 

250 visits = [int(visit) for visit in visitStrings] 

251 except Exception: 

252 raise RuntimeError("Could not parse {} as two integer visit numbers".format(visitStringPair)) 

253 visitPairs.append(visits) 

254 

255 return pipeBase.TaskRunner.getTargetList(parsedCmd, visitPairs=visitPairs, **kwargs) 

256 

257 

258def parseCmdlineNumberString(inputString): 

259 """Parse command line numerical expression sytax and return as list of int 

260 

261 Take an input of the form "'1..5:2^123..126'" as a string, and return 

262 a list of ints as [1, 3, 5, 123, 124, 125, 126] 

263 

264 Parameters 

265 ---------- 

266 inputString : `str` 

267 String to be parsed. 

268 

269 Returns 

270 ------- 

271 outList : `list` [`int`] 

272 List of integers identified in the string. 

273 """ 

274 outList = [] 

275 for subString in inputString.split("^"): 

276 mat = re.search(r"^(\d+)\.\.(\d+)(?::(\d+))?$", subString) 

277 if mat: 

278 v1 = int(mat.group(1)) 

279 v2 = int(mat.group(2)) 

280 v3 = mat.group(3) 

281 v3 = int(v3) if v3 else 1 

282 for v in range(v1, v2 + 1, v3): 

283 outList.append(int(v)) 

284 else: 

285 outList.append(int(subString)) 

286 return outList 

287 

288 

289class SingleVisitListTaskRunner(pipeBase.TaskRunner): 

290 """Subclass of TaskRunner for tasks requiring a list of visits per dataRef. 

291 

292 This transforms the processed arguments generated by the ArgumentParser 

293 into the arguments expected by tasks which require a list of visits 

294 to be supplied for each dataRef, as is common in `lsst.cp.pipe` code. 

295 

296 Such tasks' run() methods tend to take two arguments, 

297 one of which is the dataRef (as usual), and the other is the list 

298 of visits. 

299 This list is supplied on the command line as documented, 

300 and this class parses that, and passes the parsed version 

301 to the run() method. 

302 

303 See `lsst.pipe.base.TaskRunner` for more information. 

304 """ 

305 

306 @staticmethod 

307 def getTargetList(parsedCmd, **kwargs): 

308 """Parse the visit list and pass through explicitly.""" 

309 # if this has been pre-parsed and therefore doesn't have length of one 

310 # then something has gone wrong, so execution should stop here. 

311 assert len(parsedCmd.visitList) == 1, 'visitList parsing assumptions violated' 

312 visits = parseCmdlineNumberString(parsedCmd.visitList[0]) 

313 

314 return pipeBase.TaskRunner.getTargetList(parsedCmd, visitList=visits, **kwargs) 

315 

316 

317class NonexistentDatasetTaskDataIdContainer(pipeBase.DataIdContainer): 

318 """A DataIdContainer for the tasks for which the output does 

319 not yet exist.""" 

320 

321 def makeDataRefList(self, namespace): 

322 """Compute refList based on idList. 

323 

324 This method must be defined as the dataset does not exist before this 

325 task is run. 

326 

327 Parameters 

328 ---------- 

329 namespace 

330 Results of parsing the command-line. 

331 

332 Notes 

333 ----- 

334 Not called if ``add_id_argument`` called 

335 with ``doMakeDataRefList=False``. 

336 Note that this is almost a copy-and-paste of the vanilla 

337 implementation, but without checking if the datasets already exist, 

338 as this task exists to make them. 

339 """ 

340 if self.datasetType is None: 

341 raise RuntimeError("Must call setDatasetType first") 

342 butler = namespace.butler 

343 for dataId in self.idList: 

344 refList = list(butler.subset(datasetType=self.datasetType, level=self.level, dataId=dataId)) 

345 # exclude nonexistent data 

346 # this is a recursive test, e.g. for the sake of "raw" data 

347 if not refList: 

348 namespace.log.warn("No data found for dataId=%s", dataId) 

349 continue 

350 self.refList += refList 

351 

352 

353def irlsFit(initialParams, dataX, dataY, function, weightsY=None, weightType='Cauchy'): 

354 """Iteratively reweighted least squares fit. 

355 

356 This uses the `lsst.cp.pipe.utils.fitLeastSq`, but applies weights 

357 based on the Cauchy distribution by default. Other weight options 

358 are implemented. See e.g. Holland and Welsch, 1977, 

359 doi:10.1080/03610927708827533 

360 

361 Parameters 

362 ---------- 

363 initialParams : `list` [`float`] 

364 Starting parameters. 

365 dataX : `numpy.array`, (N,) 

366 Abscissa data. 

367 dataY : `numpy.array`, (N,) 

368 Ordinate data. 

369 function : callable 

370 Function to fit. 

371 weightsY : `numpy.array`, (N,) 

372 Weights to apply to the data. 

373 weightType : `str`, optional 

374 Type of weighting to use. One of Cauchy, Anderson, bisquare, 

375 box, Welsch, Huber, logistic, or Fair. 

376 

377 Returns 

378 ------- 

379 polyFit : `list` [`float`] 

380 Final best fit parameters. 

381 polyFitErr : `list` [`float`] 

382 Final errors on fit parameters. 

383 chiSq : `float` 

384 Reduced chi squared. 

385 weightsY : `list` [`float`] 

386 Final weights used for each point. 

387 

388 Raises 

389 ------ 

390 RuntimeError : 

391 Raised if an unknown weightType string is passed. 

392 """ 

393 if not weightsY: 

394 weightsY = np.ones_like(dataX) 

395 

396 polyFit, polyFitErr, chiSq = fitLeastSq(initialParams, dataX, dataY, function, weightsY=weightsY) 

397 for iteration in range(10): 

398 resid = np.abs(dataY - function(polyFit, dataX)) / np.sqrt(dataY) 

399 if weightType == 'Cauchy': 

400 # Use Cauchy weighting. This is a soft weight. 

401 # At [2, 3, 5, 10] sigma, weights are [.59, .39, .19, .05]. 

402 Z = resid / 2.385 

403 weightsY = 1.0 / (1.0 + np.square(Z)) 

404 elif weightType == 'Anderson': 

405 # Anderson+1972 weighting. This is a hard weight. 

406 # At [2, 3, 5, 10] sigma, weights are [.67, .35, 0.0, 0.0]. 

407 Z = resid / (1.339 * np.pi) 

408 weightsY = np.where(Z < 1.0, np.sinc(Z), 0.0) 

409 elif weightType == 'bisquare': 

410 # Beaton and Tukey (1974) biweight. This is a hard weight. 

411 # At [2, 3, 5, 10] sigma, weights are [.81, .59, 0.0, 0.0]. 

412 Z = resid / 4.685 

413 weightsY = np.where(Z < 1.0, 1.0 - np.square(Z), 0.0) 

414 elif weightType == 'box': 

415 # Hinich and Talwar (1975). This is a hard weight. 

416 # At [2, 3, 5, 10] sigma, weights are [1.0, 0.0, 0.0, 0.0]. 

417 weightsY = np.where(resid < 2.795, 1.0, 0.0) 

418 elif weightType == 'Welsch': 

419 # Dennis and Welsch (1976). This is a hard weight. 

420 # At [2, 3, 5, 10] sigma, weights are [.64, .36, .06, 1e-5]. 

421 Z = resid / 2.985 

422 weightsY = np.exp(-1.0 * np.square(Z)) 

423 elif weightType == 'Huber': 

424 # Huber (1964) weighting. This is a soft weight. 

425 # At [2, 3, 5, 10] sigma, weights are [.67, .45, .27, .13]. 

426 Z = resid / 1.345 

427 weightsY = np.where(Z < 1.0, 1.0, 1 / Z) 

428 elif weightType == 'logistic': 

429 # Logistic weighting. This is a soft weight. 

430 # At [2, 3, 5, 10] sigma, weights are [.56, .40, .24, .12]. 

431 Z = resid / 1.205 

432 weightsY = np.tanh(Z) / Z 

433 elif weightType == 'Fair': 

434 # Fair (1974) weighting. This is a soft weight. 

435 # At [2, 3, 5, 10] sigma, weights are [.41, .32, .22, .12]. 

436 Z = resid / 1.4 

437 weightsY = (1.0 / (1.0 + (Z))) 

438 else: 

439 raise RuntimeError(f"Unknown weighting type: {weightType}") 

440 polyFit, polyFitErr, chiSq = fitLeastSq(initialParams, dataX, dataY, function, weightsY=weightsY) 

441 

442 return polyFit, polyFitErr, chiSq, weightsY 

443 

444 

445def fitLeastSq(initialParams, dataX, dataY, function, weightsY=None): 

446 """Do a fit and estimate the parameter errors using using 

447 scipy.optimize.leastq. 

448 

449 optimize.leastsq returns the fractional covariance matrix. To 

450 estimate the standard deviation of the fit parameters, multiply 

451 the entries of this matrix by the unweighted reduced chi squared 

452 and take the square root of the diagonal elements. 

453 

454 Parameters 

455 ---------- 

456 initialParams : `list` [`float`] 

457 initial values for fit parameters. For ptcFitType=POLYNOMIAL, 

458 its length determines the degree of the polynomial. 

459 

460 dataX : `numpy.array`, (N,) 

461 Data in the abscissa axis. 

462 

463 dataY : `numpy.array`, (N,) 

464 Data in the ordinate axis. 

465 

466 function : callable object (function) 

467 Function to fit the data with. 

468 

469 weightsY : `numpy.array`, (N,) 

470 Weights of the data in the ordinate axis. 

471 

472 Return 

473 ------ 

474 pFitSingleLeastSquares : `list` [`float`] 

475 List with fitted parameters. 

476 

477 pErrSingleLeastSquares : `list` [`float`] 

478 List with errors for fitted parameters. 

479 

480 reducedChiSqSingleLeastSquares : `float` 

481 Reduced chi squared, unweighted if weightsY is not provided. 

482 """ 

483 if weightsY is None: 

484 weightsY = np.ones(len(dataX)) 

485 

486 def errFunc(p, x, y, weightsY=None): 

487 if weightsY is None: 

488 weightsY = np.ones(len(x)) 

489 return (function(p, x) - y)*weightsY 

490 

491 pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams, 

492 args=(dataX, dataY, weightsY), full_output=1, 

493 epsfcn=0.0001) 

494 

495 if (len(dataY) > len(initialParams)) and pCov is not None: 

496 reducedChiSq = calculateWeightedReducedChi2(dataY, function(pFit, dataX), weightsY, len(dataY), 

497 len(initialParams)) 

498 pCov *= reducedChiSq 

499 else: 

500 pCov = np.zeros((len(initialParams), len(initialParams))) 

501 pCov[:, :] = np.nan 

502 reducedChiSq = np.nan 

503 

504 errorVec = [] 

505 for i in range(len(pFit)): 

506 errorVec.append(np.fabs(pCov[i][i])**0.5) 

507 

508 pFitSingleLeastSquares = pFit 

509 pErrSingleLeastSquares = np.array(errorVec) 

510 

511 return pFitSingleLeastSquares, pErrSingleLeastSquares, reducedChiSq 

512 

513 

514def fitBootstrap(initialParams, dataX, dataY, function, weightsY=None, confidenceSigma=1.): 

515 """Do a fit using least squares and bootstrap to estimate parameter errors. 

516 

517 The bootstrap error bars are calculated by fitting 100 random data sets. 

518 

519 Parameters 

520 ---------- 

521 initialParams : `list` [`float`] 

522 initial values for fit parameters. For ptcFitType=POLYNOMIAL, 

523 its length determines the degree of the polynomial. 

524 

525 dataX : `numpy.array`, (N,) 

526 Data in the abscissa axis. 

527 

528 dataY : `numpy.array`, (N,) 

529 Data in the ordinate axis. 

530 

531 function : callable object (function) 

532 Function to fit the data with. 

533 

534 weightsY : `numpy.array`, (N,), optional. 

535 Weights of the data in the ordinate axis. 

536 

537 confidenceSigma : `float`, optional. 

538 Number of sigmas that determine confidence interval for the 

539 bootstrap errors. 

540 

541 Return 

542 ------ 

543 pFitBootstrap : `list` [`float`] 

544 List with fitted parameters. 

545 

546 pErrBootstrap : `list` [`float`] 

547 List with errors for fitted parameters. 

548 

549 reducedChiSqBootstrap : `float` 

550 Reduced chi squared, unweighted if weightsY is not provided. 

551 """ 

552 if weightsY is None: 

553 weightsY = np.ones(len(dataX)) 

554 

555 def errFunc(p, x, y, weightsY): 

556 if weightsY is None: 

557 weightsY = np.ones(len(x)) 

558 return (function(p, x) - y)*weightsY 

559 

560 # Fit first time 

561 pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY, weightsY), full_output=0) 

562 

563 # Get the stdev of the residuals 

564 residuals = errFunc(pFit, dataX, dataY, weightsY) 

565 # 100 random data sets are generated and fitted 

566 pars = [] 

567 for i in range(100): 

568 randomDelta = np.random.normal(0., np.fabs(residuals), len(dataY)) 

569 randomDataY = dataY + randomDelta 

570 randomFit, _ = leastsq(errFunc, initialParams, 

571 args=(dataX, randomDataY, weightsY), full_output=0) 

572 pars.append(randomFit) 

573 pars = np.array(pars) 

574 meanPfit = np.mean(pars, 0) 

575 

576 # confidence interval for parameter estimates 

577 errPfit = confidenceSigma*np.std(pars, 0) 

578 pFitBootstrap = meanPfit 

579 pErrBootstrap = errPfit 

580 

581 reducedChiSq = calculateWeightedReducedChi2(dataY, function(pFitBootstrap, dataX), weightsY, len(dataY), 

582 len(initialParams)) 

583 return pFitBootstrap, pErrBootstrap, reducedChiSq 

584 

585 

586def funcPolynomial(pars, x): 

587 """Polynomial function definition 

588 Parameters 

589 ---------- 

590 params : `list` 

591 Polynomial coefficients. Its length determines the polynomial order. 

592 

593 x : `numpy.array`, (N,) 

594 Abscisa array. 

595 

596 Returns 

597 ------- 

598 y : `numpy.array`, (N,) 

599 Ordinate array after evaluating polynomial of order 

600 len(pars)-1 at `x`. 

601 """ 

602 return poly.polyval(x, [*pars]) 

603 

604 

605def funcAstier(pars, x): 

606 """Single brighter-fatter parameter model for PTC; Equation 16 of 

607 Astier+19. 

608 

609 Parameters 

610 ---------- 

611 params : `list` 

612 Parameters of the model: a00 (brightter-fatter), gain (e/ADU), 

613 and noise (e^2). 

614 

615 x : `numpy.array`, (N,) 

616 Signal mu (ADU). 

617 

618 Returns 

619 ------- 

620 y : `numpy.array`, (N,) 

621 C_00 (variance) in ADU^2. 

622 """ 

623 a00, gain, noise = pars 

624 return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain) # C_00 

625 

626 

627def arrangeFlatsByExpTime(exposureList, exposureIdList): 

628 """Arrange exposures by exposure time. 

629 

630 Parameters 

631 ---------- 

632 exposureList : `list` [`lsst.afw.image.ExposureF`] 

633 Input list of exposures. 

634 

635 exposureIdList : `list` [`int`] 

636 List of exposure ids as obtained by dataId[`exposure`]. 

637 

638 Returns 

639 ------ 

640 flatsAtExpTime : `dict` [`float`, 

641 `list`[(`lsst.afw.image.ExposureF`, `int`)]] 

642 Dictionary that groups flat-field exposures (and their IDs) that have 

643 the same exposure time (seconds). 

644 """ 

645 flatsAtExpTime = {} 

646 assert len(exposureList) == len(exposureIdList), "Different lengths for exp. list and exp. ID lists" 

647 for exp, expId in zip(exposureList, exposureIdList): 

648 expTime = exp.getInfo().getVisitInfo().getExposureTime() 

649 listAtExpTime = flatsAtExpTime.setdefault(expTime, []) 

650 listAtExpTime.append((exp, expId)) 

651 

652 return flatsAtExpTime 

653 

654 

655def arrangeFlatsByExpId(exposureList, exposureIdList): 

656 """Arrange exposures by exposure ID. 

657 

658 There is no guarantee that this will properly group exposures, but 

659 allows a sequence of flats that have different illumination 

660 (despite having the same exposure time) to be processed. 

661 

662 Parameters 

663 ---------- 

664 exposureList : `list`[`lsst.afw.image.ExposureF`] 

665 Input list of exposures. 

666 

667 exposureIdList : `list`[`int`] 

668 List of exposure ids as obtained by dataId[`exposure`]. 

669 

670 Returns 

671 ------ 

672 flatsAtExpId : `dict` [`float`, 

673 `list`[(`lsst.afw.image.ExposureF`, `int`)]] 

674 Dictionary that groups flat-field exposures (and their IDs) 

675 sequentially by their exposure id. 

676 

677 Notes 

678 ----- 

679 

680 This algorithm sorts the input exposures by their exposure id, and 

681 then assigns each pair of exposures (exp_j, exp_{j+1}) to pair k, 

682 such that 2*k = j, where j is the python index of one of the 

683 exposures (starting from zero). By checking for the IndexError 

684 while appending, we can ensure that there will only ever be fully 

685 populated pairs. 

686 """ 

687 flatsAtExpId = {} 

688 # sortedExposures = sorted(exposureList, key=lambda exp: 

689 # exp.getInfo().getVisitInfo().getExposureId()) 

690 assert len(exposureList) == len(exposureIdList), "Different lengths for exp. list and exp. ID lists" 

691 # Sort exposures by expIds, which are in the second list `exposureIdList`. 

692 sortedExposures = sorted(zip(exposureList, exposureIdList), key=lambda pair: pair[1]) 

693 

694 for jPair, expTuple in enumerate(sortedExposures): 

695 if (jPair + 1) % 2: 

696 kPair = jPair // 2 

697 listAtExpId = flatsAtExpId.setdefault(kPair, []) 

698 try: 

699 listAtExpId.append(expTuple) 

700 listAtExpId.append(sortedExposures[jPair + 1]) 

701 except IndexError: 

702 pass 

703 

704 return flatsAtExpId 

705 

706 

707def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False): 

708 """Check the exposure lengths of two exposures are equal. 

709 

710 Parameters 

711 ---------- 

712 exp1 : `lsst.afw.image.Exposure` 

713 First exposure to check 

714 exp2 : `lsst.afw.image.Exposure` 

715 Second exposure to check 

716 v1 : `int` or `str`, optional 

717 First visit of the visit pair 

718 v2 : `int` or `str`, optional 

719 Second visit of the visit pair 

720 raiseWithMessage : `bool` 

721 If True, instead of returning a bool, raise a RuntimeError if 

722 exposure times are not equal, with a message about which 

723 visits mismatch if the information is available. 

724 

725 Returns 

726 ------- 

727 success : `bool` 

728 This is true if the exposures have equal exposure times. 

729 

730 Raises 

731 ------ 

732 RuntimeError 

733 Raised if the exposure lengths of the two exposures are not equal 

734 """ 

735 expTime1 = exp1.getInfo().getVisitInfo().getExposureTime() 

736 expTime2 = exp2.getInfo().getVisitInfo().getExposureTime() 

737 if expTime1 != expTime2: 

738 if raiseWithMessage: 

739 msg = "Exposure lengths for visit pairs must be equal. " + \ 

740 "Found %s and %s" % (expTime1, expTime2) 

741 if v1 and v2: 

742 msg += " for visit pair %s, %s" % (v1, v2) 

743 raise RuntimeError(msg) 

744 else: 

745 return False 

746 return True 

747 

748 

749def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None, 

750 checkTrim=True, logName=None): 

751 """Check that appropriate ISR settings have been selected for the task. 

752 

753 Note that this checks that the task itself is configured correctly rather 

754 than checking a config. 

755 

756 Parameters 

757 ---------- 

758 isrTask : `lsst.ip.isr.IsrTask` 

759 The task whose config is to be validated 

760 

761 mandatory : `iterable` [`str`] 

762 isr steps that must be set to True. Raises if False or missing 

763 

764 forbidden : `iterable` [`str`] 

765 isr steps that must be set to False. Raises if True, warns if missing 

766 

767 desirable : `iterable` [`str`] 

768 isr steps that should probably be set to True. Warns is False, 

769 info if missing 

770 

771 undesirable : `iterable` [`str`] 

772 isr steps that should probably be set to False. Warns is True, 

773 info if missing 

774 

775 checkTrim : `bool` 

776 Check to ensure the isrTask's assembly subtask is trimming the 

777 images. This is a separate config as it is very ugly to do 

778 this within the normal configuration lists as it is an option 

779 of a sub task. 

780 

781 Raises 

782 ------ 

783 RuntimeError 

784 Raised if ``mandatory`` config parameters are False, 

785 or if ``forbidden`` parameters are True. 

786 

787 TypeError 

788 Raised if parameter ``isrTask`` is an invalid type. 

789 

790 Notes 

791 ----- 

792 Logs warnings using an isrValidation logger for desirable/undesirable 

793 options that are of the wrong polarity or if keys are missing. 

794 """ 

795 if not isinstance(isrTask, ipIsr.IsrTask): 

796 raise TypeError(f'Must supply an instance of lsst.ip.isr.IsrTask not {type(isrTask)}') 

797 

798 configDict = isrTask.config.toDict() 

799 

800 if logName and isinstance(logName, str): 

801 log = lsst.log.getLogger(logName) 

802 else: 

803 log = lsst.log.getLogger("isrValidation") 

804 

805 if mandatory: 

806 for configParam in mandatory: 

807 if configParam not in configDict: 

808 raise RuntimeError(f"Mandatory parameter {configParam} not found in the isr configuration.") 

809 if configDict[configParam] is False: 

810 raise RuntimeError(f"Must set config.isr.{configParam} to True for this task.") 

811 

812 if forbidden: 

813 for configParam in forbidden: 

814 if configParam not in configDict: 

815 log.warn(f"Failed to find forbidden key {configParam} in the isr config. The keys in the" 

816 " forbidden list should each have an associated Field in IsrConfig:" 

817 " check that there is not a typo in this case.") 

818 continue 

819 if configDict[configParam] is True: 

820 raise RuntimeError(f"Must set config.isr.{configParam} to False for this task.") 

821 

822 if desirable: 

823 for configParam in desirable: 

824 if configParam not in configDict: 

825 log.info(f"Failed to find key {configParam} in the isr config. You probably want" 

826 " to set the equivalent for your obs_package to True.") 

827 continue 

828 if configDict[configParam] is False: 

829 log.warn(f"Found config.isr.{configParam} set to False for this task." 

830 " The cp_pipe Config recommends setting this to True.") 

831 if undesirable: 

832 for configParam in undesirable: 

833 if configParam not in configDict: 

834 log.info(f"Failed to find key {configParam} in the isr config. You probably want" 

835 " to set the equivalent for your obs_package to False.") 

836 continue 

837 if configDict[configParam] is True: 

838 log.warn(f"Found config.isr.{configParam} set to True for this task." 

839 " The cp_pipe Config recommends setting this to False.") 

840 

841 if checkTrim: # subtask setting, seems non-trivial to combine with above lists 

842 if not isrTask.assembleCcd.config.doTrim: 

843 raise RuntimeError("Must trim when assembling CCDs. Set config.isr.assembleCcd.doTrim to True") 

844 

845 

846def ddict2dict(d): 

847 """Convert nested default dictionaries to regular dictionaries. 

848 

849 This is needed to prevent yaml persistence issues. 

850 

851 Parameters 

852 ---------- 

853 d : `defaultdict` 

854 A possibly nested set of `defaultdict`. 

855 

856 Returns 

857 ------- 

858 dict : `dict` 

859 A possibly nested set of `dict`. 

860 """ 

861 for k, v in d.items(): 

862 if isinstance(v, dict): 

863 d[k] = ddict2dict(v) 

864 return dict(d)