Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22 

23__all__ = ['PairedVisitListTaskRunner', 'SingleVisitListTaskRunner', 

24 'NonexistentDatasetTaskDataIdContainer', 'parseCmdlineNumberString', 

25 'countMaskedPixels', 'checkExpLengthEqual', 'ddict2dict'] 

26 

27import re 

28import numpy as np 

29from scipy.optimize import leastsq 

30import numpy.polynomial.polynomial as poly 

31 

32import lsst.pipe.base as pipeBase 

33import lsst.ip.isr as ipIsr 

34from lsst.ip.isr import isrMock 

35import lsst.log 

36import lsst.afw.image 

37 

38import galsim 

39 

40 

41def calculateWeightedReducedChi2(measured, model, weightsMeasured, nData, nParsModel): 

42 """Calculate weighted reduced chi2. 

43 

44 Parameters 

45 ---------- 

46 

47 measured : `list` 

48 List with measured data. 

49 

50 model : `list` 

51 List with modeled data. 

52 

53 weightsMeasured : `list` 

54 List with weights for the measured data. 

55 

56 nData : `int` 

57 Number of data points. 

58 

59 nParsModel : `int` 

60 Number of parameters in the model. 

61 

62 Returns 

63 ------- 

64 

65 redWeightedChi2 : `float` 

66 Reduced weighted chi2. 

67 """ 

68 

69 wRes = (measured - model)*weightsMeasured 

70 return ((wRes*wRes).sum())/(nData-nParsModel) 

71 

72 

73def makeMockFlats(expTime, gain=1.0, readNoiseElectrons=5, fluxElectrons=1000, 

74 randomSeedFlat1=1984, randomSeedFlat2=666, powerLawBfParams=[], 

75 expId1=0, expId2=1): 

76 """Create a pair or mock flats with isrMock. 

77 

78 Parameters 

79 ---------- 

80 expTime : `float` 

81 Exposure time of the flats. 

82 

83 gain : `float`, optional 

84 Gain, in e/ADU. 

85 

86 readNoiseElectrons : `float`, optional 

87 Read noise rms, in electrons. 

88 

89 fluxElectrons : `float`, optional 

90 Flux of flats, in electrons per second. 

91 

92 randomSeedFlat1 : `int`, optional 

93 Random seed for the normal distrubutions for the mean signal and noise (flat1). 

94 

95 randomSeedFlat2 : `int`, optional 

96 Random seed for the normal distrubutions for the mean signal and noise (flat2). 

97 

98 powerLawBfParams : `list`, optional 

99 Parameters for `galsim.cdmodel.PowerLawCD` to simulate the brightter-fatter effect. 

100 

101 expId1 : `int`, optional 

102 Exposure ID for first flat. 

103 

104 expId2 : `int`, optional 

105 Exposure ID for second flat. 

106 

107 Returns 

108 ------- 

109 

110 flatExp1 : `lsst.afw.image.exposure.exposure.ExposureF` 

111 First exposure of flat field pair. 

112 

113 flatExp2 : `lsst.afw.image.exposure.exposure.ExposureF` 

114 Second exposure of flat field pair. 

115 

116 Notes 

117 ----- 

118 The parameters of `galsim.cdmodel.PowerLawCD` are `n, r0, t0, rx, tx, r, t, alpha`. For more 

119 information about their meaning, see the Galsim documentation 

120 https://galsim-developers.github.io/GalSim/_build/html/_modules/galsim/cdmodel.html 

121 and Gruen+15 (1501.02802). 

122 

123 Example: galsim.cdmodel.PowerLawCD(8, 1.1e-7, 1.1e-7, 1.0e-8, 1.0e-8, 1.0e-9, 1.0e-9, 2.0) 

124 """ 

125 flatFlux = fluxElectrons # e/s 

126 flatMean = flatFlux*expTime # e 

127 readNoise = readNoiseElectrons # e 

128 

129 mockImageConfig = isrMock.IsrMock.ConfigClass() 

130 

131 mockImageConfig.flatDrop = 0.99999 

132 mockImageConfig.isTrimmed = True 

133 

134 flatExp1 = isrMock.FlatMock(config=mockImageConfig).run() 

135 flatExp2 = flatExp1.clone() 

136 (shapeY, shapeX) = flatExp1.getDimensions() 

137 flatWidth = np.sqrt(flatMean) 

138 

139 rng1 = np.random.RandomState(randomSeedFlat1) 

140 flatData1 = rng1.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng1.normal(0.0, readNoise, 

141 (shapeX, shapeY)) 

142 rng2 = np.random.RandomState(randomSeedFlat2) 

143 flatData2 = rng2.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng2.normal(0.0, readNoise, 

144 (shapeX, shapeY)) 

145 # Simulate BF with power law model in galsim 

146 if len(powerLawBfParams): 

147 if not len(powerLawBfParams) == 8: 

148 raise RuntimeError("Wrong number of parameters for `galsim.cdmodel.PowerLawCD`. " 

149 f"Expected 8; passed {len(powerLawBfParams)}.") 

150 cd = galsim.cdmodel.PowerLawCD(*powerLawBfParams) 

151 tempFlatData1 = galsim.Image(flatData1) 

152 temp2FlatData1 = cd.applyForward(tempFlatData1) 

153 

154 tempFlatData2 = galsim.Image(flatData2) 

155 temp2FlatData2 = cd.applyForward(tempFlatData2) 

156 

157 flatExp1.image.array[:] = temp2FlatData1.array/gain # ADU 

158 flatExp2.image.array[:] = temp2FlatData2.array/gain # ADU 

159 else: 

160 flatExp1.image.array[:] = flatData1/gain # ADU 

161 flatExp2.image.array[:] = flatData2/gain # ADU 

162 

163 visitInfoExp1 = lsst.afw.image.VisitInfo(exposureId=expId1, exposureTime=expTime) 

164 visitInfoExp2 = lsst.afw.image.VisitInfo(exposureId=expId2, exposureTime=expTime) 

165 

166 flatExp1.getInfo().setVisitInfo(visitInfoExp1) 

167 flatExp2.getInfo().setVisitInfo(visitInfoExp2) 

168 

169 return flatExp1, flatExp2 

170 

171 

172def countMaskedPixels(maskedIm, maskPlane): 

173 """Count the number of pixels in a given mask plane.""" 

174 maskBit = maskedIm.mask.getPlaneBitMask(maskPlane) 

175 nPix = np.where(np.bitwise_and(maskedIm.mask.array, maskBit))[0].flatten().size 

176 return nPix 

177 

178 

179class PairedVisitListTaskRunner(pipeBase.TaskRunner): 

180 """Subclass of TaskRunner for handling intrinsically paired visits. 

181 

182 This transforms the processed arguments generated by the ArgumentParser 

183 into the arguments expected by tasks which take visit pairs for their 

184 run() methods. 

185 

186 Such tasks' run() methods tend to take two arguments, 

187 one of which is the dataRef (as usual), and the other is the list 

188 of visit-pairs, in the form of a list of tuples. 

189 This list is supplied on the command line as documented, 

190 and this class parses that, and passes the parsed version 

191 to the run() method. 

192 

193 See pipeBase.TaskRunner for more information. 

194 """ 

195 

196 @staticmethod 

197 def getTargetList(parsedCmd, **kwargs): 

198 """Parse the visit list and pass through explicitly.""" 

199 visitPairs = [] 

200 for visitStringPair in parsedCmd.visitPairs: 

201 visitStrings = visitStringPair.split(",") 

202 if len(visitStrings) != 2: 

203 raise RuntimeError("Found {} visits in {} instead of 2".format(len(visitStrings), 

204 visitStringPair)) 

205 try: 

206 visits = [int(visit) for visit in visitStrings] 

207 except Exception: 

208 raise RuntimeError("Could not parse {} as two integer visit numbers".format(visitStringPair)) 

209 visitPairs.append(visits) 

210 

211 return pipeBase.TaskRunner.getTargetList(parsedCmd, visitPairs=visitPairs, **kwargs) 

212 

213 

214def parseCmdlineNumberString(inputString): 

215 """Parse command line numerical expression sytax and return as list of int 

216 

217 Take an input of the form "'1..5:2^123..126'" as a string, and return 

218 a list of ints as [1, 3, 5, 123, 124, 125, 126] 

219 """ 

220 outList = [] 

221 for subString in inputString.split("^"): 

222 mat = re.search(r"^(\d+)\.\.(\d+)(?::(\d+))?$", subString) 

223 if mat: 

224 v1 = int(mat.group(1)) 

225 v2 = int(mat.group(2)) 

226 v3 = mat.group(3) 

227 v3 = int(v3) if v3 else 1 

228 for v in range(v1, v2 + 1, v3): 

229 outList.append(int(v)) 

230 else: 

231 outList.append(int(subString)) 

232 return outList 

233 

234 

235class SingleVisitListTaskRunner(pipeBase.TaskRunner): 

236 """Subclass of TaskRunner for tasks requiring a list of visits per dataRef. 

237 

238 This transforms the processed arguments generated by the ArgumentParser 

239 into the arguments expected by tasks which require a list of visits 

240 to be supplied for each dataRef, as is common in `lsst.cp.pipe` code. 

241 

242 Such tasks' run() methods tend to take two arguments, 

243 one of which is the dataRef (as usual), and the other is the list 

244 of visits. 

245 This list is supplied on the command line as documented, 

246 and this class parses that, and passes the parsed version 

247 to the run() method. 

248 

249 See `lsst.pipe.base.TaskRunner` for more information. 

250 """ 

251 

252 @staticmethod 

253 def getTargetList(parsedCmd, **kwargs): 

254 """Parse the visit list and pass through explicitly.""" 

255 # if this has been pre-parsed and therefore doesn't have length of one 

256 # then something has gone wrong, so execution should stop here. 

257 assert len(parsedCmd.visitList) == 1, 'visitList parsing assumptions violated' 

258 visits = parseCmdlineNumberString(parsedCmd.visitList[0]) 

259 

260 return pipeBase.TaskRunner.getTargetList(parsedCmd, visitList=visits, **kwargs) 

261 

262 

263class NonexistentDatasetTaskDataIdContainer(pipeBase.DataIdContainer): 

264 """A DataIdContainer for the tasks for which the output does 

265 not yet exist.""" 

266 

267 def makeDataRefList(self, namespace): 

268 """Compute refList based on idList. 

269 

270 This method must be defined as the dataset does not exist before this 

271 task is run. 

272 

273 Parameters 

274 ---------- 

275 namespace 

276 Results of parsing the command-line. 

277 

278 Notes 

279 ----- 

280 Not called if ``add_id_argument`` called 

281 with ``doMakeDataRefList=False``. 

282 Note that this is almost a copy-and-paste of the vanilla 

283 implementation, but without checking if the datasets already exist, 

284 as this task exists to make them. 

285 """ 

286 if self.datasetType is None: 

287 raise RuntimeError("Must call setDatasetType first") 

288 butler = namespace.butler 

289 for dataId in self.idList: 

290 refList = list(butler.subset(datasetType=self.datasetType, level=self.level, dataId=dataId)) 

291 # exclude nonexistent data 

292 # this is a recursive test, e.g. for the sake of "raw" data 

293 if not refList: 

294 namespace.log.warn("No data found for dataId=%s", dataId) 

295 continue 

296 self.refList += refList 

297 

298 

299def irlsFit(initialParams, dataX, dataY, function, weightsY=None): 

300 """Iteratively reweighted least squares fit. 

301 

302 This uses the `lsst.cp.pipe.utils.fitLeastSq`, but applies 

303 weights based on the Cauchy distribution to the fitter. See 

304 e.g. Holland and Welsch, 1977, doi:10.1080/03610927708827533 

305 

306 Parameters 

307 ---------- 

308 initialParams : `list` [`float`] 

309 Starting parameters. 

310 dataX : `numpy.array` [`float`] 

311 Abscissa data. 

312 dataY : `numpy.array` [`float`] 

313 Ordinate data. 

314 function : callable 

315 Function to fit. 

316 weightsY : `numpy.array` [`float`] 

317 Weights to apply to the data. 

318 

319 Returns 

320 ------- 

321 polyFit : `list` [`float`] 

322 Final best fit parameters. 

323 polyFitErr : `list` [`float`] 

324 Final errors on fit parameters. 

325 chiSq : `float` 

326 Reduced chi squared. 

327 weightsY : `list` [`float`] 

328 Final weights used for each point. 

329 

330 """ 

331 if not weightsY: 

332 weightsY = np.ones_like(dataX) 

333 

334 polyFit, polyFitErr, chiSq = fitLeastSq(initialParams, dataX, dataY, function, weightsY=weightsY) 

335 for iteration in range(10): 

336 # Use Cauchy weights 

337 resid = np.abs(dataY - function(polyFit, dataX)) / np.sqrt(dataY) 

338 weightsY = 1.0 / (1.0 + np.sqrt(resid / 2.385)) 

339 polyFit, polyFitErr, chiSq = fitLeastSq(initialParams, dataX, dataY, function, weightsY=weightsY) 

340 

341 return polyFit, polyFitErr, chiSq, weightsY 

342 

343 

344def fitLeastSq(initialParams, dataX, dataY, function, weightsY=None): 

345 """Do a fit and estimate the parameter errors using using scipy.optimize.leastq. 

346 

347 optimize.leastsq returns the fractional covariance matrix. To estimate the 

348 standard deviation of the fit parameters, multiply the entries of this matrix 

349 by the unweighted reduced chi squared and take the square root of the diagonal elements. 

350 

351 Parameters 

352 ---------- 

353 initialParams : `list` of `float` 

354 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length 

355 determines the degree of the polynomial. 

356 

357 dataX : `numpy.array` of `float` 

358 Data in the abscissa axis. 

359 

360 dataY : `numpy.array` of `float` 

361 Data in the ordinate axis. 

362 

363 function : callable object (function) 

364 Function to fit the data with. 

365 

366 weightsY : `numpy.array` of `float` 

367 Weights of the data in the ordinate axis. 

368 

369 Return 

370 ------ 

371 pFitSingleLeastSquares : `list` of `float` 

372 List with fitted parameters. 

373 

374 pErrSingleLeastSquares : `list` of `float` 

375 List with errors for fitted parameters. 

376 

377 reducedChiSqSingleLeastSquares : `float` 

378 Reduced chi squared, unweighted if weightsY is not provided. 

379 """ 

380 if weightsY is None: 

381 weightsY = np.ones(len(dataX)) 

382 

383 def errFunc(p, x, y, weightsY=None): 

384 if weightsY is None: 

385 weightsY = np.ones(len(x)) 

386 return (function(p, x) - y)*weightsY 

387 

388 pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams, 

389 args=(dataX, dataY, weightsY), full_output=1, 

390 epsfcn=0.0001) 

391 

392 if (len(dataY) > len(initialParams)) and pCov is not None: 

393 reducedChiSq = calculateWeightedReducedChi2(dataY, function(pFit, dataX), weightsY, len(dataY), 

394 len(initialParams)) 

395 pCov *= reducedChiSq 

396 else: 

397 pCov = np.zeros((len(initialParams), len(initialParams))) 

398 pCov[:, :] = np.nan 

399 reducedChiSq = np.nan 

400 

401 errorVec = [] 

402 for i in range(len(pFit)): 

403 errorVec.append(np.fabs(pCov[i][i])**0.5) 

404 

405 pFitSingleLeastSquares = pFit 

406 pErrSingleLeastSquares = np.array(errorVec) 

407 

408 return pFitSingleLeastSquares, pErrSingleLeastSquares, reducedChiSq 

409 

410 

411def fitBootstrap(initialParams, dataX, dataY, function, weightsY=None, confidenceSigma=1.): 

412 """Do a fit using least squares and bootstrap to estimate parameter errors. 

413 

414 The bootstrap error bars are calculated by fitting 100 random data sets. 

415 

416 Parameters 

417 ---------- 

418 initialParams : `list` of `float` 

419 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length 

420 determines the degree of the polynomial. 

421 

422 dataX : `numpy.array` of `float` 

423 Data in the abscissa axis. 

424 

425 dataY : `numpy.array` of `float` 

426 Data in the ordinate axis. 

427 

428 function : callable object (function) 

429 Function to fit the data with. 

430 

431 weightsY : `numpy.array` of `float`, optional. 

432 Weights of the data in the ordinate axis. 

433 

434 confidenceSigma : `float`, optional. 

435 Number of sigmas that determine confidence interval for the bootstrap errors. 

436 

437 Return 

438 ------ 

439 pFitBootstrap : `list` of `float` 

440 List with fitted parameters. 

441 

442 pErrBootstrap : `list` of `float` 

443 List with errors for fitted parameters. 

444 

445 reducedChiSqBootstrap : `float` 

446 Reduced chi squared, unweighted if weightsY is not provided. 

447 """ 

448 if weightsY is None: 

449 weightsY = np.ones(len(dataX)) 

450 

451 def errFunc(p, x, y, weightsY): 

452 if weightsY is None: 

453 weightsY = np.ones(len(x)) 

454 return (function(p, x) - y)*weightsY 

455 

456 # Fit first time 

457 pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY, weightsY), full_output=0) 

458 

459 # Get the stdev of the residuals 

460 residuals = errFunc(pFit, dataX, dataY, weightsY) 

461 # 100 random data sets are generated and fitted 

462 pars = [] 

463 for i in range(100): 

464 randomDelta = np.random.normal(0., np.fabs(residuals), len(dataY)) 

465 randomDataY = dataY + randomDelta 

466 randomFit, _ = leastsq(errFunc, initialParams, 

467 args=(dataX, randomDataY, weightsY), full_output=0) 

468 pars.append(randomFit) 

469 pars = np.array(pars) 

470 meanPfit = np.mean(pars, 0) 

471 

472 # confidence interval for parameter estimates 

473 errPfit = confidenceSigma*np.std(pars, 0) 

474 pFitBootstrap = meanPfit 

475 pErrBootstrap = errPfit 

476 

477 reducedChiSq = calculateWeightedReducedChi2(dataY, function(pFitBootstrap, dataX), weightsY, len(dataY), 

478 len(initialParams)) 

479 return pFitBootstrap, pErrBootstrap, reducedChiSq 

480 

481 

482def funcPolynomial(pars, x): 

483 """Polynomial function definition 

484 Parameters 

485 ---------- 

486 params : `list` 

487 Polynomial coefficients. Its length determines the polynomial order. 

488 

489 x : `numpy.array` 

490 Abscisa array. 

491 

492 Returns 

493 ------- 

494 Ordinate array after evaluating polynomial of order len(pars)-1 at `x`. 

495 """ 

496 return poly.polyval(x, [*pars]) 

497 

498 

499def funcAstier(pars, x): 

500 """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19. 

501 

502 Parameters 

503 ---------- 

504 params : `list` 

505 Parameters of the model: a00 (brightter-fatter), gain (e/ADU), and noise (e^2). 

506 

507 x : `numpy.array` 

508 Signal mu (ADU). 

509 

510 Returns 

511 ------- 

512 C_00 (variance) in ADU^2. 

513 """ 

514 a00, gain, noise = pars 

515 return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain) # C_00 

516 

517 

518def arrangeFlatsByExpTime(exposureList): 

519 """Arrange exposures by exposure time. 

520 

521 Parameters 

522 ---------- 

523 exposureList : `list`[`lsst.afw.image.exposure.exposure.ExposureF`] 

524 Input list of exposures. 

525 

526 Returns 

527 ------ 

528 flatsAtExpTime : `dict` [`float`, 

529 `list`[`lsst.afw.image.exposure.exposure.ExposureF`]] 

530 Dictionary that groups flat-field exposures that have the same 

531 exposure time (seconds). 

532 """ 

533 flatsAtExpTime = {} 

534 for exp in exposureList: 

535 tempFlat = exp 

536 expTime = tempFlat.getInfo().getVisitInfo().getExposureTime() 

537 listAtExpTime = flatsAtExpTime.setdefault(expTime, []) 

538 listAtExpTime.append(tempFlat) 

539 

540 return flatsAtExpTime 

541 

542 

543def arrangeFlatsByExpId(exposureList): 

544 """Arrange exposures by exposure ID. 

545 

546 There is no guarantee that this will properly group exposures, but 

547 allows a sequence of flats that have different illumination 

548 (despite having the same exposure time) to be processed. 

549 

550 Parameters 

551 ---------- 

552 exposureList : `list`[`lsst.afw.image.exposure.exposure.ExposureF`] 

553 Input list of exposures. 

554 

555 Returns 

556 ------ 

557 flatsAtExpId : `dict` [`float`, 

558 `list`[`lsst.afw.image.exposure.exposure.ExposureF`]] 

559 Dictionary that groups flat-field exposures sequentially by 

560 their exposure id. 

561 

562 Notes 

563 ----- 

564 

565 This algorithm sorts the input exposures by their exposure id, and 

566 then assigns each pair of exposures (exp_j, exp_{j+1}) to pair k, 

567 such that 2*k = j, where j is the python index of one of the 

568 exposures (starting from zero). By checking for the IndexError 

569 while appending, we can ensure that there will only ever be fully 

570 populated pairs. 

571 """ 

572 flatsAtExpId = {} 

573 sortedExposures = sorted(exposureList, key=lambda exp: exp.getInfo().getVisitInfo().getExposureId()) 

574 

575 for jPair, exp in enumerate(sortedExposures): 

576 if (jPair + 1) % 2: 

577 kPair = jPair // 2 

578 listAtExpId = flatsAtExpId.setdefault(kPair, []) 

579 try: 

580 listAtExpId.append(exp) 

581 listAtExpId.append(sortedExposures[jPair + 1]) 

582 except IndexError: 

583 pass 

584 

585 return flatsAtExpId 

586 

587 

588def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False): 

589 """Check the exposure lengths of two exposures are equal. 

590 

591 Parameters: 

592 ----------- 

593 exp1 : `lsst.afw.image.exposure.ExposureF` 

594 First exposure to check 

595 exp2 : `lsst.afw.image.exposure.ExposureF` 

596 Second exposure to check 

597 v1 : `int` or `str`, optional 

598 First visit of the visit pair 

599 v2 : `int` or `str`, optional 

600 Second visit of the visit pair 

601 raiseWithMessage : `bool` 

602 If True, instead of returning a bool, raise a RuntimeError if exposure 

603 times are not equal, with a message about which visits mismatch if the 

604 information is available. 

605 

606 Raises: 

607 ------- 

608 RuntimeError 

609 Raised if the exposure lengths of the two exposures are not equal 

610 """ 

611 expTime1 = exp1.getInfo().getVisitInfo().getExposureTime() 

612 expTime2 = exp2.getInfo().getVisitInfo().getExposureTime() 

613 if expTime1 != expTime2: 

614 if raiseWithMessage: 

615 msg = "Exposure lengths for visit pairs must be equal. " + \ 

616 "Found %s and %s" % (expTime1, expTime2) 

617 if v1 and v2: 

618 msg += " for visit pair %s, %s" % (v1, v2) 

619 raise RuntimeError(msg) 

620 else: 

621 return False 

622 return True 

623 

624 

625def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None, 

626 checkTrim=True, logName=None): 

627 """Check that appropriate ISR settings have been selected for the task. 

628 

629 Note that this checks that the task itself is configured correctly rather 

630 than checking a config. 

631 

632 Parameters 

633 ---------- 

634 isrTask : `lsst.ip.isr.IsrTask` 

635 The task whose config is to be validated 

636 

637 mandatory : `iterable` of `str` 

638 isr steps that must be set to True. Raises if False or missing 

639 

640 forbidden : `iterable` of `str` 

641 isr steps that must be set to False. Raises if True, warns if missing 

642 

643 desirable : `iterable` of `str` 

644 isr steps that should probably be set to True. Warns is False, info if 

645 missing 

646 

647 undesirable : `iterable` of `str` 

648 isr steps that should probably be set to False. Warns is True, info if 

649 missing 

650 

651 checkTrim : `bool` 

652 Check to ensure the isrTask's assembly subtask is trimming the images. 

653 This is a separate config as it is very ugly to do this within the 

654 normal configuration lists as it is an option of a sub task. 

655 

656 Raises 

657 ------ 

658 RuntimeError 

659 Raised if ``mandatory`` config parameters are False, 

660 or if ``forbidden`` parameters are True. 

661 

662 TypeError 

663 Raised if parameter ``isrTask`` is an invalid type. 

664 

665 Notes 

666 ----- 

667 Logs warnings using an isrValidation logger for desirable/undesirable 

668 options that are of the wrong polarity or if keys are missing. 

669 """ 

670 if not isinstance(isrTask, ipIsr.IsrTask): 

671 raise TypeError(f'Must supply an instance of lsst.ip.isr.IsrTask not {type(isrTask)}') 

672 

673 configDict = isrTask.config.toDict() 

674 

675 if logName and isinstance(logName, str): 

676 log = lsst.log.getLogger(logName) 

677 else: 

678 log = lsst.log.getLogger("isrValidation") 

679 

680 if mandatory: 

681 for configParam in mandatory: 

682 if configParam not in configDict: 

683 raise RuntimeError(f"Mandatory parameter {configParam} not found in the isr configuration.") 

684 if configDict[configParam] is False: 

685 raise RuntimeError(f"Must set config.isr.{configParam} to True for this task.") 

686 

687 if forbidden: 

688 for configParam in forbidden: 

689 if configParam not in configDict: 

690 log.warn(f"Failed to find forbidden key {configParam} in the isr config. The keys in the" 

691 " forbidden list should each have an associated Field in IsrConfig:" 

692 " check that there is not a typo in this case.") 

693 continue 

694 if configDict[configParam] is True: 

695 raise RuntimeError(f"Must set config.isr.{configParam} to False for this task.") 

696 

697 if desirable: 

698 for configParam in desirable: 

699 if configParam not in configDict: 

700 log.info(f"Failed to find key {configParam} in the isr config. You probably want" 

701 " to set the equivalent for your obs_package to True.") 

702 continue 

703 if configDict[configParam] is False: 

704 log.warn(f"Found config.isr.{configParam} set to False for this task." 

705 " The cp_pipe Config recommends setting this to True.") 

706 if undesirable: 

707 for configParam in undesirable: 

708 if configParam not in configDict: 

709 log.info(f"Failed to find key {configParam} in the isr config. You probably want" 

710 " to set the equivalent for your obs_package to False.") 

711 continue 

712 if configDict[configParam] is True: 

713 log.warn(f"Found config.isr.{configParam} set to True for this task." 

714 " The cp_pipe Config recommends setting this to False.") 

715 

716 if checkTrim: # subtask setting, seems non-trivial to combine with above lists 

717 if not isrTask.assembleCcd.config.doTrim: 

718 raise RuntimeError("Must trim when assembling CCDs. Set config.isr.assembleCcd.doTrim to True") 

719 

720 

721def ddict2dict(d): 

722 """Convert nested default dictionaries to regular dictionaries. 

723 

724 This is needed to prevent yaml persistence issues. 

725 

726 Parameters 

727 ---------- 

728 d : `defaultdict` 

729 A possibly nested set of `defaultdict`. 

730 

731 Returns 

732 ------- 

733 dict : `dict` 

734 A possibly nested set of `dict`. 

735 """ 

736 for k, v in d.items(): 

737 if isinstance(v, dict): 

738 d[k] = ddict2dict(v) 

739 return dict(d)