Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22 

23__all__ = ['PairedVisitListTaskRunner', 'SingleVisitListTaskRunner', 

24 'NonexistentDatasetTaskDataIdContainer', 'parseCmdlineNumberString', 

25 'countMaskedPixels', 'checkExpLengthEqual', 'ddict2dict'] 

26 

27import re 

28import numpy as np 

29from scipy.optimize import leastsq 

30import numpy.polynomial.polynomial as poly 

31 

32import lsst.pipe.base as pipeBase 

33import lsst.ip.isr as ipIsr 

34from lsst.ip.isr import isrMock 

35import lsst.log 

36 

37import galsim 

38 

39 

40def calculateWeightedReducedChi2(measured, model, weightsMeasured, nData, nParsModel): 

41 """Calculate weighted reduced chi2. 

42 

43 Parameters 

44 ---------- 

45 

46 measured : `list` 

47 List with measured data. 

48 

49 model : `list` 

50 List with modeled data. 

51 

52 weightsMeasured : `list` 

53 List with weights for the measured data. 

54 

55 nData : `int` 

56 Number of data points. 

57 

58 nParsModel : `int` 

59 Number of parameters in the model. 

60 

61 Returns 

62 ------- 

63 

64 redWeightedChi2 : `float` 

65 Reduced weighted chi2. 

66 """ 

67 

68 wRes = (measured - model)*weightsMeasured 

69 return ((wRes*wRes).sum())/(nData-nParsModel) 

70 

71 

72def makeMockFlats(expTime, gain=1.0, readNoiseElectrons=5, fluxElectrons=1000, 

73 randomSeedFlat1=1984, randomSeedFlat2=666, powerLawBfParams=[]): 

74 """Create a pair or mock flats with isrMock. 

75 

76 Parameters 

77 ---------- 

78 expTime : `float` 

79 Exposure time of the flats. 

80 

81 gain : `float`, optional 

82 Gain, in e/ADU. 

83 

84 readNoiseElectrons : `float`, optional 

85 Read noise rms, in electrons. 

86 

87 fluxElectrons : `float`, optional 

88 Flux of flats, in electrons per second. 

89 

90 randomSeedFlat1 : `int`, optional 

91 Random seed for the normal distrubutions for the mean signal and noise (flat1). 

92 

93 randomSeedFlat2 : `int`, optional 

94 Random seed for the normal distrubutions for the mean signal and noise (flat2). 

95 

96 powerLawBfParams : `list`, optional 

97 Parameters for `galsim.cdmodel.PowerLawCD` to simulate the brightter-fatter effect. 

98 

99 Returns 

100 ------- 

101 

102 flatExp1 : `lsst.afw.image.exposure.exposure.ExposureF` 

103 First exposure of flat field pair. 

104 

105 flatExp2 : `lsst.afw.image.exposure.exposure.ExposureF` 

106 Second exposure of flat field pair. 

107 

108 Notes 

109 ----- 

110 The parameters of `galsim.cdmodel.PowerLawCD` are `n, r0, t0, rx, tx, r, t, alpha`. For more 

111 information about their meaning, see the Galsim documentation 

112 https://galsim-developers.github.io/GalSim/_build/html/_modules/galsim/cdmodel.html 

113 and Gruen+15 (1501.02802). 

114 

115 Example: galsim.cdmodel.PowerLawCD(8, 1.1e-7, 1.1e-7, 1.0e-8, 1.0e-8, 1.0e-9, 1.0e-9, 2.0) 

116 """ 

117 flatFlux = fluxElectrons # e/s 

118 flatMean = flatFlux*expTime # e 

119 readNoise = readNoiseElectrons # e 

120 

121 mockImageConfig = isrMock.IsrMock.ConfigClass() 

122 

123 mockImageConfig.flatDrop = 0.99999 

124 mockImageConfig.isTrimmed = True 

125 

126 flatExp1 = isrMock.FlatMock(config=mockImageConfig).run() 

127 flatExp2 = flatExp1.clone() 

128 (shapeY, shapeX) = flatExp1.getDimensions() 

129 flatWidth = np.sqrt(flatMean) 

130 

131 rng1 = np.random.RandomState(randomSeedFlat1) 

132 flatData1 = rng1.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng1.normal(0.0, readNoise, 

133 (shapeX, shapeY)) 

134 rng2 = np.random.RandomState(randomSeedFlat2) 

135 flatData2 = rng2.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng2.normal(0.0, readNoise, 

136 (shapeX, shapeY)) 

137 # Simulate BF with power law model in galsim 

138 if len(powerLawBfParams): 

139 if not len(powerLawBfParams) == 8: 

140 raise RuntimeError("Wrong number of parameters for `galsim.cdmodel.PowerLawCD`. " + 

141 f"Expected 8; passed {len(powerLawBfParams)}.") 

142 cd = galsim.cdmodel.PowerLawCD(*powerLawBfParams) 

143 tempFlatData1 = galsim.Image(flatData1) 

144 temp2FlatData1 = cd.applyForward(tempFlatData1) 

145 

146 tempFlatData2 = galsim.Image(flatData2) 

147 temp2FlatData2 = cd.applyForward(tempFlatData2) 

148 

149 flatExp1.image.array[:] = temp2FlatData1.array/gain # ADU 

150 flatExp2.image.array[:] = temp2FlatData2.array/gain # ADU 

151 else: 

152 flatExp1.image.array[:] = flatData1/gain # ADU 

153 flatExp2.image.array[:] = flatData2/gain # ADU 

154 

155 return flatExp1, flatExp2 

156 

157 

158def countMaskedPixels(maskedIm, maskPlane): 

159 """Count the number of pixels in a given mask plane.""" 

160 maskBit = maskedIm.mask.getPlaneBitMask(maskPlane) 

161 nPix = np.where(np.bitwise_and(maskedIm.mask.array, maskBit))[0].flatten().size 

162 return nPix 

163 

164 

165class PairedVisitListTaskRunner(pipeBase.TaskRunner): 

166 """Subclass of TaskRunner for handling intrinsically paired visits. 

167 

168 This transforms the processed arguments generated by the ArgumentParser 

169 into the arguments expected by tasks which take visit pairs for their 

170 run() methods. 

171 

172 Such tasks' run() methods tend to take two arguments, 

173 one of which is the dataRef (as usual), and the other is the list 

174 of visit-pairs, in the form of a list of tuples. 

175 This list is supplied on the command line as documented, 

176 and this class parses that, and passes the parsed version 

177 to the run() method. 

178 

179 See pipeBase.TaskRunner for more information. 

180 """ 

181 

182 @staticmethod 

183 def getTargetList(parsedCmd, **kwargs): 

184 """Parse the visit list and pass through explicitly.""" 

185 visitPairs = [] 

186 for visitStringPair in parsedCmd.visitPairs: 

187 visitStrings = visitStringPair.split(",") 

188 if len(visitStrings) != 2: 

189 raise RuntimeError("Found {} visits in {} instead of 2".format(len(visitStrings), 

190 visitStringPair)) 

191 try: 

192 visits = [int(visit) for visit in visitStrings] 

193 except Exception: 

194 raise RuntimeError("Could not parse {} as two integer visit numbers".format(visitStringPair)) 

195 visitPairs.append(visits) 

196 

197 return pipeBase.TaskRunner.getTargetList(parsedCmd, visitPairs=visitPairs, **kwargs) 

198 

199 

200def parseCmdlineNumberString(inputString): 

201 """Parse command line numerical expression sytax and return as list of int 

202 

203 Take an input of the form "'1..5:2^123..126'" as a string, and return 

204 a list of ints as [1, 3, 5, 123, 124, 125, 126] 

205 """ 

206 outList = [] 

207 for subString in inputString.split("^"): 

208 mat = re.search(r"^(\d+)\.\.(\d+)(?::(\d+))?$", subString) 

209 if mat: 

210 v1 = int(mat.group(1)) 

211 v2 = int(mat.group(2)) 

212 v3 = mat.group(3) 

213 v3 = int(v3) if v3 else 1 

214 for v in range(v1, v2 + 1, v3): 

215 outList.append(int(v)) 

216 else: 

217 outList.append(int(subString)) 

218 return outList 

219 

220 

221class SingleVisitListTaskRunner(pipeBase.TaskRunner): 

222 """Subclass of TaskRunner for tasks requiring a list of visits per dataRef. 

223 

224 This transforms the processed arguments generated by the ArgumentParser 

225 into the arguments expected by tasks which require a list of visits 

226 to be supplied for each dataRef, as is common in `lsst.cp.pipe` code. 

227 

228 Such tasks' run() methods tend to take two arguments, 

229 one of which is the dataRef (as usual), and the other is the list 

230 of visits. 

231 This list is supplied on the command line as documented, 

232 and this class parses that, and passes the parsed version 

233 to the run() method. 

234 

235 See `lsst.pipe.base.TaskRunner` for more information. 

236 """ 

237 

238 @staticmethod 

239 def getTargetList(parsedCmd, **kwargs): 

240 """Parse the visit list and pass through explicitly.""" 

241 # if this has been pre-parsed and therefore doesn't have length of one 

242 # then something has gone wrong, so execution should stop here. 

243 assert len(parsedCmd.visitList) == 1, 'visitList parsing assumptions violated' 

244 visits = parseCmdlineNumberString(parsedCmd.visitList[0]) 

245 

246 return pipeBase.TaskRunner.getTargetList(parsedCmd, visitList=visits, **kwargs) 

247 

248 

249class NonexistentDatasetTaskDataIdContainer(pipeBase.DataIdContainer): 

250 """A DataIdContainer for the tasks for which the output does 

251 not yet exist.""" 

252 

253 def makeDataRefList(self, namespace): 

254 """Compute refList based on idList. 

255 

256 This method must be defined as the dataset does not exist before this 

257 task is run. 

258 

259 Parameters 

260 ---------- 

261 namespace 

262 Results of parsing the command-line. 

263 

264 Notes 

265 ----- 

266 Not called if ``add_id_argument`` called 

267 with ``doMakeDataRefList=False``. 

268 Note that this is almost a copy-and-paste of the vanilla 

269 implementation, but without checking if the datasets already exist, 

270 as this task exists to make them. 

271 """ 

272 if self.datasetType is None: 

273 raise RuntimeError("Must call setDatasetType first") 

274 butler = namespace.butler 

275 for dataId in self.idList: 

276 refList = list(butler.subset(datasetType=self.datasetType, level=self.level, dataId=dataId)) 

277 # exclude nonexistent data 

278 # this is a recursive test, e.g. for the sake of "raw" data 

279 if not refList: 

280 namespace.log.warn("No data found for dataId=%s", dataId) 

281 continue 

282 self.refList += refList 

283 

284 

285def irlsFit(initialParams, dataX, dataY, function, weightsY=None): 

286 """Iteratively reweighted least squares fit. 

287 

288 This uses the `lsst.cp.pipe.utils.fitLeastSq`, but applies 

289 weights based on the Cauchy distribution to the fitter. See 

290 e.g. Holland and Welsch, 1977, doi:10.1080/03610927708827533 

291 

292 Parameters 

293 ---------- 

294 initialParams : `list` [`float`] 

295 Starting parameters. 

296 dataX : `numpy.array` [`float`] 

297 Abscissa data. 

298 dataY : `numpy.array` [`float`] 

299 Ordinate data. 

300 function : callable 

301 Function to fit. 

302 weightsY : `numpy.array` [`float`] 

303 Weights to apply to the data. 

304 

305 Returns 

306 ------- 

307 polyFit : `list` [`float`] 

308 Final best fit parameters. 

309 polyFitErr : `list` [`float`] 

310 Final errors on fit parameters. 

311 chiSq : `float` 

312 Reduced chi squared. 

313 weightsY : `list` [`float`] 

314 Final weights used for each point. 

315 

316 """ 

317 if not weightsY: 

318 weightsY = np.ones_like(dataX) 

319 

320 polyFit, polyFitErr, chiSq = fitLeastSq(initialParams, dataX, dataY, function, weightsY=weightsY) 

321 for iteration in range(10): 

322 # Use Cauchy weights 

323 resid = np.abs(dataY - function(polyFit, dataX)) / np.sqrt(dataY) 

324 weightsY = 1.0 / (1.0 + np.sqrt(resid / 2.385)) 

325 polyFit, polyFitErr, chiSq = fitLeastSq(initialParams, dataX, dataY, function, weightsY=weightsY) 

326 

327 return polyFit, polyFitErr, chiSq, weightsY 

328 

329 

330def fitLeastSq(initialParams, dataX, dataY, function, weightsY=None): 

331 """Do a fit and estimate the parameter errors using using scipy.optimize.leastq. 

332 

333 optimize.leastsq returns the fractional covariance matrix. To estimate the 

334 standard deviation of the fit parameters, multiply the entries of this matrix 

335 by the unweighted reduced chi squared and take the square root of the diagonal elements. 

336 

337 Parameters 

338 ---------- 

339 initialParams : `list` of `float` 

340 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length 

341 determines the degree of the polynomial. 

342 

343 dataX : `numpy.array` of `float` 

344 Data in the abscissa axis. 

345 

346 dataY : `numpy.array` of `float` 

347 Data in the ordinate axis. 

348 

349 function : callable object (function) 

350 Function to fit the data with. 

351 

352 weightsY : `numpy.array` of `float` 

353 Weights of the data in the ordinate axis. 

354 

355 Return 

356 ------ 

357 pFitSingleLeastSquares : `list` of `float` 

358 List with fitted parameters. 

359 

360 pErrSingleLeastSquares : `list` of `float` 

361 List with errors for fitted parameters. 

362 

363 reducedChiSqSingleLeastSquares : `float` 

364 Reduced chi squared, unweighted if weightsY is not provided. 

365 """ 

366 if weightsY is None: 

367 weightsY = np.ones(len(dataX)) 

368 

369 def errFunc(p, x, y, weightsY=None): 

370 if weightsY is None: 

371 weightsY = np.ones(len(x)) 

372 return (function(p, x) - y)*weightsY 

373 

374 pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams, 

375 args=(dataX, dataY, weightsY), full_output=1, 

376 epsfcn=0.0001) 

377 

378 if (len(dataY) > len(initialParams)) and pCov is not None: 

379 reducedChiSq = calculateWeightedReducedChi2(dataY, function(pFit, dataX), weightsY, len(dataY), 

380 len(initialParams)) 

381 pCov *= reducedChiSq 

382 else: 

383 pCov = np.zeros((len(initialParams), len(initialParams))) 

384 pCov[:, :] = np.nan 

385 reducedChiSq = np.nan 

386 

387 errorVec = [] 

388 for i in range(len(pFit)): 

389 errorVec.append(np.fabs(pCov[i][i])**0.5) 

390 

391 pFitSingleLeastSquares = pFit 

392 pErrSingleLeastSquares = np.array(errorVec) 

393 

394 return pFitSingleLeastSquares, pErrSingleLeastSquares, reducedChiSq 

395 

396 

397def fitBootstrap(initialParams, dataX, dataY, function, weightsY=None, confidenceSigma=1.): 

398 """Do a fit using least squares and bootstrap to estimate parameter errors. 

399 

400 The bootstrap error bars are calculated by fitting 100 random data sets. 

401 

402 Parameters 

403 ---------- 

404 initialParams : `list` of `float` 

405 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length 

406 determines the degree of the polynomial. 

407 

408 dataX : `numpy.array` of `float` 

409 Data in the abscissa axis. 

410 

411 dataY : `numpy.array` of `float` 

412 Data in the ordinate axis. 

413 

414 function : callable object (function) 

415 Function to fit the data with. 

416 

417 weightsY : `numpy.array` of `float`, optional. 

418 Weights of the data in the ordinate axis. 

419 

420 confidenceSigma : `float`, optional. 

421 Number of sigmas that determine confidence interval for the bootstrap errors. 

422 

423 Return 

424 ------ 

425 pFitBootstrap : `list` of `float` 

426 List with fitted parameters. 

427 

428 pErrBootstrap : `list` of `float` 

429 List with errors for fitted parameters. 

430 

431 reducedChiSqBootstrap : `float` 

432 Reduced chi squared, unweighted if weightsY is not provided. 

433 """ 

434 if weightsY is None: 

435 weightsY = np.ones(len(dataX)) 

436 

437 def errFunc(p, x, y, weightsY): 

438 if weightsY is None: 

439 weightsY = np.ones(len(x)) 

440 return (function(p, x) - y)*weightsY 

441 

442 # Fit first time 

443 pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY, weightsY), full_output=0) 

444 

445 # Get the stdev of the residuals 

446 residuals = errFunc(pFit, dataX, dataY, weightsY) 

447 # 100 random data sets are generated and fitted 

448 pars = [] 

449 for i in range(100): 

450 randomDelta = np.random.normal(0., np.fabs(residuals), len(dataY)) 

451 randomDataY = dataY + randomDelta 

452 randomFit, _ = leastsq(errFunc, initialParams, 

453 args=(dataX, randomDataY, weightsY), full_output=0) 

454 pars.append(randomFit) 

455 pars = np.array(pars) 

456 meanPfit = np.mean(pars, 0) 

457 

458 # confidence interval for parameter estimates 

459 errPfit = confidenceSigma*np.std(pars, 0) 

460 pFitBootstrap = meanPfit 

461 pErrBootstrap = errPfit 

462 

463 reducedChiSq = calculateWeightedReducedChi2(dataY, function(pFitBootstrap, dataX), weightsY, len(dataY), 

464 len(initialParams)) 

465 return pFitBootstrap, pErrBootstrap, reducedChiSq 

466 

467 

468def funcPolynomial(pars, x): 

469 """Polynomial function definition 

470 Parameters 

471 ---------- 

472 params : `list` 

473 Polynomial coefficients. Its length determines the polynomial order. 

474 

475 x : `numpy.array` 

476 Abscisa array. 

477 

478 Returns 

479 ------- 

480 Ordinate array after evaluating polynomial of order len(pars)-1 at `x`. 

481 """ 

482 return poly.polyval(x, [*pars]) 

483 

484 

485def funcAstier(pars, x): 

486 """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19. 

487 

488 Parameters 

489 ---------- 

490 params : `list` 

491 Parameters of the model: a00 (brightter-fatter), gain (e/ADU), and noise (e^2). 

492 

493 x : `numpy.array` 

494 Signal mu (ADU). 

495 

496 Returns 

497 ------- 

498 C_00 (variance) in ADU^2. 

499 """ 

500 a00, gain, noise = pars 

501 return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain) # C_00 

502 

503 

504def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False): 

505 """Check the exposure lengths of two exposures are equal. 

506 

507 Parameters: 

508 ----------- 

509 exp1 : `lsst.afw.image.exposure.ExposureF` 

510 First exposure to check 

511 exp2 : `lsst.afw.image.exposure.ExposureF` 

512 Second exposure to check 

513 v1 : `int` or `str`, optional 

514 First visit of the visit pair 

515 v2 : `int` or `str`, optional 

516 Second visit of the visit pair 

517 raiseWithMessage : `bool` 

518 If True, instead of returning a bool, raise a RuntimeError if exposure 

519 times are not equal, with a message about which visits mismatch if the 

520 information is available. 

521 

522 Raises: 

523 ------- 

524 RuntimeError 

525 Raised if the exposure lengths of the two exposures are not equal 

526 """ 

527 expTime1 = exp1.getInfo().getVisitInfo().getExposureTime() 

528 expTime2 = exp2.getInfo().getVisitInfo().getExposureTime() 

529 if expTime1 != expTime2: 

530 if raiseWithMessage: 

531 msg = "Exposure lengths for visit pairs must be equal. " + \ 

532 "Found %s and %s" % (expTime1, expTime2) 

533 if v1 and v2: 

534 msg += " for visit pair %s, %s" % (v1, v2) 

535 raise RuntimeError(msg) 

536 else: 

537 return False 

538 return True 

539 

540 

541def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None, 

542 checkTrim=True, logName=None): 

543 """Check that appropriate ISR settings have been selected for the task. 

544 

545 Note that this checks that the task itself is configured correctly rather 

546 than checking a config. 

547 

548 Parameters 

549 ---------- 

550 isrTask : `lsst.ip.isr.IsrTask` 

551 The task whose config is to be validated 

552 

553 mandatory : `iterable` of `str` 

554 isr steps that must be set to True. Raises if False or missing 

555 

556 forbidden : `iterable` of `str` 

557 isr steps that must be set to False. Raises if True, warns if missing 

558 

559 desirable : `iterable` of `str` 

560 isr steps that should probably be set to True. Warns is False, info if 

561 missing 

562 

563 undesirable : `iterable` of `str` 

564 isr steps that should probably be set to False. Warns is True, info if 

565 missing 

566 

567 checkTrim : `bool` 

568 Check to ensure the isrTask's assembly subtask is trimming the images. 

569 This is a separate config as it is very ugly to do this within the 

570 normal configuration lists as it is an option of a sub task. 

571 

572 Raises 

573 ------ 

574 RuntimeError 

575 Raised if ``mandatory`` config parameters are False, 

576 or if ``forbidden`` parameters are True. 

577 

578 TypeError 

579 Raised if parameter ``isrTask`` is an invalid type. 

580 

581 Notes 

582 ----- 

583 Logs warnings using an isrValidation logger for desirable/undesirable 

584 options that are of the wrong polarity or if keys are missing. 

585 """ 

586 if not isinstance(isrTask, ipIsr.IsrTask): 

587 raise TypeError(f'Must supply an instance of lsst.ip.isr.IsrTask not {type(isrTask)}') 

588 

589 configDict = isrTask.config.toDict() 

590 

591 if logName and isinstance(logName, str): 

592 log = lsst.log.getLogger(logName) 

593 else: 

594 log = lsst.log.getLogger("isrValidation") 

595 

596 if mandatory: 

597 for configParam in mandatory: 

598 if configParam not in configDict: 

599 raise RuntimeError(f"Mandatory parameter {configParam} not found in the isr configuration.") 

600 if configDict[configParam] is False: 

601 raise RuntimeError(f"Must set config.isr.{configParam} to True for this task.") 

602 

603 if forbidden: 

604 for configParam in forbidden: 

605 if configParam not in configDict: 

606 log.warn(f"Failed to find forbidden key {configParam} in the isr config. The keys in the" 

607 " forbidden list should each have an associated Field in IsrConfig:" 

608 " check that there is not a typo in this case.") 

609 continue 

610 if configDict[configParam] is True: 

611 raise RuntimeError(f"Must set config.isr.{configParam} to False for this task.") 

612 

613 if desirable: 

614 for configParam in desirable: 

615 if configParam not in configDict: 

616 log.info(f"Failed to find key {configParam} in the isr config. You probably want" + 

617 " to set the equivalent for your obs_package to True.") 

618 continue 

619 if configDict[configParam] is False: 

620 log.warn(f"Found config.isr.{configParam} set to False for this task." + 

621 " The cp_pipe Config recommends setting this to True.") 

622 if undesirable: 

623 for configParam in undesirable: 

624 if configParam not in configDict: 

625 log.info(f"Failed to find key {configParam} in the isr config. You probably want" + 

626 " to set the equivalent for your obs_package to False.") 

627 continue 

628 if configDict[configParam] is True: 

629 log.warn(f"Found config.isr.{configParam} set to True for this task." + 

630 " The cp_pipe Config recommends setting this to False.") 

631 

632 if checkTrim: # subtask setting, seems non-trivial to combine with above lists 

633 if not isrTask.assembleCcd.config.doTrim: 

634 raise RuntimeError("Must trim when assembling CCDs. Set config.isr.assembleCcd.doTrim to True") 

635 

636 

637def ddict2dict(d): 

638 """Convert nested default dictionaries to regular dictionaries. 

639 

640 This is needed to prevent yaml persistence issues. 

641 

642 Parameters 

643 ---------- 

644 d : `defaultdict` 

645 A possibly nested set of `defaultdict`. 

646 

647 Returns 

648 ------- 

649 dict : `dict` 

650 A possibly nested set of `dict`. 

651 """ 

652 for k, v in d.items(): 

653 if isinstance(v, dict): 

654 d[k] = ddict2dict(v) 

655 return dict(d)