Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22 

23__all__ = ['PairedVisitListTaskRunner', 'SingleVisitListTaskRunner', 

24 'NonexistentDatasetTaskDataIdContainer', 'parseCmdlineNumberString', 

25 'countMaskedPixels', 'checkExpLengthEqual'] 

26 

27import re 

28import numpy as np 

29from scipy.optimize import leastsq 

30import numpy.polynomial.polynomial as poly 

31 

32import lsst.pipe.base as pipeBase 

33import lsst.ip.isr as ipIsr 

34from lsst.ip.isr import isrMock 

35import lsst.log 

36 

37import galsim 

38 

39 

40def calculateWeightedReducedChi2(measured, model, weightsMeasured, nData, nParsModel): 

41 """Calculate weighted reduced chi2. 

42 

43 Parameters 

44 ---------- 

45 

46 measured : `list` 

47 List with measured data. 

48 

49 model : `list` 

50 List with modeled data. 

51 

52 weightsMeasured : `list` 

53 List with weights for the measured data. 

54 

55 nData : `int` 

56 Number of data points. 

57 

58 nParsModel : `int` 

59 Number of parameters in the model. 

60 

61 Returns 

62 ------- 

63 

64 redWeightedChi2 : `float` 

65 Reduced weighted chi2. 

66 """ 

67 

68 wRes = (measured - model)*weightsMeasured 

69 return ((wRes*wRes).sum())/(nData-nParsModel) 

70 

71 

72def makeMockFlats(expTime, gain=1.0, readNoiseElectrons=5, fluxElectrons=1000, 

73 randomSeedFlat1=1984, randomSeedFlat2=666, powerLawBfParams=[]): 

74 """Create a pair or mock flats with isrMock. 

75 

76 Parameters 

77 ---------- 

78 expTime : `float` 

79 Exposure time of the flats. 

80 

81 gain : `float`, optional 

82 Gain, in e/ADU. 

83 

84 readNoiseElectrons : `float`, optional 

85 Read noise rms, in electrons. 

86 

87 fluxElectrons : `float`, optional 

88 Flux of flats, in electrons per second. 

89 

90 randomSeedFlat1 : `int`, optional 

91 Random seed for the normal distrubutions for the mean signal and noise (flat1). 

92 

93 randomSeedFlat2 : `int`, optional 

94 Random seed for the normal distrubutions for the mean signal and noise (flat2). 

95 

96 powerLawBfParams : `list`, optional 

97 Parameters for `galsim.cdmodel.PowerLawCD` to simulate the brightter-fatter effect. 

98 

99 Returns 

100 ------- 

101 

102 flatExp1 : `lsst.afw.image.exposure.exposure.ExposureF` 

103 First exposure of flat field pair. 

104 

105 flatExp2 : `lsst.afw.image.exposure.exposure.ExposureF` 

106 Second exposure of flat field pair. 

107 

108 Notes 

109 ----- 

110 The parameters of `galsim.cdmodel.PowerLawCD` are `n, r0, t0, rx, tx, r, t, alpha`. For more 

111 information about their meaning, see the Galsim documentation 

112 https://galsim-developers.github.io/GalSim/_build/html/_modules/galsim/cdmodel.html 

113 and Gruen+15 (1501.02802). 

114 

115 Example: galsim.cdmodel.PowerLawCD(8, 1.1e-7, 1.1e-7, 1.0e-8, 1.0e-8, 1.0e-9, 1.0e-9, 2.0) 

116 """ 

117 flatFlux = fluxElectrons # e/s 

118 flatMean = flatFlux*expTime # e 

119 readNoise = readNoiseElectrons # e 

120 

121 mockImageConfig = isrMock.IsrMock.ConfigClass() 

122 

123 mockImageConfig.flatDrop = 0.99999 

124 mockImageConfig.isTrimmed = True 

125 

126 flatExp1 = isrMock.FlatMock(config=mockImageConfig).run() 

127 flatExp2 = flatExp1.clone() 

128 (shapeY, shapeX) = flatExp1.getDimensions() 

129 flatWidth = np.sqrt(flatMean) 

130 

131 rng1 = np.random.RandomState(randomSeedFlat1) 

132 flatData1 = rng1.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng1.normal(0.0, readNoise, 

133 (shapeX, shapeY)) 

134 rng2 = np.random.RandomState(randomSeedFlat2) 

135 flatData2 = rng2.normal(flatMean, flatWidth, (shapeX, shapeY)) + rng2.normal(0.0, readNoise, 

136 (shapeX, shapeY)) 

137 # Simulate BF with power law model in galsim 

138 if len(powerLawBfParams): 

139 if not len(powerLawBfParams) == 8: 

140 raise RuntimeError("Wrong number of parameters for `galsim.cdmodel.PowerLawCD`. " + 

141 f"Expected 8; passed {len(powerLawBfParams)}.") 

142 cd = galsim.cdmodel.PowerLawCD(*powerLawBfParams) 

143 tempFlatData1 = galsim.Image(flatData1) 

144 temp2FlatData1 = cd.applyForward(tempFlatData1) 

145 

146 tempFlatData2 = galsim.Image(flatData2) 

147 temp2FlatData2 = cd.applyForward(tempFlatData2) 

148 

149 flatExp1.image.array[:] = temp2FlatData1.array/gain # ADU 

150 flatExp2.image.array[:] = temp2FlatData2.array/gain # ADU 

151 else: 

152 flatExp1.image.array[:] = flatData1/gain # ADU 

153 flatExp2.image.array[:] = flatData2/gain # ADU 

154 

155 return flatExp1, flatExp2 

156 

157 

158def countMaskedPixels(maskedIm, maskPlane): 

159 """Count the number of pixels in a given mask plane.""" 

160 maskBit = maskedIm.mask.getPlaneBitMask(maskPlane) 

161 nPix = np.where(np.bitwise_and(maskedIm.mask.array, maskBit))[0].flatten().size 

162 return nPix 

163 

164 

165class PairedVisitListTaskRunner(pipeBase.TaskRunner): 

166 """Subclass of TaskRunner for handling intrinsically paired visits. 

167 

168 This transforms the processed arguments generated by the ArgumentParser 

169 into the arguments expected by tasks which take visit pairs for their 

170 run() methods. 

171 

172 Such tasks' run() methods tend to take two arguments, 

173 one of which is the dataRef (as usual), and the other is the list 

174 of visit-pairs, in the form of a list of tuples. 

175 This list is supplied on the command line as documented, 

176 and this class parses that, and passes the parsed version 

177 to the run() method. 

178 

179 See pipeBase.TaskRunner for more information. 

180 """ 

181 

182 @staticmethod 

183 def getTargetList(parsedCmd, **kwargs): 

184 """Parse the visit list and pass through explicitly.""" 

185 visitPairs = [] 

186 for visitStringPair in parsedCmd.visitPairs: 

187 visitStrings = visitStringPair.split(",") 

188 if len(visitStrings) != 2: 

189 raise RuntimeError("Found {} visits in {} instead of 2".format(len(visitStrings), 

190 visitStringPair)) 

191 try: 

192 visits = [int(visit) for visit in visitStrings] 

193 except Exception: 

194 raise RuntimeError("Could not parse {} as two integer visit numbers".format(visitStringPair)) 

195 visitPairs.append(visits) 

196 

197 return pipeBase.TaskRunner.getTargetList(parsedCmd, visitPairs=visitPairs, **kwargs) 

198 

199 

200def parseCmdlineNumberString(inputString): 

201 """Parse command line numerical expression sytax and return as list of int 

202 

203 Take an input of the form "'1..5:2^123..126'" as a string, and return 

204 a list of ints as [1, 3, 5, 123, 124, 125, 126] 

205 """ 

206 outList = [] 

207 for subString in inputString.split("^"): 

208 mat = re.search(r"^(\d+)\.\.(\d+)(?::(\d+))?$", subString) 

209 if mat: 

210 v1 = int(mat.group(1)) 

211 v2 = int(mat.group(2)) 

212 v3 = mat.group(3) 

213 v3 = int(v3) if v3 else 1 

214 for v in range(v1, v2 + 1, v3): 

215 outList.append(int(v)) 

216 else: 

217 outList.append(int(subString)) 

218 return outList 

219 

220 

221class SingleVisitListTaskRunner(pipeBase.TaskRunner): 

222 """Subclass of TaskRunner for tasks requiring a list of visits per dataRef. 

223 

224 This transforms the processed arguments generated by the ArgumentParser 

225 into the arguments expected by tasks which require a list of visits 

226 to be supplied for each dataRef, as is common in `lsst.cp.pipe` code. 

227 

228 Such tasks' run() methods tend to take two arguments, 

229 one of which is the dataRef (as usual), and the other is the list 

230 of visits. 

231 This list is supplied on the command line as documented, 

232 and this class parses that, and passes the parsed version 

233 to the run() method. 

234 

235 See `lsst.pipe.base.TaskRunner` for more information. 

236 """ 

237 

238 @staticmethod 

239 def getTargetList(parsedCmd, **kwargs): 

240 """Parse the visit list and pass through explicitly.""" 

241 # if this has been pre-parsed and therefore doesn't have length of one 

242 # then something has gone wrong, so execution should stop here. 

243 assert len(parsedCmd.visitList) == 1, 'visitList parsing assumptions violated' 

244 visits = parseCmdlineNumberString(parsedCmd.visitList[0]) 

245 

246 return pipeBase.TaskRunner.getTargetList(parsedCmd, visitList=visits, **kwargs) 

247 

248 

249class NonexistentDatasetTaskDataIdContainer(pipeBase.DataIdContainer): 

250 """A DataIdContainer for the tasks for which the output does 

251 not yet exist.""" 

252 

253 def makeDataRefList(self, namespace): 

254 """Compute refList based on idList. 

255 

256 This method must be defined as the dataset does not exist before this 

257 task is run. 

258 

259 Parameters 

260 ---------- 

261 namespace 

262 Results of parsing the command-line. 

263 

264 Notes 

265 ----- 

266 Not called if ``add_id_argument`` called 

267 with ``doMakeDataRefList=False``. 

268 Note that this is almost a copy-and-paste of the vanilla 

269 implementation, but without checking if the datasets already exist, 

270 as this task exists to make them. 

271 """ 

272 if self.datasetType is None: 

273 raise RuntimeError("Must call setDatasetType first") 

274 butler = namespace.butler 

275 for dataId in self.idList: 

276 refList = list(butler.subset(datasetType=self.datasetType, level=self.level, dataId=dataId)) 

277 # exclude nonexistent data 

278 # this is a recursive test, e.g. for the sake of "raw" data 

279 if not refList: 

280 namespace.log.warn("No data found for dataId=%s", dataId) 

281 continue 

282 self.refList += refList 

283 

284 

285def fitLeastSq(initialParams, dataX, dataY, function, weightsY=None): 

286 """Do a fit and estimate the parameter errors using using scipy.optimize.leastq. 

287 

288 optimize.leastsq returns the fractional covariance matrix. To estimate the 

289 standard deviation of the fit parameters, multiply the entries of this matrix 

290 by the unweighted reduced chi squared and take the square root of the diagonal elements. 

291 

292 Parameters 

293 ---------- 

294 initialParams : `list` of `float` 

295 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length 

296 determines the degree of the polynomial. 

297 

298 dataX : `numpy.array` of `float` 

299 Data in the abscissa axis. 

300 

301 dataY : `numpy.array` of `float` 

302 Data in the ordinate axis. 

303 

304 function : callable object (function) 

305 Function to fit the data with. 

306 

307 weightsY : `numpy.array` of `float` 

308 Weights of the data in the ordinate axis. 

309 

310 Return 

311 ------ 

312 pFitSingleLeastSquares : `list` of `float` 

313 List with fitted parameters. 

314 

315 pErrSingleLeastSquares : `list` of `float` 

316 List with errors for fitted parameters. 

317 

318 reducedChiSqSingleLeastSquares : `float` 

319 Reduced chi squared, unweighted if weightsY is not provided. 

320 """ 

321 if weightsY is None: 

322 weightsY = np.ones(len(dataX)) 

323 

324 def errFunc(p, x, y, weightsY=None): 

325 if weightsY is None: 

326 weightsY = np.ones(len(x)) 

327 return (function(p, x) - y)*weightsY 

328 

329 pFit, pCov, infoDict, errMessage, success = leastsq(errFunc, initialParams, 

330 args=(dataX, dataY, weightsY), full_output=1, 

331 epsfcn=0.0001) 

332 

333 if (len(dataY) > len(initialParams)) and pCov is not None: 

334 reducedChiSq = calculateWeightedReducedChi2(dataY, function(pFit, dataX), weightsY, len(dataY), 

335 len(initialParams)) 

336 pCov *= reducedChiSq 

337 else: 

338 pCov = np.zeros((len(initialParams), len(initialParams))) 

339 pCov[:, :] = np.nan 

340 reducedChiSq = np.nan 

341 

342 errorVec = [] 

343 for i in range(len(pFit)): 

344 errorVec.append(np.fabs(pCov[i][i])**0.5) 

345 

346 pFitSingleLeastSquares = pFit 

347 pErrSingleLeastSquares = np.array(errorVec) 

348 

349 return pFitSingleLeastSquares, pErrSingleLeastSquares, reducedChiSq 

350 

351 

352def fitBootstrap(initialParams, dataX, dataY, function, weightsY=None, confidenceSigma=1.): 

353 """Do a fit using least squares and bootstrap to estimate parameter errors. 

354 

355 The bootstrap error bars are calculated by fitting 100 random data sets. 

356 

357 Parameters 

358 ---------- 

359 initialParams : `list` of `float` 

360 initial values for fit parameters. For ptcFitType=POLYNOMIAL, its length 

361 determines the degree of the polynomial. 

362 

363 dataX : `numpy.array` of `float` 

364 Data in the abscissa axis. 

365 

366 dataY : `numpy.array` of `float` 

367 Data in the ordinate axis. 

368 

369 function : callable object (function) 

370 Function to fit the data with. 

371 

372 weightsY : `numpy.array` of `float`, optional. 

373 Weights of the data in the ordinate axis. 

374 

375 confidenceSigma : `float`, optional. 

376 Number of sigmas that determine confidence interval for the bootstrap errors. 

377 

378 Return 

379 ------ 

380 pFitBootstrap : `list` of `float` 

381 List with fitted parameters. 

382 

383 pErrBootstrap : `list` of `float` 

384 List with errors for fitted parameters. 

385 

386 reducedChiSqBootstrap : `float` 

387 Reduced chi squared, unweighted if weightsY is not provided. 

388 """ 

389 if weightsY is None: 

390 weightsY = np.ones(len(dataX)) 

391 

392 def errFunc(p, x, y, weightsY): 

393 if weightsY is None: 

394 weightsY = np.ones(len(x)) 

395 return (function(p, x) - y)*weightsY 

396 

397 # Fit first time 

398 pFit, _ = leastsq(errFunc, initialParams, args=(dataX, dataY, weightsY), full_output=0) 

399 

400 # Get the stdev of the residuals 

401 residuals = errFunc(pFit, dataX, dataY, weightsY) 

402 # 100 random data sets are generated and fitted 

403 pars = [] 

404 for i in range(100): 

405 randomDelta = np.random.normal(0., np.fabs(residuals), len(dataY)) 

406 randomDataY = dataY + randomDelta 

407 randomFit, _ = leastsq(errFunc, initialParams, 

408 args=(dataX, randomDataY, weightsY), full_output=0) 

409 pars.append(randomFit) 

410 pars = np.array(pars) 

411 meanPfit = np.mean(pars, 0) 

412 

413 # confidence interval for parameter estimates 

414 errPfit = confidenceSigma*np.std(pars, 0) 

415 pFitBootstrap = meanPfit 

416 pErrBootstrap = errPfit 

417 

418 reducedChiSq = calculateWeightedReducedChi2(dataY, function(pFitBootstrap, dataX), weightsY, len(dataY), 

419 len(initialParams)) 

420 return pFitBootstrap, pErrBootstrap, reducedChiSq 

421 

422 

423def funcPolynomial(pars, x): 

424 """Polynomial function definition 

425 Parameters 

426 ---------- 

427 params : `list` 

428 Polynomial coefficients. Its length determines the polynomial order. 

429 

430 x : `numpy.array` 

431 Signal mu (ADU). 

432 

433 Returns 

434 ------- 

435 C_00 (variance) in ADU^2. 

436 """ 

437 return poly.polyval(x, [*pars]) # C_00 

438 

439 

440def funcAstier(pars, x): 

441 """Single brighter-fatter parameter model for PTC; Equation 16 of Astier+19. 

442 

443 Parameters 

444 ---------- 

445 params : `list` 

446 Parameters of the model: a00 (brightter-fatter), gain (e/ADU), and noise (e^2). 

447 

448 x : `numpy.array` 

449 Signal mu (ADU). 

450 

451 Returns 

452 ------- 

453 C_00 (variance) in ADU^2. 

454 """ 

455 a00, gain, noise = pars 

456 return 0.5/(a00*gain*gain)*(np.exp(2*a00*x*gain)-1) + noise/(gain*gain) # C_00 

457 

458 

459def checkExpLengthEqual(exp1, exp2, v1=None, v2=None, raiseWithMessage=False): 

460 """Check the exposure lengths of two exposures are equal. 

461 

462 Parameters: 

463 ----------- 

464 exp1 : `lsst.afw.image.exposure.ExposureF` 

465 First exposure to check 

466 exp2 : `lsst.afw.image.exposure.ExposureF` 

467 Second exposure to check 

468 v1 : `int` or `str`, optional 

469 First visit of the visit pair 

470 v2 : `int` or `str`, optional 

471 Second visit of the visit pair 

472 raiseWithMessage : `bool` 

473 If True, instead of returning a bool, raise a RuntimeError if exposure 

474 times are not equal, with a message about which visits mismatch if the 

475 information is available. 

476 

477 Raises: 

478 ------- 

479 RuntimeError 

480 Raised if the exposure lengths of the two exposures are not equal 

481 """ 

482 expTime1 = exp1.getInfo().getVisitInfo().getExposureTime() 

483 expTime2 = exp2.getInfo().getVisitInfo().getExposureTime() 

484 if expTime1 != expTime2: 

485 if raiseWithMessage: 

486 msg = "Exposure lengths for visit pairs must be equal. " + \ 

487 "Found %s and %s" % (expTime1, expTime2) 

488 if v1 and v2: 

489 msg += " for visit pair %s, %s" % (v1, v2) 

490 raise RuntimeError(msg) 

491 else: 

492 return False 

493 return True 

494 

495 

496def validateIsrConfig(isrTask, mandatory=None, forbidden=None, desirable=None, undesirable=None, 

497 checkTrim=True, logName=None): 

498 """Check that appropriate ISR settings have been selected for the task. 

499 

500 Note that this checks that the task itself is configured correctly rather 

501 than checking a config. 

502 

503 Parameters 

504 ---------- 

505 isrTask : `lsst.ip.isr.IsrTask` 

506 The task whose config is to be validated 

507 

508 mandatory : `iterable` of `str` 

509 isr steps that must be set to True. Raises if False or missing 

510 

511 forbidden : `iterable` of `str` 

512 isr steps that must be set to False. Raises if True, warns if missing 

513 

514 desirable : `iterable` of `str` 

515 isr steps that should probably be set to True. Warns is False, info if 

516 missing 

517 

518 undesirable : `iterable` of `str` 

519 isr steps that should probably be set to False. Warns is True, info if 

520 missing 

521 

522 checkTrim : `bool` 

523 Check to ensure the isrTask's assembly subtask is trimming the images. 

524 This is a separate config as it is very ugly to do this within the 

525 normal configuration lists as it is an option of a sub task. 

526 

527 Raises 

528 ------ 

529 RuntimeError 

530 Raised if ``mandatory`` config parameters are False, 

531 or if ``forbidden`` parameters are True. 

532 

533 TypeError 

534 Raised if parameter ``isrTask`` is an invalid type. 

535 

536 Notes 

537 ----- 

538 Logs warnings using an isrValidation logger for desirable/undesirable 

539 options that are of the wrong polarity or if keys are missing. 

540 """ 

541 if not isinstance(isrTask, ipIsr.IsrTask): 

542 raise TypeError(f'Must supply an instance of lsst.ip.isr.IsrTask not {type(isrTask)}') 

543 

544 configDict = isrTask.config.toDict() 

545 

546 if logName and isinstance(logName, str): 

547 log = lsst.log.getLogger(logName) 

548 else: 

549 log = lsst.log.getLogger("isrValidation") 

550 

551 if mandatory: 

552 for configParam in mandatory: 

553 if configParam not in configDict: 

554 raise RuntimeError(f"Mandatory parameter {configParam} not found in the isr configuration.") 

555 if configDict[configParam] is False: 

556 raise RuntimeError(f"Must set config.isr.{configParam} to True for this task.") 

557 

558 if forbidden: 

559 for configParam in forbidden: 

560 if configParam not in configDict: 

561 log.warn(f"Failed to find forbidden key {configParam} in the isr config. The keys in the" 

562 " forbidden list should each have an associated Field in IsrConfig:" 

563 " check that there is not a typo in this case.") 

564 continue 

565 if configDict[configParam] is True: 

566 raise RuntimeError(f"Must set config.isr.{configParam} to False for this task.") 

567 

568 if desirable: 

569 for configParam in desirable: 

570 if configParam not in configDict: 

571 log.info(f"Failed to find key {configParam} in the isr config. You probably want" + 

572 " to set the equivalent for your obs_package to True.") 

573 continue 

574 if configDict[configParam] is False: 

575 log.warn(f"Found config.isr.{configParam} set to False for this task." + 

576 " The cp_pipe Config recommends setting this to True.") 

577 if undesirable: 

578 for configParam in undesirable: 

579 if configParam not in configDict: 

580 log.info(f"Failed to find key {configParam} in the isr config. You probably want" + 

581 " to set the equivalent for your obs_package to False.") 

582 continue 

583 if configDict[configParam] is True: 

584 log.warn(f"Found config.isr.{configParam} set to True for this task." + 

585 " The cp_pipe Config recommends setting this to False.") 

586 

587 if checkTrim: # subtask setting, seems non-trivial to combine with above lists 

588 if not isrTask.assembleCcd.config.doTrim: 

589 raise RuntimeError("Must trim when assembling CCDs. Set config.isr.assembleCcd.doTrim to True")