Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_verify. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21import math 

22 

23import lsst.afw.geom as afwGeom 

24import lsst.afw.math as afwMath 

25import lsst.pex.config as pexConfig 

26import lsst.pipe.base as pipeBase 

27import lsst.pipe.base.connectionTypes as cT 

28import lsst.meas.algorithms as measAlg 

29 

30from lsst.cp.pipe.cpCombine import vignetteExposure 

31from lsst.pipe.tasks.repair import RepairTask 

32from .utils import mergeStatDict 

33 

34__all__ = ['CpVerifyStatsConfig', 'CpVerifyStatsTask'] 

35 

36 

37class CpVerifyStatsConnections(pipeBase.PipelineTaskConnections, 

38 dimensions={"instrument", "exposure", "detector"}, 

39 defaultTemplates={}): 

40 inputExp = cT.Input( 

41 name="postISRCCD", 

42 doc="Input exposure to calculate statistics for.", 

43 storageClass="Exposure", 

44 dimensions=["instrument", "exposure", "detector"], 

45 ) 

46 camera = cT.PrerequisiteInput( 

47 name="camera", 

48 storageClass="Camera", 

49 doc="Input camera.", 

50 dimensions=["instrument", ], 

51 isCalibration=True, 

52 ) 

53 

54 outputStats = cT.Output( 

55 name="detectorStats", 

56 doc="Output statistics from cp_verify.", 

57 storageClass="StructuredDataDict", 

58 dimensions=["instrument", "exposure", "detector"], 

59 ) 

60 

61 

62class CpVerifyStatsConfig(pipeBase.PipelineTaskConfig, 

63 pipelineConnections=CpVerifyStatsConnections): 

64 """Configuration parameters for CpVerifyStatsTask. 

65 """ 

66 maskNameList = pexConfig.ListField( 

67 dtype=str, 

68 doc="Mask list to exclude from statistics calculations.", 

69 default=['DETECTED', 'BAD', 'NO_DATA'], 

70 ) 

71 doVignette = pexConfig.Field( 

72 dtype=bool, 

73 doc="Mask vignetted regions?", 

74 default=False, 

75 ) 

76 doNormalize = pexConfig.Field( 

77 dtype=bool, 

78 doc="Normalize by exposure time?", 

79 default=False, 

80 ) 

81 

82 # Cosmic ray handling options. 

83 doCR = pexConfig.Field( 

84 dtype=bool, 

85 doc="Run CR rejection?", 

86 default=False, 

87 ) 

88 repair = pexConfig.ConfigurableField( 

89 target=RepairTask, 

90 doc="Repair task to use.", 

91 ) 

92 psfFwhm = pexConfig.Field( 

93 dtype=float, 

94 default=3.0, 

95 doc="Repair PSF FWHM (pixels).", 

96 ) 

97 psfSize = pexConfig.Field( 

98 dtype=int, 

99 default=21, 

100 doc="Repair PSF bounding-box size (pixels).", 

101 ) 

102 crGrow = pexConfig.Field( 

103 dtype=int, 

104 default=2, 

105 doc="Grow radius for CR (pixels).", 

106 ) 

107 

108 # Statistics options. 

109 useReadNoise = pexConfig.Field( 

110 dtype=bool, 

111 doc="Compare sigma against read noise?", 

112 default=True, 

113 ) 

114 numSigmaClip = pexConfig.Field( 

115 dtype=float, 

116 doc="Rejection threshold (sigma) for statistics clipping.", 

117 default=5.0, 

118 ) 

119 clipMaxIter = pexConfig.Field( 

120 dtype=int, 

121 doc="Max number of clipping iterations to apply.", 

122 default=3, 

123 ) 

124 

125 # Keywords and statistics to measure from different sources. 

126 imageStatKeywords = pexConfig.DictField( 

127 keytype=str, 

128 itemtype=str, 

129 doc="Image statistics to run on amplifier segments.", 

130 default={}, 

131 ) 

132 unmaskedImageStatKeywords = pexConfig.DictField( 

133 keytype=str, 

134 itemtype=str, 

135 doc="Image statistics to run on amplifier segments, ignoring masks.", 

136 default={}, 

137 ) 

138 crImageStatKeywords = pexConfig.DictField( 

139 keytype=str, 

140 itemtype=str, 

141 doc="Image statistics to run on CR cleaned amplifier segments.", 

142 default={}, 

143 ) 

144 normImageStatKeywords = pexConfig.DictField( 

145 keytype=str, 

146 itemtype=str, 

147 doc="Image statistics to run on expTime normalized amplifier segments.", 

148 default={}, 

149 ) 

150 catalogStatKeywords = pexConfig.DictField( 

151 keytype=str, 

152 itemtype=str, 

153 doc="Statistics to measure from source catalogs of objects in the exposure.", 

154 default={}, 

155 ) 

156 detectorStatKeywords = pexConfig.DictField( 

157 keytype=str, 

158 itemtype=str, 

159 doc="Statistics to create for the full detector from the per-amplifier measurements.", 

160 default={}, 

161 ) 

162 

163 

164class CpVerifyStatsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask): 

165 """Main statistic measurement and validation class. 

166 

167 This operates on a single (exposure, detector) pair, and is 

168 designed to be subclassed so specific calibrations can apply their 

169 own validation methods. 

170 """ 

171 ConfigClass = CpVerifyStatsConfig 

172 _DefaultName = 'cpVerifyStats' 

173 

174 def __init__(self, **kwargs): 

175 super().__init__(**kwargs) 

176 self.makeSubtask("repair") 

177 

178 def run(self, inputExp, camera): 

179 """Calculate quality statistics and verify they meet the requirements 

180 for a calibration. 

181 

182 Parameters 

183 ---------- 

184 inputExp : `lsst.afw.image.Exposure` 

185 The ISR processed exposure to be measured. 

186 camera : `lsst.afw.cameraGeom.Camera` 

187 The camera geometry for ``inputExp``. 

188 

189 Returns 

190 ------- 

191 result : `lsst.pipe.base.Struct` 

192 Result struct with components: 

193 - ``outputStats`` : `dict` 

194 The output measured statistics. 

195 

196 Notes 

197 ----- 

198 The outputStats should have a yaml representation of the form 

199 

200 AMP: 

201 Amp1: 

202 STAT: value 

203 STAT2: value2 

204 Amp2: 

205 Amp3: 

206 DET: 

207 STAT: value 

208 STAT2: value 

209 CATALOG: 

210 STAT: value 

211 STAT2: value 

212 VERIFY: 

213 DET: 

214 TEST: boolean 

215 CATALOG: 

216 TEST: boolean 

217 AMP: 

218 Amp1: 

219 TEST: boolean 

220 TEST2: boolean 

221 Amp2: 

222 Amp3: 

223 SUCCESS: boolean 

224 

225 """ 

226 outputStats = {} 

227 

228 if self.config.doVignette: 

229 vignetteExposure(inputExp, doUpdateMask=True, maskPlane='NO_DATA', 

230 doSetValue=False, log=self.log) 

231 

232 mask = inputExp.getMask() 

233 maskVal = mask.getPlaneBitMask(self.config.maskNameList) 

234 statControl = afwMath.StatisticsControl(self.config.numSigmaClip, 

235 self.config.clipMaxIter, 

236 maskVal) 

237 

238 # This is wrapped below to check for config lengths, as we can 

239 # make a number of different image stats. 

240 outputStats['AMP'] = self.imageStatistics(inputExp, statControl) 

241 if len(self.config.catalogStatKeywords): 

242 outputStats['CATALOG'] = self.catalogStatistics(inputExp, statControl) 

243 else: 

244 outputStats['CATALOG'] = {} 

245 if len(self.config.detectorStatKeywords): 

246 outputStats['DET'] = self.detectorStatistics(outputStats, statControl) 

247 else: 

248 outputStats['DET'] = {} 

249 

250 outputStats['VERIFY'], outputStats['SUCCESS'] = self.verify(inputExp, outputStats) 

251 

252 return pipeBase.Struct( 

253 outputStats=outputStats, 

254 ) 

255 

256 @staticmethod 

257 def _emptyAmpDict(exposure): 

258 """Construct empty dictionary indexed by amplifier names. 

259 

260 Parameters 

261 ---------- 

262 exposure : `lsst.afw.image.Exposure` 

263 Exposure to extract detector from. 

264 

265 Returns 

266 ------- 

267 outputStatistics : `dict` [`str`, `dict`] 

268 A skeleton statistics dictionary. 

269 

270 Raises 

271 ------ 

272 RuntimeError : 

273 Raised if no detector can be found. 

274 """ 

275 outputStatistics = {} 

276 detector = exposure.getDetector() 

277 if detector is None: 

278 raise RuntimeError("No detector found in exposure!") 

279 

280 for amp in detector.getAmplifiers(): 

281 outputStatistics[amp.getName()] = {} 

282 

283 return outputStatistics 

284 

285 # Image measurement methods. 

286 def imageStatistics(self, exposure, statControl): 

287 """Measure image statistics for a number of simple image 

288 modifications. 

289 

290 Parameters 

291 ---------- 

292 exposure : `lsst.afw.image.Exposure` 

293 Exposure containing the ISR processed data to measure. 

294 statControl : `lsst.afw.math.StatisticsControl` 

295 Statistics control object with parameters defined by 

296 the config. 

297 

298 Returns 

299 ------- 

300 outputStatistics : `dict` [`str`, `dict` [`str`, scalar]] 

301 A dictionary indexed by the amplifier name, containing 

302 dictionaries of the statistics measured and their values. 

303 

304 """ 

305 outputStatistics = self._emptyAmpDict(exposure) 

306 

307 if len(self.config.imageStatKeywords): 

308 outputStatistics = mergeStatDict(outputStatistics, 

309 self.amplifierStats(exposure, 

310 self.config.imageStatKeywords, 

311 statControl)) 

312 

313 if len(self.config.unmaskedImageStatKeywords): 

314 outputStatistics = mergeStatDict(outputStatistics, self.unmaskedImageStats(exposure)) 

315 

316 if len(self.config.normImageStatKeywords): 

317 outputStatistics = mergeStatDict(outputStatistics, 

318 self.normalizedImageStats(exposure, statControl)) 

319 

320 if len(self.config.crImageStatKeywords): 

321 outputStatistics = mergeStatDict(outputStatistics, 

322 self.crImageStats(exposure, statControl)) 

323 

324 return outputStatistics 

325 

326 def amplifierStats(self, exposure, keywordDict, statControl): 

327 """Measure amplifier level statistics from the exposure. 

328 

329 Parameters 

330 ---------- 

331 exposure : `lsst.afw.image.Exposure` 

332 The exposure to measure. 

333 keywordDict : `dict` [`str`, `str`] 

334 A dictionary of keys to use in the output results, with 

335 values the string name associated with the 

336 `lsst.afw.math.statistics.Property` to measure. 

337 statControl : `lsst.afw.math.StatisticsControl` 

338 Statistics control object with parameters defined by 

339 the config. 

340 

341 Returns 

342 ------- 

343 ampStats : `dict` [`str`, `dict` [`str`, scalar]] 

344 A dictionary indexed by the amplifier name, containing 

345 dictionaries of the statistics measured and their values. 

346 """ 

347 ampStats = {} 

348 

349 # Convert the string names in the keywordDict to the afwMath values. 

350 # The statisticToRun is then the bitwise-or of that set. 

351 statisticToRun = 0 

352 statAccessor = {} 

353 for k, v in keywordDict.items(): 

354 statValue = afwMath.stringToStatisticsProperty(v) 

355 statisticToRun |= statValue 

356 statAccessor[k] = statValue 

357 

358 # Measure stats on all amplifiers. 

359 for ampIdx, amp in enumerate(exposure.getDetector()): 

360 ampName = amp.getName() 

361 theseStats = {} 

362 ampExp = exposure.Factory(exposure, amp.getBBox()) 

363 stats = afwMath.makeStatistics(ampExp.getMaskedImage(), statisticToRun, statControl) 

364 

365 for k, v in statAccessor.items(): 

366 theseStats[k] = stats.getValue(v) 

367 ampStats[ampName] = theseStats 

368 

369 return ampStats 

370 

371 def unmaskedImageStats(self, exposure): 

372 """Measure amplifier level statistics on the exposure, including all 

373 pixels in the exposure, regardless of any mask planes set. 

374 

375 Parameters 

376 ---------- 

377 exposure : `lsst.afw.image.Exposure` 

378 The exposure to measure. 

379 

380 Returns 

381 ------- 

382 outputStatistics : `dict` [`str`, `dict` [`str`, scalar]] 

383 A dictionary indexed by the amplifier name, containing 

384 dictionaries of the statistics measured and their values. 

385 """ 

386 noMaskStatsControl = afwMath.StatisticsControl(self.config.numSigmaClip, 

387 self.config.clipMaxIter, 

388 0x0) 

389 return self.amplifierStats(exposure, self.config.unmaskedImageStatKeywords, noMaskStatsControl) 

390 

391 def normalizedImageStats(self, exposure, statControl): 

392 """Measure amplifier level statistics on the exposure after dividing 

393 by the exposure time. 

394 

395 Parameters 

396 ---------- 

397 exposure : `lsst.afw.image.Exposure` 

398 The exposure to measure. 

399 statControl : `lsst.afw.math.StatisticsControl` 

400 Statistics control object with parameters defined by 

401 the config. 

402 

403 Returns 

404 ------- 

405 outputStatistics : `dict` [`str`, `dict` [`str`, scalar]] 

406 A dictionary indexed by the amplifier name, containing 

407 dictionaries of the statistics measured and their values. 

408 

409 Raises 

410 ------ 

411 RuntimeError : 

412 Raised if the exposure time cannot be used for normalization. 

413 """ 

414 scaledExposure = exposure.clone() 

415 exposureTime = scaledExposure.getInfo().getVisitInfo().getExposureTime() 

416 if exposureTime <= 0: 

417 raise RuntimeError(f"Invalid exposureTime {exposureTime}.") 

418 mi = scaledExposure.getMaskedImage() 

419 mi /= exposureTime 

420 

421 return self.amplifierStats(scaledExposure, self.config.normImageStatKeywords, statControl) 

422 

423 def crImageStats(self, exposure, statControl): 

424 """Measure amplifier level statistics on the exposure, 

425 after running cosmic ray rejection. 

426 

427 Parameters 

428 ---------- 

429 exposure : `lsst.afw.image.Exposure` 

430 The exposure to measure. 

431 statControl : `lsst.afw.math.StatisticsControl` 

432 Statistics control object with parameters defined by 

433 the config. 

434 

435 Returns 

436 ------- 

437 outputStatistics : `dict` [`str`, `dict` [`str`, scalar]] 

438 A dictionary indexed by the amplifier name, containing 

439 dictionaries of the statistics measured and their values. 

440 

441 """ 

442 crRejectedExp = exposure.clone() 

443 psf = measAlg.SingleGaussianPsf(self.config.psfSize, 

444 self.config.psfSize, 

445 self.config.psfFwhm/(2*math.sqrt(2*math.log(2)))) 

446 crRejectedExp.setPsf(psf) 

447 self.repair.run(crRejectedExp, keepCRs=False) 

448 if self.config.crGrow > 0: 

449 crMask = crRejectedExp.getMaskedImage().getMask().getPlaneBitMask("CR") 

450 spans = afwGeom.SpanSet.fromMask(crRejectedExp.mask, crMask) 

451 spans = spans.dilated(self.config.crGrow) 

452 spans.setMask(crRejectedExp.mask, crMask) 

453 

454 return self.amplifierStats(crRejectedExp, self.config.crImageStatKeywords, statControl) 

455 

456 # Methods that need to be implemented by the calibration-level subclasses. 

457 def catalogStatistics(self, exposure, statControl): 

458 """Calculate statistics from a catalog. 

459 

460 Parameters 

461 ---------- 

462 exposure : `lsst.afw.image.Exposure` 

463 The exposure to measure. 

464 statControl : `lsst.afw.math.StatisticsControl` 

465 Statistics control object with parameters defined by 

466 the config. 

467 

468 Returns 

469 ------- 

470 outputStatistics : `dict` [`str`, `dict` [`str`, scalar]] 

471 A dictionary indexed by the amplifier name, containing 

472 dictionaries of the statistics measured and their values. 

473 """ 

474 raise NotImplementedError("Subclasses must implement catalog statistics method.") 

475 

476 def detectorStatistics(self, statisticsDict, statControl): 

477 """Calculate detector level statistics based on the existing 

478 per-amplifier measurements. 

479 

480 Parameters 

481 ---------- 

482 statisticsDictionary : `dict` [`str`, `dict` [`str`, scalar]], 

483 Dictionary of measured statistics. The inner dictionary 

484 should have keys that are statistic names (`str`) with 

485 values that are some sort of scalar (`int` or `float` are 

486 the mostly likely types). 

487 

488 Returns 

489 ------- 

490 outputStatistics : `dict` [`str`, scalar] 

491 A dictionary of the statistics measured and their values. 

492 

493 Raises 

494 ------ 

495 NotImplementedError : 

496 This method must be implemented by the calibration-type 

497 subclass. 

498 """ 

499 raise NotImplementedError("Subclasses must implement detector statistics method.") 

500 

501 def verify(self, exposure, statisticsDict): 

502 """Verify that the measured statistics meet the verification criteria. 

503 

504 Parameters 

505 ---------- 

506 exposure : `lsst.afw.image.Exposure` 

507 The exposure the statistics are from. 

508 statisticsDictionary : `dict` [`str`, `dict` [`str`, scalar]], 

509 Dictionary of measured statistics. The inner dictionary 

510 should have keys that are statistic names (`str`) with 

511 values that are some sort of scalar (`int` or `float` are 

512 the mostly likely types). 

513 

514 Returns 

515 ------- 

516 outputStatistics : `dict` [`str`, `dict` [`str`, `bool`]] 

517 A dictionary indexed by the amplifier name, containing 

518 dictionaries of the verification criteria. 

519 success : `bool` 

520 A boolean indicating whether all tests have passed. 

521 

522 Raises 

523 ------ 

524 NotImplementedError : 

525 This method must be implemented by the calibration-type 

526 subclass. 

527 """ 

528 raise NotImplementedError("Subclasses must implement verification criteria.")