Coverage for python/lsst/cp/verify/verifyStats.py: 25%

215 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-08 12:14 +0000

1# This file is part of cp_verify. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21import numpy as np 

22 

23from astropy.table import Table 

24 

25import lsst.afw.geom as afwGeom 

26import lsst.afw.math as afwMath 

27import lsst.pex.config as pexConfig 

28import lsst.pex.exceptions as pexException 

29import lsst.pipe.base as pipeBase 

30import lsst.pipe.base.connectionTypes as cT 

31import lsst.meas.algorithms as measAlg 

32 

33from lsst.ip.isr.vignette import maskVignettedRegion 

34from lsst.pipe.tasks.repair import RepairTask 

35from .utils import mergeStatDict 

36 

37 

38__all__ = ["CpVerifyStatsConfig", "CpVerifyStatsTask"] 

39 

40 

41class CpVerifyStatsConnections( 

42 pipeBase.PipelineTaskConnections, 

43 dimensions={"instrument", "exposure", "detector"}, 

44 defaultTemplates={}, 

45): 

46 inputExp = cT.Input( 

47 name="postISRCCD", 

48 doc="Input exposure to calculate statistics for.", 

49 storageClass="Exposure", 

50 dimensions=["instrument", "exposure", "detector"], 

51 ) 

52 uncorrectedExp = cT.Input( 

53 name="uncorrectedExp", 

54 doc="Uncorrected input exposure to calculate statistics for.", 

55 storageClass="ExposureF", 

56 dimensions=["instrument", "visit", "detector"], 

57 ) 

58 taskMetadata = cT.Input( 

59 name="isrTask_metadata", 

60 doc="Input task metadata to extract statistics from.", 

61 storageClass="TaskMetadata", 

62 dimensions=["instrument", "exposure", "detector"], 

63 ) 

64 inputCatalog = cT.Input( 

65 name="src", 

66 doc="Input catalog to calculate statistics for.", 

67 storageClass="SourceCatalog", 

68 dimensions=["instrument", "visit", "detector"], 

69 ) 

70 uncorrectedCatalog = cT.Input( 

71 name="uncorrectedSrc", 

72 doc="Input catalog without correction applied.", 

73 storageClass="SourceCatalog", 

74 dimensions=["instrument", "visit", "detector"], 

75 ) 

76 camera = cT.PrerequisiteInput( 

77 name="camera", 

78 storageClass="Camera", 

79 doc="Input camera.", 

80 dimensions=["instrument", ], 

81 isCalibration=True, 

82 ) 

83 isrStatistics = cT.Input( 

84 name="isrStatistics", 

85 storageClass="StructuredDataDict", 

86 doc="Pre-calculated statistics from IsrTask.", 

87 dimensions=["instrument", "exposure", "detector"], 

88 ) 

89 

90 outputStats = cT.Output( 

91 name="detectorStats", 

92 doc="Output statistics from cp_verify.", 

93 storageClass="StructuredDataDict", 

94 dimensions=["instrument", "exposure", "detector"], 

95 ) 

96 outputResults = cT.Output( 

97 name="detectorResults", 

98 doc="Output results from cp_verify.", 

99 storageClass="ArrowAstropy", 

100 dimensions=["instrument", "exposure", "detector"], 

101 ) 

102 outputMatrix = cT.Output( 

103 name="detectorMatrix", 

104 doc="Output matrix results from cp_verify.", 

105 storageClass="ArrowAstropy", 

106 dimensions=["instrument", "exposure", "detector"], 

107 ) 

108 

109 def __init__(self, *, config=None): 

110 super().__init__(config=config) 

111 

112 if len(config.metadataStatKeywords) < 1: 

113 self.inputs.discard("taskMetadata") 

114 

115 if len(config.catalogStatKeywords) < 1: 

116 self.inputs.discard("inputCatalog") 

117 self.inputs.discard("uncorrectedCatalog") 

118 

119 if len(config.uncorrectedImageStatKeywords) < 1: 

120 self.inputs.discard("uncorrectedExp") 

121 

122 if config.useIsrStatistics is not True: 

123 self.inputs.discard("isrStatistics") 

124 

125 if not config.hasMatrixCatalog: 

126 self.outputs.discard("outputMatrix") 

127 

128 

129class CpVerifyStatsConfig( 

130 pipeBase.PipelineTaskConfig, pipelineConnections=CpVerifyStatsConnections 

131): 

132 """Configuration parameters for CpVerifyStatsTask.""" 

133 

134 maskNameList = pexConfig.ListField( 

135 dtype=str, 

136 doc="Mask list to exclude from statistics calculations.", 

137 default=["DETECTED", "BAD", "NO_DATA"], 

138 ) 

139 doVignette = pexConfig.Field( 

140 dtype=bool, 

141 doc="Mask vignetted regions?", 

142 default=False, 

143 ) 

144 doNormalize = pexConfig.Field( 

145 dtype=bool, 

146 doc="Normalize by exposure time?", 

147 default=False, 

148 ) 

149 

150 # Cosmic ray handling options. 

151 doCR = pexConfig.Field( 

152 dtype=bool, 

153 doc="Run CR rejection?", 

154 default=False, 

155 ) 

156 repair = pexConfig.ConfigurableField( 

157 target=RepairTask, 

158 doc="Repair task to use.", 

159 ) 

160 psfFwhm = pexConfig.Field( 

161 dtype=float, 

162 default=3.0, 

163 doc="Repair PSF FWHM (pixels).", 

164 ) 

165 psfSize = pexConfig.Field( 

166 dtype=int, 

167 default=21, 

168 doc="Repair PSF bounding-box size (pixels).", 

169 ) 

170 crGrow = pexConfig.Field( 

171 dtype=int, 

172 default=0, 

173 doc="Grow radius for CR (pixels).", 

174 ) 

175 

176 # Statistics options. 

177 useReadNoise = pexConfig.Field( 

178 dtype=bool, 

179 doc="Compare sigma against read noise?", 

180 default=True, 

181 ) 

182 numSigmaClip = pexConfig.Field( 

183 dtype=float, 

184 doc="Rejection threshold (sigma) for statistics clipping.", 

185 default=5.0, 

186 ) 

187 clipMaxIter = pexConfig.Field( 

188 dtype=int, 

189 doc="Max number of clipping iterations to apply.", 

190 default=3, 

191 ) 

192 

193 # Keywords and statistics to measure from different sources. 

194 imageStatKeywords = pexConfig.DictField( 

195 keytype=str, 

196 itemtype=str, 

197 doc="Image statistics to run on amplifier segments.", 

198 default={}, 

199 ) 

200 unmaskedImageStatKeywords = pexConfig.DictField( 

201 keytype=str, 

202 itemtype=str, 

203 doc="Image statistics to run on amplifier segments, ignoring masks.", 

204 default={}, 

205 ) 

206 uncorrectedImageStatKeywords = pexConfig.DictField( 

207 keytype=str, 

208 itemtype=str, 

209 doc="Uncorrected image statistics to run on amplifier segments.", 

210 default={}, 

211 ) 

212 crImageStatKeywords = pexConfig.DictField( 

213 keytype=str, 

214 itemtype=str, 

215 doc="Image statistics to run on CR cleaned amplifier segments.", 

216 default={}, 

217 ) 

218 normImageStatKeywords = pexConfig.DictField( 

219 keytype=str, 

220 itemtype=str, 

221 doc="Image statistics to run on expTime normalized amplifier segments.", 

222 default={}, 

223 ) 

224 metadataStatKeywords = pexConfig.DictField( 

225 keytype=str, 

226 itemtype=str, 

227 doc="Statistics to measure from the metadata of the exposure.", 

228 default={}, 

229 ) 

230 catalogStatKeywords = pexConfig.DictField( 

231 keytype=str, 

232 itemtype=str, 

233 doc="Statistics to measure from source catalogs of objects in the exposure.", 

234 default={}, 

235 ) 

236 detectorStatKeywords = pexConfig.DictField( 

237 keytype=str, 

238 itemtype=str, 

239 doc="Statistics to create for the full detector from the per-amplifier measurements.", 

240 default={}, 

241 ) 

242 

243 stageName = pexConfig.Field( 

244 dtype=str, 

245 doc="Stage name to use for table columns.", 

246 default="NOSTAGE", 

247 ) 

248 useIsrStatistics = pexConfig.Field( 

249 dtype=bool, 

250 doc="Use statistics calculated by IsrTask?", 

251 default=False, 

252 ) 

253 hasMatrixCatalog = pexConfig.Field( 

254 dtype=bool, 

255 doc="Will a matrix table of results be made?", 

256 default=False, 

257 ) 

258 expectedDistributionLevels = pexConfig.ListField( 

259 dtype=float, 

260 doc="Percentile levels expected in the calibration header.", 

261 default=[0, 5, 16, 50, 84, 95, 100], 

262 ) 

263 

264 

265class CpVerifyStatsTask(pipeBase.PipelineTask): 

266 """Main statistic measurement and validation class. 

267 

268 This operates on a single (exposure, detector) pair, and is 

269 designed to be subclassed so specific calibrations can apply their 

270 own validation methods. 

271 """ 

272 

273 ConfigClass = CpVerifyStatsConfig 

274 _DefaultName = "cpVerifyStats" 

275 

276 def __init__(self, **kwargs): 

277 super().__init__(**kwargs) 

278 self.makeSubtask("repair") 

279 

280 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

281 inputs = butlerQC.get(inputRefs) 

282 

283 # Pass the full dataId, as we want to retain filter info. 

284 inputs["dimensions"] = dict(inputRefs.inputExp.dataId.mapping) 

285 print("CZW", inputs["dimensions"], dict(inputRefs.inputExp.dataId.mapping)) 

286 # import pdb; pdb.set_trace() 

287 outputs = self.run(**inputs) 

288 butlerQC.put(outputs, outputRefs) 

289 

290 def run( 

291 self, 

292 inputExp, 

293 camera, 

294 isrStatistics=None, 

295 uncorrectedExp=None, 

296 taskMetadata=None, 

297 inputCatalog=None, 

298 uncorrectedCatalog=None, 

299 dimensions=None, 

300 ): 

301 """Calculate quality statistics and verify they meet the requirements 

302 for a calibration. 

303 

304 Parameters 

305 ---------- 

306 inputExp : `lsst.afw.image.Exposure` 

307 The ISR processed exposure to be measured. 

308 camera : `lsst.afw.cameraGeom.Camera` 

309 The camera geometry for ``inputExp``. 

310 uncorrectedExp : `lsst.afw.image.Exposure` 

311 The alternate exposure to measure. 

312 taskMetadata : `lsst.pipe.base.TaskMetadata`, optional 

313 Task metadata containing additional statistics. 

314 inputCatalog : `lsst.afw.image.Table` 

315 The source catalog to measure. 

316 uncorrectedCatalog : `lsst.afw.image.Table` 

317 The alternate source catalog to measure. 

318 dimensions : `dict` 

319 Dictionary of input dictionary. 

320 

321 Returns 

322 ------- 

323 result : `lsst.pipe.base.Struct` 

324 Result struct with components: 

325 - ``outputStats`` : `dict` 

326 The output measured statistics. 

327 """ 

328 outputStats = {} 

329 

330 if self.config.doVignette: 

331 polygon = inputExp.getInfo().getValidPolygon() 

332 maskVignettedRegion( 

333 inputExp, polygon, maskPlane="NO_DATA", vignetteValue=None, log=self.log 

334 ) 

335 

336 mask = inputExp.getMask() 

337 maskVal = mask.getPlaneBitMask(self.config.maskNameList) 

338 statControl = afwMath.StatisticsControl( 

339 self.config.numSigmaClip, self.config.clipMaxIter, maskVal 

340 ) 

341 

342 # This is wrapped below to check for config lengths, as we can 

343 # make a number of different image stats. 

344 outputStats["AMP"] = self.imageStatistics(inputExp, uncorrectedExp, statControl) 

345 

346 if len(self.config.metadataStatKeywords): 

347 # These are also defined on a amp-by-amp basis. 

348 outputStats["METADATA"] = self.metadataStatistics(inputExp, taskMetadata) 

349 else: 

350 outputStats["METADATA"] = {} 

351 

352 if len(self.config.catalogStatKeywords): 

353 outputStats["CATALOG"] = self.catalogStatistics( 

354 inputExp, inputCatalog, uncorrectedCatalog, statControl 

355 ) 

356 else: 

357 outputStats["CATALOG"] = {} 

358 if len(self.config.detectorStatKeywords): 

359 outputStats["DET"] = self.detectorStatistics( 

360 outputStats, statControl, inputExp, uncorrectedExp 

361 ) 

362 else: 

363 outputStats["DET"] = {} 

364 

365 if self.config.useIsrStatistics: 

366 outputStats["ISR"] = isrStatistics 

367 

368 outputStats["VERIFY"], outputStats["SUCCESS"] = self.verify( 

369 inputExp, outputStats 

370 ) 

371 

372 outputResults, outputMatrix = self.repackStats(outputStats, dimensions) 

373 

374 return pipeBase.Struct( 

375 outputStats=outputStats, 

376 outputResults=Table(outputResults), 

377 outputMatrix=Table(outputMatrix), 

378 ) 

379 

380 @staticmethod 

381 def _emptyAmpDict(exposure): 

382 """Construct empty dictionary indexed by amplifier names. 

383 

384 Parameters 

385 ---------- 

386 exposure : `lsst.afw.image.Exposure` 

387 Exposure to extract detector from. 

388 

389 Returns 

390 ------- 

391 outputStatistics : `dict` [`str`, `dict`] 

392 A skeleton statistics dictionary. 

393 

394 Raises 

395 ------ 

396 RuntimeError : 

397 Raised if no detector can be found. 

398 """ 

399 outputStatistics = {} 

400 detector = exposure.getDetector() 

401 if detector is None: 

402 raise RuntimeError("No detector found in exposure!") 

403 

404 for amp in detector.getAmplifiers(): 

405 outputStatistics[amp.getName()] = {} 

406 

407 return outputStatistics 

408 

409 # Image measurement methods. 

410 def imageStatistics(self, exposure, uncorrectedExposure, statControl): 

411 """Measure image statistics for a number of simple image 

412 modifications. 

413 

414 Parameters 

415 ---------- 

416 exposure : `lsst.afw.image.Exposure` 

417 Exposure containing the ISR processed data to measure. 

418 uncorrectedExposure: `lsst.afw.image.Exposure` 

419 Uncorrected exposure containing the ISR processed data to measure. 

420 statControl : `lsst.afw.math.StatisticsControl` 

421 Statistics control object with parameters defined by 

422 the config. 

423 

424 Returns 

425 ------- 

426 outputStatistics : `dict` [`str`, `dict` [`str`, scalar]] 

427 A dictionary indexed by the amplifier name, containing 

428 dictionaries of the statistics measured and their values. 

429 

430 """ 

431 outputStatistics = self._emptyAmpDict(exposure) 

432 

433 if len(self.config.imageStatKeywords): 

434 outputStatistics = mergeStatDict( 

435 outputStatistics, 

436 self.amplifierStats( 

437 exposure, self.config.imageStatKeywords, statControl 

438 ), 

439 ) 

440 if len(self.config.uncorrectedImageStatKeywords): 

441 outputStatistics = mergeStatDict( 

442 outputStatistics, 

443 self.amplifierStats( 

444 uncorrectedExposure, 

445 self.config.uncorrectedImageStatKeywords, 

446 statControl, 

447 ), 

448 ) 

449 if len(self.config.unmaskedImageStatKeywords): 

450 outputStatistics = mergeStatDict( 

451 outputStatistics, self.unmaskedImageStats(exposure) 

452 ) 

453 

454 if len(self.config.normImageStatKeywords): 

455 outputStatistics = mergeStatDict( 

456 outputStatistics, self.normalizedImageStats(exposure, statControl) 

457 ) 

458 

459 if len(self.config.crImageStatKeywords): 

460 outputStatistics = mergeStatDict( 

461 outputStatistics, self.crImageStats(exposure, statControl) 

462 ) 

463 

464 return outputStatistics 

465 

466 @staticmethod 

467 def _configHelper(keywordDict): 

468 """Helper to convert keyword dictionary to stat value. 

469 

470 Convert the string names in the keywordDict to the afwMath values. 

471 The statisticToRun is then the bitwise-or of that set. 

472 

473 Parameters 

474 ---------- 

475 keywordDict : `dict` [`str`, `str`] 

476 A dictionary of keys to use in the output results, with 

477 values the string name associated with the 

478 `lsst.afw.math.statistics.Property` to measure. 

479 

480 Returns 

481 ------- 

482 statisticToRun : `int` 

483 The merged `lsst.afw.math` statistics property. 

484 statAccessor : `dict` [`str`, `int`] 

485 Dictionary containing statistics property indexed by name. 

486 """ 

487 statisticToRun = 0 

488 statAccessor = {} 

489 for k, v in keywordDict.items(): 

490 statValue = afwMath.stringToStatisticsProperty(v) 

491 statisticToRun |= statValue 

492 statAccessor[k] = statValue 

493 

494 return statisticToRun, statAccessor 

495 

496 def metadataStatistics(self, exposure, taskMetadata): 

497 """Extract task metadata information for verification. 

498 

499 Parameters 

500 ---------- 

501 exposure : `lsst.afw.image.Exposure` 

502 The exposure to measure. 

503 taskMetadata : `lsst.pipe.base.TaskMetadata` 

504 The metadata to extract values from. 

505 

506 Returns 

507 ------- 

508 ampStats : `dict` [`str`, `dict` [`str`, scalar]] 

509 A dictionary indexed by the amplifier name, containing 

510 dictionaries of the statistics measured and their values. 

511 """ 

512 metadataStats = {} 

513 keywordDict = self.config.metadataStatKeywords 

514 

515 if taskMetadata: 

516 for key, value in keywordDict.items(): 

517 if value == "AMP": 

518 metadataStats[key] = {} 

519 for ampIdx, amp in enumerate(exposure.getDetector()): 

520 ampName = amp.getName() 

521 expectedKey = f"{key} {ampName}" 

522 metadataStats[key][ampName] = None 

523 for name in taskMetadata: 

524 if expectedKey in taskMetadata[name]: 

525 metadataStats[key][ampName] = taskMetadata[name][ 

526 expectedKey 

527 ] 

528 else: 

529 # Assume it's detector-wide. 

530 expectedKey = key 

531 for name in taskMetadata: 

532 if expectedKey in taskMetadata[name]: 

533 metadataStats[key] = taskMetadata[name][expectedKey] 

534 return metadataStats 

535 

536 def amplifierStats(self, exposure, keywordDict, statControl, failAll=False): 

537 """Measure amplifier level statistics from the exposure. 

538 

539 Parameters 

540 ---------- 

541 exposure : `lsst.afw.image.Exposure` 

542 The exposure to measure. 

543 keywordDict : `dict` [`str`, `str`] 

544 A dictionary of keys to use in the output results, with 

545 values the string name associated with the 

546 `lsst.afw.math.statistics.Property` to measure. 

547 statControl : `lsst.afw.math.StatisticsControl` 

548 Statistics control object with parameters defined by 

549 the config. 

550 failAll : `bool`, optional 

551 If True, all tests will be set as failed. 

552 

553 Returns 

554 ------- 

555 ampStats : `dict` [`str`, `dict` [`str`, scalar]] 

556 A dictionary indexed by the amplifier name, containing 

557 dictionaries of the statistics measured and their values. 

558 """ 

559 ampStats = {} 

560 statisticToRun, statAccessor = self._configHelper(keywordDict) 

561 # Measure stats on all amplifiers. 

562 for ampIdx, amp in enumerate(exposure.getDetector()): 

563 ampName = amp.getName() 

564 theseStats = {} 

565 ampExp = exposure.Factory(exposure, amp.getBBox()) 

566 stats = afwMath.makeStatistics( 

567 ampExp.getMaskedImage(), statisticToRun, statControl 

568 ) 

569 

570 for k, v in statAccessor.items(): 

571 theseStats[k] = stats.getValue(v) 

572 

573 if failAll: 

574 theseStats["FORCE_FAILURE"] = failAll 

575 ampStats[ampName] = theseStats 

576 

577 return ampStats 

578 

579 def unmaskedImageStats(self, exposure): 

580 """Measure amplifier level statistics on the exposure, including all 

581 pixels in the exposure, regardless of any mask planes set. 

582 

583 Parameters 

584 ---------- 

585 exposure : `lsst.afw.image.Exposure` 

586 The exposure to measure. 

587 

588 Returns 

589 ------- 

590 outputStatistics : `dict` [`str`, `dict` [`str`, scalar]] 

591 A dictionary indexed by the amplifier name, containing 

592 dictionaries of the statistics measured and their values. 

593 """ 

594 noMaskStatsControl = afwMath.StatisticsControl( 

595 self.config.numSigmaClip, self.config.clipMaxIter, 0x0 

596 ) 

597 return self.amplifierStats( 

598 exposure, self.config.unmaskedImageStatKeywords, noMaskStatsControl 

599 ) 

600 

601 def normalizedImageStats(self, exposure, statControl): 

602 """Measure amplifier level statistics on the exposure after dividing 

603 by the exposure time. 

604 

605 Parameters 

606 ---------- 

607 exposure : `lsst.afw.image.Exposure` 

608 The exposure to measure. 

609 statControl : `lsst.afw.math.StatisticsControl` 

610 Statistics control object with parameters defined by 

611 the config. 

612 

613 Returns 

614 ------- 

615 outputStatistics : `dict` [`str`, `dict` [`str`, scalar]] 

616 A dictionary indexed by the amplifier name, containing 

617 dictionaries of the statistics measured and their values. 

618 

619 Raises 

620 ------ 

621 RuntimeError : 

622 Raised if the exposure time cannot be used for normalization. 

623 """ 

624 scaledExposure = exposure.clone() 

625 exposureTime = scaledExposure.getInfo().getVisitInfo().getExposureTime() 

626 if exposureTime <= 0: 

627 raise RuntimeError(f"Invalid exposureTime {exposureTime}.") 

628 mi = scaledExposure.getMaskedImage() 

629 mi /= exposureTime 

630 

631 return self.amplifierStats( 

632 scaledExposure, self.config.normImageStatKeywords, statControl 

633 ) 

634 

635 def crImageStats(self, exposure, statControl): 

636 """Measure amplifier level statistics on the exposure, 

637 after running cosmic ray rejection. 

638 

639 Parameters 

640 ---------- 

641 exposure : `lsst.afw.image.Exposure` 

642 The exposure to measure. 

643 statControl : `lsst.afw.math.StatisticsControl` 

644 Statistics control object with parameters defined by 

645 the config. 

646 

647 Returns 

648 ------- 

649 outputStatistics : `dict` [`str`, `dict` [`str`, scalar]] 

650 A dictionary indexed by the amplifier name, containing 

651 dictionaries of the statistics measured and their values. 

652 

653 """ 

654 crRejectedExp = exposure.clone() 

655 psf = measAlg.SingleGaussianPsf( 

656 self.config.psfSize, 

657 self.config.psfSize, 

658 self.config.psfFwhm / (2 * np.sqrt(2 * np.log(2))), 

659 ) 

660 crRejectedExp.setPsf(psf) 

661 try: 

662 self.repair.run(crRejectedExp, keepCRs=False) 

663 failAll = False 

664 except pexException.LengthError: 

665 self.log.warning( 

666 "Failure masking cosmic rays (too many found). Continuing." 

667 ) 

668 failAll = True 

669 

670 if self.config.crGrow > 0: 

671 crMask = crRejectedExp.getMaskedImage().getMask().getPlaneBitMask("CR") 

672 spans = afwGeom.SpanSet.fromMask(crRejectedExp.mask, crMask) 

673 spans = spans.dilated(self.config.crGrow) 

674 spans = spans.clippedTo(crRejectedExp.getBBox()) 

675 spans.setMask(crRejectedExp.mask, crMask) 

676 

677 return self.amplifierStats( 

678 crRejectedExp, self.config.crImageStatKeywords, statControl, failAll=failAll 

679 ) 

680 

681 # Methods that need to be implemented by the calibration-level subclasses. 

682 def catalogStatistics(self, exposure, catalog, uncorrectedCatalog, statControl): 

683 """Calculate statistics from a catalog. 

684 

685 Parameters 

686 ---------- 

687 exposure : `lsst.afw.image.Exposure` 

688 The exposure to measure. 

689 catalog : `lsst.afw.table.Table` 

690 The catalog to measure. 

691 uncorrectedCatalog : `lsst.afw.table.Table` 

692 The alternate catalog to measure. 

693 statControl : `lsst.afw.math.StatisticsControl` 

694 Statistics control object with parameters defined by 

695 the config. 

696 

697 Returns 

698 ------- 

699 outputStatistics : `dict` [`str`, `dict` [`str`, scalar]] 

700 A dictionary indexed by the amplifier name, containing 

701 dictionaries of the statistics measured and their values. 

702 """ 

703 raise NotImplementedError( 

704 "Subclasses must implement catalog statistics method." 

705 ) 

706 

707 def detectorStatistics( 

708 self, statisticsDict, statControl, exposure=None, uncorrectedExposure=None 

709 ): 

710 """Calculate detector level statistics based on the existing 

711 per-amplifier measurements. 

712 

713 Parameters 

714 ---------- 

715 statisticsDict : `dict` [`str`, scalar] 

716 Dictionary of measured statistics. The inner dictionary 

717 should have keys that are statistic names (`str`) with 

718 values that are some sort of scalar (`int` or `float` are 

719 the mostly likely types). 

720 statControl : `lsst.afw.math.StatControl` 

721 Statistics control object with parameters defined by 

722 the config. 

723 exposure : `lsst.afw.image.Exposure`, optional 

724 Exposure containing the ISR-processed data to measure. 

725 uncorrectedExposure : `lsst.afw.image.Exposure`, optional 

726 uncorrected esposure (no defects) containing the 

727 ISR-processed data to measure. 

728 

729 Returns 

730 ------- 

731 outputStatistics : `dict` [`str`, scalar] 

732 A dictionary of the statistics measured and their values. 

733 

734 Raises 

735 ------ 

736 NotImplementedError : 

737 This method must be implemented by the calibration-type 

738 subclass. 

739 """ 

740 raise NotImplementedError( 

741 "Subclasses must implement detector statistics method." 

742 ) 

743 

744 def verify(self, exposure, statisticsDict): 

745 """Verify that the measured statistics meet the verification criteria. 

746 

747 Parameters 

748 ---------- 

749 exposure : `lsst.afw.image.Exposure` 

750 The exposure the statistics are from. 

751 statisticsDictionary : `dict` [`str`, `dict` [`str`, scalar]], 

752 Dictionary of measured statistics. The inner dictionary 

753 should have keys that are statistic names (`str`) with 

754 values that are some sort of scalar (`int` or `float` are 

755 the mostly likely types). 

756 

757 Returns 

758 ------- 

759 outputStatistics : `dict` [`str`, `dict` [`str`, `bool`]] 

760 A dictionary indexed by the amplifier name, containing 

761 dictionaries of the verification criteria. 

762 success : `bool` 

763 A boolean indicating whether all tests have passed. 

764 

765 Raises 

766 ------ 

767 NotImplementedError : 

768 This method must be implemented by the calibration-type 

769 subclass. 

770 """ 

771 raise NotImplementedError("Subclasses must implement verification criteria.") 

772 

773 def repackStats(self, statisticsDict, dimensions): 

774 """Repack information into flat tables. 

775 

776 This method may be redefined in subclasses. This default 

777 version will repack simple amp-level statistics and 

778 verification results. 

779 

780 Parameters 

781 ---------- 

782 statisticsDictionary : `dict` [`str`, `dict` [`str`, scalar]], 

783 Dictionary of measured statistics. The inner dictionary 

784 should have keys that are statistic names (`str`) with 

785 values that are some sort of scalar (`int` or `float` are 

786 the mostly likely types). 

787 dimensions : `dict` 

788 The dictionary of dimensions values for this data, to be 

789 included in the output results. 

790 

791 Returns 

792 ------- 

793 outputResults : `list` [`dict`] 

794 A list of rows to add to the output table. 

795 outputMatrix : `list` [`dict`] 

796 A list of rows to add to the output matrix. 

797 """ 

798 rows = {} 

799 rowList = [] 

800 matrixRowList = None 

801 

802 if self.config.useIsrStatistics: 

803 mjd = statisticsDict["ISR"]["MJD"] 

804 else: 

805 mjd = np.nan 

806 

807 rowBase = { 

808 "instrument": dimensions["instrument"], 

809 "detector": dimensions["detector"], 

810 "mjd": mjd, 

811 } 

812 

813 # AMP results: 

814 for ampName, stats in statisticsDict["AMP"].items(): 

815 rows[ampName] = {} 

816 rows[ampName].update(rowBase) 

817 rows[ampName]["amplifier"] = ampName 

818 for key, value in stats.items(): 

819 rows[ampName][f"{self.config.stageName}_{key}"] = value 

820 

821 # VERIFY results 

822 if "AMP" in statisticsDict["VERIFY"]: 

823 for ampName, stats in statisticsDict["VERIFY"]["AMP"].items(): 

824 for key, value in stats.items(): 

825 rows[ampName][f"{self.config.stageName}_VERIFY_{key}"] = value 

826 

827 # pack final list 

828 for ampName, stats in rows.items(): 

829 rowList.append(stats) 

830 

831 return rowList, matrixRowList