Coverage for python/lsst/cp/verify/mergeResults.py: 40%

227 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-09 04:38 -0700

1# This file is part of cp_verify. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from astropy.table import vstack, Table 

22 

23import lsst.pipe.base as pipeBase 

24import lsst.pipe.base.connectionTypes as cT 

25import lsst.pex.config as pexConfig 

26 

27 

28__all__ = ['CpVerifyExpMergeConfig', 'CpVerifyExpMergeTask', 

29 'CpVerifyRunMergeConfig', 'CpVerifyRunMergeTask', 

30 'CpVerifyExpMergeByFilterConfig', 'CpVerifyExpMergeByFilterTask', 

31 'CpVerifyRunMergeByFilterConfig', 'CpVerifyRunMergeByFilterTask', 

32 'CpVerifyVisitExpMergeConfig', 'CpVerifyVisitExpMergeTask', 

33 'CpVerifyVisitRunMergeConfig', 'CpVerifyVisitRunMergeTask', 

34 'CpVerifyCalibMergeConfig', 'CpVerifyCalibMergeTask'] 

35 

36 

37class CpVerifyExpMergeConnections(pipeBase.PipelineTaskConnections, 

38 dimensions={"instrument", "exposure"}, 

39 defaultTemplates={}): 

40 inputStats = cT.Input( 

41 name="detectorStats", 

42 doc="Input statistics to merge.", 

43 storageClass="StructuredDataDict", 

44 dimensions=["instrument", "exposure", "detector"], 

45 multiple=True, 

46 ) 

47 inputResults = cT.Input( 

48 name="detectorResults", 

49 doc="Input results to merge.", 

50 storageClass="ArrowAstropy", 

51 dimensions=["instrument", "exposure", "detector"], 

52 multiple=True, 

53 ) 

54 inputMatrix = cT.Input( 

55 name="detectorMatrix", 

56 doc="Input matrix to merge.", 

57 storageClass="ArrowAstropy", 

58 dimensions=["instrument", "exposure", "detector"], 

59 multiple=True, 

60 ) 

61 camera = cT.PrerequisiteInput( 

62 name="camera", 

63 storageClass="Camera", 

64 doc="Input camera.", 

65 dimensions=["instrument", ], 

66 isCalibration=True, 

67 ) 

68 

69 outputStats = cT.Output( 

70 name="exposureStats", 

71 doc="Output statistics.", 

72 storageClass="StructuredDataDict", 

73 dimensions=["instrument", "exposure"], 

74 ) 

75 outputResults = cT.Output( 

76 name="exposureResults", 

77 doc="Output results.", 

78 storageClass="ArrowAstropy", 

79 dimensions=["instrument", "exposure"], 

80 ) 

81 outputMatrix = cT.Output( 

82 name="exposureMatrix", 

83 doc="Output matrix.", 

84 storageClass="ArrowAstropy", 

85 dimensions=["instrument", "exposure"], 

86 ) 

87 

88 def __init__(self, *, config=None): 

89 super().__init__(config=config) 

90 

91 if not self.config.hasMatrixCatalog: 

92 self.inputs.remove("inputMatrix") 

93 self.outputs.remove("outputMatrix") 

94 if not self.config.hasInputResults: 

95 self.inputs.remove("inputResults") 

96 

97 

98class CpVerifyExpMergeConfig(pipeBase.PipelineTaskConfig, 

99 pipelineConnections=CpVerifyExpMergeConnections): 

100 """Configuration parameters for exposure stats merging. 

101 """ 

102 statKeywords = pexConfig.DictField( 

103 keytype=str, 

104 itemtype=str, 

105 doc="Dictionary of statistics to run on the set of detector values. The key should be the test " 

106 "name to record in the output, and the value should be the `lsst.afw.math` statistic name string.", 

107 default={}, 

108 ) 

109 hasMatrixCatalog = pexConfig.Field( 

110 dtype=bool, 

111 doc="Is there matrix catalog to merge?", 

112 default=False, 

113 ) 

114 hasInputResults = pexConfig.Field( 

115 dtype=bool, 

116 doc="Are there results tables to merge?", 

117 default=False, 

118 ) 

119 

120 mergeDimension = pexConfig.Field( 

121 dtype=str, 

122 doc="Dimension name that these inputs will be merged over.", 

123 default="detector", 

124 ) 

125 stageName = pexConfig.Field( 

126 dtype=str, 

127 doc="Stage name to use in any further analysis.", 

128 default="stageName", 

129 ) 

130 

131 

132class CpVerifyExpMergeTask(pipeBase.PipelineTask): 

133 """Merge statistics from detectors together. 

134 """ 

135 ConfigClass = CpVerifyExpMergeConfig 

136 _DefaultName = 'cpVerifyExpMerge' 

137 

138 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

139 inputs = butlerQC.get(inputRefs) 

140 

141 dimensions = [dict(exp.dataId.required) for exp in inputRefs.inputStats] 

142 inputs['inputDims'] = dimensions 

143 

144 outputs = self.run(**inputs) 

145 butlerQC.put(outputs, outputRefs) 

146 

147 def run(self, inputStats, inputDims, camera=None, inputResults=None, inputMatrix=None,): 

148 """Merge statistics. 

149 

150 Parameters 

151 ---------- 

152 inputStats : `dict` 

153 A nested dictionary of measured statistics and 

154 verification results. 

155 inputDims : `dict` 

156 The input dimensions for each element of the inputStats. 

157 camera : `lsst.afw.cameraGeom.Camera`, optional 

158 The camera definition, used for identifying amplifier and 

159 detector names. 

160 inputResults : `astropy.Table`, optional 

161 The statistics information, formatted into a flat table. 

162 inputMatrix : `astropy.Table`, optional 

163 A table of results that represent the elements of matrices 

164 of values. 

165 

166 Returns 

167 ------- 

168 outputStats : `dict` 

169 A nested dictionary of merged statistics and verification 

170 results. 

171 outputResults : `astropy.Table` 

172 Flat table containing the merged results from all 

173 inputResults. 

174 outputMatrix : `astropy.Table` 

175 Table containing the merge results from all inputMatrix. 

176 

177 See Also 

178 -------- 

179 lsst.cp.verify.CpVerifyStatsTask 

180 """ 

181 outputStats = {} # This contains failure information 

182 success = True 

183 

184 mergedStats = {} # This contains the merged set of subcomponent stats. 

185 for inStats, dimensions in zip(inputStats, inputDims): 

186 thisId = dimensions[self.config.mergeDimension] 

187 thisName = thisId 

188 

189 if self.config.mergeDimension == 'detector': 

190 thisName = camera[thisId].getName() 

191 

192 calcStats = {} 

193 

194 mergedStats[thisName] = inStats 

195 

196 if inStats['SUCCESS'] is True: 

197 calcStats['SUCCESS'] = True 

198 else: 

199 calcStats['SUCCESS'] = False 

200 calcStats['FAILURES'] = list() 

201 success = False 

202 

203 # See if we have verify information to check: 

204 if 'VERIFY' in inStats: 

205 # See if an exposure failed 

206 if 'EXP' in inStats['VERIFY'] and len(inStats['VERIFY']['EXP']) > 0: 

207 expSuccess = inStats['VERIFY']['EXP'].pop('SUCCESS', False) 

208 if not expSuccess: 

209 for testName, testResult in inStats['VERIFY']['EXP'].items(): 

210 if testResult is False: 

211 calcStats['FAILURES'].append(testName) 

212 

213 # See if a detector failed 

214 if 'DET' in inStats['VERIFY'] and len(inStats['VERIFY']['DET']) > 0: 

215 detSuccess = inStats['VERIFY']['DET'].pop('SUCCESS', False) 

216 if not detSuccess: 

217 for testName, testResult in inStats['VERIFY']['DET'].items(): 

218 if testResult is False: 

219 calcStats['FAILURES'].append(testName) 

220 

221 # See if an amplifier failed 

222 if 'AMP' in inStats['VERIFY'] and len(inStats['VERIFY']['AMP']) > 0: 

223 for ampName, ampStats in inStats['VERIFY']['AMP'].items(): 

224 ampSuccess = ampStats.pop('SUCCESS') 

225 if not ampSuccess: 

226 for testName, testResult in ampStats.items(): 

227 if testResult is False: 

228 calcStats['FAILURES'].append(ampName + " " + testName) 

229 

230 # See if a catalog failed 

231 if 'CATALOG' in inStats['VERIFY'] and len(inStats['VERIFY']['CATALOG']) > 0: 

232 for testName, testResult in inStats['VERIFY']['CATALOG'].items(): 

233 if testResult is False: 

234 calcStats['FAILURES'].append(testName) 

235 else: 

236 # No VERIFY info? This must be partially accumulated. 

237 # But we know there are failures somewhere. 

238 # Drop any "SUCCESS" keys 

239 _ = inStats.pop("SUCCESS", False) 

240 for statKey, statDict in inStats.items(): 

241 if 'SUCCESS' in statDict and not statDict['SUCCESS']: 

242 for failure in statDict['FAILURES']: 

243 calcStats['FAILURES'].append(f"{statKey} {failure}") 

244 

245 outputStats[thisName] = calcStats 

246 

247 if self.config.mergeDimension == 'detector': 

248 outKey = 'EXP' 

249 else: 

250 outKey = 'RUN' 

251 

252 groupSuccess = True 

253 if len(self.config.statKeywords): 

254 outputStats[outKey] = self.calcStatistics(mergedStats) 

255 outputStats['VERIFY'], groupSuccess = self.verify(mergedStats, outputStats) 

256 

257 outputStats['SUCCESS'] = success & groupSuccess 

258 

259 additionalResults = None 

260 if outKey in outputStats: 

261 # This is the only new information generated here. 

262 additionalResults, _ = self.pack(outputStats, inputDims, outKey) 

263 

264 outputResults = self.mergeTable(inputResults, additionalResults) 

265 if inputMatrix is not None: 

266 outputMatrix = self.mergeTable(inputMatrix) 

267 else: 

268 outputMatrix = None 

269 

270 return pipeBase.Struct( 

271 outputStats=outputStats, 

272 outputResults=outputResults, 

273 outputMatrix=outputMatrix, 

274 ) 

275 

276 @staticmethod 

277 def mergeTable(inputResults, newStats=None): 

278 """Merge input tables. 

279 

280 Parameters 

281 ---------- 

282 inputResults : `list` [`astropy.table.Table`] 

283 Input tables to merge. 

284 newStats : `astropy.table.Table` 

285 Additional table to merge. 

286 

287 Returns 

288 ------- 

289 merged : `astropy.table.Table` 

290 "Outer-join" merged table. 

291 """ 

292 if inputResults is None: 

293 return Table() 

294 

295 if len(inputResults) > 0: 

296 outputResults = vstack(inputResults) 

297 else: 

298 outputResults = inputResults 

299 

300 if newStats: 

301 return vstack([outputResults, Table(newStats)]) 

302 else: 

303 return outputResults 

304 

305 def calcStatistics(self, statisticsDict): 

306 """Calculate exposure level statistics based on the existing 

307 per-amplifier and per-detector measurements. 

308 

309 Parameters 

310 ---------- 

311 statisticsDictionary : `dict [`str`, `dict` [`str`, scalar]], 

312 Dictionary of measured statistics. The top level 

313 dictionary is keyed on the detector names, and contains 

314 the measured statistics from the per-detector 

315 measurements. 

316 

317 Returns 

318 ------- 

319 outputStatistics : `dict` [`str, scalar] 

320 A dictionary of the statistics measured and their values. 

321 """ 

322 raise NotImplementedError("Subclasses must implement verification criteria.") 

323 

324 def verify(self, detectorStatistics, statisticsDictionary): 

325 

326 """Verify if the measured statistics meet the verification criteria. 

327 

328 Parameters 

329 ---------- 

330 detectorStatistics : `dict` [`str`, `dict` [`str`, scalar]] 

331 Merged set of input detector level statistics. 

332 statisticsDictionary : `dict` [`str`, `dict` [`str`, scalar]] 

333 Dictionary of measured statistics. The inner dictionary 

334 should have keys that are statistic names (`str`) with 

335 values that are some sort of scalar (`int` or `float` are 

336 the mostly likely types). 

337 

338 Returns 

339 ------- 

340 outputStatistics : `dict` [`str`, `dict` [`str`, `bool`]] 

341 A dictionary indexed by the amplifier name, containing 

342 dictionaries of the verification criteria. 

343 success : `bool` 

344 A boolean indicating if all tests have passed. 

345 

346 Raises 

347 ------ 

348 NotImplementedError : 

349 This method must be implemented by the calibration-type 

350 subclass. 

351 """ 

352 raise NotImplementedError("Subclasses must implement verification criteria.") 

353 

354 def pack(self, statisticsDict, dimensions, outKey): 

355 """Repack information into flat tables. 

356 

357 This method should be redefined in subclasses, if new 

358 statistics are measured. 

359 

360 Parameters 

361 ---------- 

362 statisticsDictionary : `dict` [`str`, `dict` [`str`, scalar]], 

363 Dictionary of measured statistics. The inner dictionary 

364 should have keys that are statistic names (`str`) with 

365 values that are some sort of scalar (`int` or `float` are 

366 the mostly likely types). 

367 dimensions : `dict` 

368 Dictionary of input dimensions. 

369 outKey : `str` 

370 Key to use to access the data to pack. 

371 

372 Returns 

373 ------- 

374 outputResults : `list` [`dict`] 

375 A list of rows to add to the output table. 

376 outputMatrix : `list` [`dict`] 

377 A list of rows to add to the output matrix. 

378 

379 Raises 

380 ------ 

381 NotImplementedError : 

382 This method must be implemented by the calibration-type 

383 subclass. 

384 

385 """ 

386 raise NotImplementedError("Subclasses must implement verification criteria.") 

387 

388 

389class CpVerifyRunMergeConnections(pipeBase.PipelineTaskConnections, 

390 dimensions={"instrument", }, 

391 defaultTemplates={}): 

392 inputStats = cT.Input( 

393 name="exposureStats", 

394 doc="Input statistics to merge.", 

395 storageClass="StructuredDataDict", 

396 dimensions=["instrument", "exposure"], 

397 multiple=True, 

398 ) 

399 inputResults = cT.Input( 

400 name="exposureResults", 

401 doc="Input results table to merge.", 

402 storageClass="ArrowAstropy", 

403 dimensions=["instrument", "exposure"], 

404 multiple=True, 

405 ) 

406 inputMatrix = cT.Input( 

407 name="exposureMatrix", 

408 doc="Input matrix table to merge.", 

409 storageClass="ArrowAstropy", 

410 dimensions=["instrument", "exposure"], 

411 multiple=True, 

412 ) 

413 camera = cT.PrerequisiteInput( 

414 name="camera", 

415 storageClass="Camera", 

416 doc="Input camera.", 

417 dimensions=["instrument", ], 

418 isCalibration=True, 

419 ) 

420 

421 outputStats = cT.Output( 

422 name="runStats", 

423 doc="Output statistics.", 

424 storageClass="StructuredDataDict", 

425 dimensions=["instrument", ], 

426 ) 

427 outputResults = cT.Output( 

428 name="runResults", 

429 doc="Output merged results table.", 

430 storageClass="ArrowAstropy", 

431 dimensions=["instrument",], 

432 ) 

433 outputMatrix = cT.Output( 

434 name="runMatrix", 

435 doc="Output merged matrix table.", 

436 storageClass="ArrowAstropy", 

437 dimensions=["instrument",], 

438 ) 

439 

440 def __init__(self, *, config=None): 

441 super().__init__(config=config) 

442 

443 if not self.config.hasMatrixCatalog: 

444 self.inputs.remove("inputMatrix") 

445 self.outputs.remove("outputMatrix") 

446 

447 

448class CpVerifyRunMergeConfig(CpVerifyExpMergeConfig, 

449 pipelineConnections=CpVerifyRunMergeConnections): 

450 """Configuration paramters for exposure stats merging. 

451 """ 

452 mergeDimension = pexConfig.Field( 

453 dtype=str, 

454 doc="Dimension name for this input.", 

455 default="exposure", 

456 ) 

457 

458 

459class CpVerifyRunMergeTask(CpVerifyExpMergeTask): 

460 """Merge statistics from detectors together. 

461 """ 

462 ConfigClass = CpVerifyRunMergeConfig 

463 _DefaultName = 'cpVerifyRunMerge' 

464 

465 pass 

466# End ExpMerge/RunMerge 

467 

468 

469# Begin ExpMergeByFilter/RunMergeByFilter 

470class CpVerifyExpMergeByFilterConnections(pipeBase.PipelineTaskConnections, 

471 dimensions={"instrument", "exposure", "physical_filter"}, 

472 defaultTemplates={}): 

473 inputStats = cT.Input( 

474 name="exposureStats", 

475 doc="Input statistics to merge.", 

476 storageClass="StructuredDataDict", 

477 dimensions=["instrument", "exposure", "detector", "physical_filter"], 

478 multiple=True, 

479 ) 

480 inputResults = cT.Input( 

481 name="exposureResults", 

482 doc="Input results table to merge.", 

483 storageClass="ArrowAstropy", 

484 dimensions=["instrument", "exposure", "detector", "physical_filter"], 

485 multiple=True, 

486 ) 

487 inputMatrix = cT.Input( 

488 name="exposureMatrix", 

489 doc="Input matrix table to merge.", 

490 storageClass="ArrowAstropy", 

491 dimensions=["instrument", "exposure", "detector", "physical_filter"], 

492 multiple=True, 

493 ) 

494 camera = cT.PrerequisiteInput( 

495 name="camera", 

496 storageClass="Camera", 

497 doc="Input camera.", 

498 dimensions=["instrument", ], 

499 isCalibration=True, 

500 ) 

501 

502 outputStats = cT.Output( 

503 name="runStats", 

504 doc="Output statistics.", 

505 storageClass="StructuredDataDict", 

506 dimensions=["instrument", "exposure", "physical_filter"], 

507 ) 

508 outputResults = cT.Output( 

509 name="runResults", 

510 doc="Output merged results table.", 

511 storageClass="ArrowAstropy", 

512 dimensions=["instrument", "exposure", "physical_filter"], 

513 ) 

514 outputMatrix = cT.Output( 

515 name="runMatrix", 

516 doc="Output merged matrix table.", 

517 storageClass="ArrowAstropy", 

518 dimensions=["instrument", "exposure", "physical_filter"], 

519 ) 

520 

521 def __init__(self, *, config=None): 

522 super().__init__(config=config) 

523 

524 if not self.config.hasMatrixCatalog: 

525 self.inputs.remove("inputMatrix") 

526 self.outputs.remove("outputMatrix") 

527 

528 

529class CpVerifyExpMergeByFilterConfig(CpVerifyExpMergeConfig, 

530 pipelineConnections=CpVerifyExpMergeByFilterConnections): 

531 """Configuration paramters for exposure stats merging. 

532 """ 

533 mergeDimension = pexConfig.Field( 

534 dtype=str, 

535 doc="Dimension name for this input.", 

536 default="detector", 

537 ) 

538 

539 

540class CpVerifyExpMergeByFilterTask(CpVerifyExpMergeTask): 

541 """Merge statistics from detectors together. 

542 """ 

543 ConfigClass = CpVerifyRunMergeConfig 

544 _DefaultName = 'cpVerifyRunMerge' 

545 

546 pass 

547 

548 

549class CpVerifyRunMergeByFilterConnections(pipeBase.PipelineTaskConnections, 

550 dimensions={"instrument", "physical_filter"}, 

551 defaultTemplates={}): 

552 inputStats = cT.Input( 

553 name="exposureStats", 

554 doc="Input statistics to merge.", 

555 storageClass="StructuredDataDict", 

556 dimensions=["instrument", "exposure", "physical_filter"], 

557 multiple=True, 

558 ) 

559 inputResults = cT.Input( 

560 name="exposureResults", 

561 doc="Input results table to merge.", 

562 storageClass="ArrowAstropy", 

563 dimensions=["instrument", "exposure", "physical_filter"], 

564 multiple=True, 

565 ) 

566 inputMatrix = cT.Input( 

567 name="exposureMatrix", 

568 doc="Input matrix table to merge.", 

569 storageClass="ArrowAstropy", 

570 dimensions=["instrument", "exposure", "physical_filter"], 

571 multiple=True, 

572 ) 

573 camera = cT.PrerequisiteInput( 

574 name="camera", 

575 storageClass="Camera", 

576 doc="Input camera.", 

577 dimensions=["instrument", ], 

578 isCalibration=True, 

579 ) 

580 

581 outputStats = cT.Output( 

582 name="runStats", 

583 doc="Output statistics.", 

584 storageClass="StructuredDataDict", 

585 dimensions=["instrument", "physical_filter"], 

586 ) 

587 outputResults = cT.Output( 

588 name="runResults", 

589 doc="Output merged results table.", 

590 storageClass="ArrowAstropy", 

591 dimensions=["instrument", "physical_filter"], 

592 ) 

593 outputMatrix = cT.Output( 

594 name="runMatrix", 

595 doc="Output merged matrix table.", 

596 storageClass="ArrowAstropy", 

597 dimensions=["instrument", "physical_filter"], 

598 ) 

599 

600 def __init__(self, *, config=None): 

601 super().__init__(config=config) 

602 

603 if not self.config.hasMatrixCatalog: 

604 self.inputs.remove("inputMatrix") 

605 self.outputs.remove("outputMatrix") 

606 

607 

608class CpVerifyRunMergeByFilterConfig(CpVerifyRunMergeConfig, 

609 pipelineConnections=CpVerifyRunMergeByFilterConnections): 

610 """Configuration paramters for exposure stats merging. 

611 """ 

612 pass 

613 

614 

615class CpVerifyRunMergeByFilterTask(CpVerifyExpMergeTask): 

616 """Merge statistics from detectors together. 

617 """ 

618 ConfigClass = CpVerifyRunMergeByFilterConfig 

619 _DefaultName = 'cpVerifyRunMergeByFilter' 

620 

621 pass 

622# End ExpMergeByFilter/RunMergeByFilter 

623 

624 

625# Begin ExpMergeByVisit/RunMergeByVisit 

626class CpVerifyVisitExpMergeConnections(pipeBase.PipelineTaskConnections, 

627 dimensions={"instrument", "visit"}, 

628 defaultTemplates={}): 

629 inputStats = cT.Input( 

630 name="detectorStats", 

631 doc="Input statistics to merge.", 

632 storageClass="StructuredDataDict", 

633 dimensions=["instrument", "visit", "detector"], 

634 multiple=True, 

635 ) 

636 inputResults = cT.Input( 

637 name="detectorResults", 

638 doc="Input results to merge.", 

639 storageClass="ArrowAstropy", 

640 dimensions=["instrument", "visit", "detector"], 

641 multiple=True, 

642 ) 

643 inputMatrix = cT.Input( 

644 name="detectorMatrix", 

645 doc="Input matrix to merge.", 

646 storageClass="ArrowAstropy", 

647 dimensions=["instrument", "visit", "detector"], 

648 multiple=True, 

649 ) 

650 camera = cT.PrerequisiteInput( 

651 name="camera", 

652 storageClass="Camera", 

653 doc="Input camera.", 

654 dimensions=["instrument", ], 

655 isCalibration=True, 

656 ) 

657 

658 outputStats = cT.Output( 

659 name="exposureStats", 

660 doc="Output statistics.", 

661 storageClass="StructuredDataDict", 

662 dimensions=["instrument", "visit"], 

663 ) 

664 outputResults = cT.Output( 

665 name="exposureResults", 

666 doc="Output results.", 

667 storageClass="ArrowAstropy", 

668 dimensions=["instrument", "visit"], 

669 ) 

670 outputMatrix = cT.Output( 

671 name="exposureMatrix", 

672 doc="Output matrix.", 

673 storageClass="ArrowAstropy", 

674 dimensions=["instrument", "visit"], 

675 ) 

676 

677 def __init__(self, *, config=None): 

678 super().__init__(config=config) 

679 

680 if not self.config.hasMatrixCatalog: 

681 self.inputs.remove("inputMatrix") 

682 self.outputs.remove("outputMatrix") 

683 

684 

685class CpVerifyVisitExpMergeConfig(CpVerifyExpMergeConfig, 

686 pipelineConnections=CpVerifyVisitExpMergeConnections): 

687 pass 

688 

689 

690class CpVerifyVisitExpMergeTask(CpVerifyExpMergeTask): 

691 """Merge visit based data.""" 

692 

693 ConfigClass = CpVerifyVisitExpMergeConfig 

694 _DefaultName = 'cpVerifyVisitExpMerge' 

695 

696 pass 

697 

698 

699class CpVerifyVisitRunMergeConnections(pipeBase.PipelineTaskConnections, 

700 dimensions={"instrument"}, 

701 defaultTemplates={}): 

702 inputStats = cT.Input( 

703 name="exposureStats", 

704 doc="Input statistics to merge.", 

705 storageClass="StructuredDataDict", 

706 dimensions=["instrument", "visit"], 

707 multiple=True, 

708 ) 

709 inputResults = cT.Input( 

710 name="exposureResults", 

711 doc="Input results table to merge.", 

712 storageClass="ArrowAstropy", 

713 dimensions=["instrument", "visit"], 

714 multiple=True, 

715 ) 

716 inputMatrix = cT.Input( 

717 name="exposureMatrix", 

718 doc="Input matrix table to merge.", 

719 storageClass="ArrowAstropy", 

720 dimensions=["instrument", "visit"], 

721 multiple=True, 

722 ) 

723 camera = cT.PrerequisiteInput( 

724 name="camera", 

725 storageClass="Camera", 

726 doc="Input camera.", 

727 dimensions=["instrument", ], 

728 isCalibration=True, 

729 ) 

730 

731 outputStats = cT.Output( 

732 name="runStats", 

733 doc="Output statistics.", 

734 storageClass="StructuredDataDict", 

735 dimensions=["instrument"], 

736 ) 

737 outputResults = cT.Output( 

738 name="runResults", 

739 doc="Output merged results table.", 

740 storageClass="ArrowAstropy", 

741 dimensions=["instrument", ], 

742 ) 

743 outputMatrix = cT.Output( 

744 name="runMatrix", 

745 doc="Output merged matrix table.", 

746 storageClass="ArrowAstropy", 

747 dimensions=["instrument", ], 

748 ) 

749 

750 def __init__(self, *, config=None): 

751 super().__init__(config=config) 

752 

753 if not self.config.hasMatrixCatalog: 

754 self.inputs.remove("inputMatrix") 

755 self.outputs.remove("outputMatrix") 

756 

757 

758class CpVerifyVisitRunMergeConfig(CpVerifyRunMergeConfig, 

759 pipelineConnections=CpVerifyVisitRunMergeConnections): 

760 pass 

761 

762 

763class CpVerifyVisitRunMergeTask(CpVerifyRunMergeTask): 

764 """Merge visit based data.""" 

765 

766 ConfigClass = CpVerifyVisitRunMergeConfig 

767 _DefaultName = 'cpVerifyVisitRunMerge' 

768 

769 pass 

770# End ExpMergeByVisit/RunMergeByVisit 

771 

772 

773# Begin CalibMerge (this is a one-step) 

774class CpVerifyCalibMergeConnections(pipeBase.PipelineTaskConnections, 

775 dimensions={"instrument"}, 

776 defaultTemplates={}): 

777 inputStats = cT.Input( 

778 name="exposureStats", 

779 doc="Input statistics to merge.", 

780 storageClass="StructuredDataDict", 

781 dimensions=["instrument", "detector"], 

782 multiple=True, 

783 ) 

784 inputResults = cT.Input( 

785 name="exposureResults", 

786 doc="Input results table to merge.", 

787 storageClass="ArrowAstropy", 

788 dimensions=["instrument", "detector"], 

789 multiple=True, 

790 ) 

791 inputMatrix = cT.Input( 

792 name="exposureMatrix", 

793 doc="Input matrix table to merge.", 

794 storageClass="ArrowAstropy", 

795 dimensions=["instrument", "detector"], 

796 multiple=True, 

797 ) 

798 camera = cT.PrerequisiteInput( 

799 name="camera", 

800 storageClass="Camera", 

801 doc="Input camera.", 

802 dimensions=["instrument", ], 

803 isCalibration=True, 

804 ) 

805 

806 outputStats = cT.Output( 

807 name="exposureStats", 

808 doc="Output statistics.", 

809 storageClass="StructuredDataDict", 

810 dimensions=["instrument"], 

811 ) 

812 outputResults = cT.Output( 

813 name="runResults", 

814 doc="Output merged results table.", 

815 storageClass="ArrowAstropy", 

816 dimensions=["instrument", ], 

817 ) 

818 outputMatrix = cT.Output( 

819 name="runMatrix", 

820 doc="Output merged matrix table.", 

821 storageClass="ArrowAstropy", 

822 dimensions=["instrument", ], 

823 ) 

824 

825 def __init__(self, *, config=None): 

826 super().__init__(config=config) 

827 

828 if not self.config.hasMatrixCatalog: 

829 self.inputs.remove("inputMatrix") 

830 self.outputs.remove("outputMatrix") 

831 

832 

833class CpVerifyCalibMergeConfig(CpVerifyRunMergeConfig, 

834 pipelineConnections=CpVerifyCalibMergeConnections): 

835 """Configuration paramters for exposure stats merging. 

836 """ 

837 pass 

838 

839 

840class CpVerifyCalibMergeTask(CpVerifyRunMergeTask): 

841 """Merge statistics from detectors together. 

842 """ 

843 ConfigClass = CpVerifyCalibMergeConfig 

844 _DefaultName = 'cpVerifyCalibMerge' 

845 

846 pass 

847# End CalibMerge