Coverage for python/lsst/ctrl/mpexec/singleQuantumExecutor.py: 9%

266 statements  

« prev     ^ index     » next       coverage.py v7.2.3, created at 2023-04-22 02:21 -0700

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22__all__ = ["SingleQuantumExecutor"] 

23 

24# ------------------------------- 

25# Imports of standard modules -- 

26# ------------------------------- 

27import logging 

28import os 

29import sys 

30import time 

31import warnings 

32from collections import defaultdict 

33from collections.abc import Callable 

34from itertools import chain 

35from typing import Any, Optional, Union 

36 

37from lsst.daf.butler import ( 

38 Butler, 

39 DatasetRef, 

40 DatasetType, 

41 LimitedButler, 

42 NamedKeyDict, 

43 Quantum, 

44 UnresolvedRefWarning, 

45) 

46from lsst.pipe.base import ( 

47 AdjustQuantumHelper, 

48 ButlerQuantumContext, 

49 Instrument, 

50 InvalidQuantumError, 

51 NoWorkFound, 

52 PipelineTask, 

53 RepeatableQuantumError, 

54 TaskDef, 

55 TaskFactory, 

56) 

57from lsst.pipe.base.configOverrides import ConfigOverrides 

58 

59# During metadata transition phase, determine metadata class by 

60# asking pipe_base 

61from lsst.pipe.base.task import _TASK_FULL_METADATA_TYPE, _TASK_METADATA_TYPE 

62from lsst.utils.timer import logInfo 

63 

64# ----------------------------- 

65# Imports for other modules -- 

66# ----------------------------- 

67from .cli.utils import _PipelineAction 

68from .log_capture import LogCapture 

69from .mock_task import MockButlerQuantumContext, MockPipelineTask 

70from .quantumGraphExecutor import QuantumExecutor 

71from .reports import QuantumReport 

72 

73# ---------------------------------- 

74# Local non-exported definitions -- 

75# ---------------------------------- 

76 

77_LOG = logging.getLogger(__name__) 

78 

79 

80class SingleQuantumExecutor(QuantumExecutor): 

81 """Executor class which runs one Quantum at a time. 

82 

83 Parameters 

84 ---------- 

85 butler : `~lsst.daf.butler.Butler` or `None` 

86 Data butler, `None` means that Quantum-backed butler should be used 

87 instead. 

88 taskFactory : `~lsst.pipe.base.TaskFactory` 

89 Instance of a task factory. 

90 skipExistingIn : `list` [ `str` ], optional 

91 Accepts list of collections, if all Quantum outputs already exist in 

92 the specified list of collections then that Quantum will not be rerun. 

93 clobberOutputs : `bool`, optional 

94 If `True`, then existing outputs in output run collection will be 

95 overwritten. If ``skipExistingIn`` is defined, only outputs from 

96 failed quanta will be overwritten. Only used when ``butler`` is not 

97 `None`. 

98 enableLsstDebug : `bool`, optional 

99 Enable debugging with ``lsstDebug`` facility for a task. 

100 exitOnKnownError : `bool`, optional 

101 If `True`, call `sys.exit` with the appropriate exit code for special 

102 known exceptions, after printing a traceback, instead of letting the 

103 exception propagate up to calling. This is always the behavior for 

104 InvalidQuantumError. 

105 mock : `bool`, optional 

106 If `True` then mock task execution. 

107 mock_configs : `list` [ `_PipelineAction` ], optional 

108 Optional config overrides for mock tasks. 

109 limited_butler_factory : `Callable`, optional 

110 A method that creates a `~lsst.daf.butler.LimitedButler` instance 

111 for a given Quantum. This parameter must be defined if ``butler`` is 

112 `None`. If ``butler`` is not `None` then this parameter is ignored. 

113 """ 

114 

115 def __init__( 

116 self, 

117 butler: Butler | None, 

118 taskFactory: TaskFactory, 

119 skipExistingIn: list[str] | None = None, 

120 clobberOutputs: bool = False, 

121 enableLsstDebug: bool = False, 

122 exitOnKnownError: bool = False, 

123 mock: bool = False, 

124 mock_configs: list[_PipelineAction] | None = None, 

125 limited_butler_factory: Callable[[Quantum], LimitedButler] | None = None, 

126 ): 

127 self.butler = butler 

128 self.taskFactory = taskFactory 

129 self.skipExistingIn = skipExistingIn 

130 self.enableLsstDebug = enableLsstDebug 

131 self.clobberOutputs = clobberOutputs 

132 self.exitOnKnownError = exitOnKnownError 

133 self.mock = mock 

134 self.mock_configs = mock_configs if mock_configs is not None else [] 

135 self.limited_butler_factory = limited_butler_factory 

136 self.report: Optional[QuantumReport] = None 

137 

138 if self.butler is None: 

139 assert not self.mock, "Mock execution only possible with full butler" 

140 assert limited_butler_factory is not None, "limited_butler_factory is needed when butler is None" 

141 

142 def execute(self, taskDef: TaskDef, quantum: Quantum) -> Quantum: 

143 # Docstring inherited from QuantumExecutor.execute 

144 assert quantum.dataId is not None, "Quantum DataId cannot be None" 

145 

146 if self.butler is not None: 

147 self.butler.registry.refresh() 

148 

149 # Catch any exception and make a report based on that. 

150 try: 

151 result = self._execute(taskDef, quantum) 

152 self.report = QuantumReport(dataId=quantum.dataId, taskLabel=taskDef.label) 

153 return result 

154 except Exception as exc: 

155 self.report = QuantumReport.from_exception( 

156 exception=exc, 

157 dataId=quantum.dataId, 

158 taskLabel=taskDef.label, 

159 ) 

160 raise 

161 

162 def _resolve_ref(self, ref: DatasetRef, collections: Any = None) -> DatasetRef | None: 

163 """Return resolved reference. 

164 

165 Parameters 

166 ---------- 

167 ref : `DatasetRef` 

168 Input reference, can be either resolved or unresolved. 

169 collections : 

170 Collections to search for the existing reference, only used when 

171 running with full butler. 

172 

173 Notes 

174 ----- 

175 When running with Quantum-backed butler it assumes that reference is 

176 already resolved and returns input references without any checks. When 

177 running with full butler, it always searches registry fof a reference 

178 in specified collections, even if reference is already resolved. 

179 """ 

180 if self.butler is not None: 

181 # If running with full butler, need to re-resolve it in case 

182 # collections are different. 

183 with warnings.catch_warnings(): 

184 warnings.simplefilter("ignore", category=UnresolvedRefWarning) 

185 ref = ref.unresolved() 

186 return self.butler.registry.findDataset(ref.datasetType, ref.dataId, collections=collections) 

187 else: 

188 # In case of QBB all refs must be resolved already, do not check. 

189 return ref 

190 

191 def _execute(self, taskDef: TaskDef, quantum: Quantum) -> Quantum: 

192 """Internal implementation of execute()""" 

193 startTime = time.time() 

194 

195 # Make a limited butler instance if needed (which should be QBB if full 

196 # butler is not defined). 

197 limited_butler: LimitedButler 

198 if self.butler is not None: 

199 limited_butler = self.butler 

200 else: 

201 # We check this in constructor, but mypy needs this check here. 

202 assert self.limited_butler_factory is not None 

203 limited_butler = self.limited_butler_factory(quantum) 

204 

205 if self.butler is not None: 

206 log_capture = LogCapture.from_full(self.butler) 

207 else: 

208 log_capture = LogCapture.from_limited(limited_butler) 

209 with log_capture.capture_logging(taskDef, quantum) as captureLog: 

210 # Save detailed resource usage before task start to metadata. 

211 quantumMetadata = _TASK_METADATA_TYPE() 

212 logInfo(None, "prep", metadata=quantumMetadata) # type: ignore[arg-type] 

213 

214 # check whether to skip or delete old outputs, if it returns True 

215 # or raises an exception do not try to store logs, as they may be 

216 # already in butler. 

217 captureLog.store = False 

218 if self.checkExistingOutputs(quantum, taskDef, limited_butler): 

219 _LOG.info( 

220 "Skipping already-successful quantum for label=%s dataId=%s.", 

221 taskDef.label, 

222 quantum.dataId, 

223 ) 

224 return quantum 

225 captureLog.store = True 

226 

227 try: 

228 quantum = self.updatedQuantumInputs(quantum, taskDef, limited_butler) 

229 except NoWorkFound as exc: 

230 _LOG.info( 

231 "Nothing to do for task '%s' on quantum %s; saving metadata and skipping: %s", 

232 taskDef.label, 

233 quantum.dataId, 

234 str(exc), 

235 ) 

236 # Make empty metadata that looks something like what a 

237 # do-nothing task would write (but we don't bother with empty 

238 # nested PropertySets for subtasks). This is slightly 

239 # duplicative with logic in pipe_base that we can't easily call 

240 # from here; we'll fix this on DM-29761. 

241 logInfo(None, "end", metadata=quantumMetadata) # type: ignore[arg-type] 

242 fullMetadata = _TASK_FULL_METADATA_TYPE() 

243 fullMetadata[taskDef.label] = _TASK_METADATA_TYPE() 

244 fullMetadata["quantum"] = quantumMetadata 

245 self.writeMetadata(quantum, fullMetadata, taskDef, limited_butler) 

246 return quantum 

247 

248 # enable lsstDebug debugging 

249 if self.enableLsstDebug: 

250 try: 

251 _LOG.debug("Will try to import debug.py") 

252 import debug # type: ignore # noqa:F401 

253 except ImportError: 

254 _LOG.warn("No 'debug' module found.") 

255 

256 # initialize global state 

257 self.initGlobals(quantum) 

258 

259 # Ensure that we are executing a frozen config 

260 taskDef.config.freeze() 

261 logInfo(None, "init", metadata=quantumMetadata) # type: ignore[arg-type] 

262 init_input_refs = [] 

263 for ref in quantum.initInputs.values(): 

264 resolved = self._resolve_ref(ref) 

265 if resolved is None: 

266 raise ValueError(f"Failed to resolve init input reference {ref}") 

267 init_input_refs.append(resolved) 

268 task = self.taskFactory.makeTask(taskDef, limited_butler, init_input_refs) 

269 logInfo(None, "start", metadata=quantumMetadata) # type: ignore[arg-type] 

270 try: 

271 if self.mock: 

272 # Use mock task instance to execute method. 

273 runTask = self._makeMockTask(taskDef) 

274 else: 

275 runTask = task 

276 self.runQuantum(runTask, quantum, taskDef, limited_butler) 

277 except Exception as e: 

278 _LOG.error( 

279 "Execution of task '%s' on quantum %s failed. Exception %s: %s", 

280 taskDef.label, 

281 quantum.dataId, 

282 e.__class__.__name__, 

283 str(e), 

284 ) 

285 raise 

286 logInfo(None, "end", metadata=quantumMetadata) # type: ignore[arg-type] 

287 fullMetadata = task.getFullMetadata() 

288 fullMetadata["quantum"] = quantumMetadata 

289 self.writeMetadata(quantum, fullMetadata, taskDef, limited_butler) 

290 stopTime = time.time() 

291 _LOG.info( 

292 "Execution of task '%s' on quantum %s took %.3f seconds", 

293 taskDef.label, 

294 quantum.dataId, 

295 stopTime - startTime, 

296 ) 

297 return quantum 

298 

299 def _makeMockTask(self, taskDef: TaskDef) -> PipelineTask: 

300 """Make an instance of mock task for given TaskDef.""" 

301 # Make config instance and apply overrides 

302 overrides = ConfigOverrides() 

303 for action in self.mock_configs: 

304 if action.label == taskDef.label + "-mock": 

305 if action.action == "config": 

306 key, _, value = action.value.partition("=") 

307 overrides.addValueOverride(key, value) 

308 elif action.action == "configfile": 

309 overrides.addFileOverride(os.path.expandvars(action.value)) 

310 else: 

311 raise ValueError(f"Unexpected action for mock task config overrides: {action}") 

312 config = MockPipelineTask.ConfigClass() 

313 overrides.applyTo(config) 

314 

315 task = MockPipelineTask(config=config, name=taskDef.label) 

316 return task 

317 

318 def checkExistingOutputs(self, quantum: Quantum, taskDef: TaskDef, limited_butler: LimitedButler) -> bool: 

319 """Decide whether this quantum needs to be executed. 

320 

321 If only partial outputs exist then they are removed if 

322 ``clobberOutputs`` is True, otherwise an exception is raised. 

323 

324 Parameters 

325 ---------- 

326 quantum : `~lsst.daf.butler.Quantum` 

327 Quantum to check for existing outputs 

328 taskDef : `~lsst.pipe.base.TaskDef` 

329 Task definition structure. 

330 

331 Returns 

332 ------- 

333 exist : `bool` 

334 `True` if ``self.skipExistingIn`` is defined, and a previous 

335 execution of this quanta appears to have completed successfully 

336 (either because metadata was written or all datasets were written). 

337 `False` otherwise. 

338 

339 Raises 

340 ------ 

341 RuntimeError 

342 Raised if some outputs exist and some not. 

343 """ 

344 if self.skipExistingIn and taskDef.metadataDatasetName is not None: 

345 # Metadata output exists; this is sufficient to assume the previous 

346 # run was successful and should be skipped. 

347 [metadata_ref] = quantum.outputs[taskDef.metadataDatasetName] 

348 ref = self._resolve_ref(metadata_ref, self.skipExistingIn) 

349 if ref is not None: 

350 if limited_butler.datastore.exists(ref): 

351 return True 

352 

353 # Previously we always checked for existing outputs in `butler.run`, 

354 # now logic gets more complicated as we only want to skip quantum 

355 # whose outputs exist in `self.skipExistingIn` but pruning should only 

356 # be done for outputs existing in `butler.run`. 

357 

358 def findOutputs( 

359 collections: Optional[Union[str, list[str]]] 

360 ) -> tuple[list[DatasetRef], list[DatasetRef]]: 

361 """Find quantum outputs in specified collections.""" 

362 existingRefs = [] 

363 missingRefs = [] 

364 for datasetRefs in quantum.outputs.values(): 

365 checkRefs: list[DatasetRef] = [] 

366 registryRefToQuantumRef: dict[DatasetRef, DatasetRef] = {} 

367 for datasetRef in datasetRefs: 

368 ref = self._resolve_ref(datasetRef, collections) 

369 if ref is None: 

370 missingRefs.append(datasetRef) 

371 else: 

372 checkRefs.append(ref) 

373 registryRefToQuantumRef[ref] = datasetRef 

374 

375 # More efficient to ask the datastore in bulk for ref 

376 # existence rather than one at a time. 

377 existence = limited_butler.datastore.mexists(checkRefs) 

378 for ref, exists in existence.items(): 

379 if exists: 

380 existingRefs.append(ref) 

381 else: 

382 missingRefs.append(registryRefToQuantumRef[ref]) 

383 return existingRefs, missingRefs 

384 

385 # If skipExistingIn is None this will search in butler.run. 

386 existingRefs, missingRefs = findOutputs(self.skipExistingIn) 

387 if self.skipExistingIn: 

388 if existingRefs and not missingRefs: 

389 # Everything is already there, and we do not clobber complete 

390 # outputs if skipExistingIn is specified. 

391 return True 

392 

393 # If we are to re-run quantum then prune datasets that exists in 

394 # output run collection, only if `self.clobberOutputs` is set, 

395 # that only works when we have full butler. 

396 if existingRefs and self.butler is not None: 

397 # Look at butler run instead of skipExistingIn collections. 

398 existingRefs, missingRefs = findOutputs(self.butler.run) 

399 if existingRefs and missingRefs: 

400 _LOG.debug( 

401 "Partial outputs exist for task %s dataId=%s collection=%s " 

402 "existingRefs=%s missingRefs=%s", 

403 taskDef, 

404 quantum.dataId, 

405 self.butler.run, 

406 existingRefs, 

407 missingRefs, 

408 ) 

409 if self.clobberOutputs: 

410 # only prune 

411 _LOG.info("Removing partial outputs for task %s: %s", taskDef, existingRefs) 

412 self.butler.pruneDatasets(existingRefs, disassociate=True, unstore=True, purge=True) 

413 return False 

414 else: 

415 raise RuntimeError( 

416 "Registry inconsistency while checking for existing outputs:" 

417 f" collection={self.butler.run} existingRefs={existingRefs}" 

418 f" missingRefs={missingRefs}" 

419 ) 

420 elif existingRefs and self.clobberOutputs and not self.skipExistingIn: 

421 # Clobber complete outputs if skipExistingIn is not specified. 

422 _LOG.info("Removing complete outputs for task %s: %s", taskDef, existingRefs) 

423 self.butler.pruneDatasets(existingRefs, disassociate=True, unstore=True, purge=True) 

424 return False 

425 

426 # need to re-run 

427 return False 

428 

429 def updatedQuantumInputs( 

430 self, quantum: Quantum, taskDef: TaskDef, limited_butler: LimitedButler 

431 ) -> Quantum: 

432 """Update quantum with extra information, returns a new updated 

433 Quantum. 

434 

435 Some methods may require input DatasetRefs to have non-None 

436 ``dataset_id``, but in case of intermediate dataset it may not be 

437 filled during QuantumGraph construction. This method will retrieve 

438 missing info from registry. 

439 

440 Parameters 

441 ---------- 

442 quantum : `~lsst.daf.butler.Quantum` 

443 Single Quantum instance. 

444 taskDef : `~lsst.pipe.base.TaskDef` 

445 Task definition structure. 

446 

447 Returns 

448 ------- 

449 update : `~lsst.daf.butler.Quantum` 

450 Updated Quantum instance 

451 """ 

452 anyChanges = False 

453 updatedInputs: defaultdict[DatasetType, list] = defaultdict(list) 

454 for key, refsForDatasetType in quantum.inputs.items(): 

455 newRefsForDatasetType = updatedInputs[key] 

456 for ref in refsForDatasetType: 

457 # Inputs may already be resolved even if they do not exist, but 

458 # we have to re-resolve them because IDs are ignored on output. 

459 # Check datastore for existence first to cover calibration 

460 # dataset types, as they would need a timespan for findDataset. 

461 resolvedRef: DatasetRef | None 

462 checked_datastore = False 

463 if ref.id is not None and limited_butler.datastore.exists(ref): 

464 resolvedRef = ref 

465 checked_datastore = True 

466 elif self.butler is not None: 

467 # In case of full butler try to (re-)resolve it. 

468 resolvedRef = self._resolve_ref(ref) 

469 if resolvedRef is None: 

470 _LOG.info("No dataset found for %s", ref) 

471 continue 

472 else: 

473 _LOG.debug("Updated dataset ID for %s", ref) 

474 else: 

475 # QBB with missing intermediate 

476 _LOG.info("No dataset found for %s", ref) 

477 continue 

478 

479 # In case of mock execution we check that mock dataset exists 

480 # instead. Mock execution is only possible with full butler. 

481 if self.mock and self.butler is not None: 

482 try: 

483 typeName, component = ref.datasetType.nameAndComponent() 

484 if component is not None: 

485 mockDatasetTypeName = MockButlerQuantumContext.mockDatasetTypeName(typeName) 

486 else: 

487 mockDatasetTypeName = MockButlerQuantumContext.mockDatasetTypeName( 

488 ref.datasetType.name 

489 ) 

490 

491 mockDatasetType = self.butler.registry.getDatasetType(mockDatasetTypeName) 

492 except KeyError: 

493 # means that mock dataset type is not there and this 

494 # should be a pre-existing dataset 

495 _LOG.debug("No mock dataset type for %s", ref) 

496 if self.butler.datastore.exists(resolvedRef): 

497 newRefsForDatasetType.append(resolvedRef) 

498 else: 

499 resolvedMockRef = self.butler.registry.findDataset( 

500 mockDatasetType, ref.dataId, collections=self.butler.collections 

501 ) 

502 _LOG.debug( 

503 "mockRef=(%s, %s) resolvedMockRef=%s", 

504 mockDatasetType, 

505 ref.dataId, 

506 resolvedMockRef, 

507 ) 

508 if resolvedMockRef is not None and self.butler.datastore.exists(resolvedMockRef): 

509 _LOG.debug("resolvedMockRef dataset exists") 

510 newRefsForDatasetType.append(resolvedRef) 

511 elif checked_datastore or limited_butler.datastore.exists(resolvedRef): 

512 # We need to ask datastore if the dataset actually exists 

513 # because the Registry of a local "execution butler" 

514 # cannot know this (because we prepopulate it with all of 

515 # the datasets that might be created). 

516 newRefsForDatasetType.append(resolvedRef) 

517 

518 if len(newRefsForDatasetType) != len(refsForDatasetType): 

519 anyChanges = True 

520 # If we removed any input datasets, let the task check if it has enough 

521 # to proceed and/or prune related datasets that it also doesn't 

522 # need/produce anymore. It will raise NoWorkFound if it can't run, 

523 # which we'll let propagate up. This is exactly what we run during QG 

524 # generation, because a task shouldn't care whether an input is missing 

525 # because some previous task didn't produce it, or because it just 

526 # wasn't there during QG generation. 

527 namedUpdatedInputs = NamedKeyDict[DatasetType, list[DatasetRef]](updatedInputs.items()) 

528 helper = AdjustQuantumHelper(namedUpdatedInputs, quantum.outputs) 

529 if anyChanges: 

530 assert quantum.dataId is not None, "Quantum DataId cannot be None" 

531 helper.adjust_in_place(taskDef.connections, label=taskDef.label, data_id=quantum.dataId) 

532 return Quantum( 

533 taskName=quantum.taskName, 

534 taskClass=quantum.taskClass, 

535 dataId=quantum.dataId, 

536 initInputs=quantum.initInputs, 

537 inputs=helper.inputs, 

538 outputs=helper.outputs, 

539 ) 

540 

541 def runQuantum( 

542 self, task: PipelineTask, quantum: Quantum, taskDef: TaskDef, limited_butler: LimitedButler 

543 ) -> None: 

544 """Execute task on a single quantum. 

545 

546 Parameters 

547 ---------- 

548 task : `~lsst.pipe.base.PipelineTask` 

549 Task object. 

550 quantum : `~lsst.daf.butler.Quantum` 

551 Single Quantum instance. 

552 taskDef : `~lsst.pipe.base.TaskDef` 

553 Task definition structure. 

554 """ 

555 # Create a butler that operates in the context of a quantum 

556 if self.butler is None: 

557 butlerQC = ButlerQuantumContext.from_limited(limited_butler, quantum) 

558 else: 

559 if self.mock: 

560 butlerQC = MockButlerQuantumContext(self.butler, quantum) 

561 else: 

562 butlerQC = ButlerQuantumContext.from_full(self.butler, quantum) 

563 

564 # Get the input and output references for the task 

565 inputRefs, outputRefs = taskDef.connections.buildDatasetRefs(quantum) 

566 

567 # Call task runQuantum() method. Catch a few known failure modes and 

568 # translate them into specific 

569 try: 

570 task.runQuantum(butlerQC, inputRefs, outputRefs) 

571 except NoWorkFound as err: 

572 # Not an error, just an early exit. 

573 _LOG.info("Task '%s' on quantum %s exited early: %s", taskDef.label, quantum.dataId, str(err)) 

574 pass 

575 except RepeatableQuantumError as err: 

576 if self.exitOnKnownError: 

577 _LOG.warning("Caught repeatable quantum error for %s (%s):", taskDef, quantum.dataId) 

578 _LOG.warning(err, exc_info=True) 

579 sys.exit(err.EXIT_CODE) 

580 else: 

581 raise 

582 except InvalidQuantumError as err: 

583 _LOG.fatal("Invalid quantum error for %s (%s): %s", taskDef, quantum.dataId) 

584 _LOG.fatal(err, exc_info=True) 

585 sys.exit(err.EXIT_CODE) 

586 

587 def writeMetadata( 

588 self, quantum: Quantum, metadata: Any, taskDef: TaskDef, limited_butler: LimitedButler 

589 ) -> None: 

590 if taskDef.metadataDatasetName is not None: 

591 # DatasetRef has to be in the Quantum outputs, can lookup by name 

592 try: 

593 [ref] = quantum.outputs[taskDef.metadataDatasetName] 

594 except LookupError as exc: 

595 raise InvalidQuantumError( 

596 f"Quantum outputs is missing metadata dataset type {taskDef.metadataDatasetName};" 

597 " this could happen due to inconsistent options between QuantumGraph generation" 

598 " and execution" 

599 ) from exc 

600 if self.butler is not None: 

601 # Dataset ref can already be resolved, for non-QBB executor we 

602 # have to ignore that because may be overriding run 

603 # collection. 

604 if ref.id is not None: 

605 with warnings.catch_warnings(): 

606 warnings.simplefilter("ignore", category=UnresolvedRefWarning) 

607 ref = ref.unresolved() 

608 self.butler.put(metadata, ref) 

609 else: 

610 limited_butler.put(metadata, ref) 

611 

612 def initGlobals(self, quantum: Quantum) -> None: 

613 """Initialize global state needed for task execution. 

614 

615 Parameters 

616 ---------- 

617 quantum : `~lsst.daf.butler.Quantum` 

618 Single Quantum instance. 

619 

620 Notes 

621 ----- 

622 There is an issue with initializing filters singleton which is done 

623 by instrument, to avoid requiring tasks to do it in runQuantum() 

624 we do it here when any dataId has an instrument dimension. Also for 

625 now we only allow single instrument, verify that all instrument 

626 names in all dataIds are identical. 

627 

628 This will need revision when filter singleton disappears. 

629 """ 

630 # can only work for full butler 

631 if self.butler is None: 

632 return 

633 oneInstrument = None 

634 for datasetRefs in chain(quantum.inputs.values(), quantum.outputs.values()): 

635 for datasetRef in datasetRefs: 

636 dataId = datasetRef.dataId 

637 instrument = dataId.get("instrument") 

638 if instrument is not None: 

639 if oneInstrument is not None: 

640 assert ( # type: ignore 

641 instrument == oneInstrument 

642 ), "Currently require that only one instrument is used per graph" 

643 else: 

644 oneInstrument = instrument 

645 Instrument.fromName(instrument, self.butler.registry) 

646 

647 def getReport(self) -> Optional[QuantumReport]: 

648 # Docstring inherited from base class 

649 if self.report is None: 

650 raise RuntimeError("getReport() called before execute()") 

651 return self.report