Coverage for python/lsst/ctrl/mpexec/singleQuantumExecutor.py: 11%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

271 statements  

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22__all__ = ["SingleQuantumExecutor"] 

23 

24# ------------------------------- 

25# Imports of standard modules -- 

26# ------------------------------- 

27import logging 

28import os 

29import shutil 

30import sys 

31import tempfile 

32import time 

33from collections import defaultdict 

34from contextlib import contextmanager 

35from itertools import chain 

36from logging import FileHandler 

37from typing import Dict, List 

38 

39from lsst.daf.butler import DatasetRef, DatasetType, FileDataset, NamedKeyDict, Quantum 

40from lsst.daf.butler.core.logging import ButlerLogRecordHandler, ButlerLogRecords, ButlerMDC, JsonLogFormatter 

41from lsst.obs.base import Instrument 

42from lsst.pipe.base import ( 

43 AdjustQuantumHelper, 

44 ButlerQuantumContext, 

45 InvalidQuantumError, 

46 NoWorkFound, 

47 RepeatableQuantumError, 

48) 

49from lsst.pipe.base.configOverrides import ConfigOverrides 

50 

51# During metadata transition phase, determine metadata class by 

52# asking pipe_base 

53from lsst.pipe.base.task import _TASK_FULL_METADATA_TYPE, _TASK_METADATA_TYPE 

54from lsst.utils.timer import logInfo 

55 

56# ----------------------------- 

57# Imports for other modules -- 

58# ----------------------------- 

59from .mock_task import MockButlerQuantumContext, MockPipelineTask 

60from .quantumGraphExecutor import QuantumExecutor 

61 

62# ---------------------------------- 

63# Local non-exported definitions -- 

64# ---------------------------------- 

65 

66_LOG = logging.getLogger(__name__) 

67 

68 

69class _LogCaptureFlag: 

70 """Simple flag to enable/disable log-to-butler saving.""" 

71 

72 store: bool = True 

73 

74 

75class SingleQuantumExecutor(QuantumExecutor): 

76 """Executor class which runs one Quantum at a time. 

77 

78 Parameters 

79 ---------- 

80 butler : `~lsst.daf.butler.Butler` 

81 Data butler. 

82 taskFactory : `~lsst.pipe.base.TaskFactory` 

83 Instance of a task factory. 

84 skipExistingIn : `list` [ `str` ], optional 

85 Accepts list of collections, if all Quantum outputs already exist in 

86 the specified list of collections then that Quantum will not be rerun. 

87 clobberOutputs : `bool`, optional 

88 If `True`, then existing outputs in output run collection will be 

89 overwritten. If ``skipExistingIn`` is defined, only outputs from 

90 failed quanta will be overwritten. 

91 enableLsstDebug : `bool`, optional 

92 Enable debugging with ``lsstDebug`` facility for a task. 

93 exitOnKnownError : `bool`, optional 

94 If `True`, call `sys.exit` with the appropriate exit code for special 

95 known exceptions, after printing a traceback, instead of letting the 

96 exception propagate up to calling. This is always the behavior for 

97 InvalidQuantumError. 

98 mock : `bool`, optional 

99 If `True` then mock task execution. 

100 mock_configs : `list` [ `_PipelineAction` ], optional 

101 Optional config overrides for mock tasks. 

102 """ 

103 

104 stream_json_logs = True 

105 """If True each log record is written to a temporary file and ingested 

106 when quantum completes. If False the records are accumulated in memory 

107 and stored in butler on quantum completion.""" 

108 

109 def __init__( 

110 self, 

111 taskFactory, 

112 skipExistingIn=None, 

113 clobberOutputs=False, 

114 enableLsstDebug=False, 

115 exitOnKnownError=False, 

116 mock=False, 

117 mock_configs=None, 

118 ): 

119 self.taskFactory = taskFactory 

120 self.skipExistingIn = skipExistingIn 

121 self.enableLsstDebug = enableLsstDebug 

122 self.clobberOutputs = clobberOutputs 

123 self.exitOnKnownError = exitOnKnownError 

124 self.mock = mock 

125 self.mock_configs = mock_configs 

126 self.log_handler = None 

127 

128 def execute(self, taskDef, quantum, butler): 

129 # Docstring inherited from QuantumExecutor.execute 

130 startTime = time.time() 

131 

132 with self.captureLogging(taskDef, quantum, butler) as captureLog: 

133 

134 # Save detailed resource usage before task start to metadata. 

135 quantumMetadata = _TASK_METADATA_TYPE() 

136 logInfo(None, "prep", metadata=quantumMetadata) 

137 

138 taskClass, label, config = taskDef.taskClass, taskDef.label, taskDef.config 

139 

140 # check whether to skip or delete old outputs, if it returns True 

141 # or raises an exception do not try to store logs, as they may be 

142 # already in butler. 

143 captureLog.store = False 

144 if self.checkExistingOutputs(quantum, butler, taskDef): 

145 _LOG.info( 

146 "Skipping already-successful quantum for label=%s dataId=%s.", label, quantum.dataId 

147 ) 

148 return 

149 captureLog.store = True 

150 

151 try: 

152 quantum = self.updatedQuantumInputs(quantum, butler, taskDef) 

153 except NoWorkFound as exc: 

154 _LOG.info( 

155 "Nothing to do for task '%s' on quantum %s; saving metadata and skipping: %s", 

156 taskDef.label, 

157 quantum.dataId, 

158 str(exc), 

159 ) 

160 # Make empty metadata that looks something like what a 

161 # do-nothing task would write (but we don't bother with empty 

162 # nested PropertySets for subtasks). This is slightly 

163 # duplicative with logic in pipe_base that we can't easily call 

164 # from here; we'll fix this on DM-29761. 

165 logInfo(None, "end", metadata=quantumMetadata) 

166 fullMetadata = _TASK_FULL_METADATA_TYPE() 

167 fullMetadata[taskDef.label] = _TASK_METADATA_TYPE() 

168 fullMetadata["quantum"] = quantumMetadata 

169 self.writeMetadata(quantum, fullMetadata, taskDef, butler) 

170 return 

171 

172 # enable lsstDebug debugging 

173 if self.enableLsstDebug: 

174 try: 

175 _LOG.debug("Will try to import debug.py") 

176 import debug # noqa:F401 

177 except ImportError: 

178 _LOG.warn("No 'debug' module found.") 

179 

180 # initialize global state 

181 self.initGlobals(quantum, butler) 

182 

183 # Ensure that we are executing a frozen config 

184 config.freeze() 

185 logInfo(None, "init", metadata=quantumMetadata) 

186 task = self.makeTask(taskClass, label, config, butler) 

187 logInfo(None, "start", metadata=quantumMetadata) 

188 try: 

189 if self.mock: 

190 # Use mock task instance to execute method. 

191 runTask = self._makeMockTask(taskDef) 

192 else: 

193 runTask = task 

194 self.runQuantum(runTask, quantum, taskDef, butler) 

195 except Exception as e: 

196 _LOG.error( 

197 "Execution of task '%s' on quantum %s failed. Exception %s: %s", 

198 taskDef.label, 

199 quantum.dataId, 

200 e.__class__.__name__, 

201 str(e), 

202 ) 

203 raise 

204 logInfo(None, "end", metadata=quantumMetadata) 

205 fullMetadata = task.getFullMetadata() 

206 fullMetadata["quantum"] = quantumMetadata 

207 self.writeMetadata(quantum, fullMetadata, taskDef, butler) 

208 stopTime = time.time() 

209 _LOG.info( 

210 "Execution of task '%s' on quantum %s took %.3f seconds", 

211 taskDef.label, 

212 quantum.dataId, 

213 stopTime - startTime, 

214 ) 

215 return quantum 

216 

217 def _makeMockTask(self, taskDef): 

218 """Make an instance of mock task for given TaskDef.""" 

219 # Make config instance and apply overrides 

220 overrides = ConfigOverrides() 

221 for action in self.mock_configs: 

222 if action.label == taskDef.label + "-mock": 

223 if action.action == "config": 

224 key, _, value = action.value.partition("=") 

225 overrides.addValueOverride(key, value) 

226 elif action.action == "configfile": 

227 overrides.addFileOverride(os.path.expandvars(action.value)) 

228 else: 

229 raise ValueError(f"Unexpected action for mock task config overrides: {action}") 

230 config = MockPipelineTask.ConfigClass() 

231 overrides.applyTo(config) 

232 

233 task = MockPipelineTask(config=config, name=taskDef.label) 

234 return task 

235 

236 @contextmanager 

237 def captureLogging(self, taskDef, quantum, butler): 

238 """Configure logging system to capture logs for execution of this task. 

239 

240 Parameters 

241 ---------- 

242 taskDef : `lsst.pipe.base.TaskDef` 

243 The task definition. 

244 quantum : `~lsst.daf.butler.Quantum` 

245 Single Quantum instance. 

246 butler : `~lsst.daf.butler.Butler` 

247 Butler to write logs to. 

248 

249 Notes 

250 ----- 

251 Expected to be used as a context manager to ensure that logging 

252 records are inserted into the butler once the quantum has been 

253 executed: 

254 

255 .. code-block:: py 

256 

257 with self.captureLogging(taskDef, quantum, butler): 

258 # Run quantum and capture logs. 

259 

260 Ths method can also setup logging to attach task- or 

261 quantum-specific information to log messages. Potentially this can 

262 take into account some info from task configuration as well. 

263 """ 

264 # Add a handler to the root logger to capture execution log output. 

265 # How does it get removed reliably? 

266 if taskDef.logOutputDatasetName is not None: 

267 # Either accumulate into ButlerLogRecords or stream 

268 # JSON records to file and ingest that. 

269 tmpdir = None 

270 if self.stream_json_logs: 

271 # Create the log file in a temporary directory rather than 

272 # creating a temporary file. This is necessary because 

273 # temporary files are created with restrictive permissions 

274 # and during file ingest these permissions persist in the 

275 # datastore. Using a temp directory allows us to create 

276 # a file with umask default permissions. 

277 tmpdir = tempfile.mkdtemp(prefix="butler-temp-logs-") 

278 

279 # Construct a file to receive the log records and "touch" it. 

280 log_file = os.path.join(tmpdir, f"butler-log-{taskDef.label}.json") 

281 with open(log_file, "w"): 

282 pass 

283 self.log_handler = FileHandler(log_file) 

284 self.log_handler.setFormatter(JsonLogFormatter()) 

285 else: 

286 self.log_handler = ButlerLogRecordHandler() 

287 

288 logging.getLogger().addHandler(self.log_handler) 

289 

290 # include quantum dataId and task label into MDC 

291 label = taskDef.label 

292 if quantum.dataId: 

293 label += f":{quantum.dataId}" 

294 

295 ctx = _LogCaptureFlag() 

296 try: 

297 with ButlerMDC.set_mdc({"LABEL": label, "RUN": butler.run}): 

298 yield ctx 

299 finally: 

300 # Ensure that the logs are stored in butler. 

301 self.writeLogRecords(quantum, taskDef, butler, ctx.store) 

302 if tmpdir: 

303 shutil.rmtree(tmpdir, ignore_errors=True) 

304 

305 def checkExistingOutputs(self, quantum, butler, taskDef): 

306 """Decide whether this quantum needs to be executed. 

307 

308 If only partial outputs exist then they are removed if 

309 ``clobberOutputs`` is True, otherwise an exception is raised. 

310 

311 Parameters 

312 ---------- 

313 quantum : `~lsst.daf.butler.Quantum` 

314 Quantum to check for existing outputs 

315 butler : `~lsst.daf.butler.Butler` 

316 Data butler. 

317 taskDef : `~lsst.pipe.base.TaskDef` 

318 Task definition structure. 

319 

320 Returns 

321 ------- 

322 exist : `bool` 

323 `True` if ``self.skipExistingIn`` is defined, and a previous 

324 execution of this quanta appears to have completed successfully 

325 (either because metadata was written or all datasets were written). 

326 `False` otherwise. 

327 

328 Raises 

329 ------ 

330 RuntimeError 

331 Raised if some outputs exist and some not. 

332 """ 

333 if self.skipExistingIn and taskDef.metadataDatasetName is not None: 

334 # Metadata output exists; this is sufficient to assume the previous 

335 # run was successful and should be skipped. 

336 ref = butler.registry.findDataset( 

337 taskDef.metadataDatasetName, quantum.dataId, collections=self.skipExistingIn 

338 ) 

339 if ref is not None: 

340 if butler.datastore.exists(ref): 

341 return True 

342 

343 # Previously we always checked for existing outputs in `butler.run`, 

344 # now logic gets more complicated as we only want to skip quantum 

345 # whose outputs exist in `self.skipExistingIn` but pruning should only 

346 # be done for outputs existing in `butler.run`. 

347 

348 def findOutputs(collections): 

349 """Find quantum outputs in specified collections.""" 

350 existingRefs = [] 

351 missingRefs = [] 

352 for datasetRefs in quantum.outputs.values(): 

353 checkRefs: List[DatasetRef] = [] 

354 registryRefToQuantumRef: Dict[DatasetRef, DatasetRef] = {} 

355 for datasetRef in datasetRefs: 

356 ref = butler.registry.findDataset( 

357 datasetRef.datasetType, datasetRef.dataId, collections=collections 

358 ) 

359 if ref is None: 

360 missingRefs.append(datasetRef) 

361 else: 

362 checkRefs.append(ref) 

363 registryRefToQuantumRef[ref] = datasetRef 

364 

365 # More efficient to ask the datastore in bulk for ref 

366 # existence rather than one at a time. 

367 existence = butler.datastore.mexists(checkRefs) 

368 for ref, exists in existence.items(): 

369 if exists: 

370 existingRefs.append(ref) 

371 else: 

372 missingRefs.append(registryRefToQuantumRef[ref]) 

373 return existingRefs, missingRefs 

374 

375 existingRefs, missingRefs = findOutputs(self.skipExistingIn) 

376 if self.skipExistingIn: 

377 if existingRefs and not missingRefs: 

378 # everything is already there 

379 return True 

380 

381 # If we are to re-run quantum then prune datasets that exists in 

382 # output run collection, only if `self.clobberOutputs` is set. 

383 if existingRefs: 

384 existingRefs, missingRefs = findOutputs(butler.run) 

385 if existingRefs and missingRefs: 

386 _LOG.debug( 

387 "Partial outputs exist for task %s dataId=%s collection=%s " 

388 "existingRefs=%s missingRefs=%s", 

389 taskDef, 

390 quantum.dataId, 

391 butler.run, 

392 existingRefs, 

393 missingRefs, 

394 ) 

395 if self.clobberOutputs: 

396 # only prune 

397 _LOG.info("Removing partial outputs for task %s: %s", taskDef, existingRefs) 

398 # Do not purge registry records if this looks like 

399 # an execution butler. This ensures that the UUID 

400 # of the dataset doesn't change. 

401 if butler._allow_put_of_predefined_dataset: 

402 purge = False 

403 disassociate = False 

404 else: 

405 purge = True 

406 disassociate = True 

407 butler.pruneDatasets(existingRefs, disassociate=disassociate, unstore=True, purge=purge) 

408 return False 

409 else: 

410 raise RuntimeError( 

411 f"Registry inconsistency while checking for existing outputs:" 

412 f" collection={butler.run} existingRefs={existingRefs}" 

413 f" missingRefs={missingRefs}" 

414 ) 

415 

416 # need to re-run 

417 return False 

418 

419 def makeTask(self, taskClass, name, config, butler): 

420 """Make new task instance. 

421 

422 Parameters 

423 ---------- 

424 taskClass : `type` 

425 Sub-class of `~lsst.pipe.base.PipelineTask`. 

426 name : `str` 

427 Name for this task. 

428 config : `~lsst.pipe.base.PipelineTaskConfig` 

429 Configuration object for this task 

430 

431 Returns 

432 ------- 

433 task : `~lsst.pipe.base.PipelineTask` 

434 Instance of ``taskClass`` type. 

435 butler : `~lsst.daf.butler.Butler` 

436 Data butler. 

437 """ 

438 # call task factory for that 

439 return self.taskFactory.makeTask(taskClass, name, config, None, butler) 

440 

441 def updatedQuantumInputs(self, quantum, butler, taskDef): 

442 """Update quantum with extra information, returns a new updated 

443 Quantum. 

444 

445 Some methods may require input DatasetRefs to have non-None 

446 ``dataset_id``, but in case of intermediate dataset it may not be 

447 filled during QuantumGraph construction. This method will retrieve 

448 missing info from registry. 

449 

450 Parameters 

451 ---------- 

452 quantum : `~lsst.daf.butler.Quantum` 

453 Single Quantum instance. 

454 butler : `~lsst.daf.butler.Butler` 

455 Data butler. 

456 taskDef : `~lsst.pipe.base.TaskDef` 

457 Task definition structure. 

458 

459 Returns 

460 ------- 

461 update : `~lsst.daf.butler.Quantum` 

462 Updated Quantum instance 

463 """ 

464 anyChanges = False 

465 updatedInputs = defaultdict(list) 

466 for key, refsForDatasetType in quantum.inputs.items(): 

467 newRefsForDatasetType = updatedInputs[key] 

468 for ref in refsForDatasetType: 

469 if ref.id is None: 

470 resolvedRef = butler.registry.findDataset( 

471 ref.datasetType, ref.dataId, collections=butler.collections 

472 ) 

473 if resolvedRef is None: 

474 _LOG.info("No dataset found for %s", ref) 

475 continue 

476 else: 

477 _LOG.debug("Updated dataset ID for %s", ref) 

478 else: 

479 resolvedRef = ref 

480 # We need to ask datastore if the dataset actually exists 

481 # because the Registry of a local "execution butler" cannot 

482 # know this (because we prepopulate it with all of the datasets 

483 # that might be created). In case of mock execution we check 

484 # that mock dataset exists instead. 

485 if self.mock: 

486 try: 

487 typeName, component = ref.datasetType.nameAndComponent() 

488 if component is not None: 

489 mockDatasetTypeName = MockButlerQuantumContext.mockDatasetTypeName(typeName) 

490 else: 

491 mockDatasetTypeName = MockButlerQuantumContext.mockDatasetTypeName( 

492 ref.datasetType.name 

493 ) 

494 

495 mockDatasetType = butler.registry.getDatasetType(mockDatasetTypeName) 

496 except KeyError: 

497 # means that mock dataset type is not there and this 

498 # should be a pre-existing dataset 

499 _LOG.debug("No mock dataset type for %s", ref) 

500 if butler.datastore.exists(resolvedRef): 

501 newRefsForDatasetType.append(resolvedRef) 

502 else: 

503 mockRef = DatasetRef(mockDatasetType, ref.dataId) 

504 resolvedMockRef = butler.registry.findDataset( 

505 mockRef.datasetType, mockRef.dataId, collections=butler.collections 

506 ) 

507 _LOG.debug("mockRef=%s resolvedMockRef=%s", mockRef, resolvedMockRef) 

508 if resolvedMockRef is not None and butler.datastore.exists(resolvedMockRef): 

509 _LOG.debug("resolvedMockRef dataset exists") 

510 newRefsForDatasetType.append(resolvedRef) 

511 elif butler.datastore.exists(resolvedRef): 

512 newRefsForDatasetType.append(resolvedRef) 

513 

514 if len(newRefsForDatasetType) != len(refsForDatasetType): 

515 anyChanges = True 

516 # If we removed any input datasets, let the task check if it has enough 

517 # to proceed and/or prune related datasets that it also doesn't 

518 # need/produce anymore. It will raise NoWorkFound if it can't run, 

519 # which we'll let propagate up. This is exactly what we run during QG 

520 # generation, because a task shouldn't care whether an input is missing 

521 # because some previous task didn't produce it, or because it just 

522 # wasn't there during QG generation. 

523 updatedInputs = NamedKeyDict[DatasetType, List[DatasetRef]](updatedInputs.items()) 

524 helper = AdjustQuantumHelper(updatedInputs, quantum.outputs) 

525 if anyChanges: 

526 helper.adjust_in_place(taskDef.connections, label=taskDef.label, data_id=quantum.dataId) 

527 return Quantum( 

528 taskName=quantum.taskName, 

529 taskClass=quantum.taskClass, 

530 dataId=quantum.dataId, 

531 initInputs=quantum.initInputs, 

532 inputs=helper.inputs, 

533 outputs=helper.outputs, 

534 ) 

535 

536 def runQuantum(self, task, quantum, taskDef, butler): 

537 """Execute task on a single quantum. 

538 

539 Parameters 

540 ---------- 

541 task : `~lsst.pipe.base.PipelineTask` 

542 Task object. 

543 quantum : `~lsst.daf.butler.Quantum` 

544 Single Quantum instance. 

545 taskDef : `~lsst.pipe.base.TaskDef` 

546 Task definition structure. 

547 butler : `~lsst.daf.butler.Butler` 

548 Data butler. 

549 """ 

550 # Create a butler that operates in the context of a quantum 

551 if self.mock: 

552 butlerQC = MockButlerQuantumContext(butler, quantum) 

553 else: 

554 butlerQC = ButlerQuantumContext(butler, quantum) 

555 

556 # Get the input and output references for the task 

557 inputRefs, outputRefs = taskDef.connections.buildDatasetRefs(quantum) 

558 

559 # Call task runQuantum() method. Catch a few known failure modes and 

560 # translate them into specific 

561 try: 

562 task.runQuantum(butlerQC, inputRefs, outputRefs) 

563 except NoWorkFound as err: 

564 # Not an error, just an early exit. 

565 _LOG.info("Task '%s' on quantum %s exited early: %s", taskDef.label, quantum.dataId, str(err)) 

566 pass 

567 except RepeatableQuantumError as err: 

568 if self.exitOnKnownError: 

569 _LOG.warning("Caught repeatable quantum error for %s (%s):", taskDef, quantum.dataId) 

570 _LOG.warning(err, exc_info=True) 

571 sys.exit(err.EXIT_CODE) 

572 else: 

573 raise 

574 except InvalidQuantumError as err: 

575 _LOG.fatal("Invalid quantum error for %s (%s): %s", taskDef, quantum.dataId) 

576 _LOG.fatal(err, exc_info=True) 

577 sys.exit(err.EXIT_CODE) 

578 

579 def writeMetadata(self, quantum, metadata, taskDef, butler): 

580 if taskDef.metadataDatasetName is not None: 

581 # DatasetRef has to be in the Quantum outputs, can lookup by name 

582 try: 

583 ref = quantum.outputs[taskDef.metadataDatasetName] 

584 except LookupError as exc: 

585 raise InvalidQuantumError( 

586 f"Quantum outputs is missing metadata dataset type {taskDef.metadataDatasetName};" 

587 f" this could happen due to inconsistent options between QuantumGraph generation" 

588 f" and execution" 

589 ) from exc 

590 butler.put(metadata, ref[0]) 

591 

592 def writeLogRecords(self, quantum, taskDef, butler, store): 

593 # If we are logging to an external file we must always try to 

594 # close it. 

595 filename = None 

596 if isinstance(self.log_handler, FileHandler): 

597 filename = self.log_handler.stream.name 

598 self.log_handler.close() 

599 

600 if self.log_handler is not None: 

601 # Remove the handler so we stop accumulating log messages. 

602 logging.getLogger().removeHandler(self.log_handler) 

603 

604 try: 

605 if store and taskDef.logOutputDatasetName is not None and self.log_handler is not None: 

606 # DatasetRef has to be in the Quantum outputs, can lookup by 

607 # name 

608 try: 

609 ref = quantum.outputs[taskDef.logOutputDatasetName] 

610 except LookupError as exc: 

611 raise InvalidQuantumError( 

612 f"Quantum outputs is missing log output dataset type {taskDef.logOutputDatasetName};" 

613 f" this could happen due to inconsistent options between QuantumGraph generation" 

614 f" and execution" 

615 ) from exc 

616 

617 if isinstance(self.log_handler, ButlerLogRecordHandler): 

618 butler.put(self.log_handler.records, ref[0]) 

619 

620 # Clear the records in case the handler is reused. 

621 self.log_handler.records.clear() 

622 else: 

623 assert filename is not None, "Somehow unable to extract filename from file handler" 

624 

625 # Need to ingest this file directly into butler. 

626 dataset = FileDataset(path=filename, refs=ref[0]) 

627 try: 

628 butler.ingest(dataset, transfer="move") 

629 filename = None 

630 except NotImplementedError: 

631 # Some datastores can't receive files (e.g. in-memory 

632 # datastore when testing), we store empty list for 

633 # those just to have a dataset. Alternative is to read 

634 # the file as a ButlerLogRecords object and put it. 

635 _LOG.info( 

636 "Log records could not be stored in this butler because the" 

637 " datastore can not ingest files, empty record list is stored instead." 

638 ) 

639 records = ButlerLogRecords.from_records([]) 

640 butler.put(records, ref[0]) 

641 finally: 

642 # remove file if it is not ingested 

643 if filename is not None: 

644 try: 

645 os.remove(filename) 

646 except OSError: 

647 pass 

648 

649 def initGlobals(self, quantum, butler): 

650 """Initialize global state needed for task execution. 

651 

652 Parameters 

653 ---------- 

654 quantum : `~lsst.daf.butler.Quantum` 

655 Single Quantum instance. 

656 butler : `~lsst.daf.butler.Butler` 

657 Data butler. 

658 

659 Notes 

660 ----- 

661 There is an issue with initializing filters singleton which is done 

662 by instrument, to avoid requiring tasks to do it in runQuantum() 

663 we do it here when any dataId has an instrument dimension. Also for 

664 now we only allow single instrument, verify that all instrument 

665 names in all dataIds are identical. 

666 

667 This will need revision when filter singleton disappears. 

668 """ 

669 oneInstrument = None 

670 for datasetRefs in chain(quantum.inputs.values(), quantum.outputs.values()): 

671 for datasetRef in datasetRefs: 

672 dataId = datasetRef.dataId 

673 instrument = dataId.get("instrument") 

674 if instrument is not None: 

675 if oneInstrument is not None: 

676 assert ( 

677 instrument == oneInstrument 

678 ), "Currently require that only one instrument is used per graph" 

679 else: 

680 oneInstrument = instrument 

681 Instrument.fromName(instrument, butler.registry)