Coverage for python/lsst/ctrl/mpexec/singleQuantumExecutor.py: 11%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_mpexec.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22__all__ = ["SingleQuantumExecutor"]
24# -------------------------------
25# Imports of standard modules --
26# -------------------------------
27import logging
28import os
29import shutil
30import sys
31import tempfile
32import time
33from collections import defaultdict
34from contextlib import contextmanager
35from itertools import chain
36from logging import FileHandler
37from typing import Dict, List
39from lsst.daf.butler import DatasetRef, DatasetType, FileDataset, NamedKeyDict, Quantum
40from lsst.daf.butler.core.logging import ButlerLogRecordHandler, ButlerLogRecords, ButlerMDC, JsonLogFormatter
41from lsst.obs.base import Instrument
42from lsst.pipe.base import (
43 AdjustQuantumHelper,
44 ButlerQuantumContext,
45 InvalidQuantumError,
46 NoWorkFound,
47 RepeatableQuantumError,
48)
49from lsst.pipe.base.configOverrides import ConfigOverrides
51# During metadata transition phase, determine metadata class by
52# asking pipe_base
53from lsst.pipe.base.task import _TASK_FULL_METADATA_TYPE, _TASK_METADATA_TYPE
54from lsst.utils.timer import logInfo
56# -----------------------------
57# Imports for other modules --
58# -----------------------------
59from .mock_task import MockButlerQuantumContext, MockPipelineTask
60from .quantumGraphExecutor import QuantumExecutor
62# ----------------------------------
63# Local non-exported definitions --
64# ----------------------------------
66_LOG = logging.getLogger(__name__)
69class _LogCaptureFlag:
70 """Simple flag to enable/disable log-to-butler saving."""
72 store: bool = True
75class SingleQuantumExecutor(QuantumExecutor):
76 """Executor class which runs one Quantum at a time.
78 Parameters
79 ----------
80 butler : `~lsst.daf.butler.Butler`
81 Data butler.
82 taskFactory : `~lsst.pipe.base.TaskFactory`
83 Instance of a task factory.
84 skipExistingIn : `list` [ `str` ], optional
85 Accepts list of collections, if all Quantum outputs already exist in
86 the specified list of collections then that Quantum will not be rerun.
87 clobberOutputs : `bool`, optional
88 If `True`, then existing outputs in output run collection will be
89 overwritten. If ``skipExistingIn`` is defined, only outputs from
90 failed quanta will be overwritten.
91 enableLsstDebug : `bool`, optional
92 Enable debugging with ``lsstDebug`` facility for a task.
93 exitOnKnownError : `bool`, optional
94 If `True`, call `sys.exit` with the appropriate exit code for special
95 known exceptions, after printing a traceback, instead of letting the
96 exception propagate up to calling. This is always the behavior for
97 InvalidQuantumError.
98 mock : `bool`, optional
99 If `True` then mock task execution.
100 mock_configs : `list` [ `_PipelineAction` ], optional
101 Optional config overrides for mock tasks.
102 """
104 stream_json_logs = True
105 """If True each log record is written to a temporary file and ingested
106 when quantum completes. If False the records are accumulated in memory
107 and stored in butler on quantum completion."""
109 def __init__(
110 self,
111 taskFactory,
112 skipExistingIn=None,
113 clobberOutputs=False,
114 enableLsstDebug=False,
115 exitOnKnownError=False,
116 mock=False,
117 mock_configs=None,
118 ):
119 self.taskFactory = taskFactory
120 self.skipExistingIn = skipExistingIn
121 self.enableLsstDebug = enableLsstDebug
122 self.clobberOutputs = clobberOutputs
123 self.exitOnKnownError = exitOnKnownError
124 self.mock = mock
125 self.mock_configs = mock_configs
126 self.log_handler = None
128 def execute(self, taskDef, quantum, butler):
129 # Docstring inherited from QuantumExecutor.execute
130 startTime = time.time()
132 with self.captureLogging(taskDef, quantum, butler) as captureLog:
134 # Save detailed resource usage before task start to metadata.
135 quantumMetadata = _TASK_METADATA_TYPE()
136 logInfo(None, "prep", metadata=quantumMetadata)
138 taskClass, label, config = taskDef.taskClass, taskDef.label, taskDef.config
140 # check whether to skip or delete old outputs, if it returns True
141 # or raises an exception do not try to store logs, as they may be
142 # already in butler.
143 captureLog.store = False
144 if self.checkExistingOutputs(quantum, butler, taskDef):
145 _LOG.info(
146 "Skipping already-successful quantum for label=%s dataId=%s.", label, quantum.dataId
147 )
148 return
149 captureLog.store = True
151 try:
152 quantum = self.updatedQuantumInputs(quantum, butler, taskDef)
153 except NoWorkFound as exc:
154 _LOG.info(
155 "Nothing to do for task '%s' on quantum %s; saving metadata and skipping: %s",
156 taskDef.label,
157 quantum.dataId,
158 str(exc),
159 )
160 # Make empty metadata that looks something like what a
161 # do-nothing task would write (but we don't bother with empty
162 # nested PropertySets for subtasks). This is slightly
163 # duplicative with logic in pipe_base that we can't easily call
164 # from here; we'll fix this on DM-29761.
165 logInfo(None, "end", metadata=quantumMetadata)
166 fullMetadata = _TASK_FULL_METADATA_TYPE()
167 fullMetadata[taskDef.label] = _TASK_METADATA_TYPE()
168 fullMetadata["quantum"] = quantumMetadata
169 self.writeMetadata(quantum, fullMetadata, taskDef, butler)
170 return
172 # enable lsstDebug debugging
173 if self.enableLsstDebug:
174 try:
175 _LOG.debug("Will try to import debug.py")
176 import debug # noqa:F401
177 except ImportError:
178 _LOG.warn("No 'debug' module found.")
180 # initialize global state
181 self.initGlobals(quantum, butler)
183 # Ensure that we are executing a frozen config
184 config.freeze()
185 logInfo(None, "init", metadata=quantumMetadata)
186 task = self.makeTask(taskClass, label, config, butler)
187 logInfo(None, "start", metadata=quantumMetadata)
188 try:
189 if self.mock:
190 # Use mock task instance to execute method.
191 runTask = self._makeMockTask(taskDef)
192 else:
193 runTask = task
194 self.runQuantum(runTask, quantum, taskDef, butler)
195 except Exception as e:
196 _LOG.error(
197 "Execution of task '%s' on quantum %s failed. Exception %s: %s",
198 taskDef.label,
199 quantum.dataId,
200 e.__class__.__name__,
201 str(e),
202 )
203 raise
204 logInfo(None, "end", metadata=quantumMetadata)
205 fullMetadata = task.getFullMetadata()
206 fullMetadata["quantum"] = quantumMetadata
207 self.writeMetadata(quantum, fullMetadata, taskDef, butler)
208 stopTime = time.time()
209 _LOG.info(
210 "Execution of task '%s' on quantum %s took %.3f seconds",
211 taskDef.label,
212 quantum.dataId,
213 stopTime - startTime,
214 )
215 return quantum
217 def _makeMockTask(self, taskDef):
218 """Make an instance of mock task for given TaskDef."""
219 # Make config instance and apply overrides
220 overrides = ConfigOverrides()
221 for action in self.mock_configs:
222 if action.label == taskDef.label + "-mock":
223 if action.action == "config":
224 key, _, value = action.value.partition("=")
225 overrides.addValueOverride(key, value)
226 elif action.action == "configfile":
227 overrides.addFileOverride(os.path.expandvars(action.value))
228 else:
229 raise ValueError(f"Unexpected action for mock task config overrides: {action}")
230 config = MockPipelineTask.ConfigClass()
231 overrides.applyTo(config)
233 task = MockPipelineTask(config=config, name=taskDef.label)
234 return task
236 @contextmanager
237 def captureLogging(self, taskDef, quantum, butler):
238 """Configure logging system to capture logs for execution of this task.
240 Parameters
241 ----------
242 taskDef : `lsst.pipe.base.TaskDef`
243 The task definition.
244 quantum : `~lsst.daf.butler.Quantum`
245 Single Quantum instance.
246 butler : `~lsst.daf.butler.Butler`
247 Butler to write logs to.
249 Notes
250 -----
251 Expected to be used as a context manager to ensure that logging
252 records are inserted into the butler once the quantum has been
253 executed:
255 .. code-block:: py
257 with self.captureLogging(taskDef, quantum, butler):
258 # Run quantum and capture logs.
260 Ths method can also setup logging to attach task- or
261 quantum-specific information to log messages. Potentially this can
262 take into account some info from task configuration as well.
263 """
264 # Add a handler to the root logger to capture execution log output.
265 # How does it get removed reliably?
266 if taskDef.logOutputDatasetName is not None:
267 # Either accumulate into ButlerLogRecords or stream
268 # JSON records to file and ingest that.
269 tmpdir = None
270 if self.stream_json_logs:
271 # Create the log file in a temporary directory rather than
272 # creating a temporary file. This is necessary because
273 # temporary files are created with restrictive permissions
274 # and during file ingest these permissions persist in the
275 # datastore. Using a temp directory allows us to create
276 # a file with umask default permissions.
277 tmpdir = tempfile.mkdtemp(prefix="butler-temp-logs-")
279 # Construct a file to receive the log records and "touch" it.
280 log_file = os.path.join(tmpdir, f"butler-log-{taskDef.label}.json")
281 with open(log_file, "w"):
282 pass
283 self.log_handler = FileHandler(log_file)
284 self.log_handler.setFormatter(JsonLogFormatter())
285 else:
286 self.log_handler = ButlerLogRecordHandler()
288 logging.getLogger().addHandler(self.log_handler)
290 # include quantum dataId and task label into MDC
291 label = taskDef.label
292 if quantum.dataId:
293 label += f":{quantum.dataId}"
295 ctx = _LogCaptureFlag()
296 try:
297 with ButlerMDC.set_mdc({"LABEL": label, "RUN": butler.run}):
298 yield ctx
299 finally:
300 # Ensure that the logs are stored in butler.
301 self.writeLogRecords(quantum, taskDef, butler, ctx.store)
302 if tmpdir:
303 shutil.rmtree(tmpdir, ignore_errors=True)
305 def checkExistingOutputs(self, quantum, butler, taskDef):
306 """Decide whether this quantum needs to be executed.
308 If only partial outputs exist then they are removed if
309 ``clobberOutputs`` is True, otherwise an exception is raised.
311 Parameters
312 ----------
313 quantum : `~lsst.daf.butler.Quantum`
314 Quantum to check for existing outputs
315 butler : `~lsst.daf.butler.Butler`
316 Data butler.
317 taskDef : `~lsst.pipe.base.TaskDef`
318 Task definition structure.
320 Returns
321 -------
322 exist : `bool`
323 `True` if ``self.skipExistingIn`` is defined, and a previous
324 execution of this quanta appears to have completed successfully
325 (either because metadata was written or all datasets were written).
326 `False` otherwise.
328 Raises
329 ------
330 RuntimeError
331 Raised if some outputs exist and some not.
332 """
333 if self.skipExistingIn and taskDef.metadataDatasetName is not None:
334 # Metadata output exists; this is sufficient to assume the previous
335 # run was successful and should be skipped.
336 ref = butler.registry.findDataset(
337 taskDef.metadataDatasetName, quantum.dataId, collections=self.skipExistingIn
338 )
339 if ref is not None:
340 if butler.datastore.exists(ref):
341 return True
343 # Previously we always checked for existing outputs in `butler.run`,
344 # now logic gets more complicated as we only want to skip quantum
345 # whose outputs exist in `self.skipExistingIn` but pruning should only
346 # be done for outputs existing in `butler.run`.
348 def findOutputs(collections):
349 """Find quantum outputs in specified collections."""
350 existingRefs = []
351 missingRefs = []
352 for datasetRefs in quantum.outputs.values():
353 checkRefs: List[DatasetRef] = []
354 registryRefToQuantumRef: Dict[DatasetRef, DatasetRef] = {}
355 for datasetRef in datasetRefs:
356 ref = butler.registry.findDataset(
357 datasetRef.datasetType, datasetRef.dataId, collections=collections
358 )
359 if ref is None:
360 missingRefs.append(datasetRef)
361 else:
362 checkRefs.append(ref)
363 registryRefToQuantumRef[ref] = datasetRef
365 # More efficient to ask the datastore in bulk for ref
366 # existence rather than one at a time.
367 existence = butler.datastore.mexists(checkRefs)
368 for ref, exists in existence.items():
369 if exists:
370 existingRefs.append(ref)
371 else:
372 missingRefs.append(registryRefToQuantumRef[ref])
373 return existingRefs, missingRefs
375 existingRefs, missingRefs = findOutputs(self.skipExistingIn)
376 if self.skipExistingIn:
377 if existingRefs and not missingRefs:
378 # everything is already there
379 return True
381 # If we are to re-run quantum then prune datasets that exists in
382 # output run collection, only if `self.clobberOutputs` is set.
383 if existingRefs:
384 existingRefs, missingRefs = findOutputs(butler.run)
385 if existingRefs and missingRefs:
386 _LOG.debug(
387 "Partial outputs exist for task %s dataId=%s collection=%s "
388 "existingRefs=%s missingRefs=%s",
389 taskDef,
390 quantum.dataId,
391 butler.run,
392 existingRefs,
393 missingRefs,
394 )
395 if self.clobberOutputs:
396 # only prune
397 _LOG.info("Removing partial outputs for task %s: %s", taskDef, existingRefs)
398 # Do not purge registry records if this looks like
399 # an execution butler. This ensures that the UUID
400 # of the dataset doesn't change.
401 if butler._allow_put_of_predefined_dataset:
402 purge = False
403 disassociate = False
404 else:
405 purge = True
406 disassociate = True
407 butler.pruneDatasets(existingRefs, disassociate=disassociate, unstore=True, purge=purge)
408 return False
409 else:
410 raise RuntimeError(
411 f"Registry inconsistency while checking for existing outputs:"
412 f" collection={butler.run} existingRefs={existingRefs}"
413 f" missingRefs={missingRefs}"
414 )
416 # need to re-run
417 return False
419 def makeTask(self, taskClass, name, config, butler):
420 """Make new task instance.
422 Parameters
423 ----------
424 taskClass : `type`
425 Sub-class of `~lsst.pipe.base.PipelineTask`.
426 name : `str`
427 Name for this task.
428 config : `~lsst.pipe.base.PipelineTaskConfig`
429 Configuration object for this task
431 Returns
432 -------
433 task : `~lsst.pipe.base.PipelineTask`
434 Instance of ``taskClass`` type.
435 butler : `~lsst.daf.butler.Butler`
436 Data butler.
437 """
438 # call task factory for that
439 return self.taskFactory.makeTask(taskClass, name, config, None, butler)
441 def updatedQuantumInputs(self, quantum, butler, taskDef):
442 """Update quantum with extra information, returns a new updated
443 Quantum.
445 Some methods may require input DatasetRefs to have non-None
446 ``dataset_id``, but in case of intermediate dataset it may not be
447 filled during QuantumGraph construction. This method will retrieve
448 missing info from registry.
450 Parameters
451 ----------
452 quantum : `~lsst.daf.butler.Quantum`
453 Single Quantum instance.
454 butler : `~lsst.daf.butler.Butler`
455 Data butler.
456 taskDef : `~lsst.pipe.base.TaskDef`
457 Task definition structure.
459 Returns
460 -------
461 update : `~lsst.daf.butler.Quantum`
462 Updated Quantum instance
463 """
464 anyChanges = False
465 updatedInputs = defaultdict(list)
466 for key, refsForDatasetType in quantum.inputs.items():
467 newRefsForDatasetType = updatedInputs[key]
468 for ref in refsForDatasetType:
469 if ref.id is None:
470 resolvedRef = butler.registry.findDataset(
471 ref.datasetType, ref.dataId, collections=butler.collections
472 )
473 if resolvedRef is None:
474 _LOG.info("No dataset found for %s", ref)
475 continue
476 else:
477 _LOG.debug("Updated dataset ID for %s", ref)
478 else:
479 resolvedRef = ref
480 # We need to ask datastore if the dataset actually exists
481 # because the Registry of a local "execution butler" cannot
482 # know this (because we prepopulate it with all of the datasets
483 # that might be created). In case of mock execution we check
484 # that mock dataset exists instead.
485 if self.mock:
486 try:
487 typeName, component = ref.datasetType.nameAndComponent()
488 if component is not None:
489 mockDatasetTypeName = MockButlerQuantumContext.mockDatasetTypeName(typeName)
490 else:
491 mockDatasetTypeName = MockButlerQuantumContext.mockDatasetTypeName(
492 ref.datasetType.name
493 )
495 mockDatasetType = butler.registry.getDatasetType(mockDatasetTypeName)
496 except KeyError:
497 # means that mock dataset type is not there and this
498 # should be a pre-existing dataset
499 _LOG.debug("No mock dataset type for %s", ref)
500 if butler.datastore.exists(resolvedRef):
501 newRefsForDatasetType.append(resolvedRef)
502 else:
503 mockRef = DatasetRef(mockDatasetType, ref.dataId)
504 resolvedMockRef = butler.registry.findDataset(
505 mockRef.datasetType, mockRef.dataId, collections=butler.collections
506 )
507 _LOG.debug("mockRef=%s resolvedMockRef=%s", mockRef, resolvedMockRef)
508 if resolvedMockRef is not None and butler.datastore.exists(resolvedMockRef):
509 _LOG.debug("resolvedMockRef dataset exists")
510 newRefsForDatasetType.append(resolvedRef)
511 elif butler.datastore.exists(resolvedRef):
512 newRefsForDatasetType.append(resolvedRef)
514 if len(newRefsForDatasetType) != len(refsForDatasetType):
515 anyChanges = True
516 # If we removed any input datasets, let the task check if it has enough
517 # to proceed and/or prune related datasets that it also doesn't
518 # need/produce anymore. It will raise NoWorkFound if it can't run,
519 # which we'll let propagate up. This is exactly what we run during QG
520 # generation, because a task shouldn't care whether an input is missing
521 # because some previous task didn't produce it, or because it just
522 # wasn't there during QG generation.
523 updatedInputs = NamedKeyDict[DatasetType, List[DatasetRef]](updatedInputs.items())
524 helper = AdjustQuantumHelper(updatedInputs, quantum.outputs)
525 if anyChanges:
526 helper.adjust_in_place(taskDef.connections, label=taskDef.label, data_id=quantum.dataId)
527 return Quantum(
528 taskName=quantum.taskName,
529 taskClass=quantum.taskClass,
530 dataId=quantum.dataId,
531 initInputs=quantum.initInputs,
532 inputs=helper.inputs,
533 outputs=helper.outputs,
534 )
536 def runQuantum(self, task, quantum, taskDef, butler):
537 """Execute task on a single quantum.
539 Parameters
540 ----------
541 task : `~lsst.pipe.base.PipelineTask`
542 Task object.
543 quantum : `~lsst.daf.butler.Quantum`
544 Single Quantum instance.
545 taskDef : `~lsst.pipe.base.TaskDef`
546 Task definition structure.
547 butler : `~lsst.daf.butler.Butler`
548 Data butler.
549 """
550 # Create a butler that operates in the context of a quantum
551 if self.mock:
552 butlerQC = MockButlerQuantumContext(butler, quantum)
553 else:
554 butlerQC = ButlerQuantumContext(butler, quantum)
556 # Get the input and output references for the task
557 inputRefs, outputRefs = taskDef.connections.buildDatasetRefs(quantum)
559 # Call task runQuantum() method. Catch a few known failure modes and
560 # translate them into specific
561 try:
562 task.runQuantum(butlerQC, inputRefs, outputRefs)
563 except NoWorkFound as err:
564 # Not an error, just an early exit.
565 _LOG.info("Task '%s' on quantum %s exited early: %s", taskDef.label, quantum.dataId, str(err))
566 pass
567 except RepeatableQuantumError as err:
568 if self.exitOnKnownError:
569 _LOG.warning("Caught repeatable quantum error for %s (%s):", taskDef, quantum.dataId)
570 _LOG.warning(err, exc_info=True)
571 sys.exit(err.EXIT_CODE)
572 else:
573 raise
574 except InvalidQuantumError as err:
575 _LOG.fatal("Invalid quantum error for %s (%s): %s", taskDef, quantum.dataId)
576 _LOG.fatal(err, exc_info=True)
577 sys.exit(err.EXIT_CODE)
579 def writeMetadata(self, quantum, metadata, taskDef, butler):
580 if taskDef.metadataDatasetName is not None:
581 # DatasetRef has to be in the Quantum outputs, can lookup by name
582 try:
583 ref = quantum.outputs[taskDef.metadataDatasetName]
584 except LookupError as exc:
585 raise InvalidQuantumError(
586 f"Quantum outputs is missing metadata dataset type {taskDef.metadataDatasetName};"
587 f" this could happen due to inconsistent options between QuantumGraph generation"
588 f" and execution"
589 ) from exc
590 butler.put(metadata, ref[0])
592 def writeLogRecords(self, quantum, taskDef, butler, store):
593 # If we are logging to an external file we must always try to
594 # close it.
595 filename = None
596 if isinstance(self.log_handler, FileHandler):
597 filename = self.log_handler.stream.name
598 self.log_handler.close()
600 if self.log_handler is not None:
601 # Remove the handler so we stop accumulating log messages.
602 logging.getLogger().removeHandler(self.log_handler)
604 try:
605 if store and taskDef.logOutputDatasetName is not None and self.log_handler is not None:
606 # DatasetRef has to be in the Quantum outputs, can lookup by
607 # name
608 try:
609 ref = quantum.outputs[taskDef.logOutputDatasetName]
610 except LookupError as exc:
611 raise InvalidQuantumError(
612 f"Quantum outputs is missing log output dataset type {taskDef.logOutputDatasetName};"
613 f" this could happen due to inconsistent options between QuantumGraph generation"
614 f" and execution"
615 ) from exc
617 if isinstance(self.log_handler, ButlerLogRecordHandler):
618 butler.put(self.log_handler.records, ref[0])
620 # Clear the records in case the handler is reused.
621 self.log_handler.records.clear()
622 else:
623 assert filename is not None, "Somehow unable to extract filename from file handler"
625 # Need to ingest this file directly into butler.
626 dataset = FileDataset(path=filename, refs=ref[0])
627 try:
628 butler.ingest(dataset, transfer="move")
629 filename = None
630 except NotImplementedError:
631 # Some datastores can't receive files (e.g. in-memory
632 # datastore when testing), we store empty list for
633 # those just to have a dataset. Alternative is to read
634 # the file as a ButlerLogRecords object and put it.
635 _LOG.info(
636 "Log records could not be stored in this butler because the"
637 " datastore can not ingest files, empty record list is stored instead."
638 )
639 records = ButlerLogRecords.from_records([])
640 butler.put(records, ref[0])
641 finally:
642 # remove file if it is not ingested
643 if filename is not None:
644 try:
645 os.remove(filename)
646 except OSError:
647 pass
649 def initGlobals(self, quantum, butler):
650 """Initialize global state needed for task execution.
652 Parameters
653 ----------
654 quantum : `~lsst.daf.butler.Quantum`
655 Single Quantum instance.
656 butler : `~lsst.daf.butler.Butler`
657 Data butler.
659 Notes
660 -----
661 There is an issue with initializing filters singleton which is done
662 by instrument, to avoid requiring tasks to do it in runQuantum()
663 we do it here when any dataId has an instrument dimension. Also for
664 now we only allow single instrument, verify that all instrument
665 names in all dataIds are identical.
667 This will need revision when filter singleton disappears.
668 """
669 oneInstrument = None
670 for datasetRefs in chain(quantum.inputs.values(), quantum.outputs.values()):
671 for datasetRef in datasetRefs:
672 dataId = datasetRef.dataId
673 instrument = dataId.get("instrument")
674 if instrument is not None:
675 if oneInstrument is not None:
676 assert (
677 instrument == oneInstrument
678 ), "Currently require that only one instrument is used per graph"
679 else:
680 oneInstrument = instrument
681 Instrument.fromName(instrument, butler.registry)