Coverage for python/lsst/pipe/base/graph/graph.py: 17%

381 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-02-14 02:24 -0800

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ("QuantumGraph", "IncompatibleGraphError") 

24 

25import io 

26import json 

27import lzma 

28import os 

29import pickle 

30import struct 

31import time 

32import uuid 

33import warnings 

34from collections import defaultdict, deque 

35from itertools import chain 

36from types import MappingProxyType 

37from typing import ( 

38 Any, 

39 BinaryIO, 

40 DefaultDict, 

41 Deque, 

42 Dict, 

43 FrozenSet, 

44 Generator, 

45 Iterable, 

46 List, 

47 Mapping, 

48 MutableMapping, 

49 Optional, 

50 Set, 

51 Tuple, 

52 TypeVar, 

53 Union, 

54) 

55 

56import networkx as nx 

57from lsst.daf.butler import DatasetRef, DatasetType, DimensionRecordsAccumulator, DimensionUniverse, Quantum 

58from lsst.resources import ResourcePath, ResourcePathExpression 

59from lsst.utils.introspection import get_full_type_name 

60from networkx.drawing.nx_agraph import write_dot 

61 

62from ..connections import iterConnections 

63from ..pipeline import TaskDef 

64from ._implDetails import DatasetTypeName, _DatasetTracker, _pruner 

65from ._loadHelpers import LoadHelper 

66from ._versionDeserializers import DESERIALIZER_MAP 

67from .quantumNode import BuildId, QuantumNode 

68 

69_T = TypeVar("_T", bound="QuantumGraph") 

70 

71# modify this constant any time the on disk representation of the save file 

72# changes, and update the load helpers to behave properly for each version. 

73SAVE_VERSION = 3 

74 

75# Strings used to describe the format for the preamble bytes in a file save 

76# The base is a big endian encoded unsigned short that is used to hold the 

77# file format version. This allows reading version bytes and determine which 

78# loading code should be used for the rest of the file 

79STRUCT_FMT_BASE = ">H" 

80# 

81# Version 1 

82# This marks a big endian encoded format with an unsigned short, an unsigned 

83# long long, and an unsigned long long in the byte stream 

84# Version 2 

85# A big endian encoded format with an unsigned long long byte stream used to 

86# indicate the total length of the entire header. 

87STRUCT_FMT_STRING = {1: ">QQ", 2: ">Q"} 

88 

89# magic bytes that help determine this is a graph save 

90MAGIC_BYTES = b"qgraph4\xf6\xe8\xa9" 

91 

92 

93class IncompatibleGraphError(Exception): 

94 """Exception class to indicate that a lookup by NodeId is impossible due 

95 to incompatibilities 

96 """ 

97 

98 pass 

99 

100 

101class QuantumGraph: 

102 """QuantumGraph is a directed acyclic graph of `QuantumNode` objects 

103 

104 This data structure represents a concrete workflow generated from a 

105 `Pipeline`. 

106 

107 Parameters 

108 ---------- 

109 quanta : Mapping of `TaskDef` to sets of `Quantum` 

110 This maps tasks (and their configs) to the sets of data they are to 

111 process. 

112 metadata : Optional Mapping of `str` to primitives 

113 This is an optional parameter of extra data to carry with the graph. 

114 Entries in this mapping should be able to be serialized in JSON. 

115 pruneRefs : iterable [ `DatasetRef` ], optional 

116 Set of dataset refs to exclude from a graph. 

117 initInputs : `Mapping`, optional 

118 Maps tasks to their InitInput dataset refs. Dataset refs can be either 

119 resolved or non-resolved. Presently the same dataset refs are included 

120 in each `Quantum` for the same task. 

121 initOutputs : `Mapping`, optional 

122 Maps tasks to their InitOutput dataset refs. Dataset refs can be either 

123 resolved or non-resolved. For intermediate resolved refs their dataset 

124 ID must match ``initInputs`` and Quantum ``initInputs``. 

125 globalInitOutputs : iterable [ `DatasetRef` ], optional 

126 Dataset refs for some global objects produced by pipeline. These 

127 objects include task configurations and package versions. Typically 

128 they have an empty DataId, but there is no real restriction on what 

129 can appear here. 

130 

131 Raises 

132 ------ 

133 ValueError 

134 Raised if the graph is pruned such that some tasks no longer have nodes 

135 associated with them. 

136 """ 

137 

138 def __init__( 

139 self, 

140 quanta: Mapping[TaskDef, Set[Quantum]], 

141 metadata: Optional[Mapping[str, Any]] = None, 

142 pruneRefs: Optional[Iterable[DatasetRef]] = None, 

143 universe: Optional[DimensionUniverse] = None, 

144 initInputs: Optional[Mapping[TaskDef, Iterable[DatasetRef]]] = None, 

145 initOutputs: Optional[Mapping[TaskDef, Iterable[DatasetRef]]] = None, 

146 globalInitOutputs: Optional[Iterable[DatasetRef]] = None, 

147 ): 

148 self._buildGraphs( 

149 quanta, 

150 metadata=metadata, 

151 pruneRefs=pruneRefs, 

152 universe=universe, 

153 initInputs=initInputs, 

154 initOutputs=initOutputs, 

155 globalInitOutputs=globalInitOutputs, 

156 ) 

157 

158 def _buildGraphs( 

159 self, 

160 quanta: Mapping[TaskDef, Set[Quantum]], 

161 *, 

162 _quantumToNodeId: Optional[Mapping[Quantum, uuid.UUID]] = None, 

163 _buildId: Optional[BuildId] = None, 

164 metadata: Optional[Mapping[str, Any]] = None, 

165 pruneRefs: Optional[Iterable[DatasetRef]] = None, 

166 universe: Optional[DimensionUniverse] = None, 

167 initInputs: Optional[Mapping[TaskDef, Iterable[DatasetRef]]] = None, 

168 initOutputs: Optional[Mapping[TaskDef, Iterable[DatasetRef]]] = None, 

169 globalInitOutputs: Optional[Iterable[DatasetRef]] = None, 

170 ) -> None: 

171 """Builds the graph that is used to store the relation between tasks, 

172 and the graph that holds the relations between quanta 

173 """ 

174 self._metadata = metadata 

175 self._buildId = _buildId if _buildId is not None else BuildId(f"{time.time()}-{os.getpid()}") 

176 # Data structures used to identify relations between components; 

177 # DatasetTypeName -> TaskDef for task, 

178 # and DatasetRef -> QuantumNode for the quanta 

179 self._datasetDict = _DatasetTracker[DatasetTypeName, TaskDef](createInverse=True) 

180 self._datasetRefDict = _DatasetTracker[DatasetRef, QuantumNode]() 

181 

182 self._nodeIdMap: Dict[uuid.UUID, QuantumNode] = {} 

183 self._taskToQuantumNode: MutableMapping[TaskDef, Set[QuantumNode]] = defaultdict(set) 

184 for taskDef, quantumSet in quanta.items(): 

185 connections = taskDef.connections 

186 

187 # For each type of connection in the task, add a key to the 

188 # `_DatasetTracker` for the connections name, with a value of 

189 # the TaskDef in the appropriate field 

190 for inpt in iterConnections(connections, ("inputs", "prerequisiteInputs", "initInputs")): 

191 # Have to handle components in inputs. 

192 dataset_name, _, _ = inpt.name.partition(".") 

193 self._datasetDict.addConsumer(DatasetTypeName(dataset_name), taskDef) 

194 

195 for output in iterConnections(connections, ("outputs",)): 

196 # Have to handle possible components in outputs. 

197 dataset_name, _, _ = output.name.partition(".") 

198 self._datasetDict.addProducer(DatasetTypeName(dataset_name), taskDef) 

199 

200 # For each `Quantum` in the set of all `Quantum` for this task, 

201 # add a key to the `_DatasetTracker` that is a `DatasetRef` for one 

202 # of the individual datasets inside the `Quantum`, with a value of 

203 # a newly created QuantumNode to the appropriate input/output 

204 # field. 

205 for quantum in quantumSet: 

206 if quantum.dataId is not None: 

207 if universe is None: 

208 universe = quantum.dataId.universe 

209 elif universe != quantum.dataId.universe: 

210 raise RuntimeError( 

211 "Mismatched dimension universes in QuantumGraph construction: " 

212 f"{universe} != {quantum.dataId.universe}. " 

213 ) 

214 

215 if _quantumToNodeId: 

216 if (nodeId := _quantumToNodeId.get(quantum)) is None: 

217 raise ValueError( 

218 "If _quantuMToNodeNumber is not None, all quanta must have an " 

219 "associated value in the mapping" 

220 ) 

221 else: 

222 nodeId = uuid.uuid4() 

223 

224 inits = quantum.initInputs.values() 

225 inputs = quantum.inputs.values() 

226 value = QuantumNode(quantum, taskDef, nodeId) 

227 self._taskToQuantumNode[taskDef].add(value) 

228 self._nodeIdMap[nodeId] = value 

229 

230 for dsRef in chain(inits, inputs): 

231 # unfortunately, `Quantum` allows inits to be individual 

232 # `DatasetRef`s or an Iterable of such, so there must 

233 # be an instance check here 

234 if isinstance(dsRef, Iterable): 

235 for sub in dsRef: 

236 if sub.isComponent(): 

237 sub = sub.makeCompositeRef() 

238 self._datasetRefDict.addConsumer(sub, value) 

239 else: 

240 assert isinstance(dsRef, DatasetRef) 

241 if dsRef.isComponent(): 

242 dsRef = dsRef.makeCompositeRef() 

243 self._datasetRefDict.addConsumer(dsRef, value) 

244 for dsRef in chain.from_iterable(quantum.outputs.values()): 

245 self._datasetRefDict.addProducer(dsRef, value) 

246 

247 if pruneRefs is not None: 

248 # track what refs were pruned and prune the graph 

249 prunes: Set[QuantumNode] = set() 

250 _pruner(self._datasetRefDict, pruneRefs, alreadyPruned=prunes) 

251 

252 # recreate the taskToQuantumNode dict removing nodes that have been 

253 # pruned. Keep track of task defs that now have no QuantumNodes 

254 emptyTasks: Set[str] = set() 

255 newTaskToQuantumNode: DefaultDict[TaskDef, Set[QuantumNode]] = defaultdict(set) 

256 # accumulate all types 

257 types_ = set() 

258 # tracker for any pruneRefs that have caused tasks to have no nodes 

259 # This helps the user find out what caused the issues seen. 

260 culprits = set() 

261 # Find all the types from the refs to prune 

262 for r in pruneRefs: 

263 types_.add(r.datasetType) 

264 

265 # For each of the tasks, and their associated nodes, remove any 

266 # any nodes that were pruned. If there are no nodes associated 

267 # with a task, record that task, and find out if that was due to 

268 # a type from an input ref to prune. 

269 for td, taskNodes in self._taskToQuantumNode.items(): 

270 diff = taskNodes.difference(prunes) 

271 if len(diff) == 0: 

272 if len(taskNodes) != 0: 

273 tp: DatasetType 

274 for tp in types_: 

275 if (tmpRefs := next(iter(taskNodes)).quantum.inputs.get(tp)) and not set( 

276 tmpRefs 

277 ).difference(pruneRefs): 

278 culprits.add(tp.name) 

279 emptyTasks.add(td.label) 

280 newTaskToQuantumNode[td] = diff 

281 

282 # update the internal dict 

283 self._taskToQuantumNode = newTaskToQuantumNode 

284 

285 if emptyTasks: 

286 raise ValueError( 

287 f"{', '.join(emptyTasks)} task(s) have no nodes associated with them " 

288 f"after graph pruning; {', '.join(culprits)} caused over-pruning" 

289 ) 

290 

291 # Dimension universe 

292 if universe is None: 

293 raise RuntimeError( 

294 "Dimension universe or at least one quantum with a data ID " 

295 "must be provided when constructing a QuantumGraph." 

296 ) 

297 self._universe = universe 

298 

299 # Graph of quanta relations 

300 self._connectedQuanta = self._datasetRefDict.makeNetworkXGraph() 

301 self._count = len(self._connectedQuanta) 

302 

303 # Graph of task relations, used in various methods 

304 self._taskGraph = self._datasetDict.makeNetworkXGraph() 

305 

306 # convert default dict into a regular to prevent accidental key 

307 # insertion 

308 self._taskToQuantumNode = dict(self._taskToQuantumNode.items()) 

309 

310 self._initInputRefs: Dict[TaskDef, List[DatasetRef]] = {} 

311 self._initOutputRefs: Dict[TaskDef, List[DatasetRef]] = {} 

312 self._globalInitOutputRefs: List[DatasetRef] = [] 

313 if initInputs is not None: 

314 self._initInputRefs = {taskDef: list(refs) for taskDef, refs in initInputs.items()} 

315 if initOutputs is not None: 

316 self._initOutputRefs = {taskDef: list(refs) for taskDef, refs in initOutputs.items()} 

317 if globalInitOutputs is not None: 

318 self._globalInitOutputRefs = list(globalInitOutputs) 

319 

320 @property 

321 def taskGraph(self) -> nx.DiGraph: 

322 """Return a graph representing the relations between the tasks inside 

323 the quantum graph. 

324 

325 Returns 

326 ------- 

327 taskGraph : `networkx.Digraph` 

328 Internal datastructure that holds relations of `TaskDef` objects 

329 """ 

330 return self._taskGraph 

331 

332 @property 

333 def graph(self) -> nx.DiGraph: 

334 """Return a graph representing the relations between all the 

335 `QuantumNode` objects. Largely it should be preferred to iterate 

336 over, and use methods of this class, but sometimes direct access to 

337 the networkx object may be helpful 

338 

339 Returns 

340 ------- 

341 graph : `networkx.Digraph` 

342 Internal datastructure that holds relations of `QuantumNode` 

343 objects 

344 """ 

345 return self._connectedQuanta 

346 

347 @property 

348 def inputQuanta(self) -> Iterable[QuantumNode]: 

349 """Make a `list` of all `QuantumNode` objects that are 'input' nodes 

350 to the graph, meaning those nodes to not depend on any other nodes in 

351 the graph. 

352 

353 Returns 

354 ------- 

355 inputNodes : iterable of `QuantumNode` 

356 A list of nodes that are inputs to the graph 

357 """ 

358 return (q for q, n in self._connectedQuanta.in_degree if n == 0) 

359 

360 @property 

361 def outputQuanta(self) -> Iterable[QuantumNode]: 

362 """Make a `list` of all `QuantumNode` objects that are 'output' nodes 

363 to the graph, meaning those nodes have no nodes that depend them in 

364 the graph. 

365 

366 Returns 

367 ------- 

368 outputNodes : iterable of `QuantumNode` 

369 A list of nodes that are outputs of the graph 

370 """ 

371 return [q for q, n in self._connectedQuanta.out_degree if n == 0] 

372 

373 @property 

374 def allDatasetTypes(self) -> Tuple[DatasetTypeName, ...]: 

375 """Return all the `DatasetTypeName` objects that are contained inside 

376 the graph. 

377 

378 Returns 

379 ------- 

380 tuple of `DatasetTypeName` 

381 All the data set type names that are present in the graph, not 

382 including global init-outputs. 

383 """ 

384 return tuple(self._datasetDict.keys()) 

385 

386 @property 

387 def isConnected(self) -> bool: 

388 """Return True if all of the nodes in the graph are connected, ignores 

389 directionality of connections. 

390 """ 

391 return nx.is_weakly_connected(self._connectedQuanta) 

392 

393 def pruneGraphFromRefs(self: _T, refs: Iterable[DatasetRef]) -> _T: 

394 r"""Return a graph pruned of input `~lsst.daf.butler.DatasetRef`\ s 

395 and nodes which depend on them. 

396 

397 Parameters 

398 ---------- 

399 refs : `Iterable` of `DatasetRef` 

400 Refs which should be removed from resulting graph 

401 

402 Returns 

403 ------- 

404 graph : `QuantumGraph` 

405 A graph that has been pruned of specified refs and the nodes that 

406 depend on them. 

407 """ 

408 newInst = object.__new__(type(self)) 

409 quantumMap = defaultdict(set) 

410 for node in self: 

411 quantumMap[node.taskDef].add(node.quantum) 

412 

413 # convert to standard dict to prevent accidental key insertion 

414 quantumDict: Dict[TaskDef, Set[Quantum]] = dict(quantumMap.items()) 

415 

416 newInst._buildGraphs( 

417 quantumDict, 

418 _quantumToNodeId={n.quantum: n.nodeId for n in self}, 

419 metadata=self._metadata, 

420 pruneRefs=refs, 

421 universe=self._universe, 

422 globalInitOutputs=self._globalInitOutputRefs, 

423 ) 

424 return newInst 

425 

426 def getQuantumNodeByNodeId(self, nodeId: uuid.UUID) -> QuantumNode: 

427 """Lookup a `QuantumNode` from an id associated with the node. 

428 

429 Parameters 

430 ---------- 

431 nodeId : `NodeId` 

432 The number associated with a node 

433 

434 Returns 

435 ------- 

436 node : `QuantumNode` 

437 The node corresponding with input number 

438 

439 Raises 

440 ------ 

441 KeyError 

442 Raised if the requested nodeId is not in the graph. 

443 """ 

444 return self._nodeIdMap[nodeId] 

445 

446 def getQuantaForTask(self, taskDef: TaskDef) -> FrozenSet[Quantum]: 

447 """Return all the `Quantum` associated with a `TaskDef`. 

448 

449 Parameters 

450 ---------- 

451 taskDef : `TaskDef` 

452 The `TaskDef` for which `Quantum` are to be queried 

453 

454 Returns 

455 ------- 

456 frozenset of `Quantum` 

457 The `set` of `Quantum` that is associated with the specified 

458 `TaskDef`. 

459 """ 

460 return frozenset(node.quantum for node in self._taskToQuantumNode.get(taskDef, ())) 

461 

462 def getNumberOfQuantaForTask(self, taskDef: TaskDef) -> int: 

463 """Return all the number of `Quantum` associated with a `TaskDef`. 

464 

465 Parameters 

466 ---------- 

467 taskDef : `TaskDef` 

468 The `TaskDef` for which `Quantum` are to be queried 

469 

470 Returns 

471 ------- 

472 count : int 

473 The number of `Quantum` that are associated with the specified 

474 `TaskDef`. 

475 """ 

476 return len(self._taskToQuantumNode.get(taskDef, ())) 

477 

478 def getNodesForTask(self, taskDef: TaskDef) -> FrozenSet[QuantumNode]: 

479 """Return all the `QuantumNodes` associated with a `TaskDef`. 

480 

481 Parameters 

482 ---------- 

483 taskDef : `TaskDef` 

484 The `TaskDef` for which `Quantum` are to be queried 

485 

486 Returns 

487 ------- 

488 frozenset of `QuantumNodes` 

489 The `frozenset` of `QuantumNodes` that is associated with the 

490 specified `TaskDef`. 

491 """ 

492 return frozenset(self._taskToQuantumNode[taskDef]) 

493 

494 def findTasksWithInput(self, datasetTypeName: DatasetTypeName) -> Iterable[TaskDef]: 

495 """Find all tasks that have the specified dataset type name as an 

496 input. 

497 

498 Parameters 

499 ---------- 

500 datasetTypeName : `str` 

501 A string representing the name of a dataset type to be queried, 

502 can also accept a `DatasetTypeName` which is a `NewType` of str for 

503 type safety in static type checking. 

504 

505 Returns 

506 ------- 

507 tasks : iterable of `TaskDef` 

508 `TaskDef` objects that have the specified `DatasetTypeName` as an 

509 input, list will be empty if no tasks use specified 

510 `DatasetTypeName` as an input. 

511 

512 Raises 

513 ------ 

514 KeyError 

515 Raised if the `DatasetTypeName` is not part of the `QuantumGraph` 

516 """ 

517 return (c for c in self._datasetDict.getConsumers(datasetTypeName)) 

518 

519 def findTaskWithOutput(self, datasetTypeName: DatasetTypeName) -> Optional[TaskDef]: 

520 """Find all tasks that have the specified dataset type name as an 

521 output. 

522 

523 Parameters 

524 ---------- 

525 datasetTypeName : `str` 

526 A string representing the name of a dataset type to be queried, 

527 can also accept a `DatasetTypeName` which is a `NewType` of str for 

528 type safety in static type checking. 

529 

530 Returns 

531 ------- 

532 `TaskDef` or `None` 

533 `TaskDef` that outputs `DatasetTypeName` as an output or None if 

534 none of the tasks produce this `DatasetTypeName`. 

535 

536 Raises 

537 ------ 

538 KeyError 

539 Raised if the `DatasetTypeName` is not part of the `QuantumGraph` 

540 """ 

541 return self._datasetDict.getProducer(datasetTypeName) 

542 

543 def tasksWithDSType(self, datasetTypeName: DatasetTypeName) -> Iterable[TaskDef]: 

544 """Find all tasks that are associated with the specified dataset type 

545 name. 

546 

547 Parameters 

548 ---------- 

549 datasetTypeName : `str` 

550 A string representing the name of a dataset type to be queried, 

551 can also accept a `DatasetTypeName` which is a `NewType` of str for 

552 type safety in static type checking. 

553 

554 Returns 

555 ------- 

556 result : iterable of `TaskDef` 

557 `TaskDef` objects that are associated with the specified 

558 `DatasetTypeName` 

559 

560 Raises 

561 ------ 

562 KeyError 

563 Raised if the `DatasetTypeName` is not part of the `QuantumGraph` 

564 """ 

565 return self._datasetDict.getAll(datasetTypeName) 

566 

567 def findTaskDefByName(self, taskName: str) -> List[TaskDef]: 

568 """Determine which `TaskDef` objects in this graph are associated 

569 with a `str` representing a task name (looks at the taskName property 

570 of `TaskDef` objects). 

571 

572 Returns a list of `TaskDef` objects as a `PipelineTask` may appear 

573 multiple times in a graph with different labels. 

574 

575 Parameters 

576 ---------- 

577 taskName : str 

578 Name of a task to search for 

579 

580 Returns 

581 ------- 

582 result : list of `TaskDef` 

583 List of the `TaskDef` objects that have the name specified. 

584 Multiple values are returned in the case that a task is used 

585 multiple times with different labels. 

586 """ 

587 results = [] 

588 for task in self._taskToQuantumNode.keys(): 

589 split = task.taskName.split(".") 

590 if split[-1] == taskName: 

591 results.append(task) 

592 return results 

593 

594 def findTaskDefByLabel(self, label: str) -> Optional[TaskDef]: 

595 """Determine which `TaskDef` objects in this graph are associated 

596 with a `str` representing a tasks label. 

597 

598 Parameters 

599 ---------- 

600 taskName : str 

601 Name of a task to search for 

602 

603 Returns 

604 ------- 

605 result : `TaskDef` 

606 `TaskDef` objects that has the specified label. 

607 """ 

608 for task in self._taskToQuantumNode.keys(): 

609 if label == task.label: 

610 return task 

611 return None 

612 

613 def findQuantaWithDSType(self, datasetTypeName: DatasetTypeName) -> Set[Quantum]: 

614 """Return all the `Quantum` that contain a specified `DatasetTypeName`. 

615 

616 Parameters 

617 ---------- 

618 datasetTypeName : `str` 

619 The name of the dataset type to search for as a string, 

620 can also accept a `DatasetTypeName` which is a `NewType` of str for 

621 type safety in static type checking. 

622 

623 Returns 

624 ------- 

625 result : `set` of `QuantumNode` objects 

626 A `set` of `QuantumNode`s that contain specified `DatasetTypeName` 

627 

628 Raises 

629 ------ 

630 KeyError 

631 Raised if the `DatasetTypeName` is not part of the `QuantumGraph` 

632 

633 """ 

634 tasks = self._datasetDict.getAll(datasetTypeName) 

635 result: Set[Quantum] = set() 

636 result = result.union(quantum for task in tasks for quantum in self.getQuantaForTask(task)) 

637 return result 

638 

639 def checkQuantumInGraph(self, quantum: Quantum) -> bool: 

640 """Check if specified quantum appears in the graph as part of a node. 

641 

642 Parameters 

643 ---------- 

644 quantum : `Quantum` 

645 The quantum to search for 

646 

647 Returns 

648 ------- 

649 `bool` 

650 The result of searching for the quantum 

651 """ 

652 for node in self: 

653 if quantum == node.quantum: 

654 return True 

655 return False 

656 

657 def writeDotGraph(self, output: Union[str, io.BufferedIOBase]) -> None: 

658 """Write out the graph as a dot graph. 

659 

660 Parameters 

661 ---------- 

662 output : str or `io.BufferedIOBase` 

663 Either a filesystem path to write to, or a file handle object 

664 """ 

665 write_dot(self._connectedQuanta, output) 

666 

667 def subset(self: _T, nodes: Union[QuantumNode, Iterable[QuantumNode]]) -> _T: 

668 """Create a new graph object that contains the subset of the nodes 

669 specified as input. Node number is preserved. 

670 

671 Parameters 

672 ---------- 

673 nodes : `QuantumNode` or iterable of `QuantumNode` 

674 

675 Returns 

676 ------- 

677 graph : instance of graph type 

678 An instance of the type from which the subset was created 

679 """ 

680 if not isinstance(nodes, Iterable): 

681 nodes = (nodes,) 

682 quantumSubgraph = self._connectedQuanta.subgraph(nodes).nodes 

683 quantumMap = defaultdict(set) 

684 

685 node: QuantumNode 

686 for node in quantumSubgraph: 

687 quantumMap[node.taskDef].add(node.quantum) 

688 

689 # convert to standard dict to prevent accidental key insertion 

690 quantumDict: Dict[TaskDef, Set[Quantum]] = dict(quantumMap.items()) 

691 # Create an empty graph, and then populate it with custom mapping 

692 newInst = type(self)({}, universe=self._universe) 

693 newInst._buildGraphs( 

694 quantumDict, 

695 _quantumToNodeId={n.quantum: n.nodeId for n in nodes}, 

696 _buildId=self._buildId, 

697 metadata=self._metadata, 

698 universe=self._universe, 

699 globalInitOutputs=self._globalInitOutputRefs, 

700 ) 

701 return newInst 

702 

703 def subsetToConnected(self: _T) -> Tuple[_T, ...]: 

704 """Generate a list of subgraphs where each is connected. 

705 

706 Returns 

707 ------- 

708 result : list of `QuantumGraph` 

709 A list of graphs that are each connected 

710 """ 

711 return tuple( 

712 self.subset(connectedSet) 

713 for connectedSet in nx.weakly_connected_components(self._connectedQuanta) 

714 ) 

715 

716 def determineInputsToQuantumNode(self, node: QuantumNode) -> Set[QuantumNode]: 

717 """Return a set of `QuantumNode` that are direct inputs to a specified 

718 node. 

719 

720 Parameters 

721 ---------- 

722 node : `QuantumNode` 

723 The node of the graph for which inputs are to be determined 

724 

725 Returns 

726 ------- 

727 set of `QuantumNode` 

728 All the nodes that are direct inputs to specified node 

729 """ 

730 return set(pred for pred in self._connectedQuanta.predecessors(node)) 

731 

732 def determineOutputsOfQuantumNode(self, node: QuantumNode) -> Set[QuantumNode]: 

733 """Return a set of `QuantumNode` that are direct outputs of a specified 

734 node. 

735 

736 Parameters 

737 ---------- 

738 node : `QuantumNode` 

739 The node of the graph for which outputs are to be determined 

740 

741 Returns 

742 ------- 

743 set of `QuantumNode` 

744 All the nodes that are direct outputs to specified node 

745 """ 

746 return set(succ for succ in self._connectedQuanta.successors(node)) 

747 

748 def determineConnectionsOfQuantumNode(self: _T, node: QuantumNode) -> _T: 

749 """Return a graph of `QuantumNode` that are direct inputs and outputs 

750 of a specified node. 

751 

752 Parameters 

753 ---------- 

754 node : `QuantumNode` 

755 The node of the graph for which connected nodes are to be 

756 determined. 

757 

758 Returns 

759 ------- 

760 graph : graph of `QuantumNode` 

761 All the nodes that are directly connected to specified node 

762 """ 

763 nodes = self.determineInputsToQuantumNode(node).union(self.determineOutputsOfQuantumNode(node)) 

764 nodes.add(node) 

765 return self.subset(nodes) 

766 

767 def determineAncestorsOfQuantumNode(self: _T, node: QuantumNode) -> _T: 

768 """Return a graph of the specified node and all the ancestor nodes 

769 directly reachable by walking edges. 

770 

771 Parameters 

772 ---------- 

773 node : `QuantumNode` 

774 The node for which all ansestors are to be determined 

775 

776 Returns 

777 ------- 

778 graph of `QuantumNode` 

779 Graph of node and all of its ansestors 

780 """ 

781 predecessorNodes = nx.ancestors(self._connectedQuanta, node) 

782 predecessorNodes.add(node) 

783 return self.subset(predecessorNodes) 

784 

785 def findCycle(self) -> List[Tuple[QuantumNode, QuantumNode]]: 

786 """Check a graph for the presense of cycles and returns the edges of 

787 any cycles found, or an empty list if there is no cycle. 

788 

789 Returns 

790 ------- 

791 result : list of tuple of `QuantumNode`, `QuantumNode` 

792 A list of any graph edges that form a cycle, or an empty list if 

793 there is no cycle. Empty list to so support if graph.find_cycle() 

794 syntax as an empty list is falsy. 

795 """ 

796 try: 

797 return nx.find_cycle(self._connectedQuanta) 

798 except nx.NetworkXNoCycle: 

799 return [] 

800 

801 def saveUri(self, uri: ResourcePathExpression) -> None: 

802 """Save `QuantumGraph` to the specified URI. 

803 

804 Parameters 

805 ---------- 

806 uri : convertible to `ResourcePath` 

807 URI to where the graph should be saved. 

808 """ 

809 buffer = self._buildSaveObject() 

810 path = ResourcePath(uri) 

811 if path.getExtension() not in (".qgraph"): 

812 raise TypeError(f"Can currently only save a graph in qgraph format not {uri}") 

813 path.write(buffer) # type: ignore # Ignore because bytearray is safe to use in place of bytes 

814 

815 @property 

816 def metadata(self) -> Optional[MappingProxyType[str, Any]]: 

817 """ """ 

818 if self._metadata is None: 

819 return None 

820 return MappingProxyType(self._metadata) 

821 

822 def initInputRefs(self, taskDef: TaskDef) -> Optional[List[DatasetRef]]: 

823 """Return DatasetRefs for a given task InitInputs. 

824 

825 Parameters 

826 ---------- 

827 taskDef : `TaskDef` 

828 Task definition structure. 

829 

830 Returns 

831 ------- 

832 refs : `list` [ `DatasetRef` ] or None 

833 DatasetRef for the task InitInput, can be `None`. This can return 

834 either resolved or non-resolved reference. 

835 """ 

836 return self._initInputRefs.get(taskDef) 

837 

838 def initOutputRefs(self, taskDef: TaskDef) -> Optional[List[DatasetRef]]: 

839 """Return DatasetRefs for a given task InitOutputs. 

840 

841 Parameters 

842 ---------- 

843 taskDef : `TaskDef` 

844 Task definition structure. 

845 

846 Returns 

847 ------- 

848 refs : `list` [ `DatasetRef` ] or None 

849 DatasetRefs for the task InitOutput, can be `None`. This can return 

850 either resolved or non-resolved reference. Resolved reference will 

851 match Quantum's initInputs if this is an intermediate dataset type. 

852 """ 

853 return self._initOutputRefs.get(taskDef) 

854 

855 def globalInitOutputRefs(self) -> List[DatasetRef]: 

856 """Return DatasetRefs for global InitOutputs. 

857 

858 Returns 

859 ------- 

860 refs : `list` [ `DatasetRef` ] 

861 DatasetRefs for global InitOutputs. 

862 """ 

863 return self._globalInitOutputRefs 

864 

865 @classmethod 

866 def loadUri( 

867 cls, 

868 uri: ResourcePathExpression, 

869 universe: Optional[DimensionUniverse] = None, 

870 nodes: Optional[Iterable[uuid.UUID]] = None, 

871 graphID: Optional[BuildId] = None, 

872 minimumVersion: int = 3, 

873 ) -> QuantumGraph: 

874 """Read `QuantumGraph` from a URI. 

875 

876 Parameters 

877 ---------- 

878 uri : convertible to `ResourcePath` 

879 URI from where to load the graph. 

880 universe: `~lsst.daf.butler.DimensionUniverse` optional 

881 DimensionUniverse instance, not used by the method itself but 

882 needed to ensure that registry data structures are initialized. 

883 If None it is loaded from the QuantumGraph saved structure. If 

884 supplied, the DimensionUniverse from the loaded `QuantumGraph` 

885 will be validated against the supplied argument for compatibility. 

886 nodes: iterable of `int` or None 

887 Numbers that correspond to nodes in the graph. If specified, only 

888 these nodes will be loaded. Defaults to None, in which case all 

889 nodes will be loaded. 

890 graphID : `str` or `None` 

891 If specified this ID is verified against the loaded graph prior to 

892 loading any Nodes. This defaults to None in which case no 

893 validation is done. 

894 minimumVersion : int 

895 Minimum version of a save file to load. Set to -1 to load all 

896 versions. Older versions may need to be loaded, and re-saved 

897 to upgrade them to the latest format before they can be used in 

898 production. 

899 

900 Returns 

901 ------- 

902 graph : `QuantumGraph` 

903 Resulting QuantumGraph instance. 

904 

905 Raises 

906 ------ 

907 TypeError 

908 Raised if pickle contains instance of a type other than 

909 QuantumGraph. 

910 ValueError 

911 Raised if one or more of the nodes requested is not in the 

912 `QuantumGraph` or if graphID parameter does not match the graph 

913 being loaded or if the supplied uri does not point at a valid 

914 `QuantumGraph` save file. 

915 RuntimeError 

916 Raise if Supplied DimensionUniverse is not compatible with the 

917 DimensionUniverse saved in the graph 

918 

919 

920 Notes 

921 ----- 

922 Reading Quanta from pickle requires existence of singleton 

923 DimensionUniverse which is usually instantiated during Registry 

924 initialization. To make sure that DimensionUniverse exists this method 

925 accepts dummy DimensionUniverse argument. 

926 """ 

927 uri = ResourcePath(uri) 

928 # With ResourcePath we have the choice of always using a local file 

929 # or reading in the bytes directly. Reading in bytes can be more 

930 # efficient for reasonably-sized pickle files when the resource 

931 # is remote. For now use the local file variant. For a local file 

932 # as_local() does nothing. 

933 

934 if uri.getExtension() in (".pickle", ".pkl"): 

935 with uri.as_local() as local, open(local.ospath, "rb") as fd: 

936 warnings.warn("Pickle graphs are deprecated, please re-save your graph with the save method") 

937 qgraph = pickle.load(fd) 

938 elif uri.getExtension() in (".qgraph"): 

939 with LoadHelper(uri, minimumVersion) as loader: 

940 qgraph = loader.load(universe, nodes, graphID) 

941 else: 

942 raise ValueError("Only know how to handle files saved as `pickle`, `pkl`, or `qgraph`") 

943 if not isinstance(qgraph, QuantumGraph): 

944 raise TypeError(f"QuantumGraph save file contains unexpected object type: {type(qgraph)}") 

945 return qgraph 

946 

947 @classmethod 

948 def readHeader(cls, uri: ResourcePathExpression, minimumVersion: int = 3) -> Optional[str]: 

949 """Read the header of a `QuantumGraph` pointed to by the uri parameter 

950 and return it as a string. 

951 

952 Parameters 

953 ---------- 

954 uri : convertible to `ResourcePath` 

955 The location of the `QuantumGraph` to load. If the argument is a 

956 string, it must correspond to a valid `ResourcePath` path. 

957 minimumVersion : int 

958 Minimum version of a save file to load. Set to -1 to load all 

959 versions. Older versions may need to be loaded, and re-saved 

960 to upgrade them to the latest format before they can be used in 

961 production. 

962 

963 Returns 

964 ------- 

965 header : `str` or `None` 

966 The header associated with the specified `QuantumGraph` it there is 

967 one, else `None`. 

968 

969 Raises 

970 ------ 

971 ValueError 

972 Raised if `QuantuGraph` was saved as a pickle. 

973 Raised if the extention of the file specified by uri is not a 

974 `QuantumGraph` extention. 

975 """ 

976 uri = ResourcePath(uri) 

977 if uri.getExtension() in (".pickle", ".pkl"): 

978 raise ValueError("Reading a header from a pickle save is not supported") 

979 elif uri.getExtension() in (".qgraph"): 

980 return LoadHelper(uri, minimumVersion).readHeader() 

981 else: 

982 raise ValueError("Only know how to handle files saved as `qgraph`") 

983 

984 def buildAndPrintHeader(self) -> None: 

985 """Creates a header that would be used in a save of this object and 

986 prints it out to standard out. 

987 """ 

988 _, header = self._buildSaveObject(returnHeader=True) 

989 print(json.dumps(header)) 

990 

991 def save(self, file: BinaryIO) -> None: 

992 """Save QuantumGraph to a file. 

993 

994 Parameters 

995 ---------- 

996 file : `io.BufferedIOBase` 

997 File to write pickle data open in binary mode. 

998 """ 

999 buffer = self._buildSaveObject() 

1000 file.write(buffer) # type: ignore # Ignore because bytearray is safe to use in place of bytes 

1001 

1002 def _buildSaveObject(self, returnHeader: bool = False) -> Union[bytearray, Tuple[bytearray, Dict]]: 

1003 # make some containers 

1004 jsonData: Deque[bytes] = deque() 

1005 # node map is a list because json does not accept mapping keys that 

1006 # are not strings, so we store a list of key, value pairs that will 

1007 # be converted to a mapping on load 

1008 nodeMap = [] 

1009 taskDefMap = {} 

1010 headerData: Dict[str, Any] = {} 

1011 

1012 # Store the QauntumGraph BuildId, this will allow validating BuildIds 

1013 # at load time, prior to loading any QuantumNodes. Name chosen for 

1014 # unlikely conflicts. 

1015 headerData["GraphBuildID"] = self.graphID 

1016 headerData["Metadata"] = self._metadata 

1017 

1018 # Store the universe this graph was created with 

1019 universeConfig = self._universe.dimensionConfig 

1020 headerData["universe"] = universeConfig.toDict() 

1021 

1022 # counter for the number of bytes processed thus far 

1023 count = 0 

1024 # serialize out the task Defs recording the start and end bytes of each 

1025 # taskDef 

1026 inverseLookup = self._datasetDict.inverse 

1027 taskDef: TaskDef 

1028 # sort by task label to ensure serialization happens in the same order 

1029 for taskDef in self.taskGraph: 

1030 # compressing has very little impact on saving or load time, but 

1031 # a large impact on on disk size, so it is worth doing 

1032 taskDescription: Dict[str, Any] = {} 

1033 # save the fully qualified name. 

1034 taskDescription["taskName"] = get_full_type_name(taskDef.taskClass) 

1035 # save the config as a text stream that will be un-persisted on the 

1036 # other end 

1037 stream = io.StringIO() 

1038 taskDef.config.saveToStream(stream) 

1039 taskDescription["config"] = stream.getvalue() 

1040 taskDescription["label"] = taskDef.label 

1041 if (refs := self._initInputRefs.get(taskDef)) is not None: 

1042 taskDescription["initInputRefs"] = [ref.to_json() for ref in refs] 

1043 if (refs := self._initOutputRefs.get(taskDef)) is not None: 

1044 taskDescription["initOutputRefs"] = [ref.to_json() for ref in refs] 

1045 

1046 inputs = [] 

1047 outputs = [] 

1048 

1049 # Determine the connection between all of tasks and save that in 

1050 # the header as a list of connections and edges in each task 

1051 # this will help in un-persisting, and possibly in a "quick view" 

1052 # method that does not require everything to be un-persisted 

1053 # 

1054 # Typing returns can't be parameter dependent 

1055 for connection in inverseLookup[taskDef]: # type: ignore 

1056 consumers = self._datasetDict.getConsumers(connection) 

1057 producer = self._datasetDict.getProducer(connection) 

1058 if taskDef in consumers: 

1059 # This checks if the task consumes the connection directly 

1060 # from the datastore or it is produced by another task 

1061 producerLabel = producer.label if producer is not None else "datastore" 

1062 inputs.append((producerLabel, connection)) 

1063 elif taskDef not in consumers and producer is taskDef: 

1064 # If there are no consumers for this tasks produced 

1065 # connection, the output will be said to be the datastore 

1066 # in which case the for loop will be a zero length loop 

1067 if not consumers: 

1068 outputs.append(("datastore", connection)) 

1069 for td in consumers: 

1070 outputs.append((td.label, connection)) 

1071 

1072 # dump to json string, and encode that string to bytes and then 

1073 # conpress those bytes 

1074 dump = lzma.compress(json.dumps(taskDescription).encode()) 

1075 # record the sizing and relation information 

1076 taskDefMap[taskDef.label] = { 

1077 "bytes": (count, count + len(dump)), 

1078 "inputs": inputs, 

1079 "outputs": outputs, 

1080 } 

1081 count += len(dump) 

1082 jsonData.append(dump) 

1083 

1084 headerData["TaskDefs"] = taskDefMap 

1085 

1086 # serialize the nodes, recording the start and end bytes of each node 

1087 dimAccumulator = DimensionRecordsAccumulator() 

1088 for node in self: 

1089 # compressing has very little impact on saving or load time, but 

1090 # a large impact on on disk size, so it is worth doing 

1091 simpleNode = node.to_simple(accumulator=dimAccumulator) 

1092 

1093 dump = lzma.compress(simpleNode.json().encode()) 

1094 jsonData.append(dump) 

1095 nodeMap.append( 

1096 ( 

1097 str(node.nodeId), 

1098 { 

1099 "bytes": (count, count + len(dump)), 

1100 "inputs": [str(n.nodeId) for n in self.determineInputsToQuantumNode(node)], 

1101 "outputs": [str(n.nodeId) for n in self.determineOutputsOfQuantumNode(node)], 

1102 }, 

1103 ) 

1104 ) 

1105 count += len(dump) 

1106 

1107 headerData["DimensionRecords"] = { 

1108 key: value.dict() for key, value in dimAccumulator.makeSerializedDimensionRecordMapping().items() 

1109 } 

1110 

1111 # need to serialize this as a series of key,value tuples because of 

1112 # a limitation on how json cant do anything but strings as keys 

1113 headerData["Nodes"] = nodeMap 

1114 

1115 if self._globalInitOutputRefs: 

1116 headerData["GlobalInitOutputRefs"] = [ref.to_json() for ref in self._globalInitOutputRefs] 

1117 

1118 # dump the headerData to json 

1119 header_encode = lzma.compress(json.dumps(headerData).encode()) 

1120 

1121 # record the sizes as 2 unsigned long long numbers for a total of 16 

1122 # bytes 

1123 save_bytes = struct.pack(STRUCT_FMT_BASE, SAVE_VERSION) 

1124 

1125 fmt_string = DESERIALIZER_MAP[SAVE_VERSION].FMT_STRING() 

1126 map_lengths = struct.pack(fmt_string, len(header_encode)) 

1127 

1128 # write each component of the save out in a deterministic order 

1129 # buffer = io.BytesIO() 

1130 # buffer.write(map_lengths) 

1131 # buffer.write(taskDef_pickle) 

1132 # buffer.write(map_pickle) 

1133 buffer = bytearray() 

1134 buffer.extend(MAGIC_BYTES) 

1135 buffer.extend(save_bytes) 

1136 buffer.extend(map_lengths) 

1137 buffer.extend(header_encode) 

1138 # Iterate over the length of pickleData, and for each element pop the 

1139 # leftmost element off the deque and write it out. This is to save 

1140 # memory, as the memory is added to the buffer object, it is removed 

1141 # from from the container. 

1142 # 

1143 # Only this section needs to worry about memory pressue because 

1144 # everything else written to the buffer prior to this pickle data is 

1145 # only on the order of kilobytes to low numbers of megabytes. 

1146 while jsonData: 

1147 buffer.extend(jsonData.popleft()) 

1148 if returnHeader: 

1149 return buffer, headerData 

1150 else: 

1151 return buffer 

1152 

1153 @classmethod 

1154 def load( 

1155 cls, 

1156 file: BinaryIO, 

1157 universe: Optional[DimensionUniverse] = None, 

1158 nodes: Optional[Iterable[uuid.UUID]] = None, 

1159 graphID: Optional[BuildId] = None, 

1160 minimumVersion: int = 3, 

1161 ) -> QuantumGraph: 

1162 """Read QuantumGraph from a file that was made by `save`. 

1163 

1164 Parameters 

1165 ---------- 

1166 file : `io.IO` of bytes 

1167 File with pickle data open in binary mode. 

1168 universe: `~lsst.daf.butler.DimensionUniverse`, optional 

1169 DimensionUniverse instance, not used by the method itself but 

1170 needed to ensure that registry data structures are initialized. 

1171 If None it is loaded from the QuantumGraph saved structure. If 

1172 supplied, the DimensionUniverse from the loaded `QuantumGraph` 

1173 will be validated against the supplied argument for compatibility. 

1174 nodes: iterable of `int` or None 

1175 Numbers that correspond to nodes in the graph. If specified, only 

1176 these nodes will be loaded. Defaults to None, in which case all 

1177 nodes will be loaded. 

1178 graphID : `str` or `None` 

1179 If specified this ID is verified against the loaded graph prior to 

1180 loading any Nodes. This defaults to None in which case no 

1181 validation is done. 

1182 minimumVersion : int 

1183 Minimum version of a save file to load. Set to -1 to load all 

1184 versions. Older versions may need to be loaded, and re-saved 

1185 to upgrade them to the latest format before they can be used in 

1186 production. 

1187 

1188 Returns 

1189 ------- 

1190 graph : `QuantumGraph` 

1191 Resulting QuantumGraph instance. 

1192 

1193 Raises 

1194 ------ 

1195 TypeError 

1196 Raised if pickle contains instance of a type other than 

1197 QuantumGraph. 

1198 ValueError 

1199 Raised if one or more of the nodes requested is not in the 

1200 `QuantumGraph` or if graphID parameter does not match the graph 

1201 being loaded or if the supplied uri does not point at a valid 

1202 `QuantumGraph` save file. 

1203 

1204 Notes 

1205 ----- 

1206 Reading Quanta from pickle requires existence of singleton 

1207 DimensionUniverse which is usually instantiated during Registry 

1208 initialization. To make sure that DimensionUniverse exists this method 

1209 accepts dummy DimensionUniverse argument. 

1210 """ 

1211 # Try to see if the file handle contains pickle data, this will be 

1212 # removed in the future 

1213 try: 

1214 qgraph = pickle.load(file) 

1215 warnings.warn("Pickle graphs are deprecated, please re-save your graph with the save method") 

1216 except pickle.UnpicklingError: 

1217 with LoadHelper(file, minimumVersion) as loader: 

1218 qgraph = loader.load(universe, nodes, graphID) 

1219 if not isinstance(qgraph, QuantumGraph): 

1220 raise TypeError(f"QuantumGraph pickle file has contains unexpected object type: {type(qgraph)}") 

1221 return qgraph 

1222 

1223 def iterTaskGraph(self) -> Generator[TaskDef, None, None]: 

1224 """Iterate over the `taskGraph` attribute in topological order 

1225 

1226 Yields 

1227 ------ 

1228 taskDef : `TaskDef` 

1229 `TaskDef` objects in topological order 

1230 """ 

1231 yield from nx.topological_sort(self.taskGraph) 

1232 

1233 @property 

1234 def graphID(self) -> BuildId: 

1235 """Returns the ID generated by the graph at construction time""" 

1236 return self._buildId 

1237 

1238 @property 

1239 def universe(self) -> DimensionUniverse: 

1240 """Dimension universe associated with this graph.""" 

1241 return self._universe 

1242 

1243 def __iter__(self) -> Generator[QuantumNode, None, None]: 

1244 yield from nx.topological_sort(self._connectedQuanta) 

1245 

1246 def __len__(self) -> int: 

1247 return self._count 

1248 

1249 def __contains__(self, node: QuantumNode) -> bool: 

1250 return self._connectedQuanta.has_node(node) 

1251 

1252 def __getstate__(self) -> dict: 

1253 """Stores a compact form of the graph as a list of graph nodes, and a 

1254 tuple of task labels and task configs. The full graph can be 

1255 reconstructed with this information, and it preseves the ordering of 

1256 the graph ndoes. 

1257 """ 

1258 universe: Optional[DimensionUniverse] = None 

1259 for node in self: 

1260 dId = node.quantum.dataId 

1261 if dId is None: 

1262 continue 

1263 universe = dId.graph.universe 

1264 return {"reduced": self._buildSaveObject(), "graphId": self._buildId, "universe": universe} 

1265 

1266 def __setstate__(self, state: dict) -> None: 

1267 """Reconstructs the state of the graph from the information persisted 

1268 in getstate. 

1269 """ 

1270 buffer = io.BytesIO(state["reduced"]) 

1271 with LoadHelper(buffer, minimumVersion=3) as loader: 

1272 qgraph = loader.load(state["universe"], graphID=state["graphId"]) 

1273 

1274 self._metadata = qgraph._metadata 

1275 self._buildId = qgraph._buildId 

1276 self._datasetDict = qgraph._datasetDict 

1277 self._nodeIdMap = qgraph._nodeIdMap 

1278 self._count = len(qgraph) 

1279 self._taskToQuantumNode = qgraph._taskToQuantumNode 

1280 self._taskGraph = qgraph._taskGraph 

1281 self._connectedQuanta = qgraph._connectedQuanta 

1282 self._initInputRefs = qgraph._initInputRefs 

1283 self._initOutputRefs = qgraph._initOutputRefs 

1284 

1285 def __eq__(self, other: object) -> bool: 

1286 if not isinstance(other, QuantumGraph): 

1287 return False 

1288 if len(self) != len(other): 

1289 return False 

1290 for node in self: 

1291 if node not in other: 

1292 return False 

1293 if self.determineInputsToQuantumNode(node) != other.determineInputsToQuantumNode(node): 

1294 return False 

1295 if self.determineOutputsOfQuantumNode(node) != other.determineOutputsOfQuantumNode(node): 

1296 return False 

1297 if set(self.allDatasetTypes) != set(other.allDatasetTypes): 

1298 return False 

1299 return set(self.taskGraph) == set(other.taskGraph)