Coverage for python/lsst/pipe/base/connections.py: 46%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

211 statements  

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Module defining connection classes for PipelineTask. 

23""" 

24 

25from __future__ import annotations 

26 

27__all__ = [ 

28 "AdjustQuantumHelper", 

29 "DeferredDatasetRef", 

30 "InputQuantizedConnection", 

31 "OutputQuantizedConnection", 

32 "PipelineTaskConnections", 

33 "ScalarError", 

34 "iterConnections", 

35 "ScalarError", 

36] 

37 

38import itertools 

39import string 

40import typing 

41from collections import UserDict, namedtuple 

42from dataclasses import dataclass 

43from types import SimpleNamespace 

44from typing import Iterable, Union 

45 

46from lsst.daf.butler import DataCoordinate, DatasetRef, DatasetType, NamedKeyDict, NamedKeyMapping, Quantum 

47 

48from ._status import NoWorkFound 

49from .connectionTypes import ( 

50 BaseConnection, 

51 BaseInput, 

52 InitInput, 

53 InitOutput, 

54 Input, 

55 Output, 

56 PrerequisiteInput, 

57) 

58 

59if typing.TYPE_CHECKING: 59 ↛ 60line 59 didn't jump to line 60, because the condition on line 59 was never true

60 from .config import PipelineTaskConfig 

61 

62 

63class ScalarError(TypeError): 

64 """Exception raised when dataset type is configured as scalar 

65 but there are multiple data IDs in a Quantum for that dataset. 

66 """ 

67 

68 

69class PipelineTaskConnectionDict(UserDict): 

70 """This is a special dict class used by PipelineTaskConnectionMetaclass 

71 

72 This dict is used in PipelineTaskConnection class creation, as the 

73 dictionary that is initially used as __dict__. It exists to 

74 intercept connection fields declared in a PipelineTaskConnection, and 

75 what name is used to identify them. The names are then added to class 

76 level list according to the connection type of the class attribute. The 

77 names are also used as keys in a class level dictionary associated with 

78 the corresponding class attribute. This information is a duplicate of 

79 what exists in __dict__, but provides a simple place to lookup and 

80 iterate on only these variables. 

81 """ 

82 

83 def __init__(self, *args, **kwargs): 

84 super().__init__(*args, **kwargs) 

85 # Initialize class level variables used to track any declared 

86 # class level variables that are instances of 

87 # connectionTypes.BaseConnection 

88 self.data["inputs"] = [] 

89 self.data["prerequisiteInputs"] = [] 

90 self.data["outputs"] = [] 

91 self.data["initInputs"] = [] 

92 self.data["initOutputs"] = [] 

93 self.data["allConnections"] = {} 

94 

95 def __setitem__(self, name, value): 

96 if isinstance(value, Input): 

97 self.data["inputs"].append(name) 

98 elif isinstance(value, PrerequisiteInput): 

99 self.data["prerequisiteInputs"].append(name) 

100 elif isinstance(value, Output): 

101 self.data["outputs"].append(name) 

102 elif isinstance(value, InitInput): 

103 self.data["initInputs"].append(name) 

104 elif isinstance(value, InitOutput): 

105 self.data["initOutputs"].append(name) 

106 # This should not be an elif, as it needs tested for 

107 # everything that inherits from BaseConnection 

108 if isinstance(value, BaseConnection): 

109 object.__setattr__(value, "varName", name) 

110 self.data["allConnections"][name] = value 

111 # defer to the default behavior 

112 super().__setitem__(name, value) 

113 

114 

115class PipelineTaskConnectionsMetaclass(type): 

116 """Metaclass used in the declaration of PipelineTaskConnections classes""" 

117 

118 def __prepare__(name, bases, **kwargs): # noqa: 805 

119 # Create an instance of our special dict to catch and track all 

120 # variables that are instances of connectionTypes.BaseConnection 

121 # Copy any existing connections from a parent class 

122 dct = PipelineTaskConnectionDict() 

123 for base in bases: 

124 if isinstance(base, PipelineTaskConnectionsMetaclass): 124 ↛ 123line 124 didn't jump to line 123, because the condition on line 124 was never false

125 for name, value in base.allConnections.items(): 125 ↛ 126line 125 didn't jump to line 126, because the loop on line 125 never started

126 dct[name] = value 

127 return dct 

128 

129 def __new__(cls, name, bases, dct, **kwargs): 

130 dimensionsValueError = TypeError( 

131 "PipelineTaskConnections class must be created with a dimensions " 

132 "attribute which is an iterable of dimension names" 

133 ) 

134 

135 if name != "PipelineTaskConnections": 

136 # Verify that dimensions are passed as a keyword in class 

137 # declaration 

138 if "dimensions" not in kwargs: 138 ↛ 139line 138 didn't jump to line 139, because the condition on line 138 was never true

139 for base in bases: 

140 if hasattr(base, "dimensions"): 

141 kwargs["dimensions"] = base.dimensions 

142 break 

143 if "dimensions" not in kwargs: 

144 raise dimensionsValueError 

145 try: 

146 if isinstance(kwargs["dimensions"], str): 146 ↛ 147line 146 didn't jump to line 147, because the condition on line 146 was never true

147 raise TypeError( 

148 "Dimensions must be iterable of dimensions, got str," 

149 "possibly omitted trailing comma" 

150 ) 

151 if not isinstance(kwargs["dimensions"], typing.Iterable): 151 ↛ 152line 151 didn't jump to line 152, because the condition on line 151 was never true

152 raise TypeError("Dimensions must be iterable of dimensions") 

153 dct["dimensions"] = set(kwargs["dimensions"]) 

154 except TypeError as exc: 

155 raise dimensionsValueError from exc 

156 # Lookup any python string templates that may have been used in the 

157 # declaration of the name field of a class connection attribute 

158 allTemplates = set() 

159 stringFormatter = string.Formatter() 

160 # Loop over all connections 

161 for obj in dct["allConnections"].values(): 

162 nameValue = obj.name 

163 # add all the parameters to the set of templates 

164 for param in stringFormatter.parse(nameValue): 

165 if param[1] is not None: 

166 allTemplates.add(param[1]) 

167 

168 # look up any template from base classes and merge them all 

169 # together 

170 mergeDict = {} 

171 for base in bases[::-1]: 

172 if hasattr(base, "defaultTemplates"): 172 ↛ 173line 172 didn't jump to line 173, because the condition on line 172 was never true

173 mergeDict.update(base.defaultTemplates) 

174 if "defaultTemplates" in kwargs: 

175 mergeDict.update(kwargs["defaultTemplates"]) 

176 

177 if len(mergeDict) > 0: 

178 kwargs["defaultTemplates"] = mergeDict 

179 

180 # Verify that if templated strings were used, defaults were 

181 # supplied as an argument in the declaration of the connection 

182 # class 

183 if len(allTemplates) > 0 and "defaultTemplates" not in kwargs: 183 ↛ 184line 183 didn't jump to line 184, because the condition on line 183 was never true

184 raise TypeError( 

185 "PipelineTaskConnection class contains templated attribute names, but no " 

186 "defaut templates were provided, add a dictionary attribute named " 

187 "defaultTemplates which contains the mapping between template key and value" 

188 ) 

189 if len(allTemplates) > 0: 

190 # Verify all templates have a default, and throw if they do not 

191 defaultTemplateKeys = set(kwargs["defaultTemplates"].keys()) 

192 templateDifference = allTemplates.difference(defaultTemplateKeys) 

193 if templateDifference: 193 ↛ 194line 193 didn't jump to line 194, because the condition on line 193 was never true

194 raise TypeError(f"Default template keys were not provided for {templateDifference}") 

195 # Verify that templates do not share names with variable names 

196 # used for a connection, this is needed because of how 

197 # templates are specified in an associated config class. 

198 nameTemplateIntersection = allTemplates.intersection(set(dct["allConnections"].keys())) 

199 if len(nameTemplateIntersection) > 0: 199 ↛ 200line 199 didn't jump to line 200, because the condition on line 199 was never true

200 raise TypeError( 

201 f"Template parameters cannot share names with Class attributes" 

202 f" (conflicts are {nameTemplateIntersection})." 

203 ) 

204 dct["defaultTemplates"] = kwargs.get("defaultTemplates", {}) 

205 

206 # Convert all the connection containers into frozensets so they cannot 

207 # be modified at the class scope 

208 for connectionName in ("inputs", "prerequisiteInputs", "outputs", "initInputs", "initOutputs"): 

209 dct[connectionName] = frozenset(dct[connectionName]) 

210 # our custom dict type must be turned into an actual dict to be used in 

211 # type.__new__ 

212 return super().__new__(cls, name, bases, dict(dct)) 

213 

214 def __init__(cls, name, bases, dct, **kwargs): 

215 # This overrides the default init to drop the kwargs argument. Python 

216 # metaclasses will have this argument set if any kwargs are passes at 

217 # class construction time, but should be consumed before calling 

218 # __init__ on the type metaclass. This is in accordance with python 

219 # documentation on metaclasses 

220 super().__init__(name, bases, dct) 

221 

222 

223class QuantizedConnection(SimpleNamespace): 

224 """A Namespace to map defined variable names of connections to the 

225 associated `lsst.daf.butler.DatasetRef` objects. 

226 

227 This class maps the names used to define a connection on a 

228 PipelineTaskConnectionsClass to the corresponding 

229 `lsst.daf.butler.DatasetRef`s provided by a `lsst.daf.butler.Quantum` 

230 instance. This will be a quantum of execution based on the graph created 

231 by examining all the connections defined on the 

232 `PipelineTaskConnectionsClass`. 

233 """ 

234 

235 def __init__(self, **kwargs): 

236 # Create a variable to track what attributes are added. This is used 

237 # later when iterating over this QuantizedConnection instance 

238 object.__setattr__(self, "_attributes", set()) 

239 

240 def __setattr__(self, name: str, value: typing.Union[DatasetRef, typing.List[DatasetRef]]): 

241 # Capture the attribute name as it is added to this object 

242 self._attributes.add(name) 

243 super().__setattr__(name, value) 

244 

245 def __delattr__(self, name): 

246 object.__delattr__(self, name) 

247 self._attributes.remove(name) 

248 

249 def __iter__( 

250 self, 

251 ) -> typing.Generator[typing.Tuple[str, typing.Union[DatasetRef, typing.List[DatasetRef]]], None, None]: 

252 """Make an Iterator for this QuantizedConnection 

253 

254 Iterating over a QuantizedConnection will yield a tuple with the name 

255 of an attribute and the value associated with that name. This is 

256 similar to dict.items() but is on the namespace attributes rather than 

257 dict keys. 

258 """ 

259 yield from ((name, getattr(self, name)) for name in self._attributes) 

260 

261 def keys(self) -> typing.Generator[str, None, None]: 

262 """Returns an iterator over all the attributes added to a 

263 QuantizedConnection class 

264 """ 

265 yield from self._attributes 

266 

267 

268class InputQuantizedConnection(QuantizedConnection): 

269 pass 

270 

271 

272class OutputQuantizedConnection(QuantizedConnection): 

273 pass 

274 

275 

276class DeferredDatasetRef(namedtuple("DeferredDatasetRefBase", "datasetRef")): 

277 """Class which denotes that a datasetRef should be treated as deferred when 

278 interacting with the butler 

279 

280 Parameters 

281 ---------- 

282 datasetRef : `lsst.daf.butler.DatasetRef` 

283 The `lsst.daf.butler.DatasetRef` that will be eventually used to 

284 resolve a dataset 

285 """ 

286 

287 __slots__ = () 

288 

289 

290class PipelineTaskConnections(metaclass=PipelineTaskConnectionsMetaclass): 

291 """PipelineTaskConnections is a class used to declare desired IO when a 

292 PipelineTask is run by an activator 

293 

294 Parameters 

295 ---------- 

296 config : `PipelineTaskConfig` 

297 A `PipelineTaskConfig` class instance whose class has been configured 

298 to use this `PipelineTaskConnectionsClass` 

299 

300 See also 

301 -------- 

302 iterConnections 

303 

304 Notes 

305 ----- 

306 ``PipelineTaskConnection`` classes are created by declaring class 

307 attributes of types defined in `lsst.pipe.base.connectionTypes` and are 

308 listed as follows: 

309 

310 * ``InitInput`` - Defines connections in a quantum graph which are used as 

311 inputs to the ``__init__`` function of the `PipelineTask` corresponding 

312 to this class 

313 * ``InitOuput`` - Defines connections in a quantum graph which are to be 

314 persisted using a butler at the end of the ``__init__`` function of the 

315 `PipelineTask` corresponding to this class. The variable name used to 

316 define this connection should be the same as an attribute name on the 

317 `PipelineTask` instance. E.g. if an ``InitOutput`` is declared with 

318 the name ``outputSchema`` in a ``PipelineTaskConnections`` class, then 

319 a `PipelineTask` instance should have an attribute 

320 ``self.outputSchema`` defined. Its value is what will be saved by the 

321 activator framework. 

322 * ``PrerequisiteInput`` - An input connection type that defines a 

323 `lsst.daf.butler.DatasetType` that must be present at execution time, 

324 but that will not be used during the course of creating the quantum 

325 graph to be executed. These most often are things produced outside the 

326 processing pipeline, such as reference catalogs. 

327 * ``Input`` - Input `lsst.daf.butler.DatasetType` objects that will be used 

328 in the ``run`` method of a `PipelineTask`. The name used to declare 

329 class attribute must match a function argument name in the ``run`` 

330 method of a `PipelineTask`. E.g. If the ``PipelineTaskConnections`` 

331 defines an ``Input`` with the name ``calexp``, then the corresponding 

332 signature should be ``PipelineTask.run(calexp, ...)`` 

333 * ``Output`` - A `lsst.daf.butler.DatasetType` that will be produced by an 

334 execution of a `PipelineTask`. The name used to declare the connection 

335 must correspond to an attribute of a `Struct` that is returned by a 

336 `PipelineTask` ``run`` method. E.g. if an output connection is 

337 defined with the name ``measCat``, then the corresponding 

338 ``PipelineTask.run`` method must return ``Struct(measCat=X,..)`` where 

339 X matches the ``storageClass`` type defined on the output connection. 

340 

341 The process of declaring a ``PipelineTaskConnection`` class involves 

342 parameters passed in the declaration statement. 

343 

344 The first parameter is ``dimensions`` which is an iterable of strings which 

345 defines the unit of processing the run method of a corresponding 

346 `PipelineTask` will operate on. These dimensions must match dimensions that 

347 exist in the butler registry which will be used in executing the 

348 corresponding `PipelineTask`. 

349 

350 The second parameter is labeled ``defaultTemplates`` and is conditionally 

351 optional. The name attributes of connections can be specified as python 

352 format strings, with named format arguments. If any of the name parameters 

353 on connections defined in a `PipelineTaskConnections` class contain a 

354 template, then a default template value must be specified in the 

355 ``defaultTemplates`` argument. This is done by passing a dictionary with 

356 keys corresponding to a template identifier, and values corresponding to 

357 the value to use as a default when formatting the string. For example if 

358 ``ConnectionClass.calexp.name = '{input}Coadd_calexp'`` then 

359 ``defaultTemplates`` = {'input': 'deep'}. 

360 

361 Once a `PipelineTaskConnections` class is created, it is used in the 

362 creation of a `PipelineTaskConfig`. This is further documented in the 

363 documentation of `PipelineTaskConfig`. For the purposes of this 

364 documentation, the relevant information is that the config class allows 

365 configuration of connection names by users when running a pipeline. 

366 

367 Instances of a `PipelineTaskConnections` class are used by the pipeline 

368 task execution framework to introspect what a corresponding `PipelineTask` 

369 will require, and what it will produce. 

370 

371 Examples 

372 -------- 

373 >>> from lsst.pipe.base import connectionTypes as cT 

374 >>> from lsst.pipe.base import PipelineTaskConnections 

375 >>> from lsst.pipe.base import PipelineTaskConfig 

376 >>> class ExampleConnections(PipelineTaskConnections, 

377 ... dimensions=("A", "B"), 

378 ... defaultTemplates={"foo": "Example"}): 

379 ... inputConnection = cT.Input(doc="Example input", 

380 ... dimensions=("A", "B"), 

381 ... storageClass=Exposure, 

382 ... name="{foo}Dataset") 

383 ... outputConnection = cT.Output(doc="Example output", 

384 ... dimensions=("A", "B"), 

385 ... storageClass=Exposure, 

386 ... name="{foo}output") 

387 >>> class ExampleConfig(PipelineTaskConfig, 

388 ... pipelineConnections=ExampleConnections): 

389 ... pass 

390 >>> config = ExampleConfig() 

391 >>> config.connections.foo = Modified 

392 >>> config.connections.outputConnection = "TotallyDifferent" 

393 >>> connections = ExampleConnections(config=config) 

394 >>> assert(connections.inputConnection.name == "ModifiedDataset") 

395 >>> assert(connections.outputConnection.name == "TotallyDifferent") 

396 """ 

397 

398 def __init__(self, *, config: "PipelineTaskConfig" = None): 

399 self.inputs = set(self.inputs) 

400 self.prerequisiteInputs = set(self.prerequisiteInputs) 

401 self.outputs = set(self.outputs) 

402 self.initInputs = set(self.initInputs) 

403 self.initOutputs = set(self.initOutputs) 

404 self.allConnections = dict(self.allConnections) 

405 

406 from .config import PipelineTaskConfig # local import to avoid cycle 

407 

408 if config is None or not isinstance(config, PipelineTaskConfig): 

409 raise ValueError( 

410 "PipelineTaskConnections must be instantiated with a PipelineTaskConfig instance" 

411 ) 

412 self.config = config 

413 # Extract the template names that were defined in the config instance 

414 # by looping over the keys of the defaultTemplates dict specified at 

415 # class declaration time 

416 templateValues = { 

417 name: getattr(config.connections, name) for name in getattr(self, "defaultTemplates").keys() 

418 } 

419 # Extract the configured value corresponding to each connection 

420 # variable. I.e. for each connection identifier, populate a override 

421 # for the connection.name attribute 

422 self._nameOverrides = { 

423 name: getattr(config.connections, name).format(**templateValues) 

424 for name in self.allConnections.keys() 

425 } 

426 

427 # connections.name corresponds to a dataset type name, create a reverse 

428 # mapping that goes from dataset type name to attribute identifier name 

429 # (variable name) on the connection class 

430 self._typeNameToVarName = {v: k for k, v in self._nameOverrides.items()} 

431 

432 def buildDatasetRefs( 

433 self, quantum: Quantum 

434 ) -> typing.Tuple[InputQuantizedConnection, OutputQuantizedConnection]: 

435 """Builds QuantizedConnections corresponding to input Quantum 

436 

437 Parameters 

438 ---------- 

439 quantum : `lsst.daf.butler.Quantum` 

440 Quantum object which defines the inputs and outputs for a given 

441 unit of processing 

442 

443 Returns 

444 ------- 

445 retVal : `tuple` of (`InputQuantizedConnection`, 

446 `OutputQuantizedConnection`) Namespaces mapping attribute names 

447 (identifiers of connections) to butler references defined in the 

448 input `lsst.daf.butler.Quantum` 

449 """ 

450 inputDatasetRefs = InputQuantizedConnection() 

451 outputDatasetRefs = OutputQuantizedConnection() 

452 # operate on a reference object and an interable of names of class 

453 # connection attributes 

454 for refs, names in zip( 

455 (inputDatasetRefs, outputDatasetRefs), 

456 (itertools.chain(self.inputs, self.prerequisiteInputs), self.outputs), 

457 ): 

458 # get a name of a class connection attribute 

459 for attributeName in names: 

460 # get the attribute identified by name 

461 attribute = getattr(self, attributeName) 

462 # Branch if the attribute dataset type is an input 

463 if attribute.name in quantum.inputs: 

464 # Get the DatasetRefs 

465 quantumInputRefs = quantum.inputs[attribute.name] 

466 # if the dataset is marked to load deferred, wrap it in a 

467 # DeferredDatasetRef 

468 if attribute.deferLoad: 

469 quantumInputRefs = [DeferredDatasetRef(datasetRef=ref) for ref in quantumInputRefs] 

470 # Unpack arguments that are not marked multiples (list of 

471 # length one) 

472 if not attribute.multiple: 

473 if len(quantumInputRefs) > 1: 

474 raise ScalarError( 

475 f"Received multiple datasets " 

476 f"{', '.join(str(r.dataId) for r in quantumInputRefs)} " 

477 f"for scalar connection {attributeName} " 

478 f"({quantumInputRefs[0].datasetType.name}) " 

479 f"of quantum for {quantum.taskName} with data ID {quantum.dataId}." 

480 ) 

481 if len(quantumInputRefs) == 0: 

482 continue 

483 quantumInputRefs = quantumInputRefs[0] 

484 # Add to the QuantizedConnection identifier 

485 setattr(refs, attributeName, quantumInputRefs) 

486 # Branch if the attribute dataset type is an output 

487 elif attribute.name in quantum.outputs: 

488 value = quantum.outputs[attribute.name] 

489 # Unpack arguments that are not marked multiples (list of 

490 # length one) 

491 if not attribute.multiple: 

492 value = value[0] 

493 # Add to the QuantizedConnection identifier 

494 setattr(refs, attributeName, value) 

495 # Specified attribute is not in inputs or outputs dont know how 

496 # to handle, throw 

497 else: 

498 raise ValueError( 

499 f"Attribute with name {attributeName} has no counterpoint in input quantum" 

500 ) 

501 return inputDatasetRefs, outputDatasetRefs 

502 

503 def adjustQuantum( 

504 self, 

505 inputs: typing.Dict[str, typing.Tuple[BaseInput, typing.Collection[DatasetRef]]], 

506 outputs: typing.Dict[str, typing.Tuple[Output, typing.Collection[DatasetRef]]], 

507 label: str, 

508 data_id: DataCoordinate, 

509 ) -> tuple.Tuple[ 

510 typing.Mapping[str, typing.Tuple[BaseInput, typing.Collection[DatasetRef]]], 

511 typing.Mapping[str, typing.Tuple[Output, typing.Collection[DatasetRef]]], 

512 ]: 

513 """Override to make adjustments to `lsst.daf.butler.DatasetRef` objects 

514 in the `lsst.daf.butler.core.Quantum` during the graph generation stage 

515 of the activator. 

516 

517 Parameters 

518 ---------- 

519 inputs : `dict` 

520 Dictionary whose keys are an input (regular or prerequisite) 

521 connection name and whose values are a tuple of the connection 

522 instance and a collection of associated `DatasetRef` objects. 

523 The exact type of the nested collections is unspecified; it can be 

524 assumed to be multi-pass iterable and support `len` and ``in``, but 

525 it should not be mutated in place. In contrast, the outer 

526 dictionaries are guaranteed to be temporary copies that are true 

527 `dict` instances, and hence may be modified and even returned; this 

528 is especially useful for delegating to `super` (see notes below). 

529 outputs : `Mapping` 

530 Mapping of output datasets, with the same structure as ``inputs``. 

531 label : `str` 

532 Label for this task in the pipeline (should be used in all 

533 diagnostic messages). 

534 data_id : `lsst.daf.butler.DataCoordinate` 

535 Data ID for this quantum in the pipeline (should be used in all 

536 diagnostic messages). 

537 

538 Returns 

539 ------- 

540 adjusted_inputs : `Mapping` 

541 Mapping of the same form as ``inputs`` with updated containers of 

542 input `DatasetRef` objects. Connections that are not changed 

543 should not be returned at all. Datasets may only be removed, not 

544 added. Nested collections may be of any multi-pass iterable type, 

545 and the order of iteration will set the order of iteration within 

546 `PipelineTask.runQuantum`. 

547 adjusted_outputs : `Mapping` 

548 Mapping of updated output datasets, with the same structure and 

549 interpretation as ``adjusted_inputs``. 

550 

551 Raises 

552 ------ 

553 ScalarError 

554 Raised if any `Input` or `PrerequisiteInput` connection has 

555 ``multiple`` set to `False`, but multiple datasets. 

556 NoWorkFound 

557 Raised to indicate that this quantum should not be run; not enough 

558 datasets were found for a regular `Input` connection, and the 

559 quantum should be pruned or skipped. 

560 FileNotFoundError 

561 Raised to cause QuantumGraph generation to fail (with the message 

562 included in this exception); not enough datasets were found for a 

563 `PrerequisiteInput` connection. 

564 

565 Notes 

566 ----- 

567 The base class implementation performs important checks. It always 

568 returns an empty mapping (i.e. makes no adjustments). It should 

569 always called be via `super` by custom implementations, ideally at the 

570 end of the custom implementation with already-adjusted mappings when 

571 any datasets are actually dropped, e.g.:: 

572 

573 def adjustQuantum(self, inputs, outputs, label, data_id): 

574 # Filter out some dataset refs for one connection. 

575 connection, old_refs = inputs["my_input"] 

576 new_refs = [ref for ref in old_refs if ...] 

577 adjusted_inputs = {"my_input", (connection, new_refs)} 

578 # Update the original inputs so we can pass them to super. 

579 inputs.update(adjusted_inputs) 

580 # Can ignore outputs from super because they are guaranteed 

581 # to be empty. 

582 super().adjustQuantum(inputs, outputs, label_data_id) 

583 # Return only the connections we modified. 

584 return adjusted_inputs, {} 

585 

586 Removing outputs here is guaranteed to affect what is actually 

587 passed to `PipelineTask.runQuantum`, but its effect on the larger 

588 graph may be deferred to execution, depending on the context in 

589 which `adjustQuantum` is being run: if one quantum removes an output 

590 that is needed by a second quantum as input, the second quantum may not 

591 be adjusted (and hence pruned or skipped) until that output is actually 

592 found to be missing at execution time. 

593 

594 Tasks that desire zip-iteration consistency between any combinations of 

595 connections that have the same data ID should generally implement 

596 `adjustQuantum` to achieve this, even if they could also run that 

597 logic during execution; this allows the system to see outputs that will 

598 not be produced because the corresponding input is missing as early as 

599 possible. 

600 """ 

601 for name, (connection, refs) in inputs.items(): 

602 dataset_type_name = connection.name 

603 if not connection.multiple and len(refs) > 1: 

604 raise ScalarError( 

605 f"Found multiple datasets {', '.join(str(r.dataId) for r in refs)} " 

606 f"for non-multiple input connection {label}.{name} ({dataset_type_name}) " 

607 f"for quantum data ID {data_id}." 

608 ) 

609 if len(refs) < connection.minimum: 

610 if isinstance(connection, PrerequisiteInput): 

611 # This branch should only be possible during QG generation, 

612 # or if someone deleted the dataset between making the QG 

613 # and trying to run it. Either one should be a hard error. 

614 raise FileNotFoundError( 

615 f"Not enough datasets ({len(refs)}) found for non-optional connection {label}.{name} " 

616 f"({dataset_type_name}) with minimum={connection.minimum} for quantum data ID " 

617 f"{data_id}." 

618 ) 

619 else: 

620 # This branch should be impossible during QG generation, 

621 # because that algorithm can only make quanta whose inputs 

622 # are either already present or should be created during 

623 # execution. It can trigger during execution if the input 

624 # wasn't actually created by an upstream task in the same 

625 # graph. 

626 raise NoWorkFound(label, name, connection) 

627 for name, (connection, refs) in outputs.items(): 

628 dataset_type_name = connection.name 

629 if not connection.multiple and len(refs) > 1: 

630 raise ScalarError( 

631 f"Found multiple datasets {', '.join(str(r.dataId) for r in refs)} " 

632 f"for non-multiple output connection {label}.{name} ({dataset_type_name}) " 

633 f"for quantum data ID {data_id}." 

634 ) 

635 return {}, {} 

636 

637 

638def iterConnections( 

639 connections: PipelineTaskConnections, connectionType: Union[str, Iterable[str]] 

640) -> typing.Generator[BaseConnection, None, None]: 

641 """Creates an iterator over the selected connections type which yields 

642 all the defined connections of that type. 

643 

644 Parameters 

645 ---------- 

646 connections: `PipelineTaskConnections` 

647 An instance of a `PipelineTaskConnections` object that will be iterated 

648 over. 

649 connectionType: `str` 

650 The type of connections to iterate over, valid values are inputs, 

651 outputs, prerequisiteInputs, initInputs, initOutputs. 

652 

653 Yields 

654 ------- 

655 connection: `BaseConnection` 

656 A connection defined on the input connections object of the type 

657 supplied. The yielded value Will be an derived type of 

658 `BaseConnection`. 

659 """ 

660 if isinstance(connectionType, str): 

661 connectionType = (connectionType,) 

662 for name in itertools.chain.from_iterable(getattr(connections, ct) for ct in connectionType): 

663 yield getattr(connections, name) 

664 

665 

666@dataclass 

667class AdjustQuantumHelper: 

668 """Helper class for calling `PipelineTaskConnections.adjustQuantum`. 

669 

670 This class holds `input` and `output` mappings in the form used by 

671 `Quantum` and execution harness code, i.e. with `DatasetType` keys, 

672 translating them to and from the connection-oriented mappings used inside 

673 `PipelineTaskConnections`. 

674 """ 

675 

676 inputs: NamedKeyMapping[DatasetType, typing.List[DatasetRef]] 

677 """Mapping of regular input and prerequisite input datasets, grouped by 

678 `DatasetType`. 

679 """ 

680 

681 outputs: NamedKeyMapping[DatasetType, typing.List[DatasetRef]] 

682 """Mapping of output datasets, grouped by `DatasetType`. 

683 """ 

684 

685 inputs_adjusted: bool = False 

686 """Whether any inputs were removed in the last call to `adjust_in_place`. 

687 """ 

688 

689 outputs_adjusted: bool = False 

690 """Whether any outputs were removed in the last call to `adjust_in_place`. 

691 """ 

692 

693 def adjust_in_place( 

694 self, 

695 connections: PipelineTaskConnections, 

696 label: str, 

697 data_id: DataCoordinate, 

698 ) -> None: 

699 """Call `~PipelineTaskConnections.adjustQuantum` and update ``self`` 

700 with its results. 

701 

702 Parameters 

703 ---------- 

704 connections : `PipelineTaskConnections` 

705 Instance on which to call `~PipelineTaskConnections.adjustQuantum`. 

706 label : `str` 

707 Label for this task in the pipeline (should be used in all 

708 diagnostic messages). 

709 data_id : `lsst.daf.butler.DataCoordinate` 

710 Data ID for this quantum in the pipeline (should be used in all 

711 diagnostic messages). 

712 """ 

713 # Translate self's DatasetType-keyed, Quantum-oriented mappings into 

714 # connection-keyed, PipelineTask-oriented mappings. 

715 inputs_by_connection: typing.Dict[str, typing.Tuple[BaseInput, typing.Tuple[DatasetRef, ...]]] = {} 

716 outputs_by_connection: typing.Dict[str, typing.Tuple[Output, typing.Tuple[DatasetRef, ...]]] = {} 

717 for name in itertools.chain(connections.inputs, connections.prerequisiteInputs): 

718 connection = getattr(connections, name) 

719 dataset_type_name = connection.name 

720 inputs_by_connection[name] = (connection, tuple(self.inputs.get(dataset_type_name, ()))) 

721 for name in itertools.chain(connections.outputs): 

722 connection = getattr(connections, name) 

723 dataset_type_name = connection.name 

724 outputs_by_connection[name] = (connection, tuple(self.outputs.get(dataset_type_name, ()))) 

725 # Actually call adjustQuantum. 

726 adjusted_inputs_by_connection, adjusted_outputs_by_connection = connections.adjustQuantum( 

727 inputs_by_connection, 

728 outputs_by_connection, 

729 label, 

730 data_id, 

731 ) 

732 # Translate adjustments to DatasetType-keyed, Quantum-oriented form, 

733 # installing new mappings in self if necessary. 

734 if adjusted_inputs_by_connection: 

735 adjusted_inputs = NamedKeyDict[DatasetType, typing.List[DatasetRef]](self.inputs) 

736 for name, (connection, updated_refs) in adjusted_inputs_by_connection.items(): 

737 dataset_type_name = connection.name 

738 if not set(updated_refs).issubset(self.inputs[dataset_type_name]): 

739 raise RuntimeError( 

740 f"adjustQuantum implementation for task with label {label} returned {name} " 

741 f"({dataset_type_name}) input datasets that are not a subset of those " 

742 f"it was given for data ID {data_id}." 

743 ) 

744 adjusted_inputs[dataset_type_name] = list(updated_refs) 

745 self.inputs = adjusted_inputs.freeze() 

746 self.inputs_adjusted = True 

747 else: 

748 self.inputs_adjusted = False 

749 if adjusted_outputs_by_connection: 

750 adjusted_outputs = NamedKeyDict[DatasetType, typing.List[DatasetRef]](self.outputs) 

751 for name, (connection, updated_refs) in adjusted_outputs_by_connection.items(): 

752 if not set(updated_refs).issubset(self.outputs[dataset_type_name]): 

753 raise RuntimeError( 

754 f"adjustQuantum implementation for task with label {label} returned {name} " 

755 f"({dataset_type_name}) output datasets that are not a subset of those " 

756 f"it was given for data ID {data_id}." 

757 ) 

758 adjusted_outputs[dataset_type_name] = list(updated_refs) 

759 self.outputs = adjusted_outputs.freeze() 

760 self.outputs_adjusted = True 

761 else: 

762 self.outputs_adjusted = False