Coverage for python/lsst/daf/butler/datastore/_datastore.py: 64%

275 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-05 10:00 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Support for generic data stores.""" 

29 

30from __future__ import annotations 

31 

32__all__ = ( 

33 "DatasetRefURIs", 

34 "Datastore", 

35 "DatastoreConfig", 

36 "DatastoreOpaqueTable", 

37 "DatastoreValidationError", 

38 "NullDatastore", 

39 "DatastoreTransaction", 

40) 

41 

42import contextlib 

43import dataclasses 

44import logging 

45import time 

46from abc import ABCMeta, abstractmethod 

47from collections import abc, defaultdict 

48from collections.abc import Callable, Collection, Iterable, Iterator, Mapping 

49from typing import TYPE_CHECKING, Any, ClassVar 

50 

51from lsst.utils import doImportType 

52 

53from .._config import Config, ConfigSubset 

54from .._exceptions import DatasetTypeNotSupportedError, ValidationError 

55from .._file_dataset import FileDataset 

56from .._storage_class import StorageClassFactory 

57from .constraints import Constraints 

58 

59if TYPE_CHECKING: 

60 from lsst.resources import ResourcePath, ResourcePathExpression 

61 

62 from .. import ddl 

63 from .._config_support import LookupKey 

64 from .._dataset_ref import DatasetRef 

65 from .._dataset_type import DatasetType 

66 from .._storage_class import StorageClass 

67 from ..registry.interfaces import DatasetIdRef, DatastoreRegistryBridgeManager 

68 from .record_data import DatastoreRecordData 

69 from .stored_file_info import StoredDatastoreItemInfo 

70 

71_LOG = logging.getLogger(__name__) 

72 

73 

74class DatastoreConfig(ConfigSubset): 

75 """Configuration for Datastores.""" 

76 

77 component = "datastore" 

78 requiredKeys = ("cls",) 

79 defaultConfigFile = "datastore.yaml" 

80 

81 

82class DatastoreValidationError(ValidationError): 

83 """There is a problem with the Datastore configuration.""" 

84 

85 pass 

86 

87 

88@dataclasses.dataclass(frozen=True) 

89class Event: 

90 """Representation of an event that can be rolled back.""" 

91 

92 __slots__ = {"name", "undoFunc", "args", "kwargs"} 

93 name: str 

94 undoFunc: Callable 

95 args: tuple 

96 kwargs: dict 

97 

98 

99@dataclasses.dataclass(frozen=True) 

100class DatastoreOpaqueTable: 

101 """Definition of the opaque table which stores datastore records. 

102 

103 Table definition contains `.ddl.TableSpec` for a table and a class 

104 of a record which must be a subclass of `StoredDatastoreItemInfo`. 

105 """ 

106 

107 __slots__ = {"table_spec", "record_class"} 

108 table_spec: ddl.TableSpec 

109 record_class: type[StoredDatastoreItemInfo] 

110 

111 

112class IngestPrepData: 

113 """A helper base class for `Datastore` ingest implementations. 

114 

115 Datastore implementations will generally need a custom implementation of 

116 this class. 

117 

118 Should be accessed as ``Datastore.IngestPrepData`` instead of via direct 

119 import. 

120 

121 Parameters 

122 ---------- 

123 refs : iterable of `DatasetRef` 

124 References for the datasets that can be ingested by this datastore. 

125 """ 

126 

127 def __init__(self, refs: Iterable[DatasetRef]): 

128 self.refs = {ref.id: ref for ref in refs} 

129 

130 

131class DatastoreTransaction: 

132 """Keeps a log of `Datastore` activity and allow rollback. 

133 

134 Parameters 

135 ---------- 

136 parent : `DatastoreTransaction`, optional 

137 The parent transaction (if any). 

138 """ 

139 

140 Event: ClassVar[type] = Event 

141 

142 parent: DatastoreTransaction | None 

143 """The parent transaction. (`DatastoreTransaction`, optional)""" 

144 

145 def __init__(self, parent: DatastoreTransaction | None = None): 

146 self.parent = parent 

147 self._log: list[Event] = [] 

148 

149 def registerUndo(self, name: str, undoFunc: Callable, *args: Any, **kwargs: Any) -> None: 

150 """Register event with undo function. 

151 

152 Parameters 

153 ---------- 

154 name : `str` 

155 Name of the event. 

156 undoFunc : `~collections.abc.Callable` 

157 Function to undo this event. 

158 *args : `tuple` 

159 Positional arguments to ``undoFunc``. 

160 **kwargs 

161 Keyword arguments to ``undoFunc``. 

162 """ 

163 self._log.append(self.Event(name, undoFunc, args, kwargs)) 

164 

165 @contextlib.contextmanager 

166 def undoWith(self, name: str, undoFunc: Callable, *args: Any, **kwargs: Any) -> Iterator[None]: 

167 """Register undo function if nested operation succeeds. 

168 

169 Calls `registerUndo`. 

170 

171 This can be used to wrap individual undo-able statements within a 

172 DatastoreTransaction block. Multiple statements that can fail 

173 separately should not be part of the same `undoWith` block. 

174 

175 All arguments are forwarded directly to `registerUndo`. 

176 

177 Parameters 

178 ---------- 

179 name : `str` 

180 The name to associate with this event. 

181 undoFunc : `~collections.abc.Callable` 

182 Function to undo this event. 

183 *args : `tuple` 

184 Positional arguments for ``undoFunc``. 

185 **kwargs : `typing.Any` 

186 Keyword arguments for ``undoFunc``. 

187 """ 

188 try: 

189 yield None 

190 except BaseException: 

191 raise 

192 else: 

193 self.registerUndo(name, undoFunc, *args, **kwargs) 

194 

195 def rollback(self) -> None: 

196 """Roll back all events in this transaction.""" 

197 log = logging.getLogger(__name__) 

198 while self._log: 

199 ev = self._log.pop() 

200 try: 

201 log.debug( 

202 "Rolling back transaction: %s: %s(%s,%s)", 

203 ev.name, 

204 ev.undoFunc, 

205 ",".join(str(a) for a in ev.args), 

206 ",".join(f"{k}={v}" for k, v in ev.kwargs.items()), 

207 ) 

208 except Exception: 

209 # In case we had a problem in stringification of arguments 

210 log.warning("Rolling back transaction: %s", ev.name) 

211 try: 

212 ev.undoFunc(*ev.args, **ev.kwargs) 

213 except BaseException as e: 

214 # Deliberately swallow error that may occur in unrolling 

215 log.warning("Exception: %s caught while unrolling: %s", e, ev.name) 

216 pass 

217 

218 def commit(self) -> None: 

219 """Commit this transaction.""" 

220 if self.parent is None: 

221 # Just forget about the events, they have already happened. 

222 return 

223 else: 

224 # We may still want to events from this transaction as part of 

225 # the parent. 

226 self.parent._log.extend(self._log) 

227 

228 

229@dataclasses.dataclass 

230class DatasetRefURIs(abc.Sequence): 

231 """Represents the primary and component ResourcePath(s) associated with a 

232 DatasetRef. 

233 

234 This is used in places where its members used to be represented as a tuple 

235 `(primaryURI, componentURIs)`. To maintain backward compatibility this 

236 inherits from Sequence and so instances can be treated as a two-item 

237 tuple. 

238 

239 Parameters 

240 ---------- 

241 primaryURI : `lsst.resources.ResourcePath` or `None`, optional 

242 The URI to the primary artifact associated with this dataset. If the 

243 dataset was disassembled within the datastore this may be `None`. 

244 componentURIs : `dict` [`str`, `~lsst.resources.ResourcePath`] or `None` 

245 The URIs to any components associated with the dataset artifact 

246 indexed by component name. This can be empty if there are no 

247 components. 

248 """ 

249 

250 def __init__( 

251 self, 

252 primaryURI: ResourcePath | None = None, 

253 componentURIs: dict[str, ResourcePath] | None = None, 

254 ): 

255 self.primaryURI = primaryURI 

256 self.componentURIs = componentURIs or {} 

257 

258 def __getitem__(self, index: Any) -> Any: 

259 """Get primaryURI and componentURIs by index. 

260 

261 Provides support for tuple-like access. 

262 """ 

263 if index == 0: 

264 return self.primaryURI 

265 elif index == 1: 

266 return self.componentURIs 

267 raise IndexError("list index out of range") 

268 

269 def __len__(self) -> int: 

270 """Get the number of data members. 

271 

272 Provides support for tuple-like access. 

273 """ 

274 return 2 

275 

276 def __repr__(self) -> str: 

277 return f"DatasetRefURIs({repr(self.primaryURI)}, {repr(self.componentURIs)})" 

278 

279 

280class Datastore(metaclass=ABCMeta): 

281 """Datastore interface. 

282 

283 Parameters 

284 ---------- 

285 config : `DatastoreConfig` or `str` 

286 Load configuration either from an existing config instance or by 

287 referring to a configuration file. 

288 bridgeManager : `DatastoreRegistryBridgeManager` 

289 Object that manages the interface between `Registry` and datastores. 

290 """ 

291 

292 defaultConfigFile: ClassVar[str | None] = None 

293 """Path to configuration defaults. Accessed within the ``config`` resource 

294 or relative to a search path. Can be None if no defaults specified. 

295 """ 

296 

297 containerKey: ClassVar[str | None] = None 

298 """Name of the key containing a list of subconfigurations that also 

299 need to be merged with defaults and will likely use different Python 

300 datastore classes (but all using DatastoreConfig). Assumed to be a 

301 list of configurations that can be represented in a DatastoreConfig 

302 and containing a "cls" definition. None indicates that no containers 

303 are expected in this Datastore.""" 

304 

305 isEphemeral: bool = False 

306 """Indicate whether this Datastore is ephemeral or not. An ephemeral 

307 datastore is one where the contents of the datastore will not exist 

308 across process restarts. This value can change per-instance.""" 

309 

310 config: DatastoreConfig 

311 """Configuration used to create Datastore.""" 

312 

313 name: str 

314 """Label associated with this Datastore.""" 

315 

316 storageClassFactory: StorageClassFactory 

317 """Factory for creating storage class instances from name.""" 

318 

319 constraints: Constraints 

320 """Constraints to apply when putting datasets into the datastore.""" 

321 

322 # MyPy does not like for this to be annotated as any kind of type, because 

323 # it can't do static checking on type variables that can change at runtime. 

324 IngestPrepData: ClassVar[Any] = IngestPrepData 

325 """Helper base class for ingest implementations. 

326 """ 

327 

328 @classmethod 

329 @abstractmethod 

330 def setConfigRoot(cls, root: str, config: Config, full: Config, overwrite: bool = True) -> None: 

331 """Set filesystem-dependent config options for this datastore. 

332 

333 The options will be appropriate for a new empty repository with the 

334 given root. 

335 

336 Parameters 

337 ---------- 

338 root : `str` 

339 Filesystem path to the root of the data repository. 

340 config : `Config` 

341 A `Config` to update. Only the subset understood by 

342 this component will be updated. Will not expand 

343 defaults. 

344 full : `Config` 

345 A complete config with all defaults expanded that can be 

346 converted to a `DatastoreConfig`. Read-only and will not be 

347 modified by this method. 

348 Repository-specific options that should not be obtained 

349 from defaults when Butler instances are constructed 

350 should be copied from ``full`` to ``config``. 

351 overwrite : `bool`, optional 

352 If `False`, do not modify a value in ``config`` if the value 

353 already exists. Default is always to overwrite with the provided 

354 ``root``. 

355 

356 Notes 

357 ----- 

358 If a keyword is explicitly defined in the supplied ``config`` it 

359 will not be overridden by this method if ``overwrite`` is `False`. 

360 This allows explicit values set in external configs to be retained. 

361 """ 

362 raise NotImplementedError() 

363 

364 @staticmethod 

365 def fromConfig( 

366 config: Config, 

367 bridgeManager: DatastoreRegistryBridgeManager, 

368 butlerRoot: ResourcePathExpression | None = None, 

369 ) -> Datastore: 

370 """Create datastore from type specified in config file. 

371 

372 Parameters 

373 ---------- 

374 config : `Config` or `~lsst.resources.ResourcePathExpression` 

375 Configuration instance. 

376 bridgeManager : `DatastoreRegistryBridgeManager` 

377 Object that manages the interface between `Registry` and 

378 datastores. 

379 butlerRoot : `str`, optional 

380 Butler root directory. 

381 """ 

382 config = DatastoreConfig(config) 

383 cls = doImportType(config["cls"]) 

384 if not issubclass(cls, Datastore): 

385 raise TypeError(f"Imported child class {config['cls']} is not a Datastore") 

386 return cls._create_from_config(config=config, bridgeManager=bridgeManager, butlerRoot=butlerRoot) 

387 

388 def __init__( 

389 self, 

390 config: DatastoreConfig, 

391 bridgeManager: DatastoreRegistryBridgeManager, 

392 ): 

393 self.config = config 

394 self.name = "ABCDataStore" 

395 self._transaction: DatastoreTransaction | None = None 

396 

397 # All Datastores need storage classes and constraints 

398 self.storageClassFactory = StorageClassFactory() 

399 

400 # And read the constraints list 

401 constraintsConfig = self.config.get("constraints") 

402 self.constraints = Constraints(constraintsConfig, universe=bridgeManager.universe) 

403 

404 @classmethod 

405 @abstractmethod 

406 def _create_from_config( 

407 cls, 

408 config: DatastoreConfig, 

409 bridgeManager: DatastoreRegistryBridgeManager, 

410 butlerRoot: ResourcePathExpression | None, 

411 ) -> Datastore: 

412 """`Datastore`.``fromConfig`` calls this to instantiate Datastore 

413 subclasses. This is the primary constructor for the individual 

414 Datastore subclasses. 

415 """ 

416 raise NotImplementedError() 

417 

418 @abstractmethod 

419 def clone(self, bridgeManager: DatastoreRegistryBridgeManager) -> Datastore: 

420 """Make an independent copy of this Datastore with a different 

421 `DatastoreRegistryBridgeManager` instance. 

422 

423 Parameters 

424 ---------- 

425 bridgeManager : `DatastoreRegistryBridgeManager` 

426 New `DatastoreRegistryBridgeManager` object to use when 

427 instantiating managers. 

428 

429 Returns 

430 ------- 

431 datastore : `Datastore` 

432 New `Datastore` instance with the same configuration as the 

433 existing instance. 

434 """ 

435 raise NotImplementedError() 

436 

437 def __str__(self) -> str: 

438 return self.name 

439 

440 def __repr__(self) -> str: 

441 return self.name 

442 

443 @property 

444 def names(self) -> tuple[str, ...]: 

445 """Names associated with this datastore returned as a list. 

446 

447 Can be different to ``name`` for a chaining datastore. 

448 """ 

449 # Default implementation returns solely the name itself 

450 return (self.name,) 

451 

452 @property 

453 def roots(self) -> dict[str, ResourcePath | None]: 

454 """Return the root URIs for each named datastore. 

455 

456 Mapping from datastore name to root URI. The URI can be `None` 

457 if a datastore has no concept of a root URI. 

458 (`dict` [`str`, `ResourcePath` | `None`]) 

459 """ 

460 return {self.name: None} 

461 

462 @contextlib.contextmanager 

463 def transaction(self) -> Iterator[DatastoreTransaction]: 

464 """Context manager supporting `Datastore` transactions. 

465 

466 Transactions can be nested, and are to be used in combination with 

467 `Registry.transaction`. 

468 """ 

469 self._transaction = DatastoreTransaction(self._transaction) 

470 try: 

471 yield self._transaction 

472 except BaseException: 

473 self._transaction.rollback() 

474 raise 

475 else: 

476 self._transaction.commit() 

477 self._transaction = self._transaction.parent 

478 

479 def _set_trust_mode(self, mode: bool) -> None: 

480 """Set the trust mode for this datastore. 

481 

482 Parameters 

483 ---------- 

484 mode : `bool` 

485 If `True`, get requests will be attempted even if the datastore 

486 does not know about the dataset. 

487 

488 Notes 

489 ----- 

490 This is a private method to indicate that trust mode might be a 

491 transitory property that we do not want to make fully public. For now 

492 only a `~lsst.daf.butler.datastores.FileDatastore` understands this 

493 concept. By default this method does nothing. 

494 """ 

495 return 

496 

497 @abstractmethod 

498 def knows(self, ref: DatasetRef) -> bool: 

499 """Check if the dataset is known to the datastore. 

500 

501 Does not check for existence of any artifact. 

502 

503 Parameters 

504 ---------- 

505 ref : `DatasetRef` 

506 Reference to the required dataset. 

507 

508 Returns 

509 ------- 

510 exists : `bool` 

511 `True` if the dataset is known to the datastore. 

512 """ 

513 raise NotImplementedError() 

514 

515 def knows_these(self, refs: Iterable[DatasetRef]) -> dict[DatasetRef, bool]: 

516 """Check which of the given datasets are known to this datastore. 

517 

518 This is like ``mexist()`` but does not check that the file exists. 

519 

520 Parameters 

521 ---------- 

522 refs : iterable `DatasetRef` 

523 The datasets to check. 

524 

525 Returns 

526 ------- 

527 exists : `dict`[`DatasetRef`, `bool`] 

528 Mapping of dataset to boolean indicating whether the dataset 

529 is known to the datastore. 

530 """ 

531 # Non-optimized default calls knows() repeatedly. 

532 return {ref: self.knows(ref) for ref in refs} 

533 

534 def mexists( 

535 self, refs: Iterable[DatasetRef], artifact_existence: dict[ResourcePath, bool] | None = None 

536 ) -> dict[DatasetRef, bool]: 

537 """Check the existence of multiple datasets at once. 

538 

539 Parameters 

540 ---------- 

541 refs : iterable of `DatasetRef` 

542 The datasets to be checked. 

543 artifact_existence : `dict` [`lsst.resources.ResourcePath`, `bool`] 

544 Optional mapping of datastore artifact to existence. Updated by 

545 this method with details of all artifacts tested. Can be `None` 

546 if the caller is not interested. 

547 

548 Returns 

549 ------- 

550 existence : `dict` of [`DatasetRef`, `bool`] 

551 Mapping from dataset to boolean indicating existence. 

552 """ 

553 existence: dict[DatasetRef, bool] = {} 

554 # Non-optimized default. 

555 for ref in refs: 

556 existence[ref] = self.exists(ref) 

557 return existence 

558 

559 @abstractmethod 

560 def exists(self, datasetRef: DatasetRef) -> bool: 

561 """Check if the dataset exists in the datastore. 

562 

563 Parameters 

564 ---------- 

565 datasetRef : `DatasetRef` 

566 Reference to the required dataset. 

567 

568 Returns 

569 ------- 

570 exists : `bool` 

571 `True` if the entity exists in the `Datastore`. 

572 """ 

573 raise NotImplementedError("Must be implemented by subclass") 

574 

575 @abstractmethod 

576 def get( 

577 self, 

578 datasetRef: DatasetRef, 

579 parameters: Mapping[str, Any] | None = None, 

580 storageClass: StorageClass | str | None = None, 

581 ) -> Any: 

582 """Load an `InMemoryDataset` from the store. 

583 

584 Parameters 

585 ---------- 

586 datasetRef : `DatasetRef` 

587 Reference to the required Dataset. 

588 parameters : `dict` 

589 `StorageClass`-specific parameters that specify a slice of the 

590 Dataset to be loaded. 

591 storageClass : `StorageClass` or `str`, optional 

592 The storage class to be used to override the Python type 

593 returned by this method. By default the returned type matches 

594 the dataset type definition for this dataset. Specifying a 

595 read `StorageClass` can force a different type to be returned. 

596 This type must be compatible with the original type. 

597 

598 Returns 

599 ------- 

600 inMemoryDataset : `object` 

601 Requested Dataset or slice thereof as an InMemoryDataset. 

602 """ 

603 raise NotImplementedError("Must be implemented by subclass") 

604 

605 def prepare_get_for_external_client(self, ref: DatasetRef) -> object | None: 

606 """Retrieve serializable data that can be used to execute a ``get()``. 

607 

608 Parameters 

609 ---------- 

610 ref : `DatasetRef` 

611 Reference to the required dataset. 

612 

613 Returns 

614 ------- 

615 payload : `object` | `None` 

616 Serializable payload containing the information needed to perform a 

617 get() operation. This payload may be sent over the wire to another 

618 system to perform the get(). Returns `None` if the dataset is not 

619 known to this datastore. 

620 """ 

621 raise NotImplementedError() 

622 

623 @abstractmethod 

624 def put(self, inMemoryDataset: Any, datasetRef: DatasetRef) -> None: 

625 """Write a `InMemoryDataset` with a given `DatasetRef` to the store. 

626 

627 Parameters 

628 ---------- 

629 inMemoryDataset : `object` 

630 The Dataset to store. 

631 datasetRef : `DatasetRef` 

632 Reference to the associated Dataset. 

633 """ 

634 raise NotImplementedError("Must be implemented by subclass") 

635 

636 @abstractmethod 

637 def put_new(self, in_memory_dataset: Any, ref: DatasetRef) -> Mapping[str, DatasetRef]: 

638 """Write a `InMemoryDataset` with a given `DatasetRef` to the store. 

639 

640 Parameters 

641 ---------- 

642 in_memory_dataset : `object` 

643 The Dataset to store. 

644 ref : `DatasetRef` 

645 Reference to the associated Dataset. 

646 

647 Returns 

648 ------- 

649 datastore_refs : `~collections.abc.Mapping` [`str`, `DatasetRef`] 

650 Mapping of a datastore name to dataset reference stored in that 

651 datastore, reference will include datastore records. Only 

652 non-ephemeral datastores will appear in this mapping. 

653 """ 

654 raise NotImplementedError("Must be implemented by subclass") 

655 

656 def _overrideTransferMode(self, *datasets: FileDataset, transfer: str | None = None) -> str | None: 

657 """Allow ingest transfer mode to be defaulted based on datasets. 

658 

659 Parameters 

660 ---------- 

661 *datasets : `FileDataset` 

662 Each positional argument is a struct containing information about 

663 a file to be ingested, including its path (either absolute or 

664 relative to the datastore root, if applicable), a complete 

665 `DatasetRef` (with ``dataset_id not None``), and optionally a 

666 formatter class or its fully-qualified string name. If a formatter 

667 is not provided, this method should populate that attribute with 

668 the formatter the datastore would use for `put`. Subclasses are 

669 also permitted to modify the path attribute (typically to put it 

670 in what the datastore considers its standard form). 

671 transfer : `str`, optional 

672 How (and whether) the dataset should be added to the datastore. 

673 See `ingest` for details of transfer modes. 

674 

675 Returns 

676 ------- 

677 newTransfer : `str` 

678 Transfer mode to use. Will be identical to the supplied transfer 

679 mode unless "auto" is used. 

680 """ 

681 if transfer != "auto": 

682 return transfer 

683 raise RuntimeError(f"{transfer} is not allowed without specialization.") 

684 

685 def _prepIngest(self, *datasets: FileDataset, transfer: str | None = None) -> IngestPrepData: 

686 """Process datasets to identify which ones can be ingested. 

687 

688 Parameters 

689 ---------- 

690 *datasets : `FileDataset` 

691 Each positional argument is a struct containing information about 

692 a file to be ingested, including its path (either absolute or 

693 relative to the datastore root, if applicable), a complete 

694 `DatasetRef` (with ``dataset_id not None``), and optionally a 

695 formatter class or its fully-qualified string name. If a formatter 

696 is not provided, this method should populate that attribute with 

697 the formatter the datastore would use for `put`. Subclasses are 

698 also permitted to modify the path attribute (typically to put it 

699 in what the datastore considers its standard form). 

700 transfer : `str`, optional 

701 How (and whether) the dataset should be added to the datastore. 

702 See `ingest` for details of transfer modes. 

703 

704 Returns 

705 ------- 

706 data : `IngestPrepData` 

707 An instance of a subclass of `IngestPrepData`, used to pass 

708 arbitrary data from `_prepIngest` to `_finishIngest`. This should 

709 include only the datasets this datastore can actually ingest; 

710 others should be silently ignored (`Datastore.ingest` will inspect 

711 `IngestPrepData.refs` and raise `DatasetTypeNotSupportedError` if 

712 necessary). 

713 

714 Raises 

715 ------ 

716 NotImplementedError 

717 Raised if the datastore does not support the given transfer mode 

718 (including the case where ingest is not supported at all). 

719 FileNotFoundError 

720 Raised if one of the given files does not exist. 

721 FileExistsError 

722 Raised if transfer is not `None` but the (internal) location the 

723 file would be moved to is already occupied. 

724 

725 Notes 

726 ----- 

727 This method (along with `_finishIngest`) should be implemented by 

728 subclasses to provide ingest support instead of implementing `ingest` 

729 directly. 

730 

731 `_prepIngest` should not modify the data repository or given files in 

732 any way; all changes should be deferred to `_finishIngest`. 

733 

734 When possible, exceptions should be raised in `_prepIngest` instead of 

735 `_finishIngest`. `NotImplementedError` exceptions that indicate that 

736 the transfer mode is not supported must be raised by `_prepIngest` 

737 instead of `_finishIngest`. 

738 """ 

739 raise NotImplementedError(f"Datastore {self} does not support direct file-based ingest.") 

740 

741 def _finishIngest( 

742 self, prepData: IngestPrepData, *, transfer: str | None = None, record_validation_info: bool = True 

743 ) -> None: 

744 """Complete an ingest operation. 

745 

746 Parameters 

747 ---------- 

748 prepData : `IngestPrepData` 

749 An instance of a subclass of `IngestPrepData`. Guaranteed to be 

750 the direct result of a call to `_prepIngest` on this datastore. 

751 transfer : `str`, optional 

752 How (and whether) the dataset should be added to the datastore. 

753 See `ingest` for details of transfer modes. 

754 record_validation_info : `bool`, optional 

755 If `True`, the default, the datastore can record validation 

756 information associated with the file. If `False` the datastore 

757 will not attempt to track any information such as checksums 

758 or file sizes. This can be useful if such information is tracked 

759 in an external system or if the file is to be compressed in place. 

760 It is up to the datastore whether this parameter is relevant. 

761 

762 Raises 

763 ------ 

764 FileNotFoundError 

765 Raised if one of the given files does not exist. 

766 FileExistsError 

767 Raised if transfer is not `None` but the (internal) location the 

768 file would be moved to is already occupied. 

769 

770 Notes 

771 ----- 

772 This method (along with `_prepIngest`) should be implemented by 

773 subclasses to provide ingest support instead of implementing `ingest` 

774 directly. 

775 """ 

776 raise NotImplementedError(f"Datastore {self} does not support direct file-based ingest.") 

777 

778 def ingest( 

779 self, *datasets: FileDataset, transfer: str | None = None, record_validation_info: bool = True 

780 ) -> None: 

781 """Ingest one or more files into the datastore. 

782 

783 Parameters 

784 ---------- 

785 *datasets : `FileDataset` 

786 Each positional argument is a struct containing information about 

787 a file to be ingested, including its path (either absolute or 

788 relative to the datastore root, if applicable), a complete 

789 `DatasetRef` (with ``dataset_id not None``), and optionally a 

790 formatter class or its fully-qualified string name. If a formatter 

791 is not provided, the one the datastore would use for ``put`` on 

792 that dataset is assumed. 

793 transfer : `str`, optional 

794 How (and whether) the dataset should be added to the datastore. 

795 If `None` (default), the file must already be in a location 

796 appropriate for the datastore (e.g. within its root directory), 

797 and will not be modified. Other choices include "move", "copy", 

798 "link", "symlink", "relsymlink", and "hardlink". "link" is a 

799 special transfer mode that will first try to make a hardlink and 

800 if that fails a symlink will be used instead. "relsymlink" creates 

801 a relative symlink rather than use an absolute path. 

802 Most datastores do not support all transfer modes. 

803 "auto" is a special option that will let the 

804 data store choose the most natural option for itself. 

805 record_validation_info : `bool`, optional 

806 If `True`, the default, the datastore can record validation 

807 information associated with the file. If `False` the datastore 

808 will not attempt to track any information such as checksums 

809 or file sizes. This can be useful if such information is tracked 

810 in an external system or if the file is to be compressed in place. 

811 It is up to the datastore whether this parameter is relevant. 

812 

813 Raises 

814 ------ 

815 NotImplementedError 

816 Raised if the datastore does not support the given transfer mode 

817 (including the case where ingest is not supported at all). 

818 DatasetTypeNotSupportedError 

819 Raised if one or more files to be ingested have a dataset type that 

820 is not supported by the datastore. 

821 FileNotFoundError 

822 Raised if one of the given files does not exist. 

823 FileExistsError 

824 Raised if transfer is not `None` but the (internal) location the 

825 file would be moved to is already occupied. 

826 

827 Notes 

828 ----- 

829 Subclasses should implement `_prepIngest` and `_finishIngest` instead 

830 of implementing `ingest` directly. Datastores that hold and 

831 delegate to child datastores may want to call those methods as well. 

832 

833 Subclasses are encouraged to document their supported transfer modes 

834 in their class documentation. 

835 """ 

836 # Allow a datastore to select a default transfer mode 

837 transfer = self._overrideTransferMode(*datasets, transfer=transfer) 

838 prepData = self._prepIngest(*datasets, transfer=transfer) 

839 refs = {ref.id: ref for dataset in datasets for ref in dataset.refs} 

840 if refs.keys() != prepData.refs.keys(): 

841 unsupported = refs.keys() - prepData.refs.keys() 

842 # Group unsupported refs by DatasetType for an informative 

843 # but still concise error message. 

844 byDatasetType = defaultdict(list) 

845 for datasetId in unsupported: 

846 ref = refs[datasetId] 

847 byDatasetType[ref.datasetType].append(ref) 

848 raise DatasetTypeNotSupportedError( 

849 "DatasetType(s) not supported in ingest: " 

850 + ", ".join(f"{k.name} ({len(v)} dataset(s))" for k, v in byDatasetType.items()) 

851 ) 

852 self._finishIngest(prepData, transfer=transfer, record_validation_info=record_validation_info) 

853 

854 def transfer_from( 

855 self, 

856 source_datastore: Datastore, 

857 refs: Collection[DatasetRef], 

858 transfer: str = "auto", 

859 artifact_existence: dict[ResourcePath, bool] | None = None, 

860 dry_run: bool = False, 

861 ) -> tuple[set[DatasetRef], set[DatasetRef]]: 

862 """Transfer dataset artifacts from another datastore to this one. 

863 

864 Parameters 

865 ---------- 

866 source_datastore : `Datastore` 

867 The datastore from which to transfer artifacts. That datastore 

868 must be compatible with this datastore receiving the artifacts. 

869 refs : `~collections.abc.Collection` of `DatasetRef` 

870 The datasets to transfer from the source datastore. 

871 transfer : `str`, optional 

872 How (and whether) the dataset should be added to the datastore. 

873 Choices include "move", "copy", 

874 "link", "symlink", "relsymlink", and "hardlink". "link" is a 

875 special transfer mode that will first try to make a hardlink and 

876 if that fails a symlink will be used instead. "relsymlink" creates 

877 a relative symlink rather than use an absolute path. 

878 Most datastores do not support all transfer modes. 

879 "auto" (the default) is a special option that will let the 

880 data store choose the most natural option for itself. 

881 If the source location and transfer location are identical the 

882 transfer mode will be ignored. 

883 artifact_existence : `dict` [`lsst.resources.ResourcePath`, `bool`] 

884 Optional mapping of datastore artifact to existence. Updated by 

885 this method with details of all artifacts tested. Can be `None` 

886 if the caller is not interested. 

887 dry_run : `bool`, optional 

888 Process the supplied source refs without updating the target 

889 datastore. 

890 

891 Returns 

892 ------- 

893 accepted : `set` [`DatasetRef`] 

894 The datasets that were transferred. 

895 rejected : `set` [`DatasetRef`] 

896 The datasets that were rejected due to a constraints violation. 

897 

898 Raises 

899 ------ 

900 TypeError 

901 Raised if the two datastores are not compatible. 

902 """ 

903 if type(self) is not type(source_datastore): 

904 raise TypeError( 

905 f"Datastore mismatch between this datastore ({type(self)}) and the " 

906 f"source datastore ({type(source_datastore)})." 

907 ) 

908 

909 raise NotImplementedError(f"Datastore {type(self)} must implement a transfer_from method.") 

910 

911 def getManyURIs( 

912 self, 

913 refs: Iterable[DatasetRef], 

914 predict: bool = False, 

915 allow_missing: bool = False, 

916 ) -> dict[DatasetRef, DatasetRefURIs]: 

917 """Return URIs associated with many datasets. 

918 

919 Parameters 

920 ---------- 

921 refs : iterable of `DatasetIdRef` 

922 References to the required datasets. 

923 predict : `bool`, optional 

924 If `True`, allow URIs to be returned of datasets that have not 

925 been written. 

926 allow_missing : `bool` 

927 If `False`, and ``predict`` is `False`, will raise if a 

928 `DatasetRef` does not exist. 

929 

930 Returns 

931 ------- 

932 URIs : `dict` of [`DatasetRef`, `DatasetRefUris`] 

933 A dict of primary and component URIs, indexed by the passed-in 

934 refs. 

935 

936 Raises 

937 ------ 

938 FileNotFoundError 

939 A URI has been requested for a dataset that does not exist and 

940 guessing is not allowed. 

941 

942 Notes 

943 ----- 

944 In file-based datastores, getManyURIs does not check that the file is 

945 really there, it's assuming it is if datastore is aware of the file 

946 then it actually exists. 

947 """ 

948 uris: dict[DatasetRef, DatasetRefURIs] = {} 

949 missing_refs = [] 

950 for ref in refs: 

951 try: 

952 uris[ref] = self.getURIs(ref, predict=predict) 

953 except FileNotFoundError: 

954 missing_refs.append(ref) 

955 if missing_refs and not allow_missing: 

956 num_missing = len(missing_refs) 

957 raise FileNotFoundError( 

958 f"Missing {num_missing} refs from datastore out of " 

959 f"{num_missing + len(uris)} and predict=False." 

960 ) 

961 return uris 

962 

963 @abstractmethod 

964 def getURIs(self, datasetRef: DatasetRef, predict: bool = False) -> DatasetRefURIs: 

965 """Return URIs associated with dataset. 

966 

967 Parameters 

968 ---------- 

969 datasetRef : `DatasetRef` 

970 Reference to the required dataset. 

971 predict : `bool`, optional 

972 If the datastore does not know about the dataset, controls whether 

973 it should return a predicted URI or not. 

974 

975 Returns 

976 ------- 

977 uris : `DatasetRefURIs` 

978 The URI to the primary artifact associated with this dataset (if 

979 the dataset was disassembled within the datastore this may be 

980 `None`), and the URIs to any components associated with the dataset 

981 artifact. (can be empty if there are no components). 

982 """ 

983 raise NotImplementedError() 

984 

985 @abstractmethod 

986 def getURI(self, datasetRef: DatasetRef, predict: bool = False) -> ResourcePath: 

987 """URI to the Dataset. 

988 

989 Parameters 

990 ---------- 

991 datasetRef : `DatasetRef` 

992 Reference to the required Dataset. 

993 predict : `bool` 

994 If `True` attempt to predict the URI for a dataset if it does 

995 not exist in datastore. 

996 

997 Returns 

998 ------- 

999 uri : `str` 

1000 URI string pointing to the Dataset within the datastore. If the 

1001 Dataset does not exist in the datastore, the URI may be a guess. 

1002 If the datastore does not have entities that relate well 

1003 to the concept of a URI the returned URI string will be 

1004 descriptive. The returned URI is not guaranteed to be obtainable. 

1005 

1006 Raises 

1007 ------ 

1008 FileNotFoundError 

1009 A URI has been requested for a dataset that does not exist and 

1010 guessing is not allowed. 

1011 """ 

1012 raise NotImplementedError("Must be implemented by subclass") 

1013 

1014 @abstractmethod 

1015 def retrieveArtifacts( 

1016 self, 

1017 refs: Iterable[DatasetRef], 

1018 destination: ResourcePath, 

1019 transfer: str = "auto", 

1020 preserve_path: bool = True, 

1021 overwrite: bool = False, 

1022 ) -> list[ResourcePath]: 

1023 """Retrieve the artifacts associated with the supplied refs. 

1024 

1025 Parameters 

1026 ---------- 

1027 refs : iterable of `DatasetRef` 

1028 The datasets for which artifacts are to be retrieved. 

1029 A single ref can result in multiple artifacts. The refs must 

1030 be resolved. 

1031 destination : `lsst.resources.ResourcePath` 

1032 Location to write the artifacts. 

1033 transfer : `str`, optional 

1034 Method to use to transfer the artifacts. Must be one of the options 

1035 supported by `lsst.resources.ResourcePath.transfer_from()`. 

1036 "move" is not allowed. 

1037 preserve_path : `bool`, optional 

1038 If `True` the full path of the artifact within the datastore 

1039 is preserved. If `False` the final file component of the path 

1040 is used. 

1041 overwrite : `bool`, optional 

1042 If `True` allow transfers to overwrite existing files at the 

1043 destination. 

1044 

1045 Returns 

1046 ------- 

1047 targets : `list` of `lsst.resources.ResourcePath` 

1048 URIs of file artifacts in destination location. Order is not 

1049 preserved. 

1050 

1051 Notes 

1052 ----- 

1053 For non-file datastores the artifacts written to the destination 

1054 may not match the representation inside the datastore. For example 

1055 a hierarchichal data structure in a NoSQL database may well be stored 

1056 as a JSON file. 

1057 """ 

1058 raise NotImplementedError() 

1059 

1060 @abstractmethod 

1061 def remove(self, datasetRef: DatasetRef) -> None: 

1062 """Indicate to the Datastore that a Dataset can be removed. 

1063 

1064 Parameters 

1065 ---------- 

1066 datasetRef : `DatasetRef` 

1067 Reference to the required Dataset. 

1068 

1069 Raises 

1070 ------ 

1071 FileNotFoundError 

1072 When Dataset does not exist. 

1073 

1074 Notes 

1075 ----- 

1076 Some Datastores may implement this method as a silent no-op to 

1077 disable Dataset deletion through standard interfaces. 

1078 """ 

1079 raise NotImplementedError("Must be implemented by subclass") 

1080 

1081 @abstractmethod 

1082 def forget(self, refs: Iterable[DatasetRef]) -> None: 

1083 """Indicate to the Datastore that it should remove all records of the 

1084 given datasets, without actually deleting them. 

1085 

1086 Parameters 

1087 ---------- 

1088 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

1089 References to the datasets being forgotten. 

1090 

1091 Notes 

1092 ----- 

1093 Asking a datastore to forget a `DatasetRef` it does not hold should be 

1094 a silent no-op, not an error. 

1095 """ 

1096 raise NotImplementedError("Must be implemented by subclass") 

1097 

1098 @abstractmethod 

1099 def trash(self, ref: DatasetRef | Iterable[DatasetRef], ignore_errors: bool = True) -> None: 

1100 """Indicate to the Datastore that a Dataset can be moved to the trash. 

1101 

1102 Parameters 

1103 ---------- 

1104 ref : `DatasetRef` or iterable thereof 

1105 Reference(s) to the required Dataset. 

1106 ignore_errors : `bool`, optional 

1107 Determine whether errors should be ignored. When multiple 

1108 refs are being trashed there will be no per-ref check. 

1109 

1110 Raises 

1111 ------ 

1112 FileNotFoundError 

1113 When Dataset does not exist and errors are not ignored. Only 

1114 checked if a single ref is supplied (and not in a list). 

1115 

1116 Notes 

1117 ----- 

1118 Some Datastores may implement this method as a silent no-op to 

1119 disable Dataset deletion through standard interfaces. 

1120 """ 

1121 raise NotImplementedError("Must be implemented by subclass") 

1122 

1123 @abstractmethod 

1124 def emptyTrash(self, ignore_errors: bool = True) -> None: 

1125 """Remove all datasets from the trash. 

1126 

1127 Parameters 

1128 ---------- 

1129 ignore_errors : `bool`, optional 

1130 Determine whether errors should be ignored. 

1131 

1132 Notes 

1133 ----- 

1134 Some Datastores may implement this method as a silent no-op to 

1135 disable Dataset deletion through standard interfaces. 

1136 """ 

1137 raise NotImplementedError("Must be implemented by subclass") 

1138 

1139 @abstractmethod 

1140 def transfer(self, inputDatastore: Datastore, datasetRef: DatasetRef) -> None: 

1141 """Transfer a dataset from another datastore to this datastore. 

1142 

1143 Parameters 

1144 ---------- 

1145 inputDatastore : `Datastore` 

1146 The external `Datastore` from which to retrieve the Dataset. 

1147 datasetRef : `DatasetRef` 

1148 Reference to the required Dataset. 

1149 """ 

1150 raise NotImplementedError("Must be implemented by subclass") 

1151 

1152 def export( 

1153 self, 

1154 refs: Iterable[DatasetRef], 

1155 *, 

1156 directory: ResourcePathExpression | None = None, 

1157 transfer: str | None = "auto", 

1158 ) -> Iterable[FileDataset]: 

1159 """Export datasets for transfer to another data repository. 

1160 

1161 Parameters 

1162 ---------- 

1163 refs : iterable of `DatasetRef` 

1164 Dataset references to be exported. 

1165 directory : `str`, optional 

1166 Path to a directory that should contain files corresponding to 

1167 output datasets. Ignored if ``transfer`` is explicitly `None`. 

1168 transfer : `str`, optional 

1169 Mode that should be used to move datasets out of the repository. 

1170 Valid options are the same as those of the ``transfer`` argument 

1171 to ``ingest``, and datastores may similarly signal that a transfer 

1172 mode is not supported by raising `NotImplementedError`. If "auto" 

1173 is given and no ``directory`` is specified, `None` will be 

1174 implied. 

1175 

1176 Returns 

1177 ------- 

1178 dataset : iterable of `DatasetTransfer` 

1179 Structs containing information about the exported datasets, in the 

1180 same order as ``refs``. 

1181 

1182 Raises 

1183 ------ 

1184 NotImplementedError 

1185 Raised if the given transfer mode is not supported. 

1186 """ 

1187 raise NotImplementedError(f"Transfer mode {transfer} not supported.") 

1188 

1189 @abstractmethod 

1190 def validateConfiguration( 

1191 self, entities: Iterable[DatasetRef | DatasetType | StorageClass], logFailures: bool = False 

1192 ) -> None: 

1193 """Validate some of the configuration for this datastore. 

1194 

1195 Parameters 

1196 ---------- 

1197 entities : iterable of `DatasetRef`, `DatasetType`, or `StorageClass` 

1198 Entities to test against this configuration. Can be differing 

1199 types. 

1200 logFailures : `bool`, optional 

1201 If `True`, output a log message for every validation error 

1202 detected. 

1203 

1204 Raises 

1205 ------ 

1206 DatastoreValidationError 

1207 Raised if there is a validation problem with a configuration. 

1208 

1209 Notes 

1210 ----- 

1211 Which parts of the configuration are validated is at the discretion 

1212 of each Datastore implementation. 

1213 """ 

1214 raise NotImplementedError("Must be implemented by subclass") 

1215 

1216 @abstractmethod 

1217 def validateKey(self, lookupKey: LookupKey, entity: DatasetRef | DatasetType | StorageClass) -> None: 

1218 """Validate a specific look up key with supplied entity. 

1219 

1220 Parameters 

1221 ---------- 

1222 lookupKey : `LookupKey` 

1223 Key to use to retrieve information from the datastore 

1224 configuration. 

1225 entity : `DatasetRef`, `DatasetType`, or `StorageClass` 

1226 Entity to compare with configuration retrieved using the 

1227 specified lookup key. 

1228 

1229 Raises 

1230 ------ 

1231 DatastoreValidationError 

1232 Raised if there is a problem with the combination of entity 

1233 and lookup key. 

1234 

1235 Notes 

1236 ----- 

1237 Bypasses the normal selection priorities by allowing a key that 

1238 would normally not be selected to be validated. 

1239 """ 

1240 raise NotImplementedError("Must be implemented by subclass") 

1241 

1242 @abstractmethod 

1243 def getLookupKeys(self) -> set[LookupKey]: 

1244 """Return all the lookup keys relevant to this datastore. 

1245 

1246 Returns 

1247 ------- 

1248 keys : `set` of `LookupKey` 

1249 The keys stored internally for looking up information based 

1250 on `DatasetType` name or `StorageClass`. 

1251 """ 

1252 raise NotImplementedError("Must be implemented by subclass") 

1253 

1254 def needs_expanded_data_ids( 

1255 self, 

1256 transfer: str | None, 

1257 entity: DatasetRef | DatasetType | StorageClass | None = None, 

1258 ) -> bool: 

1259 """Test whether this datastore needs expanded data IDs to ingest. 

1260 

1261 Parameters 

1262 ---------- 

1263 transfer : `str` or `None` 

1264 Transfer mode for ingest. 

1265 entity : `DatasetRef` or `DatasetType` or `StorageClass` or `None`, \ 

1266 optional 

1267 Object representing what will be ingested. If not provided (or not 

1268 specific enough), `True` may be returned even if expanded data 

1269 IDs aren't necessary. 

1270 

1271 Returns 

1272 ------- 

1273 needed : `bool` 

1274 If `True`, expanded data IDs may be needed. `False` only if 

1275 expansion definitely isn't necessary. 

1276 """ 

1277 return True 

1278 

1279 @abstractmethod 

1280 def import_records( 

1281 self, 

1282 data: Mapping[str, DatastoreRecordData], 

1283 ) -> None: 

1284 """Import datastore location and record data from an in-memory data 

1285 structure. 

1286 

1287 Parameters 

1288 ---------- 

1289 data : `~collections.abc.Mapping` [ `str`, `DatastoreRecordData` ] 

1290 Datastore records indexed by datastore name. May contain data for 

1291 other `Datastore` instances (generally because they are chained to 

1292 this one), which should be ignored. 

1293 

1294 Notes 

1295 ----- 

1296 Implementations should generally not check that any external resources 

1297 (e.g. files) referred to by these records actually exist, for 

1298 performance reasons; we expect higher-level code to guarantee that they 

1299 do. 

1300 

1301 Implementations are responsible for calling 

1302 `DatastoreRegistryBridge.insert` on all datasets in ``data.locations`` 

1303 where the key is in `names`, as well as loading any opaque table data. 

1304 

1305 Implementations may assume that datasets are either fully present or 

1306 not at all (single-component exports are not permitted). 

1307 """ 

1308 raise NotImplementedError() 

1309 

1310 @abstractmethod 

1311 def export_records( 

1312 self, 

1313 refs: Iterable[DatasetIdRef], 

1314 ) -> Mapping[str, DatastoreRecordData]: 

1315 """Export datastore records and locations to an in-memory data 

1316 structure. 

1317 

1318 Parameters 

1319 ---------- 

1320 refs : `~collections.abc.Iterable` [ `DatasetIdRef` ] 

1321 Datasets to save. This may include datasets not known to this 

1322 datastore, which should be ignored. May not include component 

1323 datasets. 

1324 

1325 Returns 

1326 ------- 

1327 data : `~collections.abc.Mapping` [ `str`, `DatastoreRecordData` ] 

1328 Exported datastore records indexed by datastore name. 

1329 """ 

1330 raise NotImplementedError() 

1331 

1332 def set_retrieve_dataset_type_method(self, method: Callable[[str], DatasetType | None] | None) -> None: 

1333 """Specify a method that can be used by datastore to retrieve 

1334 registry-defined dataset type. 

1335 

1336 Parameters 

1337 ---------- 

1338 method : `~collections.abc.Callable` | `None` 

1339 Method that takes a name of the dataset type and returns a 

1340 corresponding `DatasetType` instance as defined in Registry. If 

1341 dataset type name is not known to registry `None` is returned. 

1342 

1343 Notes 

1344 ----- 

1345 This method is only needed for a Datastore supporting a "trusted" mode 

1346 when it does not have an access to datastore records and needs to 

1347 guess dataset location based on its stored dataset type. 

1348 """ 

1349 pass 

1350 

1351 @abstractmethod 

1352 def get_opaque_table_definitions(self) -> Mapping[str, DatastoreOpaqueTable]: 

1353 """Make definitions of the opaque tables used by this Datastore. 

1354 

1355 Returns 

1356 ------- 

1357 tables : `~collections.abc.Mapping` [ `str`, `.ddl.TableSpec` ] 

1358 Mapping of opaque table names to their definitions. This can be an 

1359 empty mapping if Datastore does not use opaque tables to keep 

1360 datastore records. 

1361 """ 

1362 raise NotImplementedError() 

1363 

1364 

1365class NullDatastore(Datastore): 

1366 """A datastore that implements the `Datastore` API but always fails when 

1367 it accepts any request. 

1368 

1369 Parameters 

1370 ---------- 

1371 config : `Config` or `~lsst.resources.ResourcePathExpression` or `None` 

1372 Ignored. 

1373 bridgeManager : `DatastoreRegistryBridgeManager` or `None` 

1374 Ignored. 

1375 butlerRoot : `~lsst.resources.ResourcePathExpression` or `None` 

1376 Ignored. 

1377 """ 

1378 

1379 @classmethod 

1380 def _create_from_config( 

1381 cls, 

1382 config: Config, 

1383 bridgeManager: DatastoreRegistryBridgeManager, 

1384 butlerRoot: ResourcePathExpression | None = None, 

1385 ) -> NullDatastore: 

1386 return NullDatastore(config, bridgeManager, butlerRoot) 

1387 

1388 def clone(self, bridgeManager: DatastoreRegistryBridgeManager) -> Datastore: 

1389 return self 

1390 

1391 @classmethod 

1392 def setConfigRoot(cls, root: str, config: Config, full: Config, overwrite: bool = True) -> None: 

1393 # Nothing to do. This is not a real Datastore. 

1394 pass 

1395 

1396 def __init__( 

1397 self, 

1398 config: Config | ResourcePathExpression | None, 

1399 bridgeManager: DatastoreRegistryBridgeManager | None, 

1400 butlerRoot: ResourcePathExpression | None = None, 

1401 ): 

1402 # Name ourselves with the timestamp the datastore 

1403 # was created. 

1404 self.name = f"{type(self).__name__}@{time.time()}" 

1405 _LOG.debug("Creating datastore %s", self.name) 

1406 

1407 return 

1408 

1409 def knows(self, ref: DatasetRef) -> bool: 

1410 return False 

1411 

1412 def exists(self, datasetRef: DatasetRef) -> bool: 

1413 return False 

1414 

1415 def get( 

1416 self, 

1417 datasetRef: DatasetRef, 

1418 parameters: Mapping[str, Any] | None = None, 

1419 storageClass: StorageClass | str | None = None, 

1420 ) -> Any: 

1421 raise FileNotFoundError("This is a no-op datastore that can not access a real datastore") 

1422 

1423 def put(self, inMemoryDataset: Any, datasetRef: DatasetRef) -> None: 

1424 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1425 

1426 def put_new(self, in_memory_dataset: Any, ref: DatasetRef) -> Mapping[str, DatasetRef]: 

1427 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1428 

1429 def ingest( 

1430 self, *datasets: FileDataset, transfer: str | None = None, record_validation_info: bool = True 

1431 ) -> None: 

1432 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1433 

1434 def transfer_from( 

1435 self, 

1436 source_datastore: Datastore, 

1437 refs: Iterable[DatasetRef], 

1438 transfer: str = "auto", 

1439 artifact_existence: dict[ResourcePath, bool] | None = None, 

1440 dry_run: bool = False, 

1441 ) -> tuple[set[DatasetRef], set[DatasetRef]]: 

1442 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1443 

1444 def getURIs(self, datasetRef: DatasetRef, predict: bool = False) -> DatasetRefURIs: 

1445 raise FileNotFoundError("This is a no-op datastore that can not access a real datastore") 

1446 

1447 def getURI(self, datasetRef: DatasetRef, predict: bool = False) -> ResourcePath: 

1448 raise FileNotFoundError("This is a no-op datastore that can not access a real datastore") 

1449 

1450 def retrieveArtifacts( 

1451 self, 

1452 refs: Iterable[DatasetRef], 

1453 destination: ResourcePath, 

1454 transfer: str = "auto", 

1455 preserve_path: bool = True, 

1456 overwrite: bool = False, 

1457 ) -> list[ResourcePath]: 

1458 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1459 

1460 def remove(self, datasetRef: DatasetRef) -> None: 

1461 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1462 

1463 def forget(self, refs: Iterable[DatasetRef]) -> None: 

1464 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1465 

1466 def trash(self, ref: DatasetRef | Iterable[DatasetRef], ignore_errors: bool = True) -> None: 

1467 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1468 

1469 def emptyTrash(self, ignore_errors: bool = True) -> None: 

1470 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1471 

1472 def transfer(self, inputDatastore: Datastore, datasetRef: DatasetRef) -> None: 

1473 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1474 

1475 def export( 

1476 self, 

1477 refs: Iterable[DatasetRef], 

1478 *, 

1479 directory: ResourcePathExpression | None = None, 

1480 transfer: str | None = "auto", 

1481 ) -> Iterable[FileDataset]: 

1482 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1483 

1484 def validateConfiguration( 

1485 self, entities: Iterable[DatasetRef | DatasetType | StorageClass], logFailures: bool = False 

1486 ) -> None: 

1487 # No configuration so always validates. 

1488 pass 

1489 

1490 def validateKey(self, lookupKey: LookupKey, entity: DatasetRef | DatasetType | StorageClass) -> None: 

1491 pass 

1492 

1493 def getLookupKeys(self) -> set[LookupKey]: 

1494 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1495 

1496 def import_records( 

1497 self, 

1498 data: Mapping[str, DatastoreRecordData], 

1499 ) -> None: 

1500 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1501 

1502 def export_records( 

1503 self, 

1504 refs: Iterable[DatasetIdRef], 

1505 ) -> Mapping[str, DatastoreRecordData]: 

1506 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1507 

1508 def get_opaque_table_definitions(self) -> Mapping[str, DatastoreOpaqueTable]: 

1509 return {}