Coverage for python/lsst/daf/butler/datastore/_datastore.py: 64%

272 statements  

« prev     ^ index     » next       coverage.py v7.4.0, created at 2024-01-25 10:50 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Support for generic data stores.""" 

29 

30from __future__ import annotations 

31 

32__all__ = ( 

33 "DatasetRefURIs", 

34 "Datastore", 

35 "DatastoreConfig", 

36 "DatastoreOpaqueTable", 

37 "DatastoreValidationError", 

38 "NullDatastore", 

39 "DatastoreTransaction", 

40) 

41 

42import contextlib 

43import dataclasses 

44import logging 

45import time 

46from abc import ABCMeta, abstractmethod 

47from collections import abc, defaultdict 

48from collections.abc import Callable, Iterable, Iterator, Mapping 

49from typing import TYPE_CHECKING, Any, ClassVar 

50 

51from lsst.utils import doImportType 

52 

53from .._config import Config, ConfigSubset 

54from .._exceptions import DatasetTypeNotSupportedError, ValidationError 

55from .._file_dataset import FileDataset 

56from .._storage_class import StorageClassFactory 

57from .constraints import Constraints 

58 

59if TYPE_CHECKING: 

60 from lsst.resources import ResourcePath, ResourcePathExpression 

61 

62 from .. import ddl 

63 from .._config_support import LookupKey 

64 from .._dataset_ref import DatasetRef 

65 from .._dataset_type import DatasetType 

66 from .._storage_class import StorageClass 

67 from ..registry.interfaces import DatasetIdRef, DatastoreRegistryBridgeManager 

68 from .record_data import DatastoreRecordData 

69 from .stored_file_info import StoredDatastoreItemInfo 

70 

71_LOG = logging.getLogger(__name__) 

72 

73 

74class DatastoreConfig(ConfigSubset): 

75 """Configuration for Datastores.""" 

76 

77 component = "datastore" 

78 requiredKeys = ("cls",) 

79 defaultConfigFile = "datastore.yaml" 

80 

81 

82class DatastoreValidationError(ValidationError): 

83 """There is a problem with the Datastore configuration.""" 

84 

85 pass 

86 

87 

88@dataclasses.dataclass(frozen=True) 

89class Event: 

90 """Representation of an event that can be rolled back.""" 

91 

92 __slots__ = {"name", "undoFunc", "args", "kwargs"} 

93 name: str 

94 undoFunc: Callable 

95 args: tuple 

96 kwargs: dict 

97 

98 

99@dataclasses.dataclass(frozen=True) 

100class DatastoreOpaqueTable: 

101 """Definition of the opaque table which stores datastore records. 

102 

103 Table definition contains `.ddl.TableSpec` for a table and a class 

104 of a record which must be a subclass of `StoredDatastoreItemInfo`. 

105 """ 

106 

107 __slots__ = {"table_spec", "record_class"} 

108 table_spec: ddl.TableSpec 

109 record_class: type[StoredDatastoreItemInfo] 

110 

111 

112class IngestPrepData: 

113 """A helper base class for `Datastore` ingest implementations. 

114 

115 Datastore implementations will generally need a custom implementation of 

116 this class. 

117 

118 Should be accessed as ``Datastore.IngestPrepData`` instead of via direct 

119 import. 

120 

121 Parameters 

122 ---------- 

123 refs : iterable of `DatasetRef` 

124 References for the datasets that can be ingested by this datastore. 

125 """ 

126 

127 def __init__(self, refs: Iterable[DatasetRef]): 

128 self.refs = {ref.id: ref for ref in refs} 

129 

130 

131class DatastoreTransaction: 

132 """Keeps a log of `Datastore` activity and allow rollback. 

133 

134 Parameters 

135 ---------- 

136 parent : `DatastoreTransaction`, optional 

137 The parent transaction (if any). 

138 """ 

139 

140 Event: ClassVar[type] = Event 

141 

142 parent: DatastoreTransaction | None 

143 """The parent transaction. (`DatastoreTransaction`, optional)""" 

144 

145 def __init__(self, parent: DatastoreTransaction | None = None): 

146 self.parent = parent 

147 self._log: list[Event] = [] 

148 

149 def registerUndo(self, name: str, undoFunc: Callable, *args: Any, **kwargs: Any) -> None: 

150 """Register event with undo function. 

151 

152 Parameters 

153 ---------- 

154 name : `str` 

155 Name of the event. 

156 undoFunc : `~collections.abc.Callable` 

157 Function to undo this event. 

158 *args : `tuple` 

159 Positional arguments to ``undoFunc``. 

160 **kwargs 

161 Keyword arguments to ``undoFunc``. 

162 """ 

163 self._log.append(self.Event(name, undoFunc, args, kwargs)) 

164 

165 @contextlib.contextmanager 

166 def undoWith(self, name: str, undoFunc: Callable, *args: Any, **kwargs: Any) -> Iterator[None]: 

167 """Register undo function if nested operation succeeds. 

168 

169 Calls `registerUndo`. 

170 

171 This can be used to wrap individual undo-able statements within a 

172 DatastoreTransaction block. Multiple statements that can fail 

173 separately should not be part of the same `undoWith` block. 

174 

175 All arguments are forwarded directly to `registerUndo`. 

176 

177 Parameters 

178 ---------- 

179 name : `str` 

180 The name to associate with this event. 

181 undoFunc : `~collections.abc.Callable` 

182 Function to undo this event. 

183 *args : `tuple` 

184 Positional arguments for ``undoFunc``. 

185 **kwargs : `typing.Any` 

186 Keyword arguments for ``undoFunc``. 

187 """ 

188 try: 

189 yield None 

190 except BaseException: 

191 raise 

192 else: 

193 self.registerUndo(name, undoFunc, *args, **kwargs) 

194 

195 def rollback(self) -> None: 

196 """Roll back all events in this transaction.""" 

197 log = logging.getLogger(__name__) 

198 while self._log: 

199 ev = self._log.pop() 

200 try: 

201 log.debug( 

202 "Rolling back transaction: %s: %s(%s,%s)", 

203 ev.name, 

204 ev.undoFunc, 

205 ",".join(str(a) for a in ev.args), 

206 ",".join(f"{k}={v}" for k, v in ev.kwargs.items()), 

207 ) 

208 except Exception: 

209 # In case we had a problem in stringification of arguments 

210 log.warning("Rolling back transaction: %s", ev.name) 

211 try: 

212 ev.undoFunc(*ev.args, **ev.kwargs) 

213 except BaseException as e: 

214 # Deliberately swallow error that may occur in unrolling 

215 log.warning("Exception: %s caught while unrolling: %s", e, ev.name) 

216 pass 

217 

218 def commit(self) -> None: 

219 """Commit this transaction.""" 

220 if self.parent is None: 

221 # Just forget about the events, they have already happened. 

222 return 

223 else: 

224 # We may still want to events from this transaction as part of 

225 # the parent. 

226 self.parent._log.extend(self._log) 

227 

228 

229@dataclasses.dataclass 

230class DatasetRefURIs(abc.Sequence): 

231 """Represents the primary and component ResourcePath(s) associated with a 

232 DatasetRef. 

233 

234 This is used in places where its members used to be represented as a tuple 

235 `(primaryURI, componentURIs)`. To maintain backward compatibility this 

236 inherits from Sequence and so instances can be treated as a two-item 

237 tuple. 

238 

239 Parameters 

240 ---------- 

241 primaryURI : `lsst.resources.ResourcePath` or `None`, optional 

242 The URI to the primary artifact associated with this dataset. If the 

243 dataset was disassembled within the datastore this may be `None`. 

244 componentURIs : `dict` [`str`, `~lsst.resources.ResourcePath`] or `None` 

245 The URIs to any components associated with the dataset artifact 

246 indexed by component name. This can be empty if there are no 

247 components. 

248 """ 

249 

250 def __init__( 

251 self, 

252 primaryURI: ResourcePath | None = None, 

253 componentURIs: dict[str, ResourcePath] | None = None, 

254 ): 

255 self.primaryURI = primaryURI 

256 self.componentURIs = componentURIs or {} 

257 

258 def __getitem__(self, index: Any) -> Any: 

259 """Get primaryURI and componentURIs by index. 

260 

261 Provides support for tuple-like access. 

262 """ 

263 if index == 0: 

264 return self.primaryURI 

265 elif index == 1: 

266 return self.componentURIs 

267 raise IndexError("list index out of range") 

268 

269 def __len__(self) -> int: 

270 """Get the number of data members. 

271 

272 Provides support for tuple-like access. 

273 """ 

274 return 2 

275 

276 def __repr__(self) -> str: 

277 return f"DatasetRefURIs({repr(self.primaryURI)}, {repr(self.componentURIs)})" 

278 

279 

280class Datastore(metaclass=ABCMeta): 

281 """Datastore interface. 

282 

283 Parameters 

284 ---------- 

285 config : `DatastoreConfig` or `str` 

286 Load configuration either from an existing config instance or by 

287 referring to a configuration file. 

288 bridgeManager : `DatastoreRegistryBridgeManager` 

289 Object that manages the interface between `Registry` and datastores. 

290 """ 

291 

292 defaultConfigFile: ClassVar[str | None] = None 

293 """Path to configuration defaults. Accessed within the ``config`` resource 

294 or relative to a search path. Can be None if no defaults specified. 

295 """ 

296 

297 containerKey: ClassVar[str | None] = None 

298 """Name of the key containing a list of subconfigurations that also 

299 need to be merged with defaults and will likely use different Python 

300 datastore classes (but all using DatastoreConfig). Assumed to be a 

301 list of configurations that can be represented in a DatastoreConfig 

302 and containing a "cls" definition. None indicates that no containers 

303 are expected in this Datastore.""" 

304 

305 isEphemeral: bool = False 

306 """Indicate whether this Datastore is ephemeral or not. An ephemeral 

307 datastore is one where the contents of the datastore will not exist 

308 across process restarts. This value can change per-instance.""" 

309 

310 config: DatastoreConfig 

311 """Configuration used to create Datastore.""" 

312 

313 name: str 

314 """Label associated with this Datastore.""" 

315 

316 storageClassFactory: StorageClassFactory 

317 """Factory for creating storage class instances from name.""" 

318 

319 constraints: Constraints 

320 """Constraints to apply when putting datasets into the datastore.""" 

321 

322 # MyPy does not like for this to be annotated as any kind of type, because 

323 # it can't do static checking on type variables that can change at runtime. 

324 IngestPrepData: ClassVar[Any] = IngestPrepData 

325 """Helper base class for ingest implementations. 

326 """ 

327 

328 @classmethod 

329 @abstractmethod 

330 def setConfigRoot(cls, root: str, config: Config, full: Config, overwrite: bool = True) -> None: 

331 """Set filesystem-dependent config options for this datastore. 

332 

333 The options will be appropriate for a new empty repository with the 

334 given root. 

335 

336 Parameters 

337 ---------- 

338 root : `str` 

339 Filesystem path to the root of the data repository. 

340 config : `Config` 

341 A `Config` to update. Only the subset understood by 

342 this component will be updated. Will not expand 

343 defaults. 

344 full : `Config` 

345 A complete config with all defaults expanded that can be 

346 converted to a `DatastoreConfig`. Read-only and will not be 

347 modified by this method. 

348 Repository-specific options that should not be obtained 

349 from defaults when Butler instances are constructed 

350 should be copied from ``full`` to ``config``. 

351 overwrite : `bool`, optional 

352 If `False`, do not modify a value in ``config`` if the value 

353 already exists. Default is always to overwrite with the provided 

354 ``root``. 

355 

356 Notes 

357 ----- 

358 If a keyword is explicitly defined in the supplied ``config`` it 

359 will not be overridden by this method if ``overwrite`` is `False`. 

360 This allows explicit values set in external configs to be retained. 

361 """ 

362 raise NotImplementedError() 

363 

364 @staticmethod 

365 def fromConfig( 

366 config: Config, 

367 bridgeManager: DatastoreRegistryBridgeManager, 

368 butlerRoot: ResourcePathExpression | None = None, 

369 ) -> Datastore: 

370 """Create datastore from type specified in config file. 

371 

372 Parameters 

373 ---------- 

374 config : `Config` or `~lsst.resources.ResourcePathExpression` 

375 Configuration instance. 

376 bridgeManager : `DatastoreRegistryBridgeManager` 

377 Object that manages the interface between `Registry` and 

378 datastores. 

379 butlerRoot : `str`, optional 

380 Butler root directory. 

381 """ 

382 config = DatastoreConfig(config) 

383 cls = doImportType(config["cls"]) 

384 if not issubclass(cls, Datastore): 

385 raise TypeError(f"Imported child class {config['cls']} is not a Datastore") 

386 return cls._create_from_config(config=config, bridgeManager=bridgeManager, butlerRoot=butlerRoot) 

387 

388 def __init__( 

389 self, 

390 config: DatastoreConfig, 

391 bridgeManager: DatastoreRegistryBridgeManager, 

392 ): 

393 self.config = config 

394 self.name = "ABCDataStore" 

395 self._transaction: DatastoreTransaction | None = None 

396 

397 # All Datastores need storage classes and constraints 

398 self.storageClassFactory = StorageClassFactory() 

399 

400 # And read the constraints list 

401 constraintsConfig = self.config.get("constraints") 

402 self.constraints = Constraints(constraintsConfig, universe=bridgeManager.universe) 

403 

404 @classmethod 

405 @abstractmethod 

406 def _create_from_config( 

407 cls, 

408 config: DatastoreConfig, 

409 bridgeManager: DatastoreRegistryBridgeManager, 

410 butlerRoot: ResourcePathExpression | None, 

411 ) -> Datastore: 

412 """`Datastore`.``fromConfig`` calls this to instantiate Datastore 

413 subclasses. This is the primary constructor for the individual 

414 Datastore subclasses. 

415 """ 

416 raise NotImplementedError() 

417 

418 @abstractmethod 

419 def clone(self, bridgeManager: DatastoreRegistryBridgeManager) -> Datastore: 

420 """Make an independent copy of this Datastore with a different 

421 `DatastoreRegistryBridgeManager` instance. 

422 

423 Parameters 

424 ---------- 

425 bridgeManager : `DatastoreRegistryBridgeManager` 

426 New `DatastoreRegistryBridgeManager` object to use when 

427 instantiating managers. 

428 

429 Returns 

430 ------- 

431 datastore : `Datastore` 

432 New `Datastore` instance with the same configuration as the 

433 existing instance. 

434 """ 

435 raise NotImplementedError() 

436 

437 def __str__(self) -> str: 

438 return self.name 

439 

440 def __repr__(self) -> str: 

441 return self.name 

442 

443 @property 

444 def names(self) -> tuple[str, ...]: 

445 """Names associated with this datastore returned as a list. 

446 

447 Can be different to ``name`` for a chaining datastore. 

448 """ 

449 # Default implementation returns solely the name itself 

450 return (self.name,) 

451 

452 @property 

453 def roots(self) -> dict[str, ResourcePath | None]: 

454 """Return the root URIs for each named datastore. 

455 

456 Mapping from datastore name to root URI. The URI can be `None` 

457 if a datastore has no concept of a root URI. 

458 (`dict` [`str`, `ResourcePath` | `None`]) 

459 """ 

460 return {self.name: None} 

461 

462 @contextlib.contextmanager 

463 def transaction(self) -> Iterator[DatastoreTransaction]: 

464 """Context manager supporting `Datastore` transactions. 

465 

466 Transactions can be nested, and are to be used in combination with 

467 `Registry.transaction`. 

468 """ 

469 self._transaction = DatastoreTransaction(self._transaction) 

470 try: 

471 yield self._transaction 

472 except BaseException: 

473 self._transaction.rollback() 

474 raise 

475 else: 

476 self._transaction.commit() 

477 self._transaction = self._transaction.parent 

478 

479 @abstractmethod 

480 def knows(self, ref: DatasetRef) -> bool: 

481 """Check if the dataset is known to the datastore. 

482 

483 Does not check for existence of any artifact. 

484 

485 Parameters 

486 ---------- 

487 ref : `DatasetRef` 

488 Reference to the required dataset. 

489 

490 Returns 

491 ------- 

492 exists : `bool` 

493 `True` if the dataset is known to the datastore. 

494 """ 

495 raise NotImplementedError() 

496 

497 def knows_these(self, refs: Iterable[DatasetRef]) -> dict[DatasetRef, bool]: 

498 """Check which of the given datasets are known to this datastore. 

499 

500 This is like ``mexist()`` but does not check that the file exists. 

501 

502 Parameters 

503 ---------- 

504 refs : iterable `DatasetRef` 

505 The datasets to check. 

506 

507 Returns 

508 ------- 

509 exists : `dict`[`DatasetRef`, `bool`] 

510 Mapping of dataset to boolean indicating whether the dataset 

511 is known to the datastore. 

512 """ 

513 # Non-optimized default calls knows() repeatedly. 

514 return {ref: self.knows(ref) for ref in refs} 

515 

516 def mexists( 

517 self, refs: Iterable[DatasetRef], artifact_existence: dict[ResourcePath, bool] | None = None 

518 ) -> dict[DatasetRef, bool]: 

519 """Check the existence of multiple datasets at once. 

520 

521 Parameters 

522 ---------- 

523 refs : iterable of `DatasetRef` 

524 The datasets to be checked. 

525 artifact_existence : `dict` [`lsst.resources.ResourcePath`, `bool`] 

526 Optional mapping of datastore artifact to existence. Updated by 

527 this method with details of all artifacts tested. Can be `None` 

528 if the caller is not interested. 

529 

530 Returns 

531 ------- 

532 existence : `dict` of [`DatasetRef`, `bool`] 

533 Mapping from dataset to boolean indicating existence. 

534 """ 

535 existence: dict[DatasetRef, bool] = {} 

536 # Non-optimized default. 

537 for ref in refs: 

538 existence[ref] = self.exists(ref) 

539 return existence 

540 

541 @abstractmethod 

542 def exists(self, datasetRef: DatasetRef) -> bool: 

543 """Check if the dataset exists in the datastore. 

544 

545 Parameters 

546 ---------- 

547 datasetRef : `DatasetRef` 

548 Reference to the required dataset. 

549 

550 Returns 

551 ------- 

552 exists : `bool` 

553 `True` if the entity exists in the `Datastore`. 

554 """ 

555 raise NotImplementedError("Must be implemented by subclass") 

556 

557 @abstractmethod 

558 def get( 

559 self, 

560 datasetRef: DatasetRef, 

561 parameters: Mapping[str, Any] | None = None, 

562 storageClass: StorageClass | str | None = None, 

563 ) -> Any: 

564 """Load an `InMemoryDataset` from the store. 

565 

566 Parameters 

567 ---------- 

568 datasetRef : `DatasetRef` 

569 Reference to the required Dataset. 

570 parameters : `dict` 

571 `StorageClass`-specific parameters that specify a slice of the 

572 Dataset to be loaded. 

573 storageClass : `StorageClass` or `str`, optional 

574 The storage class to be used to override the Python type 

575 returned by this method. By default the returned type matches 

576 the dataset type definition for this dataset. Specifying a 

577 read `StorageClass` can force a different type to be returned. 

578 This type must be compatible with the original type. 

579 

580 Returns 

581 ------- 

582 inMemoryDataset : `object` 

583 Requested Dataset or slice thereof as an InMemoryDataset. 

584 """ 

585 raise NotImplementedError("Must be implemented by subclass") 

586 

587 def prepare_get_for_external_client(self, ref: DatasetRef) -> object: 

588 """Retrieve serializable data that can be used to execute a ``get()``. 

589 

590 Parameters 

591 ---------- 

592 ref : `DatasetRef` 

593 Reference to the required dataset. 

594 

595 Returns 

596 ------- 

597 payload : `object` 

598 Serializable payload containing the information needed to perform a 

599 get() operation. This payload may be sent over the wire to another 

600 system to perform the get(). 

601 """ 

602 raise NotImplementedError() 

603 

604 @abstractmethod 

605 def put(self, inMemoryDataset: Any, datasetRef: DatasetRef) -> None: 

606 """Write a `InMemoryDataset` with a given `DatasetRef` to the store. 

607 

608 Parameters 

609 ---------- 

610 inMemoryDataset : `object` 

611 The Dataset to store. 

612 datasetRef : `DatasetRef` 

613 Reference to the associated Dataset. 

614 """ 

615 raise NotImplementedError("Must be implemented by subclass") 

616 

617 @abstractmethod 

618 def put_new(self, in_memory_dataset: Any, ref: DatasetRef) -> Mapping[str, DatasetRef]: 

619 """Write a `InMemoryDataset` with a given `DatasetRef` to the store. 

620 

621 Parameters 

622 ---------- 

623 in_memory_dataset : `object` 

624 The Dataset to store. 

625 ref : `DatasetRef` 

626 Reference to the associated Dataset. 

627 

628 Returns 

629 ------- 

630 datastore_refs : `~collections.abc.Mapping` [`str`, `DatasetRef`] 

631 Mapping of a datastore name to dataset reference stored in that 

632 datastore, reference will include datastore records. Only 

633 non-ephemeral datastores will appear in this mapping. 

634 """ 

635 raise NotImplementedError("Must be implemented by subclass") 

636 

637 def _overrideTransferMode(self, *datasets: FileDataset, transfer: str | None = None) -> str | None: 

638 """Allow ingest transfer mode to be defaulted based on datasets. 

639 

640 Parameters 

641 ---------- 

642 *datasets : `FileDataset` 

643 Each positional argument is a struct containing information about 

644 a file to be ingested, including its path (either absolute or 

645 relative to the datastore root, if applicable), a complete 

646 `DatasetRef` (with ``dataset_id not None``), and optionally a 

647 formatter class or its fully-qualified string name. If a formatter 

648 is not provided, this method should populate that attribute with 

649 the formatter the datastore would use for `put`. Subclasses are 

650 also permitted to modify the path attribute (typically to put it 

651 in what the datastore considers its standard form). 

652 transfer : `str`, optional 

653 How (and whether) the dataset should be added to the datastore. 

654 See `ingest` for details of transfer modes. 

655 

656 Returns 

657 ------- 

658 newTransfer : `str` 

659 Transfer mode to use. Will be identical to the supplied transfer 

660 mode unless "auto" is used. 

661 """ 

662 if transfer != "auto": 

663 return transfer 

664 raise RuntimeError(f"{transfer} is not allowed without specialization.") 

665 

666 def _prepIngest(self, *datasets: FileDataset, transfer: str | None = None) -> IngestPrepData: 

667 """Process datasets to identify which ones can be ingested. 

668 

669 Parameters 

670 ---------- 

671 *datasets : `FileDataset` 

672 Each positional argument is a struct containing information about 

673 a file to be ingested, including its path (either absolute or 

674 relative to the datastore root, if applicable), a complete 

675 `DatasetRef` (with ``dataset_id not None``), and optionally a 

676 formatter class or its fully-qualified string name. If a formatter 

677 is not provided, this method should populate that attribute with 

678 the formatter the datastore would use for `put`. Subclasses are 

679 also permitted to modify the path attribute (typically to put it 

680 in what the datastore considers its standard form). 

681 transfer : `str`, optional 

682 How (and whether) the dataset should be added to the datastore. 

683 See `ingest` for details of transfer modes. 

684 

685 Returns 

686 ------- 

687 data : `IngestPrepData` 

688 An instance of a subclass of `IngestPrepData`, used to pass 

689 arbitrary data from `_prepIngest` to `_finishIngest`. This should 

690 include only the datasets this datastore can actually ingest; 

691 others should be silently ignored (`Datastore.ingest` will inspect 

692 `IngestPrepData.refs` and raise `DatasetTypeNotSupportedError` if 

693 necessary). 

694 

695 Raises 

696 ------ 

697 NotImplementedError 

698 Raised if the datastore does not support the given transfer mode 

699 (including the case where ingest is not supported at all). 

700 FileNotFoundError 

701 Raised if one of the given files does not exist. 

702 FileExistsError 

703 Raised if transfer is not `None` but the (internal) location the 

704 file would be moved to is already occupied. 

705 

706 Notes 

707 ----- 

708 This method (along with `_finishIngest`) should be implemented by 

709 subclasses to provide ingest support instead of implementing `ingest` 

710 directly. 

711 

712 `_prepIngest` should not modify the data repository or given files in 

713 any way; all changes should be deferred to `_finishIngest`. 

714 

715 When possible, exceptions should be raised in `_prepIngest` instead of 

716 `_finishIngest`. `NotImplementedError` exceptions that indicate that 

717 the transfer mode is not supported must be raised by `_prepIngest` 

718 instead of `_finishIngest`. 

719 """ 

720 raise NotImplementedError(f"Datastore {self} does not support direct file-based ingest.") 

721 

722 def _finishIngest( 

723 self, prepData: IngestPrepData, *, transfer: str | None = None, record_validation_info: bool = True 

724 ) -> None: 

725 """Complete an ingest operation. 

726 

727 Parameters 

728 ---------- 

729 prepData : `IngestPrepData` 

730 An instance of a subclass of `IngestPrepData`. Guaranteed to be 

731 the direct result of a call to `_prepIngest` on this datastore. 

732 transfer : `str`, optional 

733 How (and whether) the dataset should be added to the datastore. 

734 See `ingest` for details of transfer modes. 

735 record_validation_info : `bool`, optional 

736 If `True`, the default, the datastore can record validation 

737 information associated with the file. If `False` the datastore 

738 will not attempt to track any information such as checksums 

739 or file sizes. This can be useful if such information is tracked 

740 in an external system or if the file is to be compressed in place. 

741 It is up to the datastore whether this parameter is relevant. 

742 

743 Raises 

744 ------ 

745 FileNotFoundError 

746 Raised if one of the given files does not exist. 

747 FileExistsError 

748 Raised if transfer is not `None` but the (internal) location the 

749 file would be moved to is already occupied. 

750 

751 Notes 

752 ----- 

753 This method (along with `_prepIngest`) should be implemented by 

754 subclasses to provide ingest support instead of implementing `ingest` 

755 directly. 

756 """ 

757 raise NotImplementedError(f"Datastore {self} does not support direct file-based ingest.") 

758 

759 def ingest( 

760 self, *datasets: FileDataset, transfer: str | None = None, record_validation_info: bool = True 

761 ) -> None: 

762 """Ingest one or more files into the datastore. 

763 

764 Parameters 

765 ---------- 

766 *datasets : `FileDataset` 

767 Each positional argument is a struct containing information about 

768 a file to be ingested, including its path (either absolute or 

769 relative to the datastore root, if applicable), a complete 

770 `DatasetRef` (with ``dataset_id not None``), and optionally a 

771 formatter class or its fully-qualified string name. If a formatter 

772 is not provided, the one the datastore would use for ``put`` on 

773 that dataset is assumed. 

774 transfer : `str`, optional 

775 How (and whether) the dataset should be added to the datastore. 

776 If `None` (default), the file must already be in a location 

777 appropriate for the datastore (e.g. within its root directory), 

778 and will not be modified. Other choices include "move", "copy", 

779 "link", "symlink", "relsymlink", and "hardlink". "link" is a 

780 special transfer mode that will first try to make a hardlink and 

781 if that fails a symlink will be used instead. "relsymlink" creates 

782 a relative symlink rather than use an absolute path. 

783 Most datastores do not support all transfer modes. 

784 "auto" is a special option that will let the 

785 data store choose the most natural option for itself. 

786 record_validation_info : `bool`, optional 

787 If `True`, the default, the datastore can record validation 

788 information associated with the file. If `False` the datastore 

789 will not attempt to track any information such as checksums 

790 or file sizes. This can be useful if such information is tracked 

791 in an external system or if the file is to be compressed in place. 

792 It is up to the datastore whether this parameter is relevant. 

793 

794 Raises 

795 ------ 

796 NotImplementedError 

797 Raised if the datastore does not support the given transfer mode 

798 (including the case where ingest is not supported at all). 

799 DatasetTypeNotSupportedError 

800 Raised if one or more files to be ingested have a dataset type that 

801 is not supported by the datastore. 

802 FileNotFoundError 

803 Raised if one of the given files does not exist. 

804 FileExistsError 

805 Raised if transfer is not `None` but the (internal) location the 

806 file would be moved to is already occupied. 

807 

808 Notes 

809 ----- 

810 Subclasses should implement `_prepIngest` and `_finishIngest` instead 

811 of implementing `ingest` directly. Datastores that hold and 

812 delegate to child datastores may want to call those methods as well. 

813 

814 Subclasses are encouraged to document their supported transfer modes 

815 in their class documentation. 

816 """ 

817 # Allow a datastore to select a default transfer mode 

818 transfer = self._overrideTransferMode(*datasets, transfer=transfer) 

819 prepData = self._prepIngest(*datasets, transfer=transfer) 

820 refs = {ref.id: ref for dataset in datasets for ref in dataset.refs} 

821 if refs.keys() != prepData.refs.keys(): 

822 unsupported = refs.keys() - prepData.refs.keys() 

823 # Group unsupported refs by DatasetType for an informative 

824 # but still concise error message. 

825 byDatasetType = defaultdict(list) 

826 for datasetId in unsupported: 

827 ref = refs[datasetId] 

828 byDatasetType[ref.datasetType].append(ref) 

829 raise DatasetTypeNotSupportedError( 

830 "DatasetType(s) not supported in ingest: " 

831 + ", ".join(f"{k.name} ({len(v)} dataset(s))" for k, v in byDatasetType.items()) 

832 ) 

833 self._finishIngest(prepData, transfer=transfer, record_validation_info=record_validation_info) 

834 

835 def transfer_from( 

836 self, 

837 source_datastore: Datastore, 

838 refs: Iterable[DatasetRef], 

839 transfer: str = "auto", 

840 artifact_existence: dict[ResourcePath, bool] | None = None, 

841 dry_run: bool = False, 

842 ) -> tuple[set[DatasetRef], set[DatasetRef]]: 

843 """Transfer dataset artifacts from another datastore to this one. 

844 

845 Parameters 

846 ---------- 

847 source_datastore : `Datastore` 

848 The datastore from which to transfer artifacts. That datastore 

849 must be compatible with this datastore receiving the artifacts. 

850 refs : iterable of `DatasetRef` 

851 The datasets to transfer from the source datastore. 

852 transfer : `str`, optional 

853 How (and whether) the dataset should be added to the datastore. 

854 Choices include "move", "copy", 

855 "link", "symlink", "relsymlink", and "hardlink". "link" is a 

856 special transfer mode that will first try to make a hardlink and 

857 if that fails a symlink will be used instead. "relsymlink" creates 

858 a relative symlink rather than use an absolute path. 

859 Most datastores do not support all transfer modes. 

860 "auto" (the default) is a special option that will let the 

861 data store choose the most natural option for itself. 

862 If the source location and transfer location are identical the 

863 transfer mode will be ignored. 

864 artifact_existence : `dict` [`lsst.resources.ResourcePath`, `bool`] 

865 Optional mapping of datastore artifact to existence. Updated by 

866 this method with details of all artifacts tested. Can be `None` 

867 if the caller is not interested. 

868 dry_run : `bool`, optional 

869 Process the supplied source refs without updating the target 

870 datastore. 

871 

872 Returns 

873 ------- 

874 accepted : `set` [`DatasetRef`] 

875 The datasets that were transferred. 

876 rejected : `set` [`DatasetRef`] 

877 The datasets that were rejected due to a constraints violation. 

878 

879 Raises 

880 ------ 

881 TypeError 

882 Raised if the two datastores are not compatible. 

883 """ 

884 if type(self) is not type(source_datastore): 

885 raise TypeError( 

886 f"Datastore mismatch between this datastore ({type(self)}) and the " 

887 f"source datastore ({type(source_datastore)})." 

888 ) 

889 

890 raise NotImplementedError(f"Datastore {type(self)} must implement a transfer_from method.") 

891 

892 def getManyURIs( 

893 self, 

894 refs: Iterable[DatasetRef], 

895 predict: bool = False, 

896 allow_missing: bool = False, 

897 ) -> dict[DatasetRef, DatasetRefURIs]: 

898 """Return URIs associated with many datasets. 

899 

900 Parameters 

901 ---------- 

902 refs : iterable of `DatasetIdRef` 

903 References to the required datasets. 

904 predict : `bool`, optional 

905 If `True`, allow URIs to be returned of datasets that have not 

906 been written. 

907 allow_missing : `bool` 

908 If `False`, and ``predict`` is `False`, will raise if a 

909 `DatasetRef` does not exist. 

910 

911 Returns 

912 ------- 

913 URIs : `dict` of [`DatasetRef`, `DatasetRefUris`] 

914 A dict of primary and component URIs, indexed by the passed-in 

915 refs. 

916 

917 Raises 

918 ------ 

919 FileNotFoundError 

920 A URI has been requested for a dataset that does not exist and 

921 guessing is not allowed. 

922 

923 Notes 

924 ----- 

925 In file-based datastores, getManyURIs does not check that the file is 

926 really there, it's assuming it is if datastore is aware of the file 

927 then it actually exists. 

928 """ 

929 uris: dict[DatasetRef, DatasetRefURIs] = {} 

930 missing_refs = [] 

931 for ref in refs: 

932 try: 

933 uris[ref] = self.getURIs(ref, predict=predict) 

934 except FileNotFoundError: 

935 missing_refs.append(ref) 

936 if missing_refs and not allow_missing: 

937 raise FileNotFoundError( 

938 "Missing {} refs from datastore out of {} and predict=False.".format( 

939 num_missing := len(missing_refs), num_missing + len(uris) 

940 ) 

941 ) 

942 return uris 

943 

944 @abstractmethod 

945 def getURIs(self, datasetRef: DatasetRef, predict: bool = False) -> DatasetRefURIs: 

946 """Return URIs associated with dataset. 

947 

948 Parameters 

949 ---------- 

950 datasetRef : `DatasetRef` 

951 Reference to the required dataset. 

952 predict : `bool`, optional 

953 If the datastore does not know about the dataset, controls whether 

954 it should return a predicted URI or not. 

955 

956 Returns 

957 ------- 

958 uris : `DatasetRefURIs` 

959 The URI to the primary artifact associated with this dataset (if 

960 the dataset was disassembled within the datastore this may be 

961 `None`), and the URIs to any components associated with the dataset 

962 artifact. (can be empty if there are no components). 

963 """ 

964 raise NotImplementedError() 

965 

966 @abstractmethod 

967 def getURI(self, datasetRef: DatasetRef, predict: bool = False) -> ResourcePath: 

968 """URI to the Dataset. 

969 

970 Parameters 

971 ---------- 

972 datasetRef : `DatasetRef` 

973 Reference to the required Dataset. 

974 predict : `bool` 

975 If `True` attempt to predict the URI for a dataset if it does 

976 not exist in datastore. 

977 

978 Returns 

979 ------- 

980 uri : `str` 

981 URI string pointing to the Dataset within the datastore. If the 

982 Dataset does not exist in the datastore, the URI may be a guess. 

983 If the datastore does not have entities that relate well 

984 to the concept of a URI the returned URI string will be 

985 descriptive. The returned URI is not guaranteed to be obtainable. 

986 

987 Raises 

988 ------ 

989 FileNotFoundError 

990 A URI has been requested for a dataset that does not exist and 

991 guessing is not allowed. 

992 """ 

993 raise NotImplementedError("Must be implemented by subclass") 

994 

995 @abstractmethod 

996 def retrieveArtifacts( 

997 self, 

998 refs: Iterable[DatasetRef], 

999 destination: ResourcePath, 

1000 transfer: str = "auto", 

1001 preserve_path: bool = True, 

1002 overwrite: bool = False, 

1003 ) -> list[ResourcePath]: 

1004 """Retrieve the artifacts associated with the supplied refs. 

1005 

1006 Parameters 

1007 ---------- 

1008 refs : iterable of `DatasetRef` 

1009 The datasets for which artifacts are to be retrieved. 

1010 A single ref can result in multiple artifacts. The refs must 

1011 be resolved. 

1012 destination : `lsst.resources.ResourcePath` 

1013 Location to write the artifacts. 

1014 transfer : `str`, optional 

1015 Method to use to transfer the artifacts. Must be one of the options 

1016 supported by `lsst.resources.ResourcePath.transfer_from()`. 

1017 "move" is not allowed. 

1018 preserve_path : `bool`, optional 

1019 If `True` the full path of the artifact within the datastore 

1020 is preserved. If `False` the final file component of the path 

1021 is used. 

1022 overwrite : `bool`, optional 

1023 If `True` allow transfers to overwrite existing files at the 

1024 destination. 

1025 

1026 Returns 

1027 ------- 

1028 targets : `list` of `lsst.resources.ResourcePath` 

1029 URIs of file artifacts in destination location. Order is not 

1030 preserved. 

1031 

1032 Notes 

1033 ----- 

1034 For non-file datastores the artifacts written to the destination 

1035 may not match the representation inside the datastore. For example 

1036 a hierarchichal data structure in a NoSQL database may well be stored 

1037 as a JSON file. 

1038 """ 

1039 raise NotImplementedError() 

1040 

1041 @abstractmethod 

1042 def remove(self, datasetRef: DatasetRef) -> None: 

1043 """Indicate to the Datastore that a Dataset can be removed. 

1044 

1045 Parameters 

1046 ---------- 

1047 datasetRef : `DatasetRef` 

1048 Reference to the required Dataset. 

1049 

1050 Raises 

1051 ------ 

1052 FileNotFoundError 

1053 When Dataset does not exist. 

1054 

1055 Notes 

1056 ----- 

1057 Some Datastores may implement this method as a silent no-op to 

1058 disable Dataset deletion through standard interfaces. 

1059 """ 

1060 raise NotImplementedError("Must be implemented by subclass") 

1061 

1062 @abstractmethod 

1063 def forget(self, refs: Iterable[DatasetRef]) -> None: 

1064 """Indicate to the Datastore that it should remove all records of the 

1065 given datasets, without actually deleting them. 

1066 

1067 Parameters 

1068 ---------- 

1069 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

1070 References to the datasets being forgotten. 

1071 

1072 Notes 

1073 ----- 

1074 Asking a datastore to forget a `DatasetRef` it does not hold should be 

1075 a silent no-op, not an error. 

1076 """ 

1077 raise NotImplementedError("Must be implemented by subclass") 

1078 

1079 @abstractmethod 

1080 def trash(self, ref: DatasetRef | Iterable[DatasetRef], ignore_errors: bool = True) -> None: 

1081 """Indicate to the Datastore that a Dataset can be moved to the trash. 

1082 

1083 Parameters 

1084 ---------- 

1085 ref : `DatasetRef` or iterable thereof 

1086 Reference(s) to the required Dataset. 

1087 ignore_errors : `bool`, optional 

1088 Determine whether errors should be ignored. When multiple 

1089 refs are being trashed there will be no per-ref check. 

1090 

1091 Raises 

1092 ------ 

1093 FileNotFoundError 

1094 When Dataset does not exist and errors are not ignored. Only 

1095 checked if a single ref is supplied (and not in a list). 

1096 

1097 Notes 

1098 ----- 

1099 Some Datastores may implement this method as a silent no-op to 

1100 disable Dataset deletion through standard interfaces. 

1101 """ 

1102 raise NotImplementedError("Must be implemented by subclass") 

1103 

1104 @abstractmethod 

1105 def emptyTrash(self, ignore_errors: bool = True) -> None: 

1106 """Remove all datasets from the trash. 

1107 

1108 Parameters 

1109 ---------- 

1110 ignore_errors : `bool`, optional 

1111 Determine whether errors should be ignored. 

1112 

1113 Notes 

1114 ----- 

1115 Some Datastores may implement this method as a silent no-op to 

1116 disable Dataset deletion through standard interfaces. 

1117 """ 

1118 raise NotImplementedError("Must be implemented by subclass") 

1119 

1120 @abstractmethod 

1121 def transfer(self, inputDatastore: Datastore, datasetRef: DatasetRef) -> None: 

1122 """Transfer a dataset from another datastore to this datastore. 

1123 

1124 Parameters 

1125 ---------- 

1126 inputDatastore : `Datastore` 

1127 The external `Datastore` from which to retrieve the Dataset. 

1128 datasetRef : `DatasetRef` 

1129 Reference to the required Dataset. 

1130 """ 

1131 raise NotImplementedError("Must be implemented by subclass") 

1132 

1133 def export( 

1134 self, 

1135 refs: Iterable[DatasetRef], 

1136 *, 

1137 directory: ResourcePathExpression | None = None, 

1138 transfer: str | None = "auto", 

1139 ) -> Iterable[FileDataset]: 

1140 """Export datasets for transfer to another data repository. 

1141 

1142 Parameters 

1143 ---------- 

1144 refs : iterable of `DatasetRef` 

1145 Dataset references to be exported. 

1146 directory : `str`, optional 

1147 Path to a directory that should contain files corresponding to 

1148 output datasets. Ignored if ``transfer`` is explicitly `None`. 

1149 transfer : `str`, optional 

1150 Mode that should be used to move datasets out of the repository. 

1151 Valid options are the same as those of the ``transfer`` argument 

1152 to ``ingest``, and datastores may similarly signal that a transfer 

1153 mode is not supported by raising `NotImplementedError`. If "auto" 

1154 is given and no ``directory`` is specified, `None` will be 

1155 implied. 

1156 

1157 Returns 

1158 ------- 

1159 dataset : iterable of `DatasetTransfer` 

1160 Structs containing information about the exported datasets, in the 

1161 same order as ``refs``. 

1162 

1163 Raises 

1164 ------ 

1165 NotImplementedError 

1166 Raised if the given transfer mode is not supported. 

1167 """ 

1168 raise NotImplementedError(f"Transfer mode {transfer} not supported.") 

1169 

1170 @abstractmethod 

1171 def validateConfiguration( 

1172 self, entities: Iterable[DatasetRef | DatasetType | StorageClass], logFailures: bool = False 

1173 ) -> None: 

1174 """Validate some of the configuration for this datastore. 

1175 

1176 Parameters 

1177 ---------- 

1178 entities : iterable of `DatasetRef`, `DatasetType`, or `StorageClass` 

1179 Entities to test against this configuration. Can be differing 

1180 types. 

1181 logFailures : `bool`, optional 

1182 If `True`, output a log message for every validation error 

1183 detected. 

1184 

1185 Raises 

1186 ------ 

1187 DatastoreValidationError 

1188 Raised if there is a validation problem with a configuration. 

1189 

1190 Notes 

1191 ----- 

1192 Which parts of the configuration are validated is at the discretion 

1193 of each Datastore implementation. 

1194 """ 

1195 raise NotImplementedError("Must be implemented by subclass") 

1196 

1197 @abstractmethod 

1198 def validateKey(self, lookupKey: LookupKey, entity: DatasetRef | DatasetType | StorageClass) -> None: 

1199 """Validate a specific look up key with supplied entity. 

1200 

1201 Parameters 

1202 ---------- 

1203 lookupKey : `LookupKey` 

1204 Key to use to retrieve information from the datastore 

1205 configuration. 

1206 entity : `DatasetRef`, `DatasetType`, or `StorageClass` 

1207 Entity to compare with configuration retrieved using the 

1208 specified lookup key. 

1209 

1210 Raises 

1211 ------ 

1212 DatastoreValidationError 

1213 Raised if there is a problem with the combination of entity 

1214 and lookup key. 

1215 

1216 Notes 

1217 ----- 

1218 Bypasses the normal selection priorities by allowing a key that 

1219 would normally not be selected to be validated. 

1220 """ 

1221 raise NotImplementedError("Must be implemented by subclass") 

1222 

1223 @abstractmethod 

1224 def getLookupKeys(self) -> set[LookupKey]: 

1225 """Return all the lookup keys relevant to this datastore. 

1226 

1227 Returns 

1228 ------- 

1229 keys : `set` of `LookupKey` 

1230 The keys stored internally for looking up information based 

1231 on `DatasetType` name or `StorageClass`. 

1232 """ 

1233 raise NotImplementedError("Must be implemented by subclass") 

1234 

1235 def needs_expanded_data_ids( 

1236 self, 

1237 transfer: str | None, 

1238 entity: DatasetRef | DatasetType | StorageClass | None = None, 

1239 ) -> bool: 

1240 """Test whether this datastore needs expanded data IDs to ingest. 

1241 

1242 Parameters 

1243 ---------- 

1244 transfer : `str` or `None` 

1245 Transfer mode for ingest. 

1246 entity : `DatasetRef` or `DatasetType` or `StorageClass` or `None`, \ 

1247 optional 

1248 Object representing what will be ingested. If not provided (or not 

1249 specific enough), `True` may be returned even if expanded data 

1250 IDs aren't necessary. 

1251 

1252 Returns 

1253 ------- 

1254 needed : `bool` 

1255 If `True`, expanded data IDs may be needed. `False` only if 

1256 expansion definitely isn't necessary. 

1257 """ 

1258 return True 

1259 

1260 @abstractmethod 

1261 def import_records( 

1262 self, 

1263 data: Mapping[str, DatastoreRecordData], 

1264 ) -> None: 

1265 """Import datastore location and record data from an in-memory data 

1266 structure. 

1267 

1268 Parameters 

1269 ---------- 

1270 data : `~collections.abc.Mapping` [ `str`, `DatastoreRecordData` ] 

1271 Datastore records indexed by datastore name. May contain data for 

1272 other `Datastore` instances (generally because they are chained to 

1273 this one), which should be ignored. 

1274 

1275 Notes 

1276 ----- 

1277 Implementations should generally not check that any external resources 

1278 (e.g. files) referred to by these records actually exist, for 

1279 performance reasons; we expect higher-level code to guarantee that they 

1280 do. 

1281 

1282 Implementations are responsible for calling 

1283 `DatastoreRegistryBridge.insert` on all datasets in ``data.locations`` 

1284 where the key is in `names`, as well as loading any opaque table data. 

1285 

1286 Implementations may assume that datasets are either fully present or 

1287 not at all (single-component exports are not permitted). 

1288 """ 

1289 raise NotImplementedError() 

1290 

1291 @abstractmethod 

1292 def export_records( 

1293 self, 

1294 refs: Iterable[DatasetIdRef], 

1295 ) -> Mapping[str, DatastoreRecordData]: 

1296 """Export datastore records and locations to an in-memory data 

1297 structure. 

1298 

1299 Parameters 

1300 ---------- 

1301 refs : `~collections.abc.Iterable` [ `DatasetIdRef` ] 

1302 Datasets to save. This may include datasets not known to this 

1303 datastore, which should be ignored. May not include component 

1304 datasets. 

1305 

1306 Returns 

1307 ------- 

1308 data : `~collections.abc.Mapping` [ `str`, `DatastoreRecordData` ] 

1309 Exported datastore records indexed by datastore name. 

1310 """ 

1311 raise NotImplementedError() 

1312 

1313 def set_retrieve_dataset_type_method(self, method: Callable[[str], DatasetType | None] | None) -> None: 

1314 """Specify a method that can be used by datastore to retrieve 

1315 registry-defined dataset type. 

1316 

1317 Parameters 

1318 ---------- 

1319 method : `~collections.abc.Callable` | `None` 

1320 Method that takes a name of the dataset type and returns a 

1321 corresponding `DatasetType` instance as defined in Registry. If 

1322 dataset type name is not known to registry `None` is returned. 

1323 

1324 Notes 

1325 ----- 

1326 This method is only needed for a Datastore supporting a "trusted" mode 

1327 when it does not have an access to datastore records and needs to 

1328 guess dataset location based on its stored dataset type. 

1329 """ 

1330 pass 

1331 

1332 @abstractmethod 

1333 def get_opaque_table_definitions(self) -> Mapping[str, DatastoreOpaqueTable]: 

1334 """Make definitions of the opaque tables used by this Datastore. 

1335 

1336 Returns 

1337 ------- 

1338 tables : `~collections.abc.Mapping` [ `str`, `.ddl.TableSpec` ] 

1339 Mapping of opaque table names to their definitions. This can be an 

1340 empty mapping if Datastore does not use opaque tables to keep 

1341 datastore records. 

1342 """ 

1343 raise NotImplementedError() 

1344 

1345 

1346class NullDatastore(Datastore): 

1347 """A datastore that implements the `Datastore` API but always fails when 

1348 it accepts any request. 

1349 

1350 Parameters 

1351 ---------- 

1352 config : `Config` or `~lsst.resources.ResourcePathExpression` or `None` 

1353 Ignored. 

1354 bridgeManager : `DatastoreRegistryBridgeManager` or `None` 

1355 Ignored. 

1356 butlerRoot : `~lsst.resources.ResourcePathExpression` or `None` 

1357 Ignored. 

1358 """ 

1359 

1360 @classmethod 

1361 def _create_from_config( 

1362 cls, 

1363 config: Config, 

1364 bridgeManager: DatastoreRegistryBridgeManager, 

1365 butlerRoot: ResourcePathExpression | None = None, 

1366 ) -> NullDatastore: 

1367 return NullDatastore(config, bridgeManager, butlerRoot) 

1368 

1369 def clone(self, bridgeManager: DatastoreRegistryBridgeManager) -> Datastore: 

1370 return self 

1371 

1372 @classmethod 

1373 def setConfigRoot(cls, root: str, config: Config, full: Config, overwrite: bool = True) -> None: 

1374 # Nothing to do. This is not a real Datastore. 

1375 pass 

1376 

1377 def __init__( 

1378 self, 

1379 config: Config | ResourcePathExpression | None, 

1380 bridgeManager: DatastoreRegistryBridgeManager | None, 

1381 butlerRoot: ResourcePathExpression | None = None, 

1382 ): 

1383 # Name ourselves with the timestamp the datastore 

1384 # was created. 

1385 self.name = f"{type(self).__name__}@{time.time()}" 

1386 _LOG.debug("Creating datastore %s", self.name) 

1387 

1388 return 

1389 

1390 def knows(self, ref: DatasetRef) -> bool: 

1391 return False 

1392 

1393 def exists(self, datasetRef: DatasetRef) -> bool: 

1394 return False 

1395 

1396 def get( 

1397 self, 

1398 datasetRef: DatasetRef, 

1399 parameters: Mapping[str, Any] | None = None, 

1400 storageClass: StorageClass | str | None = None, 

1401 ) -> Any: 

1402 raise FileNotFoundError("This is a no-op datastore that can not access a real datastore") 

1403 

1404 def put(self, inMemoryDataset: Any, datasetRef: DatasetRef) -> None: 

1405 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1406 

1407 def put_new(self, in_memory_dataset: Any, ref: DatasetRef) -> Mapping[str, DatasetRef]: 

1408 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1409 

1410 def ingest( 

1411 self, *datasets: FileDataset, transfer: str | None = None, record_validation_info: bool = True 

1412 ) -> None: 

1413 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1414 

1415 def transfer_from( 

1416 self, 

1417 source_datastore: Datastore, 

1418 refs: Iterable[DatasetRef], 

1419 transfer: str = "auto", 

1420 artifact_existence: dict[ResourcePath, bool] | None = None, 

1421 dry_run: bool = False, 

1422 ) -> tuple[set[DatasetRef], set[DatasetRef]]: 

1423 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1424 

1425 def getURIs(self, datasetRef: DatasetRef, predict: bool = False) -> DatasetRefURIs: 

1426 raise FileNotFoundError("This is a no-op datastore that can not access a real datastore") 

1427 

1428 def getURI(self, datasetRef: DatasetRef, predict: bool = False) -> ResourcePath: 

1429 raise FileNotFoundError("This is a no-op datastore that can not access a real datastore") 

1430 

1431 def retrieveArtifacts( 

1432 self, 

1433 refs: Iterable[DatasetRef], 

1434 destination: ResourcePath, 

1435 transfer: str = "auto", 

1436 preserve_path: bool = True, 

1437 overwrite: bool = False, 

1438 ) -> list[ResourcePath]: 

1439 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1440 

1441 def remove(self, datasetRef: DatasetRef) -> None: 

1442 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1443 

1444 def forget(self, refs: Iterable[DatasetRef]) -> None: 

1445 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1446 

1447 def trash(self, ref: DatasetRef | Iterable[DatasetRef], ignore_errors: bool = True) -> None: 

1448 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1449 

1450 def emptyTrash(self, ignore_errors: bool = True) -> None: 

1451 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1452 

1453 def transfer(self, inputDatastore: Datastore, datasetRef: DatasetRef) -> None: 

1454 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1455 

1456 def export( 

1457 self, 

1458 refs: Iterable[DatasetRef], 

1459 *, 

1460 directory: ResourcePathExpression | None = None, 

1461 transfer: str | None = "auto", 

1462 ) -> Iterable[FileDataset]: 

1463 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1464 

1465 def validateConfiguration( 

1466 self, entities: Iterable[DatasetRef | DatasetType | StorageClass], logFailures: bool = False 

1467 ) -> None: 

1468 # No configuration so always validates. 

1469 pass 

1470 

1471 def validateKey(self, lookupKey: LookupKey, entity: DatasetRef | DatasetType | StorageClass) -> None: 

1472 pass 

1473 

1474 def getLookupKeys(self) -> set[LookupKey]: 

1475 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1476 

1477 def import_records( 

1478 self, 

1479 data: Mapping[str, DatastoreRecordData], 

1480 ) -> None: 

1481 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1482 

1483 def export_records( 

1484 self, 

1485 refs: Iterable[DatasetIdRef], 

1486 ) -> Mapping[str, DatastoreRecordData]: 

1487 raise NotImplementedError("This is a no-op datastore that can not access a real datastore") 

1488 

1489 def get_opaque_table_definitions(self) -> Mapping[str, DatastoreOpaqueTable]: 

1490 return {}