Coverage for python/lsst/daf/butler/datastores/inMemoryDatastore.py: 88%

193 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-11-17 02:01 -0800

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24"""In-memory datastore.""" 

25 

26__all__ = ("StoredMemoryItemInfo", "InMemoryDatastore") 

27 

28import logging 

29import time 

30from dataclasses import dataclass 

31from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Mapping, Optional, Set, Tuple, Union 

32from urllib.parse import urlencode 

33 

34from lsst.daf.butler import ( 

35 DatasetId, 

36 DatasetRef, 

37 DatasetRefURIs, 

38 DatastoreRecordData, 

39 StorageClass, 

40 StoredDatastoreItemInfo, 

41) 

42from lsst.daf.butler.core.utils import transactional 

43from lsst.daf.butler.registry.interfaces import DatastoreRegistryBridge 

44from lsst.resources import ResourcePath 

45 

46from .genericDatastore import GenericBaseDatastore 

47 

48if TYPE_CHECKING: 48 ↛ 49line 48 didn't jump to line 49, because the condition on line 48 was never true

49 from lsst.daf.butler import Config, DatasetType, LookupKey 

50 from lsst.daf.butler.registry.interfaces import DatasetIdRef, DatastoreRegistryBridgeManager 

51 

52log = logging.getLogger(__name__) 

53 

54 

55@dataclass(frozen=True) 

56class StoredMemoryItemInfo(StoredDatastoreItemInfo): 

57 """Internal InMemoryDatastore Metadata associated with a stored 

58 DatasetRef. 

59 """ 

60 

61 __slots__ = {"timestamp", "storageClass", "parentID", "dataset_id"} 

62 

63 timestamp: float 

64 """Unix timestamp indicating the time the dataset was stored.""" 

65 

66 storageClass: StorageClass 

67 """StorageClass associated with the dataset.""" 

68 

69 parentID: DatasetId 

70 """ID of the parent `DatasetRef` if this entry is a concrete 

71 composite. Not used if the dataset being stored is not a 

72 virtual component of a composite 

73 """ 

74 

75 dataset_id: DatasetId 

76 """DatasetId associated with this record.""" 

77 

78 

79class InMemoryDatastore(GenericBaseDatastore): 

80 """Basic Datastore for writing to an in memory cache. 

81 

82 This datastore is ephemeral in that the contents of the datastore 

83 disappear when the Python process completes. This also means that 

84 other processes can not access this datastore. 

85 

86 Parameters 

87 ---------- 

88 config : `DatastoreConfig` or `str` 

89 Configuration. 

90 bridgeManager : `DatastoreRegistryBridgeManager` 

91 Object that manages the interface between `Registry` and datastores. 

92 butlerRoot : `str`, optional 

93 Unused parameter. 

94 

95 Notes 

96 ----- 

97 InMemoryDatastore does not support any file-based ingest. 

98 """ 

99 

100 defaultConfigFile = "datastores/inMemoryDatastore.yaml" 

101 """Path to configuration defaults. Accessed within the ``configs`` resource 

102 or relative to a search path. Can be None if no defaults specified. 

103 """ 

104 

105 isEphemeral = True 

106 """A new datastore is created every time and datasets disappear when 

107 the process shuts down.""" 

108 

109 datasets: Dict[DatasetId, Any] 

110 """Internal storage of datasets indexed by dataset ID.""" 

111 

112 records: Dict[DatasetId, StoredMemoryItemInfo] 

113 """Internal records about stored datasets.""" 

114 

115 def __init__( 

116 self, 

117 config: Union[Config, str], 

118 bridgeManager: DatastoreRegistryBridgeManager, 

119 butlerRoot: Optional[str] = None, 

120 ): 

121 super().__init__(config, bridgeManager) 

122 

123 # Name ourselves with the timestamp the datastore 

124 # was created. 

125 self.name = "{}@{}".format(type(self).__name__, time.time()) 

126 log.debug("Creating datastore %s", self.name) 

127 

128 # Storage of datasets, keyed by dataset_id 

129 self.datasets: Dict[DatasetId, Any] = {} 

130 

131 # Records is distinct in order to track concrete composite components 

132 # where we register multiple components for a single dataset. 

133 self.records: Dict[DatasetId, StoredMemoryItemInfo] = {} 

134 

135 # Related records that share the same parent 

136 self.related: Dict[DatasetId, Set[DatasetId]] = {} 

137 

138 self._bridge = bridgeManager.register(self.name, ephemeral=True) 

139 

140 @classmethod 

141 def setConfigRoot(cls, root: str, config: Config, full: Config, overwrite: bool = True) -> None: 

142 """Set any filesystem-dependent config options for this Datastore to 

143 be appropriate for a new empty repository with the given root. 

144 

145 Does nothing in this implementation. 

146 

147 Parameters 

148 ---------- 

149 root : `str` 

150 Filesystem path to the root of the data repository. 

151 config : `Config` 

152 A `Config` to update. Only the subset understood by 

153 this component will be updated. Will not expand 

154 defaults. 

155 full : `Config` 

156 A complete config with all defaults expanded that can be 

157 converted to a `DatastoreConfig`. Read-only and will not be 

158 modified by this method. 

159 Repository-specific options that should not be obtained 

160 from defaults when Butler instances are constructed 

161 should be copied from ``full`` to ``config``. 

162 overwrite : `bool`, optional 

163 If `False`, do not modify a value in ``config`` if the value 

164 already exists. Default is always to overwrite with the provided 

165 ``root``. 

166 

167 Notes 

168 ----- 

169 If a keyword is explicitly defined in the supplied ``config`` it 

170 will not be overridden by this method if ``overwrite`` is `False`. 

171 This allows explicit values set in external configs to be retained. 

172 """ 

173 return 

174 

175 @property 

176 def bridge(self) -> DatastoreRegistryBridge: 

177 # Docstring inherited from GenericBaseDatastore. 

178 return self._bridge 

179 

180 def addStoredItemInfo(self, refs: Iterable[DatasetRef], infos: Iterable[StoredMemoryItemInfo]) -> None: 

181 # Docstring inherited from GenericBaseDatastore. 

182 for ref, info in zip(refs, infos): 

183 if ref.id is None: 183 ↛ 184line 183 didn't jump to line 184, because the condition on line 183 was never true

184 raise RuntimeError(f"Can not store unresolved DatasetRef {ref}") 

185 self.records[ref.id] = info 

186 self.related.setdefault(info.parentID, set()).add(ref.id) 

187 

188 def getStoredItemInfo(self, ref: DatasetIdRef) -> StoredMemoryItemInfo: 

189 # Docstring inherited from GenericBaseDatastore. 

190 if ref.id is None: 190 ↛ 191line 190 didn't jump to line 191, because the condition on line 190 was never true

191 raise RuntimeError(f"Can not retrieve unresolved DatasetRef {ref}") 

192 return self.records[ref.id] 

193 

194 def getStoredItemsInfo(self, ref: DatasetIdRef) -> List[StoredMemoryItemInfo]: 

195 # Docstring inherited from GenericBaseDatastore. 

196 return [self.getStoredItemInfo(ref)] 

197 

198 def removeStoredItemInfo(self, ref: DatasetIdRef) -> None: 

199 # Docstring inherited from GenericBaseDatastore. 

200 # If a component has been removed previously then we can sometimes 

201 # be asked to remove it again. Other datastores ignore this 

202 # so also ignore here 

203 if ref.id is None: 203 ↛ 204line 203 didn't jump to line 204, because the condition on line 203 was never true

204 raise RuntimeError(f"Can not remove unresolved DatasetRef {ref}") 

205 if ref.id not in self.records: 

206 return 

207 record = self.records[ref.id] 

208 del self.records[ref.id] 

209 self.related[record.parentID].remove(ref.id) 

210 

211 def _get_dataset_info(self, ref: DatasetIdRef) -> Tuple[DatasetId, StoredMemoryItemInfo]: 

212 """Check that the dataset is present and return the real ID and 

213 associated information. 

214 

215 Parameters 

216 ---------- 

217 ref : `DatasetRef` 

218 Target `DatasetRef` 

219 

220 Returns 

221 ------- 

222 realID : `int` 

223 The dataset ID associated with this ref that should be used. This 

224 could either be the ID of the supplied `DatasetRef` or the parent. 

225 storageInfo : `StoredMemoryItemInfo` 

226 Associated storage information. 

227 

228 Raises 

229 ------ 

230 FileNotFoundError 

231 Raised if the dataset is not present in this datastore. 

232 """ 

233 try: 

234 storedItemInfo = self.getStoredItemInfo(ref) 

235 except KeyError: 

236 raise FileNotFoundError(f"No such file dataset in memory: {ref}") from None 

237 realID = ref.id 

238 if storedItemInfo.parentID is not None: 238 ↛ 241line 238 didn't jump to line 241, because the condition on line 238 was never false

239 realID = storedItemInfo.parentID 

240 

241 if realID not in self.datasets: 241 ↛ 242line 241 didn't jump to line 242, because the condition on line 241 was never true

242 raise FileNotFoundError(f"No such file dataset in memory: {ref}") 

243 

244 return realID, storedItemInfo 

245 

246 def knows(self, ref: DatasetRef) -> bool: 

247 """Check if the dataset is known to the datastore. 

248 

249 This datastore does not distinguish dataset existence from knowledge 

250 of a dataset. 

251 

252 Parameters 

253 ---------- 

254 ref : `DatasetRef` 

255 Reference to the required dataset. 

256 

257 Returns 

258 ------- 

259 exists : `bool` 

260 `True` if the dataset is known to the datastore. 

261 """ 

262 return self.exists(ref) 

263 

264 def exists(self, ref: DatasetRef) -> bool: 

265 """Check if the dataset exists in the datastore. 

266 

267 Parameters 

268 ---------- 

269 ref : `DatasetRef` 

270 Reference to the required dataset. 

271 

272 Returns 

273 ------- 

274 exists : `bool` 

275 `True` if the entity exists in the `Datastore`. 

276 """ 

277 try: 

278 self._get_dataset_info(ref) 

279 except FileNotFoundError: 

280 return False 

281 return True 

282 

283 def get( 

284 self, 

285 ref: DatasetRef, 

286 parameters: Optional[Mapping[str, Any]] = None, 

287 storageClass: Optional[Union[StorageClass, str]] = None, 

288 ) -> Any: 

289 """Load an InMemoryDataset from the store. 

290 

291 Parameters 

292 ---------- 

293 ref : `DatasetRef` 

294 Reference to the required Dataset. 

295 parameters : `dict` 

296 `StorageClass`-specific parameters that specify, for example, 

297 a slice of the dataset to be loaded. 

298 storageClass : `StorageClass` or `str`, optional 

299 The storage class to be used to override the Python type 

300 returned by this method. By default the returned type matches 

301 the dataset type definition for this dataset. Specifying a 

302 read `StorageClass` can force a different type to be returned. 

303 This type must be compatible with the original type. 

304 

305 Returns 

306 ------- 

307 inMemoryDataset : `object` 

308 Requested dataset or slice thereof as an InMemoryDataset. 

309 

310 Raises 

311 ------ 

312 FileNotFoundError 

313 Requested dataset can not be retrieved. 

314 TypeError 

315 Return value from formatter has unexpected type. 

316 ValueError 

317 Formatter failed to process the dataset. 

318 """ 

319 

320 log.debug("Retrieve %s from %s with parameters %s", ref, self.name, parameters) 

321 

322 realID, storedItemInfo = self._get_dataset_info(ref) 

323 

324 # We have a write storage class and a read storage class and they 

325 # can be different for concrete composites or if overridden. 

326 if storageClass is not None: 

327 ref = ref.overrideStorageClass(storageClass) 

328 refStorageClass = ref.datasetType.storageClass 

329 writeStorageClass = storedItemInfo.storageClass 

330 

331 component = ref.datasetType.component() 

332 

333 # Check that the supplied parameters are suitable for the type read 

334 # If this is a derived component we validate against the composite 

335 isDerivedComponent = False 

336 if component in writeStorageClass.derivedComponents: 

337 writeStorageClass.validateParameters(parameters) 

338 isDerivedComponent = True 

339 else: 

340 refStorageClass.validateParameters(parameters) 

341 

342 inMemoryDataset = self.datasets[realID] 

343 

344 # if this is a read only component we need to apply parameters 

345 # before we retrieve the component. We assume that the parameters 

346 # will affect the data globally, before the derived component 

347 # is selected. 

348 if isDerivedComponent: 

349 inMemoryDataset = writeStorageClass.delegate().handleParameters(inMemoryDataset, parameters) 

350 # Then disable parameters for later 

351 parameters = {} 

352 

353 # Check if we have a component. 

354 if component: 

355 # In-memory datastore must have stored the dataset as a single 

356 # object in the write storage class. We therefore use that 

357 # storage class delegate to obtain the component. 

358 inMemoryDataset = writeStorageClass.delegate().getComponent(inMemoryDataset, component) 

359 

360 # Since there is no formatter to process parameters, they all must be 

361 # passed to the assembler. 

362 inMemoryDataset = self._post_process_get( 

363 inMemoryDataset, refStorageClass, parameters, isComponent=component is not None 

364 ) 

365 

366 # Last minute type conversion. 

367 return refStorageClass.coerce_type(inMemoryDataset) 

368 

369 def put(self, inMemoryDataset: Any, ref: DatasetRef) -> None: 

370 """Write a InMemoryDataset with a given `DatasetRef` to the store. 

371 

372 Parameters 

373 ---------- 

374 inMemoryDataset : `object` 

375 The dataset to store. 

376 ref : `DatasetRef` 

377 Reference to the associated Dataset. 

378 

379 Raises 

380 ------ 

381 TypeError 

382 Supplied object and storage class are inconsistent. 

383 DatasetTypeNotSupportedError 

384 The associated `DatasetType` is not handled by this datastore. 

385 

386 Notes 

387 ----- 

388 If the datastore is configured to reject certain dataset types it 

389 is possible that the put will fail and raise a 

390 `DatasetTypeNotSupportedError`. The main use case for this is to 

391 allow `ChainedDatastore` to put to multiple datastores without 

392 requiring that every datastore accepts the dataset. 

393 """ 

394 

395 if ref.id is None: 395 ↛ 396line 395 didn't jump to line 396, because the condition on line 395 was never true

396 raise RuntimeError(f"Can not store unresolved DatasetRef {ref}") 

397 

398 # May need to coerce the in memory dataset to the correct 

399 # python type, otherwise parameters may not work. 

400 inMemoryDataset = ref.datasetType.storageClass.coerce_type(inMemoryDataset) 

401 

402 self._validate_put_parameters(inMemoryDataset, ref) 

403 

404 self.datasets[ref.id] = inMemoryDataset 

405 log.debug("Store %s in %s", ref, self.name) 

406 

407 # Store time we received this content, to allow us to optionally 

408 # expire it. Instead of storing a filename here, we include the 

409 # ID of this datasetRef so we can find it from components. 

410 itemInfo = StoredMemoryItemInfo( 

411 time.time(), ref.datasetType.storageClass, parentID=ref.id, dataset_id=ref.getCheckedId() 

412 ) 

413 

414 # We have to register this content with registry. 

415 # Currently this assumes we have a file so we need to use stub entries 

416 # TODO: Add to ephemeral part of registry 

417 self._register_datasets([(ref, itemInfo)]) 

418 

419 if self._transaction is not None: 

420 self._transaction.registerUndo("put", self.remove, ref) 

421 

422 def getURIs(self, ref: DatasetRef, predict: bool = False) -> DatasetRefURIs: 

423 """Return URIs associated with dataset. 

424 

425 Parameters 

426 ---------- 

427 ref : `DatasetRef` 

428 Reference to the required dataset. 

429 predict : `bool`, optional 

430 If the datastore does not know about the dataset, should it 

431 return a predicted URI or not? 

432 

433 Returns 

434 ------- 

435 uris : `DatasetRefURIs` 

436 The URI to the primary artifact associated with this dataset (if 

437 the dataset was disassembled within the datastore this may be 

438 `None`), and the URIs to any components associated with the dataset 

439 artifact. (can be empty if there are no components). 

440 

441 Notes 

442 ----- 

443 The URIs returned for in-memory datastores are not usable but 

444 provide an indication of the associated dataset. 

445 """ 

446 

447 # Include the dataID as a URI query 

448 query = urlencode(ref.dataId) 

449 

450 # if this has never been written then we have to guess 

451 if not self.exists(ref): 

452 if not predict: 

453 raise FileNotFoundError("Dataset {} not in this datastore".format(ref)) 

454 name = f"{ref.datasetType.name}" 

455 fragment = "#predicted" 

456 else: 

457 realID, _ = self._get_dataset_info(ref) 

458 name = f"{id(self.datasets[realID])}?{query}" 

459 fragment = "" 

460 

461 return DatasetRefURIs(ResourcePath(f"mem://{name}?{query}{fragment}"), {}) 

462 

463 def getURI(self, ref: DatasetRef, predict: bool = False) -> ResourcePath: 

464 """URI to the Dataset. 

465 

466 Always uses "mem://" URI prefix. 

467 

468 Parameters 

469 ---------- 

470 ref : `DatasetRef` 

471 Reference to the required Dataset. 

472 predict : `bool` 

473 If `True`, allow URIs to be returned of datasets that have not 

474 been written. 

475 

476 Returns 

477 ------- 

478 uri : `str` 

479 URI pointing to the dataset within the datastore. If the 

480 dataset does not exist in the datastore, and if ``predict`` is 

481 `True`, the URI will be a prediction and will include a URI 

482 fragment "#predicted". 

483 If the datastore does not have entities that relate well 

484 to the concept of a URI the returned URI string will be 

485 descriptive. The returned URI is not guaranteed to be obtainable. 

486 

487 Raises 

488 ------ 

489 FileNotFoundError 

490 A URI has been requested for a dataset that does not exist and 

491 guessing is not allowed. 

492 AssertionError 

493 Raised if an internal error occurs. 

494 """ 

495 primary, _ = self.getURIs(ref, predict) 

496 if primary is None: 496 ↛ 499line 496 didn't jump to line 499, because the condition on line 496 was never true

497 # This should be impossible since this datastore does 

498 # not disassemble. This check also helps mypy. 

499 raise AssertionError(f"Unexpectedly got no URI for in-memory datastore for {ref}") 

500 return primary 

501 

502 def retrieveArtifacts( 

503 self, 

504 refs: Iterable[DatasetRef], 

505 destination: ResourcePath, 

506 transfer: str = "auto", 

507 preserve_path: bool = True, 

508 overwrite: Optional[bool] = False, 

509 ) -> List[ResourcePath]: 

510 """Retrieve the file artifacts associated with the supplied refs. 

511 

512 Notes 

513 ----- 

514 Not implemented by this datastore. 

515 """ 

516 # Could conceivably launch a FileDatastore to use formatters to write 

517 # the data but this is fraught with problems. 

518 raise NotImplementedError("Can not write artifacts to disk from in-memory datastore.") 

519 

520 def forget(self, refs: Iterable[DatasetRef]) -> None: 

521 # Docstring inherited. 

522 refs = list(refs) 

523 self._bridge.forget(refs) 

524 for ref in refs: 

525 self.removeStoredItemInfo(ref) 

526 

527 @transactional 

528 def trash(self, ref: Union[DatasetRef, Iterable[DatasetRef]], ignore_errors: bool = False) -> None: 

529 """Indicate to the Datastore that a dataset can be removed. 

530 

531 Parameters 

532 ---------- 

533 ref : `DatasetRef` or iterable thereof 

534 Reference to the required Dataset(s). 

535 ignore_errors: `bool`, optional 

536 Indicate that errors should be ignored. 

537 

538 Raises 

539 ------ 

540 FileNotFoundError 

541 Attempt to remove a dataset that does not exist. Only relevant 

542 if a single dataset ref is given. 

543 

544 Notes 

545 ----- 

546 Concurrency should not normally be an issue for the in memory datastore 

547 since all internal changes are isolated to solely this process and 

548 the registry only changes rows associated with this process. 

549 """ 

550 if not isinstance(ref, DatasetRef): 

551 log.debug("Bulk trashing of datasets in datastore %s", self.name) 

552 self.bridge.moveToTrash(ref, transaction=self._transaction) 

553 return 

554 

555 log.debug("Trash %s in datastore %s", ref, self.name) 

556 

557 # Check that this dataset is known to datastore 

558 try: 

559 self._get_dataset_info(ref) 

560 

561 # Move datasets to trash table 

562 self.bridge.moveToTrash([ref], transaction=self._transaction) 

563 except Exception as e: 

564 if ignore_errors: 564 ↛ 565line 564 didn't jump to line 565, because the condition on line 564 was never true

565 log.warning( 

566 "Error encountered moving dataset %s to trash in datastore %s: %s", ref, self.name, e 

567 ) 

568 else: 

569 raise 

570 

571 def emptyTrash(self, ignore_errors: bool = False) -> None: 

572 """Remove all datasets from the trash. 

573 

574 Parameters 

575 ---------- 

576 ignore_errors : `bool`, optional 

577 Ignore errors. 

578 

579 Notes 

580 ----- 

581 The internal tracking of datasets is affected by this method and 

582 transaction handling is not supported if there is a problem before 

583 the datasets themselves are deleted. 

584 

585 Concurrency should not normally be an issue for the in memory datastore 

586 since all internal changes are isolated to solely this process and 

587 the registry only changes rows associated with this process. 

588 """ 

589 log.debug("Emptying trash in datastore %s", self.name) 

590 with self._bridge.emptyTrash() as trash_data: 

591 trashed, _ = trash_data 

592 for ref, _ in trashed: 

593 try: 

594 realID, _ = self._get_dataset_info(ref) 

595 except FileNotFoundError: 595 ↛ 598line 595 didn't jump to line 598

596 # Dataset already removed so ignore it 

597 continue 

598 except Exception as e: 

599 if ignore_errors: 

600 log.warning( 

601 "Emptying trash in datastore %s but encountered an error with dataset %s: %s", 

602 self.name, 

603 ref.id, 

604 e, 

605 ) 

606 continue 

607 else: 

608 raise 

609 

610 # Determine whether all references to this dataset have been 

611 # removed and we can delete the dataset itself 

612 allRefs = self.related[realID] 

613 remainingRefs = allRefs - {ref.id} 

614 if not remainingRefs: 614 ↛ 619line 614 didn't jump to line 619, because the condition on line 614 was never false

615 log.debug("Removing artifact %s from datastore %s", realID, self.name) 

616 del self.datasets[realID] 

617 

618 # Remove this entry 

619 self.removeStoredItemInfo(ref) 

620 

621 def validateConfiguration( 

622 self, entities: Iterable[Union[DatasetRef, DatasetType, StorageClass]], logFailures: bool = False 

623 ) -> None: 

624 """Validate some of the configuration for this datastore. 

625 

626 Parameters 

627 ---------- 

628 entities : iterable of `DatasetRef`, `DatasetType`, or `StorageClass` 

629 Entities to test against this configuration. Can be differing 

630 types. 

631 logFailures : `bool`, optional 

632 If `True`, output a log message for every validation error 

633 detected. 

634 

635 Raises 

636 ------ 

637 DatastoreValidationError 

638 Raised if there is a validation problem with a configuration. 

639 All the problems are reported in a single exception. 

640 

641 Notes 

642 ----- 

643 This method is a no-op. 

644 """ 

645 return 

646 

647 def _overrideTransferMode(self, *datasets: Any, transfer: Optional[str] = None) -> Optional[str]: 

648 # Docstring is inherited from base class 

649 return transfer 

650 

651 def validateKey(self, lookupKey: LookupKey, entity: Union[DatasetRef, DatasetType, StorageClass]) -> None: 

652 # Docstring is inherited from base class 

653 return 

654 

655 def getLookupKeys(self) -> Set[LookupKey]: 

656 # Docstring is inherited from base class 

657 return self.constraints.getLookupKeys() 

658 

659 def needs_expanded_data_ids( 

660 self, 

661 transfer: Optional[str], 

662 entity: Optional[Union[DatasetRef, DatasetType, StorageClass]] = None, 

663 ) -> bool: 

664 # Docstring inherited. 

665 return False 

666 

667 def import_records(self, data: Mapping[str, DatastoreRecordData]) -> None: 

668 # Docstring inherited from the base class. 

669 return 

670 

671 def export_records(self, refs: Iterable[DatasetIdRef]) -> Mapping[str, DatastoreRecordData]: 

672 # Docstring inherited from the base class. 

673 

674 # In-memory Datastore records cannot be exported or imported 

675 return {}