Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ( 

25 "Registry", 

26) 

27 

28from abc import ABC, abstractmethod 

29import contextlib 

30import logging 

31from typing import ( 

32 Any, 

33 Dict, 

34 Iterable, 

35 Iterator, 

36 List, 

37 Mapping, 

38 Optional, 

39 Tuple, 

40 Type, 

41 TYPE_CHECKING, 

42 Union, 

43) 

44 

45from lsst.utils import doImportType 

46 

47from ..core import ( 

48 ButlerURI, 

49 Config, 

50 DataCoordinate, 

51 DataCoordinateIterable, 

52 DataId, 

53 DatasetAssociation, 

54 DatasetId, 

55 DatasetRef, 

56 DatasetType, 

57 Dimension, 

58 DimensionConfig, 

59 DimensionElement, 

60 DimensionGraph, 

61 DimensionRecord, 

62 DimensionUniverse, 

63 NameLookupMapping, 

64 StorageClassFactory, 

65 Timespan, 

66) 

67 

68from ._config import RegistryConfig 

69from ._collectionType import CollectionType 

70from ._defaults import RegistryDefaults 

71from .interfaces import DatasetIdGenEnum 

72from .wildcards import CollectionSearch 

73from .summaries import CollectionSummary 

74 

75if TYPE_CHECKING: 75 ↛ 76line 75 didn't jump to line 76, because the condition on line 75 was never true

76 from .._butlerConfig import ButlerConfig 

77 from .interfaces import ( 

78 CollectionRecord, 

79 DatastoreRegistryBridgeManager, 

80 ) 

81 

82_LOG = logging.getLogger(__name__) 

83 

84 

85class Registry(ABC): 

86 """Abstract Registry interface. 

87 

88 Each registry implementation can have its own constructor parameters. 

89 The assumption is that an instance of a specific subclass will be 

90 constructed from configuration using `Registry.fromConfig()`. 

91 The base class will look for a ``cls`` entry and call that specific 

92 `fromConfig()` method. 

93 

94 All subclasses should store `RegistryDefaults` in a ``_defaults`` 

95 property. No other properties are assumed shared between implementations. 

96 """ 

97 

98 defaultConfigFile: Optional[str] = None 

99 """Path to configuration defaults. Accessed within the ``configs`` resource 

100 or relative to a search path. Can be None if no defaults specified. 

101 """ 

102 

103 @classmethod 

104 def forceRegistryConfig(cls, config: Optional[Union[ButlerConfig, 

105 RegistryConfig, Config, str]]) -> RegistryConfig: 

106 """Force the supplied config to a `RegistryConfig`. 

107 

108 Parameters 

109 ---------- 

110 config : `RegistryConfig`, `Config` or `str` or `None` 

111 Registry configuration, if missing then default configuration will 

112 be loaded from registry.yaml. 

113 

114 Returns 

115 ------- 

116 registry_config : `RegistryConfig` 

117 A registry config. 

118 """ 

119 if not isinstance(config, RegistryConfig): 

120 if isinstance(config, (str, Config)) or config is None: 

121 config = RegistryConfig(config) 

122 else: 

123 raise ValueError(f"Incompatible Registry configuration: {config}") 

124 return config 

125 

126 @classmethod 

127 def determineTrampoline(cls, 

128 config: Optional[Union[ButlerConfig, 

129 RegistryConfig, 

130 Config, 

131 str]]) -> Tuple[Type[Registry], RegistryConfig]: 

132 """Return class to use to instantiate real registry. 

133 

134 Parameters 

135 ---------- 

136 config : `RegistryConfig` or `str`, optional 

137 Registry configuration, if missing then default configuration will 

138 be loaded from registry.yaml. 

139 

140 Returns 

141 ------- 

142 requested_cls : `type` of `Registry` 

143 The real registry class to use. 

144 registry_config : `RegistryConfig` 

145 The `RegistryConfig` to use. 

146 """ 

147 config = cls.forceRegistryConfig(config) 

148 

149 # Default to the standard registry 

150 registry_cls_name = config.get("cls", "lsst.daf.butler.registries.sql.SqlRegistry") 

151 registry_cls = doImportType(registry_cls_name) 

152 if registry_cls is cls: 

153 raise ValueError("Can not instantiate the abstract base Registry from config") 

154 if not issubclass(registry_cls, Registry): 

155 raise TypeError(f"Registry class obtained from config {registry_cls_name} is " 

156 "not a Registry class.") 

157 return registry_cls, config 

158 

159 @classmethod 

160 def createFromConfig(cls, config: Optional[Union[RegistryConfig, str]] = None, 

161 dimensionConfig: Optional[Union[DimensionConfig, str]] = None, 

162 butlerRoot: Optional[str] = None) -> Registry: 

163 """Create registry database and return `Registry` instance. 

164 

165 This method initializes database contents, database must be empty 

166 prior to calling this method. 

167 

168 Parameters 

169 ---------- 

170 config : `RegistryConfig` or `str`, optional 

171 Registry configuration, if missing then default configuration will 

172 be loaded from registry.yaml. 

173 dimensionConfig : `DimensionConfig` or `str`, optional 

174 Dimensions configuration, if missing then default configuration 

175 will be loaded from dimensions.yaml. 

176 butlerRoot : `str`, optional 

177 Path to the repository root this `Registry` will manage. 

178 

179 Returns 

180 ------- 

181 registry : `Registry` 

182 A new `Registry` instance. 

183 

184 Notes 

185 ----- 

186 This class will determine the concrete `Registry` subclass to 

187 use from configuration. Each subclass should implement this method 

188 even if it can not create a registry. 

189 """ 

190 registry_cls, registry_config = cls.determineTrampoline(config) 

191 return registry_cls.createFromConfig(registry_config, dimensionConfig, butlerRoot) 

192 

193 @classmethod 

194 def fromConfig(cls, config: Union[ButlerConfig, RegistryConfig, Config, str], 

195 butlerRoot: Optional[Union[str, ButlerURI]] = None, writeable: bool = True, 

196 defaults: Optional[RegistryDefaults] = None) -> Registry: 

197 """Create `Registry` subclass instance from `config`. 

198 

199 Registry database must be initialized prior to calling this method. 

200 

201 Parameters 

202 ---------- 

203 config : `ButlerConfig`, `RegistryConfig`, `Config` or `str` 

204 Registry configuration 

205 butlerRoot : `str` or `ButlerURI`, optional 

206 Path to the repository root this `Registry` will manage. 

207 writeable : `bool`, optional 

208 If `True` (default) create a read-write connection to the database. 

209 defaults : `RegistryDefaults`, optional 

210 Default collection search path and/or output `~CollectionType.RUN` 

211 collection. 

212 

213 Returns 

214 ------- 

215 registry : `Registry` (subclass) 

216 A new `Registry` subclass instance. 

217 

218 Notes 

219 ----- 

220 This class will determine the concrete `Registry` subclass to 

221 use from configuration. Each subclass should implement this method. 

222 """ 

223 # The base class implementation should trampoline to the correct 

224 # subclass. No implementation should ever use this implementation 

225 # directly. If no class is specified, default to the standard 

226 # registry. 

227 registry_cls, registry_config = cls.determineTrampoline(config) 

228 return registry_cls.fromConfig(config, butlerRoot, writeable, defaults) 

229 

230 @abstractmethod 

231 def isWriteable(self) -> bool: 

232 """Return `True` if this registry allows write operations, and `False` 

233 otherwise. 

234 """ 

235 raise NotImplementedError() 

236 

237 @abstractmethod 

238 def copy(self, defaults: Optional[RegistryDefaults] = None) -> Registry: 

239 """Create a new `Registry` backed by the same data repository and 

240 connection as this one, but independent defaults. 

241 

242 Parameters 

243 ---------- 

244 defaults : `RegistryDefaults`, optional 

245 Default collections and data ID values for the new registry. If 

246 not provided, ``self.defaults`` will be used (but future changes 

247 to either registry's defaults will not affect the other). 

248 

249 Returns 

250 ------- 

251 copy : `Registry` 

252 A new `Registry` instance with its own defaults. 

253 

254 Notes 

255 ----- 

256 Because the new registry shares a connection with the original, they 

257 also share transaction state (despite the fact that their `transaction` 

258 context manager methods do not reflect this), and must be used with 

259 care. 

260 """ 

261 raise NotImplementedError() 

262 

263 @property 

264 @abstractmethod 

265 def dimensions(self) -> DimensionUniverse: 

266 """All dimensions recognized by this `Registry` (`DimensionUniverse`). 

267 """ 

268 raise NotImplementedError() 

269 

270 @property 

271 def defaults(self) -> RegistryDefaults: 

272 """Default collection search path and/or output `~CollectionType.RUN` 

273 collection (`RegistryDefaults`). 

274 

275 This is an immutable struct whose components may not be set 

276 individually, but the entire struct can be set by assigning to this 

277 property. 

278 """ 

279 return self._defaults 

280 

281 @defaults.setter 

282 def defaults(self, value: RegistryDefaults) -> None: 

283 if value.run is not None: 

284 self.registerRun(value.run) 

285 value.finish(self) 

286 self._defaults = value 

287 

288 @abstractmethod 

289 def refresh(self) -> None: 

290 """Refresh all in-memory state by querying the database. 

291 

292 This may be necessary to enable querying for entities added by other 

293 registry instances after this one was constructed. 

294 """ 

295 raise NotImplementedError() 

296 

297 @contextlib.contextmanager 

298 @abstractmethod 

299 def transaction(self, *, savepoint: bool = False) -> Iterator[None]: 

300 """Return a context manager that represents a transaction. 

301 """ 

302 raise NotImplementedError() 

303 

304 def resetConnectionPool(self) -> None: 

305 """Reset connection pool for registry if relevant. 

306 

307 This operation can be used reset connections to servers when 

308 using registry with fork-based multiprocessing. This method should 

309 usually be called by the child process immediately 

310 after the fork. 

311 

312 The base class implementation is a no-op. 

313 """ 

314 pass 

315 

316 @abstractmethod 

317 def registerCollection(self, name: str, type: CollectionType = CollectionType.TAGGED, 

318 doc: Optional[str] = None) -> bool: 

319 """Add a new collection if one with the given name does not exist. 

320 

321 Parameters 

322 ---------- 

323 name : `str` 

324 The name of the collection to create. 

325 type : `CollectionType` 

326 Enum value indicating the type of collection to create. 

327 doc : `str`, optional 

328 Documentation string for the collection. 

329 

330 Returns 

331 ------- 

332 registered : `bool` 

333 Boolean indicating whether the collection was already registered 

334 or was created by this call. 

335 

336 Notes 

337 ----- 

338 This method cannot be called within transactions, as it needs to be 

339 able to perform its own transaction to be concurrent. 

340 """ 

341 raise NotImplementedError() 

342 

343 @abstractmethod 

344 def getCollectionType(self, name: str) -> CollectionType: 

345 """Return an enumeration value indicating the type of the given 

346 collection. 

347 

348 Parameters 

349 ---------- 

350 name : `str` 

351 The name of the collection. 

352 

353 Returns 

354 ------- 

355 type : `CollectionType` 

356 Enum value indicating the type of this collection. 

357 

358 Raises 

359 ------ 

360 MissingCollectionError 

361 Raised if no collection with the given name exists. 

362 """ 

363 raise NotImplementedError() 

364 

365 @abstractmethod 

366 def _get_collection_record(self, name: str) -> CollectionRecord: 

367 """Return the record for this collection. 

368 

369 Parameters 

370 ---------- 

371 name : `str` 

372 Name of the collection for which the record is to be retrieved. 

373 

374 Returns 

375 ------- 

376 record : `CollectionRecord` 

377 The record for this collection. 

378 """ 

379 raise NotImplementedError() 

380 

381 @abstractmethod 

382 def registerRun(self, name: str, doc: Optional[str] = None) -> bool: 

383 """Add a new run if one with the given name does not exist. 

384 

385 Parameters 

386 ---------- 

387 name : `str` 

388 The name of the run to create. 

389 doc : `str`, optional 

390 Documentation string for the collection. 

391 

392 Returns 

393 ------- 

394 registered : `bool` 

395 Boolean indicating whether a new run was registered. `False` 

396 if it already existed. 

397 

398 Notes 

399 ----- 

400 This method cannot be called within transactions, as it needs to be 

401 able to perform its own transaction to be concurrent. 

402 """ 

403 raise NotImplementedError() 

404 

405 @abstractmethod 

406 def removeCollection(self, name: str) -> None: 

407 """Completely remove the given collection. 

408 

409 Parameters 

410 ---------- 

411 name : `str` 

412 The name of the collection to remove. 

413 

414 Raises 

415 ------ 

416 MissingCollectionError 

417 Raised if no collection with the given name exists. 

418 

419 Notes 

420 ----- 

421 If this is a `~CollectionType.RUN` collection, all datasets and quanta 

422 in it are also fully removed. This requires that those datasets be 

423 removed (or at least trashed) from any datastores that hold them first. 

424 

425 A collection may not be deleted as long as it is referenced by a 

426 `~CollectionType.CHAINED` collection; the ``CHAINED`` collection must 

427 be deleted or redefined first. 

428 """ 

429 raise NotImplementedError() 

430 

431 @abstractmethod 

432 def getCollectionChain(self, parent: str) -> CollectionSearch: 

433 """Return the child collections in a `~CollectionType.CHAINED` 

434 collection. 

435 

436 Parameters 

437 ---------- 

438 parent : `str` 

439 Name of the chained collection. Must have already been added via 

440 a call to `Registry.registerCollection`. 

441 

442 Returns 

443 ------- 

444 children : `CollectionSearch` 

445 An object that defines the search path of the collection. 

446 See :ref:`daf_butler_collection_expressions` for more information. 

447 

448 Raises 

449 ------ 

450 MissingCollectionError 

451 Raised if ``parent`` does not exist in the `Registry`. 

452 TypeError 

453 Raised if ``parent`` does not correspond to a 

454 `~CollectionType.CHAINED` collection. 

455 """ 

456 raise NotImplementedError() 

457 

458 @abstractmethod 

459 def setCollectionChain(self, parent: str, children: Any, *, flatten: bool = False) -> None: 

460 """Define or redefine a `~CollectionType.CHAINED` collection. 

461 

462 Parameters 

463 ---------- 

464 parent : `str` 

465 Name of the chained collection. Must have already been added via 

466 a call to `Registry.registerCollection`. 

467 children : `Any` 

468 An expression defining an ordered search of child collections, 

469 generally an iterable of `str`; see 

470 :ref:`daf_butler_collection_expressions` for more information. 

471 flatten : `bool`, optional 

472 If `True` (`False` is default), recursively flatten out any nested 

473 `~CollectionType.CHAINED` collections in ``children`` first. 

474 

475 Raises 

476 ------ 

477 MissingCollectionError 

478 Raised when any of the given collections do not exist in the 

479 `Registry`. 

480 TypeError 

481 Raised if ``parent`` does not correspond to a 

482 `~CollectionType.CHAINED` collection. 

483 ValueError 

484 Raised if the given collections contains a cycle. 

485 """ 

486 raise NotImplementedError() 

487 

488 @abstractmethod 

489 def getCollectionDocumentation(self, collection: str) -> Optional[str]: 

490 """Retrieve the documentation string for a collection. 

491 

492 Parameters 

493 ---------- 

494 name : `str` 

495 Name of the collection. 

496 

497 Returns 

498 ------- 

499 docs : `str` or `None` 

500 Docstring for the collection with the given name. 

501 """ 

502 raise NotImplementedError() 

503 

504 @abstractmethod 

505 def setCollectionDocumentation(self, collection: str, doc: Optional[str]) -> None: 

506 """Set the documentation string for a collection. 

507 

508 Parameters 

509 ---------- 

510 name : `str` 

511 Name of the collection. 

512 docs : `str` or `None` 

513 Docstring for the collection with the given name; will replace any 

514 existing docstring. Passing `None` will remove any existing 

515 docstring. 

516 """ 

517 raise NotImplementedError() 

518 

519 @abstractmethod 

520 def getCollectionSummary(self, collection: str) -> CollectionSummary: 

521 """Return a summary for the given collection. 

522 

523 Parameters 

524 ---------- 

525 collection : `str` 

526 Name of the collection for which a summary is to be retrieved. 

527 

528 Returns 

529 ------- 

530 summary : `CollectionSummary` 

531 Summary of the dataset types and governor dimension values in 

532 this collection. 

533 """ 

534 raise NotImplementedError() 

535 

536 @abstractmethod 

537 def registerDatasetType(self, datasetType: DatasetType) -> bool: 

538 """ 

539 Add a new `DatasetType` to the Registry. 

540 

541 It is not an error to register the same `DatasetType` twice. 

542 

543 Parameters 

544 ---------- 

545 datasetType : `DatasetType` 

546 The `DatasetType` to be added. 

547 

548 Returns 

549 ------- 

550 inserted : `bool` 

551 `True` if ``datasetType`` was inserted, `False` if an identical 

552 existing `DatsetType` was found. Note that in either case the 

553 DatasetType is guaranteed to be defined in the Registry 

554 consistently with the given definition. 

555 

556 Raises 

557 ------ 

558 ValueError 

559 Raised if the dimensions or storage class are invalid. 

560 ConflictingDefinitionError 

561 Raised if this DatasetType is already registered with a different 

562 definition. 

563 

564 Notes 

565 ----- 

566 This method cannot be called within transactions, as it needs to be 

567 able to perform its own transaction to be concurrent. 

568 """ 

569 raise NotImplementedError() 

570 

571 @abstractmethod 

572 def removeDatasetType(self, name: str) -> None: 

573 """Remove the named `DatasetType` from the registry. 

574 

575 .. warning:: 

576 

577 Registry implementations can cache the dataset type definitions. 

578 This means that deleting the dataset type definition may result in 

579 unexpected behavior from other butler processes that are active 

580 that have not seen the deletion. 

581 

582 Parameters 

583 ---------- 

584 name : `str` 

585 Name of the type to be removed. 

586 

587 Raises 

588 ------ 

589 lsst.daf.butler.registry.OrphanedRecordError 

590 Raised if an attempt is made to remove the dataset type definition 

591 when there are already datasets associated with it. 

592 

593 Notes 

594 ----- 

595 If the dataset type is not registered the method will return without 

596 action. 

597 """ 

598 raise NotImplementedError() 

599 

600 @abstractmethod 

601 def getDatasetType(self, name: str) -> DatasetType: 

602 """Get the `DatasetType`. 

603 

604 Parameters 

605 ---------- 

606 name : `str` 

607 Name of the type. 

608 

609 Returns 

610 ------- 

611 type : `DatasetType` 

612 The `DatasetType` associated with the given name. 

613 

614 Raises 

615 ------ 

616 KeyError 

617 Requested named DatasetType could not be found in registry. 

618 """ 

619 raise NotImplementedError() 

620 

621 @abstractmethod 

622 def supportsIdGenerationMode(self, mode: DatasetIdGenEnum) -> bool: 

623 """Test whether the given dataset ID generation mode is supported by 

624 `insertDatasets`. 

625 

626 Parameters 

627 ---------- 

628 mode : `DatasetIdGenEnum` 

629 Enum value for the mode to test. 

630 

631 Returns 

632 ------- 

633 supported : `bool` 

634 Whether the given mode is supported. 

635 """ 

636 raise NotImplementedError() 

637 

638 @abstractmethod 

639 def findDataset(self, datasetType: Union[DatasetType, str], dataId: Optional[DataId] = None, *, 

640 collections: Any = None, timespan: Optional[Timespan] = None, 

641 **kwargs: Any) -> Optional[DatasetRef]: 

642 """Find a dataset given its `DatasetType` and data ID. 

643 

644 This can be used to obtain a `DatasetRef` that permits the dataset to 

645 be read from a `Datastore`. If the dataset is a component and can not 

646 be found using the provided dataset type, a dataset ref for the parent 

647 will be returned instead but with the correct dataset type. 

648 

649 Parameters 

650 ---------- 

651 datasetType : `DatasetType` or `str` 

652 A `DatasetType` or the name of one. 

653 dataId : `dict` or `DataCoordinate`, optional 

654 A `dict`-like object containing the `Dimension` links that identify 

655 the dataset within a collection. 

656 collections, optional. 

657 An expression that fully or partially identifies the collections to 

658 search for the dataset; see 

659 :ref:`daf_butler_collection_expressions` for more information. 

660 Defaults to ``self.defaults.collections``. 

661 timespan : `Timespan`, optional 

662 A timespan that the validity range of the dataset must overlap. 

663 If not provided, any `~CollectionType.CALIBRATION` collections 

664 matched by the ``collections`` argument will not be searched. 

665 **kwargs 

666 Additional keyword arguments passed to 

667 `DataCoordinate.standardize` to convert ``dataId`` to a true 

668 `DataCoordinate` or augment an existing one. 

669 

670 Returns 

671 ------- 

672 ref : `DatasetRef` 

673 A reference to the dataset, or `None` if no matching Dataset 

674 was found. 

675 

676 Raises 

677 ------ 

678 TypeError 

679 Raised if ``collections`` is `None` and 

680 ``self.defaults.collections`` is `None`. 

681 LookupError 

682 Raised if one or more data ID keys are missing. 

683 KeyError 

684 Raised if the dataset type does not exist. 

685 MissingCollectionError 

686 Raised if any of ``collections`` does not exist in the registry. 

687 

688 Notes 

689 ----- 

690 This method simply returns `None` and does not raise an exception even 

691 when the set of collections searched is intrinsically incompatible with 

692 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but 

693 only `~CollectionType.CALIBRATION` collections are being searched. 

694 This may make it harder to debug some lookup failures, but the behavior 

695 is intentional; we consider it more important that failed searches are 

696 reported consistently, regardless of the reason, and that adding 

697 additional collections that do not contain a match to the search path 

698 never changes the behavior. 

699 """ 

700 raise NotImplementedError() 

701 

702 @abstractmethod 

703 def insertDatasets(self, datasetType: Union[DatasetType, str], dataIds: Iterable[DataId], 

704 run: Optional[str] = None, expand: bool = True, 

705 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE) -> List[DatasetRef]: 

706 """Insert one or more datasets into the `Registry` 

707 

708 This always adds new datasets; to associate existing datasets with 

709 a new collection, use ``associate``. 

710 

711 Parameters 

712 ---------- 

713 datasetType : `DatasetType` or `str` 

714 A `DatasetType` or the name of one. 

715 dataIds : `~collections.abc.Iterable` of `dict` or `DataCoordinate` 

716 Dimension-based identifiers for the new datasets. 

717 run : `str`, optional 

718 The name of the run that produced the datasets. Defaults to 

719 ``self.defaults.run``. 

720 expand : `bool`, optional 

721 If `True` (default), expand data IDs as they are inserted. This is 

722 necessary in general to allow datastore to generate file templates, 

723 but it may be disabled if the caller can guarantee this is 

724 unnecessary. 

725 idGenerationMode : `DatasetIdGenEnum`, optional 

726 Specifies option for generating dataset IDs. By default unique IDs 

727 are generated for each inserted dataset. 

728 

729 Returns 

730 ------- 

731 refs : `list` of `DatasetRef` 

732 Resolved `DatasetRef` instances for all given data IDs (in the same 

733 order). 

734 

735 Raises 

736 ------ 

737 TypeError 

738 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

739 ConflictingDefinitionError 

740 If a dataset with the same dataset type and data ID as one of those 

741 given already exists in ``run``. 

742 MissingCollectionError 

743 Raised if ``run`` does not exist in the registry. 

744 """ 

745 raise NotImplementedError() 

746 

747 @abstractmethod 

748 def _importDatasets(self, datasets: Iterable[DatasetRef], expand: bool = True, 

749 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE, 

750 reuseIds: bool = False) -> List[DatasetRef]: 

751 """Import one or more datasets into the `Registry`. 

752 

753 Difference from `insertDatasets` method is that this method accepts 

754 `DatasetRef` instances which should already be resolved and have a 

755 dataset ID. If registry supports globally-unique dataset IDs (e.g. 

756 `uuid.UUID`) then datasets which already exist in the registry will be 

757 ignored if imported again. 

758 

759 Parameters 

760 ---------- 

761 datasets : `~collections.abc.Iterable` of `DatasetRef` 

762 Datasets to be inserted. All `DatasetRef` instances must have 

763 identical ``datasetType`` and ``run`` attributes. ``run`` 

764 attribute can be `None` and defaults to ``self.defaults.run``. 

765 Datasets can specify ``id`` attribute which will be used for 

766 inserted datasets. All dataset IDs must have the same type 

767 (`int` or `uuid.UUID`), if type of dataset IDs does not match 

768 configured backend then IDs will be ignored and new IDs will be 

769 generated by backend. 

770 expand : `bool`, optional 

771 If `True` (default), expand data IDs as they are inserted. This is 

772 necessary in general to allow datastore to generate file templates, 

773 but it may be disabled if the caller can guarantee this is 

774 unnecessary. 

775 idGenerationMode : `DatasetIdGenEnum`, optional 

776 Specifies option for generating dataset IDs when IDs are not 

777 provided or their type does not match backend type. By default 

778 unique IDs are generated for each inserted dataset. 

779 reuseIds : `bool`, optional 

780 If `True` then forces re-use of imported dataset IDs for integer 

781 IDs which are normally generated as auto-incremented; exception 

782 will be raised if imported IDs clash with existing ones. This 

783 option has no effect on the use of globally-unique IDs which are 

784 always re-used (or generated if integer IDs are being imported). 

785 

786 Returns 

787 ------- 

788 refs : `list` of `DatasetRef` 

789 Resolved `DatasetRef` instances for all given data IDs (in the same 

790 order). If any of ``datasets`` has an ID which already exists in 

791 the database then it will not be inserted or updated, but a 

792 resolved `DatasetRef` will be returned for it in any case. 

793 

794 Raises 

795 ------ 

796 TypeError 

797 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

798 ConflictingDefinitionError 

799 If a dataset with the same dataset type and data ID as one of those 

800 given already exists in ``run``. 

801 MissingCollectionError 

802 Raised if ``run`` does not exist in the registry. 

803 

804 Notes 

805 ----- 

806 This method is considered package-private and internal to Butler 

807 implementation. Clients outside daf_butler package should not use this 

808 method. 

809 """ 

810 raise NotImplementedError() 

811 

812 @abstractmethod 

813 def getDataset(self, id: DatasetId) -> Optional[DatasetRef]: 

814 """Retrieve a Dataset entry. 

815 

816 Parameters 

817 ---------- 

818 id : `DatasetId` 

819 The unique identifier for the dataset. 

820 

821 Returns 

822 ------- 

823 ref : `DatasetRef` or `None` 

824 A ref to the Dataset, or `None` if no matching Dataset 

825 was found. 

826 """ 

827 raise NotImplementedError() 

828 

829 @abstractmethod 

830 def removeDatasets(self, refs: Iterable[DatasetRef]) -> None: 

831 """Remove datasets from the Registry. 

832 

833 The datasets will be removed unconditionally from all collections, and 

834 any `Quantum` that consumed this dataset will instead be marked with 

835 having a NULL input. `Datastore` records will *not* be deleted; the 

836 caller is responsible for ensuring that the dataset has already been 

837 removed from all Datastores. 

838 

839 Parameters 

840 ---------- 

841 refs : `Iterable` of `DatasetRef` 

842 References to the datasets to be removed. Must include a valid 

843 ``id`` attribute, and should be considered invalidated upon return. 

844 

845 Raises 

846 ------ 

847 AmbiguousDatasetError 

848 Raised if any ``ref.id`` is `None`. 

849 OrphanedRecordError 

850 Raised if any dataset is still present in any `Datastore`. 

851 """ 

852 raise NotImplementedError() 

853 

854 @abstractmethod 

855 def associate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

856 """Add existing datasets to a `~CollectionType.TAGGED` collection. 

857 

858 If a DatasetRef with the same exact ID is already in a collection 

859 nothing is changed. If a `DatasetRef` with the same `DatasetType` and 

860 data ID but with different ID exists in the collection, 

861 `ConflictingDefinitionError` is raised. 

862 

863 Parameters 

864 ---------- 

865 collection : `str` 

866 Indicates the collection the datasets should be associated with. 

867 refs : `Iterable` [ `DatasetRef` ] 

868 An iterable of resolved `DatasetRef` instances that already exist 

869 in this `Registry`. 

870 

871 Raises 

872 ------ 

873 ConflictingDefinitionError 

874 If a Dataset with the given `DatasetRef` already exists in the 

875 given collection. 

876 AmbiguousDatasetError 

877 Raised if ``any(ref.id is None for ref in refs)``. 

878 MissingCollectionError 

879 Raised if ``collection`` does not exist in the registry. 

880 TypeError 

881 Raise adding new datasets to the given ``collection`` is not 

882 allowed. 

883 """ 

884 raise NotImplementedError() 

885 

886 @abstractmethod 

887 def disassociate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

888 """Remove existing datasets from a `~CollectionType.TAGGED` collection. 

889 

890 ``collection`` and ``ref`` combinations that are not currently 

891 associated are silently ignored. 

892 

893 Parameters 

894 ---------- 

895 collection : `str` 

896 The collection the datasets should no longer be associated with. 

897 refs : `Iterable` [ `DatasetRef` ] 

898 An iterable of resolved `DatasetRef` instances that already exist 

899 in this `Registry`. 

900 

901 Raises 

902 ------ 

903 AmbiguousDatasetError 

904 Raised if any of the given dataset references is unresolved. 

905 MissingCollectionError 

906 Raised if ``collection`` does not exist in the registry. 

907 TypeError 

908 Raise adding new datasets to the given ``collection`` is not 

909 allowed. 

910 """ 

911 raise NotImplementedError() 

912 

913 @abstractmethod 

914 def certify(self, collection: str, refs: Iterable[DatasetRef], timespan: Timespan) -> None: 

915 """Associate one or more datasets with a calibration collection and a 

916 validity range within it. 

917 

918 Parameters 

919 ---------- 

920 collection : `str` 

921 The name of an already-registered `~CollectionType.CALIBRATION` 

922 collection. 

923 refs : `Iterable` [ `DatasetRef` ] 

924 Datasets to be associated. 

925 timespan : `Timespan` 

926 The validity range for these datasets within the collection. 

927 

928 Raises 

929 ------ 

930 AmbiguousDatasetError 

931 Raised if any of the given `DatasetRef` instances is unresolved. 

932 ConflictingDefinitionError 

933 Raised if the collection already contains a different dataset with 

934 the same `DatasetType` and data ID and an overlapping validity 

935 range. 

936 TypeError 

937 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

938 collection or if one or more datasets are of a dataset type for 

939 which `DatasetType.isCalibration` returns `False`. 

940 """ 

941 raise NotImplementedError() 

942 

943 @abstractmethod 

944 def decertify(self, collection: str, datasetType: Union[str, DatasetType], timespan: Timespan, *, 

945 dataIds: Optional[Iterable[DataId]] = None) -> None: 

946 """Remove or adjust datasets to clear a validity range within a 

947 calibration collection. 

948 

949 Parameters 

950 ---------- 

951 collection : `str` 

952 The name of an already-registered `~CollectionType.CALIBRATION` 

953 collection. 

954 datasetType : `str` or `DatasetType` 

955 Name or `DatasetType` instance for the datasets to be decertified. 

956 timespan : `Timespan`, optional 

957 The validity range to remove datasets from within the collection. 

958 Datasets that overlap this range but are not contained by it will 

959 have their validity ranges adjusted to not overlap it, which may 

960 split a single dataset validity range into two. 

961 dataIds : `Iterable` [ `DataId` ], optional 

962 Data IDs that should be decertified within the given validity range 

963 If `None`, all data IDs for ``self.datasetType`` will be 

964 decertified. 

965 

966 Raises 

967 ------ 

968 TypeError 

969 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

970 collection or if ``datasetType.isCalibration() is False``. 

971 """ 

972 raise NotImplementedError() 

973 

974 @abstractmethod 

975 def getDatastoreBridgeManager(self) -> DatastoreRegistryBridgeManager: 

976 """Return an object that allows a new `Datastore` instance to 

977 communicate with this `Registry`. 

978 

979 Returns 

980 ------- 

981 manager : `DatastoreRegistryBridgeManager` 

982 Object that mediates communication between this `Registry` and its 

983 associated datastores. 

984 """ 

985 raise NotImplementedError() 

986 

987 @abstractmethod 

988 def getDatasetLocations(self, ref: DatasetRef) -> Iterable[str]: 

989 """Retrieve datastore locations for a given dataset. 

990 

991 Parameters 

992 ---------- 

993 ref : `DatasetRef` 

994 A reference to the dataset for which to retrieve storage 

995 information. 

996 

997 Returns 

998 ------- 

999 datastores : `Iterable` [ `str` ] 

1000 All the matching datastores holding this dataset. 

1001 

1002 Raises 

1003 ------ 

1004 AmbiguousDatasetError 

1005 Raised if ``ref.id`` is `None`. 

1006 """ 

1007 raise NotImplementedError() 

1008 

1009 @abstractmethod 

1010 def expandDataId(self, dataId: Optional[DataId] = None, *, graph: Optional[DimensionGraph] = None, 

1011 records: Optional[NameLookupMapping[DimensionElement, Optional[DimensionRecord]]] = None, 

1012 withDefaults: bool = True, 

1013 **kwargs: Any) -> DataCoordinate: 

1014 """Expand a dimension-based data ID to include additional information. 

1015 

1016 Parameters 

1017 ---------- 

1018 dataId : `DataCoordinate` or `dict`, optional 

1019 Data ID to be expanded; augmented and overridden by ``kwargs``. 

1020 graph : `DimensionGraph`, optional 

1021 Set of dimensions for the expanded ID. If `None`, the dimensions 

1022 will be inferred from the keys of ``dataId`` and ``kwargs``. 

1023 Dimensions that are in ``dataId`` or ``kwargs`` but not in 

1024 ``graph`` are silently ignored, providing a way to extract and 

1025 ``graph`` expand a subset of a data ID. 

1026 records : `Mapping` [`str`, `DimensionRecord`], optional 

1027 Dimension record data to use before querying the database for that 

1028 data, keyed by element name. 

1029 withDefaults : `bool`, optional 

1030 Utilize ``self.defaults.dataId`` to fill in missing governor 

1031 dimension key-value pairs. Defaults to `True` (i.e. defaults are 

1032 used). 

1033 **kwargs 

1034 Additional keywords are treated like additional key-value pairs for 

1035 ``dataId``, extending and overriding 

1036 

1037 Returns 

1038 ------- 

1039 expanded : `DataCoordinate` 

1040 A data ID that includes full metadata for all of the dimensions it 

1041 identifieds, i.e. guarantees that ``expanded.hasRecords()`` and 

1042 ``expanded.hasFull()`` both return `True`. 

1043 """ 

1044 raise NotImplementedError() 

1045 

1046 @abstractmethod 

1047 def insertDimensionData(self, element: Union[DimensionElement, str], 

1048 *data: Union[Mapping[str, Any], DimensionRecord], 

1049 conform: bool = True, 

1050 replace: bool = False) -> None: 

1051 """Insert one or more dimension records into the database. 

1052 

1053 Parameters 

1054 ---------- 

1055 element : `DimensionElement` or `str` 

1056 The `DimensionElement` or name thereof that identifies the table 

1057 records will be inserted into. 

1058 data : `dict` or `DimensionRecord` (variadic) 

1059 One or more records to insert. 

1060 conform : `bool`, optional 

1061 If `False` (`True` is default) perform no checking or conversions, 

1062 and assume that ``element`` is a `DimensionElement` instance and 

1063 ``data`` is a one or more `DimensionRecord` instances of the 

1064 appropriate subclass. 

1065 replace: `bool`, optional 

1066 If `True` (`False` is default), replace existing records in the 

1067 database if there is a conflict. 

1068 """ 

1069 raise NotImplementedError() 

1070 

1071 @abstractmethod 

1072 def syncDimensionData(self, element: Union[DimensionElement, str], 

1073 row: Union[Mapping[str, Any], DimensionRecord], 

1074 conform: bool = True, 

1075 update: bool = False) -> Union[bool, Dict[str, Any]]: 

1076 """Synchronize the given dimension record with the database, inserting 

1077 if it does not already exist and comparing values if it does. 

1078 

1079 Parameters 

1080 ---------- 

1081 element : `DimensionElement` or `str` 

1082 The `DimensionElement` or name thereof that identifies the table 

1083 records will be inserted into. 

1084 row : `dict` or `DimensionRecord` 

1085 The record to insert. 

1086 conform : `bool`, optional 

1087 If `False` (`True` is default) perform no checking or conversions, 

1088 and assume that ``element`` is a `DimensionElement` instance and 

1089 ``data`` is a one or more `DimensionRecord` instances of the 

1090 appropriate subclass. 

1091 update: `bool`, optional 

1092 If `True` (`False` is default), update the existing record in the 

1093 database if there is a conflict. 

1094 

1095 Returns 

1096 ------- 

1097 inserted_or_updated : `bool` or `dict` 

1098 `True` if a new row was inserted, `False` if no changes were 

1099 needed, or a `dict` mapping updated column names to their old 

1100 values if an update was performed (only possible if 

1101 ``update=True``). 

1102 

1103 Raises 

1104 ------ 

1105 ConflictingDefinitionError 

1106 Raised if the record exists in the database (according to primary 

1107 key lookup) but is inconsistent with the given one. 

1108 """ 

1109 raise NotImplementedError() 

1110 

1111 @abstractmethod 

1112 def queryDatasetTypes(self, expression: Any = ..., *, components: Optional[bool] = None 

1113 ) -> Iterator[DatasetType]: 

1114 """Iterate over the dataset types whose names match an expression. 

1115 

1116 Parameters 

1117 ---------- 

1118 expression : `Any`, optional 

1119 An expression that fully or partially identifies the dataset types 

1120 to return, such as a `str`, `re.Pattern`, or iterable thereof. 

1121 `...` can be used to return all dataset types, and is the default. 

1122 See :ref:`daf_butler_dataset_type_expressions` for more 

1123 information. 

1124 components : `bool`, optional 

1125 If `True`, apply all expression patterns to component dataset type 

1126 names as well. If `False`, never apply patterns to components. 

1127 If `None` (default), apply patterns to components only if their 

1128 parent datasets were not matched by the expression. 

1129 Fully-specified component datasets (`str` or `DatasetType` 

1130 instances) are always included. 

1131 

1132 Yields 

1133 ------ 

1134 datasetType : `DatasetType` 

1135 A `DatasetType` instance whose name matches ``expression``. 

1136 """ 

1137 raise NotImplementedError() 

1138 

1139 @abstractmethod 

1140 def queryCollections(self, expression: Any = ..., 

1141 datasetType: Optional[DatasetType] = None, 

1142 collectionTypes: Iterable[CollectionType] = CollectionType.all(), 

1143 flattenChains: bool = False, 

1144 includeChains: Optional[bool] = None) -> Iterator[str]: 

1145 """Iterate over the collections whose names match an expression. 

1146 

1147 Parameters 

1148 ---------- 

1149 expression : `Any`, optional 

1150 An expression that identifies the collections to return, such as 

1151 a `str` (for full matches or partial matches via globs), 

1152 `re.Pattern` (for partial matches), or iterable thereof. `...` 

1153 can be used to return all collections, and is the default. 

1154 See :ref:`daf_butler_collection_expressions` for more information. 

1155 datasetType : `DatasetType`, optional 

1156 If provided, only yield collections that may contain datasets of 

1157 this type. This is a conservative approximation in general; it may 

1158 yield collections that do not have any such datasets. 

1159 collectionTypes : `AbstractSet` [ `CollectionType` ], optional 

1160 If provided, only yield collections of these types. 

1161 flattenChains : `bool`, optional 

1162 If `True` (`False` is default), recursively yield the child 

1163 collections of matching `~CollectionType.CHAINED` collections. 

1164 includeChains : `bool`, optional 

1165 If `True`, yield records for matching `~CollectionType.CHAINED` 

1166 collections. Default is the opposite of ``flattenChains``: include 

1167 either CHAINED collections or their children, but not both. 

1168 

1169 Yields 

1170 ------ 

1171 collection : `str` 

1172 The name of a collection that matches ``expression``. 

1173 """ 

1174 raise NotImplementedError() 

1175 

1176 @abstractmethod 

1177 def queryDatasets(self, datasetType: Any, *, 

1178 collections: Any = None, 

1179 dimensions: Optional[Iterable[Union[Dimension, str]]] = None, 

1180 dataId: Optional[DataId] = None, 

1181 where: Optional[str] = None, 

1182 findFirst: bool = False, 

1183 components: Optional[bool] = None, 

1184 bind: Optional[Mapping[str, Any]] = None, 

1185 check: bool = True, 

1186 **kwargs: Any) -> Iterable[DatasetRef]: 

1187 """Query for and iterate over dataset references matching user-provided 

1188 criteria. 

1189 

1190 Parameters 

1191 ---------- 

1192 datasetType 

1193 An expression that fully or partially identifies the dataset types 

1194 to be queried. Allowed types include `DatasetType`, `str`, 

1195 `re.Pattern`, and iterables thereof. The special value `...` can 

1196 be used to query all dataset types. See 

1197 :ref:`daf_butler_dataset_type_expressions` for more information. 

1198 collections: optional 

1199 An expression that identifies the collections to search, such as a 

1200 `str` (for full matches or partial matches via globs), `re.Pattern` 

1201 (for partial matches), or iterable thereof. `...` can be used to 

1202 search all collections (actually just all `~CollectionType.RUN` 

1203 collections, because this will still find all datasets). 

1204 If not provided, ``self.default.collections`` is used. See 

1205 :ref:`daf_butler_collection_expressions` for more information. 

1206 dimensions : `~collections.abc.Iterable` of `Dimension` or `str` 

1207 Dimensions to include in the query (in addition to those used 

1208 to identify the queried dataset type(s)), either to constrain 

1209 the resulting datasets to those for which a matching dimension 

1210 exists, or to relate the dataset type's dimensions to dimensions 

1211 referenced by the ``dataId`` or ``where`` arguments. 

1212 dataId : `dict` or `DataCoordinate`, optional 

1213 A data ID whose key-value pairs are used as equality constraints 

1214 in the query. 

1215 where : `str`, optional 

1216 A string expression similar to a SQL WHERE clause. May involve 

1217 any column of a dimension table or (as a shortcut for the primary 

1218 key column of a dimension table) dimension name. See 

1219 :ref:`daf_butler_dimension_expressions` for more information. 

1220 findFirst : `bool`, optional 

1221 If `True` (`False` is default), for each result data ID, only 

1222 yield one `DatasetRef` of each `DatasetType`, from the first 

1223 collection in which a dataset of that dataset type appears 

1224 (according to the order of ``collections`` passed in). If `True`, 

1225 ``collections`` must not contain regular expressions and may not 

1226 be `...`. 

1227 components : `bool`, optional 

1228 If `True`, apply all dataset expression patterns to component 

1229 dataset type names as well. If `False`, never apply patterns to 

1230 components. If `None` (default), apply patterns to components only 

1231 if their parent datasets were not matched by the expression. 

1232 Fully-specified component datasets (`str` or `DatasetType` 

1233 instances) are always included. 

1234 bind : `Mapping`, optional 

1235 Mapping containing literal values that should be injected into the 

1236 ``where`` expression, keyed by the identifiers they replace. 

1237 check : `bool`, optional 

1238 If `True` (default) check the query for consistency before 

1239 executing it. This may reject some valid queries that resemble 

1240 common mistakes (e.g. queries for visits without specifying an 

1241 instrument). 

1242 **kwargs 

1243 Additional keyword arguments are forwarded to 

1244 `DataCoordinate.standardize` when processing the ``dataId`` 

1245 argument (and may be used to provide a constraining data ID even 

1246 when the ``dataId`` argument is `None`). 

1247 

1248 Returns 

1249 ------- 

1250 refs : `queries.DatasetQueryResults` 

1251 Dataset references matching the given query criteria. Nested data 

1252 IDs are guaranteed to include values for all implied dimensions 

1253 (i.e. `DataCoordinate.hasFull` will return `True`), but will not 

1254 include dimension records (`DataCoordinate.hasRecords` will be 

1255 `False`) unless `~queries.DatasetQueryResults.expanded` is called 

1256 on the result object (which returns a new one). 

1257 

1258 Raises 

1259 ------ 

1260 TypeError 

1261 Raised when the arguments are incompatible, such as when a 

1262 collection wildcard is passed when ``findFirst`` is `True`, or 

1263 when ``collections`` is `None` and``self.defaults.collections`` is 

1264 also `None`. 

1265 

1266 Notes 

1267 ----- 

1268 When multiple dataset types are queried in a single call, the 

1269 results of this operation are equivalent to querying for each dataset 

1270 type separately in turn, and no information about the relationships 

1271 between datasets of different types is included. In contexts where 

1272 that kind of information is important, the recommended pattern is to 

1273 use `queryDataIds` to first obtain data IDs (possibly with the 

1274 desired dataset types and collections passed as constraints to the 

1275 query), and then use multiple (generally much simpler) calls to 

1276 `queryDatasets` with the returned data IDs passed as constraints. 

1277 """ 

1278 raise NotImplementedError() 

1279 

1280 @abstractmethod 

1281 def queryDataIds(self, dimensions: Union[Iterable[Union[Dimension, str]], Dimension, str], *, 

1282 dataId: Optional[DataId] = None, 

1283 datasets: Any = None, 

1284 collections: Any = None, 

1285 where: Optional[str] = None, 

1286 components: Optional[bool] = None, 

1287 bind: Optional[Mapping[str, Any]] = None, 

1288 check: bool = True, 

1289 **kwargs: Any) -> DataCoordinateIterable: 

1290 """Query for data IDs matching user-provided criteria. 

1291 

1292 Parameters 

1293 ---------- 

1294 dimensions : `Dimension` or `str`, or iterable thereof 

1295 The dimensions of the data IDs to yield, as either `Dimension` 

1296 instances or `str`. Will be automatically expanded to a complete 

1297 `DimensionGraph`. 

1298 dataId : `dict` or `DataCoordinate`, optional 

1299 A data ID whose key-value pairs are used as equality constraints 

1300 in the query. 

1301 datasets : `Any`, optional 

1302 An expression that fully or partially identifies dataset types 

1303 that should constrain the yielded data IDs. For example, including 

1304 "raw" here would constrain the yielded ``instrument``, 

1305 ``exposure``, ``detector``, and ``physical_filter`` values to only 

1306 those for which at least one "raw" dataset exists in 

1307 ``collections``. Allowed types include `DatasetType`, `str`, 

1308 `re.Pattern`, and iterables thereof. Unlike other dataset type 

1309 expressions, ``...`` is not permitted - it doesn't make sense to 

1310 constrain data IDs on the existence of *all* datasets. 

1311 See :ref:`daf_butler_dataset_type_expressions` for more 

1312 information. 

1313 collections: `Any`, optional 

1314 An expression that identifies the collections to search for 

1315 datasets, such as a `str` (for full matches or partial matches 

1316 via globs), `re.Pattern` (for partial matches), or iterable 

1317 thereof. `...` can be used to search all collections (actually 

1318 just all `~CollectionType.RUN` collections, because this will 

1319 still find all datasets). If not provided, 

1320 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1321 is also passed. See :ref:`daf_butler_collection_expressions` for 

1322 more information. 

1323 where : `str`, optional 

1324 A string expression similar to a SQL WHERE clause. May involve 

1325 any column of a dimension table or (as a shortcut for the primary 

1326 key column of a dimension table) dimension name. See 

1327 :ref:`daf_butler_dimension_expressions` for more information. 

1328 components : `bool`, optional 

1329 If `True`, apply all dataset expression patterns to component 

1330 dataset type names as well. If `False`, never apply patterns to 

1331 components. If `None` (default), apply patterns to components only 

1332 if their parent datasets were not matched by the expression. 

1333 Fully-specified component datasets (`str` or `DatasetType` 

1334 instances) are always included. 

1335 bind : `Mapping`, optional 

1336 Mapping containing literal values that should be injected into the 

1337 ``where`` expression, keyed by the identifiers they replace. 

1338 check : `bool`, optional 

1339 If `True` (default) check the query for consistency before 

1340 executing it. This may reject some valid queries that resemble 

1341 common mistakes (e.g. queries for visits without specifying an 

1342 instrument). 

1343 **kwargs 

1344 Additional keyword arguments are forwarded to 

1345 `DataCoordinate.standardize` when processing the ``dataId`` 

1346 argument (and may be used to provide a constraining data ID even 

1347 when the ``dataId`` argument is `None`). 

1348 

1349 Returns 

1350 ------- 

1351 dataIds : `DataCoordinateQueryResults` 

1352 Data IDs matching the given query parameters. These are guaranteed 

1353 to identify all dimensions (`DataCoordinate.hasFull` returns 

1354 `True`), but will not contain `DimensionRecord` objects 

1355 (`DataCoordinate.hasRecords` returns `False`). Call 

1356 `DataCoordinateQueryResults.expanded` on the returned object to 

1357 fetch those (and consider using 

1358 `DataCoordinateQueryResults.materialize` on the returned object 

1359 first if the expected number of rows is very large). See 

1360 documentation for those methods for additional information. 

1361 

1362 Raises 

1363 ------ 

1364 TypeError 

1365 Raised if ``collections`` is `None`, ``self.defaults.collections`` 

1366 is `None`, and ``datasets`` is not `None`. 

1367 """ 

1368 raise NotImplementedError() 

1369 

1370 @abstractmethod 

1371 def queryDimensionRecords(self, element: Union[DimensionElement, str], *, 

1372 dataId: Optional[DataId] = None, 

1373 datasets: Any = None, 

1374 collections: Any = None, 

1375 where: Optional[str] = None, 

1376 components: Optional[bool] = None, 

1377 bind: Optional[Mapping[str, Any]] = None, 

1378 check: bool = True, 

1379 **kwargs: Any) -> Iterator[DimensionRecord]: 

1380 """Query for dimension information matching user-provided criteria. 

1381 

1382 Parameters 

1383 ---------- 

1384 element : `DimensionElement` or `str` 

1385 The dimension element to obtain records for. 

1386 dataId : `dict` or `DataCoordinate`, optional 

1387 A data ID whose key-value pairs are used as equality constraints 

1388 in the query. 

1389 datasets : `Any`, optional 

1390 An expression that fully or partially identifies dataset types 

1391 that should constrain the yielded records. See `queryDataIds` and 

1392 :ref:`daf_butler_dataset_type_expressions` for more information. 

1393 collections: `Any`, optional 

1394 An expression that identifies the collections to search for 

1395 datasets, such as a `str` (for full matches or partial matches 

1396 via globs), `re.Pattern` (for partial matches), or iterable 

1397 thereof. `...` can be used to search all collections (actually 

1398 just all `~CollectionType.RUN` collections, because this will 

1399 still find all datasets). If not provided, 

1400 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1401 is also passed. See :ref:`daf_butler_collection_expressions` for 

1402 more information. 

1403 where : `str`, optional 

1404 A string expression similar to a SQL WHERE clause. See 

1405 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more 

1406 information. 

1407 components : `bool`, optional 

1408 Whether to apply dataset expressions to components as well. 

1409 See `queryDataIds` for more information. 

1410 bind : `Mapping`, optional 

1411 Mapping containing literal values that should be injected into the 

1412 ``where`` expression, keyed by the identifiers they replace. 

1413 check : `bool`, optional 

1414 If `True` (default) check the query for consistency before 

1415 executing it. This may reject some valid queries that resemble 

1416 common mistakes (e.g. queries for visits without specifying an 

1417 instrument). 

1418 **kwargs 

1419 Additional keyword arguments are forwarded to 

1420 `DataCoordinate.standardize` when processing the ``dataId`` 

1421 argument (and may be used to provide a constraining data ID even 

1422 when the ``dataId`` argument is `None`). 

1423 

1424 Returns 

1425 ------- 

1426 dataIds : `DataCoordinateQueryResults` 

1427 Data IDs matching the given query parameters. 

1428 """ 

1429 raise NotImplementedError() 

1430 

1431 @abstractmethod 

1432 def queryDatasetAssociations( 

1433 self, 

1434 datasetType: Union[str, DatasetType], 

1435 collections: Any = ..., 

1436 *, 

1437 collectionTypes: Iterable[CollectionType] = CollectionType.all(), 

1438 flattenChains: bool = False, 

1439 ) -> Iterator[DatasetAssociation]: 

1440 """Iterate over dataset-collection combinations where the dataset is in 

1441 the collection. 

1442 

1443 This method is a temporary placeholder for better support for 

1444 assocation results in `queryDatasets`. It will probably be 

1445 removed in the future, and should be avoided in production code 

1446 whenever possible. 

1447 

1448 Parameters 

1449 ---------- 

1450 datasetType : `DatasetType` or `str` 

1451 A dataset type object or the name of one. 

1452 collections: `Any`, optional 

1453 An expression that identifies the collections to search for 

1454 datasets, such as a `str` (for full matches or partial matches 

1455 via globs), `re.Pattern` (for partial matches), or iterable 

1456 thereof. `...` can be used to search all collections (actually 

1457 just all `~CollectionType.RUN` collections, because this will still 

1458 find all datasets). If not provided, ``self.default.collections`` 

1459 is used. See :ref:`daf_butler_collection_expressions` for more 

1460 information. 

1461 collectionTypes : `AbstractSet` [ `CollectionType` ], optional 

1462 If provided, only yield associations from collections of these 

1463 types. 

1464 flattenChains : `bool`, optional 

1465 If `True` (default) search in the children of 

1466 `~CollectionType.CHAINED` collections. If `False`, ``CHAINED`` 

1467 collections are ignored. 

1468 

1469 Yields 

1470 ------ 

1471 association : `DatasetAssociation` 

1472 Object representing the relationship beween a single dataset and 

1473 a single collection. 

1474 

1475 Raises 

1476 ------ 

1477 TypeError 

1478 Raised if ``collections`` is `None` and 

1479 ``self.defaults.collections`` is `None`. 

1480 """ 

1481 raise NotImplementedError() 

1482 

1483 storageClasses: StorageClassFactory 

1484 """All storage classes known to the registry (`StorageClassFactory`). 

1485 """