Coverage for python/lsst/daf/butler/registry/_registry.py: 60%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

177 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ( 

25 "Registry", 

26) 

27 

28from abc import ABC, abstractmethod 

29import contextlib 

30import logging 

31from typing import ( 

32 Any, 

33 Dict, 

34 Iterable, 

35 Iterator, 

36 List, 

37 Mapping, 

38 Optional, 

39 Tuple, 

40 Type, 

41 TYPE_CHECKING, 

42 Union, 

43) 

44 

45from lsst.utils import doImportType 

46 

47from ..core import ( 

48 ButlerURI, 

49 Config, 

50 DataCoordinate, 

51 DataCoordinateIterable, 

52 DataId, 

53 DatasetAssociation, 

54 DatasetId, 

55 DatasetRef, 

56 DatasetType, 

57 Dimension, 

58 DimensionConfig, 

59 DimensionElement, 

60 DimensionGraph, 

61 DimensionRecord, 

62 DimensionUniverse, 

63 NameLookupMapping, 

64 StorageClassFactory, 

65 Timespan, 

66) 

67 

68from ._config import RegistryConfig 

69from ._collectionType import CollectionType 

70from ._defaults import RegistryDefaults 

71from .interfaces import DatasetIdGenEnum 

72from .wildcards import CollectionSearch 

73from .summaries import CollectionSummary 

74 

75if TYPE_CHECKING: 75 ↛ 76line 75 didn't jump to line 76, because the condition on line 75 was never true

76 from .._butlerConfig import ButlerConfig 

77 from .interfaces import ( 

78 CollectionRecord, 

79 DatastoreRegistryBridgeManager, 

80 ) 

81 

82_LOG = logging.getLogger(__name__) 

83 

84 

85class Registry(ABC): 

86 """Abstract Registry interface. 

87 

88 Each registry implementation can have its own constructor parameters. 

89 The assumption is that an instance of a specific subclass will be 

90 constructed from configuration using `Registry.fromConfig()`. 

91 The base class will look for a ``cls`` entry and call that specific 

92 `fromConfig()` method. 

93 

94 All subclasses should store `RegistryDefaults` in a ``_defaults`` 

95 property. No other properties are assumed shared between implementations. 

96 """ 

97 

98 defaultConfigFile: Optional[str] = None 

99 """Path to configuration defaults. Accessed within the ``configs`` resource 

100 or relative to a search path. Can be None if no defaults specified. 

101 """ 

102 

103 @classmethod 

104 def forceRegistryConfig(cls, config: Optional[Union[ButlerConfig, 

105 RegistryConfig, Config, str]]) -> RegistryConfig: 

106 """Force the supplied config to a `RegistryConfig`. 

107 

108 Parameters 

109 ---------- 

110 config : `RegistryConfig`, `Config` or `str` or `None` 

111 Registry configuration, if missing then default configuration will 

112 be loaded from registry.yaml. 

113 

114 Returns 

115 ------- 

116 registry_config : `RegistryConfig` 

117 A registry config. 

118 """ 

119 if not isinstance(config, RegistryConfig): 

120 if isinstance(config, (str, Config)) or config is None: 

121 config = RegistryConfig(config) 

122 else: 

123 raise ValueError(f"Incompatible Registry configuration: {config}") 

124 return config 

125 

126 @classmethod 

127 def determineTrampoline(cls, 

128 config: Optional[Union[ButlerConfig, 

129 RegistryConfig, 

130 Config, 

131 str]]) -> Tuple[Type[Registry], RegistryConfig]: 

132 """Return class to use to instantiate real registry. 

133 

134 Parameters 

135 ---------- 

136 config : `RegistryConfig` or `str`, optional 

137 Registry configuration, if missing then default configuration will 

138 be loaded from registry.yaml. 

139 

140 Returns 

141 ------- 

142 requested_cls : `type` of `Registry` 

143 The real registry class to use. 

144 registry_config : `RegistryConfig` 

145 The `RegistryConfig` to use. 

146 """ 

147 config = cls.forceRegistryConfig(config) 

148 

149 # Default to the standard registry 

150 registry_cls_name = config.get("cls", "lsst.daf.butler.registries.sql.SqlRegistry") 

151 registry_cls = doImportType(registry_cls_name) 

152 if registry_cls is cls: 

153 raise ValueError("Can not instantiate the abstract base Registry from config") 

154 if not issubclass(registry_cls, Registry): 

155 raise TypeError(f"Registry class obtained from config {registry_cls_name} is " 

156 "not a Registry class.") 

157 return registry_cls, config 

158 

159 @classmethod 

160 def createFromConfig(cls, config: Optional[Union[RegistryConfig, str]] = None, 

161 dimensionConfig: Optional[Union[DimensionConfig, str]] = None, 

162 butlerRoot: Optional[str] = None) -> Registry: 

163 """Create registry database and return `Registry` instance. 

164 

165 This method initializes database contents, database must be empty 

166 prior to calling this method. 

167 

168 Parameters 

169 ---------- 

170 config : `RegistryConfig` or `str`, optional 

171 Registry configuration, if missing then default configuration will 

172 be loaded from registry.yaml. 

173 dimensionConfig : `DimensionConfig` or `str`, optional 

174 Dimensions configuration, if missing then default configuration 

175 will be loaded from dimensions.yaml. 

176 butlerRoot : `str`, optional 

177 Path to the repository root this `Registry` will manage. 

178 

179 Returns 

180 ------- 

181 registry : `Registry` 

182 A new `Registry` instance. 

183 

184 Notes 

185 ----- 

186 This class will determine the concrete `Registry` subclass to 

187 use from configuration. Each subclass should implement this method 

188 even if it can not create a registry. 

189 """ 

190 registry_cls, registry_config = cls.determineTrampoline(config) 

191 return registry_cls.createFromConfig(registry_config, dimensionConfig, butlerRoot) 

192 

193 @classmethod 

194 def fromConfig(cls, config: Union[ButlerConfig, RegistryConfig, Config, str], 

195 butlerRoot: Optional[Union[str, ButlerURI]] = None, writeable: bool = True, 

196 defaults: Optional[RegistryDefaults] = None) -> Registry: 

197 """Create `Registry` subclass instance from `config`. 

198 

199 Registry database must be initialized prior to calling this method. 

200 

201 Parameters 

202 ---------- 

203 config : `ButlerConfig`, `RegistryConfig`, `Config` or `str` 

204 Registry configuration 

205 butlerRoot : `str` or `ButlerURI`, optional 

206 Path to the repository root this `Registry` will manage. 

207 writeable : `bool`, optional 

208 If `True` (default) create a read-write connection to the database. 

209 defaults : `RegistryDefaults`, optional 

210 Default collection search path and/or output `~CollectionType.RUN` 

211 collection. 

212 

213 Returns 

214 ------- 

215 registry : `Registry` (subclass) 

216 A new `Registry` subclass instance. 

217 

218 Notes 

219 ----- 

220 This class will determine the concrete `Registry` subclass to 

221 use from configuration. Each subclass should implement this method. 

222 """ 

223 # The base class implementation should trampoline to the correct 

224 # subclass. No implementation should ever use this implementation 

225 # directly. If no class is specified, default to the standard 

226 # registry. 

227 registry_cls, registry_config = cls.determineTrampoline(config) 

228 return registry_cls.fromConfig(config, butlerRoot, writeable, defaults) 

229 

230 @abstractmethod 

231 def isWriteable(self) -> bool: 

232 """Return `True` if this registry allows write operations, and `False` 

233 otherwise. 

234 """ 

235 raise NotImplementedError() 

236 

237 @abstractmethod 

238 def copy(self, defaults: Optional[RegistryDefaults] = None) -> Registry: 

239 """Create a new `Registry` backed by the same data repository and 

240 connection as this one, but independent defaults. 

241 

242 Parameters 

243 ---------- 

244 defaults : `RegistryDefaults`, optional 

245 Default collections and data ID values for the new registry. If 

246 not provided, ``self.defaults`` will be used (but future changes 

247 to either registry's defaults will not affect the other). 

248 

249 Returns 

250 ------- 

251 copy : `Registry` 

252 A new `Registry` instance with its own defaults. 

253 

254 Notes 

255 ----- 

256 Because the new registry shares a connection with the original, they 

257 also share transaction state (despite the fact that their `transaction` 

258 context manager methods do not reflect this), and must be used with 

259 care. 

260 """ 

261 raise NotImplementedError() 

262 

263 @property 

264 @abstractmethod 

265 def dimensions(self) -> DimensionUniverse: 

266 """All dimensions recognized by this `Registry` (`DimensionUniverse`). 

267 """ 

268 raise NotImplementedError() 

269 

270 @property 

271 def defaults(self) -> RegistryDefaults: 

272 """Default collection search path and/or output `~CollectionType.RUN` 

273 collection (`RegistryDefaults`). 

274 

275 This is an immutable struct whose components may not be set 

276 individually, but the entire struct can be set by assigning to this 

277 property. 

278 """ 

279 return self._defaults 

280 

281 @defaults.setter 

282 def defaults(self, value: RegistryDefaults) -> None: 

283 if value.run is not None: 

284 self.registerRun(value.run) 

285 value.finish(self) 

286 self._defaults = value 

287 

288 @abstractmethod 

289 def refresh(self) -> None: 

290 """Refresh all in-memory state by querying the database. 

291 

292 This may be necessary to enable querying for entities added by other 

293 registry instances after this one was constructed. 

294 """ 

295 raise NotImplementedError() 

296 

297 @contextlib.contextmanager 

298 @abstractmethod 

299 def transaction(self, *, savepoint: bool = False) -> Iterator[None]: 

300 """Return a context manager that represents a transaction. 

301 """ 

302 raise NotImplementedError() 

303 

304 def resetConnectionPool(self) -> None: 

305 """Reset connection pool for registry if relevant. 

306 

307 This operation can be used reset connections to servers when 

308 using registry with fork-based multiprocessing. This method should 

309 usually be called by the child process immediately 

310 after the fork. 

311 

312 The base class implementation is a no-op. 

313 """ 

314 pass 

315 

316 @abstractmethod 

317 def registerCollection(self, name: str, type: CollectionType = CollectionType.TAGGED, 

318 doc: Optional[str] = None) -> bool: 

319 """Add a new collection if one with the given name does not exist. 

320 

321 Parameters 

322 ---------- 

323 name : `str` 

324 The name of the collection to create. 

325 type : `CollectionType` 

326 Enum value indicating the type of collection to create. 

327 doc : `str`, optional 

328 Documentation string for the collection. 

329 

330 Returns 

331 ------- 

332 registered : `bool` 

333 Boolean indicating whether the collection was already registered 

334 or was created by this call. 

335 

336 Notes 

337 ----- 

338 This method cannot be called within transactions, as it needs to be 

339 able to perform its own transaction to be concurrent. 

340 """ 

341 raise NotImplementedError() 

342 

343 @abstractmethod 

344 def getCollectionType(self, name: str) -> CollectionType: 

345 """Return an enumeration value indicating the type of the given 

346 collection. 

347 

348 Parameters 

349 ---------- 

350 name : `str` 

351 The name of the collection. 

352 

353 Returns 

354 ------- 

355 type : `CollectionType` 

356 Enum value indicating the type of this collection. 

357 

358 Raises 

359 ------ 

360 MissingCollectionError 

361 Raised if no collection with the given name exists. 

362 """ 

363 raise NotImplementedError() 

364 

365 @abstractmethod 

366 def _get_collection_record(self, name: str) -> CollectionRecord: 

367 """Return the record for this collection. 

368 

369 Parameters 

370 ---------- 

371 name : `str` 

372 Name of the collection for which the record is to be retrieved. 

373 

374 Returns 

375 ------- 

376 record : `CollectionRecord` 

377 The record for this collection. 

378 """ 

379 raise NotImplementedError() 

380 

381 @abstractmethod 

382 def registerRun(self, name: str, doc: Optional[str] = None) -> bool: 

383 """Add a new run if one with the given name does not exist. 

384 

385 Parameters 

386 ---------- 

387 name : `str` 

388 The name of the run to create. 

389 doc : `str`, optional 

390 Documentation string for the collection. 

391 

392 Returns 

393 ------- 

394 registered : `bool` 

395 Boolean indicating whether a new run was registered. `False` 

396 if it already existed. 

397 

398 Notes 

399 ----- 

400 This method cannot be called within transactions, as it needs to be 

401 able to perform its own transaction to be concurrent. 

402 """ 

403 raise NotImplementedError() 

404 

405 @abstractmethod 

406 def removeCollection(self, name: str) -> None: 

407 """Completely remove the given collection. 

408 

409 Parameters 

410 ---------- 

411 name : `str` 

412 The name of the collection to remove. 

413 

414 Raises 

415 ------ 

416 MissingCollectionError 

417 Raised if no collection with the given name exists. 

418 

419 Notes 

420 ----- 

421 If this is a `~CollectionType.RUN` collection, all datasets and quanta 

422 in it are also fully removed. This requires that those datasets be 

423 removed (or at least trashed) from any datastores that hold them first. 

424 

425 A collection may not be deleted as long as it is referenced by a 

426 `~CollectionType.CHAINED` collection; the ``CHAINED`` collection must 

427 be deleted or redefined first. 

428 """ 

429 raise NotImplementedError() 

430 

431 @abstractmethod 

432 def getCollectionChain(self, parent: str) -> CollectionSearch: 

433 """Return the child collections in a `~CollectionType.CHAINED` 

434 collection. 

435 

436 Parameters 

437 ---------- 

438 parent : `str` 

439 Name of the chained collection. Must have already been added via 

440 a call to `Registry.registerCollection`. 

441 

442 Returns 

443 ------- 

444 children : `CollectionSearch` 

445 An object that defines the search path of the collection. 

446 See :ref:`daf_butler_collection_expressions` for more information. 

447 

448 Raises 

449 ------ 

450 MissingCollectionError 

451 Raised if ``parent`` does not exist in the `Registry`. 

452 TypeError 

453 Raised if ``parent`` does not correspond to a 

454 `~CollectionType.CHAINED` collection. 

455 """ 

456 raise NotImplementedError() 

457 

458 @abstractmethod 

459 def setCollectionChain(self, parent: str, children: Any, *, flatten: bool = False) -> None: 

460 """Define or redefine a `~CollectionType.CHAINED` collection. 

461 

462 Parameters 

463 ---------- 

464 parent : `str` 

465 Name of the chained collection. Must have already been added via 

466 a call to `Registry.registerCollection`. 

467 children : `Any` 

468 An expression defining an ordered search of child collections, 

469 generally an iterable of `str`; see 

470 :ref:`daf_butler_collection_expressions` for more information. 

471 flatten : `bool`, optional 

472 If `True` (`False` is default), recursively flatten out any nested 

473 `~CollectionType.CHAINED` collections in ``children`` first. 

474 

475 Raises 

476 ------ 

477 MissingCollectionError 

478 Raised when any of the given collections do not exist in the 

479 `Registry`. 

480 TypeError 

481 Raised if ``parent`` does not correspond to a 

482 `~CollectionType.CHAINED` collection. 

483 ValueError 

484 Raised if the given collections contains a cycle. 

485 """ 

486 raise NotImplementedError() 

487 

488 @abstractmethod 

489 def getCollectionDocumentation(self, collection: str) -> Optional[str]: 

490 """Retrieve the documentation string for a collection. 

491 

492 Parameters 

493 ---------- 

494 name : `str` 

495 Name of the collection. 

496 

497 Returns 

498 ------- 

499 docs : `str` or `None` 

500 Docstring for the collection with the given name. 

501 """ 

502 raise NotImplementedError() 

503 

504 @abstractmethod 

505 def setCollectionDocumentation(self, collection: str, doc: Optional[str]) -> None: 

506 """Set the documentation string for a collection. 

507 

508 Parameters 

509 ---------- 

510 name : `str` 

511 Name of the collection. 

512 docs : `str` or `None` 

513 Docstring for the collection with the given name; will replace any 

514 existing docstring. Passing `None` will remove any existing 

515 docstring. 

516 """ 

517 raise NotImplementedError() 

518 

519 @abstractmethod 

520 def getCollectionSummary(self, collection: str) -> CollectionSummary: 

521 """Return a summary for the given collection. 

522 

523 Parameters 

524 ---------- 

525 collection : `str` 

526 Name of the collection for which a summary is to be retrieved. 

527 

528 Returns 

529 ------- 

530 summary : `CollectionSummary` 

531 Summary of the dataset types and governor dimension values in 

532 this collection. 

533 """ 

534 raise NotImplementedError() 

535 

536 @abstractmethod 

537 def registerDatasetType(self, datasetType: DatasetType) -> bool: 

538 """ 

539 Add a new `DatasetType` to the Registry. 

540 

541 It is not an error to register the same `DatasetType` twice. 

542 

543 Parameters 

544 ---------- 

545 datasetType : `DatasetType` 

546 The `DatasetType` to be added. 

547 

548 Returns 

549 ------- 

550 inserted : `bool` 

551 `True` if ``datasetType`` was inserted, `False` if an identical 

552 existing `DatsetType` was found. Note that in either case the 

553 DatasetType is guaranteed to be defined in the Registry 

554 consistently with the given definition. 

555 

556 Raises 

557 ------ 

558 ValueError 

559 Raised if the dimensions or storage class are invalid. 

560 ConflictingDefinitionError 

561 Raised if this DatasetType is already registered with a different 

562 definition. 

563 

564 Notes 

565 ----- 

566 This method cannot be called within transactions, as it needs to be 

567 able to perform its own transaction to be concurrent. 

568 """ 

569 raise NotImplementedError() 

570 

571 @abstractmethod 

572 def removeDatasetType(self, name: str) -> None: 

573 """Remove the named `DatasetType` from the registry. 

574 

575 .. warning:: 

576 

577 Registry implementations can cache the dataset type definitions. 

578 This means that deleting the dataset type definition may result in 

579 unexpected behavior from other butler processes that are active 

580 that have not seen the deletion. 

581 

582 Parameters 

583 ---------- 

584 name : `str` 

585 Name of the type to be removed. 

586 

587 Raises 

588 ------ 

589 lsst.daf.butler.registry.OrphanedRecordError 

590 Raised if an attempt is made to remove the dataset type definition 

591 when there are already datasets associated with it. 

592 

593 Notes 

594 ----- 

595 If the dataset type is not registered the method will return without 

596 action. 

597 """ 

598 raise NotImplementedError() 

599 

600 @abstractmethod 

601 def getDatasetType(self, name: str) -> DatasetType: 

602 """Get the `DatasetType`. 

603 

604 Parameters 

605 ---------- 

606 name : `str` 

607 Name of the type. 

608 

609 Returns 

610 ------- 

611 type : `DatasetType` 

612 The `DatasetType` associated with the given name. 

613 

614 Raises 

615 ------ 

616 KeyError 

617 Requested named DatasetType could not be found in registry. 

618 """ 

619 raise NotImplementedError() 

620 

621 @abstractmethod 

622 def supportsIdGenerationMode(self, mode: DatasetIdGenEnum) -> bool: 

623 """Test whether the given dataset ID generation mode is supported by 

624 `insertDatasets`. 

625 

626 Parameters 

627 ---------- 

628 mode : `DatasetIdGenEnum` 

629 Enum value for the mode to test. 

630 

631 Returns 

632 ------- 

633 supported : `bool` 

634 Whether the given mode is supported. 

635 """ 

636 raise NotImplementedError() 

637 

638 @abstractmethod 

639 def findDataset(self, datasetType: Union[DatasetType, str], dataId: Optional[DataId] = None, *, 

640 collections: Any = None, timespan: Optional[Timespan] = None, 

641 **kwargs: Any) -> Optional[DatasetRef]: 

642 """Find a dataset given its `DatasetType` and data ID. 

643 

644 This can be used to obtain a `DatasetRef` that permits the dataset to 

645 be read from a `Datastore`. If the dataset is a component and can not 

646 be found using the provided dataset type, a dataset ref for the parent 

647 will be returned instead but with the correct dataset type. 

648 

649 Parameters 

650 ---------- 

651 datasetType : `DatasetType` or `str` 

652 A `DatasetType` or the name of one. 

653 dataId : `dict` or `DataCoordinate`, optional 

654 A `dict`-like object containing the `Dimension` links that identify 

655 the dataset within a collection. 

656 collections, optional. 

657 An expression that fully or partially identifies the collections to 

658 search for the dataset; see 

659 :ref:`daf_butler_collection_expressions` for more information. 

660 Defaults to ``self.defaults.collections``. 

661 timespan : `Timespan`, optional 

662 A timespan that the validity range of the dataset must overlap. 

663 If not provided, any `~CollectionType.CALIBRATION` collections 

664 matched by the ``collections`` argument will not be searched. 

665 **kwargs 

666 Additional keyword arguments passed to 

667 `DataCoordinate.standardize` to convert ``dataId`` to a true 

668 `DataCoordinate` or augment an existing one. 

669 

670 Returns 

671 ------- 

672 ref : `DatasetRef` 

673 A reference to the dataset, or `None` if no matching Dataset 

674 was found. 

675 

676 Raises 

677 ------ 

678 TypeError 

679 Raised if ``collections`` is `None` and 

680 ``self.defaults.collections`` is `None`. 

681 LookupError 

682 Raised if one or more data ID keys are missing. 

683 KeyError 

684 Raised if the dataset type does not exist. 

685 MissingCollectionError 

686 Raised if any of ``collections`` does not exist in the registry. 

687 

688 Notes 

689 ----- 

690 This method simply returns `None` and does not raise an exception even 

691 when the set of collections searched is intrinsically incompatible with 

692 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but 

693 only `~CollectionType.CALIBRATION` collections are being searched. 

694 This may make it harder to debug some lookup failures, but the behavior 

695 is intentional; we consider it more important that failed searches are 

696 reported consistently, regardless of the reason, and that adding 

697 additional collections that do not contain a match to the search path 

698 never changes the behavior. 

699 """ 

700 raise NotImplementedError() 

701 

702 @abstractmethod 

703 def insertDatasets(self, datasetType: Union[DatasetType, str], dataIds: Iterable[DataId], 

704 run: Optional[str] = None, expand: bool = True, 

705 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE) -> List[DatasetRef]: 

706 """Insert one or more datasets into the `Registry` 

707 

708 This always adds new datasets; to associate existing datasets with 

709 a new collection, use ``associate``. 

710 

711 Parameters 

712 ---------- 

713 datasetType : `DatasetType` or `str` 

714 A `DatasetType` or the name of one. 

715 dataIds : `~collections.abc.Iterable` of `dict` or `DataCoordinate` 

716 Dimension-based identifiers for the new datasets. 

717 run : `str`, optional 

718 The name of the run that produced the datasets. Defaults to 

719 ``self.defaults.run``. 

720 expand : `bool`, optional 

721 If `True` (default), expand data IDs as they are inserted. This is 

722 necessary in general to allow datastore to generate file templates, 

723 but it may be disabled if the caller can guarantee this is 

724 unnecessary. 

725 idGenerationMode : `DatasetIdGenEnum`, optional 

726 Specifies option for generating dataset IDs. By default unique IDs 

727 are generated for each inserted dataset. 

728 

729 Returns 

730 ------- 

731 refs : `list` of `DatasetRef` 

732 Resolved `DatasetRef` instances for all given data IDs (in the same 

733 order). 

734 

735 Raises 

736 ------ 

737 TypeError 

738 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

739 ConflictingDefinitionError 

740 If a dataset with the same dataset type and data ID as one of those 

741 given already exists in ``run``. 

742 MissingCollectionError 

743 Raised if ``run`` does not exist in the registry. 

744 """ 

745 raise NotImplementedError() 

746 

747 @abstractmethod 

748 def _importDatasets(self, datasets: Iterable[DatasetRef], expand: bool = True, 

749 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE, 

750 reuseIds: bool = False) -> List[DatasetRef]: 

751 """Import one or more datasets into the `Registry`. 

752 

753 Difference from `insertDatasets` method is that this method accepts 

754 `DatasetRef` instances which should already be resolved and have a 

755 dataset ID. If registry supports globally-unique dataset IDs (e.g. 

756 `uuid.UUID`) then datasets which already exist in the registry will be 

757 ignored if imported again. 

758 

759 Parameters 

760 ---------- 

761 datasets : `~collections.abc.Iterable` of `DatasetRef` 

762 Datasets to be inserted. All `DatasetRef` instances must have 

763 identical ``datasetType`` and ``run`` attributes. ``run`` 

764 attribute can be `None` and defaults to ``self.defaults.run``. 

765 Datasets can specify ``id`` attribute which will be used for 

766 inserted datasets. All dataset IDs must have the same type 

767 (`int` or `uuid.UUID`), if type of dataset IDs does not match 

768 configured backend then IDs will be ignored and new IDs will be 

769 generated by backend. 

770 expand : `bool`, optional 

771 If `True` (default), expand data IDs as they are inserted. This is 

772 necessary in general to allow datastore to generate file templates, 

773 but it may be disabled if the caller can guarantee this is 

774 unnecessary. 

775 idGenerationMode : `DatasetIdGenEnum`, optional 

776 Specifies option for generating dataset IDs when IDs are not 

777 provided or their type does not match backend type. By default 

778 unique IDs are generated for each inserted dataset. 

779 reuseIds : `bool`, optional 

780 If `True` then forces re-use of imported dataset IDs for integer 

781 IDs which are normally generated as auto-incremented; exception 

782 will be raised if imported IDs clash with existing ones. This 

783 option has no effect on the use of globally-unique IDs which are 

784 always re-used (or generated if integer IDs are being imported). 

785 

786 Returns 

787 ------- 

788 refs : `list` of `DatasetRef` 

789 Resolved `DatasetRef` instances for all given data IDs (in the same 

790 order). If any of ``datasets`` has an ID which already exists in 

791 the database then it will not be inserted or updated, but a 

792 resolved `DatasetRef` will be returned for it in any case. 

793 

794 Raises 

795 ------ 

796 TypeError 

797 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

798 ConflictingDefinitionError 

799 If a dataset with the same dataset type and data ID as one of those 

800 given already exists in ``run``. 

801 MissingCollectionError 

802 Raised if ``run`` does not exist in the registry. 

803 

804 Notes 

805 ----- 

806 This method is considered package-private and internal to Butler 

807 implementation. Clients outside daf_butler package should not use this 

808 method. 

809 """ 

810 raise NotImplementedError() 

811 

812 @abstractmethod 

813 def getDataset(self, id: DatasetId) -> Optional[DatasetRef]: 

814 """Retrieve a Dataset entry. 

815 

816 Parameters 

817 ---------- 

818 id : `DatasetId` 

819 The unique identifier for the dataset. 

820 

821 Returns 

822 ------- 

823 ref : `DatasetRef` or `None` 

824 A ref to the Dataset, or `None` if no matching Dataset 

825 was found. 

826 """ 

827 raise NotImplementedError() 

828 

829 @abstractmethod 

830 def removeDatasets(self, refs: Iterable[DatasetRef]) -> None: 

831 """Remove datasets from the Registry. 

832 

833 The datasets will be removed unconditionally from all collections, and 

834 any `Quantum` that consumed this dataset will instead be marked with 

835 having a NULL input. `Datastore` records will *not* be deleted; the 

836 caller is responsible for ensuring that the dataset has already been 

837 removed from all Datastores. 

838 

839 Parameters 

840 ---------- 

841 refs : `Iterable` of `DatasetRef` 

842 References to the datasets to be removed. Must include a valid 

843 ``id`` attribute, and should be considered invalidated upon return. 

844 

845 Raises 

846 ------ 

847 AmbiguousDatasetError 

848 Raised if any ``ref.id`` is `None`. 

849 OrphanedRecordError 

850 Raised if any dataset is still present in any `Datastore`. 

851 """ 

852 raise NotImplementedError() 

853 

854 @abstractmethod 

855 def associate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

856 """Add existing datasets to a `~CollectionType.TAGGED` collection. 

857 

858 If a DatasetRef with the same exact ID is already in a collection 

859 nothing is changed. If a `DatasetRef` with the same `DatasetType` and 

860 data ID but with different ID exists in the collection, 

861 `ConflictingDefinitionError` is raised. 

862 

863 Parameters 

864 ---------- 

865 collection : `str` 

866 Indicates the collection the datasets should be associated with. 

867 refs : `Iterable` [ `DatasetRef` ] 

868 An iterable of resolved `DatasetRef` instances that already exist 

869 in this `Registry`. 

870 

871 Raises 

872 ------ 

873 ConflictingDefinitionError 

874 If a Dataset with the given `DatasetRef` already exists in the 

875 given collection. 

876 AmbiguousDatasetError 

877 Raised if ``any(ref.id is None for ref in refs)``. 

878 MissingCollectionError 

879 Raised if ``collection`` does not exist in the registry. 

880 TypeError 

881 Raise adding new datasets to the given ``collection`` is not 

882 allowed. 

883 """ 

884 raise NotImplementedError() 

885 

886 @abstractmethod 

887 def disassociate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

888 """Remove existing datasets from a `~CollectionType.TAGGED` collection. 

889 

890 ``collection`` and ``ref`` combinations that are not currently 

891 associated are silently ignored. 

892 

893 Parameters 

894 ---------- 

895 collection : `str` 

896 The collection the datasets should no longer be associated with. 

897 refs : `Iterable` [ `DatasetRef` ] 

898 An iterable of resolved `DatasetRef` instances that already exist 

899 in this `Registry`. 

900 

901 Raises 

902 ------ 

903 AmbiguousDatasetError 

904 Raised if any of the given dataset references is unresolved. 

905 MissingCollectionError 

906 Raised if ``collection`` does not exist in the registry. 

907 TypeError 

908 Raise adding new datasets to the given ``collection`` is not 

909 allowed. 

910 """ 

911 raise NotImplementedError() 

912 

913 @abstractmethod 

914 def certify(self, collection: str, refs: Iterable[DatasetRef], timespan: Timespan) -> None: 

915 """Associate one or more datasets with a calibration collection and a 

916 validity range within it. 

917 

918 Parameters 

919 ---------- 

920 collection : `str` 

921 The name of an already-registered `~CollectionType.CALIBRATION` 

922 collection. 

923 refs : `Iterable` [ `DatasetRef` ] 

924 Datasets to be associated. 

925 timespan : `Timespan` 

926 The validity range for these datasets within the collection. 

927 

928 Raises 

929 ------ 

930 AmbiguousDatasetError 

931 Raised if any of the given `DatasetRef` instances is unresolved. 

932 ConflictingDefinitionError 

933 Raised if the collection already contains a different dataset with 

934 the same `DatasetType` and data ID and an overlapping validity 

935 range. 

936 TypeError 

937 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

938 collection or if one or more datasets are of a dataset type for 

939 which `DatasetType.isCalibration` returns `False`. 

940 """ 

941 raise NotImplementedError() 

942 

943 @abstractmethod 

944 def decertify(self, collection: str, datasetType: Union[str, DatasetType], timespan: Timespan, *, 

945 dataIds: Optional[Iterable[DataId]] = None) -> None: 

946 """Remove or adjust datasets to clear a validity range within a 

947 calibration collection. 

948 

949 Parameters 

950 ---------- 

951 collection : `str` 

952 The name of an already-registered `~CollectionType.CALIBRATION` 

953 collection. 

954 datasetType : `str` or `DatasetType` 

955 Name or `DatasetType` instance for the datasets to be decertified. 

956 timespan : `Timespan`, optional 

957 The validity range to remove datasets from within the collection. 

958 Datasets that overlap this range but are not contained by it will 

959 have their validity ranges adjusted to not overlap it, which may 

960 split a single dataset validity range into two. 

961 dataIds : `Iterable` [ `DataId` ], optional 

962 Data IDs that should be decertified within the given validity range 

963 If `None`, all data IDs for ``self.datasetType`` will be 

964 decertified. 

965 

966 Raises 

967 ------ 

968 TypeError 

969 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

970 collection or if ``datasetType.isCalibration() is False``. 

971 """ 

972 raise NotImplementedError() 

973 

974 @abstractmethod 

975 def getDatastoreBridgeManager(self) -> DatastoreRegistryBridgeManager: 

976 """Return an object that allows a new `Datastore` instance to 

977 communicate with this `Registry`. 

978 

979 Returns 

980 ------- 

981 manager : `DatastoreRegistryBridgeManager` 

982 Object that mediates communication between this `Registry` and its 

983 associated datastores. 

984 """ 

985 raise NotImplementedError() 

986 

987 @abstractmethod 

988 def getDatasetLocations(self, ref: DatasetRef) -> Iterable[str]: 

989 """Retrieve datastore locations for a given dataset. 

990 

991 Parameters 

992 ---------- 

993 ref : `DatasetRef` 

994 A reference to the dataset for which to retrieve storage 

995 information. 

996 

997 Returns 

998 ------- 

999 datastores : `Iterable` [ `str` ] 

1000 All the matching datastores holding this dataset. 

1001 

1002 Raises 

1003 ------ 

1004 AmbiguousDatasetError 

1005 Raised if ``ref.id`` is `None`. 

1006 """ 

1007 raise NotImplementedError() 

1008 

1009 @abstractmethod 

1010 def expandDataId(self, dataId: Optional[DataId] = None, *, graph: Optional[DimensionGraph] = None, 

1011 records: Optional[NameLookupMapping[DimensionElement, Optional[DimensionRecord]]] = None, 

1012 withDefaults: bool = True, 

1013 **kwargs: Any) -> DataCoordinate: 

1014 """Expand a dimension-based data ID to include additional information. 

1015 

1016 Parameters 

1017 ---------- 

1018 dataId : `DataCoordinate` or `dict`, optional 

1019 Data ID to be expanded; augmented and overridden by ``kwargs``. 

1020 graph : `DimensionGraph`, optional 

1021 Set of dimensions for the expanded ID. If `None`, the dimensions 

1022 will be inferred from the keys of ``dataId`` and ``kwargs``. 

1023 Dimensions that are in ``dataId`` or ``kwargs`` but not in 

1024 ``graph`` are silently ignored, providing a way to extract and 

1025 ``graph`` expand a subset of a data ID. 

1026 records : `Mapping` [`str`, `DimensionRecord`], optional 

1027 Dimension record data to use before querying the database for that 

1028 data, keyed by element name. 

1029 withDefaults : `bool`, optional 

1030 Utilize ``self.defaults.dataId`` to fill in missing governor 

1031 dimension key-value pairs. Defaults to `True` (i.e. defaults are 

1032 used). 

1033 **kwargs 

1034 Additional keywords are treated like additional key-value pairs for 

1035 ``dataId``, extending and overriding 

1036 

1037 Returns 

1038 ------- 

1039 expanded : `DataCoordinate` 

1040 A data ID that includes full metadata for all of the dimensions it 

1041 identifieds, i.e. guarantees that ``expanded.hasRecords()`` and 

1042 ``expanded.hasFull()`` both return `True`. 

1043 """ 

1044 raise NotImplementedError() 

1045 

1046 @abstractmethod 

1047 def insertDimensionData(self, element: Union[DimensionElement, str], 

1048 *data: Union[Mapping[str, Any], DimensionRecord], 

1049 conform: bool = True, 

1050 replace: bool = False) -> None: 

1051 """Insert one or more dimension records into the database. 

1052 

1053 Parameters 

1054 ---------- 

1055 element : `DimensionElement` or `str` 

1056 The `DimensionElement` or name thereof that identifies the table 

1057 records will be inserted into. 

1058 data : `dict` or `DimensionRecord` (variadic) 

1059 One or more records to insert. 

1060 conform : `bool`, optional 

1061 If `False` (`True` is default) perform no checking or conversions, 

1062 and assume that ``element`` is a `DimensionElement` instance and 

1063 ``data`` is a one or more `DimensionRecord` instances of the 

1064 appropriate subclass. 

1065 replace: `bool`, optional 

1066 If `True` (`False` is default), replace existing records in the 

1067 database if there is a conflict. 

1068 """ 

1069 raise NotImplementedError() 

1070 

1071 @abstractmethod 

1072 def syncDimensionData(self, element: Union[DimensionElement, str], 

1073 row: Union[Mapping[str, Any], DimensionRecord], 

1074 conform: bool = True, 

1075 update: bool = False) -> Union[bool, Dict[str, Any]]: 

1076 """Synchronize the given dimension record with the database, inserting 

1077 if it does not already exist and comparing values if it does. 

1078 

1079 Parameters 

1080 ---------- 

1081 element : `DimensionElement` or `str` 

1082 The `DimensionElement` or name thereof that identifies the table 

1083 records will be inserted into. 

1084 row : `dict` or `DimensionRecord` 

1085 The record to insert. 

1086 conform : `bool`, optional 

1087 If `False` (`True` is default) perform no checking or conversions, 

1088 and assume that ``element`` is a `DimensionElement` instance and 

1089 ``data`` is a one or more `DimensionRecord` instances of the 

1090 appropriate subclass. 

1091 update: `bool`, optional 

1092 If `True` (`False` is default), update the existing record in the 

1093 database if there is a conflict. 

1094 

1095 Returns 

1096 ------- 

1097 inserted_or_updated : `bool` or `dict` 

1098 `True` if a new row was inserted, `False` if no changes were 

1099 needed, or a `dict` mapping updated column names to their old 

1100 values if an update was performed (only possible if 

1101 ``update=True``). 

1102 

1103 Raises 

1104 ------ 

1105 ConflictingDefinitionError 

1106 Raised if the record exists in the database (according to primary 

1107 key lookup) but is inconsistent with the given one. 

1108 """ 

1109 raise NotImplementedError() 

1110 

1111 @abstractmethod 

1112 def queryDatasetTypes(self, expression: Any = ..., *, components: Optional[bool] = None, 

1113 missing: Optional[List[str]] = None, 

1114 ) -> Iterator[DatasetType]: 

1115 """Iterate over the dataset types whose names match an expression. 

1116 

1117 Parameters 

1118 ---------- 

1119 expression : `Any`, optional 

1120 An expression that fully or partially identifies the dataset types 

1121 to return, such as a `str`, `re.Pattern`, or iterable thereof. 

1122 `...` can be used to return all dataset types, and is the default. 

1123 See :ref:`daf_butler_dataset_type_expressions` for more 

1124 information. 

1125 components : `bool`, optional 

1126 If `True`, apply all expression patterns to component dataset type 

1127 names as well. If `False`, never apply patterns to components. 

1128 If `None` (default), apply patterns to components only if their 

1129 parent datasets were not matched by the expression. 

1130 Fully-specified component datasets (`str` or `DatasetType` 

1131 instances) are always included. 

1132 missing : `list` of `str`, optional 

1133 String dataset type names that were explicitly given (i.e. not 

1134 regular expression patterns) but not found will be appended to this 

1135 list, if it is provided. 

1136 

1137 Yields 

1138 ------ 

1139 datasetType : `DatasetType` 

1140 A `DatasetType` instance whose name matches ``expression``. 

1141 """ 

1142 raise NotImplementedError() 

1143 

1144 @abstractmethod 

1145 def queryCollections(self, expression: Any = ..., 

1146 datasetType: Optional[DatasetType] = None, 

1147 collectionTypes: Iterable[CollectionType] = CollectionType.all(), 

1148 flattenChains: bool = False, 

1149 includeChains: Optional[bool] = None) -> Iterator[str]: 

1150 """Iterate over the collections whose names match an expression. 

1151 

1152 Parameters 

1153 ---------- 

1154 expression : `Any`, optional 

1155 An expression that identifies the collections to return, such as 

1156 a `str` (for full matches or partial matches via globs), 

1157 `re.Pattern` (for partial matches), or iterable thereof. `...` 

1158 can be used to return all collections, and is the default. 

1159 See :ref:`daf_butler_collection_expressions` for more information. 

1160 datasetType : `DatasetType`, optional 

1161 If provided, only yield collections that may contain datasets of 

1162 this type. This is a conservative approximation in general; it may 

1163 yield collections that do not have any such datasets. 

1164 collectionTypes : `AbstractSet` [ `CollectionType` ], optional 

1165 If provided, only yield collections of these types. 

1166 flattenChains : `bool`, optional 

1167 If `True` (`False` is default), recursively yield the child 

1168 collections of matching `~CollectionType.CHAINED` collections. 

1169 includeChains : `bool`, optional 

1170 If `True`, yield records for matching `~CollectionType.CHAINED` 

1171 collections. Default is the opposite of ``flattenChains``: include 

1172 either CHAINED collections or their children, but not both. 

1173 

1174 Yields 

1175 ------ 

1176 collection : `str` 

1177 The name of a collection that matches ``expression``. 

1178 """ 

1179 raise NotImplementedError() 

1180 

1181 @abstractmethod 

1182 def queryDatasets(self, datasetType: Any, *, 

1183 collections: Any = None, 

1184 dimensions: Optional[Iterable[Union[Dimension, str]]] = None, 

1185 dataId: Optional[DataId] = None, 

1186 where: Optional[str] = None, 

1187 findFirst: bool = False, 

1188 components: Optional[bool] = None, 

1189 bind: Optional[Mapping[str, Any]] = None, 

1190 check: bool = True, 

1191 **kwargs: Any) -> Iterable[DatasetRef]: 

1192 """Query for and iterate over dataset references matching user-provided 

1193 criteria. 

1194 

1195 Parameters 

1196 ---------- 

1197 datasetType 

1198 An expression that fully or partially identifies the dataset types 

1199 to be queried. Allowed types include `DatasetType`, `str`, 

1200 `re.Pattern`, and iterables thereof. The special value `...` can 

1201 be used to query all dataset types. See 

1202 :ref:`daf_butler_dataset_type_expressions` for more information. 

1203 collections: optional 

1204 An expression that identifies the collections to search, such as a 

1205 `str` (for full matches or partial matches via globs), `re.Pattern` 

1206 (for partial matches), or iterable thereof. `...` can be used to 

1207 search all collections (actually just all `~CollectionType.RUN` 

1208 collections, because this will still find all datasets). 

1209 If not provided, ``self.default.collections`` is used. See 

1210 :ref:`daf_butler_collection_expressions` for more information. 

1211 dimensions : `~collections.abc.Iterable` of `Dimension` or `str` 

1212 Dimensions to include in the query (in addition to those used 

1213 to identify the queried dataset type(s)), either to constrain 

1214 the resulting datasets to those for which a matching dimension 

1215 exists, or to relate the dataset type's dimensions to dimensions 

1216 referenced by the ``dataId`` or ``where`` arguments. 

1217 dataId : `dict` or `DataCoordinate`, optional 

1218 A data ID whose key-value pairs are used as equality constraints 

1219 in the query. 

1220 where : `str`, optional 

1221 A string expression similar to a SQL WHERE clause. May involve 

1222 any column of a dimension table or (as a shortcut for the primary 

1223 key column of a dimension table) dimension name. See 

1224 :ref:`daf_butler_dimension_expressions` for more information. 

1225 findFirst : `bool`, optional 

1226 If `True` (`False` is default), for each result data ID, only 

1227 yield one `DatasetRef` of each `DatasetType`, from the first 

1228 collection in which a dataset of that dataset type appears 

1229 (according to the order of ``collections`` passed in). If `True`, 

1230 ``collections`` must not contain regular expressions and may not 

1231 be `...`. 

1232 components : `bool`, optional 

1233 If `True`, apply all dataset expression patterns to component 

1234 dataset type names as well. If `False`, never apply patterns to 

1235 components. If `None` (default), apply patterns to components only 

1236 if their parent datasets were not matched by the expression. 

1237 Fully-specified component datasets (`str` or `DatasetType` 

1238 instances) are always included. 

1239 bind : `Mapping`, optional 

1240 Mapping containing literal values that should be injected into the 

1241 ``where`` expression, keyed by the identifiers they replace. 

1242 check : `bool`, optional 

1243 If `True` (default) check the query for consistency before 

1244 executing it. This may reject some valid queries that resemble 

1245 common mistakes (e.g. queries for visits without specifying an 

1246 instrument). 

1247 **kwargs 

1248 Additional keyword arguments are forwarded to 

1249 `DataCoordinate.standardize` when processing the ``dataId`` 

1250 argument (and may be used to provide a constraining data ID even 

1251 when the ``dataId`` argument is `None`). 

1252 

1253 Returns 

1254 ------- 

1255 refs : `queries.DatasetQueryResults` 

1256 Dataset references matching the given query criteria. Nested data 

1257 IDs are guaranteed to include values for all implied dimensions 

1258 (i.e. `DataCoordinate.hasFull` will return `True`), but will not 

1259 include dimension records (`DataCoordinate.hasRecords` will be 

1260 `False`) unless `~queries.DatasetQueryResults.expanded` is called 

1261 on the result object (which returns a new one). 

1262 

1263 Raises 

1264 ------ 

1265 TypeError 

1266 Raised when the arguments are incompatible, such as when a 

1267 collection wildcard is passed when ``findFirst`` is `True`, or 

1268 when ``collections`` is `None` and``self.defaults.collections`` is 

1269 also `None`. 

1270 

1271 Notes 

1272 ----- 

1273 When multiple dataset types are queried in a single call, the 

1274 results of this operation are equivalent to querying for each dataset 

1275 type separately in turn, and no information about the relationships 

1276 between datasets of different types is included. In contexts where 

1277 that kind of information is important, the recommended pattern is to 

1278 use `queryDataIds` to first obtain data IDs (possibly with the 

1279 desired dataset types and collections passed as constraints to the 

1280 query), and then use multiple (generally much simpler) calls to 

1281 `queryDatasets` with the returned data IDs passed as constraints. 

1282 """ 

1283 raise NotImplementedError() 

1284 

1285 @abstractmethod 

1286 def queryDataIds(self, dimensions: Union[Iterable[Union[Dimension, str]], Dimension, str], *, 

1287 dataId: Optional[DataId] = None, 

1288 datasets: Any = None, 

1289 collections: Any = None, 

1290 where: Optional[str] = None, 

1291 components: Optional[bool] = None, 

1292 bind: Optional[Mapping[str, Any]] = None, 

1293 check: bool = True, 

1294 **kwargs: Any) -> DataCoordinateIterable: 

1295 """Query for data IDs matching user-provided criteria. 

1296 

1297 Parameters 

1298 ---------- 

1299 dimensions : `Dimension` or `str`, or iterable thereof 

1300 The dimensions of the data IDs to yield, as either `Dimension` 

1301 instances or `str`. Will be automatically expanded to a complete 

1302 `DimensionGraph`. 

1303 dataId : `dict` or `DataCoordinate`, optional 

1304 A data ID whose key-value pairs are used as equality constraints 

1305 in the query. 

1306 datasets : `Any`, optional 

1307 An expression that fully or partially identifies dataset types 

1308 that should constrain the yielded data IDs. For example, including 

1309 "raw" here would constrain the yielded ``instrument``, 

1310 ``exposure``, ``detector``, and ``physical_filter`` values to only 

1311 those for which at least one "raw" dataset exists in 

1312 ``collections``. Allowed types include `DatasetType`, `str`, 

1313 `re.Pattern`, and iterables thereof. Unlike other dataset type 

1314 expressions, ``...`` is not permitted - it doesn't make sense to 

1315 constrain data IDs on the existence of *all* datasets. 

1316 See :ref:`daf_butler_dataset_type_expressions` for more 

1317 information. 

1318 collections: `Any`, optional 

1319 An expression that identifies the collections to search for 

1320 datasets, such as a `str` (for full matches or partial matches 

1321 via globs), `re.Pattern` (for partial matches), or iterable 

1322 thereof. `...` can be used to search all collections (actually 

1323 just all `~CollectionType.RUN` collections, because this will 

1324 still find all datasets). If not provided, 

1325 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1326 is also passed. See :ref:`daf_butler_collection_expressions` for 

1327 more information. 

1328 where : `str`, optional 

1329 A string expression similar to a SQL WHERE clause. May involve 

1330 any column of a dimension table or (as a shortcut for the primary 

1331 key column of a dimension table) dimension name. See 

1332 :ref:`daf_butler_dimension_expressions` for more information. 

1333 components : `bool`, optional 

1334 If `True`, apply all dataset expression patterns to component 

1335 dataset type names as well. If `False`, never apply patterns to 

1336 components. If `None` (default), apply patterns to components only 

1337 if their parent datasets were not matched by the expression. 

1338 Fully-specified component datasets (`str` or `DatasetType` 

1339 instances) are always included. 

1340 bind : `Mapping`, optional 

1341 Mapping containing literal values that should be injected into the 

1342 ``where`` expression, keyed by the identifiers they replace. 

1343 check : `bool`, optional 

1344 If `True` (default) check the query for consistency before 

1345 executing it. This may reject some valid queries that resemble 

1346 common mistakes (e.g. queries for visits without specifying an 

1347 instrument). 

1348 **kwargs 

1349 Additional keyword arguments are forwarded to 

1350 `DataCoordinate.standardize` when processing the ``dataId`` 

1351 argument (and may be used to provide a constraining data ID even 

1352 when the ``dataId`` argument is `None`). 

1353 

1354 Returns 

1355 ------- 

1356 dataIds : `DataCoordinateQueryResults` 

1357 Data IDs matching the given query parameters. These are guaranteed 

1358 to identify all dimensions (`DataCoordinate.hasFull` returns 

1359 `True`), but will not contain `DimensionRecord` objects 

1360 (`DataCoordinate.hasRecords` returns `False`). Call 

1361 `DataCoordinateQueryResults.expanded` on the returned object to 

1362 fetch those (and consider using 

1363 `DataCoordinateQueryResults.materialize` on the returned object 

1364 first if the expected number of rows is very large). See 

1365 documentation for those methods for additional information. 

1366 

1367 Raises 

1368 ------ 

1369 TypeError 

1370 Raised if ``collections`` is `None`, ``self.defaults.collections`` 

1371 is `None`, and ``datasets`` is not `None`. 

1372 """ 

1373 raise NotImplementedError() 

1374 

1375 @abstractmethod 

1376 def queryDimensionRecords(self, element: Union[DimensionElement, str], *, 

1377 dataId: Optional[DataId] = None, 

1378 datasets: Any = None, 

1379 collections: Any = None, 

1380 where: Optional[str] = None, 

1381 components: Optional[bool] = None, 

1382 bind: Optional[Mapping[str, Any]] = None, 

1383 check: bool = True, 

1384 **kwargs: Any) -> Iterator[DimensionRecord]: 

1385 """Query for dimension information matching user-provided criteria. 

1386 

1387 Parameters 

1388 ---------- 

1389 element : `DimensionElement` or `str` 

1390 The dimension element to obtain records for. 

1391 dataId : `dict` or `DataCoordinate`, optional 

1392 A data ID whose key-value pairs are used as equality constraints 

1393 in the query. 

1394 datasets : `Any`, optional 

1395 An expression that fully or partially identifies dataset types 

1396 that should constrain the yielded records. See `queryDataIds` and 

1397 :ref:`daf_butler_dataset_type_expressions` for more information. 

1398 collections: `Any`, optional 

1399 An expression that identifies the collections to search for 

1400 datasets, such as a `str` (for full matches or partial matches 

1401 via globs), `re.Pattern` (for partial matches), or iterable 

1402 thereof. `...` can be used to search all collections (actually 

1403 just all `~CollectionType.RUN` collections, because this will 

1404 still find all datasets). If not provided, 

1405 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1406 is also passed. See :ref:`daf_butler_collection_expressions` for 

1407 more information. 

1408 where : `str`, optional 

1409 A string expression similar to a SQL WHERE clause. See 

1410 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more 

1411 information. 

1412 components : `bool`, optional 

1413 Whether to apply dataset expressions to components as well. 

1414 See `queryDataIds` for more information. 

1415 bind : `Mapping`, optional 

1416 Mapping containing literal values that should be injected into the 

1417 ``where`` expression, keyed by the identifiers they replace. 

1418 check : `bool`, optional 

1419 If `True` (default) check the query for consistency before 

1420 executing it. This may reject some valid queries that resemble 

1421 common mistakes (e.g. queries for visits without specifying an 

1422 instrument). 

1423 **kwargs 

1424 Additional keyword arguments are forwarded to 

1425 `DataCoordinate.standardize` when processing the ``dataId`` 

1426 argument (and may be used to provide a constraining data ID even 

1427 when the ``dataId`` argument is `None`). 

1428 

1429 Returns 

1430 ------- 

1431 dataIds : `DataCoordinateQueryResults` 

1432 Data IDs matching the given query parameters. 

1433 """ 

1434 raise NotImplementedError() 

1435 

1436 @abstractmethod 

1437 def queryDatasetAssociations( 

1438 self, 

1439 datasetType: Union[str, DatasetType], 

1440 collections: Any = ..., 

1441 *, 

1442 collectionTypes: Iterable[CollectionType] = CollectionType.all(), 

1443 flattenChains: bool = False, 

1444 ) -> Iterator[DatasetAssociation]: 

1445 """Iterate over dataset-collection combinations where the dataset is in 

1446 the collection. 

1447 

1448 This method is a temporary placeholder for better support for 

1449 assocation results in `queryDatasets`. It will probably be 

1450 removed in the future, and should be avoided in production code 

1451 whenever possible. 

1452 

1453 Parameters 

1454 ---------- 

1455 datasetType : `DatasetType` or `str` 

1456 A dataset type object or the name of one. 

1457 collections: `Any`, optional 

1458 An expression that identifies the collections to search for 

1459 datasets, such as a `str` (for full matches or partial matches 

1460 via globs), `re.Pattern` (for partial matches), or iterable 

1461 thereof. `...` can be used to search all collections (actually 

1462 just all `~CollectionType.RUN` collections, because this will still 

1463 find all datasets). If not provided, ``self.default.collections`` 

1464 is used. See :ref:`daf_butler_collection_expressions` for more 

1465 information. 

1466 collectionTypes : `AbstractSet` [ `CollectionType` ], optional 

1467 If provided, only yield associations from collections of these 

1468 types. 

1469 flattenChains : `bool`, optional 

1470 If `True` (default) search in the children of 

1471 `~CollectionType.CHAINED` collections. If `False`, ``CHAINED`` 

1472 collections are ignored. 

1473 

1474 Yields 

1475 ------ 

1476 association : `DatasetAssociation` 

1477 Object representing the relationship beween a single dataset and 

1478 a single collection. 

1479 

1480 Raises 

1481 ------ 

1482 TypeError 

1483 Raised if ``collections`` is `None` and 

1484 ``self.defaults.collections`` is `None`. 

1485 """ 

1486 raise NotImplementedError() 

1487 

1488 storageClasses: StorageClassFactory 

1489 """All storage classes known to the registry (`StorageClassFactory`). 

1490 """