Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ( 

25 "Registry", 

26) 

27 

28from abc import ABC, abstractmethod 

29import contextlib 

30import logging 

31from typing import ( 

32 Any, 

33 Iterable, 

34 Iterator, 

35 List, 

36 Mapping, 

37 Optional, 

38 Tuple, 

39 Type, 

40 TYPE_CHECKING, 

41 Union, 

42) 

43 

44from lsst.utils import doImport 

45 

46from ..core import ( 

47 ButlerURI, 

48 Config, 

49 DataCoordinate, 

50 DataCoordinateIterable, 

51 DataId, 

52 DatasetAssociation, 

53 DatasetId, 

54 DatasetRef, 

55 DatasetType, 

56 Dimension, 

57 DimensionConfig, 

58 DimensionElement, 

59 DimensionGraph, 

60 DimensionRecord, 

61 DimensionUniverse, 

62 NameLookupMapping, 

63 StorageClassFactory, 

64 Timespan, 

65) 

66 

67from ._config import RegistryConfig 

68from ._collectionType import CollectionType 

69from ._defaults import RegistryDefaults 

70from .interfaces import DatasetIdGenEnum 

71from .wildcards import CollectionSearch 

72from .summaries import CollectionSummary 

73 

74if TYPE_CHECKING: 74 ↛ 75line 74 didn't jump to line 75, because the condition on line 74 was never true

75 from .._butlerConfig import ButlerConfig 

76 from .interfaces import ( 

77 CollectionRecord, 

78 DatastoreRegistryBridgeManager, 

79 ) 

80 

81_LOG = logging.getLogger(__name__) 

82 

83 

84class Registry(ABC): 

85 """Abstract Registry interface. 

86 

87 Each registry implementation can have its own constructor parameters. 

88 The assumption is that an instance of a specific subclass will be 

89 constructed from configuration using `Registry.fromConfig()`. 

90 The base class will look for a ``cls`` entry and call that specific 

91 `fromConfig()` method. 

92 

93 All subclasses should store `RegistryDefaults` in a ``_defaults`` 

94 property. No other properties are assumed shared between implementations. 

95 """ 

96 

97 defaultConfigFile: Optional[str] = None 

98 """Path to configuration defaults. Accessed within the ``configs`` resource 

99 or relative to a search path. Can be None if no defaults specified. 

100 """ 

101 

102 @classmethod 

103 def forceRegistryConfig(cls, config: Optional[Union[ButlerConfig, 

104 RegistryConfig, Config, str]]) -> RegistryConfig: 

105 """Force the supplied config to a `RegistryConfig`. 

106 

107 Parameters 

108 ---------- 

109 config : `RegistryConfig`, `Config` or `str` or `None` 

110 Registry configuration, if missing then default configuration will 

111 be loaded from registry.yaml. 

112 

113 Returns 

114 ------- 

115 registry_config : `RegistryConfig` 

116 A registry config. 

117 """ 

118 if not isinstance(config, RegistryConfig): 

119 if isinstance(config, (str, Config)) or config is None: 

120 config = RegistryConfig(config) 

121 else: 

122 raise ValueError(f"Incompatible Registry configuration: {config}") 

123 return config 

124 

125 @classmethod 

126 def determineTrampoline(cls, 

127 config: Optional[Union[ButlerConfig, 

128 RegistryConfig, 

129 Config, 

130 str]]) -> Tuple[Type[Registry], RegistryConfig]: 

131 """Return class to use to instantiate real registry. 

132 

133 Parameters 

134 ---------- 

135 config : `RegistryConfig` or `str`, optional 

136 Registry configuration, if missing then default configuration will 

137 be loaded from registry.yaml. 

138 

139 Returns 

140 ------- 

141 requested_cls : `type` of `Registry` 

142 The real registry class to use. 

143 registry_config : `RegistryConfig` 

144 The `RegistryConfig` to use. 

145 """ 

146 config = cls.forceRegistryConfig(config) 

147 

148 # Default to the standard registry 

149 registry_cls = doImport(config.get("cls", "lsst.daf.butler.registries.sql.SqlRegistry")) 

150 if registry_cls is cls: 

151 raise ValueError("Can not instantiate the abstract base Registry from config") 

152 return registry_cls, config 

153 

154 @classmethod 

155 def createFromConfig(cls, config: Optional[Union[RegistryConfig, str]] = None, 

156 dimensionConfig: Optional[Union[DimensionConfig, str]] = None, 

157 butlerRoot: Optional[str] = None) -> Registry: 

158 """Create registry database and return `Registry` instance. 

159 

160 This method initializes database contents, database must be empty 

161 prior to calling this method. 

162 

163 Parameters 

164 ---------- 

165 config : `RegistryConfig` or `str`, optional 

166 Registry configuration, if missing then default configuration will 

167 be loaded from registry.yaml. 

168 dimensionConfig : `DimensionConfig` or `str`, optional 

169 Dimensions configuration, if missing then default configuration 

170 will be loaded from dimensions.yaml. 

171 butlerRoot : `str`, optional 

172 Path to the repository root this `Registry` will manage. 

173 

174 Returns 

175 ------- 

176 registry : `Registry` 

177 A new `Registry` instance. 

178 

179 Notes 

180 ----- 

181 This class will determine the concrete `Registry` subclass to 

182 use from configuration. Each subclass should implement this method 

183 even if it can not create a registry. 

184 """ 

185 registry_cls, registry_config = cls.determineTrampoline(config) 

186 return registry_cls.createFromConfig(registry_config, dimensionConfig, butlerRoot) 

187 

188 @classmethod 

189 def fromConfig(cls, config: Union[ButlerConfig, RegistryConfig, Config, str], 

190 butlerRoot: Optional[Union[str, ButlerURI]] = None, writeable: bool = True, 

191 defaults: Optional[RegistryDefaults] = None) -> Registry: 

192 """Create `Registry` subclass instance from `config`. 

193 

194 Registry database must be initialized prior to calling this method. 

195 

196 Parameters 

197 ---------- 

198 config : `ButlerConfig`, `RegistryConfig`, `Config` or `str` 

199 Registry configuration 

200 butlerRoot : `str` or `ButlerURI`, optional 

201 Path to the repository root this `Registry` will manage. 

202 writeable : `bool`, optional 

203 If `True` (default) create a read-write connection to the database. 

204 defaults : `RegistryDefaults`, optional 

205 Default collection search path and/or output `~CollectionType.RUN` 

206 collection. 

207 

208 Returns 

209 ------- 

210 registry : `Registry` (subclass) 

211 A new `Registry` subclass instance. 

212 

213 Notes 

214 ----- 

215 This class will determine the concrete `Registry` subclass to 

216 use from configuration. Each subclass should implement this method. 

217 """ 

218 # The base class implementation should trampoline to the correct 

219 # subclass. No implementation should ever use this implementation 

220 # directly. If no class is specified, default to the standard 

221 # registry. 

222 registry_cls, registry_config = cls.determineTrampoline(config) 

223 return registry_cls.fromConfig(config, butlerRoot, writeable, defaults) 

224 

225 @abstractmethod 

226 def isWriteable(self) -> bool: 

227 """Return `True` if this registry allows write operations, and `False` 

228 otherwise. 

229 """ 

230 raise NotImplementedError() 

231 

232 @abstractmethod 

233 def copy(self, defaults: Optional[RegistryDefaults] = None) -> Registry: 

234 """Create a new `Registry` backed by the same data repository and 

235 connection as this one, but independent defaults. 

236 

237 Parameters 

238 ---------- 

239 defaults : `RegistryDefaults`, optional 

240 Default collections and data ID values for the new registry. If 

241 not provided, ``self.defaults`` will be used (but future changes 

242 to either registry's defaults will not affect the other). 

243 

244 Returns 

245 ------- 

246 copy : `Registry` 

247 A new `Registry` instance with its own defaults. 

248 

249 Notes 

250 ----- 

251 Because the new registry shares a connection with the original, they 

252 also share transaction state (despite the fact that their `transaction` 

253 context manager methods do not reflect this), and must be used with 

254 care. 

255 """ 

256 raise NotImplementedError() 

257 

258 @property 

259 @abstractmethod 

260 def dimensions(self) -> DimensionUniverse: 

261 """All dimensions recognized by this `Registry` (`DimensionUniverse`). 

262 """ 

263 raise NotImplementedError() 

264 

265 @property 

266 def defaults(self) -> RegistryDefaults: 

267 """Default collection search path and/or output `~CollectionType.RUN` 

268 collection (`RegistryDefaults`). 

269 

270 This is an immutable struct whose components may not be set 

271 individually, but the entire struct can be set by assigning to this 

272 property. 

273 """ 

274 return self._defaults 

275 

276 @defaults.setter 

277 def defaults(self, value: RegistryDefaults) -> None: 

278 if value.run is not None: 

279 self.registerRun(value.run) 

280 value.finish(self) 

281 self._defaults = value 

282 

283 @abstractmethod 

284 def refresh(self) -> None: 

285 """Refresh all in-memory state by querying the database. 

286 

287 This may be necessary to enable querying for entities added by other 

288 registry instances after this one was constructed. 

289 """ 

290 raise NotImplementedError() 

291 

292 @contextlib.contextmanager 

293 @abstractmethod 

294 def transaction(self, *, savepoint: bool = False) -> Iterator[None]: 

295 """Return a context manager that represents a transaction. 

296 """ 

297 raise NotImplementedError() 

298 

299 def resetConnectionPool(self) -> None: 

300 """Reset connection pool for registry if relevant. 

301 

302 This operation can be used reset connections to servers when 

303 using registry with fork-based multiprocessing. This method should 

304 usually be called by the child process immediately 

305 after the fork. 

306 

307 The base class implementation is a no-op. 

308 """ 

309 pass 

310 

311 @abstractmethod 

312 def registerCollection(self, name: str, type: CollectionType = CollectionType.TAGGED, 

313 doc: Optional[str] = None) -> None: 

314 """Add a new collection if one with the given name does not exist. 

315 

316 Parameters 

317 ---------- 

318 name : `str` 

319 The name of the collection to create. 

320 type : `CollectionType` 

321 Enum value indicating the type of collection to create. 

322 doc : `str`, optional 

323 Documentation string for the collection. 

324 

325 Notes 

326 ----- 

327 This method cannot be called within transactions, as it needs to be 

328 able to perform its own transaction to be concurrent. 

329 """ 

330 raise NotImplementedError() 

331 

332 @abstractmethod 

333 def getCollectionType(self, name: str) -> CollectionType: 

334 """Return an enumeration value indicating the type of the given 

335 collection. 

336 

337 Parameters 

338 ---------- 

339 name : `str` 

340 The name of the collection. 

341 

342 Returns 

343 ------- 

344 type : `CollectionType` 

345 Enum value indicating the type of this collection. 

346 

347 Raises 

348 ------ 

349 MissingCollectionError 

350 Raised if no collection with the given name exists. 

351 """ 

352 raise NotImplementedError() 

353 

354 @abstractmethod 

355 def _get_collection_record(self, name: str) -> CollectionRecord: 

356 """Return the record for this collection. 

357 

358 Parameters 

359 ---------- 

360 name : `str` 

361 Name of the collection for which the record is to be retrieved. 

362 

363 Returns 

364 ------- 

365 record : `CollectionRecord` 

366 The record for this collection. 

367 """ 

368 raise NotImplementedError() 

369 

370 @abstractmethod 

371 def registerRun(self, name: str, doc: Optional[str] = None) -> None: 

372 """Add a new run if one with the given name does not exist. 

373 

374 Parameters 

375 ---------- 

376 name : `str` 

377 The name of the run to create. 

378 doc : `str`, optional 

379 Documentation string for the collection. 

380 

381 Notes 

382 ----- 

383 This method cannot be called within transactions, as it needs to be 

384 able to perform its own transaction to be concurrent. 

385 """ 

386 raise NotImplementedError() 

387 

388 @abstractmethod 

389 def removeCollection(self, name: str) -> None: 

390 """Completely remove the given collection. 

391 

392 Parameters 

393 ---------- 

394 name : `str` 

395 The name of the collection to remove. 

396 

397 Raises 

398 ------ 

399 MissingCollectionError 

400 Raised if no collection with the given name exists. 

401 

402 Notes 

403 ----- 

404 If this is a `~CollectionType.RUN` collection, all datasets and quanta 

405 in it are also fully removed. This requires that those datasets be 

406 removed (or at least trashed) from any datastores that hold them first. 

407 

408 A collection may not be deleted as long as it is referenced by a 

409 `~CollectionType.CHAINED` collection; the ``CHAINED`` collection must 

410 be deleted or redefined first. 

411 """ 

412 raise NotImplementedError() 

413 

414 @abstractmethod 

415 def getCollectionChain(self, parent: str) -> CollectionSearch: 

416 """Return the child collections in a `~CollectionType.CHAINED` 

417 collection. 

418 

419 Parameters 

420 ---------- 

421 parent : `str` 

422 Name of the chained collection. Must have already been added via 

423 a call to `Registry.registerCollection`. 

424 

425 Returns 

426 ------- 

427 children : `CollectionSearch` 

428 An object that defines the search path of the collection. 

429 See :ref:`daf_butler_collection_expressions` for more information. 

430 

431 Raises 

432 ------ 

433 MissingCollectionError 

434 Raised if ``parent`` does not exist in the `Registry`. 

435 TypeError 

436 Raised if ``parent`` does not correspond to a 

437 `~CollectionType.CHAINED` collection. 

438 """ 

439 raise NotImplementedError() 

440 

441 @abstractmethod 

442 def setCollectionChain(self, parent: str, children: Any, *, flatten: bool = False) -> None: 

443 """Define or redefine a `~CollectionType.CHAINED` collection. 

444 

445 Parameters 

446 ---------- 

447 parent : `str` 

448 Name of the chained collection. Must have already been added via 

449 a call to `Registry.registerCollection`. 

450 children : `Any` 

451 An expression defining an ordered search of child collections, 

452 generally an iterable of `str`; see 

453 :ref:`daf_butler_collection_expressions` for more information. 

454 flatten : `bool`, optional 

455 If `True` (`False` is default), recursively flatten out any nested 

456 `~CollectionType.CHAINED` collections in ``children`` first. 

457 

458 Raises 

459 ------ 

460 MissingCollectionError 

461 Raised when any of the given collections do not exist in the 

462 `Registry`. 

463 TypeError 

464 Raised if ``parent`` does not correspond to a 

465 `~CollectionType.CHAINED` collection. 

466 ValueError 

467 Raised if the given collections contains a cycle. 

468 """ 

469 raise NotImplementedError() 

470 

471 @abstractmethod 

472 def getCollectionDocumentation(self, collection: str) -> Optional[str]: 

473 """Retrieve the documentation string for a collection. 

474 

475 Parameters 

476 ---------- 

477 name : `str` 

478 Name of the collection. 

479 

480 Returns 

481 ------- 

482 docs : `str` or `None` 

483 Docstring for the collection with the given name. 

484 """ 

485 raise NotImplementedError() 

486 

487 @abstractmethod 

488 def setCollectionDocumentation(self, collection: str, doc: Optional[str]) -> None: 

489 """Set the documentation string for a collection. 

490 

491 Parameters 

492 ---------- 

493 name : `str` 

494 Name of the collection. 

495 docs : `str` or `None` 

496 Docstring for the collection with the given name; will replace any 

497 existing docstring. Passing `None` will remove any existing 

498 docstring. 

499 """ 

500 raise NotImplementedError() 

501 

502 @abstractmethod 

503 def getCollectionSummary(self, collection: str) -> CollectionSummary: 

504 """Return a summary for the given collection. 

505 

506 Parameters 

507 ---------- 

508 collection : `str` 

509 Name of the collection for which a summary is to be retrieved. 

510 

511 Returns 

512 ------- 

513 summary : `CollectionSummary` 

514 Summary of the dataset types and governor dimension values in 

515 this collection. 

516 """ 

517 raise NotImplementedError() 

518 

519 @abstractmethod 

520 def registerDatasetType(self, datasetType: DatasetType) -> bool: 

521 """ 

522 Add a new `DatasetType` to the Registry. 

523 

524 It is not an error to register the same `DatasetType` twice. 

525 

526 Parameters 

527 ---------- 

528 datasetType : `DatasetType` 

529 The `DatasetType` to be added. 

530 

531 Returns 

532 ------- 

533 inserted : `bool` 

534 `True` if ``datasetType`` was inserted, `False` if an identical 

535 existing `DatsetType` was found. Note that in either case the 

536 DatasetType is guaranteed to be defined in the Registry 

537 consistently with the given definition. 

538 

539 Raises 

540 ------ 

541 ValueError 

542 Raised if the dimensions or storage class are invalid. 

543 ConflictingDefinitionError 

544 Raised if this DatasetType is already registered with a different 

545 definition. 

546 

547 Notes 

548 ----- 

549 This method cannot be called within transactions, as it needs to be 

550 able to perform its own transaction to be concurrent. 

551 """ 

552 raise NotImplementedError() 

553 

554 @abstractmethod 

555 def removeDatasetType(self, name: str) -> None: 

556 """Remove the named `DatasetType` from the registry. 

557 

558 .. warning:: 

559 

560 Registry implementations can cache the dataset type definitions. 

561 This means that deleting the dataset type definition may result in 

562 unexpected behavior from other butler processes that are active 

563 that have not seen the deletion. 

564 

565 Parameters 

566 ---------- 

567 name : `str` 

568 Name of the type to be removed. 

569 

570 Raises 

571 ------ 

572 lsst.daf.butler.registry.OrphanedRecordError 

573 Raised if an attempt is made to remove the dataset type definition 

574 when there are already datasets associated with it. 

575 

576 Notes 

577 ----- 

578 If the dataset type is not registered the method will return without 

579 action. 

580 """ 

581 raise NotImplementedError() 

582 

583 @abstractmethod 

584 def getDatasetType(self, name: str) -> DatasetType: 

585 """Get the `DatasetType`. 

586 

587 Parameters 

588 ---------- 

589 name : `str` 

590 Name of the type. 

591 

592 Returns 

593 ------- 

594 type : `DatasetType` 

595 The `DatasetType` associated with the given name. 

596 

597 Raises 

598 ------ 

599 KeyError 

600 Requested named DatasetType could not be found in registry. 

601 """ 

602 raise NotImplementedError() 

603 

604 @abstractmethod 

605 def findDataset(self, datasetType: Union[DatasetType, str], dataId: Optional[DataId] = None, *, 

606 collections: Any = None, timespan: Optional[Timespan] = None, 

607 **kwargs: Any) -> Optional[DatasetRef]: 

608 """Find a dataset given its `DatasetType` and data ID. 

609 

610 This can be used to obtain a `DatasetRef` that permits the dataset to 

611 be read from a `Datastore`. If the dataset is a component and can not 

612 be found using the provided dataset type, a dataset ref for the parent 

613 will be returned instead but with the correct dataset type. 

614 

615 Parameters 

616 ---------- 

617 datasetType : `DatasetType` or `str` 

618 A `DatasetType` or the name of one. 

619 dataId : `dict` or `DataCoordinate`, optional 

620 A `dict`-like object containing the `Dimension` links that identify 

621 the dataset within a collection. 

622 collections, optional. 

623 An expression that fully or partially identifies the collections to 

624 search for the dataset; see 

625 :ref:`daf_butler_collection_expressions` for more information. 

626 Defaults to ``self.defaults.collections``. 

627 timespan : `Timespan`, optional 

628 A timespan that the validity range of the dataset must overlap. 

629 If not provided, any `~CollectionType.CALIBRATION` collections 

630 matched by the ``collections`` argument will not be searched. 

631 **kwargs 

632 Additional keyword arguments passed to 

633 `DataCoordinate.standardize` to convert ``dataId`` to a true 

634 `DataCoordinate` or augment an existing one. 

635 

636 Returns 

637 ------- 

638 ref : `DatasetRef` 

639 A reference to the dataset, or `None` if no matching Dataset 

640 was found. 

641 

642 Raises 

643 ------ 

644 TypeError 

645 Raised if ``collections`` is `None` and 

646 ``self.defaults.collections`` is `None`. 

647 LookupError 

648 Raised if one or more data ID keys are missing. 

649 KeyError 

650 Raised if the dataset type does not exist. 

651 MissingCollectionError 

652 Raised if any of ``collections`` does not exist in the registry. 

653 

654 Notes 

655 ----- 

656 This method simply returns `None` and does not raise an exception even 

657 when the set of collections searched is intrinsically incompatible with 

658 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but 

659 only `~CollectionType.CALIBRATION` collections are being searched. 

660 This may make it harder to debug some lookup failures, but the behavior 

661 is intentional; we consider it more important that failed searches are 

662 reported consistently, regardless of the reason, and that adding 

663 additional collections that do not contain a match to the search path 

664 never changes the behavior. 

665 """ 

666 raise NotImplementedError() 

667 

668 @abstractmethod 

669 def insertDatasets(self, datasetType: Union[DatasetType, str], dataIds: Iterable[DataId], 

670 run: Optional[str] = None, expand: bool = True, 

671 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE) -> List[DatasetRef]: 

672 """Insert one or more datasets into the `Registry` 

673 

674 This always adds new datasets; to associate existing datasets with 

675 a new collection, use ``associate``. 

676 

677 Parameters 

678 ---------- 

679 datasetType : `DatasetType` or `str` 

680 A `DatasetType` or the name of one. 

681 dataIds : `~collections.abc.Iterable` of `dict` or `DataCoordinate` 

682 Dimension-based identifiers for the new datasets. 

683 run : `str`, optional 

684 The name of the run that produced the datasets. Defaults to 

685 ``self.defaults.run``. 

686 expand : `bool`, optional 

687 If `True` (default), expand data IDs as they are inserted. This is 

688 necessary in general to allow datastore to generate file templates, 

689 but it may be disabled if the caller can guarantee this is 

690 unnecessary. 

691 idGenerationMode : `DatasetIdGenEnum`, optional 

692 Specifies option for generating dataset IDs. By default unique IDs 

693 are generated for each inserted dataset. 

694 

695 Returns 

696 ------- 

697 refs : `list` of `DatasetRef` 

698 Resolved `DatasetRef` instances for all given data IDs (in the same 

699 order). 

700 

701 Raises 

702 ------ 

703 TypeError 

704 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

705 ConflictingDefinitionError 

706 If a dataset with the same dataset type and data ID as one of those 

707 given already exists in ``run``. 

708 MissingCollectionError 

709 Raised if ``run`` does not exist in the registry. 

710 """ 

711 raise NotImplementedError() 

712 

713 @abstractmethod 

714 def _importDatasets(self, datasets: Iterable[DatasetRef], expand: bool = True, 

715 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE, 

716 reuseIds: bool = False) -> List[DatasetRef]: 

717 """Import one or more datasets into the `Registry`. 

718 

719 Difference from `insertDatasets` method is that this method accepts 

720 `DatasetRef` instances which should already be resolved and have a 

721 dataset ID. If registry supports globally-unique dataset IDs (e.g. 

722 `uuid.UUID`) then datasets which already exist in the registry will be 

723 ignored if imported again. 

724 

725 Parameters 

726 ---------- 

727 datasets : `~collections.abc.Iterable` of `DatasetRef` 

728 Datasets to be inserted. All `DatasetRef` instances must have 

729 identical ``datasetType`` and ``run`` attributes. ``run`` 

730 attribute can be `None` and defaults to ``self.defaults.run``. 

731 Datasets can specify ``id`` attribute which will be used for 

732 inserted datasets. All dataset IDs must have the same type 

733 (`int` or `uuid.UUID`), if type of dataset IDs does not match 

734 configured backend then IDs will be ignored and new IDs will be 

735 generated by backend. 

736 expand : `bool`, optional 

737 If `True` (default), expand data IDs as they are inserted. This is 

738 necessary in general to allow datastore to generate file templates, 

739 but it may be disabled if the caller can guarantee this is 

740 unnecessary. 

741 idGenerationMode : `DatasetIdGenEnum`, optional 

742 Specifies option for generating dataset IDs when IDs are not 

743 provided or their type does not match backend type. By default 

744 unique IDs are generated for each inserted dataset. 

745 reuseIds : `bool`, optional 

746 If `True` then forces re-use of imported dataset IDs for integer 

747 IDs which are normally generated as auto-incremented; exception 

748 will be raised if imported IDs clash with existing ones. This 

749 option has no effect on the use of globally-unique IDs which are 

750 always re-used (or generated if integer IDs are being imported). 

751 

752 Returns 

753 ------- 

754 refs : `list` of `DatasetRef` 

755 Resolved `DatasetRef` instances for all given data IDs (in the same 

756 order). If any of ``datasets`` has an ID which already exists in 

757 the database then it will not be inserted or updated, but a 

758 resolved `DatasetRef` will be returned for it in any case. 

759 

760 Raises 

761 ------ 

762 TypeError 

763 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

764 ConflictingDefinitionError 

765 If a dataset with the same dataset type and data ID as one of those 

766 given already exists in ``run``. 

767 MissingCollectionError 

768 Raised if ``run`` does not exist in the registry. 

769 

770 Notes 

771 ----- 

772 This method is considered package-private and internal to Butler 

773 implementation. Clients outside daf_butler package should not use this 

774 method. 

775 """ 

776 raise NotImplementedError() 

777 

778 @abstractmethod 

779 def getDataset(self, id: DatasetId) -> Optional[DatasetRef]: 

780 """Retrieve a Dataset entry. 

781 

782 Parameters 

783 ---------- 

784 id : `DatasetId` 

785 The unique identifier for the dataset. 

786 

787 Returns 

788 ------- 

789 ref : `DatasetRef` or `None` 

790 A ref to the Dataset, or `None` if no matching Dataset 

791 was found. 

792 """ 

793 raise NotImplementedError() 

794 

795 @abstractmethod 

796 def removeDatasets(self, refs: Iterable[DatasetRef]) -> None: 

797 """Remove datasets from the Registry. 

798 

799 The datasets will be removed unconditionally from all collections, and 

800 any `Quantum` that consumed this dataset will instead be marked with 

801 having a NULL input. `Datastore` records will *not* be deleted; the 

802 caller is responsible for ensuring that the dataset has already been 

803 removed from all Datastores. 

804 

805 Parameters 

806 ---------- 

807 refs : `Iterable` of `DatasetRef` 

808 References to the datasets to be removed. Must include a valid 

809 ``id`` attribute, and should be considered invalidated upon return. 

810 

811 Raises 

812 ------ 

813 AmbiguousDatasetError 

814 Raised if any ``ref.id`` is `None`. 

815 OrphanedRecordError 

816 Raised if any dataset is still present in any `Datastore`. 

817 """ 

818 raise NotImplementedError() 

819 

820 @abstractmethod 

821 def associate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

822 """Add existing datasets to a `~CollectionType.TAGGED` collection. 

823 

824 If a DatasetRef with the same exact ID is already in a collection 

825 nothing is changed. If a `DatasetRef` with the same `DatasetType` and 

826 data ID but with different ID exists in the collection, 

827 `ConflictingDefinitionError` is raised. 

828 

829 Parameters 

830 ---------- 

831 collection : `str` 

832 Indicates the collection the datasets should be associated with. 

833 refs : `Iterable` [ `DatasetRef` ] 

834 An iterable of resolved `DatasetRef` instances that already exist 

835 in this `Registry`. 

836 

837 Raises 

838 ------ 

839 ConflictingDefinitionError 

840 If a Dataset with the given `DatasetRef` already exists in the 

841 given collection. 

842 AmbiguousDatasetError 

843 Raised if ``any(ref.id is None for ref in refs)``. 

844 MissingCollectionError 

845 Raised if ``collection`` does not exist in the registry. 

846 TypeError 

847 Raise adding new datasets to the given ``collection`` is not 

848 allowed. 

849 """ 

850 raise NotImplementedError() 

851 

852 @abstractmethod 

853 def disassociate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

854 """Remove existing datasets from a `~CollectionType.TAGGED` collection. 

855 

856 ``collection`` and ``ref`` combinations that are not currently 

857 associated are silently ignored. 

858 

859 Parameters 

860 ---------- 

861 collection : `str` 

862 The collection the datasets should no longer be associated with. 

863 refs : `Iterable` [ `DatasetRef` ] 

864 An iterable of resolved `DatasetRef` instances that already exist 

865 in this `Registry`. 

866 

867 Raises 

868 ------ 

869 AmbiguousDatasetError 

870 Raised if any of the given dataset references is unresolved. 

871 MissingCollectionError 

872 Raised if ``collection`` does not exist in the registry. 

873 TypeError 

874 Raise adding new datasets to the given ``collection`` is not 

875 allowed. 

876 """ 

877 raise NotImplementedError() 

878 

879 @abstractmethod 

880 def certify(self, collection: str, refs: Iterable[DatasetRef], timespan: Timespan) -> None: 

881 """Associate one or more datasets with a calibration collection and a 

882 validity range within it. 

883 

884 Parameters 

885 ---------- 

886 collection : `str` 

887 The name of an already-registered `~CollectionType.CALIBRATION` 

888 collection. 

889 refs : `Iterable` [ `DatasetRef` ] 

890 Datasets to be associated. 

891 timespan : `Timespan` 

892 The validity range for these datasets within the collection. 

893 

894 Raises 

895 ------ 

896 AmbiguousDatasetError 

897 Raised if any of the given `DatasetRef` instances is unresolved. 

898 ConflictingDefinitionError 

899 Raised if the collection already contains a different dataset with 

900 the same `DatasetType` and data ID and an overlapping validity 

901 range. 

902 TypeError 

903 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

904 collection or if one or more datasets are of a dataset type for 

905 which `DatasetType.isCalibration` returns `False`. 

906 """ 

907 raise NotImplementedError() 

908 

909 @abstractmethod 

910 def decertify(self, collection: str, datasetType: Union[str, DatasetType], timespan: Timespan, *, 

911 dataIds: Optional[Iterable[DataId]] = None) -> None: 

912 """Remove or adjust datasets to clear a validity range within a 

913 calibration collection. 

914 

915 Parameters 

916 ---------- 

917 collection : `str` 

918 The name of an already-registered `~CollectionType.CALIBRATION` 

919 collection. 

920 datasetType : `str` or `DatasetType` 

921 Name or `DatasetType` instance for the datasets to be decertified. 

922 timespan : `Timespan`, optional 

923 The validity range to remove datasets from within the collection. 

924 Datasets that overlap this range but are not contained by it will 

925 have their validity ranges adjusted to not overlap it, which may 

926 split a single dataset validity range into two. 

927 dataIds : `Iterable` [ `DataId` ], optional 

928 Data IDs that should be decertified within the given validity range 

929 If `None`, all data IDs for ``self.datasetType`` will be 

930 decertified. 

931 

932 Raises 

933 ------ 

934 TypeError 

935 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

936 collection or if ``datasetType.isCalibration() is False``. 

937 """ 

938 raise NotImplementedError() 

939 

940 @abstractmethod 

941 def getDatastoreBridgeManager(self) -> DatastoreRegistryBridgeManager: 

942 """Return an object that allows a new `Datastore` instance to 

943 communicate with this `Registry`. 

944 

945 Returns 

946 ------- 

947 manager : `DatastoreRegistryBridgeManager` 

948 Object that mediates communication between this `Registry` and its 

949 associated datastores. 

950 """ 

951 raise NotImplementedError() 

952 

953 @abstractmethod 

954 def getDatasetLocations(self, ref: DatasetRef) -> Iterable[str]: 

955 """Retrieve datastore locations for a given dataset. 

956 

957 Parameters 

958 ---------- 

959 ref : `DatasetRef` 

960 A reference to the dataset for which to retrieve storage 

961 information. 

962 

963 Returns 

964 ------- 

965 datastores : `Iterable` [ `str` ] 

966 All the matching datastores holding this dataset. 

967 

968 Raises 

969 ------ 

970 AmbiguousDatasetError 

971 Raised if ``ref.id`` is `None`. 

972 """ 

973 raise NotImplementedError() 

974 

975 @abstractmethod 

976 def expandDataId(self, dataId: Optional[DataId] = None, *, graph: Optional[DimensionGraph] = None, 

977 records: Optional[NameLookupMapping[DimensionElement, Optional[DimensionRecord]]] = None, 

978 withDefaults: bool = True, 

979 **kwargs: Any) -> DataCoordinate: 

980 """Expand a dimension-based data ID to include additional information. 

981 

982 Parameters 

983 ---------- 

984 dataId : `DataCoordinate` or `dict`, optional 

985 Data ID to be expanded; augmented and overridden by ``kwds``. 

986 graph : `DimensionGraph`, optional 

987 Set of dimensions for the expanded ID. If `None`, the dimensions 

988 will be inferred from the keys of ``dataId`` and ``kwds``. 

989 Dimensions that are in ``dataId`` or ``kwds`` but not in ``graph`` 

990 are silently ignored, providing a way to extract and expand a 

991 subset of a data ID. 

992 records : `Mapping` [`str`, `DimensionRecord`], optional 

993 Dimension record data to use before querying the database for that 

994 data, keyed by element name. 

995 withDefaults : `bool`, optional 

996 Utilize ``self.defaults.dataId`` to fill in missing governor 

997 dimension key-value pairs. Defaults to `True` (i.e. defaults are 

998 used). 

999 **kwargs 

1000 Additional keywords are treated like additional key-value pairs for 

1001 ``dataId``, extending and overriding 

1002 

1003 Returns 

1004 ------- 

1005 expanded : `DataCoordinate` 

1006 A data ID that includes full metadata for all of the dimensions it 

1007 identifieds, i.e. guarantees that ``expanded.hasRecords()`` and 

1008 ``expanded.hasFull()`` both return `True`. 

1009 """ 

1010 raise NotImplementedError() 

1011 

1012 @abstractmethod 

1013 def insertDimensionData(self, element: Union[DimensionElement, str], 

1014 *data: Union[Mapping[str, Any], DimensionRecord], 

1015 conform: bool = True) -> None: 

1016 """Insert one or more dimension records into the database. 

1017 

1018 Parameters 

1019 ---------- 

1020 element : `DimensionElement` or `str` 

1021 The `DimensionElement` or name thereof that identifies the table 

1022 records will be inserted into. 

1023 data : `dict` or `DimensionRecord` (variadic) 

1024 One or more records to insert. 

1025 conform : `bool`, optional 

1026 If `False` (`True` is default) perform no checking or conversions, 

1027 and assume that ``element`` is a `DimensionElement` instance and 

1028 ``data`` is a one or more `DimensionRecord` instances of the 

1029 appropriate subclass. 

1030 """ 

1031 raise NotImplementedError() 

1032 

1033 @abstractmethod 

1034 def syncDimensionData(self, element: Union[DimensionElement, str], 

1035 row: Union[Mapping[str, Any], DimensionRecord], 

1036 conform: bool = True) -> bool: 

1037 """Synchronize the given dimension record with the database, inserting 

1038 if it does not already exist and comparing values if it does. 

1039 

1040 Parameters 

1041 ---------- 

1042 element : `DimensionElement` or `str` 

1043 The `DimensionElement` or name thereof that identifies the table 

1044 records will be inserted into. 

1045 row : `dict` or `DimensionRecord` 

1046 The record to insert. 

1047 conform : `bool`, optional 

1048 If `False` (`True` is default) perform no checking or conversions, 

1049 and assume that ``element`` is a `DimensionElement` instance and 

1050 ``data`` is a one or more `DimensionRecord` instances of the 

1051 appropriate subclass. 

1052 

1053 Returns 

1054 ------- 

1055 inserted : `bool` 

1056 `True` if a new row was inserted, `False` otherwise. 

1057 

1058 Raises 

1059 ------ 

1060 ConflictingDefinitionError 

1061 Raised if the record exists in the database (according to primary 

1062 key lookup) but is inconsistent with the given one. 

1063 """ 

1064 raise NotImplementedError() 

1065 

1066 @abstractmethod 

1067 def queryDatasetTypes(self, expression: Any = ..., *, components: Optional[bool] = None 

1068 ) -> Iterator[DatasetType]: 

1069 """Iterate over the dataset types whose names match an expression. 

1070 

1071 Parameters 

1072 ---------- 

1073 expression : `Any`, optional 

1074 An expression that fully or partially identifies the dataset types 

1075 to return, such as a `str`, `re.Pattern`, or iterable thereof. 

1076 `...` can be used to return all dataset types, and is the default. 

1077 See :ref:`daf_butler_dataset_type_expressions` for more 

1078 information. 

1079 components : `bool`, optional 

1080 If `True`, apply all expression patterns to component dataset type 

1081 names as well. If `False`, never apply patterns to components. 

1082 If `None` (default), apply patterns to components only if their 

1083 parent datasets were not matched by the expression. 

1084 Fully-specified component datasets (`str` or `DatasetType` 

1085 instances) are always included. 

1086 

1087 Yields 

1088 ------ 

1089 datasetType : `DatasetType` 

1090 A `DatasetType` instance whose name matches ``expression``. 

1091 """ 

1092 raise NotImplementedError() 

1093 

1094 @abstractmethod 

1095 def queryCollections(self, expression: Any = ..., 

1096 datasetType: Optional[DatasetType] = None, 

1097 collectionTypes: Iterable[CollectionType] = CollectionType.all(), 

1098 flattenChains: bool = False, 

1099 includeChains: Optional[bool] = None) -> Iterator[str]: 

1100 """Iterate over the collections whose names match an expression. 

1101 

1102 Parameters 

1103 ---------- 

1104 expression : `Any`, optional 

1105 An expression that identifies the collections to return, such as 

1106 a `str` (for full matches or partial matches via globs), 

1107 `re.Pattern` (for partial matches), or iterable thereof. `...` 

1108 can be used to return all collections, and is the default. 

1109 See :ref:`daf_butler_collection_expressions` for more information. 

1110 datasetType : `DatasetType`, optional 

1111 If provided, only yield collections that may contain datasets of 

1112 this type. This is a conservative approximation in general; it may 

1113 yield collections that do not have any such datasets. 

1114 collectionTypes : `AbstractSet` [ `CollectionType` ], optional 

1115 If provided, only yield collections of these types. 

1116 flattenChains : `bool`, optional 

1117 If `True` (`False` is default), recursively yield the child 

1118 collections of matching `~CollectionType.CHAINED` collections. 

1119 includeChains : `bool`, optional 

1120 If `True`, yield records for matching `~CollectionType.CHAINED` 

1121 collections. Default is the opposite of ``flattenChains``: include 

1122 either CHAINED collections or their children, but not both. 

1123 

1124 Yields 

1125 ------ 

1126 collection : `str` 

1127 The name of a collection that matches ``expression``. 

1128 """ 

1129 raise NotImplementedError() 

1130 

1131 @abstractmethod 

1132 def queryDatasets(self, datasetType: Any, *, 

1133 collections: Any = None, 

1134 dimensions: Optional[Iterable[Union[Dimension, str]]] = None, 

1135 dataId: Optional[DataId] = None, 

1136 where: Optional[str] = None, 

1137 findFirst: bool = False, 

1138 components: Optional[bool] = None, 

1139 bind: Optional[Mapping[str, Any]] = None, 

1140 check: bool = True, 

1141 **kwargs: Any) -> Iterable[DatasetRef]: 

1142 """Query for and iterate over dataset references matching user-provided 

1143 criteria. 

1144 

1145 Parameters 

1146 ---------- 

1147 datasetType 

1148 An expression that fully or partially identifies the dataset types 

1149 to be queried. Allowed types include `DatasetType`, `str`, 

1150 `re.Pattern`, and iterables thereof. The special value `...` can 

1151 be used to query all dataset types. See 

1152 :ref:`daf_butler_dataset_type_expressions` for more information. 

1153 collections: optional 

1154 An expression that identifies the collections to search, such as a 

1155 `str` (for full matches or partial matches via globs), `re.Pattern` 

1156 (for partial matches), or iterable thereof. `...` can be used to 

1157 search all collections (actually just all `~CollectionType.RUN` 

1158 collections, because this will still find all datasets). 

1159 If not provided, ``self.default.collections`` is used. See 

1160 :ref:`daf_butler_collection_expressions` for more information. 

1161 dimensions : `~collections.abc.Iterable` of `Dimension` or `str` 

1162 Dimensions to include in the query (in addition to those used 

1163 to identify the queried dataset type(s)), either to constrain 

1164 the resulting datasets to those for which a matching dimension 

1165 exists, or to relate the dataset type's dimensions to dimensions 

1166 referenced by the ``dataId`` or ``where`` arguments. 

1167 dataId : `dict` or `DataCoordinate`, optional 

1168 A data ID whose key-value pairs are used as equality constraints 

1169 in the query. 

1170 where : `str`, optional 

1171 A string expression similar to a SQL WHERE clause. May involve 

1172 any column of a dimension table or (as a shortcut for the primary 

1173 key column of a dimension table) dimension name. See 

1174 :ref:`daf_butler_dimension_expressions` for more information. 

1175 findFirst : `bool`, optional 

1176 If `True` (`False` is default), for each result data ID, only 

1177 yield one `DatasetRef` of each `DatasetType`, from the first 

1178 collection in which a dataset of that dataset type appears 

1179 (according to the order of ``collections`` passed in). If `True`, 

1180 ``collections`` must not contain regular expressions and may not 

1181 be `...`. 

1182 components : `bool`, optional 

1183 If `True`, apply all dataset expression patterns to component 

1184 dataset type names as well. If `False`, never apply patterns to 

1185 components. If `None` (default), apply patterns to components only 

1186 if their parent datasets were not matched by the expression. 

1187 Fully-specified component datasets (`str` or `DatasetType` 

1188 instances) are always included. 

1189 bind : `Mapping`, optional 

1190 Mapping containing literal values that should be injected into the 

1191 ``where`` expression, keyed by the identifiers they replace. 

1192 check : `bool`, optional 

1193 If `True` (default) check the query for consistency before 

1194 executing it. This may reject some valid queries that resemble 

1195 common mistakes (e.g. queries for visits without specifying an 

1196 instrument). 

1197 **kwargs 

1198 Additional keyword arguments are forwarded to 

1199 `DataCoordinate.standardize` when processing the ``dataId`` 

1200 argument (and may be used to provide a constraining data ID even 

1201 when the ``dataId`` argument is `None`). 

1202 

1203 Returns 

1204 ------- 

1205 refs : `queries.DatasetQueryResults` 

1206 Dataset references matching the given query criteria. Nested data 

1207 IDs are guaranteed to include values for all implied dimensions 

1208 (i.e. `DataCoordinate.hasFull` will return `True`), but will not 

1209 include dimension records (`DataCoordinate.hasRecords` will be 

1210 `False`) unless `~queries.DatasetQueryResults.expanded` is called 

1211 on the result object (which returns a new one). 

1212 

1213 Raises 

1214 ------ 

1215 TypeError 

1216 Raised when the arguments are incompatible, such as when a 

1217 collection wildcard is passed when ``findFirst`` is `True`, or 

1218 when ``collections`` is `None` and``self.defaults.collections`` is 

1219 also `None`. 

1220 

1221 Notes 

1222 ----- 

1223 When multiple dataset types are queried in a single call, the 

1224 results of this operation are equivalent to querying for each dataset 

1225 type separately in turn, and no information about the relationships 

1226 between datasets of different types is included. In contexts where 

1227 that kind of information is important, the recommended pattern is to 

1228 use `queryDataIds` to first obtain data IDs (possibly with the 

1229 desired dataset types and collections passed as constraints to the 

1230 query), and then use multiple (generally much simpler) calls to 

1231 `queryDatasets` with the returned data IDs passed as constraints. 

1232 """ 

1233 raise NotImplementedError() 

1234 

1235 @abstractmethod 

1236 def queryDataIds(self, dimensions: Union[Iterable[Union[Dimension, str]], Dimension, str], *, 

1237 dataId: Optional[DataId] = None, 

1238 datasets: Any = None, 

1239 collections: Any = None, 

1240 where: Optional[str] = None, 

1241 components: Optional[bool] = None, 

1242 bind: Optional[Mapping[str, Any]] = None, 

1243 check: bool = True, 

1244 **kwargs: Any) -> DataCoordinateIterable: 

1245 """Query for data IDs matching user-provided criteria. 

1246 

1247 Parameters 

1248 ---------- 

1249 dimensions : `Dimension` or `str`, or iterable thereof 

1250 The dimensions of the data IDs to yield, as either `Dimension` 

1251 instances or `str`. Will be automatically expanded to a complete 

1252 `DimensionGraph`. 

1253 dataId : `dict` or `DataCoordinate`, optional 

1254 A data ID whose key-value pairs are used as equality constraints 

1255 in the query. 

1256 datasets : `Any`, optional 

1257 An expression that fully or partially identifies dataset types 

1258 that should constrain the yielded data IDs. For example, including 

1259 "raw" here would constrain the yielded ``instrument``, 

1260 ``exposure``, ``detector``, and ``physical_filter`` values to only 

1261 those for which at least one "raw" dataset exists in 

1262 ``collections``. Allowed types include `DatasetType`, `str`, 

1263 `re.Pattern`, and iterables thereof. Unlike other dataset type 

1264 expressions, ``...`` is not permitted - it doesn't make sense to 

1265 constrain data IDs on the existence of *all* datasets. 

1266 See :ref:`daf_butler_dataset_type_expressions` for more 

1267 information. 

1268 collections: `Any`, optional 

1269 An expression that identifies the collections to search for 

1270 datasets, such as a `str` (for full matches or partial matches 

1271 via globs), `re.Pattern` (for partial matches), or iterable 

1272 thereof. `...` can be used to search all collections (actually 

1273 just all `~CollectionType.RUN` collections, because this will 

1274 still find all datasets). If not provided, 

1275 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1276 is also passed. See :ref:`daf_butler_collection_expressions` for 

1277 more information. 

1278 where : `str`, optional 

1279 A string expression similar to a SQL WHERE clause. May involve 

1280 any column of a dimension table or (as a shortcut for the primary 

1281 key column of a dimension table) dimension name. See 

1282 :ref:`daf_butler_dimension_expressions` for more information. 

1283 components : `bool`, optional 

1284 If `True`, apply all dataset expression patterns to component 

1285 dataset type names as well. If `False`, never apply patterns to 

1286 components. If `None` (default), apply patterns to components only 

1287 if their parent datasets were not matched by the expression. 

1288 Fully-specified component datasets (`str` or `DatasetType` 

1289 instances) are always included. 

1290 bind : `Mapping`, optional 

1291 Mapping containing literal values that should be injected into the 

1292 ``where`` expression, keyed by the identifiers they replace. 

1293 check : `bool`, optional 

1294 If `True` (default) check the query for consistency before 

1295 executing it. This may reject some valid queries that resemble 

1296 common mistakes (e.g. queries for visits without specifying an 

1297 instrument). 

1298 **kwargs 

1299 Additional keyword arguments are forwarded to 

1300 `DataCoordinate.standardize` when processing the ``dataId`` 

1301 argument (and may be used to provide a constraining data ID even 

1302 when the ``dataId`` argument is `None`). 

1303 

1304 Returns 

1305 ------- 

1306 dataIds : `DataCoordinateQueryResults` 

1307 Data IDs matching the given query parameters. These are guaranteed 

1308 to identify all dimensions (`DataCoordinate.hasFull` returns 

1309 `True`), but will not contain `DimensionRecord` objects 

1310 (`DataCoordinate.hasRecords` returns `False`). Call 

1311 `DataCoordinateQueryResults.expanded` on the returned object to 

1312 fetch those (and consider using 

1313 `DataCoordinateQueryResults.materialize` on the returned object 

1314 first if the expected number of rows is very large). See 

1315 documentation for those methods for additional information. 

1316 

1317 Raises 

1318 ------ 

1319 TypeError 

1320 Raised if ``collections`` is `None`, ``self.defaults.collections`` 

1321 is `None`, and ``datasets`` is not `None`. 

1322 """ 

1323 raise NotImplementedError() 

1324 

1325 @abstractmethod 

1326 def queryDimensionRecords(self, element: Union[DimensionElement, str], *, 

1327 dataId: Optional[DataId] = None, 

1328 datasets: Any = None, 

1329 collections: Any = None, 

1330 where: Optional[str] = None, 

1331 components: Optional[bool] = None, 

1332 bind: Optional[Mapping[str, Any]] = None, 

1333 check: bool = True, 

1334 **kwargs: Any) -> Iterator[DimensionRecord]: 

1335 """Query for dimension information matching user-provided criteria. 

1336 

1337 Parameters 

1338 ---------- 

1339 element : `DimensionElement` or `str` 

1340 The dimension element to obtain records for. 

1341 dataId : `dict` or `DataCoordinate`, optional 

1342 A data ID whose key-value pairs are used as equality constraints 

1343 in the query. 

1344 datasets : `Any`, optional 

1345 An expression that fully or partially identifies dataset types 

1346 that should constrain the yielded records. See `queryDataIds` and 

1347 :ref:`daf_butler_dataset_type_expressions` for more information. 

1348 collections: `Any`, optional 

1349 An expression that identifies the collections to search for 

1350 datasets, such as a `str` (for full matches or partial matches 

1351 via globs), `re.Pattern` (for partial matches), or iterable 

1352 thereof. `...` can be used to search all collections (actually 

1353 just all `~CollectionType.RUN` collections, because this will 

1354 still find all datasets). If not provided, 

1355 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1356 is also passed. See :ref:`daf_butler_collection_expressions` for 

1357 more information. 

1358 where : `str`, optional 

1359 A string expression similar to a SQL WHERE clause. See 

1360 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more 

1361 information. 

1362 components : `bool`, optional 

1363 Whether to apply dataset expressions to components as well. 

1364 See `queryDataIds` for more information. 

1365 bind : `Mapping`, optional 

1366 Mapping containing literal values that should be injected into the 

1367 ``where`` expression, keyed by the identifiers they replace. 

1368 check : `bool`, optional 

1369 If `True` (default) check the query for consistency before 

1370 executing it. This may reject some valid queries that resemble 

1371 common mistakes (e.g. queries for visits without specifying an 

1372 instrument). 

1373 **kwargs 

1374 Additional keyword arguments are forwarded to 

1375 `DataCoordinate.standardize` when processing the ``dataId`` 

1376 argument (and may be used to provide a constraining data ID even 

1377 when the ``dataId`` argument is `None`). 

1378 

1379 Returns 

1380 ------- 

1381 dataIds : `DataCoordinateQueryResults` 

1382 Data IDs matching the given query parameters. 

1383 """ 

1384 raise NotImplementedError() 

1385 

1386 @abstractmethod 

1387 def queryDatasetAssociations( 

1388 self, 

1389 datasetType: Union[str, DatasetType], 

1390 collections: Any = ..., 

1391 *, 

1392 collectionTypes: Iterable[CollectionType] = CollectionType.all(), 

1393 flattenChains: bool = False, 

1394 ) -> Iterator[DatasetAssociation]: 

1395 """Iterate over dataset-collection combinations where the dataset is in 

1396 the collection. 

1397 

1398 This method is a temporary placeholder for better support for 

1399 assocation results in `queryDatasets`. It will probably be 

1400 removed in the future, and should be avoided in production code 

1401 whenever possible. 

1402 

1403 Parameters 

1404 ---------- 

1405 datasetType : `DatasetType` or `str` 

1406 A dataset type object or the name of one. 

1407 collections: `Any`, optional 

1408 An expression that identifies the collections to search for 

1409 datasets, such as a `str` (for full matches or partial matches 

1410 via globs), `re.Pattern` (for partial matches), or iterable 

1411 thereof. `...` can be used to search all collections (actually 

1412 just all `~CollectionType.RUN` collections, because this will still 

1413 find all datasets). If not provided, ``self.default.collections`` 

1414 is used. See :ref:`daf_butler_collection_expressions` for more 

1415 information. 

1416 collectionTypes : `AbstractSet` [ `CollectionType` ], optional 

1417 If provided, only yield associations from collections of these 

1418 types. 

1419 flattenChains : `bool`, optional 

1420 If `True` (default) search in the children of 

1421 `~CollectionType.CHAINED` collections. If `False`, ``CHAINED`` 

1422 collections are ignored. 

1423 

1424 Yields 

1425 ------ 

1426 association : `DatasetAssociation` 

1427 Object representing the relationship beween a single dataset and 

1428 a single collection. 

1429 

1430 Raises 

1431 ------ 

1432 TypeError 

1433 Raised if ``collections`` is `None` and 

1434 ``self.defaults.collections`` is `None`. 

1435 """ 

1436 raise NotImplementedError() 

1437 

1438 storageClasses: StorageClassFactory 

1439 """All storage classes known to the registry (`StorageClassFactory`). 

1440 """