Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ( 

25 "Registry", 

26) 

27 

28from abc import ABC, abstractmethod 

29import contextlib 

30import logging 

31from typing import ( 

32 Any, 

33 Dict, 

34 Iterable, 

35 Iterator, 

36 List, 

37 Mapping, 

38 Optional, 

39 Tuple, 

40 Type, 

41 TYPE_CHECKING, 

42 Union, 

43) 

44 

45from lsst.utils import doImport 

46 

47from ..core import ( 

48 ButlerURI, 

49 Config, 

50 DataCoordinate, 

51 DataCoordinateIterable, 

52 DataId, 

53 DatasetAssociation, 

54 DatasetId, 

55 DatasetRef, 

56 DatasetType, 

57 Dimension, 

58 DimensionConfig, 

59 DimensionElement, 

60 DimensionGraph, 

61 DimensionRecord, 

62 DimensionUniverse, 

63 NameLookupMapping, 

64 StorageClassFactory, 

65 Timespan, 

66) 

67 

68from ._config import RegistryConfig 

69from ._collectionType import CollectionType 

70from ._defaults import RegistryDefaults 

71from .interfaces import DatasetIdGenEnum 

72from .wildcards import CollectionSearch 

73from .summaries import CollectionSummary 

74 

75if TYPE_CHECKING: 75 ↛ 76line 75 didn't jump to line 76, because the condition on line 75 was never true

76 from .._butlerConfig import ButlerConfig 

77 from .interfaces import ( 

78 CollectionRecord, 

79 DatastoreRegistryBridgeManager, 

80 ) 

81 

82_LOG = logging.getLogger(__name__) 

83 

84 

85class Registry(ABC): 

86 """Abstract Registry interface. 

87 

88 Each registry implementation can have its own constructor parameters. 

89 The assumption is that an instance of a specific subclass will be 

90 constructed from configuration using `Registry.fromConfig()`. 

91 The base class will look for a ``cls`` entry and call that specific 

92 `fromConfig()` method. 

93 

94 All subclasses should store `RegistryDefaults` in a ``_defaults`` 

95 property. No other properties are assumed shared between implementations. 

96 """ 

97 

98 defaultConfigFile: Optional[str] = None 

99 """Path to configuration defaults. Accessed within the ``configs`` resource 

100 or relative to a search path. Can be None if no defaults specified. 

101 """ 

102 

103 @classmethod 

104 def forceRegistryConfig(cls, config: Optional[Union[ButlerConfig, 

105 RegistryConfig, Config, str]]) -> RegistryConfig: 

106 """Force the supplied config to a `RegistryConfig`. 

107 

108 Parameters 

109 ---------- 

110 config : `RegistryConfig`, `Config` or `str` or `None` 

111 Registry configuration, if missing then default configuration will 

112 be loaded from registry.yaml. 

113 

114 Returns 

115 ------- 

116 registry_config : `RegistryConfig` 

117 A registry config. 

118 """ 

119 if not isinstance(config, RegistryConfig): 

120 if isinstance(config, (str, Config)) or config is None: 

121 config = RegistryConfig(config) 

122 else: 

123 raise ValueError(f"Incompatible Registry configuration: {config}") 

124 return config 

125 

126 @classmethod 

127 def determineTrampoline(cls, 

128 config: Optional[Union[ButlerConfig, 

129 RegistryConfig, 

130 Config, 

131 str]]) -> Tuple[Type[Registry], RegistryConfig]: 

132 """Return class to use to instantiate real registry. 

133 

134 Parameters 

135 ---------- 

136 config : `RegistryConfig` or `str`, optional 

137 Registry configuration, if missing then default configuration will 

138 be loaded from registry.yaml. 

139 

140 Returns 

141 ------- 

142 requested_cls : `type` of `Registry` 

143 The real registry class to use. 

144 registry_config : `RegistryConfig` 

145 The `RegistryConfig` to use. 

146 """ 

147 config = cls.forceRegistryConfig(config) 

148 

149 # Default to the standard registry 

150 registry_cls = doImport(config.get("cls", "lsst.daf.butler.registries.sql.SqlRegistry")) 

151 if registry_cls is cls: 

152 raise ValueError("Can not instantiate the abstract base Registry from config") 

153 return registry_cls, config 

154 

155 @classmethod 

156 def createFromConfig(cls, config: Optional[Union[RegistryConfig, str]] = None, 

157 dimensionConfig: Optional[Union[DimensionConfig, str]] = None, 

158 butlerRoot: Optional[str] = None) -> Registry: 

159 """Create registry database and return `Registry` instance. 

160 

161 This method initializes database contents, database must be empty 

162 prior to calling this method. 

163 

164 Parameters 

165 ---------- 

166 config : `RegistryConfig` or `str`, optional 

167 Registry configuration, if missing then default configuration will 

168 be loaded from registry.yaml. 

169 dimensionConfig : `DimensionConfig` or `str`, optional 

170 Dimensions configuration, if missing then default configuration 

171 will be loaded from dimensions.yaml. 

172 butlerRoot : `str`, optional 

173 Path to the repository root this `Registry` will manage. 

174 

175 Returns 

176 ------- 

177 registry : `Registry` 

178 A new `Registry` instance. 

179 

180 Notes 

181 ----- 

182 This class will determine the concrete `Registry` subclass to 

183 use from configuration. Each subclass should implement this method 

184 even if it can not create a registry. 

185 """ 

186 registry_cls, registry_config = cls.determineTrampoline(config) 

187 return registry_cls.createFromConfig(registry_config, dimensionConfig, butlerRoot) 

188 

189 @classmethod 

190 def fromConfig(cls, config: Union[ButlerConfig, RegistryConfig, Config, str], 

191 butlerRoot: Optional[Union[str, ButlerURI]] = None, writeable: bool = True, 

192 defaults: Optional[RegistryDefaults] = None) -> Registry: 

193 """Create `Registry` subclass instance from `config`. 

194 

195 Registry database must be initialized prior to calling this method. 

196 

197 Parameters 

198 ---------- 

199 config : `ButlerConfig`, `RegistryConfig`, `Config` or `str` 

200 Registry configuration 

201 butlerRoot : `str` or `ButlerURI`, optional 

202 Path to the repository root this `Registry` will manage. 

203 writeable : `bool`, optional 

204 If `True` (default) create a read-write connection to the database. 

205 defaults : `RegistryDefaults`, optional 

206 Default collection search path and/or output `~CollectionType.RUN` 

207 collection. 

208 

209 Returns 

210 ------- 

211 registry : `Registry` (subclass) 

212 A new `Registry` subclass instance. 

213 

214 Notes 

215 ----- 

216 This class will determine the concrete `Registry` subclass to 

217 use from configuration. Each subclass should implement this method. 

218 """ 

219 # The base class implementation should trampoline to the correct 

220 # subclass. No implementation should ever use this implementation 

221 # directly. If no class is specified, default to the standard 

222 # registry. 

223 registry_cls, registry_config = cls.determineTrampoline(config) 

224 return registry_cls.fromConfig(config, butlerRoot, writeable, defaults) 

225 

226 @abstractmethod 

227 def isWriteable(self) -> bool: 

228 """Return `True` if this registry allows write operations, and `False` 

229 otherwise. 

230 """ 

231 raise NotImplementedError() 

232 

233 @abstractmethod 

234 def copy(self, defaults: Optional[RegistryDefaults] = None) -> Registry: 

235 """Create a new `Registry` backed by the same data repository and 

236 connection as this one, but independent defaults. 

237 

238 Parameters 

239 ---------- 

240 defaults : `RegistryDefaults`, optional 

241 Default collections and data ID values for the new registry. If 

242 not provided, ``self.defaults`` will be used (but future changes 

243 to either registry's defaults will not affect the other). 

244 

245 Returns 

246 ------- 

247 copy : `Registry` 

248 A new `Registry` instance with its own defaults. 

249 

250 Notes 

251 ----- 

252 Because the new registry shares a connection with the original, they 

253 also share transaction state (despite the fact that their `transaction` 

254 context manager methods do not reflect this), and must be used with 

255 care. 

256 """ 

257 raise NotImplementedError() 

258 

259 @property 

260 @abstractmethod 

261 def dimensions(self) -> DimensionUniverse: 

262 """All dimensions recognized by this `Registry` (`DimensionUniverse`). 

263 """ 

264 raise NotImplementedError() 

265 

266 @property 

267 def defaults(self) -> RegistryDefaults: 

268 """Default collection search path and/or output `~CollectionType.RUN` 

269 collection (`RegistryDefaults`). 

270 

271 This is an immutable struct whose components may not be set 

272 individually, but the entire struct can be set by assigning to this 

273 property. 

274 """ 

275 return self._defaults 

276 

277 @defaults.setter 

278 def defaults(self, value: RegistryDefaults) -> None: 

279 if value.run is not None: 

280 self.registerRun(value.run) 

281 value.finish(self) 

282 self._defaults = value 

283 

284 @abstractmethod 

285 def refresh(self) -> None: 

286 """Refresh all in-memory state by querying the database. 

287 

288 This may be necessary to enable querying for entities added by other 

289 registry instances after this one was constructed. 

290 """ 

291 raise NotImplementedError() 

292 

293 @contextlib.contextmanager 

294 @abstractmethod 

295 def transaction(self, *, savepoint: bool = False) -> Iterator[None]: 

296 """Return a context manager that represents a transaction. 

297 """ 

298 raise NotImplementedError() 

299 

300 def resetConnectionPool(self) -> None: 

301 """Reset connection pool for registry if relevant. 

302 

303 This operation can be used reset connections to servers when 

304 using registry with fork-based multiprocessing. This method should 

305 usually be called by the child process immediately 

306 after the fork. 

307 

308 The base class implementation is a no-op. 

309 """ 

310 pass 

311 

312 @abstractmethod 

313 def registerCollection(self, name: str, type: CollectionType = CollectionType.TAGGED, 

314 doc: Optional[str] = None) -> None: 

315 """Add a new collection if one with the given name does not exist. 

316 

317 Parameters 

318 ---------- 

319 name : `str` 

320 The name of the collection to create. 

321 type : `CollectionType` 

322 Enum value indicating the type of collection to create. 

323 doc : `str`, optional 

324 Documentation string for the collection. 

325 

326 Notes 

327 ----- 

328 This method cannot be called within transactions, as it needs to be 

329 able to perform its own transaction to be concurrent. 

330 """ 

331 raise NotImplementedError() 

332 

333 @abstractmethod 

334 def getCollectionType(self, name: str) -> CollectionType: 

335 """Return an enumeration value indicating the type of the given 

336 collection. 

337 

338 Parameters 

339 ---------- 

340 name : `str` 

341 The name of the collection. 

342 

343 Returns 

344 ------- 

345 type : `CollectionType` 

346 Enum value indicating the type of this collection. 

347 

348 Raises 

349 ------ 

350 MissingCollectionError 

351 Raised if no collection with the given name exists. 

352 """ 

353 raise NotImplementedError() 

354 

355 @abstractmethod 

356 def _get_collection_record(self, name: str) -> CollectionRecord: 

357 """Return the record for this collection. 

358 

359 Parameters 

360 ---------- 

361 name : `str` 

362 Name of the collection for which the record is to be retrieved. 

363 

364 Returns 

365 ------- 

366 record : `CollectionRecord` 

367 The record for this collection. 

368 """ 

369 raise NotImplementedError() 

370 

371 @abstractmethod 

372 def registerRun(self, name: str, doc: Optional[str] = None) -> None: 

373 """Add a new run if one with the given name does not exist. 

374 

375 Parameters 

376 ---------- 

377 name : `str` 

378 The name of the run to create. 

379 doc : `str`, optional 

380 Documentation string for the collection. 

381 

382 Notes 

383 ----- 

384 This method cannot be called within transactions, as it needs to be 

385 able to perform its own transaction to be concurrent. 

386 """ 

387 raise NotImplementedError() 

388 

389 @abstractmethod 

390 def removeCollection(self, name: str) -> None: 

391 """Completely remove the given collection. 

392 

393 Parameters 

394 ---------- 

395 name : `str` 

396 The name of the collection to remove. 

397 

398 Raises 

399 ------ 

400 MissingCollectionError 

401 Raised if no collection with the given name exists. 

402 

403 Notes 

404 ----- 

405 If this is a `~CollectionType.RUN` collection, all datasets and quanta 

406 in it are also fully removed. This requires that those datasets be 

407 removed (or at least trashed) from any datastores that hold them first. 

408 

409 A collection may not be deleted as long as it is referenced by a 

410 `~CollectionType.CHAINED` collection; the ``CHAINED`` collection must 

411 be deleted or redefined first. 

412 """ 

413 raise NotImplementedError() 

414 

415 @abstractmethod 

416 def getCollectionChain(self, parent: str) -> CollectionSearch: 

417 """Return the child collections in a `~CollectionType.CHAINED` 

418 collection. 

419 

420 Parameters 

421 ---------- 

422 parent : `str` 

423 Name of the chained collection. Must have already been added via 

424 a call to `Registry.registerCollection`. 

425 

426 Returns 

427 ------- 

428 children : `CollectionSearch` 

429 An object that defines the search path of the collection. 

430 See :ref:`daf_butler_collection_expressions` for more information. 

431 

432 Raises 

433 ------ 

434 MissingCollectionError 

435 Raised if ``parent`` does not exist in the `Registry`. 

436 TypeError 

437 Raised if ``parent`` does not correspond to a 

438 `~CollectionType.CHAINED` collection. 

439 """ 

440 raise NotImplementedError() 

441 

442 @abstractmethod 

443 def setCollectionChain(self, parent: str, children: Any, *, flatten: bool = False) -> None: 

444 """Define or redefine a `~CollectionType.CHAINED` collection. 

445 

446 Parameters 

447 ---------- 

448 parent : `str` 

449 Name of the chained collection. Must have already been added via 

450 a call to `Registry.registerCollection`. 

451 children : `Any` 

452 An expression defining an ordered search of child collections, 

453 generally an iterable of `str`; see 

454 :ref:`daf_butler_collection_expressions` for more information. 

455 flatten : `bool`, optional 

456 If `True` (`False` is default), recursively flatten out any nested 

457 `~CollectionType.CHAINED` collections in ``children`` first. 

458 

459 Raises 

460 ------ 

461 MissingCollectionError 

462 Raised when any of the given collections do not exist in the 

463 `Registry`. 

464 TypeError 

465 Raised if ``parent`` does not correspond to a 

466 `~CollectionType.CHAINED` collection. 

467 ValueError 

468 Raised if the given collections contains a cycle. 

469 """ 

470 raise NotImplementedError() 

471 

472 @abstractmethod 

473 def getCollectionDocumentation(self, collection: str) -> Optional[str]: 

474 """Retrieve the documentation string for a collection. 

475 

476 Parameters 

477 ---------- 

478 name : `str` 

479 Name of the collection. 

480 

481 Returns 

482 ------- 

483 docs : `str` or `None` 

484 Docstring for the collection with the given name. 

485 """ 

486 raise NotImplementedError() 

487 

488 @abstractmethod 

489 def setCollectionDocumentation(self, collection: str, doc: Optional[str]) -> None: 

490 """Set the documentation string for a collection. 

491 

492 Parameters 

493 ---------- 

494 name : `str` 

495 Name of the collection. 

496 docs : `str` or `None` 

497 Docstring for the collection with the given name; will replace any 

498 existing docstring. Passing `None` will remove any existing 

499 docstring. 

500 """ 

501 raise NotImplementedError() 

502 

503 @abstractmethod 

504 def getCollectionSummary(self, collection: str) -> CollectionSummary: 

505 """Return a summary for the given collection. 

506 

507 Parameters 

508 ---------- 

509 collection : `str` 

510 Name of the collection for which a summary is to be retrieved. 

511 

512 Returns 

513 ------- 

514 summary : `CollectionSummary` 

515 Summary of the dataset types and governor dimension values in 

516 this collection. 

517 """ 

518 raise NotImplementedError() 

519 

520 @abstractmethod 

521 def registerDatasetType(self, datasetType: DatasetType) -> bool: 

522 """ 

523 Add a new `DatasetType` to the Registry. 

524 

525 It is not an error to register the same `DatasetType` twice. 

526 

527 Parameters 

528 ---------- 

529 datasetType : `DatasetType` 

530 The `DatasetType` to be added. 

531 

532 Returns 

533 ------- 

534 inserted : `bool` 

535 `True` if ``datasetType`` was inserted, `False` if an identical 

536 existing `DatsetType` was found. Note that in either case the 

537 DatasetType is guaranteed to be defined in the Registry 

538 consistently with the given definition. 

539 

540 Raises 

541 ------ 

542 ValueError 

543 Raised if the dimensions or storage class are invalid. 

544 ConflictingDefinitionError 

545 Raised if this DatasetType is already registered with a different 

546 definition. 

547 

548 Notes 

549 ----- 

550 This method cannot be called within transactions, as it needs to be 

551 able to perform its own transaction to be concurrent. 

552 """ 

553 raise NotImplementedError() 

554 

555 @abstractmethod 

556 def removeDatasetType(self, name: str) -> None: 

557 """Remove the named `DatasetType` from the registry. 

558 

559 .. warning:: 

560 

561 Registry implementations can cache the dataset type definitions. 

562 This means that deleting the dataset type definition may result in 

563 unexpected behavior from other butler processes that are active 

564 that have not seen the deletion. 

565 

566 Parameters 

567 ---------- 

568 name : `str` 

569 Name of the type to be removed. 

570 

571 Raises 

572 ------ 

573 lsst.daf.butler.registry.OrphanedRecordError 

574 Raised if an attempt is made to remove the dataset type definition 

575 when there are already datasets associated with it. 

576 

577 Notes 

578 ----- 

579 If the dataset type is not registered the method will return without 

580 action. 

581 """ 

582 raise NotImplementedError() 

583 

584 @abstractmethod 

585 def getDatasetType(self, name: str) -> DatasetType: 

586 """Get the `DatasetType`. 

587 

588 Parameters 

589 ---------- 

590 name : `str` 

591 Name of the type. 

592 

593 Returns 

594 ------- 

595 type : `DatasetType` 

596 The `DatasetType` associated with the given name. 

597 

598 Raises 

599 ------ 

600 KeyError 

601 Requested named DatasetType could not be found in registry. 

602 """ 

603 raise NotImplementedError() 

604 

605 @abstractmethod 

606 def findDataset(self, datasetType: Union[DatasetType, str], dataId: Optional[DataId] = None, *, 

607 collections: Any = None, timespan: Optional[Timespan] = None, 

608 **kwargs: Any) -> Optional[DatasetRef]: 

609 """Find a dataset given its `DatasetType` and data ID. 

610 

611 This can be used to obtain a `DatasetRef` that permits the dataset to 

612 be read from a `Datastore`. If the dataset is a component and can not 

613 be found using the provided dataset type, a dataset ref for the parent 

614 will be returned instead but with the correct dataset type. 

615 

616 Parameters 

617 ---------- 

618 datasetType : `DatasetType` or `str` 

619 A `DatasetType` or the name of one. 

620 dataId : `dict` or `DataCoordinate`, optional 

621 A `dict`-like object containing the `Dimension` links that identify 

622 the dataset within a collection. 

623 collections, optional. 

624 An expression that fully or partially identifies the collections to 

625 search for the dataset; see 

626 :ref:`daf_butler_collection_expressions` for more information. 

627 Defaults to ``self.defaults.collections``. 

628 timespan : `Timespan`, optional 

629 A timespan that the validity range of the dataset must overlap. 

630 If not provided, any `~CollectionType.CALIBRATION` collections 

631 matched by the ``collections`` argument will not be searched. 

632 **kwargs 

633 Additional keyword arguments passed to 

634 `DataCoordinate.standardize` to convert ``dataId`` to a true 

635 `DataCoordinate` or augment an existing one. 

636 

637 Returns 

638 ------- 

639 ref : `DatasetRef` 

640 A reference to the dataset, or `None` if no matching Dataset 

641 was found. 

642 

643 Raises 

644 ------ 

645 TypeError 

646 Raised if ``collections`` is `None` and 

647 ``self.defaults.collections`` is `None`. 

648 LookupError 

649 Raised if one or more data ID keys are missing. 

650 KeyError 

651 Raised if the dataset type does not exist. 

652 MissingCollectionError 

653 Raised if any of ``collections`` does not exist in the registry. 

654 

655 Notes 

656 ----- 

657 This method simply returns `None` and does not raise an exception even 

658 when the set of collections searched is intrinsically incompatible with 

659 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but 

660 only `~CollectionType.CALIBRATION` collections are being searched. 

661 This may make it harder to debug some lookup failures, but the behavior 

662 is intentional; we consider it more important that failed searches are 

663 reported consistently, regardless of the reason, and that adding 

664 additional collections that do not contain a match to the search path 

665 never changes the behavior. 

666 """ 

667 raise NotImplementedError() 

668 

669 @abstractmethod 

670 def insertDatasets(self, datasetType: Union[DatasetType, str], dataIds: Iterable[DataId], 

671 run: Optional[str] = None, expand: bool = True, 

672 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE) -> List[DatasetRef]: 

673 """Insert one or more datasets into the `Registry` 

674 

675 This always adds new datasets; to associate existing datasets with 

676 a new collection, use ``associate``. 

677 

678 Parameters 

679 ---------- 

680 datasetType : `DatasetType` or `str` 

681 A `DatasetType` or the name of one. 

682 dataIds : `~collections.abc.Iterable` of `dict` or `DataCoordinate` 

683 Dimension-based identifiers for the new datasets. 

684 run : `str`, optional 

685 The name of the run that produced the datasets. Defaults to 

686 ``self.defaults.run``. 

687 expand : `bool`, optional 

688 If `True` (default), expand data IDs as they are inserted. This is 

689 necessary in general to allow datastore to generate file templates, 

690 but it may be disabled if the caller can guarantee this is 

691 unnecessary. 

692 idGenerationMode : `DatasetIdGenEnum`, optional 

693 Specifies option for generating dataset IDs. By default unique IDs 

694 are generated for each inserted dataset. 

695 

696 Returns 

697 ------- 

698 refs : `list` of `DatasetRef` 

699 Resolved `DatasetRef` instances for all given data IDs (in the same 

700 order). 

701 

702 Raises 

703 ------ 

704 TypeError 

705 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

706 ConflictingDefinitionError 

707 If a dataset with the same dataset type and data ID as one of those 

708 given already exists in ``run``. 

709 MissingCollectionError 

710 Raised if ``run`` does not exist in the registry. 

711 """ 

712 raise NotImplementedError() 

713 

714 @abstractmethod 

715 def _importDatasets(self, datasets: Iterable[DatasetRef], expand: bool = True, 

716 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE, 

717 reuseIds: bool = False) -> List[DatasetRef]: 

718 """Import one or more datasets into the `Registry`. 

719 

720 Difference from `insertDatasets` method is that this method accepts 

721 `DatasetRef` instances which should already be resolved and have a 

722 dataset ID. If registry supports globally-unique dataset IDs (e.g. 

723 `uuid.UUID`) then datasets which already exist in the registry will be 

724 ignored if imported again. 

725 

726 Parameters 

727 ---------- 

728 datasets : `~collections.abc.Iterable` of `DatasetRef` 

729 Datasets to be inserted. All `DatasetRef` instances must have 

730 identical ``datasetType`` and ``run`` attributes. ``run`` 

731 attribute can be `None` and defaults to ``self.defaults.run``. 

732 Datasets can specify ``id`` attribute which will be used for 

733 inserted datasets. All dataset IDs must have the same type 

734 (`int` or `uuid.UUID`), if type of dataset IDs does not match 

735 configured backend then IDs will be ignored and new IDs will be 

736 generated by backend. 

737 expand : `bool`, optional 

738 If `True` (default), expand data IDs as they are inserted. This is 

739 necessary in general to allow datastore to generate file templates, 

740 but it may be disabled if the caller can guarantee this is 

741 unnecessary. 

742 idGenerationMode : `DatasetIdGenEnum`, optional 

743 Specifies option for generating dataset IDs when IDs are not 

744 provided or their type does not match backend type. By default 

745 unique IDs are generated for each inserted dataset. 

746 reuseIds : `bool`, optional 

747 If `True` then forces re-use of imported dataset IDs for integer 

748 IDs which are normally generated as auto-incremented; exception 

749 will be raised if imported IDs clash with existing ones. This 

750 option has no effect on the use of globally-unique IDs which are 

751 always re-used (or generated if integer IDs are being imported). 

752 

753 Returns 

754 ------- 

755 refs : `list` of `DatasetRef` 

756 Resolved `DatasetRef` instances for all given data IDs (in the same 

757 order). If any of ``datasets`` has an ID which already exists in 

758 the database then it will not be inserted or updated, but a 

759 resolved `DatasetRef` will be returned for it in any case. 

760 

761 Raises 

762 ------ 

763 TypeError 

764 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

765 ConflictingDefinitionError 

766 If a dataset with the same dataset type and data ID as one of those 

767 given already exists in ``run``. 

768 MissingCollectionError 

769 Raised if ``run`` does not exist in the registry. 

770 

771 Notes 

772 ----- 

773 This method is considered package-private and internal to Butler 

774 implementation. Clients outside daf_butler package should not use this 

775 method. 

776 """ 

777 raise NotImplementedError() 

778 

779 @abstractmethod 

780 def getDataset(self, id: DatasetId) -> Optional[DatasetRef]: 

781 """Retrieve a Dataset entry. 

782 

783 Parameters 

784 ---------- 

785 id : `DatasetId` 

786 The unique identifier for the dataset. 

787 

788 Returns 

789 ------- 

790 ref : `DatasetRef` or `None` 

791 A ref to the Dataset, or `None` if no matching Dataset 

792 was found. 

793 """ 

794 raise NotImplementedError() 

795 

796 @abstractmethod 

797 def removeDatasets(self, refs: Iterable[DatasetRef]) -> None: 

798 """Remove datasets from the Registry. 

799 

800 The datasets will be removed unconditionally from all collections, and 

801 any `Quantum` that consumed this dataset will instead be marked with 

802 having a NULL input. `Datastore` records will *not* be deleted; the 

803 caller is responsible for ensuring that the dataset has already been 

804 removed from all Datastores. 

805 

806 Parameters 

807 ---------- 

808 refs : `Iterable` of `DatasetRef` 

809 References to the datasets to be removed. Must include a valid 

810 ``id`` attribute, and should be considered invalidated upon return. 

811 

812 Raises 

813 ------ 

814 AmbiguousDatasetError 

815 Raised if any ``ref.id`` is `None`. 

816 OrphanedRecordError 

817 Raised if any dataset is still present in any `Datastore`. 

818 """ 

819 raise NotImplementedError() 

820 

821 @abstractmethod 

822 def associate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

823 """Add existing datasets to a `~CollectionType.TAGGED` collection. 

824 

825 If a DatasetRef with the same exact ID is already in a collection 

826 nothing is changed. If a `DatasetRef` with the same `DatasetType` and 

827 data ID but with different ID exists in the collection, 

828 `ConflictingDefinitionError` is raised. 

829 

830 Parameters 

831 ---------- 

832 collection : `str` 

833 Indicates the collection the datasets should be associated with. 

834 refs : `Iterable` [ `DatasetRef` ] 

835 An iterable of resolved `DatasetRef` instances that already exist 

836 in this `Registry`. 

837 

838 Raises 

839 ------ 

840 ConflictingDefinitionError 

841 If a Dataset with the given `DatasetRef` already exists in the 

842 given collection. 

843 AmbiguousDatasetError 

844 Raised if ``any(ref.id is None for ref in refs)``. 

845 MissingCollectionError 

846 Raised if ``collection`` does not exist in the registry. 

847 TypeError 

848 Raise adding new datasets to the given ``collection`` is not 

849 allowed. 

850 """ 

851 raise NotImplementedError() 

852 

853 @abstractmethod 

854 def disassociate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

855 """Remove existing datasets from a `~CollectionType.TAGGED` collection. 

856 

857 ``collection`` and ``ref`` combinations that are not currently 

858 associated are silently ignored. 

859 

860 Parameters 

861 ---------- 

862 collection : `str` 

863 The collection the datasets should no longer be associated with. 

864 refs : `Iterable` [ `DatasetRef` ] 

865 An iterable of resolved `DatasetRef` instances that already exist 

866 in this `Registry`. 

867 

868 Raises 

869 ------ 

870 AmbiguousDatasetError 

871 Raised if any of the given dataset references is unresolved. 

872 MissingCollectionError 

873 Raised if ``collection`` does not exist in the registry. 

874 TypeError 

875 Raise adding new datasets to the given ``collection`` is not 

876 allowed. 

877 """ 

878 raise NotImplementedError() 

879 

880 @abstractmethod 

881 def certify(self, collection: str, refs: Iterable[DatasetRef], timespan: Timespan) -> None: 

882 """Associate one or more datasets with a calibration collection and a 

883 validity range within it. 

884 

885 Parameters 

886 ---------- 

887 collection : `str` 

888 The name of an already-registered `~CollectionType.CALIBRATION` 

889 collection. 

890 refs : `Iterable` [ `DatasetRef` ] 

891 Datasets to be associated. 

892 timespan : `Timespan` 

893 The validity range for these datasets within the collection. 

894 

895 Raises 

896 ------ 

897 AmbiguousDatasetError 

898 Raised if any of the given `DatasetRef` instances is unresolved. 

899 ConflictingDefinitionError 

900 Raised if the collection already contains a different dataset with 

901 the same `DatasetType` and data ID and an overlapping validity 

902 range. 

903 TypeError 

904 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

905 collection or if one or more datasets are of a dataset type for 

906 which `DatasetType.isCalibration` returns `False`. 

907 """ 

908 raise NotImplementedError() 

909 

910 @abstractmethod 

911 def decertify(self, collection: str, datasetType: Union[str, DatasetType], timespan: Timespan, *, 

912 dataIds: Optional[Iterable[DataId]] = None) -> None: 

913 """Remove or adjust datasets to clear a validity range within a 

914 calibration collection. 

915 

916 Parameters 

917 ---------- 

918 collection : `str` 

919 The name of an already-registered `~CollectionType.CALIBRATION` 

920 collection. 

921 datasetType : `str` or `DatasetType` 

922 Name or `DatasetType` instance for the datasets to be decertified. 

923 timespan : `Timespan`, optional 

924 The validity range to remove datasets from within the collection. 

925 Datasets that overlap this range but are not contained by it will 

926 have their validity ranges adjusted to not overlap it, which may 

927 split a single dataset validity range into two. 

928 dataIds : `Iterable` [ `DataId` ], optional 

929 Data IDs that should be decertified within the given validity range 

930 If `None`, all data IDs for ``self.datasetType`` will be 

931 decertified. 

932 

933 Raises 

934 ------ 

935 TypeError 

936 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

937 collection or if ``datasetType.isCalibration() is False``. 

938 """ 

939 raise NotImplementedError() 

940 

941 @abstractmethod 

942 def getDatastoreBridgeManager(self) -> DatastoreRegistryBridgeManager: 

943 """Return an object that allows a new `Datastore` instance to 

944 communicate with this `Registry`. 

945 

946 Returns 

947 ------- 

948 manager : `DatastoreRegistryBridgeManager` 

949 Object that mediates communication between this `Registry` and its 

950 associated datastores. 

951 """ 

952 raise NotImplementedError() 

953 

954 @abstractmethod 

955 def getDatasetLocations(self, ref: DatasetRef) -> Iterable[str]: 

956 """Retrieve datastore locations for a given dataset. 

957 

958 Parameters 

959 ---------- 

960 ref : `DatasetRef` 

961 A reference to the dataset for which to retrieve storage 

962 information. 

963 

964 Returns 

965 ------- 

966 datastores : `Iterable` [ `str` ] 

967 All the matching datastores holding this dataset. 

968 

969 Raises 

970 ------ 

971 AmbiguousDatasetError 

972 Raised if ``ref.id`` is `None`. 

973 """ 

974 raise NotImplementedError() 

975 

976 @abstractmethod 

977 def expandDataId(self, dataId: Optional[DataId] = None, *, graph: Optional[DimensionGraph] = None, 

978 records: Optional[NameLookupMapping[DimensionElement, Optional[DimensionRecord]]] = None, 

979 withDefaults: bool = True, 

980 **kwargs: Any) -> DataCoordinate: 

981 """Expand a dimension-based data ID to include additional information. 

982 

983 Parameters 

984 ---------- 

985 dataId : `DataCoordinate` or `dict`, optional 

986 Data ID to be expanded; augmented and overridden by ``kwargs``. 

987 graph : `DimensionGraph`, optional 

988 Set of dimensions for the expanded ID. If `None`, the dimensions 

989 will be inferred from the keys of ``dataId`` and ``kwargs``. 

990 Dimensions that are in ``dataId`` or ``kwargs`` but not in 

991 ``graph`` are silently ignored, providing a way to extract and 

992 ``graph`` expand a subset of a data ID. 

993 records : `Mapping` [`str`, `DimensionRecord`], optional 

994 Dimension record data to use before querying the database for that 

995 data, keyed by element name. 

996 withDefaults : `bool`, optional 

997 Utilize ``self.defaults.dataId`` to fill in missing governor 

998 dimension key-value pairs. Defaults to `True` (i.e. defaults are 

999 used). 

1000 **kwargs 

1001 Additional keywords are treated like additional key-value pairs for 

1002 ``dataId``, extending and overriding 

1003 

1004 Returns 

1005 ------- 

1006 expanded : `DataCoordinate` 

1007 A data ID that includes full metadata for all of the dimensions it 

1008 identifieds, i.e. guarantees that ``expanded.hasRecords()`` and 

1009 ``expanded.hasFull()`` both return `True`. 

1010 """ 

1011 raise NotImplementedError() 

1012 

1013 @abstractmethod 

1014 def insertDimensionData(self, element: Union[DimensionElement, str], 

1015 *data: Union[Mapping[str, Any], DimensionRecord], 

1016 conform: bool = True, 

1017 replace: bool = False) -> None: 

1018 """Insert one or more dimension records into the database. 

1019 

1020 Parameters 

1021 ---------- 

1022 element : `DimensionElement` or `str` 

1023 The `DimensionElement` or name thereof that identifies the table 

1024 records will be inserted into. 

1025 data : `dict` or `DimensionRecord` (variadic) 

1026 One or more records to insert. 

1027 conform : `bool`, optional 

1028 If `False` (`True` is default) perform no checking or conversions, 

1029 and assume that ``element`` is a `DimensionElement` instance and 

1030 ``data`` is a one or more `DimensionRecord` instances of the 

1031 appropriate subclass. 

1032 replace: `bool`, optional 

1033 If `True` (`False` is default), replace existing records in the 

1034 database if there is a conflict. 

1035 """ 

1036 raise NotImplementedError() 

1037 

1038 @abstractmethod 

1039 def syncDimensionData(self, element: Union[DimensionElement, str], 

1040 row: Union[Mapping[str, Any], DimensionRecord], 

1041 conform: bool = True, 

1042 update: bool = False) -> Union[bool, Dict[str, Any]]: 

1043 """Synchronize the given dimension record with the database, inserting 

1044 if it does not already exist and comparing values if it does. 

1045 

1046 Parameters 

1047 ---------- 

1048 element : `DimensionElement` or `str` 

1049 The `DimensionElement` or name thereof that identifies the table 

1050 records will be inserted into. 

1051 row : `dict` or `DimensionRecord` 

1052 The record to insert. 

1053 conform : `bool`, optional 

1054 If `False` (`True` is default) perform no checking or conversions, 

1055 and assume that ``element`` is a `DimensionElement` instance and 

1056 ``data`` is a one or more `DimensionRecord` instances of the 

1057 appropriate subclass. 

1058 update: `bool`, optional 

1059 If `True` (`False` is default), update the existing record in the 

1060 database if there is a conflict. 

1061 

1062 Returns 

1063 ------- 

1064 inserted_or_updated : `bool` or `dict` 

1065 `True` if a new row was inserted, `False` if no changes were 

1066 needed, or a `dict` mapping updated column names to their old 

1067 values if an update was performed (only possible if 

1068 ``update=True``). 

1069 

1070 Raises 

1071 ------ 

1072 ConflictingDefinitionError 

1073 Raised if the record exists in the database (according to primary 

1074 key lookup) but is inconsistent with the given one. 

1075 """ 

1076 raise NotImplementedError() 

1077 

1078 @abstractmethod 

1079 def queryDatasetTypes(self, expression: Any = ..., *, components: Optional[bool] = None 

1080 ) -> Iterator[DatasetType]: 

1081 """Iterate over the dataset types whose names match an expression. 

1082 

1083 Parameters 

1084 ---------- 

1085 expression : `Any`, optional 

1086 An expression that fully or partially identifies the dataset types 

1087 to return, such as a `str`, `re.Pattern`, or iterable thereof. 

1088 `...` can be used to return all dataset types, and is the default. 

1089 See :ref:`daf_butler_dataset_type_expressions` for more 

1090 information. 

1091 components : `bool`, optional 

1092 If `True`, apply all expression patterns to component dataset type 

1093 names as well. If `False`, never apply patterns to components. 

1094 If `None` (default), apply patterns to components only if their 

1095 parent datasets were not matched by the expression. 

1096 Fully-specified component datasets (`str` or `DatasetType` 

1097 instances) are always included. 

1098 

1099 Yields 

1100 ------ 

1101 datasetType : `DatasetType` 

1102 A `DatasetType` instance whose name matches ``expression``. 

1103 """ 

1104 raise NotImplementedError() 

1105 

1106 @abstractmethod 

1107 def queryCollections(self, expression: Any = ..., 

1108 datasetType: Optional[DatasetType] = None, 

1109 collectionTypes: Iterable[CollectionType] = CollectionType.all(), 

1110 flattenChains: bool = False, 

1111 includeChains: Optional[bool] = None) -> Iterator[str]: 

1112 """Iterate over the collections whose names match an expression. 

1113 

1114 Parameters 

1115 ---------- 

1116 expression : `Any`, optional 

1117 An expression that identifies the collections to return, such as 

1118 a `str` (for full matches or partial matches via globs), 

1119 `re.Pattern` (for partial matches), or iterable thereof. `...` 

1120 can be used to return all collections, and is the default. 

1121 See :ref:`daf_butler_collection_expressions` for more information. 

1122 datasetType : `DatasetType`, optional 

1123 If provided, only yield collections that may contain datasets of 

1124 this type. This is a conservative approximation in general; it may 

1125 yield collections that do not have any such datasets. 

1126 collectionTypes : `AbstractSet` [ `CollectionType` ], optional 

1127 If provided, only yield collections of these types. 

1128 flattenChains : `bool`, optional 

1129 If `True` (`False` is default), recursively yield the child 

1130 collections of matching `~CollectionType.CHAINED` collections. 

1131 includeChains : `bool`, optional 

1132 If `True`, yield records for matching `~CollectionType.CHAINED` 

1133 collections. Default is the opposite of ``flattenChains``: include 

1134 either CHAINED collections or their children, but not both. 

1135 

1136 Yields 

1137 ------ 

1138 collection : `str` 

1139 The name of a collection that matches ``expression``. 

1140 """ 

1141 raise NotImplementedError() 

1142 

1143 @abstractmethod 

1144 def queryDatasets(self, datasetType: Any, *, 

1145 collections: Any = None, 

1146 dimensions: Optional[Iterable[Union[Dimension, str]]] = None, 

1147 dataId: Optional[DataId] = None, 

1148 where: Optional[str] = None, 

1149 findFirst: bool = False, 

1150 components: Optional[bool] = None, 

1151 bind: Optional[Mapping[str, Any]] = None, 

1152 check: bool = True, 

1153 **kwargs: Any) -> Iterable[DatasetRef]: 

1154 """Query for and iterate over dataset references matching user-provided 

1155 criteria. 

1156 

1157 Parameters 

1158 ---------- 

1159 datasetType 

1160 An expression that fully or partially identifies the dataset types 

1161 to be queried. Allowed types include `DatasetType`, `str`, 

1162 `re.Pattern`, and iterables thereof. The special value `...` can 

1163 be used to query all dataset types. See 

1164 :ref:`daf_butler_dataset_type_expressions` for more information. 

1165 collections: optional 

1166 An expression that identifies the collections to search, such as a 

1167 `str` (for full matches or partial matches via globs), `re.Pattern` 

1168 (for partial matches), or iterable thereof. `...` can be used to 

1169 search all collections (actually just all `~CollectionType.RUN` 

1170 collections, because this will still find all datasets). 

1171 If not provided, ``self.default.collections`` is used. See 

1172 :ref:`daf_butler_collection_expressions` for more information. 

1173 dimensions : `~collections.abc.Iterable` of `Dimension` or `str` 

1174 Dimensions to include in the query (in addition to those used 

1175 to identify the queried dataset type(s)), either to constrain 

1176 the resulting datasets to those for which a matching dimension 

1177 exists, or to relate the dataset type's dimensions to dimensions 

1178 referenced by the ``dataId`` or ``where`` arguments. 

1179 dataId : `dict` or `DataCoordinate`, optional 

1180 A data ID whose key-value pairs are used as equality constraints 

1181 in the query. 

1182 where : `str`, optional 

1183 A string expression similar to a SQL WHERE clause. May involve 

1184 any column of a dimension table or (as a shortcut for the primary 

1185 key column of a dimension table) dimension name. See 

1186 :ref:`daf_butler_dimension_expressions` for more information. 

1187 findFirst : `bool`, optional 

1188 If `True` (`False` is default), for each result data ID, only 

1189 yield one `DatasetRef` of each `DatasetType`, from the first 

1190 collection in which a dataset of that dataset type appears 

1191 (according to the order of ``collections`` passed in). If `True`, 

1192 ``collections`` must not contain regular expressions and may not 

1193 be `...`. 

1194 components : `bool`, optional 

1195 If `True`, apply all dataset expression patterns to component 

1196 dataset type names as well. If `False`, never apply patterns to 

1197 components. If `None` (default), apply patterns to components only 

1198 if their parent datasets were not matched by the expression. 

1199 Fully-specified component datasets (`str` or `DatasetType` 

1200 instances) are always included. 

1201 bind : `Mapping`, optional 

1202 Mapping containing literal values that should be injected into the 

1203 ``where`` expression, keyed by the identifiers they replace. 

1204 check : `bool`, optional 

1205 If `True` (default) check the query for consistency before 

1206 executing it. This may reject some valid queries that resemble 

1207 common mistakes (e.g. queries for visits without specifying an 

1208 instrument). 

1209 **kwargs 

1210 Additional keyword arguments are forwarded to 

1211 `DataCoordinate.standardize` when processing the ``dataId`` 

1212 argument (and may be used to provide a constraining data ID even 

1213 when the ``dataId`` argument is `None`). 

1214 

1215 Returns 

1216 ------- 

1217 refs : `queries.DatasetQueryResults` 

1218 Dataset references matching the given query criteria. Nested data 

1219 IDs are guaranteed to include values for all implied dimensions 

1220 (i.e. `DataCoordinate.hasFull` will return `True`), but will not 

1221 include dimension records (`DataCoordinate.hasRecords` will be 

1222 `False`) unless `~queries.DatasetQueryResults.expanded` is called 

1223 on the result object (which returns a new one). 

1224 

1225 Raises 

1226 ------ 

1227 TypeError 

1228 Raised when the arguments are incompatible, such as when a 

1229 collection wildcard is passed when ``findFirst`` is `True`, or 

1230 when ``collections`` is `None` and``self.defaults.collections`` is 

1231 also `None`. 

1232 

1233 Notes 

1234 ----- 

1235 When multiple dataset types are queried in a single call, the 

1236 results of this operation are equivalent to querying for each dataset 

1237 type separately in turn, and no information about the relationships 

1238 between datasets of different types is included. In contexts where 

1239 that kind of information is important, the recommended pattern is to 

1240 use `queryDataIds` to first obtain data IDs (possibly with the 

1241 desired dataset types and collections passed as constraints to the 

1242 query), and then use multiple (generally much simpler) calls to 

1243 `queryDatasets` with the returned data IDs passed as constraints. 

1244 """ 

1245 raise NotImplementedError() 

1246 

1247 @abstractmethod 

1248 def queryDataIds(self, dimensions: Union[Iterable[Union[Dimension, str]], Dimension, str], *, 

1249 dataId: Optional[DataId] = None, 

1250 datasets: Any = None, 

1251 collections: Any = None, 

1252 where: Optional[str] = None, 

1253 components: Optional[bool] = None, 

1254 bind: Optional[Mapping[str, Any]] = None, 

1255 check: bool = True, 

1256 **kwargs: Any) -> DataCoordinateIterable: 

1257 """Query for data IDs matching user-provided criteria. 

1258 

1259 Parameters 

1260 ---------- 

1261 dimensions : `Dimension` or `str`, or iterable thereof 

1262 The dimensions of the data IDs to yield, as either `Dimension` 

1263 instances or `str`. Will be automatically expanded to a complete 

1264 `DimensionGraph`. 

1265 dataId : `dict` or `DataCoordinate`, optional 

1266 A data ID whose key-value pairs are used as equality constraints 

1267 in the query. 

1268 datasets : `Any`, optional 

1269 An expression that fully or partially identifies dataset types 

1270 that should constrain the yielded data IDs. For example, including 

1271 "raw" here would constrain the yielded ``instrument``, 

1272 ``exposure``, ``detector``, and ``physical_filter`` values to only 

1273 those for which at least one "raw" dataset exists in 

1274 ``collections``. Allowed types include `DatasetType`, `str`, 

1275 `re.Pattern`, and iterables thereof. Unlike other dataset type 

1276 expressions, ``...`` is not permitted - it doesn't make sense to 

1277 constrain data IDs on the existence of *all* datasets. 

1278 See :ref:`daf_butler_dataset_type_expressions` for more 

1279 information. 

1280 collections: `Any`, optional 

1281 An expression that identifies the collections to search for 

1282 datasets, such as a `str` (for full matches or partial matches 

1283 via globs), `re.Pattern` (for partial matches), or iterable 

1284 thereof. `...` can be used to search all collections (actually 

1285 just all `~CollectionType.RUN` collections, because this will 

1286 still find all datasets). If not provided, 

1287 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1288 is also passed. See :ref:`daf_butler_collection_expressions` for 

1289 more information. 

1290 where : `str`, optional 

1291 A string expression similar to a SQL WHERE clause. May involve 

1292 any column of a dimension table or (as a shortcut for the primary 

1293 key column of a dimension table) dimension name. See 

1294 :ref:`daf_butler_dimension_expressions` for more information. 

1295 components : `bool`, optional 

1296 If `True`, apply all dataset expression patterns to component 

1297 dataset type names as well. If `False`, never apply patterns to 

1298 components. If `None` (default), apply patterns to components only 

1299 if their parent datasets were not matched by the expression. 

1300 Fully-specified component datasets (`str` or `DatasetType` 

1301 instances) are always included. 

1302 bind : `Mapping`, optional 

1303 Mapping containing literal values that should be injected into the 

1304 ``where`` expression, keyed by the identifiers they replace. 

1305 check : `bool`, optional 

1306 If `True` (default) check the query for consistency before 

1307 executing it. This may reject some valid queries that resemble 

1308 common mistakes (e.g. queries for visits without specifying an 

1309 instrument). 

1310 **kwargs 

1311 Additional keyword arguments are forwarded to 

1312 `DataCoordinate.standardize` when processing the ``dataId`` 

1313 argument (and may be used to provide a constraining data ID even 

1314 when the ``dataId`` argument is `None`). 

1315 

1316 Returns 

1317 ------- 

1318 dataIds : `DataCoordinateQueryResults` 

1319 Data IDs matching the given query parameters. These are guaranteed 

1320 to identify all dimensions (`DataCoordinate.hasFull` returns 

1321 `True`), but will not contain `DimensionRecord` objects 

1322 (`DataCoordinate.hasRecords` returns `False`). Call 

1323 `DataCoordinateQueryResults.expanded` on the returned object to 

1324 fetch those (and consider using 

1325 `DataCoordinateQueryResults.materialize` on the returned object 

1326 first if the expected number of rows is very large). See 

1327 documentation for those methods for additional information. 

1328 

1329 Raises 

1330 ------ 

1331 TypeError 

1332 Raised if ``collections`` is `None`, ``self.defaults.collections`` 

1333 is `None`, and ``datasets`` is not `None`. 

1334 """ 

1335 raise NotImplementedError() 

1336 

1337 @abstractmethod 

1338 def queryDimensionRecords(self, element: Union[DimensionElement, str], *, 

1339 dataId: Optional[DataId] = None, 

1340 datasets: Any = None, 

1341 collections: Any = None, 

1342 where: Optional[str] = None, 

1343 components: Optional[bool] = None, 

1344 bind: Optional[Mapping[str, Any]] = None, 

1345 check: bool = True, 

1346 **kwargs: Any) -> Iterator[DimensionRecord]: 

1347 """Query for dimension information matching user-provided criteria. 

1348 

1349 Parameters 

1350 ---------- 

1351 element : `DimensionElement` or `str` 

1352 The dimension element to obtain records for. 

1353 dataId : `dict` or `DataCoordinate`, optional 

1354 A data ID whose key-value pairs are used as equality constraints 

1355 in the query. 

1356 datasets : `Any`, optional 

1357 An expression that fully or partially identifies dataset types 

1358 that should constrain the yielded records. See `queryDataIds` and 

1359 :ref:`daf_butler_dataset_type_expressions` for more information. 

1360 collections: `Any`, optional 

1361 An expression that identifies the collections to search for 

1362 datasets, such as a `str` (for full matches or partial matches 

1363 via globs), `re.Pattern` (for partial matches), or iterable 

1364 thereof. `...` can be used to search all collections (actually 

1365 just all `~CollectionType.RUN` collections, because this will 

1366 still find all datasets). If not provided, 

1367 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1368 is also passed. See :ref:`daf_butler_collection_expressions` for 

1369 more information. 

1370 where : `str`, optional 

1371 A string expression similar to a SQL WHERE clause. See 

1372 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more 

1373 information. 

1374 components : `bool`, optional 

1375 Whether to apply dataset expressions to components as well. 

1376 See `queryDataIds` for more information. 

1377 bind : `Mapping`, optional 

1378 Mapping containing literal values that should be injected into the 

1379 ``where`` expression, keyed by the identifiers they replace. 

1380 check : `bool`, optional 

1381 If `True` (default) check the query for consistency before 

1382 executing it. This may reject some valid queries that resemble 

1383 common mistakes (e.g. queries for visits without specifying an 

1384 instrument). 

1385 **kwargs 

1386 Additional keyword arguments are forwarded to 

1387 `DataCoordinate.standardize` when processing the ``dataId`` 

1388 argument (and may be used to provide a constraining data ID even 

1389 when the ``dataId`` argument is `None`). 

1390 

1391 Returns 

1392 ------- 

1393 dataIds : `DataCoordinateQueryResults` 

1394 Data IDs matching the given query parameters. 

1395 """ 

1396 raise NotImplementedError() 

1397 

1398 @abstractmethod 

1399 def queryDatasetAssociations( 

1400 self, 

1401 datasetType: Union[str, DatasetType], 

1402 collections: Any = ..., 

1403 *, 

1404 collectionTypes: Iterable[CollectionType] = CollectionType.all(), 

1405 flattenChains: bool = False, 

1406 ) -> Iterator[DatasetAssociation]: 

1407 """Iterate over dataset-collection combinations where the dataset is in 

1408 the collection. 

1409 

1410 This method is a temporary placeholder for better support for 

1411 assocation results in `queryDatasets`. It will probably be 

1412 removed in the future, and should be avoided in production code 

1413 whenever possible. 

1414 

1415 Parameters 

1416 ---------- 

1417 datasetType : `DatasetType` or `str` 

1418 A dataset type object or the name of one. 

1419 collections: `Any`, optional 

1420 An expression that identifies the collections to search for 

1421 datasets, such as a `str` (for full matches or partial matches 

1422 via globs), `re.Pattern` (for partial matches), or iterable 

1423 thereof. `...` can be used to search all collections (actually 

1424 just all `~CollectionType.RUN` collections, because this will still 

1425 find all datasets). If not provided, ``self.default.collections`` 

1426 is used. See :ref:`daf_butler_collection_expressions` for more 

1427 information. 

1428 collectionTypes : `AbstractSet` [ `CollectionType` ], optional 

1429 If provided, only yield associations from collections of these 

1430 types. 

1431 flattenChains : `bool`, optional 

1432 If `True` (default) search in the children of 

1433 `~CollectionType.CHAINED` collections. If `False`, ``CHAINED`` 

1434 collections are ignored. 

1435 

1436 Yields 

1437 ------ 

1438 association : `DatasetAssociation` 

1439 Object representing the relationship beween a single dataset and 

1440 a single collection. 

1441 

1442 Raises 

1443 ------ 

1444 TypeError 

1445 Raised if ``collections`` is `None` and 

1446 ``self.defaults.collections`` is `None`. 

1447 """ 

1448 raise NotImplementedError() 

1449 

1450 storageClasses: StorageClassFactory 

1451 """All storage classes known to the registry (`StorageClassFactory`). 

1452 """