Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ( 

25 "Registry", 

26) 

27 

28from abc import ABC, abstractmethod 

29import contextlib 

30import logging 

31from typing import ( 

32 Any, 

33 Dict, 

34 Iterable, 

35 Iterator, 

36 List, 

37 Mapping, 

38 Optional, 

39 Tuple, 

40 Type, 

41 TYPE_CHECKING, 

42 Union, 

43) 

44 

45from lsst.utils import doImport 

46 

47from ..core import ( 

48 ButlerURI, 

49 Config, 

50 DataCoordinate, 

51 DataCoordinateIterable, 

52 DataId, 

53 DatasetAssociation, 

54 DatasetId, 

55 DatasetRef, 

56 DatasetType, 

57 Dimension, 

58 DimensionConfig, 

59 DimensionElement, 

60 DimensionGraph, 

61 DimensionRecord, 

62 DimensionUniverse, 

63 NameLookupMapping, 

64 StorageClassFactory, 

65 Timespan, 

66) 

67 

68from ._config import RegistryConfig 

69from ._collectionType import CollectionType 

70from ._defaults import RegistryDefaults 

71from .interfaces import DatasetIdGenEnum 

72from .wildcards import CollectionSearch 

73from .summaries import CollectionSummary 

74 

75if TYPE_CHECKING: 75 ↛ 76line 75 didn't jump to line 76, because the condition on line 75 was never true

76 from .._butlerConfig import ButlerConfig 

77 from .interfaces import ( 

78 CollectionRecord, 

79 DatastoreRegistryBridgeManager, 

80 ) 

81 

82_LOG = logging.getLogger(__name__) 

83 

84 

85class Registry(ABC): 

86 """Abstract Registry interface. 

87 

88 Each registry implementation can have its own constructor parameters. 

89 The assumption is that an instance of a specific subclass will be 

90 constructed from configuration using `Registry.fromConfig()`. 

91 The base class will look for a ``cls`` entry and call that specific 

92 `fromConfig()` method. 

93 

94 All subclasses should store `RegistryDefaults` in a ``_defaults`` 

95 property. No other properties are assumed shared between implementations. 

96 """ 

97 

98 defaultConfigFile: Optional[str] = None 

99 """Path to configuration defaults. Accessed within the ``configs`` resource 

100 or relative to a search path. Can be None if no defaults specified. 

101 """ 

102 

103 @classmethod 

104 def forceRegistryConfig(cls, config: Optional[Union[ButlerConfig, 

105 RegistryConfig, Config, str]]) -> RegistryConfig: 

106 """Force the supplied config to a `RegistryConfig`. 

107 

108 Parameters 

109 ---------- 

110 config : `RegistryConfig`, `Config` or `str` or `None` 

111 Registry configuration, if missing then default configuration will 

112 be loaded from registry.yaml. 

113 

114 Returns 

115 ------- 

116 registry_config : `RegistryConfig` 

117 A registry config. 

118 """ 

119 if not isinstance(config, RegistryConfig): 

120 if isinstance(config, (str, Config)) or config is None: 

121 config = RegistryConfig(config) 

122 else: 

123 raise ValueError(f"Incompatible Registry configuration: {config}") 

124 return config 

125 

126 @classmethod 

127 def determineTrampoline(cls, 

128 config: Optional[Union[ButlerConfig, 

129 RegistryConfig, 

130 Config, 

131 str]]) -> Tuple[Type[Registry], RegistryConfig]: 

132 """Return class to use to instantiate real registry. 

133 

134 Parameters 

135 ---------- 

136 config : `RegistryConfig` or `str`, optional 

137 Registry configuration, if missing then default configuration will 

138 be loaded from registry.yaml. 

139 

140 Returns 

141 ------- 

142 requested_cls : `type` of `Registry` 

143 The real registry class to use. 

144 registry_config : `RegistryConfig` 

145 The `RegistryConfig` to use. 

146 """ 

147 config = cls.forceRegistryConfig(config) 

148 

149 # Default to the standard registry 

150 registry_cls = doImport(config.get("cls", "lsst.daf.butler.registries.sql.SqlRegistry")) 

151 if registry_cls is cls: 

152 raise ValueError("Can not instantiate the abstract base Registry from config") 

153 return registry_cls, config 

154 

155 @classmethod 

156 def createFromConfig(cls, config: Optional[Union[RegistryConfig, str]] = None, 

157 dimensionConfig: Optional[Union[DimensionConfig, str]] = None, 

158 butlerRoot: Optional[str] = None) -> Registry: 

159 """Create registry database and return `Registry` instance. 

160 

161 This method initializes database contents, database must be empty 

162 prior to calling this method. 

163 

164 Parameters 

165 ---------- 

166 config : `RegistryConfig` or `str`, optional 

167 Registry configuration, if missing then default configuration will 

168 be loaded from registry.yaml. 

169 dimensionConfig : `DimensionConfig` or `str`, optional 

170 Dimensions configuration, if missing then default configuration 

171 will be loaded from dimensions.yaml. 

172 butlerRoot : `str`, optional 

173 Path to the repository root this `Registry` will manage. 

174 

175 Returns 

176 ------- 

177 registry : `Registry` 

178 A new `Registry` instance. 

179 

180 Notes 

181 ----- 

182 This class will determine the concrete `Registry` subclass to 

183 use from configuration. Each subclass should implement this method 

184 even if it can not create a registry. 

185 """ 

186 registry_cls, registry_config = cls.determineTrampoline(config) 

187 return registry_cls.createFromConfig(registry_config, dimensionConfig, butlerRoot) 

188 

189 @classmethod 

190 def fromConfig(cls, config: Union[ButlerConfig, RegistryConfig, Config, str], 

191 butlerRoot: Optional[Union[str, ButlerURI]] = None, writeable: bool = True, 

192 defaults: Optional[RegistryDefaults] = None) -> Registry: 

193 """Create `Registry` subclass instance from `config`. 

194 

195 Registry database must be initialized prior to calling this method. 

196 

197 Parameters 

198 ---------- 

199 config : `ButlerConfig`, `RegistryConfig`, `Config` or `str` 

200 Registry configuration 

201 butlerRoot : `str` or `ButlerURI`, optional 

202 Path to the repository root this `Registry` will manage. 

203 writeable : `bool`, optional 

204 If `True` (default) create a read-write connection to the database. 

205 defaults : `RegistryDefaults`, optional 

206 Default collection search path and/or output `~CollectionType.RUN` 

207 collection. 

208 

209 Returns 

210 ------- 

211 registry : `Registry` (subclass) 

212 A new `Registry` subclass instance. 

213 

214 Notes 

215 ----- 

216 This class will determine the concrete `Registry` subclass to 

217 use from configuration. Each subclass should implement this method. 

218 """ 

219 # The base class implementation should trampoline to the correct 

220 # subclass. No implementation should ever use this implementation 

221 # directly. If no class is specified, default to the standard 

222 # registry. 

223 registry_cls, registry_config = cls.determineTrampoline(config) 

224 return registry_cls.fromConfig(config, butlerRoot, writeable, defaults) 

225 

226 @abstractmethod 

227 def isWriteable(self) -> bool: 

228 """Return `True` if this registry allows write operations, and `False` 

229 otherwise. 

230 """ 

231 raise NotImplementedError() 

232 

233 @abstractmethod 

234 def copy(self, defaults: Optional[RegistryDefaults] = None) -> Registry: 

235 """Create a new `Registry` backed by the same data repository and 

236 connection as this one, but independent defaults. 

237 

238 Parameters 

239 ---------- 

240 defaults : `RegistryDefaults`, optional 

241 Default collections and data ID values for the new registry. If 

242 not provided, ``self.defaults`` will be used (but future changes 

243 to either registry's defaults will not affect the other). 

244 

245 Returns 

246 ------- 

247 copy : `Registry` 

248 A new `Registry` instance with its own defaults. 

249 

250 Notes 

251 ----- 

252 Because the new registry shares a connection with the original, they 

253 also share transaction state (despite the fact that their `transaction` 

254 context manager methods do not reflect this), and must be used with 

255 care. 

256 """ 

257 raise NotImplementedError() 

258 

259 @property 

260 @abstractmethod 

261 def dimensions(self) -> DimensionUniverse: 

262 """All dimensions recognized by this `Registry` (`DimensionUniverse`). 

263 """ 

264 raise NotImplementedError() 

265 

266 @property 

267 def defaults(self) -> RegistryDefaults: 

268 """Default collection search path and/or output `~CollectionType.RUN` 

269 collection (`RegistryDefaults`). 

270 

271 This is an immutable struct whose components may not be set 

272 individually, but the entire struct can be set by assigning to this 

273 property. 

274 """ 

275 return self._defaults 

276 

277 @defaults.setter 

278 def defaults(self, value: RegistryDefaults) -> None: 

279 if value.run is not None: 

280 self.registerRun(value.run) 

281 value.finish(self) 

282 self._defaults = value 

283 

284 @abstractmethod 

285 def refresh(self) -> None: 

286 """Refresh all in-memory state by querying the database. 

287 

288 This may be necessary to enable querying for entities added by other 

289 registry instances after this one was constructed. 

290 """ 

291 raise NotImplementedError() 

292 

293 @contextlib.contextmanager 

294 @abstractmethod 

295 def transaction(self, *, savepoint: bool = False) -> Iterator[None]: 

296 """Return a context manager that represents a transaction. 

297 """ 

298 raise NotImplementedError() 

299 

300 def resetConnectionPool(self) -> None: 

301 """Reset connection pool for registry if relevant. 

302 

303 This operation can be used reset connections to servers when 

304 using registry with fork-based multiprocessing. This method should 

305 usually be called by the child process immediately 

306 after the fork. 

307 

308 The base class implementation is a no-op. 

309 """ 

310 pass 

311 

312 @abstractmethod 

313 def registerCollection(self, name: str, type: CollectionType = CollectionType.TAGGED, 

314 doc: Optional[str] = None) -> None: 

315 """Add a new collection if one with the given name does not exist. 

316 

317 Parameters 

318 ---------- 

319 name : `str` 

320 The name of the collection to create. 

321 type : `CollectionType` 

322 Enum value indicating the type of collection to create. 

323 doc : `str`, optional 

324 Documentation string for the collection. 

325 

326 Notes 

327 ----- 

328 This method cannot be called within transactions, as it needs to be 

329 able to perform its own transaction to be concurrent. 

330 """ 

331 raise NotImplementedError() 

332 

333 @abstractmethod 

334 def getCollectionType(self, name: str) -> CollectionType: 

335 """Return an enumeration value indicating the type of the given 

336 collection. 

337 

338 Parameters 

339 ---------- 

340 name : `str` 

341 The name of the collection. 

342 

343 Returns 

344 ------- 

345 type : `CollectionType` 

346 Enum value indicating the type of this collection. 

347 

348 Raises 

349 ------ 

350 MissingCollectionError 

351 Raised if no collection with the given name exists. 

352 """ 

353 raise NotImplementedError() 

354 

355 @abstractmethod 

356 def _get_collection_record(self, name: str) -> CollectionRecord: 

357 """Return the record for this collection. 

358 

359 Parameters 

360 ---------- 

361 name : `str` 

362 Name of the collection for which the record is to be retrieved. 

363 

364 Returns 

365 ------- 

366 record : `CollectionRecord` 

367 The record for this collection. 

368 """ 

369 raise NotImplementedError() 

370 

371 @abstractmethod 

372 def registerRun(self, name: str, doc: Optional[str] = None) -> None: 

373 """Add a new run if one with the given name does not exist. 

374 

375 Parameters 

376 ---------- 

377 name : `str` 

378 The name of the run to create. 

379 doc : `str`, optional 

380 Documentation string for the collection. 

381 

382 Notes 

383 ----- 

384 This method cannot be called within transactions, as it needs to be 

385 able to perform its own transaction to be concurrent. 

386 """ 

387 raise NotImplementedError() 

388 

389 @abstractmethod 

390 def removeCollection(self, name: str) -> None: 

391 """Completely remove the given collection. 

392 

393 Parameters 

394 ---------- 

395 name : `str` 

396 The name of the collection to remove. 

397 

398 Raises 

399 ------ 

400 MissingCollectionError 

401 Raised if no collection with the given name exists. 

402 

403 Notes 

404 ----- 

405 If this is a `~CollectionType.RUN` collection, all datasets and quanta 

406 in it are also fully removed. This requires that those datasets be 

407 removed (or at least trashed) from any datastores that hold them first. 

408 

409 A collection may not be deleted as long as it is referenced by a 

410 `~CollectionType.CHAINED` collection; the ``CHAINED`` collection must 

411 be deleted or redefined first. 

412 """ 

413 raise NotImplementedError() 

414 

415 @abstractmethod 

416 def getCollectionChain(self, parent: str) -> CollectionSearch: 

417 """Return the child collections in a `~CollectionType.CHAINED` 

418 collection. 

419 

420 Parameters 

421 ---------- 

422 parent : `str` 

423 Name of the chained collection. Must have already been added via 

424 a call to `Registry.registerCollection`. 

425 

426 Returns 

427 ------- 

428 children : `CollectionSearch` 

429 An object that defines the search path of the collection. 

430 See :ref:`daf_butler_collection_expressions` for more information. 

431 

432 Raises 

433 ------ 

434 MissingCollectionError 

435 Raised if ``parent`` does not exist in the `Registry`. 

436 TypeError 

437 Raised if ``parent`` does not correspond to a 

438 `~CollectionType.CHAINED` collection. 

439 """ 

440 raise NotImplementedError() 

441 

442 @abstractmethod 

443 def setCollectionChain(self, parent: str, children: Any, *, flatten: bool = False) -> None: 

444 """Define or redefine a `~CollectionType.CHAINED` collection. 

445 

446 Parameters 

447 ---------- 

448 parent : `str` 

449 Name of the chained collection. Must have already been added via 

450 a call to `Registry.registerCollection`. 

451 children : `Any` 

452 An expression defining an ordered search of child collections, 

453 generally an iterable of `str`; see 

454 :ref:`daf_butler_collection_expressions` for more information. 

455 flatten : `bool`, optional 

456 If `True` (`False` is default), recursively flatten out any nested 

457 `~CollectionType.CHAINED` collections in ``children`` first. 

458 

459 Raises 

460 ------ 

461 MissingCollectionError 

462 Raised when any of the given collections do not exist in the 

463 `Registry`. 

464 TypeError 

465 Raised if ``parent`` does not correspond to a 

466 `~CollectionType.CHAINED` collection. 

467 ValueError 

468 Raised if the given collections contains a cycle. 

469 """ 

470 raise NotImplementedError() 

471 

472 @abstractmethod 

473 def getCollectionDocumentation(self, collection: str) -> Optional[str]: 

474 """Retrieve the documentation string for a collection. 

475 

476 Parameters 

477 ---------- 

478 name : `str` 

479 Name of the collection. 

480 

481 Returns 

482 ------- 

483 docs : `str` or `None` 

484 Docstring for the collection with the given name. 

485 """ 

486 raise NotImplementedError() 

487 

488 @abstractmethod 

489 def setCollectionDocumentation(self, collection: str, doc: Optional[str]) -> None: 

490 """Set the documentation string for a collection. 

491 

492 Parameters 

493 ---------- 

494 name : `str` 

495 Name of the collection. 

496 docs : `str` or `None` 

497 Docstring for the collection with the given name; will replace any 

498 existing docstring. Passing `None` will remove any existing 

499 docstring. 

500 """ 

501 raise NotImplementedError() 

502 

503 @abstractmethod 

504 def getCollectionSummary(self, collection: str) -> CollectionSummary: 

505 """Return a summary for the given collection. 

506 

507 Parameters 

508 ---------- 

509 collection : `str` 

510 Name of the collection for which a summary is to be retrieved. 

511 

512 Returns 

513 ------- 

514 summary : `CollectionSummary` 

515 Summary of the dataset types and governor dimension values in 

516 this collection. 

517 """ 

518 raise NotImplementedError() 

519 

520 @abstractmethod 

521 def registerDatasetType(self, datasetType: DatasetType) -> bool: 

522 """ 

523 Add a new `DatasetType` to the Registry. 

524 

525 It is not an error to register the same `DatasetType` twice. 

526 

527 Parameters 

528 ---------- 

529 datasetType : `DatasetType` 

530 The `DatasetType` to be added. 

531 

532 Returns 

533 ------- 

534 inserted : `bool` 

535 `True` if ``datasetType`` was inserted, `False` if an identical 

536 existing `DatsetType` was found. Note that in either case the 

537 DatasetType is guaranteed to be defined in the Registry 

538 consistently with the given definition. 

539 

540 Raises 

541 ------ 

542 ValueError 

543 Raised if the dimensions or storage class are invalid. 

544 ConflictingDefinitionError 

545 Raised if this DatasetType is already registered with a different 

546 definition. 

547 

548 Notes 

549 ----- 

550 This method cannot be called within transactions, as it needs to be 

551 able to perform its own transaction to be concurrent. 

552 """ 

553 raise NotImplementedError() 

554 

555 @abstractmethod 

556 def removeDatasetType(self, name: str) -> None: 

557 """Remove the named `DatasetType` from the registry. 

558 

559 .. warning:: 

560 

561 Registry implementations can cache the dataset type definitions. 

562 This means that deleting the dataset type definition may result in 

563 unexpected behavior from other butler processes that are active 

564 that have not seen the deletion. 

565 

566 Parameters 

567 ---------- 

568 name : `str` 

569 Name of the type to be removed. 

570 

571 Raises 

572 ------ 

573 lsst.daf.butler.registry.OrphanedRecordError 

574 Raised if an attempt is made to remove the dataset type definition 

575 when there are already datasets associated with it. 

576 

577 Notes 

578 ----- 

579 If the dataset type is not registered the method will return without 

580 action. 

581 """ 

582 raise NotImplementedError() 

583 

584 @abstractmethod 

585 def getDatasetType(self, name: str) -> DatasetType: 

586 """Get the `DatasetType`. 

587 

588 Parameters 

589 ---------- 

590 name : `str` 

591 Name of the type. 

592 

593 Returns 

594 ------- 

595 type : `DatasetType` 

596 The `DatasetType` associated with the given name. 

597 

598 Raises 

599 ------ 

600 KeyError 

601 Requested named DatasetType could not be found in registry. 

602 """ 

603 raise NotImplementedError() 

604 

605 @abstractmethod 

606 def supportsIdGenerationMode(self, mode: DatasetIdGenEnum) -> bool: 

607 """Test whether the given dataset ID generation mode is supported by 

608 `insertDatasets`. 

609 

610 Parameters 

611 ---------- 

612 mode : `DatasetIdGenEnum` 

613 Enum value for the mode to test. 

614 

615 Returns 

616 ------- 

617 supported : `bool` 

618 Whether the given mode is supported. 

619 """ 

620 raise NotImplementedError() 

621 

622 @abstractmethod 

623 def findDataset(self, datasetType: Union[DatasetType, str], dataId: Optional[DataId] = None, *, 

624 collections: Any = None, timespan: Optional[Timespan] = None, 

625 **kwargs: Any) -> Optional[DatasetRef]: 

626 """Find a dataset given its `DatasetType` and data ID. 

627 

628 This can be used to obtain a `DatasetRef` that permits the dataset to 

629 be read from a `Datastore`. If the dataset is a component and can not 

630 be found using the provided dataset type, a dataset ref for the parent 

631 will be returned instead but with the correct dataset type. 

632 

633 Parameters 

634 ---------- 

635 datasetType : `DatasetType` or `str` 

636 A `DatasetType` or the name of one. 

637 dataId : `dict` or `DataCoordinate`, optional 

638 A `dict`-like object containing the `Dimension` links that identify 

639 the dataset within a collection. 

640 collections, optional. 

641 An expression that fully or partially identifies the collections to 

642 search for the dataset; see 

643 :ref:`daf_butler_collection_expressions` for more information. 

644 Defaults to ``self.defaults.collections``. 

645 timespan : `Timespan`, optional 

646 A timespan that the validity range of the dataset must overlap. 

647 If not provided, any `~CollectionType.CALIBRATION` collections 

648 matched by the ``collections`` argument will not be searched. 

649 **kwargs 

650 Additional keyword arguments passed to 

651 `DataCoordinate.standardize` to convert ``dataId`` to a true 

652 `DataCoordinate` or augment an existing one. 

653 

654 Returns 

655 ------- 

656 ref : `DatasetRef` 

657 A reference to the dataset, or `None` if no matching Dataset 

658 was found. 

659 

660 Raises 

661 ------ 

662 TypeError 

663 Raised if ``collections`` is `None` and 

664 ``self.defaults.collections`` is `None`. 

665 LookupError 

666 Raised if one or more data ID keys are missing. 

667 KeyError 

668 Raised if the dataset type does not exist. 

669 MissingCollectionError 

670 Raised if any of ``collections`` does not exist in the registry. 

671 

672 Notes 

673 ----- 

674 This method simply returns `None` and does not raise an exception even 

675 when the set of collections searched is intrinsically incompatible with 

676 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but 

677 only `~CollectionType.CALIBRATION` collections are being searched. 

678 This may make it harder to debug some lookup failures, but the behavior 

679 is intentional; we consider it more important that failed searches are 

680 reported consistently, regardless of the reason, and that adding 

681 additional collections that do not contain a match to the search path 

682 never changes the behavior. 

683 """ 

684 raise NotImplementedError() 

685 

686 @abstractmethod 

687 def insertDatasets(self, datasetType: Union[DatasetType, str], dataIds: Iterable[DataId], 

688 run: Optional[str] = None, expand: bool = True, 

689 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE) -> List[DatasetRef]: 

690 """Insert one or more datasets into the `Registry` 

691 

692 This always adds new datasets; to associate existing datasets with 

693 a new collection, use ``associate``. 

694 

695 Parameters 

696 ---------- 

697 datasetType : `DatasetType` or `str` 

698 A `DatasetType` or the name of one. 

699 dataIds : `~collections.abc.Iterable` of `dict` or `DataCoordinate` 

700 Dimension-based identifiers for the new datasets. 

701 run : `str`, optional 

702 The name of the run that produced the datasets. Defaults to 

703 ``self.defaults.run``. 

704 expand : `bool`, optional 

705 If `True` (default), expand data IDs as they are inserted. This is 

706 necessary in general to allow datastore to generate file templates, 

707 but it may be disabled if the caller can guarantee this is 

708 unnecessary. 

709 idGenerationMode : `DatasetIdGenEnum`, optional 

710 Specifies option for generating dataset IDs. By default unique IDs 

711 are generated for each inserted dataset. 

712 

713 Returns 

714 ------- 

715 refs : `list` of `DatasetRef` 

716 Resolved `DatasetRef` instances for all given data IDs (in the same 

717 order). 

718 

719 Raises 

720 ------ 

721 TypeError 

722 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

723 ConflictingDefinitionError 

724 If a dataset with the same dataset type and data ID as one of those 

725 given already exists in ``run``. 

726 MissingCollectionError 

727 Raised if ``run`` does not exist in the registry. 

728 """ 

729 raise NotImplementedError() 

730 

731 @abstractmethod 

732 def _importDatasets(self, datasets: Iterable[DatasetRef], expand: bool = True, 

733 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE, 

734 reuseIds: bool = False) -> List[DatasetRef]: 

735 """Import one or more datasets into the `Registry`. 

736 

737 Difference from `insertDatasets` method is that this method accepts 

738 `DatasetRef` instances which should already be resolved and have a 

739 dataset ID. If registry supports globally-unique dataset IDs (e.g. 

740 `uuid.UUID`) then datasets which already exist in the registry will be 

741 ignored if imported again. 

742 

743 Parameters 

744 ---------- 

745 datasets : `~collections.abc.Iterable` of `DatasetRef` 

746 Datasets to be inserted. All `DatasetRef` instances must have 

747 identical ``datasetType`` and ``run`` attributes. ``run`` 

748 attribute can be `None` and defaults to ``self.defaults.run``. 

749 Datasets can specify ``id`` attribute which will be used for 

750 inserted datasets. All dataset IDs must have the same type 

751 (`int` or `uuid.UUID`), if type of dataset IDs does not match 

752 configured backend then IDs will be ignored and new IDs will be 

753 generated by backend. 

754 expand : `bool`, optional 

755 If `True` (default), expand data IDs as they are inserted. This is 

756 necessary in general to allow datastore to generate file templates, 

757 but it may be disabled if the caller can guarantee this is 

758 unnecessary. 

759 idGenerationMode : `DatasetIdGenEnum`, optional 

760 Specifies option for generating dataset IDs when IDs are not 

761 provided or their type does not match backend type. By default 

762 unique IDs are generated for each inserted dataset. 

763 reuseIds : `bool`, optional 

764 If `True` then forces re-use of imported dataset IDs for integer 

765 IDs which are normally generated as auto-incremented; exception 

766 will be raised if imported IDs clash with existing ones. This 

767 option has no effect on the use of globally-unique IDs which are 

768 always re-used (or generated if integer IDs are being imported). 

769 

770 Returns 

771 ------- 

772 refs : `list` of `DatasetRef` 

773 Resolved `DatasetRef` instances for all given data IDs (in the same 

774 order). If any of ``datasets`` has an ID which already exists in 

775 the database then it will not be inserted or updated, but a 

776 resolved `DatasetRef` will be returned for it in any case. 

777 

778 Raises 

779 ------ 

780 TypeError 

781 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

782 ConflictingDefinitionError 

783 If a dataset with the same dataset type and data ID as one of those 

784 given already exists in ``run``. 

785 MissingCollectionError 

786 Raised if ``run`` does not exist in the registry. 

787 

788 Notes 

789 ----- 

790 This method is considered package-private and internal to Butler 

791 implementation. Clients outside daf_butler package should not use this 

792 method. 

793 """ 

794 raise NotImplementedError() 

795 

796 @abstractmethod 

797 def getDataset(self, id: DatasetId) -> Optional[DatasetRef]: 

798 """Retrieve a Dataset entry. 

799 

800 Parameters 

801 ---------- 

802 id : `DatasetId` 

803 The unique identifier for the dataset. 

804 

805 Returns 

806 ------- 

807 ref : `DatasetRef` or `None` 

808 A ref to the Dataset, or `None` if no matching Dataset 

809 was found. 

810 """ 

811 raise NotImplementedError() 

812 

813 @abstractmethod 

814 def removeDatasets(self, refs: Iterable[DatasetRef]) -> None: 

815 """Remove datasets from the Registry. 

816 

817 The datasets will be removed unconditionally from all collections, and 

818 any `Quantum` that consumed this dataset will instead be marked with 

819 having a NULL input. `Datastore` records will *not* be deleted; the 

820 caller is responsible for ensuring that the dataset has already been 

821 removed from all Datastores. 

822 

823 Parameters 

824 ---------- 

825 refs : `Iterable` of `DatasetRef` 

826 References to the datasets to be removed. Must include a valid 

827 ``id`` attribute, and should be considered invalidated upon return. 

828 

829 Raises 

830 ------ 

831 AmbiguousDatasetError 

832 Raised if any ``ref.id`` is `None`. 

833 OrphanedRecordError 

834 Raised if any dataset is still present in any `Datastore`. 

835 """ 

836 raise NotImplementedError() 

837 

838 @abstractmethod 

839 def associate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

840 """Add existing datasets to a `~CollectionType.TAGGED` collection. 

841 

842 If a DatasetRef with the same exact ID is already in a collection 

843 nothing is changed. If a `DatasetRef` with the same `DatasetType` and 

844 data ID but with different ID exists in the collection, 

845 `ConflictingDefinitionError` is raised. 

846 

847 Parameters 

848 ---------- 

849 collection : `str` 

850 Indicates the collection the datasets should be associated with. 

851 refs : `Iterable` [ `DatasetRef` ] 

852 An iterable of resolved `DatasetRef` instances that already exist 

853 in this `Registry`. 

854 

855 Raises 

856 ------ 

857 ConflictingDefinitionError 

858 If a Dataset with the given `DatasetRef` already exists in the 

859 given collection. 

860 AmbiguousDatasetError 

861 Raised if ``any(ref.id is None for ref in refs)``. 

862 MissingCollectionError 

863 Raised if ``collection`` does not exist in the registry. 

864 TypeError 

865 Raise adding new datasets to the given ``collection`` is not 

866 allowed. 

867 """ 

868 raise NotImplementedError() 

869 

870 @abstractmethod 

871 def disassociate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

872 """Remove existing datasets from a `~CollectionType.TAGGED` collection. 

873 

874 ``collection`` and ``ref`` combinations that are not currently 

875 associated are silently ignored. 

876 

877 Parameters 

878 ---------- 

879 collection : `str` 

880 The collection the datasets should no longer be associated with. 

881 refs : `Iterable` [ `DatasetRef` ] 

882 An iterable of resolved `DatasetRef` instances that already exist 

883 in this `Registry`. 

884 

885 Raises 

886 ------ 

887 AmbiguousDatasetError 

888 Raised if any of the given dataset references is unresolved. 

889 MissingCollectionError 

890 Raised if ``collection`` does not exist in the registry. 

891 TypeError 

892 Raise adding new datasets to the given ``collection`` is not 

893 allowed. 

894 """ 

895 raise NotImplementedError() 

896 

897 @abstractmethod 

898 def certify(self, collection: str, refs: Iterable[DatasetRef], timespan: Timespan) -> None: 

899 """Associate one or more datasets with a calibration collection and a 

900 validity range within it. 

901 

902 Parameters 

903 ---------- 

904 collection : `str` 

905 The name of an already-registered `~CollectionType.CALIBRATION` 

906 collection. 

907 refs : `Iterable` [ `DatasetRef` ] 

908 Datasets to be associated. 

909 timespan : `Timespan` 

910 The validity range for these datasets within the collection. 

911 

912 Raises 

913 ------ 

914 AmbiguousDatasetError 

915 Raised if any of the given `DatasetRef` instances is unresolved. 

916 ConflictingDefinitionError 

917 Raised if the collection already contains a different dataset with 

918 the same `DatasetType` and data ID and an overlapping validity 

919 range. 

920 TypeError 

921 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

922 collection or if one or more datasets are of a dataset type for 

923 which `DatasetType.isCalibration` returns `False`. 

924 """ 

925 raise NotImplementedError() 

926 

927 @abstractmethod 

928 def decertify(self, collection: str, datasetType: Union[str, DatasetType], timespan: Timespan, *, 

929 dataIds: Optional[Iterable[DataId]] = None) -> None: 

930 """Remove or adjust datasets to clear a validity range within a 

931 calibration collection. 

932 

933 Parameters 

934 ---------- 

935 collection : `str` 

936 The name of an already-registered `~CollectionType.CALIBRATION` 

937 collection. 

938 datasetType : `str` or `DatasetType` 

939 Name or `DatasetType` instance for the datasets to be decertified. 

940 timespan : `Timespan`, optional 

941 The validity range to remove datasets from within the collection. 

942 Datasets that overlap this range but are not contained by it will 

943 have their validity ranges adjusted to not overlap it, which may 

944 split a single dataset validity range into two. 

945 dataIds : `Iterable` [ `DataId` ], optional 

946 Data IDs that should be decertified within the given validity range 

947 If `None`, all data IDs for ``self.datasetType`` will be 

948 decertified. 

949 

950 Raises 

951 ------ 

952 TypeError 

953 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

954 collection or if ``datasetType.isCalibration() is False``. 

955 """ 

956 raise NotImplementedError() 

957 

958 @abstractmethod 

959 def getDatastoreBridgeManager(self) -> DatastoreRegistryBridgeManager: 

960 """Return an object that allows a new `Datastore` instance to 

961 communicate with this `Registry`. 

962 

963 Returns 

964 ------- 

965 manager : `DatastoreRegistryBridgeManager` 

966 Object that mediates communication between this `Registry` and its 

967 associated datastores. 

968 """ 

969 raise NotImplementedError() 

970 

971 @abstractmethod 

972 def getDatasetLocations(self, ref: DatasetRef) -> Iterable[str]: 

973 """Retrieve datastore locations for a given dataset. 

974 

975 Parameters 

976 ---------- 

977 ref : `DatasetRef` 

978 A reference to the dataset for which to retrieve storage 

979 information. 

980 

981 Returns 

982 ------- 

983 datastores : `Iterable` [ `str` ] 

984 All the matching datastores holding this dataset. 

985 

986 Raises 

987 ------ 

988 AmbiguousDatasetError 

989 Raised if ``ref.id`` is `None`. 

990 """ 

991 raise NotImplementedError() 

992 

993 @abstractmethod 

994 def expandDataId(self, dataId: Optional[DataId] = None, *, graph: Optional[DimensionGraph] = None, 

995 records: Optional[NameLookupMapping[DimensionElement, Optional[DimensionRecord]]] = None, 

996 withDefaults: bool = True, 

997 **kwargs: Any) -> DataCoordinate: 

998 """Expand a dimension-based data ID to include additional information. 

999 

1000 Parameters 

1001 ---------- 

1002 dataId : `DataCoordinate` or `dict`, optional 

1003 Data ID to be expanded; augmented and overridden by ``kwargs``. 

1004 graph : `DimensionGraph`, optional 

1005 Set of dimensions for the expanded ID. If `None`, the dimensions 

1006 will be inferred from the keys of ``dataId`` and ``kwargs``. 

1007 Dimensions that are in ``dataId`` or ``kwargs`` but not in 

1008 ``graph`` are silently ignored, providing a way to extract and 

1009 ``graph`` expand a subset of a data ID. 

1010 records : `Mapping` [`str`, `DimensionRecord`], optional 

1011 Dimension record data to use before querying the database for that 

1012 data, keyed by element name. 

1013 withDefaults : `bool`, optional 

1014 Utilize ``self.defaults.dataId`` to fill in missing governor 

1015 dimension key-value pairs. Defaults to `True` (i.e. defaults are 

1016 used). 

1017 **kwargs 

1018 Additional keywords are treated like additional key-value pairs for 

1019 ``dataId``, extending and overriding 

1020 

1021 Returns 

1022 ------- 

1023 expanded : `DataCoordinate` 

1024 A data ID that includes full metadata for all of the dimensions it 

1025 identifieds, i.e. guarantees that ``expanded.hasRecords()`` and 

1026 ``expanded.hasFull()`` both return `True`. 

1027 """ 

1028 raise NotImplementedError() 

1029 

1030 @abstractmethod 

1031 def insertDimensionData(self, element: Union[DimensionElement, str], 

1032 *data: Union[Mapping[str, Any], DimensionRecord], 

1033 conform: bool = True, 

1034 replace: bool = False) -> None: 

1035 """Insert one or more dimension records into the database. 

1036 

1037 Parameters 

1038 ---------- 

1039 element : `DimensionElement` or `str` 

1040 The `DimensionElement` or name thereof that identifies the table 

1041 records will be inserted into. 

1042 data : `dict` or `DimensionRecord` (variadic) 

1043 One or more records to insert. 

1044 conform : `bool`, optional 

1045 If `False` (`True` is default) perform no checking or conversions, 

1046 and assume that ``element`` is a `DimensionElement` instance and 

1047 ``data`` is a one or more `DimensionRecord` instances of the 

1048 appropriate subclass. 

1049 replace: `bool`, optional 

1050 If `True` (`False` is default), replace existing records in the 

1051 database if there is a conflict. 

1052 """ 

1053 raise NotImplementedError() 

1054 

1055 @abstractmethod 

1056 def syncDimensionData(self, element: Union[DimensionElement, str], 

1057 row: Union[Mapping[str, Any], DimensionRecord], 

1058 conform: bool = True, 

1059 update: bool = False) -> Union[bool, Dict[str, Any]]: 

1060 """Synchronize the given dimension record with the database, inserting 

1061 if it does not already exist and comparing values if it does. 

1062 

1063 Parameters 

1064 ---------- 

1065 element : `DimensionElement` or `str` 

1066 The `DimensionElement` or name thereof that identifies the table 

1067 records will be inserted into. 

1068 row : `dict` or `DimensionRecord` 

1069 The record to insert. 

1070 conform : `bool`, optional 

1071 If `False` (`True` is default) perform no checking or conversions, 

1072 and assume that ``element`` is a `DimensionElement` instance and 

1073 ``data`` is a one or more `DimensionRecord` instances of the 

1074 appropriate subclass. 

1075 update: `bool`, optional 

1076 If `True` (`False` is default), update the existing record in the 

1077 database if there is a conflict. 

1078 

1079 Returns 

1080 ------- 

1081 inserted_or_updated : `bool` or `dict` 

1082 `True` if a new row was inserted, `False` if no changes were 

1083 needed, or a `dict` mapping updated column names to their old 

1084 values if an update was performed (only possible if 

1085 ``update=True``). 

1086 

1087 Raises 

1088 ------ 

1089 ConflictingDefinitionError 

1090 Raised if the record exists in the database (according to primary 

1091 key lookup) but is inconsistent with the given one. 

1092 """ 

1093 raise NotImplementedError() 

1094 

1095 @abstractmethod 

1096 def queryDatasetTypes(self, expression: Any = ..., *, components: Optional[bool] = None 

1097 ) -> Iterator[DatasetType]: 

1098 """Iterate over the dataset types whose names match an expression. 

1099 

1100 Parameters 

1101 ---------- 

1102 expression : `Any`, optional 

1103 An expression that fully or partially identifies the dataset types 

1104 to return, such as a `str`, `re.Pattern`, or iterable thereof. 

1105 `...` can be used to return all dataset types, and is the default. 

1106 See :ref:`daf_butler_dataset_type_expressions` for more 

1107 information. 

1108 components : `bool`, optional 

1109 If `True`, apply all expression patterns to component dataset type 

1110 names as well. If `False`, never apply patterns to components. 

1111 If `None` (default), apply patterns to components only if their 

1112 parent datasets were not matched by the expression. 

1113 Fully-specified component datasets (`str` or `DatasetType` 

1114 instances) are always included. 

1115 

1116 Yields 

1117 ------ 

1118 datasetType : `DatasetType` 

1119 A `DatasetType` instance whose name matches ``expression``. 

1120 """ 

1121 raise NotImplementedError() 

1122 

1123 @abstractmethod 

1124 def queryCollections(self, expression: Any = ..., 

1125 datasetType: Optional[DatasetType] = None, 

1126 collectionTypes: Iterable[CollectionType] = CollectionType.all(), 

1127 flattenChains: bool = False, 

1128 includeChains: Optional[bool] = None) -> Iterator[str]: 

1129 """Iterate over the collections whose names match an expression. 

1130 

1131 Parameters 

1132 ---------- 

1133 expression : `Any`, optional 

1134 An expression that identifies the collections to return, such as 

1135 a `str` (for full matches or partial matches via globs), 

1136 `re.Pattern` (for partial matches), or iterable thereof. `...` 

1137 can be used to return all collections, and is the default. 

1138 See :ref:`daf_butler_collection_expressions` for more information. 

1139 datasetType : `DatasetType`, optional 

1140 If provided, only yield collections that may contain datasets of 

1141 this type. This is a conservative approximation in general; it may 

1142 yield collections that do not have any such datasets. 

1143 collectionTypes : `AbstractSet` [ `CollectionType` ], optional 

1144 If provided, only yield collections of these types. 

1145 flattenChains : `bool`, optional 

1146 If `True` (`False` is default), recursively yield the child 

1147 collections of matching `~CollectionType.CHAINED` collections. 

1148 includeChains : `bool`, optional 

1149 If `True`, yield records for matching `~CollectionType.CHAINED` 

1150 collections. Default is the opposite of ``flattenChains``: include 

1151 either CHAINED collections or their children, but not both. 

1152 

1153 Yields 

1154 ------ 

1155 collection : `str` 

1156 The name of a collection that matches ``expression``. 

1157 """ 

1158 raise NotImplementedError() 

1159 

1160 @abstractmethod 

1161 def queryDatasets(self, datasetType: Any, *, 

1162 collections: Any = None, 

1163 dimensions: Optional[Iterable[Union[Dimension, str]]] = None, 

1164 dataId: Optional[DataId] = None, 

1165 where: Optional[str] = None, 

1166 findFirst: bool = False, 

1167 components: Optional[bool] = None, 

1168 bind: Optional[Mapping[str, Any]] = None, 

1169 check: bool = True, 

1170 **kwargs: Any) -> Iterable[DatasetRef]: 

1171 """Query for and iterate over dataset references matching user-provided 

1172 criteria. 

1173 

1174 Parameters 

1175 ---------- 

1176 datasetType 

1177 An expression that fully or partially identifies the dataset types 

1178 to be queried. Allowed types include `DatasetType`, `str`, 

1179 `re.Pattern`, and iterables thereof. The special value `...` can 

1180 be used to query all dataset types. See 

1181 :ref:`daf_butler_dataset_type_expressions` for more information. 

1182 collections: optional 

1183 An expression that identifies the collections to search, such as a 

1184 `str` (for full matches or partial matches via globs), `re.Pattern` 

1185 (for partial matches), or iterable thereof. `...` can be used to 

1186 search all collections (actually just all `~CollectionType.RUN` 

1187 collections, because this will still find all datasets). 

1188 If not provided, ``self.default.collections`` is used. See 

1189 :ref:`daf_butler_collection_expressions` for more information. 

1190 dimensions : `~collections.abc.Iterable` of `Dimension` or `str` 

1191 Dimensions to include in the query (in addition to those used 

1192 to identify the queried dataset type(s)), either to constrain 

1193 the resulting datasets to those for which a matching dimension 

1194 exists, or to relate the dataset type's dimensions to dimensions 

1195 referenced by the ``dataId`` or ``where`` arguments. 

1196 dataId : `dict` or `DataCoordinate`, optional 

1197 A data ID whose key-value pairs are used as equality constraints 

1198 in the query. 

1199 where : `str`, optional 

1200 A string expression similar to a SQL WHERE clause. May involve 

1201 any column of a dimension table or (as a shortcut for the primary 

1202 key column of a dimension table) dimension name. See 

1203 :ref:`daf_butler_dimension_expressions` for more information. 

1204 findFirst : `bool`, optional 

1205 If `True` (`False` is default), for each result data ID, only 

1206 yield one `DatasetRef` of each `DatasetType`, from the first 

1207 collection in which a dataset of that dataset type appears 

1208 (according to the order of ``collections`` passed in). If `True`, 

1209 ``collections`` must not contain regular expressions and may not 

1210 be `...`. 

1211 components : `bool`, optional 

1212 If `True`, apply all dataset expression patterns to component 

1213 dataset type names as well. If `False`, never apply patterns to 

1214 components. If `None` (default), apply patterns to components only 

1215 if their parent datasets were not matched by the expression. 

1216 Fully-specified component datasets (`str` or `DatasetType` 

1217 instances) are always included. 

1218 bind : `Mapping`, optional 

1219 Mapping containing literal values that should be injected into the 

1220 ``where`` expression, keyed by the identifiers they replace. 

1221 check : `bool`, optional 

1222 If `True` (default) check the query for consistency before 

1223 executing it. This may reject some valid queries that resemble 

1224 common mistakes (e.g. queries for visits without specifying an 

1225 instrument). 

1226 **kwargs 

1227 Additional keyword arguments are forwarded to 

1228 `DataCoordinate.standardize` when processing the ``dataId`` 

1229 argument (and may be used to provide a constraining data ID even 

1230 when the ``dataId`` argument is `None`). 

1231 

1232 Returns 

1233 ------- 

1234 refs : `queries.DatasetQueryResults` 

1235 Dataset references matching the given query criteria. Nested data 

1236 IDs are guaranteed to include values for all implied dimensions 

1237 (i.e. `DataCoordinate.hasFull` will return `True`), but will not 

1238 include dimension records (`DataCoordinate.hasRecords` will be 

1239 `False`) unless `~queries.DatasetQueryResults.expanded` is called 

1240 on the result object (which returns a new one). 

1241 

1242 Raises 

1243 ------ 

1244 TypeError 

1245 Raised when the arguments are incompatible, such as when a 

1246 collection wildcard is passed when ``findFirst`` is `True`, or 

1247 when ``collections`` is `None` and``self.defaults.collections`` is 

1248 also `None`. 

1249 

1250 Notes 

1251 ----- 

1252 When multiple dataset types are queried in a single call, the 

1253 results of this operation are equivalent to querying for each dataset 

1254 type separately in turn, and no information about the relationships 

1255 between datasets of different types is included. In contexts where 

1256 that kind of information is important, the recommended pattern is to 

1257 use `queryDataIds` to first obtain data IDs (possibly with the 

1258 desired dataset types and collections passed as constraints to the 

1259 query), and then use multiple (generally much simpler) calls to 

1260 `queryDatasets` with the returned data IDs passed as constraints. 

1261 """ 

1262 raise NotImplementedError() 

1263 

1264 @abstractmethod 

1265 def queryDataIds(self, dimensions: Union[Iterable[Union[Dimension, str]], Dimension, str], *, 

1266 dataId: Optional[DataId] = None, 

1267 datasets: Any = None, 

1268 collections: Any = None, 

1269 where: Optional[str] = None, 

1270 components: Optional[bool] = None, 

1271 bind: Optional[Mapping[str, Any]] = None, 

1272 check: bool = True, 

1273 **kwargs: Any) -> DataCoordinateIterable: 

1274 """Query for data IDs matching user-provided criteria. 

1275 

1276 Parameters 

1277 ---------- 

1278 dimensions : `Dimension` or `str`, or iterable thereof 

1279 The dimensions of the data IDs to yield, as either `Dimension` 

1280 instances or `str`. Will be automatically expanded to a complete 

1281 `DimensionGraph`. 

1282 dataId : `dict` or `DataCoordinate`, optional 

1283 A data ID whose key-value pairs are used as equality constraints 

1284 in the query. 

1285 datasets : `Any`, optional 

1286 An expression that fully or partially identifies dataset types 

1287 that should constrain the yielded data IDs. For example, including 

1288 "raw" here would constrain the yielded ``instrument``, 

1289 ``exposure``, ``detector``, and ``physical_filter`` values to only 

1290 those for which at least one "raw" dataset exists in 

1291 ``collections``. Allowed types include `DatasetType`, `str`, 

1292 `re.Pattern`, and iterables thereof. Unlike other dataset type 

1293 expressions, ``...`` is not permitted - it doesn't make sense to 

1294 constrain data IDs on the existence of *all* datasets. 

1295 See :ref:`daf_butler_dataset_type_expressions` for more 

1296 information. 

1297 collections: `Any`, optional 

1298 An expression that identifies the collections to search for 

1299 datasets, such as a `str` (for full matches or partial matches 

1300 via globs), `re.Pattern` (for partial matches), or iterable 

1301 thereof. `...` can be used to search all collections (actually 

1302 just all `~CollectionType.RUN` collections, because this will 

1303 still find all datasets). If not provided, 

1304 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1305 is also passed. See :ref:`daf_butler_collection_expressions` for 

1306 more information. 

1307 where : `str`, optional 

1308 A string expression similar to a SQL WHERE clause. May involve 

1309 any column of a dimension table or (as a shortcut for the primary 

1310 key column of a dimension table) dimension name. See 

1311 :ref:`daf_butler_dimension_expressions` for more information. 

1312 components : `bool`, optional 

1313 If `True`, apply all dataset expression patterns to component 

1314 dataset type names as well. If `False`, never apply patterns to 

1315 components. If `None` (default), apply patterns to components only 

1316 if their parent datasets were not matched by the expression. 

1317 Fully-specified component datasets (`str` or `DatasetType` 

1318 instances) are always included. 

1319 bind : `Mapping`, optional 

1320 Mapping containing literal values that should be injected into the 

1321 ``where`` expression, keyed by the identifiers they replace. 

1322 check : `bool`, optional 

1323 If `True` (default) check the query for consistency before 

1324 executing it. This may reject some valid queries that resemble 

1325 common mistakes (e.g. queries for visits without specifying an 

1326 instrument). 

1327 **kwargs 

1328 Additional keyword arguments are forwarded to 

1329 `DataCoordinate.standardize` when processing the ``dataId`` 

1330 argument (and may be used to provide a constraining data ID even 

1331 when the ``dataId`` argument is `None`). 

1332 

1333 Returns 

1334 ------- 

1335 dataIds : `DataCoordinateQueryResults` 

1336 Data IDs matching the given query parameters. These are guaranteed 

1337 to identify all dimensions (`DataCoordinate.hasFull` returns 

1338 `True`), but will not contain `DimensionRecord` objects 

1339 (`DataCoordinate.hasRecords` returns `False`). Call 

1340 `DataCoordinateQueryResults.expanded` on the returned object to 

1341 fetch those (and consider using 

1342 `DataCoordinateQueryResults.materialize` on the returned object 

1343 first if the expected number of rows is very large). See 

1344 documentation for those methods for additional information. 

1345 

1346 Raises 

1347 ------ 

1348 TypeError 

1349 Raised if ``collections`` is `None`, ``self.defaults.collections`` 

1350 is `None`, and ``datasets`` is not `None`. 

1351 """ 

1352 raise NotImplementedError() 

1353 

1354 @abstractmethod 

1355 def queryDimensionRecords(self, element: Union[DimensionElement, str], *, 

1356 dataId: Optional[DataId] = None, 

1357 datasets: Any = None, 

1358 collections: Any = None, 

1359 where: Optional[str] = None, 

1360 components: Optional[bool] = None, 

1361 bind: Optional[Mapping[str, Any]] = None, 

1362 check: bool = True, 

1363 **kwargs: Any) -> Iterator[DimensionRecord]: 

1364 """Query for dimension information matching user-provided criteria. 

1365 

1366 Parameters 

1367 ---------- 

1368 element : `DimensionElement` or `str` 

1369 The dimension element to obtain records for. 

1370 dataId : `dict` or `DataCoordinate`, optional 

1371 A data ID whose key-value pairs are used as equality constraints 

1372 in the query. 

1373 datasets : `Any`, optional 

1374 An expression that fully or partially identifies dataset types 

1375 that should constrain the yielded records. See `queryDataIds` and 

1376 :ref:`daf_butler_dataset_type_expressions` for more information. 

1377 collections: `Any`, optional 

1378 An expression that identifies the collections to search for 

1379 datasets, such as a `str` (for full matches or partial matches 

1380 via globs), `re.Pattern` (for partial matches), or iterable 

1381 thereof. `...` can be used to search all collections (actually 

1382 just all `~CollectionType.RUN` collections, because this will 

1383 still find all datasets). If not provided, 

1384 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1385 is also passed. See :ref:`daf_butler_collection_expressions` for 

1386 more information. 

1387 where : `str`, optional 

1388 A string expression similar to a SQL WHERE clause. See 

1389 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more 

1390 information. 

1391 components : `bool`, optional 

1392 Whether to apply dataset expressions to components as well. 

1393 See `queryDataIds` for more information. 

1394 bind : `Mapping`, optional 

1395 Mapping containing literal values that should be injected into the 

1396 ``where`` expression, keyed by the identifiers they replace. 

1397 check : `bool`, optional 

1398 If `True` (default) check the query for consistency before 

1399 executing it. This may reject some valid queries that resemble 

1400 common mistakes (e.g. queries for visits without specifying an 

1401 instrument). 

1402 **kwargs 

1403 Additional keyword arguments are forwarded to 

1404 `DataCoordinate.standardize` when processing the ``dataId`` 

1405 argument (and may be used to provide a constraining data ID even 

1406 when the ``dataId`` argument is `None`). 

1407 

1408 Returns 

1409 ------- 

1410 dataIds : `DataCoordinateQueryResults` 

1411 Data IDs matching the given query parameters. 

1412 """ 

1413 raise NotImplementedError() 

1414 

1415 @abstractmethod 

1416 def queryDatasetAssociations( 

1417 self, 

1418 datasetType: Union[str, DatasetType], 

1419 collections: Any = ..., 

1420 *, 

1421 collectionTypes: Iterable[CollectionType] = CollectionType.all(), 

1422 flattenChains: bool = False, 

1423 ) -> Iterator[DatasetAssociation]: 

1424 """Iterate over dataset-collection combinations where the dataset is in 

1425 the collection. 

1426 

1427 This method is a temporary placeholder for better support for 

1428 assocation results in `queryDatasets`. It will probably be 

1429 removed in the future, and should be avoided in production code 

1430 whenever possible. 

1431 

1432 Parameters 

1433 ---------- 

1434 datasetType : `DatasetType` or `str` 

1435 A dataset type object or the name of one. 

1436 collections: `Any`, optional 

1437 An expression that identifies the collections to search for 

1438 datasets, such as a `str` (for full matches or partial matches 

1439 via globs), `re.Pattern` (for partial matches), or iterable 

1440 thereof. `...` can be used to search all collections (actually 

1441 just all `~CollectionType.RUN` collections, because this will still 

1442 find all datasets). If not provided, ``self.default.collections`` 

1443 is used. See :ref:`daf_butler_collection_expressions` for more 

1444 information. 

1445 collectionTypes : `AbstractSet` [ `CollectionType` ], optional 

1446 If provided, only yield associations from collections of these 

1447 types. 

1448 flattenChains : `bool`, optional 

1449 If `True` (default) search in the children of 

1450 `~CollectionType.CHAINED` collections. If `False`, ``CHAINED`` 

1451 collections are ignored. 

1452 

1453 Yields 

1454 ------ 

1455 association : `DatasetAssociation` 

1456 Object representing the relationship beween a single dataset and 

1457 a single collection. 

1458 

1459 Raises 

1460 ------ 

1461 TypeError 

1462 Raised if ``collections`` is `None` and 

1463 ``self.defaults.collections`` is `None`. 

1464 """ 

1465 raise NotImplementedError() 

1466 

1467 storageClasses: StorageClassFactory 

1468 """All storage classes known to the registry (`StorageClassFactory`). 

1469 """