Coverage for python/lsst/daf/butler/registry/_registry.py: 99%

115 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-04-30 02:52 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ("Registry", "CollectionArgType") 

31 

32import contextlib 

33import logging 

34import re 

35from abc import ABC, abstractmethod 

36from collections.abc import Iterable, Iterator, Mapping, Sequence 

37from types import EllipsisType 

38from typing import TYPE_CHECKING, Any, TypeAlias 

39 

40from .._dataset_association import DatasetAssociation 

41from .._dataset_ref import DatasetId, DatasetIdGenEnum, DatasetRef 

42from .._dataset_type import DatasetType 

43from .._named import NameLookupMapping 

44from .._storage_class import StorageClassFactory 

45from .._timespan import Timespan 

46from ..dimensions import ( 

47 DataCoordinate, 

48 DataId, 

49 Dimension, 

50 DimensionElement, 

51 DimensionGraph, 

52 DimensionGroup, 

53 DimensionRecord, 

54 DimensionUniverse, 

55) 

56from ._collection_summary import CollectionSummary 

57from ._collection_type import CollectionType 

58from ._defaults import RegistryDefaults 

59from .queries import DataCoordinateQueryResults, DatasetQueryResults, DimensionRecordQueryResults 

60from .wildcards import CollectionWildcard 

61 

62if TYPE_CHECKING: 

63 from .interfaces import ObsCoreTableManager 

64 

65_LOG = logging.getLogger(__name__) 

66 

67# TYpe alias for `collections` arguments. 

68CollectionArgType: TypeAlias = ( 

69 str | re.Pattern | Iterable[str | re.Pattern] | EllipsisType | CollectionWildcard 

70) 

71 

72 

73class Registry(ABC): 

74 """Abstract Registry interface. 

75 

76 All subclasses should store `~lsst.daf.butler.registry.RegistryDefaults` in 

77 a ``_defaults`` property. No other properties are assumed shared between 

78 implementations. 

79 

80 See Also 

81 -------- 

82 lsst.daf.butler.Butler 

83 """ 

84 

85 @abstractmethod 

86 def isWriteable(self) -> bool: 

87 """Return `True` if this registry allows write operations, and `False` 

88 otherwise. 

89 """ 

90 raise NotImplementedError() 

91 

92 @property 

93 @abstractmethod 

94 def dimensions(self) -> DimensionUniverse: 

95 """Definitions of all dimensions recognized by this `Registry` 

96 (`DimensionUniverse`). 

97 """ 

98 raise NotImplementedError() 

99 

100 @property 

101 @abstractmethod 

102 def defaults(self) -> RegistryDefaults: 

103 """Default collection search path and/or output `~CollectionType.RUN` 

104 collection (`~lsst.daf.butler.registry.RegistryDefaults`). 

105 

106 This is an immutable struct whose components may not be set 

107 individually, but the entire struct can be set by assigning to this 

108 property. 

109 """ 

110 raise NotImplementedError() 

111 

112 @defaults.setter 

113 @abstractmethod 

114 def defaults(self, value: RegistryDefaults) -> None: 

115 raise NotImplementedError() 

116 

117 @abstractmethod 

118 def refresh(self) -> None: 

119 """Refresh all in-memory state by querying the database. 

120 

121 This may be necessary to enable querying for entities added by other 

122 registry instances after this one was constructed. 

123 """ 

124 raise NotImplementedError() 

125 

126 @abstractmethod 

127 def caching_context(self) -> contextlib.AbstractContextManager[None]: 

128 """Context manager that enables caching.""" 

129 raise NotImplementedError() 

130 

131 @contextlib.contextmanager 

132 @abstractmethod 

133 def transaction(self, *, savepoint: bool = False) -> Iterator[None]: 

134 """Return a context manager that represents a transaction. 

135 

136 Parameters 

137 ---------- 

138 savepoint : `bool` 

139 Whether to create a SAVEPOINT or not. 

140 """ 

141 raise NotImplementedError() 

142 

143 def resetConnectionPool(self) -> None: 

144 """Reset connection pool for registry if relevant. 

145 

146 This operation can be used reset connections to servers when 

147 using registry with fork-based multiprocessing. This method should 

148 usually be called by the child process immediately 

149 after the fork. 

150 

151 The base class implementation is a no-op. 

152 """ 

153 pass 

154 

155 @abstractmethod 

156 def registerCollection( 

157 self, name: str, type: CollectionType = CollectionType.TAGGED, doc: str | None = None 

158 ) -> bool: 

159 """Add a new collection if one with the given name does not exist. 

160 

161 Parameters 

162 ---------- 

163 name : `str` 

164 The name of the collection to create. 

165 type : `CollectionType` 

166 Enum value indicating the type of collection to create. 

167 doc : `str`, optional 

168 Documentation string for the collection. 

169 

170 Returns 

171 ------- 

172 registered : `bool` 

173 Boolean indicating whether the collection was already registered 

174 or was created by this call. 

175 

176 Notes 

177 ----- 

178 This method cannot be called within transactions, as it needs to be 

179 able to perform its own transaction to be concurrent. 

180 """ 

181 raise NotImplementedError() 

182 

183 @abstractmethod 

184 def getCollectionType(self, name: str) -> CollectionType: 

185 """Return an enumeration value indicating the type of the given 

186 collection. 

187 

188 Parameters 

189 ---------- 

190 name : `str` 

191 The name of the collection. 

192 

193 Returns 

194 ------- 

195 type : `CollectionType` 

196 Enum value indicating the type of this collection. 

197 

198 Raises 

199 ------ 

200 lsst.daf.butler.registry.MissingCollectionError 

201 Raised if no collection with the given name exists. 

202 """ 

203 raise NotImplementedError() 

204 

205 @abstractmethod 

206 def registerRun(self, name: str, doc: str | None = None) -> bool: 

207 """Add a new run if one with the given name does not exist. 

208 

209 Parameters 

210 ---------- 

211 name : `str` 

212 The name of the run to create. 

213 doc : `str`, optional 

214 Documentation string for the collection. 

215 

216 Returns 

217 ------- 

218 registered : `bool` 

219 Boolean indicating whether a new run was registered. `False` 

220 if it already existed. 

221 

222 Notes 

223 ----- 

224 This method cannot be called within transactions, as it needs to be 

225 able to perform its own transaction to be concurrent. 

226 """ 

227 raise NotImplementedError() 

228 

229 @abstractmethod 

230 def removeCollection(self, name: str) -> None: 

231 """Remove the given collection from the registry. 

232 

233 Parameters 

234 ---------- 

235 name : `str` 

236 The name of the collection to remove. 

237 

238 Raises 

239 ------ 

240 lsst.daf.butler.registry.MissingCollectionError 

241 Raised if no collection with the given name exists. 

242 sqlalchemy.exc.IntegrityError 

243 Raised if the database rows associated with the collection are 

244 still referenced by some other table, such as a dataset in a 

245 datastore (for `~CollectionType.RUN` collections only) or a 

246 `~CollectionType.CHAINED` collection of which this collection is 

247 a child. 

248 

249 Notes 

250 ----- 

251 If this is a `~CollectionType.RUN` collection, all datasets and quanta 

252 in it will removed from the `Registry` database. This requires that 

253 those datasets be removed (or at least trashed) from any datastores 

254 that hold them first. 

255 

256 A collection may not be deleted as long as it is referenced by a 

257 `~CollectionType.CHAINED` collection; the ``CHAINED`` collection must 

258 be deleted or redefined first. 

259 """ 

260 raise NotImplementedError() 

261 

262 @abstractmethod 

263 def getCollectionChain(self, parent: str) -> Sequence[str]: 

264 """Return the child collections in a `~CollectionType.CHAINED` 

265 collection. 

266 

267 Parameters 

268 ---------- 

269 parent : `str` 

270 Name of the chained collection. Must have already been added via 

271 a call to `Registry.registerCollection`. 

272 

273 Returns 

274 ------- 

275 children : `~collections.abc.Sequence` [ `str` ] 

276 An ordered sequence of collection names that are searched when the 

277 given chained collection is searched. 

278 

279 Raises 

280 ------ 

281 lsst.daf.butler.registry.MissingCollectionError 

282 Raised if ``parent`` does not exist in the `Registry`. 

283 lsst.daf.butler.registry.CollectionTypeError 

284 Raised if ``parent`` does not correspond to a 

285 `~CollectionType.CHAINED` collection. 

286 """ 

287 raise NotImplementedError() 

288 

289 @abstractmethod 

290 def setCollectionChain(self, parent: str, children: Any, *, flatten: bool = False) -> None: 

291 """Define or redefine a `~CollectionType.CHAINED` collection. 

292 

293 Parameters 

294 ---------- 

295 parent : `str` 

296 Name of the chained collection. Must have already been added via 

297 a call to `Registry.registerCollection`. 

298 children : collection expression 

299 An expression defining an ordered search of child collections, 

300 generally an iterable of `str`; see 

301 :ref:`daf_butler_collection_expressions` for more information. 

302 flatten : `bool`, optional 

303 If `True` (`False` is default), recursively flatten out any nested 

304 `~CollectionType.CHAINED` collections in ``children`` first. 

305 

306 Raises 

307 ------ 

308 lsst.daf.butler.registry.MissingCollectionError 

309 Raised when any of the given collections do not exist in the 

310 `Registry`. 

311 lsst.daf.butler.registry.CollectionTypeError 

312 Raised if ``parent`` does not correspond to a 

313 `~CollectionType.CHAINED` collection. 

314 ValueError 

315 Raised if the given collections contains a cycle. 

316 """ 

317 raise NotImplementedError() 

318 

319 @abstractmethod 

320 def getCollectionParentChains(self, collection: str) -> set[str]: 

321 """Return the CHAINED collections that directly contain the given one. 

322 

323 Parameters 

324 ---------- 

325 collection : `str` 

326 Name of the collection. 

327 

328 Returns 

329 ------- 

330 chains : `set` of `str` 

331 Set of `~CollectionType.CHAINED` collection names. 

332 """ 

333 raise NotImplementedError() 

334 

335 @abstractmethod 

336 def getCollectionDocumentation(self, collection: str) -> str | None: 

337 """Retrieve the documentation string for a collection. 

338 

339 Parameters 

340 ---------- 

341 collection : `str` 

342 Name of the collection. 

343 

344 Returns 

345 ------- 

346 docs : `str` or `None` 

347 Docstring for the collection with the given name. 

348 """ 

349 raise NotImplementedError() 

350 

351 @abstractmethod 

352 def setCollectionDocumentation(self, collection: str, doc: str | None) -> None: 

353 """Set the documentation string for a collection. 

354 

355 Parameters 

356 ---------- 

357 collection : `str` 

358 Name of the collection. 

359 doc : `str` or `None` 

360 Docstring for the collection with the given name; will replace any 

361 existing docstring. Passing `None` will remove any existing 

362 docstring. 

363 """ 

364 raise NotImplementedError() 

365 

366 @abstractmethod 

367 def getCollectionSummary(self, collection: str) -> CollectionSummary: 

368 """Return a summary for the given collection. 

369 

370 Parameters 

371 ---------- 

372 collection : `str` 

373 Name of the collection for which a summary is to be retrieved. 

374 

375 Returns 

376 ------- 

377 summary : `~lsst.daf.butler.registry.CollectionSummary` 

378 Summary of the dataset types and governor dimension values in 

379 this collection. 

380 """ 

381 raise NotImplementedError() 

382 

383 @abstractmethod 

384 def registerDatasetType(self, datasetType: DatasetType) -> bool: 

385 """Add a new `DatasetType` to the Registry. 

386 

387 It is not an error to register the same `DatasetType` twice. 

388 

389 Parameters 

390 ---------- 

391 datasetType : `DatasetType` 

392 The `DatasetType` to be added. 

393 

394 Returns 

395 ------- 

396 inserted : `bool` 

397 `True` if ``datasetType`` was inserted, `False` if an identical 

398 existing `DatasetType` was found. Note that in either case the 

399 DatasetType is guaranteed to be defined in the Registry 

400 consistently with the given definition. 

401 

402 Raises 

403 ------ 

404 ValueError 

405 Raised if the dimensions or storage class are invalid. 

406 lsst.daf.butler.registry.ConflictingDefinitionError 

407 Raised if this `DatasetType` is already registered with a different 

408 definition. 

409 

410 Notes 

411 ----- 

412 This method cannot be called within transactions, as it needs to be 

413 able to perform its own transaction to be concurrent. 

414 """ 

415 raise NotImplementedError() 

416 

417 @abstractmethod 

418 def removeDatasetType(self, name: str | tuple[str, ...]) -> None: 

419 """Remove the named `DatasetType` from the registry. 

420 

421 .. warning:: 

422 

423 Registry implementations can cache the dataset type definitions. 

424 This means that deleting the dataset type definition may result in 

425 unexpected behavior from other butler processes that are active 

426 that have not seen the deletion. 

427 

428 Parameters 

429 ---------- 

430 name : `str` or `tuple` [`str`] 

431 Name of the type to be removed or tuple containing a list of type 

432 names to be removed. Wildcards are allowed. 

433 

434 Raises 

435 ------ 

436 lsst.daf.butler.registry.OrphanedRecordError 

437 Raised if an attempt is made to remove the dataset type definition 

438 when there are already datasets associated with it. 

439 

440 Notes 

441 ----- 

442 If the dataset type is not registered the method will return without 

443 action. 

444 """ 

445 raise NotImplementedError() 

446 

447 @abstractmethod 

448 def getDatasetType(self, name: str) -> DatasetType: 

449 """Get the `DatasetType`. 

450 

451 Parameters 

452 ---------- 

453 name : `str` 

454 Name of the type. 

455 

456 Returns 

457 ------- 

458 type : `DatasetType` 

459 The `DatasetType` associated with the given name. 

460 

461 Raises 

462 ------ 

463 lsst.daf.butler.registry.MissingDatasetTypeError 

464 Raised if the requested dataset type has not been registered. 

465 

466 Notes 

467 ----- 

468 This method handles component dataset types automatically, though most 

469 other registry operations do not. 

470 """ 

471 raise NotImplementedError() 

472 

473 @abstractmethod 

474 def supportsIdGenerationMode(self, mode: DatasetIdGenEnum) -> bool: 

475 """Test whether the given dataset ID generation mode is supported by 

476 `insertDatasets`. 

477 

478 Parameters 

479 ---------- 

480 mode : `DatasetIdGenEnum` 

481 Enum value for the mode to test. 

482 

483 Returns 

484 ------- 

485 supported : `bool` 

486 Whether the given mode is supported. 

487 """ 

488 raise NotImplementedError() 

489 

490 @abstractmethod 

491 def findDataset( 

492 self, 

493 datasetType: DatasetType | str, 

494 dataId: DataId | None = None, 

495 *, 

496 collections: CollectionArgType | None = None, 

497 timespan: Timespan | None = None, 

498 datastore_records: bool = False, 

499 **kwargs: Any, 

500 ) -> DatasetRef | None: 

501 """Find a dataset given its `DatasetType` and data ID. 

502 

503 This can be used to obtain a `DatasetRef` that permits the dataset to 

504 be read from a `Datastore`. If the dataset is a component and can not 

505 be found using the provided dataset type, a dataset ref for the parent 

506 will be returned instead but with the correct dataset type. 

507 

508 Parameters 

509 ---------- 

510 datasetType : `DatasetType` or `str` 

511 A `DatasetType` or the name of one. If this is a `DatasetType` 

512 instance, its storage class will be respected and propagated to 

513 the output, even if it differs from the dataset type definition 

514 in the registry, as long as the storage classes are convertible. 

515 dataId : `dict` or `DataCoordinate`, optional 

516 A `dict`-like object containing the `Dimension` links that identify 

517 the dataset within a collection. 

518 collections : collection expression, optional 

519 An expression that fully or partially identifies the collections to 

520 search for the dataset; see 

521 :ref:`daf_butler_collection_expressions` for more information. 

522 Defaults to ``self.defaults.collections``. 

523 timespan : `Timespan`, optional 

524 A timespan that the validity range of the dataset must overlap. 

525 If not provided, any `~CollectionType.CALIBRATION` collections 

526 matched by the ``collections`` argument will not be searched. 

527 datastore_records : `bool`, optional 

528 Whether to attach datastore records to the `DatasetRef`. 

529 **kwargs 

530 Additional keyword arguments passed to 

531 `DataCoordinate.standardize` to convert ``dataId`` to a true 

532 `DataCoordinate` or augment an existing one. 

533 

534 Returns 

535 ------- 

536 ref : `DatasetRef` 

537 A reference to the dataset, or `None` if no matching Dataset 

538 was found. 

539 

540 Raises 

541 ------ 

542 lsst.daf.butler.registry.NoDefaultCollectionError 

543 Raised if ``collections`` is `None` and 

544 ``self.defaults.collections`` is `None`. 

545 LookupError 

546 Raised if one or more data ID keys are missing. 

547 lsst.daf.butler.registry.MissingDatasetTypeError 

548 Raised if the dataset type does not exist. 

549 lsst.daf.butler.registry.MissingCollectionError 

550 Raised if any of ``collections`` does not exist in the registry. 

551 

552 Notes 

553 ----- 

554 This method simply returns `None` and does not raise an exception even 

555 when the set of collections searched is intrinsically incompatible with 

556 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but 

557 only `~CollectionType.CALIBRATION` collections are being searched. 

558 This may make it harder to debug some lookup failures, but the behavior 

559 is intentional; we consider it more important that failed searches are 

560 reported consistently, regardless of the reason, and that adding 

561 additional collections that do not contain a match to the search path 

562 never changes the behavior. 

563 

564 This method handles component dataset types automatically, though most 

565 other registry operations do not. 

566 """ 

567 raise NotImplementedError() 

568 

569 @abstractmethod 

570 def insertDatasets( 

571 self, 

572 datasetType: DatasetType | str, 

573 dataIds: Iterable[DataId], 

574 run: str | None = None, 

575 expand: bool = True, 

576 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE, 

577 ) -> list[DatasetRef]: 

578 """Insert one or more datasets into the `Registry`. 

579 

580 This always adds new datasets; to associate existing datasets with 

581 a new collection, use ``associate``. 

582 

583 Parameters 

584 ---------- 

585 datasetType : `DatasetType` or `str` 

586 A `DatasetType` or the name of one. 

587 dataIds : `~collections.abc.Iterable` of `dict` or `DataCoordinate` 

588 Dimension-based identifiers for the new datasets. 

589 run : `str`, optional 

590 The name of the run that produced the datasets. Defaults to 

591 ``self.defaults.run``. 

592 expand : `bool`, optional 

593 If `True` (default), expand data IDs as they are inserted. This is 

594 necessary in general to allow datastore to generate file templates, 

595 but it may be disabled if the caller can guarantee this is 

596 unnecessary. 

597 idGenerationMode : `DatasetIdGenEnum`, optional 

598 Specifies option for generating dataset IDs. By default unique IDs 

599 are generated for each inserted dataset. 

600 

601 Returns 

602 ------- 

603 refs : `list` of `DatasetRef` 

604 Resolved `DatasetRef` instances for all given data IDs (in the same 

605 order). 

606 

607 Raises 

608 ------ 

609 lsst.daf.butler.registry.DatasetTypeError 

610 Raised if ``datasetType`` is not known to registry. 

611 lsst.daf.butler.registry.CollectionTypeError 

612 Raised if ``run`` collection type is not `~CollectionType.RUN`. 

613 lsst.daf.butler.registry.NoDefaultCollectionError 

614 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

615 lsst.daf.butler.registry.ConflictingDefinitionError 

616 If a dataset with the same dataset type and data ID as one of those 

617 given already exists in ``run``. 

618 lsst.daf.butler.registry.MissingCollectionError 

619 Raised if ``run`` does not exist in the registry. 

620 """ 

621 raise NotImplementedError() 

622 

623 @abstractmethod 

624 def _importDatasets( 

625 self, 

626 datasets: Iterable[DatasetRef], 

627 expand: bool = True, 

628 ) -> list[DatasetRef]: 

629 """Import one or more datasets into the `Registry`. 

630 

631 Difference from `insertDatasets` method is that this method accepts 

632 `DatasetRef` instances which should already be resolved and have a 

633 dataset ID. If registry supports globally-unique dataset IDs (e.g. 

634 `uuid.UUID`) then datasets which already exist in the registry will be 

635 ignored if imported again. 

636 

637 Parameters 

638 ---------- 

639 datasets : `~collections.abc.Iterable` of `DatasetRef` 

640 Datasets to be inserted. All `DatasetRef` instances must have 

641 identical ``datasetType`` and ``run`` attributes. ``run`` 

642 attribute can be `None` and defaults to ``self.defaults.run``. 

643 Datasets can specify ``id`` attribute which will be used for 

644 inserted datasets. All dataset IDs must have the same type 

645 (`int` or `uuid.UUID`), if type of dataset IDs does not match 

646 configured backend then IDs will be ignored and new IDs will be 

647 generated by backend. 

648 expand : `bool`, optional 

649 If `True` (default), expand data IDs as they are inserted. This is 

650 necessary in general, but it may be disabled if the caller can 

651 guarantee this is unnecessary. 

652 

653 Returns 

654 ------- 

655 refs : `list` of `DatasetRef` 

656 Resolved `DatasetRef` instances for all given data IDs (in the same 

657 order). If any of ``datasets`` has an ID which already exists in 

658 the database then it will not be inserted or updated, but a 

659 resolved `DatasetRef` will be returned for it in any case. 

660 

661 Raises 

662 ------ 

663 lsst.daf.butler.registry.NoDefaultCollectionError 

664 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

665 lsst.daf.butler.registry.DatasetTypeError 

666 Raised if datasets correspond to more than one dataset type or 

667 dataset type is not known to registry. 

668 lsst.daf.butler.registry.ConflictingDefinitionError 

669 If a dataset with the same dataset type and data ID as one of those 

670 given already exists in ``run``. 

671 lsst.daf.butler.registry.MissingCollectionError 

672 Raised if ``run`` does not exist in the registry. 

673 

674 Notes 

675 ----- 

676 This method is considered package-private and internal to Butler 

677 implementation. Clients outside daf_butler package should not use this 

678 method. 

679 """ 

680 raise NotImplementedError() 

681 

682 @abstractmethod 

683 def getDataset(self, id: DatasetId) -> DatasetRef | None: 

684 """Retrieve a Dataset entry. 

685 

686 Parameters 

687 ---------- 

688 id : `DatasetId` 

689 The unique identifier for the dataset. 

690 

691 Returns 

692 ------- 

693 ref : `DatasetRef` or `None` 

694 A ref to the Dataset, or `None` if no matching Dataset 

695 was found. 

696 """ 

697 raise NotImplementedError() 

698 

699 @abstractmethod 

700 def removeDatasets(self, refs: Iterable[DatasetRef]) -> None: 

701 """Remove datasets from the Registry. 

702 

703 The datasets will be removed unconditionally from all collections, and 

704 any `Quantum` that consumed this dataset will instead be marked with 

705 having a NULL input. `Datastore` records will *not* be deleted; the 

706 caller is responsible for ensuring that the dataset has already been 

707 removed from all Datastores. 

708 

709 Parameters 

710 ---------- 

711 refs : `~collections.abc.Iterable` [`DatasetRef`] 

712 References to the datasets to be removed. Must include a valid 

713 ``id`` attribute, and should be considered invalidated upon return. 

714 

715 Raises 

716 ------ 

717 lsst.daf.butler.AmbiguousDatasetError 

718 Raised if any ``ref.id`` is `None`. 

719 lsst.daf.butler.registry.OrphanedRecordError 

720 Raised if any dataset is still present in any `Datastore`. 

721 """ 

722 raise NotImplementedError() 

723 

724 @abstractmethod 

725 def associate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

726 """Add existing datasets to a `~CollectionType.TAGGED` collection. 

727 

728 If a DatasetRef with the same exact ID is already in a collection 

729 nothing is changed. If a `DatasetRef` with the same `DatasetType` and 

730 data ID but with different ID exists in the collection, 

731 `~lsst.daf.butler.registry.ConflictingDefinitionError` is raised. 

732 

733 Parameters 

734 ---------- 

735 collection : `str` 

736 Indicates the collection the datasets should be associated with. 

737 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

738 An iterable of resolved `DatasetRef` instances that already exist 

739 in this `Registry`. 

740 

741 Raises 

742 ------ 

743 lsst.daf.butler.registry.ConflictingDefinitionError 

744 If a Dataset with the given `DatasetRef` already exists in the 

745 given collection. 

746 lsst.daf.butler.registry.MissingCollectionError 

747 Raised if ``collection`` does not exist in the registry. 

748 lsst.daf.butler.registry.CollectionTypeError 

749 Raise adding new datasets to the given ``collection`` is not 

750 allowed. 

751 """ 

752 raise NotImplementedError() 

753 

754 @abstractmethod 

755 def disassociate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

756 """Remove existing datasets from a `~CollectionType.TAGGED` collection. 

757 

758 ``collection`` and ``ref`` combinations that are not currently 

759 associated are silently ignored. 

760 

761 Parameters 

762 ---------- 

763 collection : `str` 

764 The collection the datasets should no longer be associated with. 

765 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

766 An iterable of resolved `DatasetRef` instances that already exist 

767 in this `Registry`. 

768 

769 Raises 

770 ------ 

771 lsst.daf.butler.AmbiguousDatasetError 

772 Raised if any of the given dataset references is unresolved. 

773 lsst.daf.butler.registry.MissingCollectionError 

774 Raised if ``collection`` does not exist in the registry. 

775 lsst.daf.butler.registry.CollectionTypeError 

776 Raise adding new datasets to the given ``collection`` is not 

777 allowed. 

778 """ 

779 raise NotImplementedError() 

780 

781 @abstractmethod 

782 def certify(self, collection: str, refs: Iterable[DatasetRef], timespan: Timespan) -> None: 

783 """Associate one or more datasets with a calibration collection and a 

784 validity range within it. 

785 

786 Parameters 

787 ---------- 

788 collection : `str` 

789 The name of an already-registered `~CollectionType.CALIBRATION` 

790 collection. 

791 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

792 Datasets to be associated. 

793 timespan : `Timespan` 

794 The validity range for these datasets within the collection. 

795 

796 Raises 

797 ------ 

798 lsst.daf.butler.AmbiguousDatasetError 

799 Raised if any of the given `DatasetRef` instances is unresolved. 

800 lsst.daf.butler.registry.ConflictingDefinitionError 

801 Raised if the collection already contains a different dataset with 

802 the same `DatasetType` and data ID and an overlapping validity 

803 range. 

804 lsst.daf.butler.registry.CollectionTypeError 

805 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

806 collection or if one or more datasets are of a dataset type for 

807 which `DatasetType.isCalibration` returns `False`. 

808 """ 

809 raise NotImplementedError() 

810 

811 @abstractmethod 

812 def decertify( 

813 self, 

814 collection: str, 

815 datasetType: str | DatasetType, 

816 timespan: Timespan, 

817 *, 

818 dataIds: Iterable[DataId] | None = None, 

819 ) -> None: 

820 """Remove or adjust datasets to clear a validity range within a 

821 calibration collection. 

822 

823 Parameters 

824 ---------- 

825 collection : `str` 

826 The name of an already-registered `~CollectionType.CALIBRATION` 

827 collection. 

828 datasetType : `str` or `DatasetType` 

829 Name or `DatasetType` instance for the datasets to be decertified. 

830 timespan : `Timespan`, optional 

831 The validity range to remove datasets from within the collection. 

832 Datasets that overlap this range but are not contained by it will 

833 have their validity ranges adjusted to not overlap it, which may 

834 split a single dataset validity range into two. 

835 dataIds : iterable [`dict` or `DataCoordinate`], optional 

836 Data IDs that should be decertified within the given validity range 

837 If `None`, all data IDs for ``self.datasetType`` will be 

838 decertified. 

839 

840 Raises 

841 ------ 

842 lsst.daf.butler.registry.CollectionTypeError 

843 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

844 collection or if ``datasetType.isCalibration() is False``. 

845 """ 

846 raise NotImplementedError() 

847 

848 @abstractmethod 

849 def getDatasetLocations(self, ref: DatasetRef) -> Iterable[str]: 

850 """Retrieve datastore locations for a given dataset. 

851 

852 Parameters 

853 ---------- 

854 ref : `DatasetRef` 

855 A reference to the dataset for which to retrieve storage 

856 information. 

857 

858 Returns 

859 ------- 

860 datastores : `~collections.abc.Iterable` [ `str` ] 

861 All the matching datastores holding this dataset. 

862 

863 Raises 

864 ------ 

865 lsst.daf.butler.AmbiguousDatasetError 

866 Raised if ``ref.id`` is `None`. 

867 """ 

868 raise NotImplementedError() 

869 

870 @abstractmethod 

871 def expandDataId( 

872 self, 

873 dataId: DataId | None = None, 

874 *, 

875 dimensions: Iterable[str] | DimensionGroup | DimensionGraph | None = None, 

876 graph: DimensionGraph | None = None, 

877 records: NameLookupMapping[DimensionElement, DimensionRecord | None] | None = None, 

878 withDefaults: bool = True, 

879 **kwargs: Any, 

880 ) -> DataCoordinate: 

881 """Expand a dimension-based data ID to include additional information. 

882 

883 Parameters 

884 ---------- 

885 dataId : `DataCoordinate` or `dict`, optional 

886 Data ID to be expanded; augmented and overridden by ``kwargs``. 

887 dimensions : `~collections.abc.Iterable` [ `str` ], \ 

888 `DimensionGroup`, or `DimensionGraph`, optional 

889 The dimensions to be identified by the new `DataCoordinate`. 

890 If not provided, will be inferred from the keys of ``mapping`` and 

891 ``**kwargs``, and ``universe`` must be provided unless ``mapping`` 

892 is already a `DataCoordinate`. 

893 graph : `DimensionGraph`, optional 

894 Like ``dimensions``, but as a ``DimensionGraph`` instance. Ignored 

895 if ``dimensions`` is provided. Deprecated and will be removed 

896 after v27. 

897 records : `~collections.abc.Mapping` [`str`, `DimensionRecord`], \ 

898 optional 

899 Dimension record data to use before querying the database for that 

900 data, keyed by element name. 

901 withDefaults : `bool`, optional 

902 Utilize ``self.defaults.dataId`` to fill in missing governor 

903 dimension key-value pairs. Defaults to `True` (i.e. defaults are 

904 used). 

905 **kwargs 

906 Additional keywords are treated like additional key-value pairs for 

907 ``dataId``, extending and overriding. 

908 

909 Returns 

910 ------- 

911 expanded : `DataCoordinate` 

912 A data ID that includes full metadata for all of the dimensions it 

913 identifies, i.e. guarantees that ``expanded.hasRecords()`` and 

914 ``expanded.hasFull()`` both return `True`. 

915 

916 Raises 

917 ------ 

918 lsst.daf.butler.registry.DataIdError 

919 Raised when ``dataId`` or keyword arguments specify unknown 

920 dimensions or values, or when a resulting data ID contains 

921 contradictory key-value pairs, according to dimension 

922 relationships. 

923 

924 Notes 

925 ----- 

926 This method cannot be relied upon to reject invalid data ID values 

927 for dimensions that do actually not have any record columns. For 

928 efficiency reasons the records for these dimensions (which have only 

929 dimension key values that are given by the caller) may be constructed 

930 directly rather than obtained from the registry database. 

931 """ 

932 raise NotImplementedError() 

933 

934 @abstractmethod 

935 def insertDimensionData( 

936 self, 

937 element: DimensionElement | str, 

938 *data: Mapping[str, Any] | DimensionRecord, 

939 conform: bool = True, 

940 replace: bool = False, 

941 skip_existing: bool = False, 

942 ) -> None: 

943 """Insert one or more dimension records into the database. 

944 

945 Parameters 

946 ---------- 

947 element : `DimensionElement` or `str` 

948 The `DimensionElement` or name thereof that identifies the table 

949 records will be inserted into. 

950 *data : `dict` or `DimensionRecord` 

951 One or more records to insert. 

952 conform : `bool`, optional 

953 If `False` (`True` is default) perform no checking or conversions, 

954 and assume that ``element`` is a `DimensionElement` instance and 

955 ``data`` is a one or more `DimensionRecord` instances of the 

956 appropriate subclass. 

957 replace : `bool`, optional 

958 If `True` (`False` is default), replace existing records in the 

959 database if there is a conflict. 

960 skip_existing : `bool`, optional 

961 If `True` (`False` is default), skip insertion if a record with 

962 the same primary key values already exists. Unlike 

963 `syncDimensionData`, this will not detect when the given record 

964 differs from what is in the database, and should not be used when 

965 this is a concern. 

966 """ 

967 raise NotImplementedError() 

968 

969 @abstractmethod 

970 def syncDimensionData( 

971 self, 

972 element: DimensionElement | str, 

973 row: Mapping[str, Any] | DimensionRecord, 

974 conform: bool = True, 

975 update: bool = False, 

976 ) -> bool | dict[str, Any]: 

977 """Synchronize the given dimension record with the database, inserting 

978 if it does not already exist and comparing values if it does. 

979 

980 Parameters 

981 ---------- 

982 element : `DimensionElement` or `str` 

983 The `DimensionElement` or name thereof that identifies the table 

984 records will be inserted into. 

985 row : `dict` or `DimensionRecord` 

986 The record to insert. 

987 conform : `bool`, optional 

988 If `False` (`True` is default) perform no checking or conversions, 

989 and assume that ``element`` is a `DimensionElement` instance and 

990 ``data`` is a one or more `DimensionRecord` instances of the 

991 appropriate subclass. 

992 update : `bool`, optional 

993 If `True` (`False` is default), update the existing record in the 

994 database if there is a conflict. 

995 

996 Returns 

997 ------- 

998 inserted_or_updated : `bool` or `dict` 

999 `True` if a new row was inserted, `False` if no changes were 

1000 needed, or a `dict` mapping updated column names to their old 

1001 values if an update was performed (only possible if 

1002 ``update=True``). 

1003 

1004 Raises 

1005 ------ 

1006 lsst.daf.butler.registry.ConflictingDefinitionError 

1007 Raised if the record exists in the database (according to primary 

1008 key lookup) but is inconsistent with the given one. 

1009 """ 

1010 raise NotImplementedError() 

1011 

1012 @abstractmethod 

1013 def queryDatasetTypes( 

1014 self, 

1015 expression: Any = ..., 

1016 *, 

1017 components: bool = False, 

1018 missing: list[str] | None = None, 

1019 ) -> Iterable[DatasetType]: 

1020 """Iterate over the dataset types whose names match an expression. 

1021 

1022 Parameters 

1023 ---------- 

1024 expression : dataset type expression, optional 

1025 An expression that fully or partially identifies the dataset types 

1026 to return, such as a `str`, `re.Pattern`, or iterable thereof. 

1027 ``...`` can be used to return all dataset types, and is the 

1028 default. See :ref:`daf_butler_dataset_type_expressions` for more 

1029 information. 

1030 components : `bool`, optional 

1031 Must be `False`. Provided only for backwards compatibility. After 

1032 v27 this argument will be removed entirely. 

1033 missing : `list` of `str`, optional 

1034 String dataset type names that were explicitly given (i.e. not 

1035 regular expression patterns) but not found will be appended to this 

1036 list, if it is provided. 

1037 

1038 Returns 

1039 ------- 

1040 dataset_types : `~collections.abc.Iterable` [ `DatasetType`] 

1041 An `~collections.abc.Iterable` of `DatasetType` instances whose 

1042 names match ``expression``. 

1043 

1044 Raises 

1045 ------ 

1046 lsst.daf.butler.registry.DatasetTypeExpressionError 

1047 Raised when ``expression`` is invalid. 

1048 """ 

1049 raise NotImplementedError() 

1050 

1051 @abstractmethod 

1052 def queryCollections( 

1053 self, 

1054 expression: Any = ..., 

1055 datasetType: DatasetType | None = None, 

1056 collectionTypes: Iterable[CollectionType] | CollectionType = CollectionType.all(), 

1057 flattenChains: bool = False, 

1058 includeChains: bool | None = None, 

1059 ) -> Sequence[str]: 

1060 """Iterate over the collections whose names match an expression. 

1061 

1062 Parameters 

1063 ---------- 

1064 expression : collection expression, optional 

1065 An expression that identifies the collections to return, such as 

1066 a `str` (for full matches or partial matches via globs), 

1067 `re.Pattern` (for partial matches), or iterable thereof. ``...`` 

1068 can be used to return all collections, and is the default. 

1069 See :ref:`daf_butler_collection_expressions` for more information. 

1070 datasetType : `DatasetType`, optional 

1071 If provided, only yield collections that may contain datasets of 

1072 this type. This is a conservative approximation in general; it may 

1073 yield collections that do not have any such datasets. 

1074 collectionTypes : `~collections.abc.Set` [`CollectionType`] or \ 

1075 `CollectionType`, optional 

1076 If provided, only yield collections of these types. 

1077 flattenChains : `bool`, optional 

1078 If `True` (`False` is default), recursively yield the child 

1079 collections of matching `~CollectionType.CHAINED` collections. 

1080 includeChains : `bool`, optional 

1081 If `True`, yield records for matching `~CollectionType.CHAINED` 

1082 collections. Default is the opposite of ``flattenChains``: include 

1083 either CHAINED collections or their children, but not both. 

1084 

1085 Returns 

1086 ------- 

1087 collections : `~collections.abc.Sequence` [ `str` ] 

1088 The names of collections that match ``expression``. 

1089 

1090 Raises 

1091 ------ 

1092 lsst.daf.butler.registry.CollectionExpressionError 

1093 Raised when ``expression`` is invalid. 

1094 

1095 Notes 

1096 ----- 

1097 The order in which collections are returned is unspecified, except that 

1098 the children of a `~CollectionType.CHAINED` collection are guaranteed 

1099 to be in the order in which they are searched. When multiple parent 

1100 `~CollectionType.CHAINED` collections match the same criteria, the 

1101 order in which the two lists appear is unspecified, and the lists of 

1102 children may be incomplete if a child has multiple parents. 

1103 """ 

1104 raise NotImplementedError() 

1105 

1106 @abstractmethod 

1107 def queryDatasets( 

1108 self, 

1109 datasetType: Any, 

1110 *, 

1111 collections: CollectionArgType | None = None, 

1112 dimensions: Iterable[Dimension | str] | None = None, 

1113 dataId: DataId | None = None, 

1114 where: str = "", 

1115 findFirst: bool = False, 

1116 components: bool = False, 

1117 bind: Mapping[str, Any] | None = None, 

1118 check: bool = True, 

1119 **kwargs: Any, 

1120 ) -> DatasetQueryResults: 

1121 """Query for and iterate over dataset references matching user-provided 

1122 criteria. 

1123 

1124 Parameters 

1125 ---------- 

1126 datasetType : dataset type expression 

1127 An expression that fully or partially identifies the dataset types 

1128 to be queried. Allowed types include `DatasetType`, `str`, 

1129 `re.Pattern`, and iterables thereof. The special value ``...`` can 

1130 be used to query all dataset types. See 

1131 :ref:`daf_butler_dataset_type_expressions` for more information. 

1132 collections : collection expression, optional 

1133 An expression that identifies the collections to search, such as a 

1134 `str` (for full matches or partial matches via globs), `re.Pattern` 

1135 (for partial matches), or iterable thereof. ``...`` can be used to 

1136 search all collections (actually just all `~CollectionType.RUN` 

1137 collections, because this will still find all datasets). 

1138 If not provided, ``self.default.collections`` is used. See 

1139 :ref:`daf_butler_collection_expressions` for more information. 

1140 dimensions : `~collections.abc.Iterable` of `Dimension` or `str` 

1141 Dimensions to include in the query (in addition to those used 

1142 to identify the queried dataset type(s)), either to constrain 

1143 the resulting datasets to those for which a matching dimension 

1144 exists, or to relate the dataset type's dimensions to dimensions 

1145 referenced by the ``dataId`` or ``where`` arguments. 

1146 dataId : `dict` or `DataCoordinate`, optional 

1147 A data ID whose key-value pairs are used as equality constraints 

1148 in the query. 

1149 where : `str`, optional 

1150 A string expression similar to a SQL WHERE clause. May involve 

1151 any column of a dimension table or (as a shortcut for the primary 

1152 key column of a dimension table) dimension name. See 

1153 :ref:`daf_butler_dimension_expressions` for more information. 

1154 findFirst : `bool`, optional 

1155 If `True` (`False` is default), for each result data ID, only 

1156 yield one `DatasetRef` of each `DatasetType`, from the first 

1157 collection in which a dataset of that dataset type appears 

1158 (according to the order of ``collections`` passed in). If `True`, 

1159 ``collections`` must not contain regular expressions and may not 

1160 be ``...``. 

1161 components : `bool`, optional 

1162 Must be `False`. Provided only for backwards compatibility. After 

1163 v27 this argument will be removed entirely. 

1164 bind : `~collections.abc.Mapping`, optional 

1165 Mapping containing literal values that should be injected into the 

1166 ``where`` expression, keyed by the identifiers they replace. 

1167 Values of collection type can be expanded in some cases; see 

1168 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1169 information. 

1170 check : `bool`, optional 

1171 If `True` (default) check the query for consistency before 

1172 executing it. This may reject some valid queries that resemble 

1173 common mistakes (e.g. queries for visits without specifying an 

1174 instrument). 

1175 **kwargs 

1176 Additional keyword arguments are forwarded to 

1177 `DataCoordinate.standardize` when processing the ``dataId`` 

1178 argument (and may be used to provide a constraining data ID even 

1179 when the ``dataId`` argument is `None`). 

1180 

1181 Returns 

1182 ------- 

1183 refs : `.queries.DatasetQueryResults` 

1184 Dataset references matching the given query criteria. Nested data 

1185 IDs are guaranteed to include values for all implied dimensions 

1186 (i.e. `DataCoordinate.hasFull` will return `True`), but will not 

1187 include dimension records (`DataCoordinate.hasRecords` will be 

1188 `False`) unless `~.queries.DatasetQueryResults.expanded` is 

1189 called on the result object (which returns a new one). 

1190 

1191 Raises 

1192 ------ 

1193 lsst.daf.butler.registry.DatasetTypeExpressionError 

1194 Raised when ``datasetType`` expression is invalid. 

1195 TypeError 

1196 Raised when the arguments are incompatible, such as when a 

1197 collection wildcard is passed when ``findFirst`` is `True`, or 

1198 when ``collections`` is `None` and ``self.defaults.collections`` is 

1199 also `None`. 

1200 lsst.daf.butler.registry.DataIdError 

1201 Raised when ``dataId`` or keyword arguments specify unknown 

1202 dimensions or values, or when they contain inconsistent values. 

1203 lsst.daf.butler.registry.UserExpressionError 

1204 Raised when ``where`` expression is invalid. 

1205 

1206 Notes 

1207 ----- 

1208 When multiple dataset types are queried in a single call, the 

1209 results of this operation are equivalent to querying for each dataset 

1210 type separately in turn, and no information about the relationships 

1211 between datasets of different types is included. In contexts where 

1212 that kind of information is important, the recommended pattern is to 

1213 use `queryDataIds` to first obtain data IDs (possibly with the 

1214 desired dataset types and collections passed as constraints to the 

1215 query), and then use multiple (generally much simpler) calls to 

1216 `queryDatasets` with the returned data IDs passed as constraints. 

1217 """ 

1218 raise NotImplementedError() 

1219 

1220 @abstractmethod 

1221 def queryDataIds( 

1222 self, 

1223 # TODO: Drop `Dimension` objects on DM-41326. 

1224 dimensions: DimensionGroup | Iterable[Dimension | str] | Dimension | str, 

1225 *, 

1226 dataId: DataId | None = None, 

1227 datasets: Any = None, 

1228 collections: CollectionArgType | None = None, 

1229 where: str = "", 

1230 components: bool = False, 

1231 bind: Mapping[str, Any] | None = None, 

1232 check: bool = True, 

1233 **kwargs: Any, 

1234 ) -> DataCoordinateQueryResults: 

1235 """Query for data IDs matching user-provided criteria. 

1236 

1237 Parameters 

1238 ---------- 

1239 dimensions : `DimensionGroup`, `Dimension`, or `str`, or \ 

1240 `~collections.abc.Iterable` [ `Dimension` or `str` ] 

1241 The dimensions of the data IDs to yield, as either `Dimension` 

1242 instances or `str`. Will be automatically expanded to a complete 

1243 `DimensionGroup`. Support for `Dimension` instances is deprecated 

1244 and will not be supported after v27. 

1245 dataId : `dict` or `DataCoordinate`, optional 

1246 A data ID whose key-value pairs are used as equality constraints 

1247 in the query. 

1248 datasets : dataset type expression, optional 

1249 An expression that fully or partially identifies dataset types 

1250 that should constrain the yielded data IDs. For example, including 

1251 "raw" here would constrain the yielded ``instrument``, 

1252 ``exposure``, ``detector``, and ``physical_filter`` values to only 

1253 those for which at least one "raw" dataset exists in 

1254 ``collections``. Allowed types include `DatasetType`, `str`, 

1255 and iterables thereof. Regular expression objects (i.e. 

1256 `re.Pattern`) are deprecated and will be removed after the v26 

1257 release. See :ref:`daf_butler_dataset_type_expressions` for more 

1258 information. 

1259 collections : collection expression, optional 

1260 An expression that identifies the collections to search for 

1261 datasets, such as a `str` (for full matches or partial matches 

1262 via globs), `re.Pattern` (for partial matches), or iterable 

1263 thereof. ``...`` can be used to search all collections (actually 

1264 just all `~CollectionType.RUN` collections, because this will 

1265 still find all datasets). If not provided, 

1266 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1267 is also passed. See :ref:`daf_butler_collection_expressions` for 

1268 more information. 

1269 where : `str`, optional 

1270 A string expression similar to a SQL WHERE clause. May involve 

1271 any column of a dimension table or (as a shortcut for the primary 

1272 key column of a dimension table) dimension name. See 

1273 :ref:`daf_butler_dimension_expressions` for more information. 

1274 components : `bool`, optional 

1275 Must be `False`. Provided only for backwards compatibility. After 

1276 v27 this argument will be removed entirely. 

1277 bind : `~collections.abc.Mapping`, optional 

1278 Mapping containing literal values that should be injected into the 

1279 ``where`` expression, keyed by the identifiers they replace. 

1280 Values of collection type can be expanded in some cases; see 

1281 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1282 information. 

1283 check : `bool`, optional 

1284 If `True` (default) check the query for consistency before 

1285 executing it. This may reject some valid queries that resemble 

1286 common mistakes (e.g. queries for visits without specifying an 

1287 instrument). 

1288 **kwargs 

1289 Additional keyword arguments are forwarded to 

1290 `DataCoordinate.standardize` when processing the ``dataId`` 

1291 argument (and may be used to provide a constraining data ID even 

1292 when the ``dataId`` argument is `None`). 

1293 

1294 Returns 

1295 ------- 

1296 dataIds : `.queries.DataCoordinateQueryResults` 

1297 Data IDs matching the given query parameters. These are guaranteed 

1298 to identify all dimensions (`DataCoordinate.hasFull` returns 

1299 `True`), but will not contain `DimensionRecord` objects 

1300 (`DataCoordinate.hasRecords` returns `False`). Call 

1301 `~.queries.DataCoordinateQueryResults.expanded` on the 

1302 returned object to fetch those (and consider using 

1303 `~.queries.DataCoordinateQueryResults.materialize` on the 

1304 returned object first if the expected number of rows is very 

1305 large). See documentation for those methods for additional 

1306 information. 

1307 

1308 Raises 

1309 ------ 

1310 lsst.daf.butler.registry.NoDefaultCollectionError 

1311 Raised if ``collections`` is `None` and 

1312 ``self.defaults.collections`` is `None`. 

1313 lsst.daf.butler.registry.CollectionExpressionError 

1314 Raised when ``collections`` expression is invalid. 

1315 lsst.daf.butler.registry.DataIdError 

1316 Raised when ``dataId`` or keyword arguments specify unknown 

1317 dimensions or values, or when they contain inconsistent values. 

1318 lsst.daf.butler.registry.DatasetTypeExpressionError 

1319 Raised when ``datasetType`` expression is invalid. 

1320 lsst.daf.butler.registry.UserExpressionError 

1321 Raised when ``where`` expression is invalid. 

1322 """ 

1323 raise NotImplementedError() 

1324 

1325 @abstractmethod 

1326 def queryDimensionRecords( 

1327 self, 

1328 element: DimensionElement | str, 

1329 *, 

1330 dataId: DataId | None = None, 

1331 datasets: Any = None, 

1332 collections: CollectionArgType | None = None, 

1333 where: str = "", 

1334 components: bool = False, 

1335 bind: Mapping[str, Any] | None = None, 

1336 check: bool = True, 

1337 **kwargs: Any, 

1338 ) -> DimensionRecordQueryResults: 

1339 """Query for dimension information matching user-provided criteria. 

1340 

1341 Parameters 

1342 ---------- 

1343 element : `DimensionElement` or `str` 

1344 The dimension element to obtain records for. 

1345 dataId : `dict` or `DataCoordinate`, optional 

1346 A data ID whose key-value pairs are used as equality constraints 

1347 in the query. 

1348 datasets : dataset type expression, optional 

1349 An expression that fully or partially identifies dataset types 

1350 that should constrain the yielded records. See `queryDataIds` and 

1351 :ref:`daf_butler_dataset_type_expressions` for more information. 

1352 collections : collection expression, optional 

1353 An expression that identifies the collections to search for 

1354 datasets, such as a `str` (for full matches or partial matches 

1355 via globs), `re.Pattern` (for partial matches), or iterable 

1356 thereof. ``...`` can be used to search all collections (actually 

1357 just all `~CollectionType.RUN` collections, because this will 

1358 still find all datasets). If not provided, 

1359 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1360 is also passed. See :ref:`daf_butler_collection_expressions` for 

1361 more information. 

1362 where : `str`, optional 

1363 A string expression similar to a SQL WHERE clause. See 

1364 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more 

1365 information. 

1366 components : `bool`, optional 

1367 Must be `False`. Provided only for backwards compatibility. After 

1368 v27 this argument will be removed entirely. 

1369 bind : `~collections.abc.Mapping`, optional 

1370 Mapping containing literal values that should be injected into the 

1371 ``where`` expression, keyed by the identifiers they replace. 

1372 Values of collection type can be expanded in some cases; see 

1373 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1374 information. 

1375 check : `bool`, optional 

1376 If `True` (default) check the query for consistency before 

1377 executing it. This may reject some valid queries that resemble 

1378 common mistakes (e.g. queries for visits without specifying an 

1379 instrument). 

1380 **kwargs 

1381 Additional keyword arguments are forwarded to 

1382 `DataCoordinate.standardize` when processing the ``dataId`` 

1383 argument (and may be used to provide a constraining data ID even 

1384 when the ``dataId`` argument is `None`). 

1385 

1386 Returns 

1387 ------- 

1388 dataIds : `.queries.DimensionRecordQueryResults` 

1389 Data IDs matching the given query parameters. 

1390 

1391 Raises 

1392 ------ 

1393 lsst.daf.butler.registry.NoDefaultCollectionError 

1394 Raised if ``collections`` is `None` and 

1395 ``self.defaults.collections`` is `None`. 

1396 lsst.daf.butler.registry.CollectionExpressionError 

1397 Raised when ``collections`` expression is invalid. 

1398 lsst.daf.butler.registry.DataIdError 

1399 Raised when ``dataId`` or keyword arguments specify unknown 

1400 dimensions or values, or when they contain inconsistent values. 

1401 lsst.daf.butler.registry.DatasetTypeExpressionError 

1402 Raised when ``datasetType`` expression is invalid. 

1403 lsst.daf.butler.registry.UserExpressionError 

1404 Raised when ``where`` expression is invalid. 

1405 """ 

1406 raise NotImplementedError() 

1407 

1408 @abstractmethod 

1409 def queryDatasetAssociations( 

1410 self, 

1411 datasetType: str | DatasetType, 

1412 collections: CollectionArgType | None = ..., 

1413 *, 

1414 collectionTypes: Iterable[CollectionType] = CollectionType.all(), 

1415 flattenChains: bool = False, 

1416 ) -> Iterator[DatasetAssociation]: 

1417 """Iterate over dataset-collection combinations where the dataset is in 

1418 the collection. 

1419 

1420 This method is a temporary placeholder for better support for 

1421 association results in `queryDatasets`. It will probably be 

1422 removed in the future, and should be avoided in production code 

1423 whenever possible. 

1424 

1425 Parameters 

1426 ---------- 

1427 datasetType : `DatasetType` or `str` 

1428 A dataset type object or the name of one. 

1429 collections : collection expression, optional 

1430 An expression that identifies the collections to search for 

1431 datasets, such as a `str` (for full matches or partial matches 

1432 via globs), `re.Pattern` (for partial matches), or iterable 

1433 thereof. ``...`` can be used to search all collections (actually 

1434 just all `~CollectionType.RUN` collections, because this will still 

1435 find all datasets). If not provided, ``self.default.collections`` 

1436 is used. See :ref:`daf_butler_collection_expressions` for more 

1437 information. 

1438 collectionTypes : `~collections.abc.Set` [ `CollectionType` ], optional 

1439 If provided, only yield associations from collections of these 

1440 types. 

1441 flattenChains : `bool`, optional 

1442 If `True`, search in the children of `~CollectionType.CHAINED` 

1443 collections. If `False`, ``CHAINED`` collections are ignored. 

1444 

1445 Yields 

1446 ------ 

1447 association : `.DatasetAssociation` 

1448 Object representing the relationship between a single dataset and 

1449 a single collection. 

1450 

1451 Raises 

1452 ------ 

1453 lsst.daf.butler.registry.NoDefaultCollectionError 

1454 Raised if ``collections`` is `None` and 

1455 ``self.defaults.collections`` is `None`. 

1456 lsst.daf.butler.registry.CollectionExpressionError 

1457 Raised when ``collections`` expression is invalid. 

1458 """ 

1459 raise NotImplementedError() 

1460 

1461 @property 

1462 def obsCoreTableManager(self) -> ObsCoreTableManager | None: 

1463 """The ObsCore manager instance for this registry 

1464 (`~.interfaces.ObsCoreTableManager` 

1465 or `None`). 

1466 

1467 ObsCore manager may not be implemented for all registry backend, or 

1468 may not be enabled for many repositories. 

1469 """ 

1470 return None 

1471 

1472 storageClasses: StorageClassFactory 

1473 """All storage classes known to the registry (`StorageClassFactory`). 

1474 """