Coverage for python/lsst/daf/butler/registry/_registry.py: 99%

115 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-04 02:54 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ("Registry", "CollectionArgType") 

31 

32import contextlib 

33import logging 

34import re 

35from abc import ABC, abstractmethod 

36from collections.abc import Iterable, Iterator, Mapping, Sequence 

37from types import EllipsisType 

38from typing import TYPE_CHECKING, Any, TypeAlias 

39 

40from .._dataset_association import DatasetAssociation 

41from .._dataset_ref import DatasetId, DatasetIdGenEnum, DatasetRef 

42from .._dataset_type import DatasetType 

43from .._named import NameLookupMapping 

44from .._storage_class import StorageClassFactory 

45from .._timespan import Timespan 

46from ..dimensions import ( 

47 DataCoordinate, 

48 DataId, 

49 Dimension, 

50 DimensionElement, 

51 DimensionGraph, 

52 DimensionGroup, 

53 DimensionRecord, 

54 DimensionUniverse, 

55) 

56from ._collection_summary import CollectionSummary 

57from ._collection_type import CollectionType 

58from ._defaults import RegistryDefaults 

59from .queries import DataCoordinateQueryResults, DatasetQueryResults, DimensionRecordQueryResults 

60from .wildcards import CollectionWildcard 

61 

62if TYPE_CHECKING: 

63 from .interfaces import ObsCoreTableManager 

64 

65_LOG = logging.getLogger(__name__) 

66 

67# TYpe alias for `collections` arguments. 

68CollectionArgType: TypeAlias = ( 

69 str | re.Pattern | Iterable[str | re.Pattern] | EllipsisType | CollectionWildcard 

70) 

71 

72 

73class Registry(ABC): 

74 """Abstract Registry interface. 

75 

76 All subclasses should store `~lsst.daf.butler.registry.RegistryDefaults` in 

77 a ``_defaults`` property. No other properties are assumed shared between 

78 implementations. 

79 """ 

80 

81 @abstractmethod 

82 def isWriteable(self) -> bool: 

83 """Return `True` if this registry allows write operations, and `False` 

84 otherwise. 

85 """ 

86 raise NotImplementedError() 

87 

88 @property 

89 @abstractmethod 

90 def dimensions(self) -> DimensionUniverse: 

91 """Definitions of all dimensions recognized by this `Registry` 

92 (`DimensionUniverse`). 

93 """ 

94 raise NotImplementedError() 

95 

96 @property 

97 @abstractmethod 

98 def defaults(self) -> RegistryDefaults: 

99 """Default collection search path and/or output `~CollectionType.RUN` 

100 collection (`~lsst.daf.butler.registry.RegistryDefaults`). 

101 

102 This is an immutable struct whose components may not be set 

103 individually, but the entire struct can be set by assigning to this 

104 property. 

105 """ 

106 raise NotImplementedError() 

107 

108 @defaults.setter 

109 @abstractmethod 

110 def defaults(self, value: RegistryDefaults) -> None: 

111 raise NotImplementedError() 

112 

113 @abstractmethod 

114 def refresh(self) -> None: 

115 """Refresh all in-memory state by querying the database. 

116 

117 This may be necessary to enable querying for entities added by other 

118 registry instances after this one was constructed. 

119 """ 

120 raise NotImplementedError() 

121 

122 @abstractmethod 

123 def caching_context(self) -> contextlib.AbstractContextManager[None]: 

124 """Context manager that enables caching.""" 

125 raise NotImplementedError() 

126 

127 @contextlib.contextmanager 

128 @abstractmethod 

129 def transaction(self, *, savepoint: bool = False) -> Iterator[None]: 

130 """Return a context manager that represents a transaction. 

131 

132 Parameters 

133 ---------- 

134 savepoint : `bool` 

135 Whether to create a SAVEPOINT or not. 

136 """ 

137 raise NotImplementedError() 

138 

139 def resetConnectionPool(self) -> None: 

140 """Reset connection pool for registry if relevant. 

141 

142 This operation can be used reset connections to servers when 

143 using registry with fork-based multiprocessing. This method should 

144 usually be called by the child process immediately 

145 after the fork. 

146 

147 The base class implementation is a no-op. 

148 """ 

149 pass 

150 

151 @abstractmethod 

152 def registerCollection( 

153 self, name: str, type: CollectionType = CollectionType.TAGGED, doc: str | None = None 

154 ) -> bool: 

155 """Add a new collection if one with the given name does not exist. 

156 

157 Parameters 

158 ---------- 

159 name : `str` 

160 The name of the collection to create. 

161 type : `CollectionType` 

162 Enum value indicating the type of collection to create. 

163 doc : `str`, optional 

164 Documentation string for the collection. 

165 

166 Returns 

167 ------- 

168 registered : `bool` 

169 Boolean indicating whether the collection was already registered 

170 or was created by this call. 

171 

172 Notes 

173 ----- 

174 This method cannot be called within transactions, as it needs to be 

175 able to perform its own transaction to be concurrent. 

176 """ 

177 raise NotImplementedError() 

178 

179 @abstractmethod 

180 def getCollectionType(self, name: str) -> CollectionType: 

181 """Return an enumeration value indicating the type of the given 

182 collection. 

183 

184 Parameters 

185 ---------- 

186 name : `str` 

187 The name of the collection. 

188 

189 Returns 

190 ------- 

191 type : `CollectionType` 

192 Enum value indicating the type of this collection. 

193 

194 Raises 

195 ------ 

196 lsst.daf.butler.registry.MissingCollectionError 

197 Raised if no collection with the given name exists. 

198 """ 

199 raise NotImplementedError() 

200 

201 @abstractmethod 

202 def registerRun(self, name: str, doc: str | None = None) -> bool: 

203 """Add a new run if one with the given name does not exist. 

204 

205 Parameters 

206 ---------- 

207 name : `str` 

208 The name of the run to create. 

209 doc : `str`, optional 

210 Documentation string for the collection. 

211 

212 Returns 

213 ------- 

214 registered : `bool` 

215 Boolean indicating whether a new run was registered. `False` 

216 if it already existed. 

217 

218 Notes 

219 ----- 

220 This method cannot be called within transactions, as it needs to be 

221 able to perform its own transaction to be concurrent. 

222 """ 

223 raise NotImplementedError() 

224 

225 @abstractmethod 

226 def removeCollection(self, name: str) -> None: 

227 """Remove the given collection from the registry. 

228 

229 Parameters 

230 ---------- 

231 name : `str` 

232 The name of the collection to remove. 

233 

234 Raises 

235 ------ 

236 lsst.daf.butler.registry.MissingCollectionError 

237 Raised if no collection with the given name exists. 

238 sqlalchemy.exc.IntegrityError 

239 Raised if the database rows associated with the collection are 

240 still referenced by some other table, such as a dataset in a 

241 datastore (for `~CollectionType.RUN` collections only) or a 

242 `~CollectionType.CHAINED` collection of which this collection is 

243 a child. 

244 

245 Notes 

246 ----- 

247 If this is a `~CollectionType.RUN` collection, all datasets and quanta 

248 in it will removed from the `Registry` database. This requires that 

249 those datasets be removed (or at least trashed) from any datastores 

250 that hold them first. 

251 

252 A collection may not be deleted as long as it is referenced by a 

253 `~CollectionType.CHAINED` collection; the ``CHAINED`` collection must 

254 be deleted or redefined first. 

255 """ 

256 raise NotImplementedError() 

257 

258 @abstractmethod 

259 def getCollectionChain(self, parent: str) -> Sequence[str]: 

260 """Return the child collections in a `~CollectionType.CHAINED` 

261 collection. 

262 

263 Parameters 

264 ---------- 

265 parent : `str` 

266 Name of the chained collection. Must have already been added via 

267 a call to `Registry.registerCollection`. 

268 

269 Returns 

270 ------- 

271 children : `~collections.abc.Sequence` [ `str` ] 

272 An ordered sequence of collection names that are searched when the 

273 given chained collection is searched. 

274 

275 Raises 

276 ------ 

277 lsst.daf.butler.registry.MissingCollectionError 

278 Raised if ``parent`` does not exist in the `Registry`. 

279 lsst.daf.butler.registry.CollectionTypeError 

280 Raised if ``parent`` does not correspond to a 

281 `~CollectionType.CHAINED` collection. 

282 """ 

283 raise NotImplementedError() 

284 

285 @abstractmethod 

286 def setCollectionChain(self, parent: str, children: Any, *, flatten: bool = False) -> None: 

287 """Define or redefine a `~CollectionType.CHAINED` collection. 

288 

289 Parameters 

290 ---------- 

291 parent : `str` 

292 Name of the chained collection. Must have already been added via 

293 a call to `Registry.registerCollection`. 

294 children : collection expression 

295 An expression defining an ordered search of child collections, 

296 generally an iterable of `str`; see 

297 :ref:`daf_butler_collection_expressions` for more information. 

298 flatten : `bool`, optional 

299 If `True` (`False` is default), recursively flatten out any nested 

300 `~CollectionType.CHAINED` collections in ``children`` first. 

301 

302 Raises 

303 ------ 

304 lsst.daf.butler.registry.MissingCollectionError 

305 Raised when any of the given collections do not exist in the 

306 `Registry`. 

307 lsst.daf.butler.registry.CollectionTypeError 

308 Raised if ``parent`` does not correspond to a 

309 `~CollectionType.CHAINED` collection. 

310 ValueError 

311 Raised if the given collections contains a cycle. 

312 """ 

313 raise NotImplementedError() 

314 

315 @abstractmethod 

316 def getCollectionParentChains(self, collection: str) -> set[str]: 

317 """Return the CHAINED collections that directly contain the given one. 

318 

319 Parameters 

320 ---------- 

321 collection : `str` 

322 Name of the collection. 

323 

324 Returns 

325 ------- 

326 chains : `set` of `str` 

327 Set of `~CollectionType.CHAINED` collection names. 

328 """ 

329 raise NotImplementedError() 

330 

331 @abstractmethod 

332 def getCollectionDocumentation(self, collection: str) -> str | None: 

333 """Retrieve the documentation string for a collection. 

334 

335 Parameters 

336 ---------- 

337 collection : `str` 

338 Name of the collection. 

339 

340 Returns 

341 ------- 

342 docs : `str` or `None` 

343 Docstring for the collection with the given name. 

344 """ 

345 raise NotImplementedError() 

346 

347 @abstractmethod 

348 def setCollectionDocumentation(self, collection: str, doc: str | None) -> None: 

349 """Set the documentation string for a collection. 

350 

351 Parameters 

352 ---------- 

353 collection : `str` 

354 Name of the collection. 

355 doc : `str` or `None` 

356 Docstring for the collection with the given name; will replace any 

357 existing docstring. Passing `None` will remove any existing 

358 docstring. 

359 """ 

360 raise NotImplementedError() 

361 

362 @abstractmethod 

363 def getCollectionSummary(self, collection: str) -> CollectionSummary: 

364 """Return a summary for the given collection. 

365 

366 Parameters 

367 ---------- 

368 collection : `str` 

369 Name of the collection for which a summary is to be retrieved. 

370 

371 Returns 

372 ------- 

373 summary : `~lsst.daf.butler.registry.CollectionSummary` 

374 Summary of the dataset types and governor dimension values in 

375 this collection. 

376 """ 

377 raise NotImplementedError() 

378 

379 @abstractmethod 

380 def registerDatasetType(self, datasetType: DatasetType) -> bool: 

381 """Add a new `DatasetType` to the Registry. 

382 

383 It is not an error to register the same `DatasetType` twice. 

384 

385 Parameters 

386 ---------- 

387 datasetType : `DatasetType` 

388 The `DatasetType` to be added. 

389 

390 Returns 

391 ------- 

392 inserted : `bool` 

393 `True` if ``datasetType`` was inserted, `False` if an identical 

394 existing `DatasetType` was found. Note that in either case the 

395 DatasetType is guaranteed to be defined in the Registry 

396 consistently with the given definition. 

397 

398 Raises 

399 ------ 

400 ValueError 

401 Raised if the dimensions or storage class are invalid. 

402 lsst.daf.butler.registry.ConflictingDefinitionError 

403 Raised if this `DatasetType` is already registered with a different 

404 definition. 

405 

406 Notes 

407 ----- 

408 This method cannot be called within transactions, as it needs to be 

409 able to perform its own transaction to be concurrent. 

410 """ 

411 raise NotImplementedError() 

412 

413 @abstractmethod 

414 def removeDatasetType(self, name: str | tuple[str, ...]) -> None: 

415 """Remove the named `DatasetType` from the registry. 

416 

417 .. warning:: 

418 

419 Registry implementations can cache the dataset type definitions. 

420 This means that deleting the dataset type definition may result in 

421 unexpected behavior from other butler processes that are active 

422 that have not seen the deletion. 

423 

424 Parameters 

425 ---------- 

426 name : `str` or `tuple` [`str`] 

427 Name of the type to be removed or tuple containing a list of type 

428 names to be removed. Wildcards are allowed. 

429 

430 Raises 

431 ------ 

432 lsst.daf.butler.registry.OrphanedRecordError 

433 Raised if an attempt is made to remove the dataset type definition 

434 when there are already datasets associated with it. 

435 

436 Notes 

437 ----- 

438 If the dataset type is not registered the method will return without 

439 action. 

440 """ 

441 raise NotImplementedError() 

442 

443 @abstractmethod 

444 def getDatasetType(self, name: str) -> DatasetType: 

445 """Get the `DatasetType`. 

446 

447 Parameters 

448 ---------- 

449 name : `str` 

450 Name of the type. 

451 

452 Returns 

453 ------- 

454 type : `DatasetType` 

455 The `DatasetType` associated with the given name. 

456 

457 Raises 

458 ------ 

459 lsst.daf.butler.registry.MissingDatasetTypeError 

460 Raised if the requested dataset type has not been registered. 

461 

462 Notes 

463 ----- 

464 This method handles component dataset types automatically, though most 

465 other registry operations do not. 

466 """ 

467 raise NotImplementedError() 

468 

469 @abstractmethod 

470 def supportsIdGenerationMode(self, mode: DatasetIdGenEnum) -> bool: 

471 """Test whether the given dataset ID generation mode is supported by 

472 `insertDatasets`. 

473 

474 Parameters 

475 ---------- 

476 mode : `DatasetIdGenEnum` 

477 Enum value for the mode to test. 

478 

479 Returns 

480 ------- 

481 supported : `bool` 

482 Whether the given mode is supported. 

483 """ 

484 raise NotImplementedError() 

485 

486 @abstractmethod 

487 def findDataset( 

488 self, 

489 datasetType: DatasetType | str, 

490 dataId: DataId | None = None, 

491 *, 

492 collections: CollectionArgType | None = None, 

493 timespan: Timespan | None = None, 

494 datastore_records: bool = False, 

495 **kwargs: Any, 

496 ) -> DatasetRef | None: 

497 """Find a dataset given its `DatasetType` and data ID. 

498 

499 This can be used to obtain a `DatasetRef` that permits the dataset to 

500 be read from a `Datastore`. If the dataset is a component and can not 

501 be found using the provided dataset type, a dataset ref for the parent 

502 will be returned instead but with the correct dataset type. 

503 

504 Parameters 

505 ---------- 

506 datasetType : `DatasetType` or `str` 

507 A `DatasetType` or the name of one. If this is a `DatasetType` 

508 instance, its storage class will be respected and propagated to 

509 the output, even if it differs from the dataset type definition 

510 in the registry, as long as the storage classes are convertible. 

511 dataId : `dict` or `DataCoordinate`, optional 

512 A `dict`-like object containing the `Dimension` links that identify 

513 the dataset within a collection. 

514 collections : collection expression, optional 

515 An expression that fully or partially identifies the collections to 

516 search for the dataset; see 

517 :ref:`daf_butler_collection_expressions` for more information. 

518 Defaults to ``self.defaults.collections``. 

519 timespan : `Timespan`, optional 

520 A timespan that the validity range of the dataset must overlap. 

521 If not provided, any `~CollectionType.CALIBRATION` collections 

522 matched by the ``collections`` argument will not be searched. 

523 datastore_records : `bool`, optional 

524 Whether to attach datastore records to the `DatasetRef`. 

525 **kwargs 

526 Additional keyword arguments passed to 

527 `DataCoordinate.standardize` to convert ``dataId`` to a true 

528 `DataCoordinate` or augment an existing one. 

529 

530 Returns 

531 ------- 

532 ref : `DatasetRef` 

533 A reference to the dataset, or `None` if no matching Dataset 

534 was found. 

535 

536 Raises 

537 ------ 

538 lsst.daf.butler.registry.NoDefaultCollectionError 

539 Raised if ``collections`` is `None` and 

540 ``self.defaults.collections`` is `None`. 

541 LookupError 

542 Raised if one or more data ID keys are missing. 

543 lsst.daf.butler.registry.MissingDatasetTypeError 

544 Raised if the dataset type does not exist. 

545 lsst.daf.butler.registry.MissingCollectionError 

546 Raised if any of ``collections`` does not exist in the registry. 

547 

548 Notes 

549 ----- 

550 This method simply returns `None` and does not raise an exception even 

551 when the set of collections searched is intrinsically incompatible with 

552 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but 

553 only `~CollectionType.CALIBRATION` collections are being searched. 

554 This may make it harder to debug some lookup failures, but the behavior 

555 is intentional; we consider it more important that failed searches are 

556 reported consistently, regardless of the reason, and that adding 

557 additional collections that do not contain a match to the search path 

558 never changes the behavior. 

559 

560 This method handles component dataset types automatically, though most 

561 other registry operations do not. 

562 """ 

563 raise NotImplementedError() 

564 

565 @abstractmethod 

566 def insertDatasets( 

567 self, 

568 datasetType: DatasetType | str, 

569 dataIds: Iterable[DataId], 

570 run: str | None = None, 

571 expand: bool = True, 

572 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE, 

573 ) -> list[DatasetRef]: 

574 """Insert one or more datasets into the `Registry`. 

575 

576 This always adds new datasets; to associate existing datasets with 

577 a new collection, use ``associate``. 

578 

579 Parameters 

580 ---------- 

581 datasetType : `DatasetType` or `str` 

582 A `DatasetType` or the name of one. 

583 dataIds : `~collections.abc.Iterable` of `dict` or `DataCoordinate` 

584 Dimension-based identifiers for the new datasets. 

585 run : `str`, optional 

586 The name of the run that produced the datasets. Defaults to 

587 ``self.defaults.run``. 

588 expand : `bool`, optional 

589 If `True` (default), expand data IDs as they are inserted. This is 

590 necessary in general to allow datastore to generate file templates, 

591 but it may be disabled if the caller can guarantee this is 

592 unnecessary. 

593 idGenerationMode : `DatasetIdGenEnum`, optional 

594 Specifies option for generating dataset IDs. By default unique IDs 

595 are generated for each inserted dataset. 

596 

597 Returns 

598 ------- 

599 refs : `list` of `DatasetRef` 

600 Resolved `DatasetRef` instances for all given data IDs (in the same 

601 order). 

602 

603 Raises 

604 ------ 

605 lsst.daf.butler.registry.DatasetTypeError 

606 Raised if ``datasetType`` is not known to registry. 

607 lsst.daf.butler.registry.CollectionTypeError 

608 Raised if ``run`` collection type is not `~CollectionType.RUN`. 

609 lsst.daf.butler.registry.NoDefaultCollectionError 

610 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

611 lsst.daf.butler.registry.ConflictingDefinitionError 

612 If a dataset with the same dataset type and data ID as one of those 

613 given already exists in ``run``. 

614 lsst.daf.butler.registry.MissingCollectionError 

615 Raised if ``run`` does not exist in the registry. 

616 """ 

617 raise NotImplementedError() 

618 

619 @abstractmethod 

620 def _importDatasets( 

621 self, 

622 datasets: Iterable[DatasetRef], 

623 expand: bool = True, 

624 ) -> list[DatasetRef]: 

625 """Import one or more datasets into the `Registry`. 

626 

627 Difference from `insertDatasets` method is that this method accepts 

628 `DatasetRef` instances which should already be resolved and have a 

629 dataset ID. If registry supports globally-unique dataset IDs (e.g. 

630 `uuid.UUID`) then datasets which already exist in the registry will be 

631 ignored if imported again. 

632 

633 Parameters 

634 ---------- 

635 datasets : `~collections.abc.Iterable` of `DatasetRef` 

636 Datasets to be inserted. All `DatasetRef` instances must have 

637 identical ``datasetType`` and ``run`` attributes. ``run`` 

638 attribute can be `None` and defaults to ``self.defaults.run``. 

639 Datasets can specify ``id`` attribute which will be used for 

640 inserted datasets. All dataset IDs must have the same type 

641 (`int` or `uuid.UUID`), if type of dataset IDs does not match 

642 configured backend then IDs will be ignored and new IDs will be 

643 generated by backend. 

644 expand : `bool`, optional 

645 If `True` (default), expand data IDs as they are inserted. This is 

646 necessary in general, but it may be disabled if the caller can 

647 guarantee this is unnecessary. 

648 

649 Returns 

650 ------- 

651 refs : `list` of `DatasetRef` 

652 Resolved `DatasetRef` instances for all given data IDs (in the same 

653 order). If any of ``datasets`` has an ID which already exists in 

654 the database then it will not be inserted or updated, but a 

655 resolved `DatasetRef` will be returned for it in any case. 

656 

657 Raises 

658 ------ 

659 lsst.daf.butler.registry.NoDefaultCollectionError 

660 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

661 lsst.daf.butler.registry.DatasetTypeError 

662 Raised if datasets correspond to more than one dataset type or 

663 dataset type is not known to registry. 

664 lsst.daf.butler.registry.ConflictingDefinitionError 

665 If a dataset with the same dataset type and data ID as one of those 

666 given already exists in ``run``. 

667 lsst.daf.butler.registry.MissingCollectionError 

668 Raised if ``run`` does not exist in the registry. 

669 

670 Notes 

671 ----- 

672 This method is considered package-private and internal to Butler 

673 implementation. Clients outside daf_butler package should not use this 

674 method. 

675 """ 

676 raise NotImplementedError() 

677 

678 @abstractmethod 

679 def getDataset(self, id: DatasetId) -> DatasetRef | None: 

680 """Retrieve a Dataset entry. 

681 

682 Parameters 

683 ---------- 

684 id : `DatasetId` 

685 The unique identifier for the dataset. 

686 

687 Returns 

688 ------- 

689 ref : `DatasetRef` or `None` 

690 A ref to the Dataset, or `None` if no matching Dataset 

691 was found. 

692 """ 

693 raise NotImplementedError() 

694 

695 @abstractmethod 

696 def removeDatasets(self, refs: Iterable[DatasetRef]) -> None: 

697 """Remove datasets from the Registry. 

698 

699 The datasets will be removed unconditionally from all collections, and 

700 any `Quantum` that consumed this dataset will instead be marked with 

701 having a NULL input. `Datastore` records will *not* be deleted; the 

702 caller is responsible for ensuring that the dataset has already been 

703 removed from all Datastores. 

704 

705 Parameters 

706 ---------- 

707 refs : `~collections.abc.Iterable` [`DatasetRef`] 

708 References to the datasets to be removed. Must include a valid 

709 ``id`` attribute, and should be considered invalidated upon return. 

710 

711 Raises 

712 ------ 

713 lsst.daf.butler.AmbiguousDatasetError 

714 Raised if any ``ref.id`` is `None`. 

715 lsst.daf.butler.registry.OrphanedRecordError 

716 Raised if any dataset is still present in any `Datastore`. 

717 """ 

718 raise NotImplementedError() 

719 

720 @abstractmethod 

721 def associate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

722 """Add existing datasets to a `~CollectionType.TAGGED` collection. 

723 

724 If a DatasetRef with the same exact ID is already in a collection 

725 nothing is changed. If a `DatasetRef` with the same `DatasetType` and 

726 data ID but with different ID exists in the collection, 

727 `~lsst.daf.butler.registry.ConflictingDefinitionError` is raised. 

728 

729 Parameters 

730 ---------- 

731 collection : `str` 

732 Indicates the collection the datasets should be associated with. 

733 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

734 An iterable of resolved `DatasetRef` instances that already exist 

735 in this `Registry`. 

736 

737 Raises 

738 ------ 

739 lsst.daf.butler.registry.ConflictingDefinitionError 

740 If a Dataset with the given `DatasetRef` already exists in the 

741 given collection. 

742 lsst.daf.butler.registry.MissingCollectionError 

743 Raised if ``collection`` does not exist in the registry. 

744 lsst.daf.butler.registry.CollectionTypeError 

745 Raise adding new datasets to the given ``collection`` is not 

746 allowed. 

747 """ 

748 raise NotImplementedError() 

749 

750 @abstractmethod 

751 def disassociate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

752 """Remove existing datasets from a `~CollectionType.TAGGED` collection. 

753 

754 ``collection`` and ``ref`` combinations that are not currently 

755 associated are silently ignored. 

756 

757 Parameters 

758 ---------- 

759 collection : `str` 

760 The collection the datasets should no longer be associated with. 

761 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

762 An iterable of resolved `DatasetRef` instances that already exist 

763 in this `Registry`. 

764 

765 Raises 

766 ------ 

767 lsst.daf.butler.AmbiguousDatasetError 

768 Raised if any of the given dataset references is unresolved. 

769 lsst.daf.butler.registry.MissingCollectionError 

770 Raised if ``collection`` does not exist in the registry. 

771 lsst.daf.butler.registry.CollectionTypeError 

772 Raise adding new datasets to the given ``collection`` is not 

773 allowed. 

774 """ 

775 raise NotImplementedError() 

776 

777 @abstractmethod 

778 def certify(self, collection: str, refs: Iterable[DatasetRef], timespan: Timespan) -> None: 

779 """Associate one or more datasets with a calibration collection and a 

780 validity range within it. 

781 

782 Parameters 

783 ---------- 

784 collection : `str` 

785 The name of an already-registered `~CollectionType.CALIBRATION` 

786 collection. 

787 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

788 Datasets to be associated. 

789 timespan : `Timespan` 

790 The validity range for these datasets within the collection. 

791 

792 Raises 

793 ------ 

794 lsst.daf.butler.AmbiguousDatasetError 

795 Raised if any of the given `DatasetRef` instances is unresolved. 

796 lsst.daf.butler.registry.ConflictingDefinitionError 

797 Raised if the collection already contains a different dataset with 

798 the same `DatasetType` and data ID and an overlapping validity 

799 range. 

800 lsst.daf.butler.registry.CollectionTypeError 

801 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

802 collection or if one or more datasets are of a dataset type for 

803 which `DatasetType.isCalibration` returns `False`. 

804 """ 

805 raise NotImplementedError() 

806 

807 @abstractmethod 

808 def decertify( 

809 self, 

810 collection: str, 

811 datasetType: str | DatasetType, 

812 timespan: Timespan, 

813 *, 

814 dataIds: Iterable[DataId] | None = None, 

815 ) -> None: 

816 """Remove or adjust datasets to clear a validity range within a 

817 calibration collection. 

818 

819 Parameters 

820 ---------- 

821 collection : `str` 

822 The name of an already-registered `~CollectionType.CALIBRATION` 

823 collection. 

824 datasetType : `str` or `DatasetType` 

825 Name or `DatasetType` instance for the datasets to be decertified. 

826 timespan : `Timespan`, optional 

827 The validity range to remove datasets from within the collection. 

828 Datasets that overlap this range but are not contained by it will 

829 have their validity ranges adjusted to not overlap it, which may 

830 split a single dataset validity range into two. 

831 dataIds : iterable [`dict` or `DataCoordinate`], optional 

832 Data IDs that should be decertified within the given validity range 

833 If `None`, all data IDs for ``self.datasetType`` will be 

834 decertified. 

835 

836 Raises 

837 ------ 

838 lsst.daf.butler.registry.CollectionTypeError 

839 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

840 collection or if ``datasetType.isCalibration() is False``. 

841 """ 

842 raise NotImplementedError() 

843 

844 @abstractmethod 

845 def getDatasetLocations(self, ref: DatasetRef) -> Iterable[str]: 

846 """Retrieve datastore locations for a given dataset. 

847 

848 Parameters 

849 ---------- 

850 ref : `DatasetRef` 

851 A reference to the dataset for which to retrieve storage 

852 information. 

853 

854 Returns 

855 ------- 

856 datastores : `~collections.abc.Iterable` [ `str` ] 

857 All the matching datastores holding this dataset. 

858 

859 Raises 

860 ------ 

861 lsst.daf.butler.AmbiguousDatasetError 

862 Raised if ``ref.id`` is `None`. 

863 """ 

864 raise NotImplementedError() 

865 

866 @abstractmethod 

867 def expandDataId( 

868 self, 

869 dataId: DataId | None = None, 

870 *, 

871 dimensions: Iterable[str] | DimensionGroup | DimensionGraph | None = None, 

872 graph: DimensionGraph | None = None, 

873 records: NameLookupMapping[DimensionElement, DimensionRecord | None] | None = None, 

874 withDefaults: bool = True, 

875 **kwargs: Any, 

876 ) -> DataCoordinate: 

877 """Expand a dimension-based data ID to include additional information. 

878 

879 Parameters 

880 ---------- 

881 dataId : `DataCoordinate` or `dict`, optional 

882 Data ID to be expanded; augmented and overridden by ``kwargs``. 

883 dimensions : `~collections.abc.Iterable` [ `str` ], \ 

884 `DimensionGroup`, or `DimensionGraph`, optional 

885 The dimensions to be identified by the new `DataCoordinate`. 

886 If not provided, will be inferred from the keys of ``mapping`` and 

887 ``**kwargs``, and ``universe`` must be provided unless ``mapping`` 

888 is already a `DataCoordinate`. 

889 graph : `DimensionGraph`, optional 

890 Like ``dimensions``, but as a ``DimensionGraph`` instance. Ignored 

891 if ``dimensions`` is provided. Deprecated and will be removed 

892 after v27. 

893 records : `~collections.abc.Mapping` [`str`, `DimensionRecord`], \ 

894 optional 

895 Dimension record data to use before querying the database for that 

896 data, keyed by element name. 

897 withDefaults : `bool`, optional 

898 Utilize ``self.defaults.dataId`` to fill in missing governor 

899 dimension key-value pairs. Defaults to `True` (i.e. defaults are 

900 used). 

901 **kwargs 

902 Additional keywords are treated like additional key-value pairs for 

903 ``dataId``, extending and overriding. 

904 

905 Returns 

906 ------- 

907 expanded : `DataCoordinate` 

908 A data ID that includes full metadata for all of the dimensions it 

909 identifies, i.e. guarantees that ``expanded.hasRecords()`` and 

910 ``expanded.hasFull()`` both return `True`. 

911 

912 Raises 

913 ------ 

914 lsst.daf.butler.registry.DataIdError 

915 Raised when ``dataId`` or keyword arguments specify unknown 

916 dimensions or values, or when a resulting data ID contains 

917 contradictory key-value pairs, according to dimension 

918 relationships. 

919 

920 Notes 

921 ----- 

922 This method cannot be relied upon to reject invalid data ID values 

923 for dimensions that do actually not have any record columns. For 

924 efficiency reasons the records for these dimensions (which have only 

925 dimension key values that are given by the caller) may be constructed 

926 directly rather than obtained from the registry database. 

927 """ 

928 raise NotImplementedError() 

929 

930 @abstractmethod 

931 def insertDimensionData( 

932 self, 

933 element: DimensionElement | str, 

934 *data: Mapping[str, Any] | DimensionRecord, 

935 conform: bool = True, 

936 replace: bool = False, 

937 skip_existing: bool = False, 

938 ) -> None: 

939 """Insert one or more dimension records into the database. 

940 

941 Parameters 

942 ---------- 

943 element : `DimensionElement` or `str` 

944 The `DimensionElement` or name thereof that identifies the table 

945 records will be inserted into. 

946 *data : `dict` or `DimensionRecord` 

947 One or more records to insert. 

948 conform : `bool`, optional 

949 If `False` (`True` is default) perform no checking or conversions, 

950 and assume that ``element`` is a `DimensionElement` instance and 

951 ``data`` is a one or more `DimensionRecord` instances of the 

952 appropriate subclass. 

953 replace : `bool`, optional 

954 If `True` (`False` is default), replace existing records in the 

955 database if there is a conflict. 

956 skip_existing : `bool`, optional 

957 If `True` (`False` is default), skip insertion if a record with 

958 the same primary key values already exists. Unlike 

959 `syncDimensionData`, this will not detect when the given record 

960 differs from what is in the database, and should not be used when 

961 this is a concern. 

962 """ 

963 raise NotImplementedError() 

964 

965 @abstractmethod 

966 def syncDimensionData( 

967 self, 

968 element: DimensionElement | str, 

969 row: Mapping[str, Any] | DimensionRecord, 

970 conform: bool = True, 

971 update: bool = False, 

972 ) -> bool | dict[str, Any]: 

973 """Synchronize the given dimension record with the database, inserting 

974 if it does not already exist and comparing values if it does. 

975 

976 Parameters 

977 ---------- 

978 element : `DimensionElement` or `str` 

979 The `DimensionElement` or name thereof that identifies the table 

980 records will be inserted into. 

981 row : `dict` or `DimensionRecord` 

982 The record to insert. 

983 conform : `bool`, optional 

984 If `False` (`True` is default) perform no checking or conversions, 

985 and assume that ``element`` is a `DimensionElement` instance and 

986 ``data`` is a one or more `DimensionRecord` instances of the 

987 appropriate subclass. 

988 update : `bool`, optional 

989 If `True` (`False` is default), update the existing record in the 

990 database if there is a conflict. 

991 

992 Returns 

993 ------- 

994 inserted_or_updated : `bool` or `dict` 

995 `True` if a new row was inserted, `False` if no changes were 

996 needed, or a `dict` mapping updated column names to their old 

997 values if an update was performed (only possible if 

998 ``update=True``). 

999 

1000 Raises 

1001 ------ 

1002 lsst.daf.butler.registry.ConflictingDefinitionError 

1003 Raised if the record exists in the database (according to primary 

1004 key lookup) but is inconsistent with the given one. 

1005 """ 

1006 raise NotImplementedError() 

1007 

1008 @abstractmethod 

1009 def queryDatasetTypes( 

1010 self, 

1011 expression: Any = ..., 

1012 *, 

1013 components: bool = False, 

1014 missing: list[str] | None = None, 

1015 ) -> Iterable[DatasetType]: 

1016 """Iterate over the dataset types whose names match an expression. 

1017 

1018 Parameters 

1019 ---------- 

1020 expression : dataset type expression, optional 

1021 An expression that fully or partially identifies the dataset types 

1022 to return, such as a `str`, `re.Pattern`, or iterable thereof. 

1023 ``...`` can be used to return all dataset types, and is the 

1024 default. See :ref:`daf_butler_dataset_type_expressions` for more 

1025 information. 

1026 components : `bool`, optional 

1027 Must be `False`. Provided only for backwards compatibility. After 

1028 v27 this argument will be removed entirely. 

1029 missing : `list` of `str`, optional 

1030 String dataset type names that were explicitly given (i.e. not 

1031 regular expression patterns) but not found will be appended to this 

1032 list, if it is provided. 

1033 

1034 Returns 

1035 ------- 

1036 dataset_types : `~collections.abc.Iterable` [ `DatasetType`] 

1037 An `~collections.abc.Iterable` of `DatasetType` instances whose 

1038 names match ``expression``. 

1039 

1040 Raises 

1041 ------ 

1042 lsst.daf.butler.registry.DatasetTypeExpressionError 

1043 Raised when ``expression`` is invalid. 

1044 """ 

1045 raise NotImplementedError() 

1046 

1047 @abstractmethod 

1048 def queryCollections( 

1049 self, 

1050 expression: Any = ..., 

1051 datasetType: DatasetType | None = None, 

1052 collectionTypes: Iterable[CollectionType] | CollectionType = CollectionType.all(), 

1053 flattenChains: bool = False, 

1054 includeChains: bool | None = None, 

1055 ) -> Sequence[str]: 

1056 """Iterate over the collections whose names match an expression. 

1057 

1058 Parameters 

1059 ---------- 

1060 expression : collection expression, optional 

1061 An expression that identifies the collections to return, such as 

1062 a `str` (for full matches or partial matches via globs), 

1063 `re.Pattern` (for partial matches), or iterable thereof. ``...`` 

1064 can be used to return all collections, and is the default. 

1065 See :ref:`daf_butler_collection_expressions` for more information. 

1066 datasetType : `DatasetType`, optional 

1067 If provided, only yield collections that may contain datasets of 

1068 this type. This is a conservative approximation in general; it may 

1069 yield collections that do not have any such datasets. 

1070 collectionTypes : `~collections.abc.Set` [`CollectionType`] or \ 

1071 `CollectionType`, optional 

1072 If provided, only yield collections of these types. 

1073 flattenChains : `bool`, optional 

1074 If `True` (`False` is default), recursively yield the child 

1075 collections of matching `~CollectionType.CHAINED` collections. 

1076 includeChains : `bool`, optional 

1077 If `True`, yield records for matching `~CollectionType.CHAINED` 

1078 collections. Default is the opposite of ``flattenChains``: include 

1079 either CHAINED collections or their children, but not both. 

1080 

1081 Returns 

1082 ------- 

1083 collections : `~collections.abc.Sequence` [ `str` ] 

1084 The names of collections that match ``expression``. 

1085 

1086 Raises 

1087 ------ 

1088 lsst.daf.butler.registry.CollectionExpressionError 

1089 Raised when ``expression`` is invalid. 

1090 

1091 Notes 

1092 ----- 

1093 The order in which collections are returned is unspecified, except that 

1094 the children of a `~CollectionType.CHAINED` collection are guaranteed 

1095 to be in the order in which they are searched. When multiple parent 

1096 `~CollectionType.CHAINED` collections match the same criteria, the 

1097 order in which the two lists appear is unspecified, and the lists of 

1098 children may be incomplete if a child has multiple parents. 

1099 """ 

1100 raise NotImplementedError() 

1101 

1102 @abstractmethod 

1103 def queryDatasets( 

1104 self, 

1105 datasetType: Any, 

1106 *, 

1107 collections: CollectionArgType | None = None, 

1108 dimensions: Iterable[Dimension | str] | None = None, 

1109 dataId: DataId | None = None, 

1110 where: str = "", 

1111 findFirst: bool = False, 

1112 components: bool = False, 

1113 bind: Mapping[str, Any] | None = None, 

1114 check: bool = True, 

1115 **kwargs: Any, 

1116 ) -> DatasetQueryResults: 

1117 """Query for and iterate over dataset references matching user-provided 

1118 criteria. 

1119 

1120 Parameters 

1121 ---------- 

1122 datasetType : dataset type expression 

1123 An expression that fully or partially identifies the dataset types 

1124 to be queried. Allowed types include `DatasetType`, `str`, 

1125 `re.Pattern`, and iterables thereof. The special value ``...`` can 

1126 be used to query all dataset types. See 

1127 :ref:`daf_butler_dataset_type_expressions` for more information. 

1128 collections : collection expression, optional 

1129 An expression that identifies the collections to search, such as a 

1130 `str` (for full matches or partial matches via globs), `re.Pattern` 

1131 (for partial matches), or iterable thereof. ``...`` can be used to 

1132 search all collections (actually just all `~CollectionType.RUN` 

1133 collections, because this will still find all datasets). 

1134 If not provided, ``self.default.collections`` is used. See 

1135 :ref:`daf_butler_collection_expressions` for more information. 

1136 dimensions : `~collections.abc.Iterable` of `Dimension` or `str` 

1137 Dimensions to include in the query (in addition to those used 

1138 to identify the queried dataset type(s)), either to constrain 

1139 the resulting datasets to those for which a matching dimension 

1140 exists, or to relate the dataset type's dimensions to dimensions 

1141 referenced by the ``dataId`` or ``where`` arguments. 

1142 dataId : `dict` or `DataCoordinate`, optional 

1143 A data ID whose key-value pairs are used as equality constraints 

1144 in the query. 

1145 where : `str`, optional 

1146 A string expression similar to a SQL WHERE clause. May involve 

1147 any column of a dimension table or (as a shortcut for the primary 

1148 key column of a dimension table) dimension name. See 

1149 :ref:`daf_butler_dimension_expressions` for more information. 

1150 findFirst : `bool`, optional 

1151 If `True` (`False` is default), for each result data ID, only 

1152 yield one `DatasetRef` of each `DatasetType`, from the first 

1153 collection in which a dataset of that dataset type appears 

1154 (according to the order of ``collections`` passed in). If `True`, 

1155 ``collections`` must not contain regular expressions and may not 

1156 be ``...``. 

1157 components : `bool`, optional 

1158 Must be `False`. Provided only for backwards compatibility. After 

1159 v27 this argument will be removed entirely. 

1160 bind : `~collections.abc.Mapping`, optional 

1161 Mapping containing literal values that should be injected into the 

1162 ``where`` expression, keyed by the identifiers they replace. 

1163 Values of collection type can be expanded in some cases; see 

1164 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1165 information. 

1166 check : `bool`, optional 

1167 If `True` (default) check the query for consistency before 

1168 executing it. This may reject some valid queries that resemble 

1169 common mistakes (e.g. queries for visits without specifying an 

1170 instrument). 

1171 **kwargs 

1172 Additional keyword arguments are forwarded to 

1173 `DataCoordinate.standardize` when processing the ``dataId`` 

1174 argument (and may be used to provide a constraining data ID even 

1175 when the ``dataId`` argument is `None`). 

1176 

1177 Returns 

1178 ------- 

1179 refs : `.queries.DatasetQueryResults` 

1180 Dataset references matching the given query criteria. Nested data 

1181 IDs are guaranteed to include values for all implied dimensions 

1182 (i.e. `DataCoordinate.hasFull` will return `True`), but will not 

1183 include dimension records (`DataCoordinate.hasRecords` will be 

1184 `False`) unless `~.queries.DatasetQueryResults.expanded` is 

1185 called on the result object (which returns a new one). 

1186 

1187 Raises 

1188 ------ 

1189 lsst.daf.butler.registry.DatasetTypeExpressionError 

1190 Raised when ``datasetType`` expression is invalid. 

1191 TypeError 

1192 Raised when the arguments are incompatible, such as when a 

1193 collection wildcard is passed when ``findFirst`` is `True`, or 

1194 when ``collections`` is `None` and ``self.defaults.collections`` is 

1195 also `None`. 

1196 lsst.daf.butler.registry.DataIdError 

1197 Raised when ``dataId`` or keyword arguments specify unknown 

1198 dimensions or values, or when they contain inconsistent values. 

1199 lsst.daf.butler.registry.UserExpressionError 

1200 Raised when ``where`` expression is invalid. 

1201 

1202 Notes 

1203 ----- 

1204 When multiple dataset types are queried in a single call, the 

1205 results of this operation are equivalent to querying for each dataset 

1206 type separately in turn, and no information about the relationships 

1207 between datasets of different types is included. In contexts where 

1208 that kind of information is important, the recommended pattern is to 

1209 use `queryDataIds` to first obtain data IDs (possibly with the 

1210 desired dataset types and collections passed as constraints to the 

1211 query), and then use multiple (generally much simpler) calls to 

1212 `queryDatasets` with the returned data IDs passed as constraints. 

1213 """ 

1214 raise NotImplementedError() 

1215 

1216 @abstractmethod 

1217 def queryDataIds( 

1218 self, 

1219 # TODO: Drop `Dimension` objects on DM-41326. 

1220 dimensions: DimensionGroup | Iterable[Dimension | str] | Dimension | str, 

1221 *, 

1222 dataId: DataId | None = None, 

1223 datasets: Any = None, 

1224 collections: CollectionArgType | None = None, 

1225 where: str = "", 

1226 components: bool = False, 

1227 bind: Mapping[str, Any] | None = None, 

1228 check: bool = True, 

1229 **kwargs: Any, 

1230 ) -> DataCoordinateQueryResults: 

1231 """Query for data IDs matching user-provided criteria. 

1232 

1233 Parameters 

1234 ---------- 

1235 dimensions : `DimensionGroup`, `Dimension`, or `str`, or \ 

1236 `~collections.abc.Iterable` [ `Dimension` or `str` ] 

1237 The dimensions of the data IDs to yield, as either `Dimension` 

1238 instances or `str`. Will be automatically expanded to a complete 

1239 `DimensionGroup`. Support for `Dimension` instances is deprecated 

1240 and will not be supported after v27. 

1241 dataId : `dict` or `DataCoordinate`, optional 

1242 A data ID whose key-value pairs are used as equality constraints 

1243 in the query. 

1244 datasets : dataset type expression, optional 

1245 An expression that fully or partially identifies dataset types 

1246 that should constrain the yielded data IDs. For example, including 

1247 "raw" here would constrain the yielded ``instrument``, 

1248 ``exposure``, ``detector``, and ``physical_filter`` values to only 

1249 those for which at least one "raw" dataset exists in 

1250 ``collections``. Allowed types include `DatasetType`, `str`, 

1251 and iterables thereof. Regular expression objects (i.e. 

1252 `re.Pattern`) are deprecated and will be removed after the v26 

1253 release. See :ref:`daf_butler_dataset_type_expressions` for more 

1254 information. 

1255 collections : collection expression, optional 

1256 An expression that identifies the collections to search for 

1257 datasets, such as a `str` (for full matches or partial matches 

1258 via globs), `re.Pattern` (for partial matches), or iterable 

1259 thereof. ``...`` can be used to search all collections (actually 

1260 just all `~CollectionType.RUN` collections, because this will 

1261 still find all datasets). If not provided, 

1262 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1263 is also passed. See :ref:`daf_butler_collection_expressions` for 

1264 more information. 

1265 where : `str`, optional 

1266 A string expression similar to a SQL WHERE clause. May involve 

1267 any column of a dimension table or (as a shortcut for the primary 

1268 key column of a dimension table) dimension name. See 

1269 :ref:`daf_butler_dimension_expressions` for more information. 

1270 components : `bool`, optional 

1271 Must be `False`. Provided only for backwards compatibility. After 

1272 v27 this argument will be removed entirely. 

1273 bind : `~collections.abc.Mapping`, optional 

1274 Mapping containing literal values that should be injected into the 

1275 ``where`` expression, keyed by the identifiers they replace. 

1276 Values of collection type can be expanded in some cases; see 

1277 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1278 information. 

1279 check : `bool`, optional 

1280 If `True` (default) check the query for consistency before 

1281 executing it. This may reject some valid queries that resemble 

1282 common mistakes (e.g. queries for visits without specifying an 

1283 instrument). 

1284 **kwargs 

1285 Additional keyword arguments are forwarded to 

1286 `DataCoordinate.standardize` when processing the ``dataId`` 

1287 argument (and may be used to provide a constraining data ID even 

1288 when the ``dataId`` argument is `None`). 

1289 

1290 Returns 

1291 ------- 

1292 dataIds : `.queries.DataCoordinateQueryResults` 

1293 Data IDs matching the given query parameters. These are guaranteed 

1294 to identify all dimensions (`DataCoordinate.hasFull` returns 

1295 `True`), but will not contain `DimensionRecord` objects 

1296 (`DataCoordinate.hasRecords` returns `False`). Call 

1297 `~.queries.DataCoordinateQueryResults.expanded` on the 

1298 returned object to fetch those (and consider using 

1299 `~.queries.DataCoordinateQueryResults.materialize` on the 

1300 returned object first if the expected number of rows is very 

1301 large). See documentation for those methods for additional 

1302 information. 

1303 

1304 Raises 

1305 ------ 

1306 lsst.daf.butler.registry.NoDefaultCollectionError 

1307 Raised if ``collections`` is `None` and 

1308 ``self.defaults.collections`` is `None`. 

1309 lsst.daf.butler.registry.CollectionExpressionError 

1310 Raised when ``collections`` expression is invalid. 

1311 lsst.daf.butler.registry.DataIdError 

1312 Raised when ``dataId`` or keyword arguments specify unknown 

1313 dimensions or values, or when they contain inconsistent values. 

1314 lsst.daf.butler.registry.DatasetTypeExpressionError 

1315 Raised when ``datasetType`` expression is invalid. 

1316 lsst.daf.butler.registry.UserExpressionError 

1317 Raised when ``where`` expression is invalid. 

1318 """ 

1319 raise NotImplementedError() 

1320 

1321 @abstractmethod 

1322 def queryDimensionRecords( 

1323 self, 

1324 element: DimensionElement | str, 

1325 *, 

1326 dataId: DataId | None = None, 

1327 datasets: Any = None, 

1328 collections: CollectionArgType | None = None, 

1329 where: str = "", 

1330 components: bool = False, 

1331 bind: Mapping[str, Any] | None = None, 

1332 check: bool = True, 

1333 **kwargs: Any, 

1334 ) -> DimensionRecordQueryResults: 

1335 """Query for dimension information matching user-provided criteria. 

1336 

1337 Parameters 

1338 ---------- 

1339 element : `DimensionElement` or `str` 

1340 The dimension element to obtain records for. 

1341 dataId : `dict` or `DataCoordinate`, optional 

1342 A data ID whose key-value pairs are used as equality constraints 

1343 in the query. 

1344 datasets : dataset type expression, optional 

1345 An expression that fully or partially identifies dataset types 

1346 that should constrain the yielded records. See `queryDataIds` and 

1347 :ref:`daf_butler_dataset_type_expressions` for more information. 

1348 collections : collection expression, optional 

1349 An expression that identifies the collections to search for 

1350 datasets, such as a `str` (for full matches or partial matches 

1351 via globs), `re.Pattern` (for partial matches), or iterable 

1352 thereof. ``...`` can be used to search all collections (actually 

1353 just all `~CollectionType.RUN` collections, because this will 

1354 still find all datasets). If not provided, 

1355 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1356 is also passed. See :ref:`daf_butler_collection_expressions` for 

1357 more information. 

1358 where : `str`, optional 

1359 A string expression similar to a SQL WHERE clause. See 

1360 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more 

1361 information. 

1362 components : `bool`, optional 

1363 Must be `False`. Provided only for backwards compatibility. After 

1364 v27 this argument will be removed entirely. 

1365 bind : `~collections.abc.Mapping`, optional 

1366 Mapping containing literal values that should be injected into the 

1367 ``where`` expression, keyed by the identifiers they replace. 

1368 Values of collection type can be expanded in some cases; see 

1369 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1370 information. 

1371 check : `bool`, optional 

1372 If `True` (default) check the query for consistency before 

1373 executing it. This may reject some valid queries that resemble 

1374 common mistakes (e.g. queries for visits without specifying an 

1375 instrument). 

1376 **kwargs 

1377 Additional keyword arguments are forwarded to 

1378 `DataCoordinate.standardize` when processing the ``dataId`` 

1379 argument (and may be used to provide a constraining data ID even 

1380 when the ``dataId`` argument is `None`). 

1381 

1382 Returns 

1383 ------- 

1384 dataIds : `.queries.DimensionRecordQueryResults` 

1385 Data IDs matching the given query parameters. 

1386 

1387 Raises 

1388 ------ 

1389 lsst.daf.butler.registry.NoDefaultCollectionError 

1390 Raised if ``collections`` is `None` and 

1391 ``self.defaults.collections`` is `None`. 

1392 lsst.daf.butler.registry.CollectionExpressionError 

1393 Raised when ``collections`` expression is invalid. 

1394 lsst.daf.butler.registry.DataIdError 

1395 Raised when ``dataId`` or keyword arguments specify unknown 

1396 dimensions or values, or when they contain inconsistent values. 

1397 lsst.daf.butler.registry.DatasetTypeExpressionError 

1398 Raised when ``datasetType`` expression is invalid. 

1399 lsst.daf.butler.registry.UserExpressionError 

1400 Raised when ``where`` expression is invalid. 

1401 """ 

1402 raise NotImplementedError() 

1403 

1404 @abstractmethod 

1405 def queryDatasetAssociations( 

1406 self, 

1407 datasetType: str | DatasetType, 

1408 collections: CollectionArgType | None = ..., 

1409 *, 

1410 collectionTypes: Iterable[CollectionType] = CollectionType.all(), 

1411 flattenChains: bool = False, 

1412 ) -> Iterator[DatasetAssociation]: 

1413 """Iterate over dataset-collection combinations where the dataset is in 

1414 the collection. 

1415 

1416 This method is a temporary placeholder for better support for 

1417 association results in `queryDatasets`. It will probably be 

1418 removed in the future, and should be avoided in production code 

1419 whenever possible. 

1420 

1421 Parameters 

1422 ---------- 

1423 datasetType : `DatasetType` or `str` 

1424 A dataset type object or the name of one. 

1425 collections : collection expression, optional 

1426 An expression that identifies the collections to search for 

1427 datasets, such as a `str` (for full matches or partial matches 

1428 via globs), `re.Pattern` (for partial matches), or iterable 

1429 thereof. ``...`` can be used to search all collections (actually 

1430 just all `~CollectionType.RUN` collections, because this will still 

1431 find all datasets). If not provided, ``self.default.collections`` 

1432 is used. See :ref:`daf_butler_collection_expressions` for more 

1433 information. 

1434 collectionTypes : `~collections.abc.Set` [ `CollectionType` ], optional 

1435 If provided, only yield associations from collections of these 

1436 types. 

1437 flattenChains : `bool`, optional 

1438 If `True`, search in the children of `~CollectionType.CHAINED` 

1439 collections. If `False`, ``CHAINED`` collections are ignored. 

1440 

1441 Yields 

1442 ------ 

1443 association : `.DatasetAssociation` 

1444 Object representing the relationship between a single dataset and 

1445 a single collection. 

1446 

1447 Raises 

1448 ------ 

1449 lsst.daf.butler.registry.NoDefaultCollectionError 

1450 Raised if ``collections`` is `None` and 

1451 ``self.defaults.collections`` is `None`. 

1452 lsst.daf.butler.registry.CollectionExpressionError 

1453 Raised when ``collections`` expression is invalid. 

1454 """ 

1455 raise NotImplementedError() 

1456 

1457 @property 

1458 def obsCoreTableManager(self) -> ObsCoreTableManager | None: 

1459 """The ObsCore manager instance for this registry 

1460 (`~.interfaces.ObsCoreTableManager` 

1461 or `None`). 

1462 

1463 ObsCore manager may not be implemented for all registry backend, or 

1464 may not be enabled for many repositories. 

1465 """ 

1466 return None 

1467 

1468 storageClasses: StorageClassFactory 

1469 """All storage classes known to the registry (`StorageClassFactory`). 

1470 """