Coverage for python/lsst/daf/butler/registry/_registry.py: 99%

115 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-12-06 10:52 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ("Registry", "CollectionArgType") 

31 

32import contextlib 

33import logging 

34import re 

35from abc import ABC, abstractmethod 

36from collections.abc import Iterable, Iterator, Mapping, Sequence 

37from types import EllipsisType 

38from typing import TYPE_CHECKING, Any, TypeAlias 

39 

40from .._dataset_association import DatasetAssociation 

41from .._dataset_ref import DatasetId, DatasetIdGenEnum, DatasetRef 

42from .._dataset_type import DatasetType 

43from .._named import NameLookupMapping 

44from .._storage_class import StorageClassFactory 

45from .._timespan import Timespan 

46from ..dimensions import ( 

47 DataCoordinate, 

48 DataId, 

49 Dimension, 

50 DimensionElement, 

51 DimensionGraph, 

52 DimensionGroup, 

53 DimensionRecord, 

54 DimensionUniverse, 

55) 

56from ._collection_summary import CollectionSummary 

57from ._collection_type import CollectionType 

58from ._defaults import RegistryDefaults 

59from .queries import DataCoordinateQueryResults, DatasetQueryResults, DimensionRecordQueryResults 

60from .wildcards import CollectionWildcard 

61 

62if TYPE_CHECKING: 

63 from .interfaces import ObsCoreTableManager 

64 

65_LOG = logging.getLogger(__name__) 

66 

67# TYpe alias for `collections` arguments. 

68CollectionArgType: TypeAlias = ( 

69 str | re.Pattern | Iterable[str | re.Pattern] | EllipsisType | CollectionWildcard 

70) 

71 

72 

73class Registry(ABC): 

74 """Abstract Registry interface. 

75 

76 All subclasses should store `~lsst.daf.butler.registry.RegistryDefaults` in 

77 a ``_defaults`` property. No other properties are assumed shared between 

78 implementations. 

79 """ 

80 

81 @abstractmethod 

82 def isWriteable(self) -> bool: 

83 """Return `True` if this registry allows write operations, and `False` 

84 otherwise. 

85 """ 

86 raise NotImplementedError() 

87 

88 @property 

89 @abstractmethod 

90 def dimensions(self) -> DimensionUniverse: 

91 """Definitions of all dimensions recognized by this `Registry` 

92 (`DimensionUniverse`). 

93 """ 

94 raise NotImplementedError() 

95 

96 @property 

97 @abstractmethod 

98 def defaults(self) -> RegistryDefaults: 

99 """Default collection search path and/or output `~CollectionType.RUN` 

100 collection (`~lsst.daf.butler.registry.RegistryDefaults`). 

101 

102 This is an immutable struct whose components may not be set 

103 individually, but the entire struct can be set by assigning to this 

104 property. 

105 """ 

106 raise NotImplementedError() 

107 

108 @defaults.setter 

109 @abstractmethod 

110 def defaults(self, value: RegistryDefaults) -> None: 

111 raise NotImplementedError() 

112 

113 @abstractmethod 

114 def refresh(self) -> None: 

115 """Refresh all in-memory state by querying the database. 

116 

117 This may be necessary to enable querying for entities added by other 

118 registry instances after this one was constructed. 

119 """ 

120 raise NotImplementedError() 

121 

122 @abstractmethod 

123 def caching_context(self) -> contextlib.AbstractContextManager[None]: 

124 """Context manager that enables caching.""" 

125 raise NotImplementedError() 

126 

127 @contextlib.contextmanager 

128 @abstractmethod 

129 def transaction(self, *, savepoint: bool = False) -> Iterator[None]: 

130 """Return a context manager that represents a transaction.""" 

131 raise NotImplementedError() 

132 

133 def resetConnectionPool(self) -> None: 

134 """Reset connection pool for registry if relevant. 

135 

136 This operation can be used reset connections to servers when 

137 using registry with fork-based multiprocessing. This method should 

138 usually be called by the child process immediately 

139 after the fork. 

140 

141 The base class implementation is a no-op. 

142 """ 

143 pass 

144 

145 @abstractmethod 

146 def registerCollection( 

147 self, name: str, type: CollectionType = CollectionType.TAGGED, doc: str | None = None 

148 ) -> bool: 

149 """Add a new collection if one with the given name does not exist. 

150 

151 Parameters 

152 ---------- 

153 name : `str` 

154 The name of the collection to create. 

155 type : `CollectionType` 

156 Enum value indicating the type of collection to create. 

157 doc : `str`, optional 

158 Documentation string for the collection. 

159 

160 Returns 

161 ------- 

162 registered : `bool` 

163 Boolean indicating whether the collection was already registered 

164 or was created by this call. 

165 

166 Notes 

167 ----- 

168 This method cannot be called within transactions, as it needs to be 

169 able to perform its own transaction to be concurrent. 

170 """ 

171 raise NotImplementedError() 

172 

173 @abstractmethod 

174 def getCollectionType(self, name: str) -> CollectionType: 

175 """Return an enumeration value indicating the type of the given 

176 collection. 

177 

178 Parameters 

179 ---------- 

180 name : `str` 

181 The name of the collection. 

182 

183 Returns 

184 ------- 

185 type : `CollectionType` 

186 Enum value indicating the type of this collection. 

187 

188 Raises 

189 ------ 

190 lsst.daf.butler.registry.MissingCollectionError 

191 Raised if no collection with the given name exists. 

192 """ 

193 raise NotImplementedError() 

194 

195 @abstractmethod 

196 def registerRun(self, name: str, doc: str | None = None) -> bool: 

197 """Add a new run if one with the given name does not exist. 

198 

199 Parameters 

200 ---------- 

201 name : `str` 

202 The name of the run to create. 

203 doc : `str`, optional 

204 Documentation string for the collection. 

205 

206 Returns 

207 ------- 

208 registered : `bool` 

209 Boolean indicating whether a new run was registered. `False` 

210 if it already existed. 

211 

212 Notes 

213 ----- 

214 This method cannot be called within transactions, as it needs to be 

215 able to perform its own transaction to be concurrent. 

216 """ 

217 raise NotImplementedError() 

218 

219 @abstractmethod 

220 def removeCollection(self, name: str) -> None: 

221 """Remove the given collection from the registry. 

222 

223 Parameters 

224 ---------- 

225 name : `str` 

226 The name of the collection to remove. 

227 

228 Raises 

229 ------ 

230 lsst.daf.butler.registry.MissingCollectionError 

231 Raised if no collection with the given name exists. 

232 sqlalchemy.exc.IntegrityError 

233 Raised if the database rows associated with the collection are 

234 still referenced by some other table, such as a dataset in a 

235 datastore (for `~CollectionType.RUN` collections only) or a 

236 `~CollectionType.CHAINED` collection of which this collection is 

237 a child. 

238 

239 Notes 

240 ----- 

241 If this is a `~CollectionType.RUN` collection, all datasets and quanta 

242 in it will removed from the `Registry` database. This requires that 

243 those datasets be removed (or at least trashed) from any datastores 

244 that hold them first. 

245 

246 A collection may not be deleted as long as it is referenced by a 

247 `~CollectionType.CHAINED` collection; the ``CHAINED`` collection must 

248 be deleted or redefined first. 

249 """ 

250 raise NotImplementedError() 

251 

252 @abstractmethod 

253 def getCollectionChain(self, parent: str) -> Sequence[str]: 

254 """Return the child collections in a `~CollectionType.CHAINED` 

255 collection. 

256 

257 Parameters 

258 ---------- 

259 parent : `str` 

260 Name of the chained collection. Must have already been added via 

261 a call to `Registry.registerCollection`. 

262 

263 Returns 

264 ------- 

265 children : `~collections.abc.Sequence` [ `str` ] 

266 An ordered sequence of collection names that are searched when the 

267 given chained collection is searched. 

268 

269 Raises 

270 ------ 

271 lsst.daf.butler.registry.MissingCollectionError 

272 Raised if ``parent`` does not exist in the `Registry`. 

273 lsst.daf.butler.registry.CollectionTypeError 

274 Raised if ``parent`` does not correspond to a 

275 `~CollectionType.CHAINED` collection. 

276 """ 

277 raise NotImplementedError() 

278 

279 @abstractmethod 

280 def setCollectionChain(self, parent: str, children: Any, *, flatten: bool = False) -> None: 

281 """Define or redefine a `~CollectionType.CHAINED` collection. 

282 

283 Parameters 

284 ---------- 

285 parent : `str` 

286 Name of the chained collection. Must have already been added via 

287 a call to `Registry.registerCollection`. 

288 children : collection expression 

289 An expression defining an ordered search of child collections, 

290 generally an iterable of `str`; see 

291 :ref:`daf_butler_collection_expressions` for more information. 

292 flatten : `bool`, optional 

293 If `True` (`False` is default), recursively flatten out any nested 

294 `~CollectionType.CHAINED` collections in ``children`` first. 

295 

296 Raises 

297 ------ 

298 lsst.daf.butler.registry.MissingCollectionError 

299 Raised when any of the given collections do not exist in the 

300 `Registry`. 

301 lsst.daf.butler.registry.CollectionTypeError 

302 Raised if ``parent`` does not correspond to a 

303 `~CollectionType.CHAINED` collection. 

304 ValueError 

305 Raised if the given collections contains a cycle. 

306 """ 

307 raise NotImplementedError() 

308 

309 @abstractmethod 

310 def getCollectionParentChains(self, collection: str) -> set[str]: 

311 """Return the CHAINED collections that directly contain the given one. 

312 

313 Parameters 

314 ---------- 

315 name : `str` 

316 Name of the collection. 

317 

318 Returns 

319 ------- 

320 chains : `set` of `str` 

321 Set of `~CollectionType.CHAINED` collection names. 

322 """ 

323 raise NotImplementedError() 

324 

325 @abstractmethod 

326 def getCollectionDocumentation(self, collection: str) -> str | None: 

327 """Retrieve the documentation string for a collection. 

328 

329 Parameters 

330 ---------- 

331 name : `str` 

332 Name of the collection. 

333 

334 Returns 

335 ------- 

336 docs : `str` or `None` 

337 Docstring for the collection with the given name. 

338 """ 

339 raise NotImplementedError() 

340 

341 @abstractmethod 

342 def setCollectionDocumentation(self, collection: str, doc: str | None) -> None: 

343 """Set the documentation string for a collection. 

344 

345 Parameters 

346 ---------- 

347 name : `str` 

348 Name of the collection. 

349 docs : `str` or `None` 

350 Docstring for the collection with the given name; will replace any 

351 existing docstring. Passing `None` will remove any existing 

352 docstring. 

353 """ 

354 raise NotImplementedError() 

355 

356 @abstractmethod 

357 def getCollectionSummary(self, collection: str) -> CollectionSummary: 

358 """Return a summary for the given collection. 

359 

360 Parameters 

361 ---------- 

362 collection : `str` 

363 Name of the collection for which a summary is to be retrieved. 

364 

365 Returns 

366 ------- 

367 summary : `~lsst.daf.butler.registry.CollectionSummary` 

368 Summary of the dataset types and governor dimension values in 

369 this collection. 

370 """ 

371 raise NotImplementedError() 

372 

373 @abstractmethod 

374 def registerDatasetType(self, datasetType: DatasetType) -> bool: 

375 """Add a new `DatasetType` to the Registry. 

376 

377 It is not an error to register the same `DatasetType` twice. 

378 

379 Parameters 

380 ---------- 

381 datasetType : `DatasetType` 

382 The `DatasetType` to be added. 

383 

384 Returns 

385 ------- 

386 inserted : `bool` 

387 `True` if ``datasetType`` was inserted, `False` if an identical 

388 existing `DatasetType` was found. Note that in either case the 

389 DatasetType is guaranteed to be defined in the Registry 

390 consistently with the given definition. 

391 

392 Raises 

393 ------ 

394 ValueError 

395 Raised if the dimensions or storage class are invalid. 

396 lsst.daf.butler.registry.ConflictingDefinitionError 

397 Raised if this `DatasetType` is already registered with a different 

398 definition. 

399 

400 Notes 

401 ----- 

402 This method cannot be called within transactions, as it needs to be 

403 able to perform its own transaction to be concurrent. 

404 """ 

405 raise NotImplementedError() 

406 

407 @abstractmethod 

408 def removeDatasetType(self, name: str | tuple[str, ...]) -> None: 

409 """Remove the named `DatasetType` from the registry. 

410 

411 .. warning:: 

412 

413 Registry implementations can cache the dataset type definitions. 

414 This means that deleting the dataset type definition may result in 

415 unexpected behavior from other butler processes that are active 

416 that have not seen the deletion. 

417 

418 Parameters 

419 ---------- 

420 name : `str` or `tuple` [`str`] 

421 Name of the type to be removed or tuple containing a list of type 

422 names to be removed. Wildcards are allowed. 

423 

424 Raises 

425 ------ 

426 lsst.daf.butler.registry.OrphanedRecordError 

427 Raised if an attempt is made to remove the dataset type definition 

428 when there are already datasets associated with it. 

429 

430 Notes 

431 ----- 

432 If the dataset type is not registered the method will return without 

433 action. 

434 """ 

435 raise NotImplementedError() 

436 

437 @abstractmethod 

438 def getDatasetType(self, name: str) -> DatasetType: 

439 """Get the `DatasetType`. 

440 

441 Parameters 

442 ---------- 

443 name : `str` 

444 Name of the type. 

445 

446 Returns 

447 ------- 

448 type : `DatasetType` 

449 The `DatasetType` associated with the given name. 

450 

451 Raises 

452 ------ 

453 lsst.daf.butler.registry.MissingDatasetTypeError 

454 Raised if the requested dataset type has not been registered. 

455 

456 Notes 

457 ----- 

458 This method handles component dataset types automatically, though most 

459 other registry operations do not. 

460 """ 

461 raise NotImplementedError() 

462 

463 @abstractmethod 

464 def supportsIdGenerationMode(self, mode: DatasetIdGenEnum) -> bool: 

465 """Test whether the given dataset ID generation mode is supported by 

466 `insertDatasets`. 

467 

468 Parameters 

469 ---------- 

470 mode : `DatasetIdGenEnum` 

471 Enum value for the mode to test. 

472 

473 Returns 

474 ------- 

475 supported : `bool` 

476 Whether the given mode is supported. 

477 """ 

478 raise NotImplementedError() 

479 

480 @abstractmethod 

481 def findDataset( 

482 self, 

483 datasetType: DatasetType | str, 

484 dataId: DataId | None = None, 

485 *, 

486 collections: CollectionArgType | None = None, 

487 timespan: Timespan | None = None, 

488 datastore_records: bool = False, 

489 **kwargs: Any, 

490 ) -> DatasetRef | None: 

491 """Find a dataset given its `DatasetType` and data ID. 

492 

493 This can be used to obtain a `DatasetRef` that permits the dataset to 

494 be read from a `Datastore`. If the dataset is a component and can not 

495 be found using the provided dataset type, a dataset ref for the parent 

496 will be returned instead but with the correct dataset type. 

497 

498 Parameters 

499 ---------- 

500 datasetType : `DatasetType` or `str` 

501 A `DatasetType` or the name of one. If this is a `DatasetType` 

502 instance, its storage class will be respected and propagated to 

503 the output, even if it differs from the dataset type definition 

504 in the registry, as long as the storage classes are convertible. 

505 dataId : `dict` or `DataCoordinate`, optional 

506 A `dict`-like object containing the `Dimension` links that identify 

507 the dataset within a collection. 

508 collections : collection expression, optional 

509 An expression that fully or partially identifies the collections to 

510 search for the dataset; see 

511 :ref:`daf_butler_collection_expressions` for more information. 

512 Defaults to ``self.defaults.collections``. 

513 timespan : `Timespan`, optional 

514 A timespan that the validity range of the dataset must overlap. 

515 If not provided, any `~CollectionType.CALIBRATION` collections 

516 matched by the ``collections`` argument will not be searched. 

517 **kwargs 

518 Additional keyword arguments passed to 

519 `DataCoordinate.standardize` to convert ``dataId`` to a true 

520 `DataCoordinate` or augment an existing one. 

521 

522 Returns 

523 ------- 

524 ref : `DatasetRef` 

525 A reference to the dataset, or `None` if no matching Dataset 

526 was found. 

527 

528 Raises 

529 ------ 

530 lsst.daf.butler.registry.NoDefaultCollectionError 

531 Raised if ``collections`` is `None` and 

532 ``self.defaults.collections`` is `None`. 

533 LookupError 

534 Raised if one or more data ID keys are missing. 

535 lsst.daf.butler.registry.MissingDatasetTypeError 

536 Raised if the dataset type does not exist. 

537 lsst.daf.butler.registry.MissingCollectionError 

538 Raised if any of ``collections`` does not exist in the registry. 

539 

540 Notes 

541 ----- 

542 This method simply returns `None` and does not raise an exception even 

543 when the set of collections searched is intrinsically incompatible with 

544 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but 

545 only `~CollectionType.CALIBRATION` collections are being searched. 

546 This may make it harder to debug some lookup failures, but the behavior 

547 is intentional; we consider it more important that failed searches are 

548 reported consistently, regardless of the reason, and that adding 

549 additional collections that do not contain a match to the search path 

550 never changes the behavior. 

551 

552 This method handles component dataset types automatically, though most 

553 other registry operations do not. 

554 """ 

555 raise NotImplementedError() 

556 

557 @abstractmethod 

558 def insertDatasets( 

559 self, 

560 datasetType: DatasetType | str, 

561 dataIds: Iterable[DataId], 

562 run: str | None = None, 

563 expand: bool = True, 

564 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE, 

565 ) -> list[DatasetRef]: 

566 """Insert one or more datasets into the `Registry`. 

567 

568 This always adds new datasets; to associate existing datasets with 

569 a new collection, use ``associate``. 

570 

571 Parameters 

572 ---------- 

573 datasetType : `DatasetType` or `str` 

574 A `DatasetType` or the name of one. 

575 dataIds : `~collections.abc.Iterable` of `dict` or `DataCoordinate` 

576 Dimension-based identifiers for the new datasets. 

577 run : `str`, optional 

578 The name of the run that produced the datasets. Defaults to 

579 ``self.defaults.run``. 

580 expand : `bool`, optional 

581 If `True` (default), expand data IDs as they are inserted. This is 

582 necessary in general to allow datastore to generate file templates, 

583 but it may be disabled if the caller can guarantee this is 

584 unnecessary. 

585 idGenerationMode : `DatasetIdGenEnum`, optional 

586 Specifies option for generating dataset IDs. By default unique IDs 

587 are generated for each inserted dataset. 

588 

589 Returns 

590 ------- 

591 refs : `list` of `DatasetRef` 

592 Resolved `DatasetRef` instances for all given data IDs (in the same 

593 order). 

594 

595 Raises 

596 ------ 

597 lsst.daf.butler.registry.DatasetTypeError 

598 Raised if ``datasetType`` is not known to registry. 

599 lsst.daf.butler.registry.CollectionTypeError 

600 Raised if ``run`` collection type is not `~CollectionType.RUN`. 

601 lsst.daf.butler.registry.NoDefaultCollectionError 

602 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

603 lsst.daf.butler.registry.ConflictingDefinitionError 

604 If a dataset with the same dataset type and data ID as one of those 

605 given already exists in ``run``. 

606 lsst.daf.butler.registry.MissingCollectionError 

607 Raised if ``run`` does not exist in the registry. 

608 """ 

609 raise NotImplementedError() 

610 

611 @abstractmethod 

612 def _importDatasets( 

613 self, 

614 datasets: Iterable[DatasetRef], 

615 expand: bool = True, 

616 ) -> list[DatasetRef]: 

617 """Import one or more datasets into the `Registry`. 

618 

619 Difference from `insertDatasets` method is that this method accepts 

620 `DatasetRef` instances which should already be resolved and have a 

621 dataset ID. If registry supports globally-unique dataset IDs (e.g. 

622 `uuid.UUID`) then datasets which already exist in the registry will be 

623 ignored if imported again. 

624 

625 Parameters 

626 ---------- 

627 datasets : `~collections.abc.Iterable` of `DatasetRef` 

628 Datasets to be inserted. All `DatasetRef` instances must have 

629 identical ``datasetType`` and ``run`` attributes. ``run`` 

630 attribute can be `None` and defaults to ``self.defaults.run``. 

631 Datasets can specify ``id`` attribute which will be used for 

632 inserted datasets. All dataset IDs must have the same type 

633 (`int` or `uuid.UUID`), if type of dataset IDs does not match 

634 configured backend then IDs will be ignored and new IDs will be 

635 generated by backend. 

636 expand : `bool`, optional 

637 If `True` (default), expand data IDs as they are inserted. This is 

638 necessary in general, but it may be disabled if the caller can 

639 guarantee this is unnecessary. 

640 

641 Returns 

642 ------- 

643 refs : `list` of `DatasetRef` 

644 Resolved `DatasetRef` instances for all given data IDs (in the same 

645 order). If any of ``datasets`` has an ID which already exists in 

646 the database then it will not be inserted or updated, but a 

647 resolved `DatasetRef` will be returned for it in any case. 

648 

649 Raises 

650 ------ 

651 lsst.daf.butler.registry.NoDefaultCollectionError 

652 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

653 lsst.daf.butler.registry.DatasetTypeError 

654 Raised if datasets correspond to more than one dataset type or 

655 dataset type is not known to registry. 

656 lsst.daf.butler.registry.ConflictingDefinitionError 

657 If a dataset with the same dataset type and data ID as one of those 

658 given already exists in ``run``. 

659 lsst.daf.butler.registry.MissingCollectionError 

660 Raised if ``run`` does not exist in the registry. 

661 

662 Notes 

663 ----- 

664 This method is considered package-private and internal to Butler 

665 implementation. Clients outside daf_butler package should not use this 

666 method. 

667 """ 

668 raise NotImplementedError() 

669 

670 @abstractmethod 

671 def getDataset(self, id: DatasetId) -> DatasetRef | None: 

672 """Retrieve a Dataset entry. 

673 

674 Parameters 

675 ---------- 

676 id : `DatasetId` 

677 The unique identifier for the dataset. 

678 

679 Returns 

680 ------- 

681 ref : `DatasetRef` or `None` 

682 A ref to the Dataset, or `None` if no matching Dataset 

683 was found. 

684 """ 

685 raise NotImplementedError() 

686 

687 @abstractmethod 

688 def removeDatasets(self, refs: Iterable[DatasetRef]) -> None: 

689 """Remove datasets from the Registry. 

690 

691 The datasets will be removed unconditionally from all collections, and 

692 any `Quantum` that consumed this dataset will instead be marked with 

693 having a NULL input. `Datastore` records will *not* be deleted; the 

694 caller is responsible for ensuring that the dataset has already been 

695 removed from all Datastores. 

696 

697 Parameters 

698 ---------- 

699 refs : `~collections.abc.Iterable` [`DatasetRef`] 

700 References to the datasets to be removed. Must include a valid 

701 ``id`` attribute, and should be considered invalidated upon return. 

702 

703 Raises 

704 ------ 

705 lsst.daf.butler.AmbiguousDatasetError 

706 Raised if any ``ref.id`` is `None`. 

707 lsst.daf.butler.registry.OrphanedRecordError 

708 Raised if any dataset is still present in any `Datastore`. 

709 """ 

710 raise NotImplementedError() 

711 

712 @abstractmethod 

713 def associate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

714 """Add existing datasets to a `~CollectionType.TAGGED` collection. 

715 

716 If a DatasetRef with the same exact ID is already in a collection 

717 nothing is changed. If a `DatasetRef` with the same `DatasetType` and 

718 data ID but with different ID exists in the collection, 

719 `~lsst.daf.butler.registry.ConflictingDefinitionError` is raised. 

720 

721 Parameters 

722 ---------- 

723 collection : `str` 

724 Indicates the collection the datasets should be associated with. 

725 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

726 An iterable of resolved `DatasetRef` instances that already exist 

727 in this `Registry`. 

728 

729 Raises 

730 ------ 

731 lsst.daf.butler.registry.ConflictingDefinitionError 

732 If a Dataset with the given `DatasetRef` already exists in the 

733 given collection. 

734 lsst.daf.butler.registry.MissingCollectionError 

735 Raised if ``collection`` does not exist in the registry. 

736 lsst.daf.butler.registry.CollectionTypeError 

737 Raise adding new datasets to the given ``collection`` is not 

738 allowed. 

739 """ 

740 raise NotImplementedError() 

741 

742 @abstractmethod 

743 def disassociate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

744 """Remove existing datasets from a `~CollectionType.TAGGED` collection. 

745 

746 ``collection`` and ``ref`` combinations that are not currently 

747 associated are silently ignored. 

748 

749 Parameters 

750 ---------- 

751 collection : `str` 

752 The collection the datasets should no longer be associated with. 

753 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

754 An iterable of resolved `DatasetRef` instances that already exist 

755 in this `Registry`. 

756 

757 Raises 

758 ------ 

759 lsst.daf.butler.AmbiguousDatasetError 

760 Raised if any of the given dataset references is unresolved. 

761 lsst.daf.butler.registry.MissingCollectionError 

762 Raised if ``collection`` does not exist in the registry. 

763 lsst.daf.butler.registry.CollectionTypeError 

764 Raise adding new datasets to the given ``collection`` is not 

765 allowed. 

766 """ 

767 raise NotImplementedError() 

768 

769 @abstractmethod 

770 def certify(self, collection: str, refs: Iterable[DatasetRef], timespan: Timespan) -> None: 

771 """Associate one or more datasets with a calibration collection and a 

772 validity range within it. 

773 

774 Parameters 

775 ---------- 

776 collection : `str` 

777 The name of an already-registered `~CollectionType.CALIBRATION` 

778 collection. 

779 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

780 Datasets to be associated. 

781 timespan : `Timespan` 

782 The validity range for these datasets within the collection. 

783 

784 Raises 

785 ------ 

786 lsst.daf.butler.AmbiguousDatasetError 

787 Raised if any of the given `DatasetRef` instances is unresolved. 

788 lsst.daf.butler.registry.ConflictingDefinitionError 

789 Raised if the collection already contains a different dataset with 

790 the same `DatasetType` and data ID and an overlapping validity 

791 range. 

792 lsst.daf.butler.registry.CollectionTypeError 

793 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

794 collection or if one or more datasets are of a dataset type for 

795 which `DatasetType.isCalibration` returns `False`. 

796 """ 

797 raise NotImplementedError() 

798 

799 @abstractmethod 

800 def decertify( 

801 self, 

802 collection: str, 

803 datasetType: str | DatasetType, 

804 timespan: Timespan, 

805 *, 

806 dataIds: Iterable[DataId] | None = None, 

807 ) -> None: 

808 """Remove or adjust datasets to clear a validity range within a 

809 calibration collection. 

810 

811 Parameters 

812 ---------- 

813 collection : `str` 

814 The name of an already-registered `~CollectionType.CALIBRATION` 

815 collection. 

816 datasetType : `str` or `DatasetType` 

817 Name or `DatasetType` instance for the datasets to be decertified. 

818 timespan : `Timespan`, optional 

819 The validity range to remove datasets from within the collection. 

820 Datasets that overlap this range but are not contained by it will 

821 have their validity ranges adjusted to not overlap it, which may 

822 split a single dataset validity range into two. 

823 dataIds : iterable [`dict` or `DataCoordinate`], optional 

824 Data IDs that should be decertified within the given validity range 

825 If `None`, all data IDs for ``self.datasetType`` will be 

826 decertified. 

827 

828 Raises 

829 ------ 

830 lsst.daf.butler.registry.CollectionTypeError 

831 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

832 collection or if ``datasetType.isCalibration() is False``. 

833 """ 

834 raise NotImplementedError() 

835 

836 @abstractmethod 

837 def getDatasetLocations(self, ref: DatasetRef) -> Iterable[str]: 

838 """Retrieve datastore locations for a given dataset. 

839 

840 Parameters 

841 ---------- 

842 ref : `DatasetRef` 

843 A reference to the dataset for which to retrieve storage 

844 information. 

845 

846 Returns 

847 ------- 

848 datastores : `~collections.abc.Iterable` [ `str` ] 

849 All the matching datastores holding this dataset. 

850 

851 Raises 

852 ------ 

853 lsst.daf.butler.AmbiguousDatasetError 

854 Raised if ``ref.id`` is `None`. 

855 """ 

856 raise NotImplementedError() 

857 

858 @abstractmethod 

859 def expandDataId( 

860 self, 

861 dataId: DataId | None = None, 

862 *, 

863 dimensions: Iterable[str] | DimensionGroup | DimensionGraph | None = None, 

864 graph: DimensionGraph | None = None, 

865 records: NameLookupMapping[DimensionElement, DimensionRecord | None] | None = None, 

866 withDefaults: bool = True, 

867 **kwargs: Any, 

868 ) -> DataCoordinate: 

869 """Expand a dimension-based data ID to include additional information. 

870 

871 Parameters 

872 ---------- 

873 dataId : `DataCoordinate` or `dict`, optional 

874 Data ID to be expanded; augmented and overridden by ``kwargs``. 

875 dimensions : `~collections.abc.Iterable` [ `str` ], \ 

876 `DimensionGroup`, or `DimensionGraph`, optional 

877 The dimensions to be identified by the new `DataCoordinate`. 

878 If not provided, will be inferred from the keys of ``mapping`` and 

879 ``**kwargs``, and ``universe`` must be provided unless ``mapping`` 

880 is already a `DataCoordinate`. 

881 graph : `DimensionGraph`, optional 

882 Like ``dimensions``, but as a ``DimensionGraph`` instance. Ignored 

883 if ``dimensions`` is provided. Deprecated and will be removed 

884 after v27. 

885 records : `~collections.abc.Mapping` [`str`, `DimensionRecord`], \ 

886 optional 

887 Dimension record data to use before querying the database for that 

888 data, keyed by element name. 

889 withDefaults : `bool`, optional 

890 Utilize ``self.defaults.dataId`` to fill in missing governor 

891 dimension key-value pairs. Defaults to `True` (i.e. defaults are 

892 used). 

893 **kwargs 

894 Additional keywords are treated like additional key-value pairs for 

895 ``dataId``, extending and overriding 

896 

897 Returns 

898 ------- 

899 expanded : `DataCoordinate` 

900 A data ID that includes full metadata for all of the dimensions it 

901 identifies, i.e. guarantees that ``expanded.hasRecords()`` and 

902 ``expanded.hasFull()`` both return `True`. 

903 

904 Raises 

905 ------ 

906 lsst.daf.butler.registry.DataIdError 

907 Raised when ``dataId`` or keyword arguments specify unknown 

908 dimensions or values, or when a resulting data ID contains 

909 contradictory key-value pairs, according to dimension 

910 relationships. 

911 

912 Notes 

913 ----- 

914 This method cannot be relied upon to reject invalid data ID values 

915 for dimensions that do actually not have any record columns. For 

916 efficiency reasons the records for these dimensions (which have only 

917 dimension key values that are given by the caller) may be constructed 

918 directly rather than obtained from the registry database. 

919 """ 

920 raise NotImplementedError() 

921 

922 @abstractmethod 

923 def insertDimensionData( 

924 self, 

925 element: DimensionElement | str, 

926 *data: Mapping[str, Any] | DimensionRecord, 

927 conform: bool = True, 

928 replace: bool = False, 

929 skip_existing: bool = False, 

930 ) -> None: 

931 """Insert one or more dimension records into the database. 

932 

933 Parameters 

934 ---------- 

935 element : `DimensionElement` or `str` 

936 The `DimensionElement` or name thereof that identifies the table 

937 records will be inserted into. 

938 *data : `dict` or `DimensionRecord` 

939 One or more records to insert. 

940 conform : `bool`, optional 

941 If `False` (`True` is default) perform no checking or conversions, 

942 and assume that ``element`` is a `DimensionElement` instance and 

943 ``data`` is a one or more `DimensionRecord` instances of the 

944 appropriate subclass. 

945 replace : `bool`, optional 

946 If `True` (`False` is default), replace existing records in the 

947 database if there is a conflict. 

948 skip_existing : `bool`, optional 

949 If `True` (`False` is default), skip insertion if a record with 

950 the same primary key values already exists. Unlike 

951 `syncDimensionData`, this will not detect when the given record 

952 differs from what is in the database, and should not be used when 

953 this is a concern. 

954 """ 

955 raise NotImplementedError() 

956 

957 @abstractmethod 

958 def syncDimensionData( 

959 self, 

960 element: DimensionElement | str, 

961 row: Mapping[str, Any] | DimensionRecord, 

962 conform: bool = True, 

963 update: bool = False, 

964 ) -> bool | dict[str, Any]: 

965 """Synchronize the given dimension record with the database, inserting 

966 if it does not already exist and comparing values if it does. 

967 

968 Parameters 

969 ---------- 

970 element : `DimensionElement` or `str` 

971 The `DimensionElement` or name thereof that identifies the table 

972 records will be inserted into. 

973 row : `dict` or `DimensionRecord` 

974 The record to insert. 

975 conform : `bool`, optional 

976 If `False` (`True` is default) perform no checking or conversions, 

977 and assume that ``element`` is a `DimensionElement` instance and 

978 ``data`` is a one or more `DimensionRecord` instances of the 

979 appropriate subclass. 

980 update : `bool`, optional 

981 If `True` (`False` is default), update the existing record in the 

982 database if there is a conflict. 

983 

984 Returns 

985 ------- 

986 inserted_or_updated : `bool` or `dict` 

987 `True` if a new row was inserted, `False` if no changes were 

988 needed, or a `dict` mapping updated column names to their old 

989 values if an update was performed (only possible if 

990 ``update=True``). 

991 

992 Raises 

993 ------ 

994 lsst.daf.butler.registry.ConflictingDefinitionError 

995 Raised if the record exists in the database (according to primary 

996 key lookup) but is inconsistent with the given one. 

997 """ 

998 raise NotImplementedError() 

999 

1000 @abstractmethod 

1001 def queryDatasetTypes( 

1002 self, 

1003 expression: Any = ..., 

1004 *, 

1005 components: bool | None = False, 

1006 missing: list[str] | None = None, 

1007 ) -> Iterable[DatasetType]: 

1008 """Iterate over the dataset types whose names match an expression. 

1009 

1010 Parameters 

1011 ---------- 

1012 expression : dataset type expression, optional 

1013 An expression that fully or partially identifies the dataset types 

1014 to return, such as a `str`, `re.Pattern`, or iterable thereof. 

1015 ``...`` can be used to return all dataset types, and is the 

1016 default. See :ref:`daf_butler_dataset_type_expressions` for more 

1017 information. 

1018 components : `bool`, optional 

1019 If `True`, apply all expression patterns to component dataset type 

1020 names as well. If `False`, never apply patterns to components. 

1021 If `None`, apply patterns to components only if their 

1022 parent datasets were not matched by the expression. 

1023 Fully-specified component datasets (`str` or `DatasetType` 

1024 instances) are always included. 

1025 

1026 Values other than `False` are deprecated, and only `False` will be 

1027 supported after v26. After v27 this argument will be removed 

1028 entirely. 

1029 missing : `list` of `str`, optional 

1030 String dataset type names that were explicitly given (i.e. not 

1031 regular expression patterns) but not found will be appended to this 

1032 list, if it is provided. 

1033 

1034 Returns 

1035 ------- 

1036 dataset_types : `~collections.abc.Iterable` [ `DatasetType`] 

1037 An `~collections.abc.Iterable` of `DatasetType` instances whose 

1038 names match ``expression``. 

1039 

1040 Raises 

1041 ------ 

1042 lsst.daf.butler.registry.DatasetTypeExpressionError 

1043 Raised when ``expression`` is invalid. 

1044 """ 

1045 raise NotImplementedError() 

1046 

1047 @abstractmethod 

1048 def queryCollections( 

1049 self, 

1050 expression: Any = ..., 

1051 datasetType: DatasetType | None = None, 

1052 collectionTypes: Iterable[CollectionType] | CollectionType = CollectionType.all(), 

1053 flattenChains: bool = False, 

1054 includeChains: bool | None = None, 

1055 ) -> Sequence[str]: 

1056 """Iterate over the collections whose names match an expression. 

1057 

1058 Parameters 

1059 ---------- 

1060 expression : collection expression, optional 

1061 An expression that identifies the collections to return, such as 

1062 a `str` (for full matches or partial matches via globs), 

1063 `re.Pattern` (for partial matches), or iterable thereof. ``...`` 

1064 can be used to return all collections, and is the default. 

1065 See :ref:`daf_butler_collection_expressions` for more information. 

1066 datasetType : `DatasetType`, optional 

1067 If provided, only yield collections that may contain datasets of 

1068 this type. This is a conservative approximation in general; it may 

1069 yield collections that do not have any such datasets. 

1070 collectionTypes : `~collections.abc.Set` [`CollectionType`] or \ 

1071 `CollectionType`, optional 

1072 If provided, only yield collections of these types. 

1073 flattenChains : `bool`, optional 

1074 If `True` (`False` is default), recursively yield the child 

1075 collections of matching `~CollectionType.CHAINED` collections. 

1076 includeChains : `bool`, optional 

1077 If `True`, yield records for matching `~CollectionType.CHAINED` 

1078 collections. Default is the opposite of ``flattenChains``: include 

1079 either CHAINED collections or their children, but not both. 

1080 

1081 Returns 

1082 ------- 

1083 collections : `~collections.abc.Sequence` [ `str` ] 

1084 The names of collections that match ``expression``. 

1085 

1086 Raises 

1087 ------ 

1088 lsst.daf.butler.registry.CollectionExpressionError 

1089 Raised when ``expression`` is invalid. 

1090 

1091 Notes 

1092 ----- 

1093 The order in which collections are returned is unspecified, except that 

1094 the children of a `~CollectionType.CHAINED` collection are guaranteed 

1095 to be in the order in which they are searched. When multiple parent 

1096 `~CollectionType.CHAINED` collections match the same criteria, the 

1097 order in which the two lists appear is unspecified, and the lists of 

1098 children may be incomplete if a child has multiple parents. 

1099 """ 

1100 raise NotImplementedError() 

1101 

1102 @abstractmethod 

1103 def queryDatasets( 

1104 self, 

1105 datasetType: Any, 

1106 *, 

1107 collections: CollectionArgType | None = None, 

1108 dimensions: Iterable[Dimension | str] | None = None, 

1109 dataId: DataId | None = None, 

1110 where: str = "", 

1111 findFirst: bool = False, 

1112 components: bool | None = False, 

1113 bind: Mapping[str, Any] | None = None, 

1114 check: bool = True, 

1115 **kwargs: Any, 

1116 ) -> DatasetQueryResults: 

1117 """Query for and iterate over dataset references matching user-provided 

1118 criteria. 

1119 

1120 Parameters 

1121 ---------- 

1122 datasetType : dataset type expression 

1123 An expression that fully or partially identifies the dataset types 

1124 to be queried. Allowed types include `DatasetType`, `str`, 

1125 `re.Pattern`, and iterables thereof. The special value ``...`` can 

1126 be used to query all dataset types. See 

1127 :ref:`daf_butler_dataset_type_expressions` for more information. 

1128 collections : collection expression, optional 

1129 An expression that identifies the collections to search, such as a 

1130 `str` (for full matches or partial matches via globs), `re.Pattern` 

1131 (for partial matches), or iterable thereof. ``...`` can be used to 

1132 search all collections (actually just all `~CollectionType.RUN` 

1133 collections, because this will still find all datasets). 

1134 If not provided, ``self.default.collections`` is used. See 

1135 :ref:`daf_butler_collection_expressions` for more information. 

1136 dimensions : `~collections.abc.Iterable` of `Dimension` or `str` 

1137 Dimensions to include in the query (in addition to those used 

1138 to identify the queried dataset type(s)), either to constrain 

1139 the resulting datasets to those for which a matching dimension 

1140 exists, or to relate the dataset type's dimensions to dimensions 

1141 referenced by the ``dataId`` or ``where`` arguments. 

1142 dataId : `dict` or `DataCoordinate`, optional 

1143 A data ID whose key-value pairs are used as equality constraints 

1144 in the query. 

1145 where : `str`, optional 

1146 A string expression similar to a SQL WHERE clause. May involve 

1147 any column of a dimension table or (as a shortcut for the primary 

1148 key column of a dimension table) dimension name. See 

1149 :ref:`daf_butler_dimension_expressions` for more information. 

1150 findFirst : `bool`, optional 

1151 If `True` (`False` is default), for each result data ID, only 

1152 yield one `DatasetRef` of each `DatasetType`, from the first 

1153 collection in which a dataset of that dataset type appears 

1154 (according to the order of ``collections`` passed in). If `True`, 

1155 ``collections`` must not contain regular expressions and may not 

1156 be ``...``. 

1157 components : `bool`, optional 

1158 If `True`, apply all dataset expression patterns to component 

1159 dataset type names as well. If `False`, never apply patterns to 

1160 components. If `None`, apply patterns to components only 

1161 if their parent datasets were not matched by the expression. 

1162 Fully-specified component datasets (`str` or `DatasetType` 

1163 instances) are always included. 

1164 

1165 Values other than `False` are deprecated, and only `False` will be 

1166 supported after v26. After v27 this argument will be removed 

1167 entirely. 

1168 bind : `~collections.abc.Mapping`, optional 

1169 Mapping containing literal values that should be injected into the 

1170 ``where`` expression, keyed by the identifiers they replace. 

1171 Values of collection type can be expanded in some cases; see 

1172 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1173 information. 

1174 check : `bool`, optional 

1175 If `True` (default) check the query for consistency before 

1176 executing it. This may reject some valid queries that resemble 

1177 common mistakes (e.g. queries for visits without specifying an 

1178 instrument). 

1179 **kwargs 

1180 Additional keyword arguments are forwarded to 

1181 `DataCoordinate.standardize` when processing the ``dataId`` 

1182 argument (and may be used to provide a constraining data ID even 

1183 when the ``dataId`` argument is `None`). 

1184 

1185 Returns 

1186 ------- 

1187 refs : `.queries.DatasetQueryResults` 

1188 Dataset references matching the given query criteria. Nested data 

1189 IDs are guaranteed to include values for all implied dimensions 

1190 (i.e. `DataCoordinate.hasFull` will return `True`), but will not 

1191 include dimension records (`DataCoordinate.hasRecords` will be 

1192 `False`) unless `~.queries.DatasetQueryResults.expanded` is 

1193 called on the result object (which returns a new one). 

1194 

1195 Raises 

1196 ------ 

1197 lsst.daf.butler.registry.DatasetTypeExpressionError 

1198 Raised when ``datasetType`` expression is invalid. 

1199 TypeError 

1200 Raised when the arguments are incompatible, such as when a 

1201 collection wildcard is passed when ``findFirst`` is `True`, or 

1202 when ``collections`` is `None` and ``self.defaults.collections`` is 

1203 also `None`. 

1204 lsst.daf.butler.registry.DataIdError 

1205 Raised when ``dataId`` or keyword arguments specify unknown 

1206 dimensions or values, or when they contain inconsistent values. 

1207 lsst.daf.butler.registry.UserExpressionError 

1208 Raised when ``where`` expression is invalid. 

1209 

1210 Notes 

1211 ----- 

1212 When multiple dataset types are queried in a single call, the 

1213 results of this operation are equivalent to querying for each dataset 

1214 type separately in turn, and no information about the relationships 

1215 between datasets of different types is included. In contexts where 

1216 that kind of information is important, the recommended pattern is to 

1217 use `queryDataIds` to first obtain data IDs (possibly with the 

1218 desired dataset types and collections passed as constraints to the 

1219 query), and then use multiple (generally much simpler) calls to 

1220 `queryDatasets` with the returned data IDs passed as constraints. 

1221 """ 

1222 raise NotImplementedError() 

1223 

1224 @abstractmethod 

1225 def queryDataIds( 

1226 self, 

1227 # TODO: Drop `Dimension` objects on DM-41326. 

1228 dimensions: DimensionGroup | Iterable[Dimension | str] | Dimension | str, 

1229 *, 

1230 dataId: DataId | None = None, 

1231 datasets: Any = None, 

1232 collections: CollectionArgType | None = None, 

1233 where: str = "", 

1234 components: bool | None = False, 

1235 bind: Mapping[str, Any] | None = None, 

1236 check: bool = True, 

1237 **kwargs: Any, 

1238 ) -> DataCoordinateQueryResults: 

1239 """Query for data IDs matching user-provided criteria. 

1240 

1241 Parameters 

1242 ---------- 

1243 dimensions : `DimensionGroup`, `Dimension`, or `str`, or \ 

1244 `~collections.abc.Iterable` [ `Dimension` or `str` ] 

1245 The dimensions of the data IDs to yield, as either `Dimension` 

1246 instances or `str`. Will be automatically expanded to a complete 

1247 `DimensionGroup`. Support for `Dimension` instances is deprecated 

1248 and will not be supported after v27. 

1249 dataId : `dict` or `DataCoordinate`, optional 

1250 A data ID whose key-value pairs are used as equality constraints 

1251 in the query. 

1252 datasets : dataset type expression, optional 

1253 An expression that fully or partially identifies dataset types 

1254 that should constrain the yielded data IDs. For example, including 

1255 "raw" here would constrain the yielded ``instrument``, 

1256 ``exposure``, ``detector``, and ``physical_filter`` values to only 

1257 those for which at least one "raw" dataset exists in 

1258 ``collections``. Allowed types include `DatasetType`, `str`, 

1259 and iterables thereof. Regular expression objects (i.e. 

1260 `re.Pattern`) are deprecated and will be removed after the v26 

1261 release. See :ref:`daf_butler_dataset_type_expressions` for more 

1262 information. 

1263 collections : collection expression, optional 

1264 An expression that identifies the collections to search for 

1265 datasets, such as a `str` (for full matches or partial matches 

1266 via globs), `re.Pattern` (for partial matches), or iterable 

1267 thereof. ``...`` can be used to search all collections (actually 

1268 just all `~CollectionType.RUN` collections, because this will 

1269 still find all datasets). If not provided, 

1270 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1271 is also passed. See :ref:`daf_butler_collection_expressions` for 

1272 more information. 

1273 where : `str`, optional 

1274 A string expression similar to a SQL WHERE clause. May involve 

1275 any column of a dimension table or (as a shortcut for the primary 

1276 key column of a dimension table) dimension name. See 

1277 :ref:`daf_butler_dimension_expressions` for more information. 

1278 components : `bool`, optional 

1279 If `True`, apply all dataset expression patterns to component 

1280 dataset type names as well. If `False`, never apply patterns to 

1281 components. If `None`, apply patterns to components only 

1282 if their parent datasets were not matched by the expression. 

1283 Fully-specified component datasets (`str` or `DatasetType` 

1284 instances) are always included. 

1285 

1286 Values other than `False` are deprecated, and only `False` will be 

1287 supported after v26. After v27 this argument will be removed 

1288 entirely. 

1289 bind : `~collections.abc.Mapping`, optional 

1290 Mapping containing literal values that should be injected into the 

1291 ``where`` expression, keyed by the identifiers they replace. 

1292 Values of collection type can be expanded in some cases; see 

1293 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1294 information. 

1295 check : `bool`, optional 

1296 If `True` (default) check the query for consistency before 

1297 executing it. This may reject some valid queries that resemble 

1298 common mistakes (e.g. queries for visits without specifying an 

1299 instrument). 

1300 **kwargs 

1301 Additional keyword arguments are forwarded to 

1302 `DataCoordinate.standardize` when processing the ``dataId`` 

1303 argument (and may be used to provide a constraining data ID even 

1304 when the ``dataId`` argument is `None`). 

1305 

1306 Returns 

1307 ------- 

1308 dataIds : `.queries.DataCoordinateQueryResults` 

1309 Data IDs matching the given query parameters. These are guaranteed 

1310 to identify all dimensions (`DataCoordinate.hasFull` returns 

1311 `True`), but will not contain `DimensionRecord` objects 

1312 (`DataCoordinate.hasRecords` returns `False`). Call 

1313 `~.queries.DataCoordinateQueryResults.expanded` on the 

1314 returned object to fetch those (and consider using 

1315 `~.queries.DataCoordinateQueryResults.materialize` on the 

1316 returned object first if the expected number of rows is very 

1317 large). See documentation for those methods for additional 

1318 information. 

1319 

1320 Raises 

1321 ------ 

1322 lsst.daf.butler.registry.NoDefaultCollectionError 

1323 Raised if ``collections`` is `None` and 

1324 ``self.defaults.collections`` is `None`. 

1325 lsst.daf.butler.registry.CollectionExpressionError 

1326 Raised when ``collections`` expression is invalid. 

1327 lsst.daf.butler.registry.DataIdError 

1328 Raised when ``dataId`` or keyword arguments specify unknown 

1329 dimensions or values, or when they contain inconsistent values. 

1330 lsst.daf.butler.registry.DatasetTypeExpressionError 

1331 Raised when ``datasetType`` expression is invalid. 

1332 lsst.daf.butler.registry.UserExpressionError 

1333 Raised when ``where`` expression is invalid. 

1334 """ 

1335 raise NotImplementedError() 

1336 

1337 @abstractmethod 

1338 def queryDimensionRecords( 

1339 self, 

1340 element: DimensionElement | str, 

1341 *, 

1342 dataId: DataId | None = None, 

1343 datasets: Any = None, 

1344 collections: CollectionArgType | None = None, 

1345 where: str = "", 

1346 components: bool | None = False, 

1347 bind: Mapping[str, Any] | None = None, 

1348 check: bool = True, 

1349 **kwargs: Any, 

1350 ) -> DimensionRecordQueryResults: 

1351 """Query for dimension information matching user-provided criteria. 

1352 

1353 Parameters 

1354 ---------- 

1355 element : `DimensionElement` or `str` 

1356 The dimension element to obtain records for. 

1357 dataId : `dict` or `DataCoordinate`, optional 

1358 A data ID whose key-value pairs are used as equality constraints 

1359 in the query. 

1360 datasets : dataset type expression, optional 

1361 An expression that fully or partially identifies dataset types 

1362 that should constrain the yielded records. See `queryDataIds` and 

1363 :ref:`daf_butler_dataset_type_expressions` for more information. 

1364 collections : collection expression, optional 

1365 An expression that identifies the collections to search for 

1366 datasets, such as a `str` (for full matches or partial matches 

1367 via globs), `re.Pattern` (for partial matches), or iterable 

1368 thereof. ``...`` can be used to search all collections (actually 

1369 just all `~CollectionType.RUN` collections, because this will 

1370 still find all datasets). If not provided, 

1371 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1372 is also passed. See :ref:`daf_butler_collection_expressions` for 

1373 more information. 

1374 where : `str`, optional 

1375 A string expression similar to a SQL WHERE clause. See 

1376 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more 

1377 information. 

1378 components : `bool`, optional 

1379 Whether to apply dataset expressions to components as well. 

1380 See `queryDataIds` for more information. 

1381 

1382 Values other than `False` are deprecated, and only `False` will be 

1383 supported after v26. After v27 this argument will be removed 

1384 entirely. 

1385 bind : `~collections.abc.Mapping`, optional 

1386 Mapping containing literal values that should be injected into the 

1387 ``where`` expression, keyed by the identifiers they replace. 

1388 Values of collection type can be expanded in some cases; see 

1389 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1390 information. 

1391 check : `bool`, optional 

1392 If `True` (default) check the query for consistency before 

1393 executing it. This may reject some valid queries that resemble 

1394 common mistakes (e.g. queries for visits without specifying an 

1395 instrument). 

1396 **kwargs 

1397 Additional keyword arguments are forwarded to 

1398 `DataCoordinate.standardize` when processing the ``dataId`` 

1399 argument (and may be used to provide a constraining data ID even 

1400 when the ``dataId`` argument is `None`). 

1401 

1402 Returns 

1403 ------- 

1404 dataIds : `.queries.DimensionRecordQueryResults` 

1405 Data IDs matching the given query parameters. 

1406 

1407 Raises 

1408 ------ 

1409 lsst.daf.butler.registry.NoDefaultCollectionError 

1410 Raised if ``collections`` is `None` and 

1411 ``self.defaults.collections`` is `None`. 

1412 lsst.daf.butler.registry.CollectionExpressionError 

1413 Raised when ``collections`` expression is invalid. 

1414 lsst.daf.butler.registry.DataIdError 

1415 Raised when ``dataId`` or keyword arguments specify unknown 

1416 dimensions or values, or when they contain inconsistent values. 

1417 lsst.daf.butler.registry.DatasetTypeExpressionError 

1418 Raised when ``datasetType`` expression is invalid. 

1419 lsst.daf.butler.registry.UserExpressionError 

1420 Raised when ``where`` expression is invalid. 

1421 """ 

1422 raise NotImplementedError() 

1423 

1424 @abstractmethod 

1425 def queryDatasetAssociations( 

1426 self, 

1427 datasetType: str | DatasetType, 

1428 collections: CollectionArgType | None = ..., 

1429 *, 

1430 collectionTypes: Iterable[CollectionType] = CollectionType.all(), 

1431 flattenChains: bool = False, 

1432 ) -> Iterator[DatasetAssociation]: 

1433 """Iterate over dataset-collection combinations where the dataset is in 

1434 the collection. 

1435 

1436 This method is a temporary placeholder for better support for 

1437 association results in `queryDatasets`. It will probably be 

1438 removed in the future, and should be avoided in production code 

1439 whenever possible. 

1440 

1441 Parameters 

1442 ---------- 

1443 datasetType : `DatasetType` or `str` 

1444 A dataset type object or the name of one. 

1445 collections : collection expression, optional 

1446 An expression that identifies the collections to search for 

1447 datasets, such as a `str` (for full matches or partial matches 

1448 via globs), `re.Pattern` (for partial matches), or iterable 

1449 thereof. ``...`` can be used to search all collections (actually 

1450 just all `~CollectionType.RUN` collections, because this will still 

1451 find all datasets). If not provided, ``self.default.collections`` 

1452 is used. See :ref:`daf_butler_collection_expressions` for more 

1453 information. 

1454 collectionTypes : `~collections.abc.Set` [ `CollectionType` ], optional 

1455 If provided, only yield associations from collections of these 

1456 types. 

1457 flattenChains : `bool`, optional 

1458 If `True`, search in the children of `~CollectionType.CHAINED` 

1459 collections. If `False`, ``CHAINED`` collections are ignored. 

1460 

1461 Yields 

1462 ------ 

1463 association : `.DatasetAssociation` 

1464 Object representing the relationship between a single dataset and 

1465 a single collection. 

1466 

1467 Raises 

1468 ------ 

1469 lsst.daf.butler.registry.NoDefaultCollectionError 

1470 Raised if ``collections`` is `None` and 

1471 ``self.defaults.collections`` is `None`. 

1472 lsst.daf.butler.registry.CollectionExpressionError 

1473 Raised when ``collections`` expression is invalid. 

1474 """ 

1475 raise NotImplementedError() 

1476 

1477 @property 

1478 def obsCoreTableManager(self) -> ObsCoreTableManager | None: 

1479 """The ObsCore manager instance for this registry 

1480 (`~.interfaces.ObsCoreTableManager` 

1481 or `None`). 

1482 

1483 ObsCore manager may not be implemented for all registry backend, or 

1484 may not be enabled for many repositories. 

1485 """ 

1486 return None 

1487 

1488 storageClasses: StorageClassFactory 

1489 """All storage classes known to the registry (`StorageClassFactory`). 

1490 """