Coverage for python/lsst/daf/butler/registry/_registry.py: 99%

113 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-10-27 09:43 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ("Registry",) 

31 

32import contextlib 

33import logging 

34import re 

35from abc import ABC, abstractmethod 

36from collections.abc import Iterable, Iterator, Mapping, Sequence 

37from types import EllipsisType 

38from typing import TYPE_CHECKING, Any 

39 

40from .._dataset_association import DatasetAssociation 

41from .._dataset_ref import DatasetId, DatasetIdGenEnum, DatasetRef 

42from .._dataset_type import DatasetType 

43from .._named import NameLookupMapping 

44from .._storage_class import StorageClassFactory 

45from .._timespan import Timespan 

46from ..dimensions import ( 

47 DataCoordinate, 

48 DataId, 

49 Dimension, 

50 DimensionElement, 

51 DimensionGraph, 

52 DimensionRecord, 

53 DimensionUniverse, 

54) 

55from ._collection_summary import CollectionSummary 

56from ._collection_type import CollectionType 

57from ._defaults import RegistryDefaults 

58from .queries import DataCoordinateQueryResults, DatasetQueryResults, DimensionRecordQueryResults 

59from .wildcards import CollectionWildcard 

60 

61if TYPE_CHECKING: 

62 from .interfaces import ObsCoreTableManager 

63 

64_LOG = logging.getLogger(__name__) 

65 

66# TYpe alias for `collections` arguments. 

67CollectionArgType = str | re.Pattern | Iterable[str | re.Pattern] | EllipsisType | CollectionWildcard 

68 

69 

70class Registry(ABC): 

71 """Abstract Registry interface. 

72 

73 All subclasses should store `~lsst.daf.butler.registry.RegistryDefaults` in 

74 a ``_defaults`` property. No other properties are assumed shared between 

75 implementations. 

76 """ 

77 

78 @abstractmethod 

79 def isWriteable(self) -> bool: 

80 """Return `True` if this registry allows write operations, and `False` 

81 otherwise. 

82 """ 

83 raise NotImplementedError() 

84 

85 @property 

86 @abstractmethod 

87 def dimensions(self) -> DimensionUniverse: 

88 """Definitions of all dimensions recognized by this `Registry` 

89 (`DimensionUniverse`). 

90 """ 

91 raise NotImplementedError() 

92 

93 @property 

94 @abstractmethod 

95 def defaults(self) -> RegistryDefaults: 

96 """Default collection search path and/or output `~CollectionType.RUN` 

97 collection (`~lsst.daf.butler.registry.RegistryDefaults`). 

98 

99 This is an immutable struct whose components may not be set 

100 individually, but the entire struct can be set by assigning to this 

101 property. 

102 """ 

103 raise NotImplementedError() 

104 

105 @defaults.setter 

106 @abstractmethod 

107 def defaults(self, value: RegistryDefaults) -> None: 

108 raise NotImplementedError() 

109 

110 @abstractmethod 

111 def refresh(self) -> None: 

112 """Refresh all in-memory state by querying the database. 

113 

114 This may be necessary to enable querying for entities added by other 

115 registry instances after this one was constructed. 

116 """ 

117 raise NotImplementedError() 

118 

119 @contextlib.contextmanager 

120 @abstractmethod 

121 def transaction(self, *, savepoint: bool = False) -> Iterator[None]: 

122 """Return a context manager that represents a transaction.""" 

123 raise NotImplementedError() 

124 

125 def resetConnectionPool(self) -> None: 

126 """Reset connection pool for registry if relevant. 

127 

128 This operation can be used reset connections to servers when 

129 using registry with fork-based multiprocessing. This method should 

130 usually be called by the child process immediately 

131 after the fork. 

132 

133 The base class implementation is a no-op. 

134 """ 

135 pass 

136 

137 @abstractmethod 

138 def registerCollection( 

139 self, name: str, type: CollectionType = CollectionType.TAGGED, doc: str | None = None 

140 ) -> bool: 

141 """Add a new collection if one with the given name does not exist. 

142 

143 Parameters 

144 ---------- 

145 name : `str` 

146 The name of the collection to create. 

147 type : `CollectionType` 

148 Enum value indicating the type of collection to create. 

149 doc : `str`, optional 

150 Documentation string for the collection. 

151 

152 Returns 

153 ------- 

154 registered : `bool` 

155 Boolean indicating whether the collection was already registered 

156 or was created by this call. 

157 

158 Notes 

159 ----- 

160 This method cannot be called within transactions, as it needs to be 

161 able to perform its own transaction to be concurrent. 

162 """ 

163 raise NotImplementedError() 

164 

165 @abstractmethod 

166 def getCollectionType(self, name: str) -> CollectionType: 

167 """Return an enumeration value indicating the type of the given 

168 collection. 

169 

170 Parameters 

171 ---------- 

172 name : `str` 

173 The name of the collection. 

174 

175 Returns 

176 ------- 

177 type : `CollectionType` 

178 Enum value indicating the type of this collection. 

179 

180 Raises 

181 ------ 

182 lsst.daf.butler.registry.MissingCollectionError 

183 Raised if no collection with the given name exists. 

184 """ 

185 raise NotImplementedError() 

186 

187 @abstractmethod 

188 def registerRun(self, name: str, doc: str | None = None) -> bool: 

189 """Add a new run if one with the given name does not exist. 

190 

191 Parameters 

192 ---------- 

193 name : `str` 

194 The name of the run to create. 

195 doc : `str`, optional 

196 Documentation string for the collection. 

197 

198 Returns 

199 ------- 

200 registered : `bool` 

201 Boolean indicating whether a new run was registered. `False` 

202 if it already existed. 

203 

204 Notes 

205 ----- 

206 This method cannot be called within transactions, as it needs to be 

207 able to perform its own transaction to be concurrent. 

208 """ 

209 raise NotImplementedError() 

210 

211 @abstractmethod 

212 def removeCollection(self, name: str) -> None: 

213 """Remove the given collection from the registry. 

214 

215 Parameters 

216 ---------- 

217 name : `str` 

218 The name of the collection to remove. 

219 

220 Raises 

221 ------ 

222 lsst.daf.butler.registry.MissingCollectionError 

223 Raised if no collection with the given name exists. 

224 sqlalchemy.exc.IntegrityError 

225 Raised if the database rows associated with the collection are 

226 still referenced by some other table, such as a dataset in a 

227 datastore (for `~CollectionType.RUN` collections only) or a 

228 `~CollectionType.CHAINED` collection of which this collection is 

229 a child. 

230 

231 Notes 

232 ----- 

233 If this is a `~CollectionType.RUN` collection, all datasets and quanta 

234 in it will removed from the `Registry` database. This requires that 

235 those datasets be removed (or at least trashed) from any datastores 

236 that hold them first. 

237 

238 A collection may not be deleted as long as it is referenced by a 

239 `~CollectionType.CHAINED` collection; the ``CHAINED`` collection must 

240 be deleted or redefined first. 

241 """ 

242 raise NotImplementedError() 

243 

244 @abstractmethod 

245 def getCollectionChain(self, parent: str) -> Sequence[str]: 

246 """Return the child collections in a `~CollectionType.CHAINED` 

247 collection. 

248 

249 Parameters 

250 ---------- 

251 parent : `str` 

252 Name of the chained collection. Must have already been added via 

253 a call to `Registry.registerCollection`. 

254 

255 Returns 

256 ------- 

257 children : `~collections.abc.Sequence` [ `str` ] 

258 An ordered sequence of collection names that are searched when the 

259 given chained collection is searched. 

260 

261 Raises 

262 ------ 

263 lsst.daf.butler.registry.MissingCollectionError 

264 Raised if ``parent`` does not exist in the `Registry`. 

265 lsst.daf.butler.registry.CollectionTypeError 

266 Raised if ``parent`` does not correspond to a 

267 `~CollectionType.CHAINED` collection. 

268 """ 

269 raise NotImplementedError() 

270 

271 @abstractmethod 

272 def setCollectionChain(self, parent: str, children: Any, *, flatten: bool = False) -> None: 

273 """Define or redefine a `~CollectionType.CHAINED` collection. 

274 

275 Parameters 

276 ---------- 

277 parent : `str` 

278 Name of the chained collection. Must have already been added via 

279 a call to `Registry.registerCollection`. 

280 children : collection expression 

281 An expression defining an ordered search of child collections, 

282 generally an iterable of `str`; see 

283 :ref:`daf_butler_collection_expressions` for more information. 

284 flatten : `bool`, optional 

285 If `True` (`False` is default), recursively flatten out any nested 

286 `~CollectionType.CHAINED` collections in ``children`` first. 

287 

288 Raises 

289 ------ 

290 lsst.daf.butler.registry.MissingCollectionError 

291 Raised when any of the given collections do not exist in the 

292 `Registry`. 

293 lsst.daf.butler.registry.CollectionTypeError 

294 Raised if ``parent`` does not correspond to a 

295 `~CollectionType.CHAINED` collection. 

296 ValueError 

297 Raised if the given collections contains a cycle. 

298 """ 

299 raise NotImplementedError() 

300 

301 @abstractmethod 

302 def getCollectionParentChains(self, collection: str) -> set[str]: 

303 """Return the CHAINED collections that directly contain the given one. 

304 

305 Parameters 

306 ---------- 

307 name : `str` 

308 Name of the collection. 

309 

310 Returns 

311 ------- 

312 chains : `set` of `str` 

313 Set of `~CollectionType.CHAINED` collection names. 

314 """ 

315 raise NotImplementedError() 

316 

317 @abstractmethod 

318 def getCollectionDocumentation(self, collection: str) -> str | None: 

319 """Retrieve the documentation string for a collection. 

320 

321 Parameters 

322 ---------- 

323 name : `str` 

324 Name of the collection. 

325 

326 Returns 

327 ------- 

328 docs : `str` or `None` 

329 Docstring for the collection with the given name. 

330 """ 

331 raise NotImplementedError() 

332 

333 @abstractmethod 

334 def setCollectionDocumentation(self, collection: str, doc: str | None) -> None: 

335 """Set the documentation string for a collection. 

336 

337 Parameters 

338 ---------- 

339 name : `str` 

340 Name of the collection. 

341 docs : `str` or `None` 

342 Docstring for the collection with the given name; will replace any 

343 existing docstring. Passing `None` will remove any existing 

344 docstring. 

345 """ 

346 raise NotImplementedError() 

347 

348 @abstractmethod 

349 def getCollectionSummary(self, collection: str) -> CollectionSummary: 

350 """Return a summary for the given collection. 

351 

352 Parameters 

353 ---------- 

354 collection : `str` 

355 Name of the collection for which a summary is to be retrieved. 

356 

357 Returns 

358 ------- 

359 summary : `~lsst.daf.butler.registry.CollectionSummary` 

360 Summary of the dataset types and governor dimension values in 

361 this collection. 

362 """ 

363 raise NotImplementedError() 

364 

365 @abstractmethod 

366 def registerDatasetType(self, datasetType: DatasetType) -> bool: 

367 """Add a new `DatasetType` to the Registry. 

368 

369 It is not an error to register the same `DatasetType` twice. 

370 

371 Parameters 

372 ---------- 

373 datasetType : `DatasetType` 

374 The `DatasetType` to be added. 

375 

376 Returns 

377 ------- 

378 inserted : `bool` 

379 `True` if ``datasetType`` was inserted, `False` if an identical 

380 existing `DatasetType` was found. Note that in either case the 

381 DatasetType is guaranteed to be defined in the Registry 

382 consistently with the given definition. 

383 

384 Raises 

385 ------ 

386 ValueError 

387 Raised if the dimensions or storage class are invalid. 

388 lsst.daf.butler.registry.ConflictingDefinitionError 

389 Raised if this `DatasetType` is already registered with a different 

390 definition. 

391 

392 Notes 

393 ----- 

394 This method cannot be called within transactions, as it needs to be 

395 able to perform its own transaction to be concurrent. 

396 """ 

397 raise NotImplementedError() 

398 

399 @abstractmethod 

400 def removeDatasetType(self, name: str | tuple[str, ...]) -> None: 

401 """Remove the named `DatasetType` from the registry. 

402 

403 .. warning:: 

404 

405 Registry implementations can cache the dataset type definitions. 

406 This means that deleting the dataset type definition may result in 

407 unexpected behavior from other butler processes that are active 

408 that have not seen the deletion. 

409 

410 Parameters 

411 ---------- 

412 name : `str` or `tuple` [`str`] 

413 Name of the type to be removed or tuple containing a list of type 

414 names to be removed. Wildcards are allowed. 

415 

416 Raises 

417 ------ 

418 lsst.daf.butler.registry.OrphanedRecordError 

419 Raised if an attempt is made to remove the dataset type definition 

420 when there are already datasets associated with it. 

421 

422 Notes 

423 ----- 

424 If the dataset type is not registered the method will return without 

425 action. 

426 """ 

427 raise NotImplementedError() 

428 

429 @abstractmethod 

430 def getDatasetType(self, name: str) -> DatasetType: 

431 """Get the `DatasetType`. 

432 

433 Parameters 

434 ---------- 

435 name : `str` 

436 Name of the type. 

437 

438 Returns 

439 ------- 

440 type : `DatasetType` 

441 The `DatasetType` associated with the given name. 

442 

443 Raises 

444 ------ 

445 lsst.daf.butler.registry.MissingDatasetTypeError 

446 Raised if the requested dataset type has not been registered. 

447 

448 Notes 

449 ----- 

450 This method handles component dataset types automatically, though most 

451 other registry operations do not. 

452 """ 

453 raise NotImplementedError() 

454 

455 @abstractmethod 

456 def supportsIdGenerationMode(self, mode: DatasetIdGenEnum) -> bool: 

457 """Test whether the given dataset ID generation mode is supported by 

458 `insertDatasets`. 

459 

460 Parameters 

461 ---------- 

462 mode : `DatasetIdGenEnum` 

463 Enum value for the mode to test. 

464 

465 Returns 

466 ------- 

467 supported : `bool` 

468 Whether the given mode is supported. 

469 """ 

470 raise NotImplementedError() 

471 

472 @abstractmethod 

473 def findDataset( 

474 self, 

475 datasetType: DatasetType | str, 

476 dataId: DataId | None = None, 

477 *, 

478 collections: CollectionArgType | None = None, 

479 timespan: Timespan | None = None, 

480 datastore_records: bool = False, 

481 **kwargs: Any, 

482 ) -> DatasetRef | None: 

483 """Find a dataset given its `DatasetType` and data ID. 

484 

485 This can be used to obtain a `DatasetRef` that permits the dataset to 

486 be read from a `Datastore`. If the dataset is a component and can not 

487 be found using the provided dataset type, a dataset ref for the parent 

488 will be returned instead but with the correct dataset type. 

489 

490 Parameters 

491 ---------- 

492 datasetType : `DatasetType` or `str` 

493 A `DatasetType` or the name of one. If this is a `DatasetType` 

494 instance, its storage class will be respected and propagated to 

495 the output, even if it differs from the dataset type definition 

496 in the registry, as long as the storage classes are convertible. 

497 dataId : `dict` or `DataCoordinate`, optional 

498 A `dict`-like object containing the `Dimension` links that identify 

499 the dataset within a collection. 

500 collections : collection expression, optional 

501 An expression that fully or partially identifies the collections to 

502 search for the dataset; see 

503 :ref:`daf_butler_collection_expressions` for more information. 

504 Defaults to ``self.defaults.collections``. 

505 timespan : `Timespan`, optional 

506 A timespan that the validity range of the dataset must overlap. 

507 If not provided, any `~CollectionType.CALIBRATION` collections 

508 matched by the ``collections`` argument will not be searched. 

509 **kwargs 

510 Additional keyword arguments passed to 

511 `DataCoordinate.standardize` to convert ``dataId`` to a true 

512 `DataCoordinate` or augment an existing one. 

513 

514 Returns 

515 ------- 

516 ref : `DatasetRef` 

517 A reference to the dataset, or `None` if no matching Dataset 

518 was found. 

519 

520 Raises 

521 ------ 

522 lsst.daf.butler.registry.NoDefaultCollectionError 

523 Raised if ``collections`` is `None` and 

524 ``self.defaults.collections`` is `None`. 

525 LookupError 

526 Raised if one or more data ID keys are missing. 

527 lsst.daf.butler.registry.MissingDatasetTypeError 

528 Raised if the dataset type does not exist. 

529 lsst.daf.butler.registry.MissingCollectionError 

530 Raised if any of ``collections`` does not exist in the registry. 

531 

532 Notes 

533 ----- 

534 This method simply returns `None` and does not raise an exception even 

535 when the set of collections searched is intrinsically incompatible with 

536 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but 

537 only `~CollectionType.CALIBRATION` collections are being searched. 

538 This may make it harder to debug some lookup failures, but the behavior 

539 is intentional; we consider it more important that failed searches are 

540 reported consistently, regardless of the reason, and that adding 

541 additional collections that do not contain a match to the search path 

542 never changes the behavior. 

543 

544 This method handles component dataset types automatically, though most 

545 other registry operations do not. 

546 """ 

547 raise NotImplementedError() 

548 

549 @abstractmethod 

550 def insertDatasets( 

551 self, 

552 datasetType: DatasetType | str, 

553 dataIds: Iterable[DataId], 

554 run: str | None = None, 

555 expand: bool = True, 

556 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE, 

557 ) -> list[DatasetRef]: 

558 """Insert one or more datasets into the `Registry`. 

559 

560 This always adds new datasets; to associate existing datasets with 

561 a new collection, use ``associate``. 

562 

563 Parameters 

564 ---------- 

565 datasetType : `DatasetType` or `str` 

566 A `DatasetType` or the name of one. 

567 dataIds : `~collections.abc.Iterable` of `dict` or `DataCoordinate` 

568 Dimension-based identifiers for the new datasets. 

569 run : `str`, optional 

570 The name of the run that produced the datasets. Defaults to 

571 ``self.defaults.run``. 

572 expand : `bool`, optional 

573 If `True` (default), expand data IDs as they are inserted. This is 

574 necessary in general to allow datastore to generate file templates, 

575 but it may be disabled if the caller can guarantee this is 

576 unnecessary. 

577 idGenerationMode : `DatasetIdGenEnum`, optional 

578 Specifies option for generating dataset IDs. By default unique IDs 

579 are generated for each inserted dataset. 

580 

581 Returns 

582 ------- 

583 refs : `list` of `DatasetRef` 

584 Resolved `DatasetRef` instances for all given data IDs (in the same 

585 order). 

586 

587 Raises 

588 ------ 

589 lsst.daf.butler.registry.DatasetTypeError 

590 Raised if ``datasetType`` is not known to registry. 

591 lsst.daf.butler.registry.CollectionTypeError 

592 Raised if ``run`` collection type is not `~CollectionType.RUN`. 

593 lsst.daf.butler.registry.NoDefaultCollectionError 

594 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

595 lsst.daf.butler.registry.ConflictingDefinitionError 

596 If a dataset with the same dataset type and data ID as one of those 

597 given already exists in ``run``. 

598 lsst.daf.butler.registry.MissingCollectionError 

599 Raised if ``run`` does not exist in the registry. 

600 """ 

601 raise NotImplementedError() 

602 

603 @abstractmethod 

604 def _importDatasets( 

605 self, 

606 datasets: Iterable[DatasetRef], 

607 expand: bool = True, 

608 ) -> list[DatasetRef]: 

609 """Import one or more datasets into the `Registry`. 

610 

611 Difference from `insertDatasets` method is that this method accepts 

612 `DatasetRef` instances which should already be resolved and have a 

613 dataset ID. If registry supports globally-unique dataset IDs (e.g. 

614 `uuid.UUID`) then datasets which already exist in the registry will be 

615 ignored if imported again. 

616 

617 Parameters 

618 ---------- 

619 datasets : `~collections.abc.Iterable` of `DatasetRef` 

620 Datasets to be inserted. All `DatasetRef` instances must have 

621 identical ``datasetType`` and ``run`` attributes. ``run`` 

622 attribute can be `None` and defaults to ``self.defaults.run``. 

623 Datasets can specify ``id`` attribute which will be used for 

624 inserted datasets. All dataset IDs must have the same type 

625 (`int` or `uuid.UUID`), if type of dataset IDs does not match 

626 configured backend then IDs will be ignored and new IDs will be 

627 generated by backend. 

628 expand : `bool`, optional 

629 If `True` (default), expand data IDs as they are inserted. This is 

630 necessary in general, but it may be disabled if the caller can 

631 guarantee this is unnecessary. 

632 

633 Returns 

634 ------- 

635 refs : `list` of `DatasetRef` 

636 Resolved `DatasetRef` instances for all given data IDs (in the same 

637 order). If any of ``datasets`` has an ID which already exists in 

638 the database then it will not be inserted or updated, but a 

639 resolved `DatasetRef` will be returned for it in any case. 

640 

641 Raises 

642 ------ 

643 lsst.daf.butler.registry.NoDefaultCollectionError 

644 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

645 lsst.daf.butler.registry.DatasetTypeError 

646 Raised if datasets correspond to more than one dataset type or 

647 dataset type is not known to registry. 

648 lsst.daf.butler.registry.ConflictingDefinitionError 

649 If a dataset with the same dataset type and data ID as one of those 

650 given already exists in ``run``. 

651 lsst.daf.butler.registry.MissingCollectionError 

652 Raised if ``run`` does not exist in the registry. 

653 

654 Notes 

655 ----- 

656 This method is considered package-private and internal to Butler 

657 implementation. Clients outside daf_butler package should not use this 

658 method. 

659 """ 

660 raise NotImplementedError() 

661 

662 @abstractmethod 

663 def getDataset(self, id: DatasetId) -> DatasetRef | None: 

664 """Retrieve a Dataset entry. 

665 

666 Parameters 

667 ---------- 

668 id : `DatasetId` 

669 The unique identifier for the dataset. 

670 

671 Returns 

672 ------- 

673 ref : `DatasetRef` or `None` 

674 A ref to the Dataset, or `None` if no matching Dataset 

675 was found. 

676 """ 

677 raise NotImplementedError() 

678 

679 @abstractmethod 

680 def removeDatasets(self, refs: Iterable[DatasetRef]) -> None: 

681 """Remove datasets from the Registry. 

682 

683 The datasets will be removed unconditionally from all collections, and 

684 any `Quantum` that consumed this dataset will instead be marked with 

685 having a NULL input. `Datastore` records will *not* be deleted; the 

686 caller is responsible for ensuring that the dataset has already been 

687 removed from all Datastores. 

688 

689 Parameters 

690 ---------- 

691 refs : `~collections.abc.Iterable` [`DatasetRef`] 

692 References to the datasets to be removed. Must include a valid 

693 ``id`` attribute, and should be considered invalidated upon return. 

694 

695 Raises 

696 ------ 

697 lsst.daf.butler.AmbiguousDatasetError 

698 Raised if any ``ref.id`` is `None`. 

699 lsst.daf.butler.registry.OrphanedRecordError 

700 Raised if any dataset is still present in any `Datastore`. 

701 """ 

702 raise NotImplementedError() 

703 

704 @abstractmethod 

705 def associate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

706 """Add existing datasets to a `~CollectionType.TAGGED` collection. 

707 

708 If a DatasetRef with the same exact ID is already in a collection 

709 nothing is changed. If a `DatasetRef` with the same `DatasetType` and 

710 data ID but with different ID exists in the collection, 

711 `~lsst.daf.butler.registry.ConflictingDefinitionError` is raised. 

712 

713 Parameters 

714 ---------- 

715 collection : `str` 

716 Indicates the collection the datasets should be associated with. 

717 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

718 An iterable of resolved `DatasetRef` instances that already exist 

719 in this `Registry`. 

720 

721 Raises 

722 ------ 

723 lsst.daf.butler.registry.ConflictingDefinitionError 

724 If a Dataset with the given `DatasetRef` already exists in the 

725 given collection. 

726 lsst.daf.butler.registry.MissingCollectionError 

727 Raised if ``collection`` does not exist in the registry. 

728 lsst.daf.butler.registry.CollectionTypeError 

729 Raise adding new datasets to the given ``collection`` is not 

730 allowed. 

731 """ 

732 raise NotImplementedError() 

733 

734 @abstractmethod 

735 def disassociate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

736 """Remove existing datasets from a `~CollectionType.TAGGED` collection. 

737 

738 ``collection`` and ``ref`` combinations that are not currently 

739 associated are silently ignored. 

740 

741 Parameters 

742 ---------- 

743 collection : `str` 

744 The collection the datasets should no longer be associated with. 

745 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

746 An iterable of resolved `DatasetRef` instances that already exist 

747 in this `Registry`. 

748 

749 Raises 

750 ------ 

751 lsst.daf.butler.AmbiguousDatasetError 

752 Raised if any of the given dataset references is unresolved. 

753 lsst.daf.butler.registry.MissingCollectionError 

754 Raised if ``collection`` does not exist in the registry. 

755 lsst.daf.butler.registry.CollectionTypeError 

756 Raise adding new datasets to the given ``collection`` is not 

757 allowed. 

758 """ 

759 raise NotImplementedError() 

760 

761 @abstractmethod 

762 def certify(self, collection: str, refs: Iterable[DatasetRef], timespan: Timespan) -> None: 

763 """Associate one or more datasets with a calibration collection and a 

764 validity range within it. 

765 

766 Parameters 

767 ---------- 

768 collection : `str` 

769 The name of an already-registered `~CollectionType.CALIBRATION` 

770 collection. 

771 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

772 Datasets to be associated. 

773 timespan : `Timespan` 

774 The validity range for these datasets within the collection. 

775 

776 Raises 

777 ------ 

778 lsst.daf.butler.AmbiguousDatasetError 

779 Raised if any of the given `DatasetRef` instances is unresolved. 

780 lsst.daf.butler.registry.ConflictingDefinitionError 

781 Raised if the collection already contains a different dataset with 

782 the same `DatasetType` and data ID and an overlapping validity 

783 range. 

784 lsst.daf.butler.registry.CollectionTypeError 

785 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

786 collection or if one or more datasets are of a dataset type for 

787 which `DatasetType.isCalibration` returns `False`. 

788 """ 

789 raise NotImplementedError() 

790 

791 @abstractmethod 

792 def decertify( 

793 self, 

794 collection: str, 

795 datasetType: str | DatasetType, 

796 timespan: Timespan, 

797 *, 

798 dataIds: Iterable[DataId] | None = None, 

799 ) -> None: 

800 """Remove or adjust datasets to clear a validity range within a 

801 calibration collection. 

802 

803 Parameters 

804 ---------- 

805 collection : `str` 

806 The name of an already-registered `~CollectionType.CALIBRATION` 

807 collection. 

808 datasetType : `str` or `DatasetType` 

809 Name or `DatasetType` instance for the datasets to be decertified. 

810 timespan : `Timespan`, optional 

811 The validity range to remove datasets from within the collection. 

812 Datasets that overlap this range but are not contained by it will 

813 have their validity ranges adjusted to not overlap it, which may 

814 split a single dataset validity range into two. 

815 dataIds : iterable [`dict` or `DataCoordinate`], optional 

816 Data IDs that should be decertified within the given validity range 

817 If `None`, all data IDs for ``self.datasetType`` will be 

818 decertified. 

819 

820 Raises 

821 ------ 

822 lsst.daf.butler.registry.CollectionTypeError 

823 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

824 collection or if ``datasetType.isCalibration() is False``. 

825 """ 

826 raise NotImplementedError() 

827 

828 @abstractmethod 

829 def getDatasetLocations(self, ref: DatasetRef) -> Iterable[str]: 

830 """Retrieve datastore locations for a given dataset. 

831 

832 Parameters 

833 ---------- 

834 ref : `DatasetRef` 

835 A reference to the dataset for which to retrieve storage 

836 information. 

837 

838 Returns 

839 ------- 

840 datastores : `~collections.abc.Iterable` [ `str` ] 

841 All the matching datastores holding this dataset. 

842 

843 Raises 

844 ------ 

845 lsst.daf.butler.AmbiguousDatasetError 

846 Raised if ``ref.id`` is `None`. 

847 """ 

848 raise NotImplementedError() 

849 

850 @abstractmethod 

851 def expandDataId( 

852 self, 

853 dataId: DataId | None = None, 

854 *, 

855 graph: DimensionGraph | None = None, 

856 records: NameLookupMapping[DimensionElement, DimensionRecord | None] | None = None, 

857 withDefaults: bool = True, 

858 **kwargs: Any, 

859 ) -> DataCoordinate: 

860 """Expand a dimension-based data ID to include additional information. 

861 

862 Parameters 

863 ---------- 

864 dataId : `DataCoordinate` or `dict`, optional 

865 Data ID to be expanded; augmented and overridden by ``kwargs``. 

866 graph : `DimensionGraph`, optional 

867 Set of dimensions for the expanded ID. If `None`, the dimensions 

868 will be inferred from the keys of ``dataId`` and ``kwargs``. 

869 Dimensions that are in ``dataId`` or ``kwargs`` but not in 

870 ``graph`` are silently ignored, providing a way to extract and 

871 ``graph`` expand a subset of a data ID. 

872 records : `~collections.abc.Mapping` [`str`, `DimensionRecord`], \ 

873 optional 

874 Dimension record data to use before querying the database for that 

875 data, keyed by element name. 

876 withDefaults : `bool`, optional 

877 Utilize ``self.defaults.dataId`` to fill in missing governor 

878 dimension key-value pairs. Defaults to `True` (i.e. defaults are 

879 used). 

880 **kwargs 

881 Additional keywords are treated like additional key-value pairs for 

882 ``dataId``, extending and overriding 

883 

884 Returns 

885 ------- 

886 expanded : `DataCoordinate` 

887 A data ID that includes full metadata for all of the dimensions it 

888 identifies, i.e. guarantees that ``expanded.hasRecords()`` and 

889 ``expanded.hasFull()`` both return `True`. 

890 

891 Raises 

892 ------ 

893 lsst.daf.butler.registry.DataIdError 

894 Raised when ``dataId`` or keyword arguments specify unknown 

895 dimensions or values, or when a resulting data ID contains 

896 contradictory key-value pairs, according to dimension 

897 relationships. 

898 

899 Notes 

900 ----- 

901 This method cannot be relied upon to reject invalid data ID values 

902 for dimensions that do actually not have any record columns. For 

903 efficiency reasons the records for these dimensions (which have only 

904 dimension key values that are given by the caller) may be constructed 

905 directly rather than obtained from the registry database. 

906 """ 

907 raise NotImplementedError() 

908 

909 @abstractmethod 

910 def insertDimensionData( 

911 self, 

912 element: DimensionElement | str, 

913 *data: Mapping[str, Any] | DimensionRecord, 

914 conform: bool = True, 

915 replace: bool = False, 

916 skip_existing: bool = False, 

917 ) -> None: 

918 """Insert one or more dimension records into the database. 

919 

920 Parameters 

921 ---------- 

922 element : `DimensionElement` or `str` 

923 The `DimensionElement` or name thereof that identifies the table 

924 records will be inserted into. 

925 *data : `dict` or `DimensionRecord` 

926 One or more records to insert. 

927 conform : `bool`, optional 

928 If `False` (`True` is default) perform no checking or conversions, 

929 and assume that ``element`` is a `DimensionElement` instance and 

930 ``data`` is a one or more `DimensionRecord` instances of the 

931 appropriate subclass. 

932 replace : `bool`, optional 

933 If `True` (`False` is default), replace existing records in the 

934 database if there is a conflict. 

935 skip_existing : `bool`, optional 

936 If `True` (`False` is default), skip insertion if a record with 

937 the same primary key values already exists. Unlike 

938 `syncDimensionData`, this will not detect when the given record 

939 differs from what is in the database, and should not be used when 

940 this is a concern. 

941 """ 

942 raise NotImplementedError() 

943 

944 @abstractmethod 

945 def syncDimensionData( 

946 self, 

947 element: DimensionElement | str, 

948 row: Mapping[str, Any] | DimensionRecord, 

949 conform: bool = True, 

950 update: bool = False, 

951 ) -> bool | dict[str, Any]: 

952 """Synchronize the given dimension record with the database, inserting 

953 if it does not already exist and comparing values if it does. 

954 

955 Parameters 

956 ---------- 

957 element : `DimensionElement` or `str` 

958 The `DimensionElement` or name thereof that identifies the table 

959 records will be inserted into. 

960 row : `dict` or `DimensionRecord` 

961 The record to insert. 

962 conform : `bool`, optional 

963 If `False` (`True` is default) perform no checking or conversions, 

964 and assume that ``element`` is a `DimensionElement` instance and 

965 ``data`` is a one or more `DimensionRecord` instances of the 

966 appropriate subclass. 

967 update : `bool`, optional 

968 If `True` (`False` is default), update the existing record in the 

969 database if there is a conflict. 

970 

971 Returns 

972 ------- 

973 inserted_or_updated : `bool` or `dict` 

974 `True` if a new row was inserted, `False` if no changes were 

975 needed, or a `dict` mapping updated column names to their old 

976 values if an update was performed (only possible if 

977 ``update=True``). 

978 

979 Raises 

980 ------ 

981 lsst.daf.butler.registry.ConflictingDefinitionError 

982 Raised if the record exists in the database (according to primary 

983 key lookup) but is inconsistent with the given one. 

984 """ 

985 raise NotImplementedError() 

986 

987 @abstractmethod 

988 def queryDatasetTypes( 

989 self, 

990 expression: Any = ..., 

991 *, 

992 components: bool | None = False, 

993 missing: list[str] | None = None, 

994 ) -> Iterable[DatasetType]: 

995 """Iterate over the dataset types whose names match an expression. 

996 

997 Parameters 

998 ---------- 

999 expression : dataset type expression, optional 

1000 An expression that fully or partially identifies the dataset types 

1001 to return, such as a `str`, `re.Pattern`, or iterable thereof. 

1002 ``...`` can be used to return all dataset types, and is the 

1003 default. See :ref:`daf_butler_dataset_type_expressions` for more 

1004 information. 

1005 components : `bool`, optional 

1006 If `True`, apply all expression patterns to component dataset type 

1007 names as well. If `False`, never apply patterns to components. 

1008 If `None`, apply patterns to components only if their 

1009 parent datasets were not matched by the expression. 

1010 Fully-specified component datasets (`str` or `DatasetType` 

1011 instances) are always included. 

1012 

1013 Values other than `False` are deprecated, and only `False` will be 

1014 supported after v26. After v27 this argument will be removed 

1015 entirely. 

1016 missing : `list` of `str`, optional 

1017 String dataset type names that were explicitly given (i.e. not 

1018 regular expression patterns) but not found will be appended to this 

1019 list, if it is provided. 

1020 

1021 Returns 

1022 ------- 

1023 dataset_types : `~collections.abc.Iterable` [ `DatasetType`] 

1024 An `~collections.abc.Iterable` of `DatasetType` instances whose 

1025 names match ``expression``. 

1026 

1027 Raises 

1028 ------ 

1029 lsst.daf.butler.registry.DatasetTypeExpressionError 

1030 Raised when ``expression`` is invalid. 

1031 """ 

1032 raise NotImplementedError() 

1033 

1034 @abstractmethod 

1035 def queryCollections( 

1036 self, 

1037 expression: Any = ..., 

1038 datasetType: DatasetType | None = None, 

1039 collectionTypes: Iterable[CollectionType] | CollectionType = CollectionType.all(), 

1040 flattenChains: bool = False, 

1041 includeChains: bool | None = None, 

1042 ) -> Sequence[str]: 

1043 """Iterate over the collections whose names match an expression. 

1044 

1045 Parameters 

1046 ---------- 

1047 expression : collection expression, optional 

1048 An expression that identifies the collections to return, such as 

1049 a `str` (for full matches or partial matches via globs), 

1050 `re.Pattern` (for partial matches), or iterable thereof. ``...`` 

1051 can be used to return all collections, and is the default. 

1052 See :ref:`daf_butler_collection_expressions` for more information. 

1053 datasetType : `DatasetType`, optional 

1054 If provided, only yield collections that may contain datasets of 

1055 this type. This is a conservative approximation in general; it may 

1056 yield collections that do not have any such datasets. 

1057 collectionTypes : `~collections.abc.Set` [`CollectionType`] or \ 

1058 `CollectionType`, optional 

1059 If provided, only yield collections of these types. 

1060 flattenChains : `bool`, optional 

1061 If `True` (`False` is default), recursively yield the child 

1062 collections of matching `~CollectionType.CHAINED` collections. 

1063 includeChains : `bool`, optional 

1064 If `True`, yield records for matching `~CollectionType.CHAINED` 

1065 collections. Default is the opposite of ``flattenChains``: include 

1066 either CHAINED collections or their children, but not both. 

1067 

1068 Returns 

1069 ------- 

1070 collections : `~collections.abc.Sequence` [ `str` ] 

1071 The names of collections that match ``expression``. 

1072 

1073 Raises 

1074 ------ 

1075 lsst.daf.butler.registry.CollectionExpressionError 

1076 Raised when ``expression`` is invalid. 

1077 

1078 Notes 

1079 ----- 

1080 The order in which collections are returned is unspecified, except that 

1081 the children of a `~CollectionType.CHAINED` collection are guaranteed 

1082 to be in the order in which they are searched. When multiple parent 

1083 `~CollectionType.CHAINED` collections match the same criteria, the 

1084 order in which the two lists appear is unspecified, and the lists of 

1085 children may be incomplete if a child has multiple parents. 

1086 """ 

1087 raise NotImplementedError() 

1088 

1089 @abstractmethod 

1090 def queryDatasets( 

1091 self, 

1092 datasetType: Any, 

1093 *, 

1094 collections: CollectionArgType | None = None, 

1095 dimensions: Iterable[Dimension | str] | None = None, 

1096 dataId: DataId | None = None, 

1097 where: str = "", 

1098 findFirst: bool = False, 

1099 components: bool | None = False, 

1100 bind: Mapping[str, Any] | None = None, 

1101 check: bool = True, 

1102 **kwargs: Any, 

1103 ) -> DatasetQueryResults: 

1104 """Query for and iterate over dataset references matching user-provided 

1105 criteria. 

1106 

1107 Parameters 

1108 ---------- 

1109 datasetType : dataset type expression 

1110 An expression that fully or partially identifies the dataset types 

1111 to be queried. Allowed types include `DatasetType`, `str`, 

1112 `re.Pattern`, and iterables thereof. The special value ``...`` can 

1113 be used to query all dataset types. See 

1114 :ref:`daf_butler_dataset_type_expressions` for more information. 

1115 collections : collection expression, optional 

1116 An expression that identifies the collections to search, such as a 

1117 `str` (for full matches or partial matches via globs), `re.Pattern` 

1118 (for partial matches), or iterable thereof. ``...`` can be used to 

1119 search all collections (actually just all `~CollectionType.RUN` 

1120 collections, because this will still find all datasets). 

1121 If not provided, ``self.default.collections`` is used. See 

1122 :ref:`daf_butler_collection_expressions` for more information. 

1123 dimensions : `~collections.abc.Iterable` of `Dimension` or `str` 

1124 Dimensions to include in the query (in addition to those used 

1125 to identify the queried dataset type(s)), either to constrain 

1126 the resulting datasets to those for which a matching dimension 

1127 exists, or to relate the dataset type's dimensions to dimensions 

1128 referenced by the ``dataId`` or ``where`` arguments. 

1129 dataId : `dict` or `DataCoordinate`, optional 

1130 A data ID whose key-value pairs are used as equality constraints 

1131 in the query. 

1132 where : `str`, optional 

1133 A string expression similar to a SQL WHERE clause. May involve 

1134 any column of a dimension table or (as a shortcut for the primary 

1135 key column of a dimension table) dimension name. See 

1136 :ref:`daf_butler_dimension_expressions` for more information. 

1137 findFirst : `bool`, optional 

1138 If `True` (`False` is default), for each result data ID, only 

1139 yield one `DatasetRef` of each `DatasetType`, from the first 

1140 collection in which a dataset of that dataset type appears 

1141 (according to the order of ``collections`` passed in). If `True`, 

1142 ``collections`` must not contain regular expressions and may not 

1143 be ``...``. 

1144 components : `bool`, optional 

1145 If `True`, apply all dataset expression patterns to component 

1146 dataset type names as well. If `False`, never apply patterns to 

1147 components. If `None`, apply patterns to components only 

1148 if their parent datasets were not matched by the expression. 

1149 Fully-specified component datasets (`str` or `DatasetType` 

1150 instances) are always included. 

1151 

1152 Values other than `False` are deprecated, and only `False` will be 

1153 supported after v26. After v27 this argument will be removed 

1154 entirely. 

1155 bind : `~collections.abc.Mapping`, optional 

1156 Mapping containing literal values that should be injected into the 

1157 ``where`` expression, keyed by the identifiers they replace. 

1158 Values of collection type can be expanded in some cases; see 

1159 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1160 information. 

1161 check : `bool`, optional 

1162 If `True` (default) check the query for consistency before 

1163 executing it. This may reject some valid queries that resemble 

1164 common mistakes (e.g. queries for visits without specifying an 

1165 instrument). 

1166 **kwargs 

1167 Additional keyword arguments are forwarded to 

1168 `DataCoordinate.standardize` when processing the ``dataId`` 

1169 argument (and may be used to provide a constraining data ID even 

1170 when the ``dataId`` argument is `None`). 

1171 

1172 Returns 

1173 ------- 

1174 refs : `.queries.DatasetQueryResults` 

1175 Dataset references matching the given query criteria. Nested data 

1176 IDs are guaranteed to include values for all implied dimensions 

1177 (i.e. `DataCoordinate.hasFull` will return `True`), but will not 

1178 include dimension records (`DataCoordinate.hasRecords` will be 

1179 `False`) unless `~.queries.DatasetQueryResults.expanded` is 

1180 called on the result object (which returns a new one). 

1181 

1182 Raises 

1183 ------ 

1184 lsst.daf.butler.registry.DatasetTypeExpressionError 

1185 Raised when ``datasetType`` expression is invalid. 

1186 TypeError 

1187 Raised when the arguments are incompatible, such as when a 

1188 collection wildcard is passed when ``findFirst`` is `True`, or 

1189 when ``collections`` is `None` and ``self.defaults.collections`` is 

1190 also `None`. 

1191 lsst.daf.butler.registry.DataIdError 

1192 Raised when ``dataId`` or keyword arguments specify unknown 

1193 dimensions or values, or when they contain inconsistent values. 

1194 lsst.daf.butler.registry.UserExpressionError 

1195 Raised when ``where`` expression is invalid. 

1196 

1197 Notes 

1198 ----- 

1199 When multiple dataset types are queried in a single call, the 

1200 results of this operation are equivalent to querying for each dataset 

1201 type separately in turn, and no information about the relationships 

1202 between datasets of different types is included. In contexts where 

1203 that kind of information is important, the recommended pattern is to 

1204 use `queryDataIds` to first obtain data IDs (possibly with the 

1205 desired dataset types and collections passed as constraints to the 

1206 query), and then use multiple (generally much simpler) calls to 

1207 `queryDatasets` with the returned data IDs passed as constraints. 

1208 """ 

1209 raise NotImplementedError() 

1210 

1211 @abstractmethod 

1212 def queryDataIds( 

1213 self, 

1214 dimensions: Iterable[Dimension | str] | Dimension | str, 

1215 *, 

1216 dataId: DataId | None = None, 

1217 datasets: Any = None, 

1218 collections: CollectionArgType | None = None, 

1219 where: str = "", 

1220 components: bool | None = False, 

1221 bind: Mapping[str, Any] | None = None, 

1222 check: bool = True, 

1223 **kwargs: Any, 

1224 ) -> DataCoordinateQueryResults: 

1225 """Query for data IDs matching user-provided criteria. 

1226 

1227 Parameters 

1228 ---------- 

1229 dimensions : `Dimension` or `str`, or iterable thereof 

1230 The dimensions of the data IDs to yield, as either `Dimension` 

1231 instances or `str`. Will be automatically expanded to a complete 

1232 `DimensionGraph`. 

1233 dataId : `dict` or `DataCoordinate`, optional 

1234 A data ID whose key-value pairs are used as equality constraints 

1235 in the query. 

1236 datasets : dataset type expression, optional 

1237 An expression that fully or partially identifies dataset types 

1238 that should constrain the yielded data IDs. For example, including 

1239 "raw" here would constrain the yielded ``instrument``, 

1240 ``exposure``, ``detector``, and ``physical_filter`` values to only 

1241 those for which at least one "raw" dataset exists in 

1242 ``collections``. Allowed types include `DatasetType`, `str`, 

1243 and iterables thereof. Regular expression objects (i.e. 

1244 `re.Pattern`) are deprecated and will be removed after the v26 

1245 release. See :ref:`daf_butler_dataset_type_expressions` for more 

1246 information. 

1247 collections : collection expression, optional 

1248 An expression that identifies the collections to search for 

1249 datasets, such as a `str` (for full matches or partial matches 

1250 via globs), `re.Pattern` (for partial matches), or iterable 

1251 thereof. ``...`` can be used to search all collections (actually 

1252 just all `~CollectionType.RUN` collections, because this will 

1253 still find all datasets). If not provided, 

1254 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1255 is also passed. See :ref:`daf_butler_collection_expressions` for 

1256 more information. 

1257 where : `str`, optional 

1258 A string expression similar to a SQL WHERE clause. May involve 

1259 any column of a dimension table or (as a shortcut for the primary 

1260 key column of a dimension table) dimension name. See 

1261 :ref:`daf_butler_dimension_expressions` for more information. 

1262 components : `bool`, optional 

1263 If `True`, apply all dataset expression patterns to component 

1264 dataset type names as well. If `False`, never apply patterns to 

1265 components. If `None`, apply patterns to components only 

1266 if their parent datasets were not matched by the expression. 

1267 Fully-specified component datasets (`str` or `DatasetType` 

1268 instances) are always included. 

1269 

1270 Values other than `False` are deprecated, and only `False` will be 

1271 supported after v26. After v27 this argument will be removed 

1272 entirely. 

1273 bind : `~collections.abc.Mapping`, optional 

1274 Mapping containing literal values that should be injected into the 

1275 ``where`` expression, keyed by the identifiers they replace. 

1276 Values of collection type can be expanded in some cases; see 

1277 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1278 information. 

1279 check : `bool`, optional 

1280 If `True` (default) check the query for consistency before 

1281 executing it. This may reject some valid queries that resemble 

1282 common mistakes (e.g. queries for visits without specifying an 

1283 instrument). 

1284 **kwargs 

1285 Additional keyword arguments are forwarded to 

1286 `DataCoordinate.standardize` when processing the ``dataId`` 

1287 argument (and may be used to provide a constraining data ID even 

1288 when the ``dataId`` argument is `None`). 

1289 

1290 Returns 

1291 ------- 

1292 dataIds : `.queries.DataCoordinateQueryResults` 

1293 Data IDs matching the given query parameters. These are guaranteed 

1294 to identify all dimensions (`DataCoordinate.hasFull` returns 

1295 `True`), but will not contain `DimensionRecord` objects 

1296 (`DataCoordinate.hasRecords` returns `False`). Call 

1297 `~.queries.DataCoordinateQueryResults.expanded` on the 

1298 returned object to fetch those (and consider using 

1299 `~.queries.DataCoordinateQueryResults.materialize` on the 

1300 returned object first if the expected number of rows is very 

1301 large). See documentation for those methods for additional 

1302 information. 

1303 

1304 Raises 

1305 ------ 

1306 lsst.daf.butler.registry.NoDefaultCollectionError 

1307 Raised if ``collections`` is `None` and 

1308 ``self.defaults.collections`` is `None`. 

1309 lsst.daf.butler.registry.CollectionExpressionError 

1310 Raised when ``collections`` expression is invalid. 

1311 lsst.daf.butler.registry.DataIdError 

1312 Raised when ``dataId`` or keyword arguments specify unknown 

1313 dimensions or values, or when they contain inconsistent values. 

1314 lsst.daf.butler.registry.DatasetTypeExpressionError 

1315 Raised when ``datasetType`` expression is invalid. 

1316 lsst.daf.butler.registry.UserExpressionError 

1317 Raised when ``where`` expression is invalid. 

1318 """ 

1319 raise NotImplementedError() 

1320 

1321 @abstractmethod 

1322 def queryDimensionRecords( 

1323 self, 

1324 element: DimensionElement | str, 

1325 *, 

1326 dataId: DataId | None = None, 

1327 datasets: Any = None, 

1328 collections: CollectionArgType | None = None, 

1329 where: str = "", 

1330 components: bool | None = False, 

1331 bind: Mapping[str, Any] | None = None, 

1332 check: bool = True, 

1333 **kwargs: Any, 

1334 ) -> DimensionRecordQueryResults: 

1335 """Query for dimension information matching user-provided criteria. 

1336 

1337 Parameters 

1338 ---------- 

1339 element : `DimensionElement` or `str` 

1340 The dimension element to obtain records for. 

1341 dataId : `dict` or `DataCoordinate`, optional 

1342 A data ID whose key-value pairs are used as equality constraints 

1343 in the query. 

1344 datasets : dataset type expression, optional 

1345 An expression that fully or partially identifies dataset types 

1346 that should constrain the yielded records. See `queryDataIds` and 

1347 :ref:`daf_butler_dataset_type_expressions` for more information. 

1348 collections : collection expression, optional 

1349 An expression that identifies the collections to search for 

1350 datasets, such as a `str` (for full matches or partial matches 

1351 via globs), `re.Pattern` (for partial matches), or iterable 

1352 thereof. ``...`` can be used to search all collections (actually 

1353 just all `~CollectionType.RUN` collections, because this will 

1354 still find all datasets). If not provided, 

1355 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1356 is also passed. See :ref:`daf_butler_collection_expressions` for 

1357 more information. 

1358 where : `str`, optional 

1359 A string expression similar to a SQL WHERE clause. See 

1360 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more 

1361 information. 

1362 components : `bool`, optional 

1363 Whether to apply dataset expressions to components as well. 

1364 See `queryDataIds` for more information. 

1365 

1366 Values other than `False` are deprecated, and only `False` will be 

1367 supported after v26. After v27 this argument will be removed 

1368 entirely. 

1369 bind : `~collections.abc.Mapping`, optional 

1370 Mapping containing literal values that should be injected into the 

1371 ``where`` expression, keyed by the identifiers they replace. 

1372 Values of collection type can be expanded in some cases; see 

1373 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1374 information. 

1375 check : `bool`, optional 

1376 If `True` (default) check the query for consistency before 

1377 executing it. This may reject some valid queries that resemble 

1378 common mistakes (e.g. queries for visits without specifying an 

1379 instrument). 

1380 **kwargs 

1381 Additional keyword arguments are forwarded to 

1382 `DataCoordinate.standardize` when processing the ``dataId`` 

1383 argument (and may be used to provide a constraining data ID even 

1384 when the ``dataId`` argument is `None`). 

1385 

1386 Returns 

1387 ------- 

1388 dataIds : `.queries.DimensionRecordQueryResults` 

1389 Data IDs matching the given query parameters. 

1390 

1391 Raises 

1392 ------ 

1393 lsst.daf.butler.registry.NoDefaultCollectionError 

1394 Raised if ``collections`` is `None` and 

1395 ``self.defaults.collections`` is `None`. 

1396 lsst.daf.butler.registry.CollectionExpressionError 

1397 Raised when ``collections`` expression is invalid. 

1398 lsst.daf.butler.registry.DataIdError 

1399 Raised when ``dataId`` or keyword arguments specify unknown 

1400 dimensions or values, or when they contain inconsistent values. 

1401 lsst.daf.butler.registry.DatasetTypeExpressionError 

1402 Raised when ``datasetType`` expression is invalid. 

1403 lsst.daf.butler.registry.UserExpressionError 

1404 Raised when ``where`` expression is invalid. 

1405 """ 

1406 raise NotImplementedError() 

1407 

1408 @abstractmethod 

1409 def queryDatasetAssociations( 

1410 self, 

1411 datasetType: str | DatasetType, 

1412 collections: CollectionArgType | None = ..., 

1413 *, 

1414 collectionTypes: Iterable[CollectionType] = CollectionType.all(), 

1415 flattenChains: bool = False, 

1416 ) -> Iterator[DatasetAssociation]: 

1417 """Iterate over dataset-collection combinations where the dataset is in 

1418 the collection. 

1419 

1420 This method is a temporary placeholder for better support for 

1421 association results in `queryDatasets`. It will probably be 

1422 removed in the future, and should be avoided in production code 

1423 whenever possible. 

1424 

1425 Parameters 

1426 ---------- 

1427 datasetType : `DatasetType` or `str` 

1428 A dataset type object or the name of one. 

1429 collections : collection expression, optional 

1430 An expression that identifies the collections to search for 

1431 datasets, such as a `str` (for full matches or partial matches 

1432 via globs), `re.Pattern` (for partial matches), or iterable 

1433 thereof. ``...`` can be used to search all collections (actually 

1434 just all `~CollectionType.RUN` collections, because this will still 

1435 find all datasets). If not provided, ``self.default.collections`` 

1436 is used. See :ref:`daf_butler_collection_expressions` for more 

1437 information. 

1438 collectionTypes : `~collections.abc.Set` [ `CollectionType` ], optional 

1439 If provided, only yield associations from collections of these 

1440 types. 

1441 flattenChains : `bool`, optional 

1442 If `True`, search in the children of `~CollectionType.CHAINED` 

1443 collections. If `False`, ``CHAINED`` collections are ignored. 

1444 

1445 Yields 

1446 ------ 

1447 association : `.DatasetAssociation` 

1448 Object representing the relationship between a single dataset and 

1449 a single collection. 

1450 

1451 Raises 

1452 ------ 

1453 lsst.daf.butler.registry.NoDefaultCollectionError 

1454 Raised if ``collections`` is `None` and 

1455 ``self.defaults.collections`` is `None`. 

1456 lsst.daf.butler.registry.CollectionExpressionError 

1457 Raised when ``collections`` expression is invalid. 

1458 """ 

1459 raise NotImplementedError() 

1460 

1461 @property 

1462 def obsCoreTableManager(self) -> ObsCoreTableManager | None: 

1463 """The ObsCore manager instance for this registry 

1464 (`~.interfaces.ObsCoreTableManager` 

1465 or `None`). 

1466 

1467 ObsCore manager may not be implemented for all registry backend, or 

1468 may not be enabled for many repositories. 

1469 """ 

1470 return None 

1471 

1472 storageClasses: StorageClassFactory 

1473 """All storage classes known to the registry (`StorageClassFactory`). 

1474 """