Coverage for python/lsst/daf/butler/registry/_registry.py: 99%

113 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-11-04 09:45 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ("Registry", "CollectionArgType") 

31 

32import contextlib 

33import logging 

34import re 

35from abc import ABC, abstractmethod 

36from collections.abc import Iterable, Iterator, Mapping, Sequence 

37from types import EllipsisType 

38from typing import TYPE_CHECKING, Any, TypeAlias 

39 

40from .._dataset_association import DatasetAssociation 

41from .._dataset_ref import DatasetId, DatasetIdGenEnum, DatasetRef 

42from .._dataset_type import DatasetType 

43from .._named import NameLookupMapping 

44from .._storage_class import StorageClassFactory 

45from .._timespan import Timespan 

46from ..dimensions import ( 

47 DataCoordinate, 

48 DataId, 

49 Dimension, 

50 DimensionElement, 

51 DimensionGraph, 

52 DimensionRecord, 

53 DimensionUniverse, 

54) 

55from ._collection_summary import CollectionSummary 

56from ._collection_type import CollectionType 

57from ._defaults import RegistryDefaults 

58from .queries import DataCoordinateQueryResults, DatasetQueryResults, DimensionRecordQueryResults 

59from .wildcards import CollectionWildcard 

60 

61if TYPE_CHECKING: 

62 from .interfaces import ObsCoreTableManager 

63 

64_LOG = logging.getLogger(__name__) 

65 

66# TYpe alias for `collections` arguments. 

67CollectionArgType: TypeAlias = ( 

68 str | re.Pattern | Iterable[str | re.Pattern] | EllipsisType | CollectionWildcard 

69) 

70 

71 

72class Registry(ABC): 

73 """Abstract Registry interface. 

74 

75 All subclasses should store `~lsst.daf.butler.registry.RegistryDefaults` in 

76 a ``_defaults`` property. No other properties are assumed shared between 

77 implementations. 

78 """ 

79 

80 @abstractmethod 

81 def isWriteable(self) -> bool: 

82 """Return `True` if this registry allows write operations, and `False` 

83 otherwise. 

84 """ 

85 raise NotImplementedError() 

86 

87 @property 

88 @abstractmethod 

89 def dimensions(self) -> DimensionUniverse: 

90 """Definitions of all dimensions recognized by this `Registry` 

91 (`DimensionUniverse`). 

92 """ 

93 raise NotImplementedError() 

94 

95 @property 

96 @abstractmethod 

97 def defaults(self) -> RegistryDefaults: 

98 """Default collection search path and/or output `~CollectionType.RUN` 

99 collection (`~lsst.daf.butler.registry.RegistryDefaults`). 

100 

101 This is an immutable struct whose components may not be set 

102 individually, but the entire struct can be set by assigning to this 

103 property. 

104 """ 

105 raise NotImplementedError() 

106 

107 @defaults.setter 

108 @abstractmethod 

109 def defaults(self, value: RegistryDefaults) -> None: 

110 raise NotImplementedError() 

111 

112 @abstractmethod 

113 def refresh(self) -> None: 

114 """Refresh all in-memory state by querying the database. 

115 

116 This may be necessary to enable querying for entities added by other 

117 registry instances after this one was constructed. 

118 """ 

119 raise NotImplementedError() 

120 

121 @contextlib.contextmanager 

122 @abstractmethod 

123 def transaction(self, *, savepoint: bool = False) -> Iterator[None]: 

124 """Return a context manager that represents a transaction.""" 

125 raise NotImplementedError() 

126 

127 def resetConnectionPool(self) -> None: 

128 """Reset connection pool for registry if relevant. 

129 

130 This operation can be used reset connections to servers when 

131 using registry with fork-based multiprocessing. This method should 

132 usually be called by the child process immediately 

133 after the fork. 

134 

135 The base class implementation is a no-op. 

136 """ 

137 pass 

138 

139 @abstractmethod 

140 def registerCollection( 

141 self, name: str, type: CollectionType = CollectionType.TAGGED, doc: str | None = None 

142 ) -> bool: 

143 """Add a new collection if one with the given name does not exist. 

144 

145 Parameters 

146 ---------- 

147 name : `str` 

148 The name of the collection to create. 

149 type : `CollectionType` 

150 Enum value indicating the type of collection to create. 

151 doc : `str`, optional 

152 Documentation string for the collection. 

153 

154 Returns 

155 ------- 

156 registered : `bool` 

157 Boolean indicating whether the collection was already registered 

158 or was created by this call. 

159 

160 Notes 

161 ----- 

162 This method cannot be called within transactions, as it needs to be 

163 able to perform its own transaction to be concurrent. 

164 """ 

165 raise NotImplementedError() 

166 

167 @abstractmethod 

168 def getCollectionType(self, name: str) -> CollectionType: 

169 """Return an enumeration value indicating the type of the given 

170 collection. 

171 

172 Parameters 

173 ---------- 

174 name : `str` 

175 The name of the collection. 

176 

177 Returns 

178 ------- 

179 type : `CollectionType` 

180 Enum value indicating the type of this collection. 

181 

182 Raises 

183 ------ 

184 lsst.daf.butler.registry.MissingCollectionError 

185 Raised if no collection with the given name exists. 

186 """ 

187 raise NotImplementedError() 

188 

189 @abstractmethod 

190 def registerRun(self, name: str, doc: str | None = None) -> bool: 

191 """Add a new run if one with the given name does not exist. 

192 

193 Parameters 

194 ---------- 

195 name : `str` 

196 The name of the run to create. 

197 doc : `str`, optional 

198 Documentation string for the collection. 

199 

200 Returns 

201 ------- 

202 registered : `bool` 

203 Boolean indicating whether a new run was registered. `False` 

204 if it already existed. 

205 

206 Notes 

207 ----- 

208 This method cannot be called within transactions, as it needs to be 

209 able to perform its own transaction to be concurrent. 

210 """ 

211 raise NotImplementedError() 

212 

213 @abstractmethod 

214 def removeCollection(self, name: str) -> None: 

215 """Remove the given collection from the registry. 

216 

217 Parameters 

218 ---------- 

219 name : `str` 

220 The name of the collection to remove. 

221 

222 Raises 

223 ------ 

224 lsst.daf.butler.registry.MissingCollectionError 

225 Raised if no collection with the given name exists. 

226 sqlalchemy.exc.IntegrityError 

227 Raised if the database rows associated with the collection are 

228 still referenced by some other table, such as a dataset in a 

229 datastore (for `~CollectionType.RUN` collections only) or a 

230 `~CollectionType.CHAINED` collection of which this collection is 

231 a child. 

232 

233 Notes 

234 ----- 

235 If this is a `~CollectionType.RUN` collection, all datasets and quanta 

236 in it will removed from the `Registry` database. This requires that 

237 those datasets be removed (or at least trashed) from any datastores 

238 that hold them first. 

239 

240 A collection may not be deleted as long as it is referenced by a 

241 `~CollectionType.CHAINED` collection; the ``CHAINED`` collection must 

242 be deleted or redefined first. 

243 """ 

244 raise NotImplementedError() 

245 

246 @abstractmethod 

247 def getCollectionChain(self, parent: str) -> Sequence[str]: 

248 """Return the child collections in a `~CollectionType.CHAINED` 

249 collection. 

250 

251 Parameters 

252 ---------- 

253 parent : `str` 

254 Name of the chained collection. Must have already been added via 

255 a call to `Registry.registerCollection`. 

256 

257 Returns 

258 ------- 

259 children : `~collections.abc.Sequence` [ `str` ] 

260 An ordered sequence of collection names that are searched when the 

261 given chained collection is searched. 

262 

263 Raises 

264 ------ 

265 lsst.daf.butler.registry.MissingCollectionError 

266 Raised if ``parent`` does not exist in the `Registry`. 

267 lsst.daf.butler.registry.CollectionTypeError 

268 Raised if ``parent`` does not correspond to a 

269 `~CollectionType.CHAINED` collection. 

270 """ 

271 raise NotImplementedError() 

272 

273 @abstractmethod 

274 def setCollectionChain(self, parent: str, children: Any, *, flatten: bool = False) -> None: 

275 """Define or redefine a `~CollectionType.CHAINED` collection. 

276 

277 Parameters 

278 ---------- 

279 parent : `str` 

280 Name of the chained collection. Must have already been added via 

281 a call to `Registry.registerCollection`. 

282 children : collection expression 

283 An expression defining an ordered search of child collections, 

284 generally an iterable of `str`; see 

285 :ref:`daf_butler_collection_expressions` for more information. 

286 flatten : `bool`, optional 

287 If `True` (`False` is default), recursively flatten out any nested 

288 `~CollectionType.CHAINED` collections in ``children`` first. 

289 

290 Raises 

291 ------ 

292 lsst.daf.butler.registry.MissingCollectionError 

293 Raised when any of the given collections do not exist in the 

294 `Registry`. 

295 lsst.daf.butler.registry.CollectionTypeError 

296 Raised if ``parent`` does not correspond to a 

297 `~CollectionType.CHAINED` collection. 

298 ValueError 

299 Raised if the given collections contains a cycle. 

300 """ 

301 raise NotImplementedError() 

302 

303 @abstractmethod 

304 def getCollectionParentChains(self, collection: str) -> set[str]: 

305 """Return the CHAINED collections that directly contain the given one. 

306 

307 Parameters 

308 ---------- 

309 name : `str` 

310 Name of the collection. 

311 

312 Returns 

313 ------- 

314 chains : `set` of `str` 

315 Set of `~CollectionType.CHAINED` collection names. 

316 """ 

317 raise NotImplementedError() 

318 

319 @abstractmethod 

320 def getCollectionDocumentation(self, collection: str) -> str | None: 

321 """Retrieve the documentation string for a collection. 

322 

323 Parameters 

324 ---------- 

325 name : `str` 

326 Name of the collection. 

327 

328 Returns 

329 ------- 

330 docs : `str` or `None` 

331 Docstring for the collection with the given name. 

332 """ 

333 raise NotImplementedError() 

334 

335 @abstractmethod 

336 def setCollectionDocumentation(self, collection: str, doc: str | None) -> None: 

337 """Set the documentation string for a collection. 

338 

339 Parameters 

340 ---------- 

341 name : `str` 

342 Name of the collection. 

343 docs : `str` or `None` 

344 Docstring for the collection with the given name; will replace any 

345 existing docstring. Passing `None` will remove any existing 

346 docstring. 

347 """ 

348 raise NotImplementedError() 

349 

350 @abstractmethod 

351 def getCollectionSummary(self, collection: str) -> CollectionSummary: 

352 """Return a summary for the given collection. 

353 

354 Parameters 

355 ---------- 

356 collection : `str` 

357 Name of the collection for which a summary is to be retrieved. 

358 

359 Returns 

360 ------- 

361 summary : `~lsst.daf.butler.registry.CollectionSummary` 

362 Summary of the dataset types and governor dimension values in 

363 this collection. 

364 """ 

365 raise NotImplementedError() 

366 

367 @abstractmethod 

368 def registerDatasetType(self, datasetType: DatasetType) -> bool: 

369 """Add a new `DatasetType` to the Registry. 

370 

371 It is not an error to register the same `DatasetType` twice. 

372 

373 Parameters 

374 ---------- 

375 datasetType : `DatasetType` 

376 The `DatasetType` to be added. 

377 

378 Returns 

379 ------- 

380 inserted : `bool` 

381 `True` if ``datasetType`` was inserted, `False` if an identical 

382 existing `DatasetType` was found. Note that in either case the 

383 DatasetType is guaranteed to be defined in the Registry 

384 consistently with the given definition. 

385 

386 Raises 

387 ------ 

388 ValueError 

389 Raised if the dimensions or storage class are invalid. 

390 lsst.daf.butler.registry.ConflictingDefinitionError 

391 Raised if this `DatasetType` is already registered with a different 

392 definition. 

393 

394 Notes 

395 ----- 

396 This method cannot be called within transactions, as it needs to be 

397 able to perform its own transaction to be concurrent. 

398 """ 

399 raise NotImplementedError() 

400 

401 @abstractmethod 

402 def removeDatasetType(self, name: str | tuple[str, ...]) -> None: 

403 """Remove the named `DatasetType` from the registry. 

404 

405 .. warning:: 

406 

407 Registry implementations can cache the dataset type definitions. 

408 This means that deleting the dataset type definition may result in 

409 unexpected behavior from other butler processes that are active 

410 that have not seen the deletion. 

411 

412 Parameters 

413 ---------- 

414 name : `str` or `tuple` [`str`] 

415 Name of the type to be removed or tuple containing a list of type 

416 names to be removed. Wildcards are allowed. 

417 

418 Raises 

419 ------ 

420 lsst.daf.butler.registry.OrphanedRecordError 

421 Raised if an attempt is made to remove the dataset type definition 

422 when there are already datasets associated with it. 

423 

424 Notes 

425 ----- 

426 If the dataset type is not registered the method will return without 

427 action. 

428 """ 

429 raise NotImplementedError() 

430 

431 @abstractmethod 

432 def getDatasetType(self, name: str) -> DatasetType: 

433 """Get the `DatasetType`. 

434 

435 Parameters 

436 ---------- 

437 name : `str` 

438 Name of the type. 

439 

440 Returns 

441 ------- 

442 type : `DatasetType` 

443 The `DatasetType` associated with the given name. 

444 

445 Raises 

446 ------ 

447 lsst.daf.butler.registry.MissingDatasetTypeError 

448 Raised if the requested dataset type has not been registered. 

449 

450 Notes 

451 ----- 

452 This method handles component dataset types automatically, though most 

453 other registry operations do not. 

454 """ 

455 raise NotImplementedError() 

456 

457 @abstractmethod 

458 def supportsIdGenerationMode(self, mode: DatasetIdGenEnum) -> bool: 

459 """Test whether the given dataset ID generation mode is supported by 

460 `insertDatasets`. 

461 

462 Parameters 

463 ---------- 

464 mode : `DatasetIdGenEnum` 

465 Enum value for the mode to test. 

466 

467 Returns 

468 ------- 

469 supported : `bool` 

470 Whether the given mode is supported. 

471 """ 

472 raise NotImplementedError() 

473 

474 @abstractmethod 

475 def findDataset( 

476 self, 

477 datasetType: DatasetType | str, 

478 dataId: DataId | None = None, 

479 *, 

480 collections: CollectionArgType | None = None, 

481 timespan: Timespan | None = None, 

482 datastore_records: bool = False, 

483 **kwargs: Any, 

484 ) -> DatasetRef | None: 

485 """Find a dataset given its `DatasetType` and data ID. 

486 

487 This can be used to obtain a `DatasetRef` that permits the dataset to 

488 be read from a `Datastore`. If the dataset is a component and can not 

489 be found using the provided dataset type, a dataset ref for the parent 

490 will be returned instead but with the correct dataset type. 

491 

492 Parameters 

493 ---------- 

494 datasetType : `DatasetType` or `str` 

495 A `DatasetType` or the name of one. If this is a `DatasetType` 

496 instance, its storage class will be respected and propagated to 

497 the output, even if it differs from the dataset type definition 

498 in the registry, as long as the storage classes are convertible. 

499 dataId : `dict` or `DataCoordinate`, optional 

500 A `dict`-like object containing the `Dimension` links that identify 

501 the dataset within a collection. 

502 collections : collection expression, optional 

503 An expression that fully or partially identifies the collections to 

504 search for the dataset; see 

505 :ref:`daf_butler_collection_expressions` for more information. 

506 Defaults to ``self.defaults.collections``. 

507 timespan : `Timespan`, optional 

508 A timespan that the validity range of the dataset must overlap. 

509 If not provided, any `~CollectionType.CALIBRATION` collections 

510 matched by the ``collections`` argument will not be searched. 

511 **kwargs 

512 Additional keyword arguments passed to 

513 `DataCoordinate.standardize` to convert ``dataId`` to a true 

514 `DataCoordinate` or augment an existing one. 

515 

516 Returns 

517 ------- 

518 ref : `DatasetRef` 

519 A reference to the dataset, or `None` if no matching Dataset 

520 was found. 

521 

522 Raises 

523 ------ 

524 lsst.daf.butler.registry.NoDefaultCollectionError 

525 Raised if ``collections`` is `None` and 

526 ``self.defaults.collections`` is `None`. 

527 LookupError 

528 Raised if one or more data ID keys are missing. 

529 lsst.daf.butler.registry.MissingDatasetTypeError 

530 Raised if the dataset type does not exist. 

531 lsst.daf.butler.registry.MissingCollectionError 

532 Raised if any of ``collections`` does not exist in the registry. 

533 

534 Notes 

535 ----- 

536 This method simply returns `None` and does not raise an exception even 

537 when the set of collections searched is intrinsically incompatible with 

538 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but 

539 only `~CollectionType.CALIBRATION` collections are being searched. 

540 This may make it harder to debug some lookup failures, but the behavior 

541 is intentional; we consider it more important that failed searches are 

542 reported consistently, regardless of the reason, and that adding 

543 additional collections that do not contain a match to the search path 

544 never changes the behavior. 

545 

546 This method handles component dataset types automatically, though most 

547 other registry operations do not. 

548 """ 

549 raise NotImplementedError() 

550 

551 @abstractmethod 

552 def insertDatasets( 

553 self, 

554 datasetType: DatasetType | str, 

555 dataIds: Iterable[DataId], 

556 run: str | None = None, 

557 expand: bool = True, 

558 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE, 

559 ) -> list[DatasetRef]: 

560 """Insert one or more datasets into the `Registry`. 

561 

562 This always adds new datasets; to associate existing datasets with 

563 a new collection, use ``associate``. 

564 

565 Parameters 

566 ---------- 

567 datasetType : `DatasetType` or `str` 

568 A `DatasetType` or the name of one. 

569 dataIds : `~collections.abc.Iterable` of `dict` or `DataCoordinate` 

570 Dimension-based identifiers for the new datasets. 

571 run : `str`, optional 

572 The name of the run that produced the datasets. Defaults to 

573 ``self.defaults.run``. 

574 expand : `bool`, optional 

575 If `True` (default), expand data IDs as they are inserted. This is 

576 necessary in general to allow datastore to generate file templates, 

577 but it may be disabled if the caller can guarantee this is 

578 unnecessary. 

579 idGenerationMode : `DatasetIdGenEnum`, optional 

580 Specifies option for generating dataset IDs. By default unique IDs 

581 are generated for each inserted dataset. 

582 

583 Returns 

584 ------- 

585 refs : `list` of `DatasetRef` 

586 Resolved `DatasetRef` instances for all given data IDs (in the same 

587 order). 

588 

589 Raises 

590 ------ 

591 lsst.daf.butler.registry.DatasetTypeError 

592 Raised if ``datasetType`` is not known to registry. 

593 lsst.daf.butler.registry.CollectionTypeError 

594 Raised if ``run`` collection type is not `~CollectionType.RUN`. 

595 lsst.daf.butler.registry.NoDefaultCollectionError 

596 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

597 lsst.daf.butler.registry.ConflictingDefinitionError 

598 If a dataset with the same dataset type and data ID as one of those 

599 given already exists in ``run``. 

600 lsst.daf.butler.registry.MissingCollectionError 

601 Raised if ``run`` does not exist in the registry. 

602 """ 

603 raise NotImplementedError() 

604 

605 @abstractmethod 

606 def _importDatasets( 

607 self, 

608 datasets: Iterable[DatasetRef], 

609 expand: bool = True, 

610 ) -> list[DatasetRef]: 

611 """Import one or more datasets into the `Registry`. 

612 

613 Difference from `insertDatasets` method is that this method accepts 

614 `DatasetRef` instances which should already be resolved and have a 

615 dataset ID. If registry supports globally-unique dataset IDs (e.g. 

616 `uuid.UUID`) then datasets which already exist in the registry will be 

617 ignored if imported again. 

618 

619 Parameters 

620 ---------- 

621 datasets : `~collections.abc.Iterable` of `DatasetRef` 

622 Datasets to be inserted. All `DatasetRef` instances must have 

623 identical ``datasetType`` and ``run`` attributes. ``run`` 

624 attribute can be `None` and defaults to ``self.defaults.run``. 

625 Datasets can specify ``id`` attribute which will be used for 

626 inserted datasets. All dataset IDs must have the same type 

627 (`int` or `uuid.UUID`), if type of dataset IDs does not match 

628 configured backend then IDs will be ignored and new IDs will be 

629 generated by backend. 

630 expand : `bool`, optional 

631 If `True` (default), expand data IDs as they are inserted. This is 

632 necessary in general, but it may be disabled if the caller can 

633 guarantee this is unnecessary. 

634 

635 Returns 

636 ------- 

637 refs : `list` of `DatasetRef` 

638 Resolved `DatasetRef` instances for all given data IDs (in the same 

639 order). If any of ``datasets`` has an ID which already exists in 

640 the database then it will not be inserted or updated, but a 

641 resolved `DatasetRef` will be returned for it in any case. 

642 

643 Raises 

644 ------ 

645 lsst.daf.butler.registry.NoDefaultCollectionError 

646 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

647 lsst.daf.butler.registry.DatasetTypeError 

648 Raised if datasets correspond to more than one dataset type or 

649 dataset type is not known to registry. 

650 lsst.daf.butler.registry.ConflictingDefinitionError 

651 If a dataset with the same dataset type and data ID as one of those 

652 given already exists in ``run``. 

653 lsst.daf.butler.registry.MissingCollectionError 

654 Raised if ``run`` does not exist in the registry. 

655 

656 Notes 

657 ----- 

658 This method is considered package-private and internal to Butler 

659 implementation. Clients outside daf_butler package should not use this 

660 method. 

661 """ 

662 raise NotImplementedError() 

663 

664 @abstractmethod 

665 def getDataset(self, id: DatasetId) -> DatasetRef | None: 

666 """Retrieve a Dataset entry. 

667 

668 Parameters 

669 ---------- 

670 id : `DatasetId` 

671 The unique identifier for the dataset. 

672 

673 Returns 

674 ------- 

675 ref : `DatasetRef` or `None` 

676 A ref to the Dataset, or `None` if no matching Dataset 

677 was found. 

678 """ 

679 raise NotImplementedError() 

680 

681 @abstractmethod 

682 def removeDatasets(self, refs: Iterable[DatasetRef]) -> None: 

683 """Remove datasets from the Registry. 

684 

685 The datasets will be removed unconditionally from all collections, and 

686 any `Quantum` that consumed this dataset will instead be marked with 

687 having a NULL input. `Datastore` records will *not* be deleted; the 

688 caller is responsible for ensuring that the dataset has already been 

689 removed from all Datastores. 

690 

691 Parameters 

692 ---------- 

693 refs : `~collections.abc.Iterable` [`DatasetRef`] 

694 References to the datasets to be removed. Must include a valid 

695 ``id`` attribute, and should be considered invalidated upon return. 

696 

697 Raises 

698 ------ 

699 lsst.daf.butler.AmbiguousDatasetError 

700 Raised if any ``ref.id`` is `None`. 

701 lsst.daf.butler.registry.OrphanedRecordError 

702 Raised if any dataset is still present in any `Datastore`. 

703 """ 

704 raise NotImplementedError() 

705 

706 @abstractmethod 

707 def associate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

708 """Add existing datasets to a `~CollectionType.TAGGED` collection. 

709 

710 If a DatasetRef with the same exact ID is already in a collection 

711 nothing is changed. If a `DatasetRef` with the same `DatasetType` and 

712 data ID but with different ID exists in the collection, 

713 `~lsst.daf.butler.registry.ConflictingDefinitionError` is raised. 

714 

715 Parameters 

716 ---------- 

717 collection : `str` 

718 Indicates the collection the datasets should be associated with. 

719 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

720 An iterable of resolved `DatasetRef` instances that already exist 

721 in this `Registry`. 

722 

723 Raises 

724 ------ 

725 lsst.daf.butler.registry.ConflictingDefinitionError 

726 If a Dataset with the given `DatasetRef` already exists in the 

727 given collection. 

728 lsst.daf.butler.registry.MissingCollectionError 

729 Raised if ``collection`` does not exist in the registry. 

730 lsst.daf.butler.registry.CollectionTypeError 

731 Raise adding new datasets to the given ``collection`` is not 

732 allowed. 

733 """ 

734 raise NotImplementedError() 

735 

736 @abstractmethod 

737 def disassociate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

738 """Remove existing datasets from a `~CollectionType.TAGGED` collection. 

739 

740 ``collection`` and ``ref`` combinations that are not currently 

741 associated are silently ignored. 

742 

743 Parameters 

744 ---------- 

745 collection : `str` 

746 The collection the datasets should no longer be associated with. 

747 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

748 An iterable of resolved `DatasetRef` instances that already exist 

749 in this `Registry`. 

750 

751 Raises 

752 ------ 

753 lsst.daf.butler.AmbiguousDatasetError 

754 Raised if any of the given dataset references is unresolved. 

755 lsst.daf.butler.registry.MissingCollectionError 

756 Raised if ``collection`` does not exist in the registry. 

757 lsst.daf.butler.registry.CollectionTypeError 

758 Raise adding new datasets to the given ``collection`` is not 

759 allowed. 

760 """ 

761 raise NotImplementedError() 

762 

763 @abstractmethod 

764 def certify(self, collection: str, refs: Iterable[DatasetRef], timespan: Timespan) -> None: 

765 """Associate one or more datasets with a calibration collection and a 

766 validity range within it. 

767 

768 Parameters 

769 ---------- 

770 collection : `str` 

771 The name of an already-registered `~CollectionType.CALIBRATION` 

772 collection. 

773 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

774 Datasets to be associated. 

775 timespan : `Timespan` 

776 The validity range for these datasets within the collection. 

777 

778 Raises 

779 ------ 

780 lsst.daf.butler.AmbiguousDatasetError 

781 Raised if any of the given `DatasetRef` instances is unresolved. 

782 lsst.daf.butler.registry.ConflictingDefinitionError 

783 Raised if the collection already contains a different dataset with 

784 the same `DatasetType` and data ID and an overlapping validity 

785 range. 

786 lsst.daf.butler.registry.CollectionTypeError 

787 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

788 collection or if one or more datasets are of a dataset type for 

789 which `DatasetType.isCalibration` returns `False`. 

790 """ 

791 raise NotImplementedError() 

792 

793 @abstractmethod 

794 def decertify( 

795 self, 

796 collection: str, 

797 datasetType: str | DatasetType, 

798 timespan: Timespan, 

799 *, 

800 dataIds: Iterable[DataId] | None = None, 

801 ) -> None: 

802 """Remove or adjust datasets to clear a validity range within a 

803 calibration collection. 

804 

805 Parameters 

806 ---------- 

807 collection : `str` 

808 The name of an already-registered `~CollectionType.CALIBRATION` 

809 collection. 

810 datasetType : `str` or `DatasetType` 

811 Name or `DatasetType` instance for the datasets to be decertified. 

812 timespan : `Timespan`, optional 

813 The validity range to remove datasets from within the collection. 

814 Datasets that overlap this range but are not contained by it will 

815 have their validity ranges adjusted to not overlap it, which may 

816 split a single dataset validity range into two. 

817 dataIds : iterable [`dict` or `DataCoordinate`], optional 

818 Data IDs that should be decertified within the given validity range 

819 If `None`, all data IDs for ``self.datasetType`` will be 

820 decertified. 

821 

822 Raises 

823 ------ 

824 lsst.daf.butler.registry.CollectionTypeError 

825 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

826 collection or if ``datasetType.isCalibration() is False``. 

827 """ 

828 raise NotImplementedError() 

829 

830 @abstractmethod 

831 def getDatasetLocations(self, ref: DatasetRef) -> Iterable[str]: 

832 """Retrieve datastore locations for a given dataset. 

833 

834 Parameters 

835 ---------- 

836 ref : `DatasetRef` 

837 A reference to the dataset for which to retrieve storage 

838 information. 

839 

840 Returns 

841 ------- 

842 datastores : `~collections.abc.Iterable` [ `str` ] 

843 All the matching datastores holding this dataset. 

844 

845 Raises 

846 ------ 

847 lsst.daf.butler.AmbiguousDatasetError 

848 Raised if ``ref.id`` is `None`. 

849 """ 

850 raise NotImplementedError() 

851 

852 @abstractmethod 

853 def expandDataId( 

854 self, 

855 dataId: DataId | None = None, 

856 *, 

857 graph: DimensionGraph | None = None, 

858 records: NameLookupMapping[DimensionElement, DimensionRecord | None] | None = None, 

859 withDefaults: bool = True, 

860 **kwargs: Any, 

861 ) -> DataCoordinate: 

862 """Expand a dimension-based data ID to include additional information. 

863 

864 Parameters 

865 ---------- 

866 dataId : `DataCoordinate` or `dict`, optional 

867 Data ID to be expanded; augmented and overridden by ``kwargs``. 

868 graph : `DimensionGraph`, optional 

869 Set of dimensions for the expanded ID. If `None`, the dimensions 

870 will be inferred from the keys of ``dataId`` and ``kwargs``. 

871 Dimensions that are in ``dataId`` or ``kwargs`` but not in 

872 ``graph`` are silently ignored, providing a way to extract and 

873 ``graph`` expand a subset of a data ID. 

874 records : `~collections.abc.Mapping` [`str`, `DimensionRecord`], \ 

875 optional 

876 Dimension record data to use before querying the database for that 

877 data, keyed by element name. 

878 withDefaults : `bool`, optional 

879 Utilize ``self.defaults.dataId`` to fill in missing governor 

880 dimension key-value pairs. Defaults to `True` (i.e. defaults are 

881 used). 

882 **kwargs 

883 Additional keywords are treated like additional key-value pairs for 

884 ``dataId``, extending and overriding 

885 

886 Returns 

887 ------- 

888 expanded : `DataCoordinate` 

889 A data ID that includes full metadata for all of the dimensions it 

890 identifies, i.e. guarantees that ``expanded.hasRecords()`` and 

891 ``expanded.hasFull()`` both return `True`. 

892 

893 Raises 

894 ------ 

895 lsst.daf.butler.registry.DataIdError 

896 Raised when ``dataId`` or keyword arguments specify unknown 

897 dimensions or values, or when a resulting data ID contains 

898 contradictory key-value pairs, according to dimension 

899 relationships. 

900 

901 Notes 

902 ----- 

903 This method cannot be relied upon to reject invalid data ID values 

904 for dimensions that do actually not have any record columns. For 

905 efficiency reasons the records for these dimensions (which have only 

906 dimension key values that are given by the caller) may be constructed 

907 directly rather than obtained from the registry database. 

908 """ 

909 raise NotImplementedError() 

910 

911 @abstractmethod 

912 def insertDimensionData( 

913 self, 

914 element: DimensionElement | str, 

915 *data: Mapping[str, Any] | DimensionRecord, 

916 conform: bool = True, 

917 replace: bool = False, 

918 skip_existing: bool = False, 

919 ) -> None: 

920 """Insert one or more dimension records into the database. 

921 

922 Parameters 

923 ---------- 

924 element : `DimensionElement` or `str` 

925 The `DimensionElement` or name thereof that identifies the table 

926 records will be inserted into. 

927 *data : `dict` or `DimensionRecord` 

928 One or more records to insert. 

929 conform : `bool`, optional 

930 If `False` (`True` is default) perform no checking or conversions, 

931 and assume that ``element`` is a `DimensionElement` instance and 

932 ``data`` is a one or more `DimensionRecord` instances of the 

933 appropriate subclass. 

934 replace : `bool`, optional 

935 If `True` (`False` is default), replace existing records in the 

936 database if there is a conflict. 

937 skip_existing : `bool`, optional 

938 If `True` (`False` is default), skip insertion if a record with 

939 the same primary key values already exists. Unlike 

940 `syncDimensionData`, this will not detect when the given record 

941 differs from what is in the database, and should not be used when 

942 this is a concern. 

943 """ 

944 raise NotImplementedError() 

945 

946 @abstractmethod 

947 def syncDimensionData( 

948 self, 

949 element: DimensionElement | str, 

950 row: Mapping[str, Any] | DimensionRecord, 

951 conform: bool = True, 

952 update: bool = False, 

953 ) -> bool | dict[str, Any]: 

954 """Synchronize the given dimension record with the database, inserting 

955 if it does not already exist and comparing values if it does. 

956 

957 Parameters 

958 ---------- 

959 element : `DimensionElement` or `str` 

960 The `DimensionElement` or name thereof that identifies the table 

961 records will be inserted into. 

962 row : `dict` or `DimensionRecord` 

963 The record to insert. 

964 conform : `bool`, optional 

965 If `False` (`True` is default) perform no checking or conversions, 

966 and assume that ``element`` is a `DimensionElement` instance and 

967 ``data`` is a one or more `DimensionRecord` instances of the 

968 appropriate subclass. 

969 update : `bool`, optional 

970 If `True` (`False` is default), update the existing record in the 

971 database if there is a conflict. 

972 

973 Returns 

974 ------- 

975 inserted_or_updated : `bool` or `dict` 

976 `True` if a new row was inserted, `False` if no changes were 

977 needed, or a `dict` mapping updated column names to their old 

978 values if an update was performed (only possible if 

979 ``update=True``). 

980 

981 Raises 

982 ------ 

983 lsst.daf.butler.registry.ConflictingDefinitionError 

984 Raised if the record exists in the database (according to primary 

985 key lookup) but is inconsistent with the given one. 

986 """ 

987 raise NotImplementedError() 

988 

989 @abstractmethod 

990 def queryDatasetTypes( 

991 self, 

992 expression: Any = ..., 

993 *, 

994 components: bool | None = False, 

995 missing: list[str] | None = None, 

996 ) -> Iterable[DatasetType]: 

997 """Iterate over the dataset types whose names match an expression. 

998 

999 Parameters 

1000 ---------- 

1001 expression : dataset type expression, optional 

1002 An expression that fully or partially identifies the dataset types 

1003 to return, such as a `str`, `re.Pattern`, or iterable thereof. 

1004 ``...`` can be used to return all dataset types, and is the 

1005 default. See :ref:`daf_butler_dataset_type_expressions` for more 

1006 information. 

1007 components : `bool`, optional 

1008 If `True`, apply all expression patterns to component dataset type 

1009 names as well. If `False`, never apply patterns to components. 

1010 If `None`, apply patterns to components only if their 

1011 parent datasets were not matched by the expression. 

1012 Fully-specified component datasets (`str` or `DatasetType` 

1013 instances) are always included. 

1014 

1015 Values other than `False` are deprecated, and only `False` will be 

1016 supported after v26. After v27 this argument will be removed 

1017 entirely. 

1018 missing : `list` of `str`, optional 

1019 String dataset type names that were explicitly given (i.e. not 

1020 regular expression patterns) but not found will be appended to this 

1021 list, if it is provided. 

1022 

1023 Returns 

1024 ------- 

1025 dataset_types : `~collections.abc.Iterable` [ `DatasetType`] 

1026 An `~collections.abc.Iterable` of `DatasetType` instances whose 

1027 names match ``expression``. 

1028 

1029 Raises 

1030 ------ 

1031 lsst.daf.butler.registry.DatasetTypeExpressionError 

1032 Raised when ``expression`` is invalid. 

1033 """ 

1034 raise NotImplementedError() 

1035 

1036 @abstractmethod 

1037 def queryCollections( 

1038 self, 

1039 expression: Any = ..., 

1040 datasetType: DatasetType | None = None, 

1041 collectionTypes: Iterable[CollectionType] | CollectionType = CollectionType.all(), 

1042 flattenChains: bool = False, 

1043 includeChains: bool | None = None, 

1044 ) -> Sequence[str]: 

1045 """Iterate over the collections whose names match an expression. 

1046 

1047 Parameters 

1048 ---------- 

1049 expression : collection expression, optional 

1050 An expression that identifies the collections to return, such as 

1051 a `str` (for full matches or partial matches via globs), 

1052 `re.Pattern` (for partial matches), or iterable thereof. ``...`` 

1053 can be used to return all collections, and is the default. 

1054 See :ref:`daf_butler_collection_expressions` for more information. 

1055 datasetType : `DatasetType`, optional 

1056 If provided, only yield collections that may contain datasets of 

1057 this type. This is a conservative approximation in general; it may 

1058 yield collections that do not have any such datasets. 

1059 collectionTypes : `~collections.abc.Set` [`CollectionType`] or \ 

1060 `CollectionType`, optional 

1061 If provided, only yield collections of these types. 

1062 flattenChains : `bool`, optional 

1063 If `True` (`False` is default), recursively yield the child 

1064 collections of matching `~CollectionType.CHAINED` collections. 

1065 includeChains : `bool`, optional 

1066 If `True`, yield records for matching `~CollectionType.CHAINED` 

1067 collections. Default is the opposite of ``flattenChains``: include 

1068 either CHAINED collections or their children, but not both. 

1069 

1070 Returns 

1071 ------- 

1072 collections : `~collections.abc.Sequence` [ `str` ] 

1073 The names of collections that match ``expression``. 

1074 

1075 Raises 

1076 ------ 

1077 lsst.daf.butler.registry.CollectionExpressionError 

1078 Raised when ``expression`` is invalid. 

1079 

1080 Notes 

1081 ----- 

1082 The order in which collections are returned is unspecified, except that 

1083 the children of a `~CollectionType.CHAINED` collection are guaranteed 

1084 to be in the order in which they are searched. When multiple parent 

1085 `~CollectionType.CHAINED` collections match the same criteria, the 

1086 order in which the two lists appear is unspecified, and the lists of 

1087 children may be incomplete if a child has multiple parents. 

1088 """ 

1089 raise NotImplementedError() 

1090 

1091 @abstractmethod 

1092 def queryDatasets( 

1093 self, 

1094 datasetType: Any, 

1095 *, 

1096 collections: CollectionArgType | None = None, 

1097 dimensions: Iterable[Dimension | str] | None = None, 

1098 dataId: DataId | None = None, 

1099 where: str = "", 

1100 findFirst: bool = False, 

1101 components: bool | None = False, 

1102 bind: Mapping[str, Any] | None = None, 

1103 check: bool = True, 

1104 **kwargs: Any, 

1105 ) -> DatasetQueryResults: 

1106 """Query for and iterate over dataset references matching user-provided 

1107 criteria. 

1108 

1109 Parameters 

1110 ---------- 

1111 datasetType : dataset type expression 

1112 An expression that fully or partially identifies the dataset types 

1113 to be queried. Allowed types include `DatasetType`, `str`, 

1114 `re.Pattern`, and iterables thereof. The special value ``...`` can 

1115 be used to query all dataset types. See 

1116 :ref:`daf_butler_dataset_type_expressions` for more information. 

1117 collections : collection expression, optional 

1118 An expression that identifies the collections to search, such as a 

1119 `str` (for full matches or partial matches via globs), `re.Pattern` 

1120 (for partial matches), or iterable thereof. ``...`` can be used to 

1121 search all collections (actually just all `~CollectionType.RUN` 

1122 collections, because this will still find all datasets). 

1123 If not provided, ``self.default.collections`` is used. See 

1124 :ref:`daf_butler_collection_expressions` for more information. 

1125 dimensions : `~collections.abc.Iterable` of `Dimension` or `str` 

1126 Dimensions to include in the query (in addition to those used 

1127 to identify the queried dataset type(s)), either to constrain 

1128 the resulting datasets to those for which a matching dimension 

1129 exists, or to relate the dataset type's dimensions to dimensions 

1130 referenced by the ``dataId`` or ``where`` arguments. 

1131 dataId : `dict` or `DataCoordinate`, optional 

1132 A data ID whose key-value pairs are used as equality constraints 

1133 in the query. 

1134 where : `str`, optional 

1135 A string expression similar to a SQL WHERE clause. May involve 

1136 any column of a dimension table or (as a shortcut for the primary 

1137 key column of a dimension table) dimension name. See 

1138 :ref:`daf_butler_dimension_expressions` for more information. 

1139 findFirst : `bool`, optional 

1140 If `True` (`False` is default), for each result data ID, only 

1141 yield one `DatasetRef` of each `DatasetType`, from the first 

1142 collection in which a dataset of that dataset type appears 

1143 (according to the order of ``collections`` passed in). If `True`, 

1144 ``collections`` must not contain regular expressions and may not 

1145 be ``...``. 

1146 components : `bool`, optional 

1147 If `True`, apply all dataset expression patterns to component 

1148 dataset type names as well. If `False`, never apply patterns to 

1149 components. If `None`, apply patterns to components only 

1150 if their parent datasets were not matched by the expression. 

1151 Fully-specified component datasets (`str` or `DatasetType` 

1152 instances) are always included. 

1153 

1154 Values other than `False` are deprecated, and only `False` will be 

1155 supported after v26. After v27 this argument will be removed 

1156 entirely. 

1157 bind : `~collections.abc.Mapping`, optional 

1158 Mapping containing literal values that should be injected into the 

1159 ``where`` expression, keyed by the identifiers they replace. 

1160 Values of collection type can be expanded in some cases; see 

1161 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1162 information. 

1163 check : `bool`, optional 

1164 If `True` (default) check the query for consistency before 

1165 executing it. This may reject some valid queries that resemble 

1166 common mistakes (e.g. queries for visits without specifying an 

1167 instrument). 

1168 **kwargs 

1169 Additional keyword arguments are forwarded to 

1170 `DataCoordinate.standardize` when processing the ``dataId`` 

1171 argument (and may be used to provide a constraining data ID even 

1172 when the ``dataId`` argument is `None`). 

1173 

1174 Returns 

1175 ------- 

1176 refs : `.queries.DatasetQueryResults` 

1177 Dataset references matching the given query criteria. Nested data 

1178 IDs are guaranteed to include values for all implied dimensions 

1179 (i.e. `DataCoordinate.hasFull` will return `True`), but will not 

1180 include dimension records (`DataCoordinate.hasRecords` will be 

1181 `False`) unless `~.queries.DatasetQueryResults.expanded` is 

1182 called on the result object (which returns a new one). 

1183 

1184 Raises 

1185 ------ 

1186 lsst.daf.butler.registry.DatasetTypeExpressionError 

1187 Raised when ``datasetType`` expression is invalid. 

1188 TypeError 

1189 Raised when the arguments are incompatible, such as when a 

1190 collection wildcard is passed when ``findFirst`` is `True`, or 

1191 when ``collections`` is `None` and ``self.defaults.collections`` is 

1192 also `None`. 

1193 lsst.daf.butler.registry.DataIdError 

1194 Raised when ``dataId`` or keyword arguments specify unknown 

1195 dimensions or values, or when they contain inconsistent values. 

1196 lsst.daf.butler.registry.UserExpressionError 

1197 Raised when ``where`` expression is invalid. 

1198 

1199 Notes 

1200 ----- 

1201 When multiple dataset types are queried in a single call, the 

1202 results of this operation are equivalent to querying for each dataset 

1203 type separately in turn, and no information about the relationships 

1204 between datasets of different types is included. In contexts where 

1205 that kind of information is important, the recommended pattern is to 

1206 use `queryDataIds` to first obtain data IDs (possibly with the 

1207 desired dataset types and collections passed as constraints to the 

1208 query), and then use multiple (generally much simpler) calls to 

1209 `queryDatasets` with the returned data IDs passed as constraints. 

1210 """ 

1211 raise NotImplementedError() 

1212 

1213 @abstractmethod 

1214 def queryDataIds( 

1215 self, 

1216 dimensions: Iterable[Dimension | str] | Dimension | str, 

1217 *, 

1218 dataId: DataId | None = None, 

1219 datasets: Any = None, 

1220 collections: CollectionArgType | None = None, 

1221 where: str = "", 

1222 components: bool | None = False, 

1223 bind: Mapping[str, Any] | None = None, 

1224 check: bool = True, 

1225 **kwargs: Any, 

1226 ) -> DataCoordinateQueryResults: 

1227 """Query for data IDs matching user-provided criteria. 

1228 

1229 Parameters 

1230 ---------- 

1231 dimensions : `Dimension` or `str`, or iterable thereof 

1232 The dimensions of the data IDs to yield, as either `Dimension` 

1233 instances or `str`. Will be automatically expanded to a complete 

1234 `DimensionGraph`. 

1235 dataId : `dict` or `DataCoordinate`, optional 

1236 A data ID whose key-value pairs are used as equality constraints 

1237 in the query. 

1238 datasets : dataset type expression, optional 

1239 An expression that fully or partially identifies dataset types 

1240 that should constrain the yielded data IDs. For example, including 

1241 "raw" here would constrain the yielded ``instrument``, 

1242 ``exposure``, ``detector``, and ``physical_filter`` values to only 

1243 those for which at least one "raw" dataset exists in 

1244 ``collections``. Allowed types include `DatasetType`, `str`, 

1245 and iterables thereof. Regular expression objects (i.e. 

1246 `re.Pattern`) are deprecated and will be removed after the v26 

1247 release. See :ref:`daf_butler_dataset_type_expressions` for more 

1248 information. 

1249 collections : collection expression, optional 

1250 An expression that identifies the collections to search for 

1251 datasets, such as a `str` (for full matches or partial matches 

1252 via globs), `re.Pattern` (for partial matches), or iterable 

1253 thereof. ``...`` can be used to search all collections (actually 

1254 just all `~CollectionType.RUN` collections, because this will 

1255 still find all datasets). If not provided, 

1256 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1257 is also passed. See :ref:`daf_butler_collection_expressions` for 

1258 more information. 

1259 where : `str`, optional 

1260 A string expression similar to a SQL WHERE clause. May involve 

1261 any column of a dimension table or (as a shortcut for the primary 

1262 key column of a dimension table) dimension name. See 

1263 :ref:`daf_butler_dimension_expressions` for more information. 

1264 components : `bool`, optional 

1265 If `True`, apply all dataset expression patterns to component 

1266 dataset type names as well. If `False`, never apply patterns to 

1267 components. If `None`, apply patterns to components only 

1268 if their parent datasets were not matched by the expression. 

1269 Fully-specified component datasets (`str` or `DatasetType` 

1270 instances) are always included. 

1271 

1272 Values other than `False` are deprecated, and only `False` will be 

1273 supported after v26. After v27 this argument will be removed 

1274 entirely. 

1275 bind : `~collections.abc.Mapping`, optional 

1276 Mapping containing literal values that should be injected into the 

1277 ``where`` expression, keyed by the identifiers they replace. 

1278 Values of collection type can be expanded in some cases; see 

1279 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1280 information. 

1281 check : `bool`, optional 

1282 If `True` (default) check the query for consistency before 

1283 executing it. This may reject some valid queries that resemble 

1284 common mistakes (e.g. queries for visits without specifying an 

1285 instrument). 

1286 **kwargs 

1287 Additional keyword arguments are forwarded to 

1288 `DataCoordinate.standardize` when processing the ``dataId`` 

1289 argument (and may be used to provide a constraining data ID even 

1290 when the ``dataId`` argument is `None`). 

1291 

1292 Returns 

1293 ------- 

1294 dataIds : `.queries.DataCoordinateQueryResults` 

1295 Data IDs matching the given query parameters. These are guaranteed 

1296 to identify all dimensions (`DataCoordinate.hasFull` returns 

1297 `True`), but will not contain `DimensionRecord` objects 

1298 (`DataCoordinate.hasRecords` returns `False`). Call 

1299 `~.queries.DataCoordinateQueryResults.expanded` on the 

1300 returned object to fetch those (and consider using 

1301 `~.queries.DataCoordinateQueryResults.materialize` on the 

1302 returned object first if the expected number of rows is very 

1303 large). See documentation for those methods for additional 

1304 information. 

1305 

1306 Raises 

1307 ------ 

1308 lsst.daf.butler.registry.NoDefaultCollectionError 

1309 Raised if ``collections`` is `None` and 

1310 ``self.defaults.collections`` is `None`. 

1311 lsst.daf.butler.registry.CollectionExpressionError 

1312 Raised when ``collections`` expression is invalid. 

1313 lsst.daf.butler.registry.DataIdError 

1314 Raised when ``dataId`` or keyword arguments specify unknown 

1315 dimensions or values, or when they contain inconsistent values. 

1316 lsst.daf.butler.registry.DatasetTypeExpressionError 

1317 Raised when ``datasetType`` expression is invalid. 

1318 lsst.daf.butler.registry.UserExpressionError 

1319 Raised when ``where`` expression is invalid. 

1320 """ 

1321 raise NotImplementedError() 

1322 

1323 @abstractmethod 

1324 def queryDimensionRecords( 

1325 self, 

1326 element: DimensionElement | str, 

1327 *, 

1328 dataId: DataId | None = None, 

1329 datasets: Any = None, 

1330 collections: CollectionArgType | None = None, 

1331 where: str = "", 

1332 components: bool | None = False, 

1333 bind: Mapping[str, Any] | None = None, 

1334 check: bool = True, 

1335 **kwargs: Any, 

1336 ) -> DimensionRecordQueryResults: 

1337 """Query for dimension information matching user-provided criteria. 

1338 

1339 Parameters 

1340 ---------- 

1341 element : `DimensionElement` or `str` 

1342 The dimension element to obtain records for. 

1343 dataId : `dict` or `DataCoordinate`, optional 

1344 A data ID whose key-value pairs are used as equality constraints 

1345 in the query. 

1346 datasets : dataset type expression, optional 

1347 An expression that fully or partially identifies dataset types 

1348 that should constrain the yielded records. See `queryDataIds` and 

1349 :ref:`daf_butler_dataset_type_expressions` for more information. 

1350 collections : collection expression, optional 

1351 An expression that identifies the collections to search for 

1352 datasets, such as a `str` (for full matches or partial matches 

1353 via globs), `re.Pattern` (for partial matches), or iterable 

1354 thereof. ``...`` can be used to search all collections (actually 

1355 just all `~CollectionType.RUN` collections, because this will 

1356 still find all datasets). If not provided, 

1357 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1358 is also passed. See :ref:`daf_butler_collection_expressions` for 

1359 more information. 

1360 where : `str`, optional 

1361 A string expression similar to a SQL WHERE clause. See 

1362 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more 

1363 information. 

1364 components : `bool`, optional 

1365 Whether to apply dataset expressions to components as well. 

1366 See `queryDataIds` for more information. 

1367 

1368 Values other than `False` are deprecated, and only `False` will be 

1369 supported after v26. After v27 this argument will be removed 

1370 entirely. 

1371 bind : `~collections.abc.Mapping`, optional 

1372 Mapping containing literal values that should be injected into the 

1373 ``where`` expression, keyed by the identifiers they replace. 

1374 Values of collection type can be expanded in some cases; see 

1375 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1376 information. 

1377 check : `bool`, optional 

1378 If `True` (default) check the query for consistency before 

1379 executing it. This may reject some valid queries that resemble 

1380 common mistakes (e.g. queries for visits without specifying an 

1381 instrument). 

1382 **kwargs 

1383 Additional keyword arguments are forwarded to 

1384 `DataCoordinate.standardize` when processing the ``dataId`` 

1385 argument (and may be used to provide a constraining data ID even 

1386 when the ``dataId`` argument is `None`). 

1387 

1388 Returns 

1389 ------- 

1390 dataIds : `.queries.DimensionRecordQueryResults` 

1391 Data IDs matching the given query parameters. 

1392 

1393 Raises 

1394 ------ 

1395 lsst.daf.butler.registry.NoDefaultCollectionError 

1396 Raised if ``collections`` is `None` and 

1397 ``self.defaults.collections`` is `None`. 

1398 lsst.daf.butler.registry.CollectionExpressionError 

1399 Raised when ``collections`` expression is invalid. 

1400 lsst.daf.butler.registry.DataIdError 

1401 Raised when ``dataId`` or keyword arguments specify unknown 

1402 dimensions or values, or when they contain inconsistent values. 

1403 lsst.daf.butler.registry.DatasetTypeExpressionError 

1404 Raised when ``datasetType`` expression is invalid. 

1405 lsst.daf.butler.registry.UserExpressionError 

1406 Raised when ``where`` expression is invalid. 

1407 """ 

1408 raise NotImplementedError() 

1409 

1410 @abstractmethod 

1411 def queryDatasetAssociations( 

1412 self, 

1413 datasetType: str | DatasetType, 

1414 collections: CollectionArgType | None = ..., 

1415 *, 

1416 collectionTypes: Iterable[CollectionType] = CollectionType.all(), 

1417 flattenChains: bool = False, 

1418 ) -> Iterator[DatasetAssociation]: 

1419 """Iterate over dataset-collection combinations where the dataset is in 

1420 the collection. 

1421 

1422 This method is a temporary placeholder for better support for 

1423 association results in `queryDatasets`. It will probably be 

1424 removed in the future, and should be avoided in production code 

1425 whenever possible. 

1426 

1427 Parameters 

1428 ---------- 

1429 datasetType : `DatasetType` or `str` 

1430 A dataset type object or the name of one. 

1431 collections : collection expression, optional 

1432 An expression that identifies the collections to search for 

1433 datasets, such as a `str` (for full matches or partial matches 

1434 via globs), `re.Pattern` (for partial matches), or iterable 

1435 thereof. ``...`` can be used to search all collections (actually 

1436 just all `~CollectionType.RUN` collections, because this will still 

1437 find all datasets). If not provided, ``self.default.collections`` 

1438 is used. See :ref:`daf_butler_collection_expressions` for more 

1439 information. 

1440 collectionTypes : `~collections.abc.Set` [ `CollectionType` ], optional 

1441 If provided, only yield associations from collections of these 

1442 types. 

1443 flattenChains : `bool`, optional 

1444 If `True`, search in the children of `~CollectionType.CHAINED` 

1445 collections. If `False`, ``CHAINED`` collections are ignored. 

1446 

1447 Yields 

1448 ------ 

1449 association : `.DatasetAssociation` 

1450 Object representing the relationship between a single dataset and 

1451 a single collection. 

1452 

1453 Raises 

1454 ------ 

1455 lsst.daf.butler.registry.NoDefaultCollectionError 

1456 Raised if ``collections`` is `None` and 

1457 ``self.defaults.collections`` is `None`. 

1458 lsst.daf.butler.registry.CollectionExpressionError 

1459 Raised when ``collections`` expression is invalid. 

1460 """ 

1461 raise NotImplementedError() 

1462 

1463 @property 

1464 def obsCoreTableManager(self) -> ObsCoreTableManager | None: 

1465 """The ObsCore manager instance for this registry 

1466 (`~.interfaces.ObsCoreTableManager` 

1467 or `None`). 

1468 

1469 ObsCore manager may not be implemented for all registry backend, or 

1470 may not be enabled for many repositories. 

1471 """ 

1472 return None 

1473 

1474 storageClasses: StorageClassFactory 

1475 """All storage classes known to the registry (`StorageClassFactory`). 

1476 """