Coverage for python/lsst/daf/butler/registry/_registry.py: 95%

122 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-10-12 09:44 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ("Registry",) 

31 

32import contextlib 

33import logging 

34import re 

35from abc import ABC, abstractmethod 

36from collections.abc import Iterable, Iterator, Mapping, Sequence 

37from types import EllipsisType 

38from typing import TYPE_CHECKING, Any 

39 

40from lsst.resources import ResourcePathExpression 

41 

42from .._dataset_association import DatasetAssociation 

43from .._dataset_ref import DatasetId, DatasetIdGenEnum, DatasetRef 

44from .._dataset_type import DatasetType 

45from .._named import NameLookupMapping 

46from .._storage_class import StorageClassFactory 

47from .._timespan import Timespan 

48from ..dimensions import ( 

49 DataCoordinate, 

50 DataId, 

51 Dimension, 

52 DimensionConfig, 

53 DimensionElement, 

54 DimensionGraph, 

55 DimensionRecord, 

56 DimensionUniverse, 

57) 

58from ._collection_summary import CollectionSummary 

59from ._collection_type import CollectionType 

60from ._config import RegistryConfig 

61from ._defaults import RegistryDefaults 

62from .queries import DataCoordinateQueryResults, DatasetQueryResults, DimensionRecordQueryResults 

63from .wildcards import CollectionWildcard 

64 

65if TYPE_CHECKING: 

66 from .interfaces import ObsCoreTableManager 

67 

68_LOG = logging.getLogger(__name__) 

69 

70# TYpe alias for `collections` arguments. 

71CollectionArgType = str | re.Pattern | Iterable[str | re.Pattern] | EllipsisType | CollectionWildcard 

72 

73 

74class Registry(ABC): 

75 """Abstract Registry interface. 

76 

77 All subclasses should store `~lsst.daf.butler.registry.RegistryDefaults` in 

78 a ``_defaults`` property. No other properties are assumed shared between 

79 implementations. 

80 """ 

81 

82 @classmethod 

83 def createFromConfig( 

84 cls, 

85 config: RegistryConfig | str | None = None, 

86 dimensionConfig: DimensionConfig | str | None = None, 

87 butlerRoot: ResourcePathExpression | None = None, 

88 ) -> Registry: 

89 """Create registry database and return `Registry` instance. 

90 

91 This method initializes database contents, database must be empty 

92 prior to calling this method. 

93 

94 Parameters 

95 ---------- 

96 config : `RegistryConfig` or `str`, optional 

97 Registry configuration, if missing then default configuration will 

98 be loaded from registry.yaml. 

99 dimensionConfig : `DimensionConfig` or `str`, optional 

100 Dimensions configuration, if missing then default configuration 

101 will be loaded from dimensions.yaml. 

102 butlerRoot : convertible to `lsst.resources.ResourcePath`, optional 

103 Path to the repository root this `Registry` will manage. 

104 

105 Returns 

106 ------- 

107 registry : `Registry` 

108 A new `Registry` instance. 

109 

110 Notes 

111 ----- 

112 This method is for backward compatibility only, until all clients 

113 migrate to use new `~lsst.daf.butler.registry._RegistryFactory` factory 

114 class. Regular clients of registry class do not use this method, it is 

115 only used by tests in multiple packages. 

116 """ 

117 from ._registry_factory import _RegistryFactory 

118 

119 return _RegistryFactory(config).create_from_config(dimensionConfig, butlerRoot) 

120 

121 @abstractmethod 

122 def isWriteable(self) -> bool: 

123 """Return `True` if this registry allows write operations, and `False` 

124 otherwise. 

125 """ 

126 raise NotImplementedError() 

127 

128 @property 

129 @abstractmethod 

130 def dimensions(self) -> DimensionUniverse: 

131 """Definitions of all dimensions recognized by this `Registry` 

132 (`DimensionUniverse`). 

133 """ 

134 raise NotImplementedError() 

135 

136 @property 

137 def defaults(self) -> RegistryDefaults: 

138 """Default collection search path and/or output `~CollectionType.RUN` 

139 collection (`~lsst.daf.butler.registry.RegistryDefaults`). 

140 

141 This is an immutable struct whose components may not be set 

142 individually, but the entire struct can be set by assigning to this 

143 property. 

144 """ 

145 return self._defaults 

146 

147 @defaults.setter 

148 def defaults(self, value: RegistryDefaults) -> None: 

149 if value.run is not None: 

150 self.registerRun(value.run) 

151 value.finish(self) 

152 self._defaults = value 

153 

154 @abstractmethod 

155 def refresh(self) -> None: 

156 """Refresh all in-memory state by querying the database. 

157 

158 This may be necessary to enable querying for entities added by other 

159 registry instances after this one was constructed. 

160 """ 

161 raise NotImplementedError() 

162 

163 @contextlib.contextmanager 

164 @abstractmethod 

165 def transaction(self, *, savepoint: bool = False) -> Iterator[None]: 

166 """Return a context manager that represents a transaction.""" 

167 raise NotImplementedError() 

168 

169 def resetConnectionPool(self) -> None: 

170 """Reset connection pool for registry if relevant. 

171 

172 This operation can be used reset connections to servers when 

173 using registry with fork-based multiprocessing. This method should 

174 usually be called by the child process immediately 

175 after the fork. 

176 

177 The base class implementation is a no-op. 

178 """ 

179 pass 

180 

181 @abstractmethod 

182 def registerCollection( 

183 self, name: str, type: CollectionType = CollectionType.TAGGED, doc: str | None = None 

184 ) -> bool: 

185 """Add a new collection if one with the given name does not exist. 

186 

187 Parameters 

188 ---------- 

189 name : `str` 

190 The name of the collection to create. 

191 type : `CollectionType` 

192 Enum value indicating the type of collection to create. 

193 doc : `str`, optional 

194 Documentation string for the collection. 

195 

196 Returns 

197 ------- 

198 registered : `bool` 

199 Boolean indicating whether the collection was already registered 

200 or was created by this call. 

201 

202 Notes 

203 ----- 

204 This method cannot be called within transactions, as it needs to be 

205 able to perform its own transaction to be concurrent. 

206 """ 

207 raise NotImplementedError() 

208 

209 @abstractmethod 

210 def getCollectionType(self, name: str) -> CollectionType: 

211 """Return an enumeration value indicating the type of the given 

212 collection. 

213 

214 Parameters 

215 ---------- 

216 name : `str` 

217 The name of the collection. 

218 

219 Returns 

220 ------- 

221 type : `CollectionType` 

222 Enum value indicating the type of this collection. 

223 

224 Raises 

225 ------ 

226 lsst.daf.butler.registry.MissingCollectionError 

227 Raised if no collection with the given name exists. 

228 """ 

229 raise NotImplementedError() 

230 

231 @abstractmethod 

232 def registerRun(self, name: str, doc: str | None = None) -> bool: 

233 """Add a new run if one with the given name does not exist. 

234 

235 Parameters 

236 ---------- 

237 name : `str` 

238 The name of the run to create. 

239 doc : `str`, optional 

240 Documentation string for the collection. 

241 

242 Returns 

243 ------- 

244 registered : `bool` 

245 Boolean indicating whether a new run was registered. `False` 

246 if it already existed. 

247 

248 Notes 

249 ----- 

250 This method cannot be called within transactions, as it needs to be 

251 able to perform its own transaction to be concurrent. 

252 """ 

253 raise NotImplementedError() 

254 

255 @abstractmethod 

256 def removeCollection(self, name: str) -> None: 

257 """Remove the given collection from the registry. 

258 

259 Parameters 

260 ---------- 

261 name : `str` 

262 The name of the collection to remove. 

263 

264 Raises 

265 ------ 

266 lsst.daf.butler.registry.MissingCollectionError 

267 Raised if no collection with the given name exists. 

268 sqlalchemy.exc.IntegrityError 

269 Raised if the database rows associated with the collection are 

270 still referenced by some other table, such as a dataset in a 

271 datastore (for `~CollectionType.RUN` collections only) or a 

272 `~CollectionType.CHAINED` collection of which this collection is 

273 a child. 

274 

275 Notes 

276 ----- 

277 If this is a `~CollectionType.RUN` collection, all datasets and quanta 

278 in it will removed from the `Registry` database. This requires that 

279 those datasets be removed (or at least trashed) from any datastores 

280 that hold them first. 

281 

282 A collection may not be deleted as long as it is referenced by a 

283 `~CollectionType.CHAINED` collection; the ``CHAINED`` collection must 

284 be deleted or redefined first. 

285 """ 

286 raise NotImplementedError() 

287 

288 @abstractmethod 

289 def getCollectionChain(self, parent: str) -> Sequence[str]: 

290 """Return the child collections in a `~CollectionType.CHAINED` 

291 collection. 

292 

293 Parameters 

294 ---------- 

295 parent : `str` 

296 Name of the chained collection. Must have already been added via 

297 a call to `Registry.registerCollection`. 

298 

299 Returns 

300 ------- 

301 children : `~collections.abc.Sequence` [ `str` ] 

302 An ordered sequence of collection names that are searched when the 

303 given chained collection is searched. 

304 

305 Raises 

306 ------ 

307 lsst.daf.butler.registry.MissingCollectionError 

308 Raised if ``parent`` does not exist in the `Registry`. 

309 lsst.daf.butler.registry.CollectionTypeError 

310 Raised if ``parent`` does not correspond to a 

311 `~CollectionType.CHAINED` collection. 

312 """ 

313 raise NotImplementedError() 

314 

315 @abstractmethod 

316 def setCollectionChain(self, parent: str, children: Any, *, flatten: bool = False) -> None: 

317 """Define or redefine a `~CollectionType.CHAINED` collection. 

318 

319 Parameters 

320 ---------- 

321 parent : `str` 

322 Name of the chained collection. Must have already been added via 

323 a call to `Registry.registerCollection`. 

324 children : collection expression 

325 An expression defining an ordered search of child collections, 

326 generally an iterable of `str`; see 

327 :ref:`daf_butler_collection_expressions` for more information. 

328 flatten : `bool`, optional 

329 If `True` (`False` is default), recursively flatten out any nested 

330 `~CollectionType.CHAINED` collections in ``children`` first. 

331 

332 Raises 

333 ------ 

334 lsst.daf.butler.registry.MissingCollectionError 

335 Raised when any of the given collections do not exist in the 

336 `Registry`. 

337 lsst.daf.butler.registry.CollectionTypeError 

338 Raised if ``parent`` does not correspond to a 

339 `~CollectionType.CHAINED` collection. 

340 ValueError 

341 Raised if the given collections contains a cycle. 

342 """ 

343 raise NotImplementedError() 

344 

345 @abstractmethod 

346 def getCollectionParentChains(self, collection: str) -> set[str]: 

347 """Return the CHAINED collections that directly contain the given one. 

348 

349 Parameters 

350 ---------- 

351 name : `str` 

352 Name of the collection. 

353 

354 Returns 

355 ------- 

356 chains : `set` of `str` 

357 Set of `~CollectionType.CHAINED` collection names. 

358 """ 

359 raise NotImplementedError() 

360 

361 @abstractmethod 

362 def getCollectionDocumentation(self, collection: str) -> str | None: 

363 """Retrieve the documentation string for a collection. 

364 

365 Parameters 

366 ---------- 

367 name : `str` 

368 Name of the collection. 

369 

370 Returns 

371 ------- 

372 docs : `str` or `None` 

373 Docstring for the collection with the given name. 

374 """ 

375 raise NotImplementedError() 

376 

377 @abstractmethod 

378 def setCollectionDocumentation(self, collection: str, doc: str | None) -> None: 

379 """Set the documentation string for a collection. 

380 

381 Parameters 

382 ---------- 

383 name : `str` 

384 Name of the collection. 

385 docs : `str` or `None` 

386 Docstring for the collection with the given name; will replace any 

387 existing docstring. Passing `None` will remove any existing 

388 docstring. 

389 """ 

390 raise NotImplementedError() 

391 

392 @abstractmethod 

393 def getCollectionSummary(self, collection: str) -> CollectionSummary: 

394 """Return a summary for the given collection. 

395 

396 Parameters 

397 ---------- 

398 collection : `str` 

399 Name of the collection for which a summary is to be retrieved. 

400 

401 Returns 

402 ------- 

403 summary : `~lsst.daf.butler.registry.CollectionSummary` 

404 Summary of the dataset types and governor dimension values in 

405 this collection. 

406 """ 

407 raise NotImplementedError() 

408 

409 @abstractmethod 

410 def registerDatasetType(self, datasetType: DatasetType) -> bool: 

411 """Add a new `DatasetType` to the Registry. 

412 

413 It is not an error to register the same `DatasetType` twice. 

414 

415 Parameters 

416 ---------- 

417 datasetType : `DatasetType` 

418 The `DatasetType` to be added. 

419 

420 Returns 

421 ------- 

422 inserted : `bool` 

423 `True` if ``datasetType`` was inserted, `False` if an identical 

424 existing `DatasetType` was found. Note that in either case the 

425 DatasetType is guaranteed to be defined in the Registry 

426 consistently with the given definition. 

427 

428 Raises 

429 ------ 

430 ValueError 

431 Raised if the dimensions or storage class are invalid. 

432 lsst.daf.butler.registry.ConflictingDefinitionError 

433 Raised if this `DatasetType` is already registered with a different 

434 definition. 

435 

436 Notes 

437 ----- 

438 This method cannot be called within transactions, as it needs to be 

439 able to perform its own transaction to be concurrent. 

440 """ 

441 raise NotImplementedError() 

442 

443 @abstractmethod 

444 def removeDatasetType(self, name: str | tuple[str, ...]) -> None: 

445 """Remove the named `DatasetType` from the registry. 

446 

447 .. warning:: 

448 

449 Registry implementations can cache the dataset type definitions. 

450 This means that deleting the dataset type definition may result in 

451 unexpected behavior from other butler processes that are active 

452 that have not seen the deletion. 

453 

454 Parameters 

455 ---------- 

456 name : `str` or `tuple` [`str`] 

457 Name of the type to be removed or tuple containing a list of type 

458 names to be removed. Wildcards are allowed. 

459 

460 Raises 

461 ------ 

462 lsst.daf.butler.registry.OrphanedRecordError 

463 Raised if an attempt is made to remove the dataset type definition 

464 when there are already datasets associated with it. 

465 

466 Notes 

467 ----- 

468 If the dataset type is not registered the method will return without 

469 action. 

470 """ 

471 raise NotImplementedError() 

472 

473 @abstractmethod 

474 def getDatasetType(self, name: str) -> DatasetType: 

475 """Get the `DatasetType`. 

476 

477 Parameters 

478 ---------- 

479 name : `str` 

480 Name of the type. 

481 

482 Returns 

483 ------- 

484 type : `DatasetType` 

485 The `DatasetType` associated with the given name. 

486 

487 Raises 

488 ------ 

489 lsst.daf.butler.registry.MissingDatasetTypeError 

490 Raised if the requested dataset type has not been registered. 

491 

492 Notes 

493 ----- 

494 This method handles component dataset types automatically, though most 

495 other registry operations do not. 

496 """ 

497 raise NotImplementedError() 

498 

499 @abstractmethod 

500 def supportsIdGenerationMode(self, mode: DatasetIdGenEnum) -> bool: 

501 """Test whether the given dataset ID generation mode is supported by 

502 `insertDatasets`. 

503 

504 Parameters 

505 ---------- 

506 mode : `DatasetIdGenEnum` 

507 Enum value for the mode to test. 

508 

509 Returns 

510 ------- 

511 supported : `bool` 

512 Whether the given mode is supported. 

513 """ 

514 raise NotImplementedError() 

515 

516 @abstractmethod 

517 def findDataset( 

518 self, 

519 datasetType: DatasetType | str, 

520 dataId: DataId | None = None, 

521 *, 

522 collections: CollectionArgType | None = None, 

523 timespan: Timespan | None = None, 

524 **kwargs: Any, 

525 ) -> DatasetRef | None: 

526 """Find a dataset given its `DatasetType` and data ID. 

527 

528 This can be used to obtain a `DatasetRef` that permits the dataset to 

529 be read from a `Datastore`. If the dataset is a component and can not 

530 be found using the provided dataset type, a dataset ref for the parent 

531 will be returned instead but with the correct dataset type. 

532 

533 Parameters 

534 ---------- 

535 datasetType : `DatasetType` or `str` 

536 A `DatasetType` or the name of one. If this is a `DatasetType` 

537 instance, its storage class will be respected and propagated to 

538 the output, even if it differs from the dataset type definition 

539 in the registry, as long as the storage classes are convertible. 

540 dataId : `dict` or `DataCoordinate`, optional 

541 A `dict`-like object containing the `Dimension` links that identify 

542 the dataset within a collection. 

543 collections : collection expression, optional 

544 An expression that fully or partially identifies the collections to 

545 search for the dataset; see 

546 :ref:`daf_butler_collection_expressions` for more information. 

547 Defaults to ``self.defaults.collections``. 

548 timespan : `Timespan`, optional 

549 A timespan that the validity range of the dataset must overlap. 

550 If not provided, any `~CollectionType.CALIBRATION` collections 

551 matched by the ``collections`` argument will not be searched. 

552 **kwargs 

553 Additional keyword arguments passed to 

554 `DataCoordinate.standardize` to convert ``dataId`` to a true 

555 `DataCoordinate` or augment an existing one. 

556 

557 Returns 

558 ------- 

559 ref : `DatasetRef` 

560 A reference to the dataset, or `None` if no matching Dataset 

561 was found. 

562 

563 Raises 

564 ------ 

565 lsst.daf.butler.registry.NoDefaultCollectionError 

566 Raised if ``collections`` is `None` and 

567 ``self.defaults.collections`` is `None`. 

568 LookupError 

569 Raised if one or more data ID keys are missing. 

570 lsst.daf.butler.registry.MissingDatasetTypeError 

571 Raised if the dataset type does not exist. 

572 lsst.daf.butler.registry.MissingCollectionError 

573 Raised if any of ``collections`` does not exist in the registry. 

574 

575 Notes 

576 ----- 

577 This method simply returns `None` and does not raise an exception even 

578 when the set of collections searched is intrinsically incompatible with 

579 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but 

580 only `~CollectionType.CALIBRATION` collections are being searched. 

581 This may make it harder to debug some lookup failures, but the behavior 

582 is intentional; we consider it more important that failed searches are 

583 reported consistently, regardless of the reason, and that adding 

584 additional collections that do not contain a match to the search path 

585 never changes the behavior. 

586 

587 This method handles component dataset types automatically, though most 

588 other registry operations do not. 

589 """ 

590 raise NotImplementedError() 

591 

592 @abstractmethod 

593 def insertDatasets( 

594 self, 

595 datasetType: DatasetType | str, 

596 dataIds: Iterable[DataId], 

597 run: str | None = None, 

598 expand: bool = True, 

599 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE, 

600 ) -> list[DatasetRef]: 

601 """Insert one or more datasets into the `Registry`. 

602 

603 This always adds new datasets; to associate existing datasets with 

604 a new collection, use ``associate``. 

605 

606 Parameters 

607 ---------- 

608 datasetType : `DatasetType` or `str` 

609 A `DatasetType` or the name of one. 

610 dataIds : `~collections.abc.Iterable` of `dict` or `DataCoordinate` 

611 Dimension-based identifiers for the new datasets. 

612 run : `str`, optional 

613 The name of the run that produced the datasets. Defaults to 

614 ``self.defaults.run``. 

615 expand : `bool`, optional 

616 If `True` (default), expand data IDs as they are inserted. This is 

617 necessary in general to allow datastore to generate file templates, 

618 but it may be disabled if the caller can guarantee this is 

619 unnecessary. 

620 idGenerationMode : `DatasetIdGenEnum`, optional 

621 Specifies option for generating dataset IDs. By default unique IDs 

622 are generated for each inserted dataset. 

623 

624 Returns 

625 ------- 

626 refs : `list` of `DatasetRef` 

627 Resolved `DatasetRef` instances for all given data IDs (in the same 

628 order). 

629 

630 Raises 

631 ------ 

632 lsst.daf.butler.registry.DatasetTypeError 

633 Raised if ``datasetType`` is not known to registry. 

634 lsst.daf.butler.registry.CollectionTypeError 

635 Raised if ``run`` collection type is not `~CollectionType.RUN`. 

636 lsst.daf.butler.registry.NoDefaultCollectionError 

637 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

638 lsst.daf.butler.registry.ConflictingDefinitionError 

639 If a dataset with the same dataset type and data ID as one of those 

640 given already exists in ``run``. 

641 lsst.daf.butler.registry.MissingCollectionError 

642 Raised if ``run`` does not exist in the registry. 

643 """ 

644 raise NotImplementedError() 

645 

646 @abstractmethod 

647 def _importDatasets( 

648 self, 

649 datasets: Iterable[DatasetRef], 

650 expand: bool = True, 

651 ) -> list[DatasetRef]: 

652 """Import one or more datasets into the `Registry`. 

653 

654 Difference from `insertDatasets` method is that this method accepts 

655 `DatasetRef` instances which should already be resolved and have a 

656 dataset ID. If registry supports globally-unique dataset IDs (e.g. 

657 `uuid.UUID`) then datasets which already exist in the registry will be 

658 ignored if imported again. 

659 

660 Parameters 

661 ---------- 

662 datasets : `~collections.abc.Iterable` of `DatasetRef` 

663 Datasets to be inserted. All `DatasetRef` instances must have 

664 identical ``datasetType`` and ``run`` attributes. ``run`` 

665 attribute can be `None` and defaults to ``self.defaults.run``. 

666 Datasets can specify ``id`` attribute which will be used for 

667 inserted datasets. All dataset IDs must have the same type 

668 (`int` or `uuid.UUID`), if type of dataset IDs does not match 

669 configured backend then IDs will be ignored and new IDs will be 

670 generated by backend. 

671 expand : `bool`, optional 

672 If `True` (default), expand data IDs as they are inserted. This is 

673 necessary in general to allow datastore to generate file templates, 

674 but it may be disabled if the caller can guarantee this is 

675 unnecessary. 

676 

677 Returns 

678 ------- 

679 refs : `list` of `DatasetRef` 

680 Resolved `DatasetRef` instances for all given data IDs (in the same 

681 order). If any of ``datasets`` has an ID which already exists in 

682 the database then it will not be inserted or updated, but a 

683 resolved `DatasetRef` will be returned for it in any case. 

684 

685 Raises 

686 ------ 

687 lsst.daf.butler.registry.NoDefaultCollectionError 

688 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

689 lsst.daf.butler.registry.DatasetTypeError 

690 Raised if datasets correspond to more than one dataset type or 

691 dataset type is not known to registry. 

692 lsst.daf.butler.registry.ConflictingDefinitionError 

693 If a dataset with the same dataset type and data ID as one of those 

694 given already exists in ``run``. 

695 lsst.daf.butler.registry.MissingCollectionError 

696 Raised if ``run`` does not exist in the registry. 

697 

698 Notes 

699 ----- 

700 This method is considered package-private and internal to Butler 

701 implementation. Clients outside daf_butler package should not use this 

702 method. 

703 """ 

704 raise NotImplementedError() 

705 

706 @abstractmethod 

707 def getDataset(self, id: DatasetId) -> DatasetRef | None: 

708 """Retrieve a Dataset entry. 

709 

710 Parameters 

711 ---------- 

712 id : `DatasetId` 

713 The unique identifier for the dataset. 

714 

715 Returns 

716 ------- 

717 ref : `DatasetRef` or `None` 

718 A ref to the Dataset, or `None` if no matching Dataset 

719 was found. 

720 """ 

721 raise NotImplementedError() 

722 

723 @abstractmethod 

724 def removeDatasets(self, refs: Iterable[DatasetRef]) -> None: 

725 """Remove datasets from the Registry. 

726 

727 The datasets will be removed unconditionally from all collections, and 

728 any `Quantum` that consumed this dataset will instead be marked with 

729 having a NULL input. `Datastore` records will *not* be deleted; the 

730 caller is responsible for ensuring that the dataset has already been 

731 removed from all Datastores. 

732 

733 Parameters 

734 ---------- 

735 refs : `~collections.abc.Iterable` [`DatasetRef`] 

736 References to the datasets to be removed. Must include a valid 

737 ``id`` attribute, and should be considered invalidated upon return. 

738 

739 Raises 

740 ------ 

741 lsst.daf.butler.AmbiguousDatasetError 

742 Raised if any ``ref.id`` is `None`. 

743 lsst.daf.butler.registry.OrphanedRecordError 

744 Raised if any dataset is still present in any `Datastore`. 

745 """ 

746 raise NotImplementedError() 

747 

748 @abstractmethod 

749 def associate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

750 """Add existing datasets to a `~CollectionType.TAGGED` collection. 

751 

752 If a DatasetRef with the same exact ID is already in a collection 

753 nothing is changed. If a `DatasetRef` with the same `DatasetType` and 

754 data ID but with different ID exists in the collection, 

755 `~lsst.daf.butler.registry.ConflictingDefinitionError` is raised. 

756 

757 Parameters 

758 ---------- 

759 collection : `str` 

760 Indicates the collection the datasets should be associated with. 

761 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

762 An iterable of resolved `DatasetRef` instances that already exist 

763 in this `Registry`. 

764 

765 Raises 

766 ------ 

767 lsst.daf.butler.registry.ConflictingDefinitionError 

768 If a Dataset with the given `DatasetRef` already exists in the 

769 given collection. 

770 lsst.daf.butler.registry.MissingCollectionError 

771 Raised if ``collection`` does not exist in the registry. 

772 lsst.daf.butler.registry.CollectionTypeError 

773 Raise adding new datasets to the given ``collection`` is not 

774 allowed. 

775 """ 

776 raise NotImplementedError() 

777 

778 @abstractmethod 

779 def disassociate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

780 """Remove existing datasets from a `~CollectionType.TAGGED` collection. 

781 

782 ``collection`` and ``ref`` combinations that are not currently 

783 associated are silently ignored. 

784 

785 Parameters 

786 ---------- 

787 collection : `str` 

788 The collection the datasets should no longer be associated with. 

789 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

790 An iterable of resolved `DatasetRef` instances that already exist 

791 in this `Registry`. 

792 

793 Raises 

794 ------ 

795 lsst.daf.butler.AmbiguousDatasetError 

796 Raised if any of the given dataset references is unresolved. 

797 lsst.daf.butler.registry.MissingCollectionError 

798 Raised if ``collection`` does not exist in the registry. 

799 lsst.daf.butler.registry.CollectionTypeError 

800 Raise adding new datasets to the given ``collection`` is not 

801 allowed. 

802 """ 

803 raise NotImplementedError() 

804 

805 @abstractmethod 

806 def certify(self, collection: str, refs: Iterable[DatasetRef], timespan: Timespan) -> None: 

807 """Associate one or more datasets with a calibration collection and a 

808 validity range within it. 

809 

810 Parameters 

811 ---------- 

812 collection : `str` 

813 The name of an already-registered `~CollectionType.CALIBRATION` 

814 collection. 

815 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

816 Datasets to be associated. 

817 timespan : `Timespan` 

818 The validity range for these datasets within the collection. 

819 

820 Raises 

821 ------ 

822 lsst.daf.butler.AmbiguousDatasetError 

823 Raised if any of the given `DatasetRef` instances is unresolved. 

824 lsst.daf.butler.registry.ConflictingDefinitionError 

825 Raised if the collection already contains a different dataset with 

826 the same `DatasetType` and data ID and an overlapping validity 

827 range. 

828 lsst.daf.butler.registry.CollectionTypeError 

829 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

830 collection or if one or more datasets are of a dataset type for 

831 which `DatasetType.isCalibration` returns `False`. 

832 """ 

833 raise NotImplementedError() 

834 

835 @abstractmethod 

836 def decertify( 

837 self, 

838 collection: str, 

839 datasetType: str | DatasetType, 

840 timespan: Timespan, 

841 *, 

842 dataIds: Iterable[DataId] | None = None, 

843 ) -> None: 

844 """Remove or adjust datasets to clear a validity range within a 

845 calibration collection. 

846 

847 Parameters 

848 ---------- 

849 collection : `str` 

850 The name of an already-registered `~CollectionType.CALIBRATION` 

851 collection. 

852 datasetType : `str` or `DatasetType` 

853 Name or `DatasetType` instance for the datasets to be decertified. 

854 timespan : `Timespan`, optional 

855 The validity range to remove datasets from within the collection. 

856 Datasets that overlap this range but are not contained by it will 

857 have their validity ranges adjusted to not overlap it, which may 

858 split a single dataset validity range into two. 

859 dataIds : iterable [`dict` or `DataCoordinate`], optional 

860 Data IDs that should be decertified within the given validity range 

861 If `None`, all data IDs for ``self.datasetType`` will be 

862 decertified. 

863 

864 Raises 

865 ------ 

866 lsst.daf.butler.registry.CollectionTypeError 

867 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

868 collection or if ``datasetType.isCalibration() is False``. 

869 """ 

870 raise NotImplementedError() 

871 

872 @abstractmethod 

873 def getDatasetLocations(self, ref: DatasetRef) -> Iterable[str]: 

874 """Retrieve datastore locations for a given dataset. 

875 

876 Parameters 

877 ---------- 

878 ref : `DatasetRef` 

879 A reference to the dataset for which to retrieve storage 

880 information. 

881 

882 Returns 

883 ------- 

884 datastores : `~collections.abc.Iterable` [ `str` ] 

885 All the matching datastores holding this dataset. 

886 

887 Raises 

888 ------ 

889 lsst.daf.butler.AmbiguousDatasetError 

890 Raised if ``ref.id`` is `None`. 

891 """ 

892 raise NotImplementedError() 

893 

894 @abstractmethod 

895 def expandDataId( 

896 self, 

897 dataId: DataId | None = None, 

898 *, 

899 graph: DimensionGraph | None = None, 

900 records: NameLookupMapping[DimensionElement, DimensionRecord | None] | None = None, 

901 withDefaults: bool = True, 

902 **kwargs: Any, 

903 ) -> DataCoordinate: 

904 """Expand a dimension-based data ID to include additional information. 

905 

906 Parameters 

907 ---------- 

908 dataId : `DataCoordinate` or `dict`, optional 

909 Data ID to be expanded; augmented and overridden by ``kwargs``. 

910 graph : `DimensionGraph`, optional 

911 Set of dimensions for the expanded ID. If `None`, the dimensions 

912 will be inferred from the keys of ``dataId`` and ``kwargs``. 

913 Dimensions that are in ``dataId`` or ``kwargs`` but not in 

914 ``graph`` are silently ignored, providing a way to extract and 

915 ``graph`` expand a subset of a data ID. 

916 records : `~collections.abc.Mapping` [`str`, `DimensionRecord`], \ 

917 optional 

918 Dimension record data to use before querying the database for that 

919 data, keyed by element name. 

920 withDefaults : `bool`, optional 

921 Utilize ``self.defaults.dataId`` to fill in missing governor 

922 dimension key-value pairs. Defaults to `True` (i.e. defaults are 

923 used). 

924 **kwargs 

925 Additional keywords are treated like additional key-value pairs for 

926 ``dataId``, extending and overriding 

927 

928 Returns 

929 ------- 

930 expanded : `DataCoordinate` 

931 A data ID that includes full metadata for all of the dimensions it 

932 identifies, i.e. guarantees that ``expanded.hasRecords()`` and 

933 ``expanded.hasFull()`` both return `True`. 

934 

935 Raises 

936 ------ 

937 lsst.daf.butler.registry.DataIdError 

938 Raised when ``dataId`` or keyword arguments specify unknown 

939 dimensions or values, or when a resulting data ID contains 

940 contradictory key-value pairs, according to dimension 

941 relationships. 

942 

943 Notes 

944 ----- 

945 This method cannot be relied upon to reject invalid data ID values 

946 for dimensions that do actually not have any record columns. For 

947 efficiency reasons the records for these dimensions (which have only 

948 dimension key values that are given by the caller) may be constructed 

949 directly rather than obtained from the registry database. 

950 """ 

951 raise NotImplementedError() 

952 

953 @abstractmethod 

954 def insertDimensionData( 

955 self, 

956 element: DimensionElement | str, 

957 *data: Mapping[str, Any] | DimensionRecord, 

958 conform: bool = True, 

959 replace: bool = False, 

960 skip_existing: bool = False, 

961 ) -> None: 

962 """Insert one or more dimension records into the database. 

963 

964 Parameters 

965 ---------- 

966 element : `DimensionElement` or `str` 

967 The `DimensionElement` or name thereof that identifies the table 

968 records will be inserted into. 

969 *data : `dict` or `DimensionRecord` 

970 One or more records to insert. 

971 conform : `bool`, optional 

972 If `False` (`True` is default) perform no checking or conversions, 

973 and assume that ``element`` is a `DimensionElement` instance and 

974 ``data`` is a one or more `DimensionRecord` instances of the 

975 appropriate subclass. 

976 replace : `bool`, optional 

977 If `True` (`False` is default), replace existing records in the 

978 database if there is a conflict. 

979 skip_existing : `bool`, optional 

980 If `True` (`False` is default), skip insertion if a record with 

981 the same primary key values already exists. Unlike 

982 `syncDimensionData`, this will not detect when the given record 

983 differs from what is in the database, and should not be used when 

984 this is a concern. 

985 """ 

986 raise NotImplementedError() 

987 

988 @abstractmethod 

989 def syncDimensionData( 

990 self, 

991 element: DimensionElement | str, 

992 row: Mapping[str, Any] | DimensionRecord, 

993 conform: bool = True, 

994 update: bool = False, 

995 ) -> bool | dict[str, Any]: 

996 """Synchronize the given dimension record with the database, inserting 

997 if it does not already exist and comparing values if it does. 

998 

999 Parameters 

1000 ---------- 

1001 element : `DimensionElement` or `str` 

1002 The `DimensionElement` or name thereof that identifies the table 

1003 records will be inserted into. 

1004 row : `dict` or `DimensionRecord` 

1005 The record to insert. 

1006 conform : `bool`, optional 

1007 If `False` (`True` is default) perform no checking or conversions, 

1008 and assume that ``element`` is a `DimensionElement` instance and 

1009 ``data`` is a one or more `DimensionRecord` instances of the 

1010 appropriate subclass. 

1011 update : `bool`, optional 

1012 If `True` (`False` is default), update the existing record in the 

1013 database if there is a conflict. 

1014 

1015 Returns 

1016 ------- 

1017 inserted_or_updated : `bool` or `dict` 

1018 `True` if a new row was inserted, `False` if no changes were 

1019 needed, or a `dict` mapping updated column names to their old 

1020 values if an update was performed (only possible if 

1021 ``update=True``). 

1022 

1023 Raises 

1024 ------ 

1025 lsst.daf.butler.registry.ConflictingDefinitionError 

1026 Raised if the record exists in the database (according to primary 

1027 key lookup) but is inconsistent with the given one. 

1028 """ 

1029 raise NotImplementedError() 

1030 

1031 @abstractmethod 

1032 def queryDatasetTypes( 

1033 self, 

1034 expression: Any = ..., 

1035 *, 

1036 components: bool | None = False, 

1037 missing: list[str] | None = None, 

1038 ) -> Iterable[DatasetType]: 

1039 """Iterate over the dataset types whose names match an expression. 

1040 

1041 Parameters 

1042 ---------- 

1043 expression : dataset type expression, optional 

1044 An expression that fully or partially identifies the dataset types 

1045 to return, such as a `str`, `re.Pattern`, or iterable thereof. 

1046 ``...`` can be used to return all dataset types, and is the 

1047 default. See :ref:`daf_butler_dataset_type_expressions` for more 

1048 information. 

1049 components : `bool`, optional 

1050 If `True`, apply all expression patterns to component dataset type 

1051 names as well. If `False`, never apply patterns to components. 

1052 If `None`, apply patterns to components only if their 

1053 parent datasets were not matched by the expression. 

1054 Fully-specified component datasets (`str` or `DatasetType` 

1055 instances) are always included. 

1056 

1057 Values other than `False` are deprecated, and only `False` will be 

1058 supported after v26. After v27 this argument will be removed 

1059 entirely. 

1060 missing : `list` of `str`, optional 

1061 String dataset type names that were explicitly given (i.e. not 

1062 regular expression patterns) but not found will be appended to this 

1063 list, if it is provided. 

1064 

1065 Returns 

1066 ------- 

1067 dataset_types : `~collections.abc.Iterable` [ `DatasetType`] 

1068 An `~collections.abc.Iterable` of `DatasetType` instances whose 

1069 names match ``expression``. 

1070 

1071 Raises 

1072 ------ 

1073 lsst.daf.butler.registry.DatasetTypeExpressionError 

1074 Raised when ``expression`` is invalid. 

1075 """ 

1076 raise NotImplementedError() 

1077 

1078 @abstractmethod 

1079 def queryCollections( 

1080 self, 

1081 expression: Any = ..., 

1082 datasetType: DatasetType | None = None, 

1083 collectionTypes: Iterable[CollectionType] | CollectionType = CollectionType.all(), 

1084 flattenChains: bool = False, 

1085 includeChains: bool | None = None, 

1086 ) -> Sequence[str]: 

1087 """Iterate over the collections whose names match an expression. 

1088 

1089 Parameters 

1090 ---------- 

1091 expression : collection expression, optional 

1092 An expression that identifies the collections to return, such as 

1093 a `str` (for full matches or partial matches via globs), 

1094 `re.Pattern` (for partial matches), or iterable thereof. ``...`` 

1095 can be used to return all collections, and is the default. 

1096 See :ref:`daf_butler_collection_expressions` for more information. 

1097 datasetType : `DatasetType`, optional 

1098 If provided, only yield collections that may contain datasets of 

1099 this type. This is a conservative approximation in general; it may 

1100 yield collections that do not have any such datasets. 

1101 collectionTypes : `~collections.abc.Set` [`CollectionType`] or \ 

1102 `CollectionType`, optional 

1103 If provided, only yield collections of these types. 

1104 flattenChains : `bool`, optional 

1105 If `True` (`False` is default), recursively yield the child 

1106 collections of matching `~CollectionType.CHAINED` collections. 

1107 includeChains : `bool`, optional 

1108 If `True`, yield records for matching `~CollectionType.CHAINED` 

1109 collections. Default is the opposite of ``flattenChains``: include 

1110 either CHAINED collections or their children, but not both. 

1111 

1112 Returns 

1113 ------- 

1114 collections : `~collections.abc.Sequence` [ `str` ] 

1115 The names of collections that match ``expression``. 

1116 

1117 Raises 

1118 ------ 

1119 lsst.daf.butler.registry.CollectionExpressionError 

1120 Raised when ``expression`` is invalid. 

1121 

1122 Notes 

1123 ----- 

1124 The order in which collections are returned is unspecified, except that 

1125 the children of a `~CollectionType.CHAINED` collection are guaranteed 

1126 to be in the order in which they are searched. When multiple parent 

1127 `~CollectionType.CHAINED` collections match the same criteria, the 

1128 order in which the two lists appear is unspecified, and the lists of 

1129 children may be incomplete if a child has multiple parents. 

1130 """ 

1131 raise NotImplementedError() 

1132 

1133 @abstractmethod 

1134 def queryDatasets( 

1135 self, 

1136 datasetType: Any, 

1137 *, 

1138 collections: CollectionArgType | None = None, 

1139 dimensions: Iterable[Dimension | str] | None = None, 

1140 dataId: DataId | None = None, 

1141 where: str = "", 

1142 findFirst: bool = False, 

1143 components: bool | None = False, 

1144 bind: Mapping[str, Any] | None = None, 

1145 check: bool = True, 

1146 **kwargs: Any, 

1147 ) -> DatasetQueryResults: 

1148 """Query for and iterate over dataset references matching user-provided 

1149 criteria. 

1150 

1151 Parameters 

1152 ---------- 

1153 datasetType : dataset type expression 

1154 An expression that fully or partially identifies the dataset types 

1155 to be queried. Allowed types include `DatasetType`, `str`, 

1156 `re.Pattern`, and iterables thereof. The special value ``...`` can 

1157 be used to query all dataset types. See 

1158 :ref:`daf_butler_dataset_type_expressions` for more information. 

1159 collections : collection expression, optional 

1160 An expression that identifies the collections to search, such as a 

1161 `str` (for full matches or partial matches via globs), `re.Pattern` 

1162 (for partial matches), or iterable thereof. ``...`` can be used to 

1163 search all collections (actually just all `~CollectionType.RUN` 

1164 collections, because this will still find all datasets). 

1165 If not provided, ``self.default.collections`` is used. See 

1166 :ref:`daf_butler_collection_expressions` for more information. 

1167 dimensions : `~collections.abc.Iterable` of `Dimension` or `str` 

1168 Dimensions to include in the query (in addition to those used 

1169 to identify the queried dataset type(s)), either to constrain 

1170 the resulting datasets to those for which a matching dimension 

1171 exists, or to relate the dataset type's dimensions to dimensions 

1172 referenced by the ``dataId`` or ``where`` arguments. 

1173 dataId : `dict` or `DataCoordinate`, optional 

1174 A data ID whose key-value pairs are used as equality constraints 

1175 in the query. 

1176 where : `str`, optional 

1177 A string expression similar to a SQL WHERE clause. May involve 

1178 any column of a dimension table or (as a shortcut for the primary 

1179 key column of a dimension table) dimension name. See 

1180 :ref:`daf_butler_dimension_expressions` for more information. 

1181 findFirst : `bool`, optional 

1182 If `True` (`False` is default), for each result data ID, only 

1183 yield one `DatasetRef` of each `DatasetType`, from the first 

1184 collection in which a dataset of that dataset type appears 

1185 (according to the order of ``collections`` passed in). If `True`, 

1186 ``collections`` must not contain regular expressions and may not 

1187 be ``...``. 

1188 components : `bool`, optional 

1189 If `True`, apply all dataset expression patterns to component 

1190 dataset type names as well. If `False`, never apply patterns to 

1191 components. If `None`, apply patterns to components only 

1192 if their parent datasets were not matched by the expression. 

1193 Fully-specified component datasets (`str` or `DatasetType` 

1194 instances) are always included. 

1195 

1196 Values other than `False` are deprecated, and only `False` will be 

1197 supported after v26. After v27 this argument will be removed 

1198 entirely. 

1199 bind : `~collections.abc.Mapping`, optional 

1200 Mapping containing literal values that should be injected into the 

1201 ``where`` expression, keyed by the identifiers they replace. 

1202 Values of collection type can be expanded in some cases; see 

1203 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1204 information. 

1205 check : `bool`, optional 

1206 If `True` (default) check the query for consistency before 

1207 executing it. This may reject some valid queries that resemble 

1208 common mistakes (e.g. queries for visits without specifying an 

1209 instrument). 

1210 **kwargs 

1211 Additional keyword arguments are forwarded to 

1212 `DataCoordinate.standardize` when processing the ``dataId`` 

1213 argument (and may be used to provide a constraining data ID even 

1214 when the ``dataId`` argument is `None`). 

1215 

1216 Returns 

1217 ------- 

1218 refs : `.queries.DatasetQueryResults` 

1219 Dataset references matching the given query criteria. Nested data 

1220 IDs are guaranteed to include values for all implied dimensions 

1221 (i.e. `DataCoordinate.hasFull` will return `True`), but will not 

1222 include dimension records (`DataCoordinate.hasRecords` will be 

1223 `False`) unless `~.queries.DatasetQueryResults.expanded` is 

1224 called on the result object (which returns a new one). 

1225 

1226 Raises 

1227 ------ 

1228 lsst.daf.butler.registry.DatasetTypeExpressionError 

1229 Raised when ``datasetType`` expression is invalid. 

1230 TypeError 

1231 Raised when the arguments are incompatible, such as when a 

1232 collection wildcard is passed when ``findFirst`` is `True`, or 

1233 when ``collections`` is `None` and ``self.defaults.collections`` is 

1234 also `None`. 

1235 lsst.daf.butler.registry.DataIdError 

1236 Raised when ``dataId`` or keyword arguments specify unknown 

1237 dimensions or values, or when they contain inconsistent values. 

1238 lsst.daf.butler.registry.UserExpressionError 

1239 Raised when ``where`` expression is invalid. 

1240 

1241 Notes 

1242 ----- 

1243 When multiple dataset types are queried in a single call, the 

1244 results of this operation are equivalent to querying for each dataset 

1245 type separately in turn, and no information about the relationships 

1246 between datasets of different types is included. In contexts where 

1247 that kind of information is important, the recommended pattern is to 

1248 use `queryDataIds` to first obtain data IDs (possibly with the 

1249 desired dataset types and collections passed as constraints to the 

1250 query), and then use multiple (generally much simpler) calls to 

1251 `queryDatasets` with the returned data IDs passed as constraints. 

1252 """ 

1253 raise NotImplementedError() 

1254 

1255 @abstractmethod 

1256 def queryDataIds( 

1257 self, 

1258 dimensions: Iterable[Dimension | str] | Dimension | str, 

1259 *, 

1260 dataId: DataId | None = None, 

1261 datasets: Any = None, 

1262 collections: CollectionArgType | None = None, 

1263 where: str = "", 

1264 components: bool | None = False, 

1265 bind: Mapping[str, Any] | None = None, 

1266 check: bool = True, 

1267 **kwargs: Any, 

1268 ) -> DataCoordinateQueryResults: 

1269 """Query for data IDs matching user-provided criteria. 

1270 

1271 Parameters 

1272 ---------- 

1273 dimensions : `Dimension` or `str`, or iterable thereof 

1274 The dimensions of the data IDs to yield, as either `Dimension` 

1275 instances or `str`. Will be automatically expanded to a complete 

1276 `DimensionGraph`. 

1277 dataId : `dict` or `DataCoordinate`, optional 

1278 A data ID whose key-value pairs are used as equality constraints 

1279 in the query. 

1280 datasets : dataset type expression, optional 

1281 An expression that fully or partially identifies dataset types 

1282 that should constrain the yielded data IDs. For example, including 

1283 "raw" here would constrain the yielded ``instrument``, 

1284 ``exposure``, ``detector``, and ``physical_filter`` values to only 

1285 those for which at least one "raw" dataset exists in 

1286 ``collections``. Allowed types include `DatasetType`, `str`, 

1287 and iterables thereof. Regular expression objects (i.e. 

1288 `re.Pattern`) are deprecated and will be removed after the v26 

1289 release. See :ref:`daf_butler_dataset_type_expressions` for more 

1290 information. 

1291 collections : collection expression, optional 

1292 An expression that identifies the collections to search for 

1293 datasets, such as a `str` (for full matches or partial matches 

1294 via globs), `re.Pattern` (for partial matches), or iterable 

1295 thereof. ``...`` can be used to search all collections (actually 

1296 just all `~CollectionType.RUN` collections, because this will 

1297 still find all datasets). If not provided, 

1298 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1299 is also passed. See :ref:`daf_butler_collection_expressions` for 

1300 more information. 

1301 where : `str`, optional 

1302 A string expression similar to a SQL WHERE clause. May involve 

1303 any column of a dimension table or (as a shortcut for the primary 

1304 key column of a dimension table) dimension name. See 

1305 :ref:`daf_butler_dimension_expressions` for more information. 

1306 components : `bool`, optional 

1307 If `True`, apply all dataset expression patterns to component 

1308 dataset type names as well. If `False`, never apply patterns to 

1309 components. If `None`, apply patterns to components only 

1310 if their parent datasets were not matched by the expression. 

1311 Fully-specified component datasets (`str` or `DatasetType` 

1312 instances) are always included. 

1313 

1314 Values other than `False` are deprecated, and only `False` will be 

1315 supported after v26. After v27 this argument will be removed 

1316 entirely. 

1317 bind : `~collections.abc.Mapping`, optional 

1318 Mapping containing literal values that should be injected into the 

1319 ``where`` expression, keyed by the identifiers they replace. 

1320 Values of collection type can be expanded in some cases; see 

1321 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1322 information. 

1323 check : `bool`, optional 

1324 If `True` (default) check the query for consistency before 

1325 executing it. This may reject some valid queries that resemble 

1326 common mistakes (e.g. queries for visits without specifying an 

1327 instrument). 

1328 **kwargs 

1329 Additional keyword arguments are forwarded to 

1330 `DataCoordinate.standardize` when processing the ``dataId`` 

1331 argument (and may be used to provide a constraining data ID even 

1332 when the ``dataId`` argument is `None`). 

1333 

1334 Returns 

1335 ------- 

1336 dataIds : `.queries.DataCoordinateQueryResults` 

1337 Data IDs matching the given query parameters. These are guaranteed 

1338 to identify all dimensions (`DataCoordinate.hasFull` returns 

1339 `True`), but will not contain `DimensionRecord` objects 

1340 (`DataCoordinate.hasRecords` returns `False`). Call 

1341 `~.queries.DataCoordinateQueryResults.expanded` on the 

1342 returned object to fetch those (and consider using 

1343 `~.queries.DataCoordinateQueryResults.materialize` on the 

1344 returned object first if the expected number of rows is very 

1345 large). See documentation for those methods for additional 

1346 information. 

1347 

1348 Raises 

1349 ------ 

1350 lsst.daf.butler.registry.NoDefaultCollectionError 

1351 Raised if ``collections`` is `None` and 

1352 ``self.defaults.collections`` is `None`. 

1353 lsst.daf.butler.registry.CollectionExpressionError 

1354 Raised when ``collections`` expression is invalid. 

1355 lsst.daf.butler.registry.DataIdError 

1356 Raised when ``dataId`` or keyword arguments specify unknown 

1357 dimensions or values, or when they contain inconsistent values. 

1358 lsst.daf.butler.registry.DatasetTypeExpressionError 

1359 Raised when ``datasetType`` expression is invalid. 

1360 lsst.daf.butler.registry.UserExpressionError 

1361 Raised when ``where`` expression is invalid. 

1362 """ 

1363 raise NotImplementedError() 

1364 

1365 @abstractmethod 

1366 def queryDimensionRecords( 

1367 self, 

1368 element: DimensionElement | str, 

1369 *, 

1370 dataId: DataId | None = None, 

1371 datasets: Any = None, 

1372 collections: CollectionArgType | None = None, 

1373 where: str = "", 

1374 components: bool | None = False, 

1375 bind: Mapping[str, Any] | None = None, 

1376 check: bool = True, 

1377 **kwargs: Any, 

1378 ) -> DimensionRecordQueryResults: 

1379 """Query for dimension information matching user-provided criteria. 

1380 

1381 Parameters 

1382 ---------- 

1383 element : `DimensionElement` or `str` 

1384 The dimension element to obtain records for. 

1385 dataId : `dict` or `DataCoordinate`, optional 

1386 A data ID whose key-value pairs are used as equality constraints 

1387 in the query. 

1388 datasets : dataset type expression, optional 

1389 An expression that fully or partially identifies dataset types 

1390 that should constrain the yielded records. See `queryDataIds` and 

1391 :ref:`daf_butler_dataset_type_expressions` for more information. 

1392 collections : collection expression, optional 

1393 An expression that identifies the collections to search for 

1394 datasets, such as a `str` (for full matches or partial matches 

1395 via globs), `re.Pattern` (for partial matches), or iterable 

1396 thereof. ``...`` can be used to search all collections (actually 

1397 just all `~CollectionType.RUN` collections, because this will 

1398 still find all datasets). If not provided, 

1399 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1400 is also passed. See :ref:`daf_butler_collection_expressions` for 

1401 more information. 

1402 where : `str`, optional 

1403 A string expression similar to a SQL WHERE clause. See 

1404 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more 

1405 information. 

1406 components : `bool`, optional 

1407 Whether to apply dataset expressions to components as well. 

1408 See `queryDataIds` for more information. 

1409 

1410 Values other than `False` are deprecated, and only `False` will be 

1411 supported after v26. After v27 this argument will be removed 

1412 entirely. 

1413 bind : `~collections.abc.Mapping`, optional 

1414 Mapping containing literal values that should be injected into the 

1415 ``where`` expression, keyed by the identifiers they replace. 

1416 Values of collection type can be expanded in some cases; see 

1417 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1418 information. 

1419 check : `bool`, optional 

1420 If `True` (default) check the query for consistency before 

1421 executing it. This may reject some valid queries that resemble 

1422 common mistakes (e.g. queries for visits without specifying an 

1423 instrument). 

1424 **kwargs 

1425 Additional keyword arguments are forwarded to 

1426 `DataCoordinate.standardize` when processing the ``dataId`` 

1427 argument (and may be used to provide a constraining data ID even 

1428 when the ``dataId`` argument is `None`). 

1429 

1430 Returns 

1431 ------- 

1432 dataIds : `.queries.DimensionRecordQueryResults` 

1433 Data IDs matching the given query parameters. 

1434 

1435 Raises 

1436 ------ 

1437 lsst.daf.butler.registry.NoDefaultCollectionError 

1438 Raised if ``collections`` is `None` and 

1439 ``self.defaults.collections`` is `None`. 

1440 lsst.daf.butler.registry.CollectionExpressionError 

1441 Raised when ``collections`` expression is invalid. 

1442 lsst.daf.butler.registry.DataIdError 

1443 Raised when ``dataId`` or keyword arguments specify unknown 

1444 dimensions or values, or when they contain inconsistent values. 

1445 lsst.daf.butler.registry.DatasetTypeExpressionError 

1446 Raised when ``datasetType`` expression is invalid. 

1447 lsst.daf.butler.registry.UserExpressionError 

1448 Raised when ``where`` expression is invalid. 

1449 """ 

1450 raise NotImplementedError() 

1451 

1452 @abstractmethod 

1453 def queryDatasetAssociations( 

1454 self, 

1455 datasetType: str | DatasetType, 

1456 collections: CollectionArgType | None = ..., 

1457 *, 

1458 collectionTypes: Iterable[CollectionType] = CollectionType.all(), 

1459 flattenChains: bool = False, 

1460 ) -> Iterator[DatasetAssociation]: 

1461 """Iterate over dataset-collection combinations where the dataset is in 

1462 the collection. 

1463 

1464 This method is a temporary placeholder for better support for 

1465 association results in `queryDatasets`. It will probably be 

1466 removed in the future, and should be avoided in production code 

1467 whenever possible. 

1468 

1469 Parameters 

1470 ---------- 

1471 datasetType : `DatasetType` or `str` 

1472 A dataset type object or the name of one. 

1473 collections : collection expression, optional 

1474 An expression that identifies the collections to search for 

1475 datasets, such as a `str` (for full matches or partial matches 

1476 via globs), `re.Pattern` (for partial matches), or iterable 

1477 thereof. ``...`` can be used to search all collections (actually 

1478 just all `~CollectionType.RUN` collections, because this will still 

1479 find all datasets). If not provided, ``self.default.collections`` 

1480 is used. See :ref:`daf_butler_collection_expressions` for more 

1481 information. 

1482 collectionTypes : `~collections.abc.Set` [ `CollectionType` ], optional 

1483 If provided, only yield associations from collections of these 

1484 types. 

1485 flattenChains : `bool`, optional 

1486 If `True`, search in the children of `~CollectionType.CHAINED` 

1487 collections. If `False`, ``CHAINED`` collections are ignored. 

1488 

1489 Yields 

1490 ------ 

1491 association : `.DatasetAssociation` 

1492 Object representing the relationship between a single dataset and 

1493 a single collection. 

1494 

1495 Raises 

1496 ------ 

1497 lsst.daf.butler.registry.NoDefaultCollectionError 

1498 Raised if ``collections`` is `None` and 

1499 ``self.defaults.collections`` is `None`. 

1500 lsst.daf.butler.registry.CollectionExpressionError 

1501 Raised when ``collections`` expression is invalid. 

1502 """ 

1503 raise NotImplementedError() 

1504 

1505 @property 

1506 def obsCoreTableManager(self) -> ObsCoreTableManager | None: 

1507 """The ObsCore manager instance for this registry 

1508 (`~.interfaces.ObsCoreTableManager` 

1509 or `None`). 

1510 

1511 ObsCore manager may not be implemented for all registry backend, or 

1512 may not be enabled for many repositories. 

1513 """ 

1514 return None 

1515 

1516 storageClasses: StorageClassFactory 

1517 """All storage classes known to the registry (`StorageClassFactory`). 

1518 """