Coverage for python/lsst/daf/butler/registry/_registry.py: 95%

116 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-10-02 08:00 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ("Registry",) 

31 

32import contextlib 

33import logging 

34import re 

35from abc import ABC, abstractmethod 

36from collections.abc import Iterable, Iterator, Mapping, Sequence 

37from types import EllipsisType 

38from typing import TYPE_CHECKING, Any 

39 

40from lsst.resources import ResourcePathExpression 

41 

42from ..core import ( 

43 DataCoordinate, 

44 DataId, 

45 DatasetAssociation, 

46 DatasetId, 

47 DatasetIdGenEnum, 

48 DatasetRef, 

49 DatasetType, 

50 Dimension, 

51 DimensionConfig, 

52 DimensionElement, 

53 DimensionGraph, 

54 DimensionRecord, 

55 DimensionUniverse, 

56 NameLookupMapping, 

57 StorageClassFactory, 

58 Timespan, 

59) 

60from ._collection_summary import CollectionSummary 

61from ._collectionType import CollectionType 

62from ._config import RegistryConfig 

63from ._defaults import RegistryDefaults 

64from .queries import DataCoordinateQueryResults, DatasetQueryResults, DimensionRecordQueryResults 

65from .wildcards import CollectionWildcard 

66 

67if TYPE_CHECKING: 

68 from .interfaces import ObsCoreTableManager 

69 

70_LOG = logging.getLogger(__name__) 

71 

72# TYpe alias for `collections` arguments. 

73CollectionArgType = str | re.Pattern | Iterable[str | re.Pattern] | EllipsisType | CollectionWildcard 

74 

75 

76class Registry(ABC): 

77 """Abstract Registry interface. 

78 

79 All subclasses should store `~lsst.daf.butler.registry.RegistryDefaults` in 

80 a ``_defaults`` property. No other properties are assumed shared between 

81 implementations. 

82 """ 

83 

84 @classmethod 

85 def createFromConfig( 

86 cls, 

87 config: RegistryConfig | str | None = None, 

88 dimensionConfig: DimensionConfig | str | None = None, 

89 butlerRoot: ResourcePathExpression | None = None, 

90 ) -> Registry: 

91 """Create registry database and return `Registry` instance. 

92 

93 This method initializes database contents, database must be empty 

94 prior to calling this method. 

95 

96 Parameters 

97 ---------- 

98 config : `RegistryConfig` or `str`, optional 

99 Registry configuration, if missing then default configuration will 

100 be loaded from registry.yaml. 

101 dimensionConfig : `DimensionConfig` or `str`, optional 

102 Dimensions configuration, if missing then default configuration 

103 will be loaded from dimensions.yaml. 

104 butlerRoot : convertible to `lsst.resources.ResourcePath`, optional 

105 Path to the repository root this `Registry` will manage. 

106 

107 Returns 

108 ------- 

109 registry : `Registry` 

110 A new `Registry` instance. 

111 

112 Notes 

113 ----- 

114 This method is for backward compatibility only, until all clients 

115 migrate to use new `~lsst.daf.butler.registry._RegistryFactory` factory 

116 class. Regular clients of registry class do not use this method, it is 

117 only used by tests in multiple packages. 

118 """ 

119 from ._registry_factory import _RegistryFactory 

120 

121 return _RegistryFactory(config).create_from_config(dimensionConfig, butlerRoot) 

122 

123 @abstractmethod 

124 def isWriteable(self) -> bool: 

125 """Return `True` if this registry allows write operations, and `False` 

126 otherwise. 

127 """ 

128 raise NotImplementedError() 

129 

130 @property 

131 @abstractmethod 

132 def dimensions(self) -> DimensionUniverse: 

133 """Definitions of all dimensions recognized by this `Registry` 

134 (`DimensionUniverse`). 

135 """ 

136 raise NotImplementedError() 

137 

138 @property 

139 def defaults(self) -> RegistryDefaults: 

140 """Default collection search path and/or output `~CollectionType.RUN` 

141 collection (`~lsst.daf.butler.registry.RegistryDefaults`). 

142 

143 This is an immutable struct whose components may not be set 

144 individually, but the entire struct can be set by assigning to this 

145 property. 

146 """ 

147 return self._defaults 

148 

149 @defaults.setter 

150 def defaults(self, value: RegistryDefaults) -> None: 

151 if value.run is not None: 

152 self.registerRun(value.run) 

153 value.finish(self) 

154 self._defaults = value 

155 

156 @abstractmethod 

157 def refresh(self) -> None: 

158 """Refresh all in-memory state by querying the database. 

159 

160 This may be necessary to enable querying for entities added by other 

161 registry instances after this one was constructed. 

162 """ 

163 raise NotImplementedError() 

164 

165 @contextlib.contextmanager 

166 @abstractmethod 

167 def transaction(self, *, savepoint: bool = False) -> Iterator[None]: 

168 """Return a context manager that represents a transaction.""" 

169 raise NotImplementedError() 

170 

171 def resetConnectionPool(self) -> None: 

172 """Reset connection pool for registry if relevant. 

173 

174 This operation can be used reset connections to servers when 

175 using registry with fork-based multiprocessing. This method should 

176 usually be called by the child process immediately 

177 after the fork. 

178 

179 The base class implementation is a no-op. 

180 """ 

181 pass 

182 

183 @abstractmethod 

184 def registerCollection( 

185 self, name: str, type: CollectionType = CollectionType.TAGGED, doc: str | None = None 

186 ) -> bool: 

187 """Add a new collection if one with the given name does not exist. 

188 

189 Parameters 

190 ---------- 

191 name : `str` 

192 The name of the collection to create. 

193 type : `CollectionType` 

194 Enum value indicating the type of collection to create. 

195 doc : `str`, optional 

196 Documentation string for the collection. 

197 

198 Returns 

199 ------- 

200 registered : `bool` 

201 Boolean indicating whether the collection was already registered 

202 or was created by this call. 

203 

204 Notes 

205 ----- 

206 This method cannot be called within transactions, as it needs to be 

207 able to perform its own transaction to be concurrent. 

208 """ 

209 raise NotImplementedError() 

210 

211 @abstractmethod 

212 def getCollectionType(self, name: str) -> CollectionType: 

213 """Return an enumeration value indicating the type of the given 

214 collection. 

215 

216 Parameters 

217 ---------- 

218 name : `str` 

219 The name of the collection. 

220 

221 Returns 

222 ------- 

223 type : `CollectionType` 

224 Enum value indicating the type of this collection. 

225 

226 Raises 

227 ------ 

228 lsst.daf.butler.registry.MissingCollectionError 

229 Raised if no collection with the given name exists. 

230 """ 

231 raise NotImplementedError() 

232 

233 @abstractmethod 

234 def registerRun(self, name: str, doc: str | None = None) -> bool: 

235 """Add a new run if one with the given name does not exist. 

236 

237 Parameters 

238 ---------- 

239 name : `str` 

240 The name of the run to create. 

241 doc : `str`, optional 

242 Documentation string for the collection. 

243 

244 Returns 

245 ------- 

246 registered : `bool` 

247 Boolean indicating whether a new run was registered. `False` 

248 if it already existed. 

249 

250 Notes 

251 ----- 

252 This method cannot be called within transactions, as it needs to be 

253 able to perform its own transaction to be concurrent. 

254 """ 

255 raise NotImplementedError() 

256 

257 @abstractmethod 

258 def removeCollection(self, name: str) -> None: 

259 """Remove the given collection from the registry. 

260 

261 Parameters 

262 ---------- 

263 name : `str` 

264 The name of the collection to remove. 

265 

266 Raises 

267 ------ 

268 lsst.daf.butler.registry.MissingCollectionError 

269 Raised if no collection with the given name exists. 

270 sqlalchemy.exc.IntegrityError 

271 Raised if the database rows associated with the collection are 

272 still referenced by some other table, such as a dataset in a 

273 datastore (for `~CollectionType.RUN` collections only) or a 

274 `~CollectionType.CHAINED` collection of which this collection is 

275 a child. 

276 

277 Notes 

278 ----- 

279 If this is a `~CollectionType.RUN` collection, all datasets and quanta 

280 in it will removed from the `Registry` database. This requires that 

281 those datasets be removed (or at least trashed) from any datastores 

282 that hold them first. 

283 

284 A collection may not be deleted as long as it is referenced by a 

285 `~CollectionType.CHAINED` collection; the ``CHAINED`` collection must 

286 be deleted or redefined first. 

287 """ 

288 raise NotImplementedError() 

289 

290 @abstractmethod 

291 def getCollectionChain(self, parent: str) -> Sequence[str]: 

292 """Return the child collections in a `~CollectionType.CHAINED` 

293 collection. 

294 

295 Parameters 

296 ---------- 

297 parent : `str` 

298 Name of the chained collection. Must have already been added via 

299 a call to `Registry.registerCollection`. 

300 

301 Returns 

302 ------- 

303 children : `~collections.abc.Sequence` [ `str` ] 

304 An ordered sequence of collection names that are searched when the 

305 given chained collection is searched. 

306 

307 Raises 

308 ------ 

309 lsst.daf.butler.registry.MissingCollectionError 

310 Raised if ``parent`` does not exist in the `Registry`. 

311 lsst.daf.butler.registry.CollectionTypeError 

312 Raised if ``parent`` does not correspond to a 

313 `~CollectionType.CHAINED` collection. 

314 """ 

315 raise NotImplementedError() 

316 

317 @abstractmethod 

318 def setCollectionChain(self, parent: str, children: Any, *, flatten: bool = False) -> None: 

319 """Define or redefine a `~CollectionType.CHAINED` collection. 

320 

321 Parameters 

322 ---------- 

323 parent : `str` 

324 Name of the chained collection. Must have already been added via 

325 a call to `Registry.registerCollection`. 

326 children : collection expression 

327 An expression defining an ordered search of child collections, 

328 generally an iterable of `str`; see 

329 :ref:`daf_butler_collection_expressions` for more information. 

330 flatten : `bool`, optional 

331 If `True` (`False` is default), recursively flatten out any nested 

332 `~CollectionType.CHAINED` collections in ``children`` first. 

333 

334 Raises 

335 ------ 

336 lsst.daf.butler.registry.MissingCollectionError 

337 Raised when any of the given collections do not exist in the 

338 `Registry`. 

339 lsst.daf.butler.registry.CollectionTypeError 

340 Raised if ``parent`` does not correspond to a 

341 `~CollectionType.CHAINED` collection. 

342 ValueError 

343 Raised if the given collections contains a cycle. 

344 """ 

345 raise NotImplementedError() 

346 

347 @abstractmethod 

348 def getCollectionParentChains(self, collection: str) -> set[str]: 

349 """Return the CHAINED collections that directly contain the given one. 

350 

351 Parameters 

352 ---------- 

353 name : `str` 

354 Name of the collection. 

355 

356 Returns 

357 ------- 

358 chains : `set` of `str` 

359 Set of `~CollectionType.CHAINED` collection names. 

360 """ 

361 raise NotImplementedError() 

362 

363 @abstractmethod 

364 def getCollectionDocumentation(self, collection: str) -> str | None: 

365 """Retrieve the documentation string for a collection. 

366 

367 Parameters 

368 ---------- 

369 name : `str` 

370 Name of the collection. 

371 

372 Returns 

373 ------- 

374 docs : `str` or `None` 

375 Docstring for the collection with the given name. 

376 """ 

377 raise NotImplementedError() 

378 

379 @abstractmethod 

380 def setCollectionDocumentation(self, collection: str, doc: str | None) -> None: 

381 """Set the documentation string for a collection. 

382 

383 Parameters 

384 ---------- 

385 name : `str` 

386 Name of the collection. 

387 docs : `str` or `None` 

388 Docstring for the collection with the given name; will replace any 

389 existing docstring. Passing `None` will remove any existing 

390 docstring. 

391 """ 

392 raise NotImplementedError() 

393 

394 @abstractmethod 

395 def getCollectionSummary(self, collection: str) -> CollectionSummary: 

396 """Return a summary for the given collection. 

397 

398 Parameters 

399 ---------- 

400 collection : `str` 

401 Name of the collection for which a summary is to be retrieved. 

402 

403 Returns 

404 ------- 

405 summary : `~lsst.daf.butler.registry.CollectionSummary` 

406 Summary of the dataset types and governor dimension values in 

407 this collection. 

408 """ 

409 raise NotImplementedError() 

410 

411 @abstractmethod 

412 def registerDatasetType(self, datasetType: DatasetType) -> bool: 

413 """Add a new `DatasetType` to the Registry. 

414 

415 It is not an error to register the same `DatasetType` twice. 

416 

417 Parameters 

418 ---------- 

419 datasetType : `DatasetType` 

420 The `DatasetType` to be added. 

421 

422 Returns 

423 ------- 

424 inserted : `bool` 

425 `True` if ``datasetType`` was inserted, `False` if an identical 

426 existing `DatasetType` was found. Note that in either case the 

427 DatasetType is guaranteed to be defined in the Registry 

428 consistently with the given definition. 

429 

430 Raises 

431 ------ 

432 ValueError 

433 Raised if the dimensions or storage class are invalid. 

434 lsst.daf.butler.registry.ConflictingDefinitionError 

435 Raised if this `DatasetType` is already registered with a different 

436 definition. 

437 

438 Notes 

439 ----- 

440 This method cannot be called within transactions, as it needs to be 

441 able to perform its own transaction to be concurrent. 

442 """ 

443 raise NotImplementedError() 

444 

445 @abstractmethod 

446 def removeDatasetType(self, name: str | tuple[str, ...]) -> None: 

447 """Remove the named `DatasetType` from the registry. 

448 

449 .. warning:: 

450 

451 Registry implementations can cache the dataset type definitions. 

452 This means that deleting the dataset type definition may result in 

453 unexpected behavior from other butler processes that are active 

454 that have not seen the deletion. 

455 

456 Parameters 

457 ---------- 

458 name : `str` or `tuple` [`str`] 

459 Name of the type to be removed or tuple containing a list of type 

460 names to be removed. Wildcards are allowed. 

461 

462 Raises 

463 ------ 

464 lsst.daf.butler.registry.OrphanedRecordError 

465 Raised if an attempt is made to remove the dataset type definition 

466 when there are already datasets associated with it. 

467 

468 Notes 

469 ----- 

470 If the dataset type is not registered the method will return without 

471 action. 

472 """ 

473 raise NotImplementedError() 

474 

475 @abstractmethod 

476 def getDatasetType(self, name: str) -> DatasetType: 

477 """Get the `DatasetType`. 

478 

479 Parameters 

480 ---------- 

481 name : `str` 

482 Name of the type. 

483 

484 Returns 

485 ------- 

486 type : `DatasetType` 

487 The `DatasetType` associated with the given name. 

488 

489 Raises 

490 ------ 

491 lsst.daf.butler.registry.MissingDatasetTypeError 

492 Raised if the requested dataset type has not been registered. 

493 

494 Notes 

495 ----- 

496 This method handles component dataset types automatically, though most 

497 other registry operations do not. 

498 """ 

499 raise NotImplementedError() 

500 

501 @abstractmethod 

502 def supportsIdGenerationMode(self, mode: DatasetIdGenEnum) -> bool: 

503 """Test whether the given dataset ID generation mode is supported by 

504 `insertDatasets`. 

505 

506 Parameters 

507 ---------- 

508 mode : `DatasetIdGenEnum` 

509 Enum value for the mode to test. 

510 

511 Returns 

512 ------- 

513 supported : `bool` 

514 Whether the given mode is supported. 

515 """ 

516 raise NotImplementedError() 

517 

518 @abstractmethod 

519 def findDataset( 

520 self, 

521 datasetType: DatasetType | str, 

522 dataId: DataId | None = None, 

523 *, 

524 collections: CollectionArgType | None = None, 

525 timespan: Timespan | None = None, 

526 **kwargs: Any, 

527 ) -> DatasetRef | None: 

528 """Find a dataset given its `DatasetType` and data ID. 

529 

530 This can be used to obtain a `DatasetRef` that permits the dataset to 

531 be read from a `Datastore`. If the dataset is a component and can not 

532 be found using the provided dataset type, a dataset ref for the parent 

533 will be returned instead but with the correct dataset type. 

534 

535 Parameters 

536 ---------- 

537 datasetType : `DatasetType` or `str` 

538 A `DatasetType` or the name of one. If this is a `DatasetType` 

539 instance, its storage class will be respected and propagated to 

540 the output, even if it differs from the dataset type definition 

541 in the registry, as long as the storage classes are convertible. 

542 dataId : `dict` or `DataCoordinate`, optional 

543 A `dict`-like object containing the `Dimension` links that identify 

544 the dataset within a collection. 

545 collections : collection expression, optional 

546 An expression that fully or partially identifies the collections to 

547 search for the dataset; see 

548 :ref:`daf_butler_collection_expressions` for more information. 

549 Defaults to ``self.defaults.collections``. 

550 timespan : `Timespan`, optional 

551 A timespan that the validity range of the dataset must overlap. 

552 If not provided, any `~CollectionType.CALIBRATION` collections 

553 matched by the ``collections`` argument will not be searched. 

554 **kwargs 

555 Additional keyword arguments passed to 

556 `DataCoordinate.standardize` to convert ``dataId`` to a true 

557 `DataCoordinate` or augment an existing one. 

558 

559 Returns 

560 ------- 

561 ref : `DatasetRef` 

562 A reference to the dataset, or `None` if no matching Dataset 

563 was found. 

564 

565 Raises 

566 ------ 

567 lsst.daf.butler.registry.NoDefaultCollectionError 

568 Raised if ``collections`` is `None` and 

569 ``self.defaults.collections`` is `None`. 

570 LookupError 

571 Raised if one or more data ID keys are missing. 

572 lsst.daf.butler.registry.MissingDatasetTypeError 

573 Raised if the dataset type does not exist. 

574 lsst.daf.butler.registry.MissingCollectionError 

575 Raised if any of ``collections`` does not exist in the registry. 

576 

577 Notes 

578 ----- 

579 This method simply returns `None` and does not raise an exception even 

580 when the set of collections searched is intrinsically incompatible with 

581 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but 

582 only `~CollectionType.CALIBRATION` collections are being searched. 

583 This may make it harder to debug some lookup failures, but the behavior 

584 is intentional; we consider it more important that failed searches are 

585 reported consistently, regardless of the reason, and that adding 

586 additional collections that do not contain a match to the search path 

587 never changes the behavior. 

588 

589 This method handles component dataset types automatically, though most 

590 other registry operations do not. 

591 """ 

592 raise NotImplementedError() 

593 

594 @abstractmethod 

595 def insertDatasets( 

596 self, 

597 datasetType: DatasetType | str, 

598 dataIds: Iterable[DataId], 

599 run: str | None = None, 

600 expand: bool = True, 

601 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE, 

602 ) -> list[DatasetRef]: 

603 """Insert one or more datasets into the `Registry`. 

604 

605 This always adds new datasets; to associate existing datasets with 

606 a new collection, use ``associate``. 

607 

608 Parameters 

609 ---------- 

610 datasetType : `DatasetType` or `str` 

611 A `DatasetType` or the name of one. 

612 dataIds : `~collections.abc.Iterable` of `dict` or `DataCoordinate` 

613 Dimension-based identifiers for the new datasets. 

614 run : `str`, optional 

615 The name of the run that produced the datasets. Defaults to 

616 ``self.defaults.run``. 

617 expand : `bool`, optional 

618 If `True` (default), expand data IDs as they are inserted. This is 

619 necessary in general to allow datastore to generate file templates, 

620 but it may be disabled if the caller can guarantee this is 

621 unnecessary. 

622 idGenerationMode : `DatasetIdGenEnum`, optional 

623 Specifies option for generating dataset IDs. By default unique IDs 

624 are generated for each inserted dataset. 

625 

626 Returns 

627 ------- 

628 refs : `list` of `DatasetRef` 

629 Resolved `DatasetRef` instances for all given data IDs (in the same 

630 order). 

631 

632 Raises 

633 ------ 

634 lsst.daf.butler.registry.DatasetTypeError 

635 Raised if ``datasetType`` is not known to registry. 

636 lsst.daf.butler.registry.CollectionTypeError 

637 Raised if ``run`` collection type is not `~CollectionType.RUN`. 

638 lsst.daf.butler.registry.NoDefaultCollectionError 

639 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

640 lsst.daf.butler.registry.ConflictingDefinitionError 

641 If a dataset with the same dataset type and data ID as one of those 

642 given already exists in ``run``. 

643 lsst.daf.butler.registry.MissingCollectionError 

644 Raised if ``run`` does not exist in the registry. 

645 """ 

646 raise NotImplementedError() 

647 

648 @abstractmethod 

649 def _importDatasets( 

650 self, 

651 datasets: Iterable[DatasetRef], 

652 expand: bool = True, 

653 ) -> list[DatasetRef]: 

654 """Import one or more datasets into the `Registry`. 

655 

656 Difference from `insertDatasets` method is that this method accepts 

657 `DatasetRef` instances which should already be resolved and have a 

658 dataset ID. If registry supports globally-unique dataset IDs (e.g. 

659 `uuid.UUID`) then datasets which already exist in the registry will be 

660 ignored if imported again. 

661 

662 Parameters 

663 ---------- 

664 datasets : `~collections.abc.Iterable` of `DatasetRef` 

665 Datasets to be inserted. All `DatasetRef` instances must have 

666 identical ``datasetType`` and ``run`` attributes. ``run`` 

667 attribute can be `None` and defaults to ``self.defaults.run``. 

668 Datasets can specify ``id`` attribute which will be used for 

669 inserted datasets. All dataset IDs must have the same type 

670 (`int` or `uuid.UUID`), if type of dataset IDs does not match 

671 configured backend then IDs will be ignored and new IDs will be 

672 generated by backend. 

673 expand : `bool`, optional 

674 If `True` (default), expand data IDs as they are inserted. This is 

675 necessary in general to allow datastore to generate file templates, 

676 but it may be disabled if the caller can guarantee this is 

677 unnecessary. 

678 

679 Returns 

680 ------- 

681 refs : `list` of `DatasetRef` 

682 Resolved `DatasetRef` instances for all given data IDs (in the same 

683 order). If any of ``datasets`` has an ID which already exists in 

684 the database then it will not be inserted or updated, but a 

685 resolved `DatasetRef` will be returned for it in any case. 

686 

687 Raises 

688 ------ 

689 lsst.daf.butler.registry.NoDefaultCollectionError 

690 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

691 lsst.daf.butler.registry.DatasetTypeError 

692 Raised if datasets correspond to more than one dataset type or 

693 dataset type is not known to registry. 

694 lsst.daf.butler.registry.ConflictingDefinitionError 

695 If a dataset with the same dataset type and data ID as one of those 

696 given already exists in ``run``. 

697 lsst.daf.butler.registry.MissingCollectionError 

698 Raised if ``run`` does not exist in the registry. 

699 

700 Notes 

701 ----- 

702 This method is considered package-private and internal to Butler 

703 implementation. Clients outside daf_butler package should not use this 

704 method. 

705 """ 

706 raise NotImplementedError() 

707 

708 @abstractmethod 

709 def getDataset(self, id: DatasetId) -> DatasetRef | None: 

710 """Retrieve a Dataset entry. 

711 

712 Parameters 

713 ---------- 

714 id : `DatasetId` 

715 The unique identifier for the dataset. 

716 

717 Returns 

718 ------- 

719 ref : `DatasetRef` or `None` 

720 A ref to the Dataset, or `None` if no matching Dataset 

721 was found. 

722 """ 

723 raise NotImplementedError() 

724 

725 @abstractmethod 

726 def removeDatasets(self, refs: Iterable[DatasetRef]) -> None: 

727 """Remove datasets from the Registry. 

728 

729 The datasets will be removed unconditionally from all collections, and 

730 any `Quantum` that consumed this dataset will instead be marked with 

731 having a NULL input. `Datastore` records will *not* be deleted; the 

732 caller is responsible for ensuring that the dataset has already been 

733 removed from all Datastores. 

734 

735 Parameters 

736 ---------- 

737 refs : `~collections.abc.Iterable` [`DatasetRef`] 

738 References to the datasets to be removed. Must include a valid 

739 ``id`` attribute, and should be considered invalidated upon return. 

740 

741 Raises 

742 ------ 

743 lsst.daf.butler.AmbiguousDatasetError 

744 Raised if any ``ref.id`` is `None`. 

745 lsst.daf.butler.registry.OrphanedRecordError 

746 Raised if any dataset is still present in any `Datastore`. 

747 """ 

748 raise NotImplementedError() 

749 

750 @abstractmethod 

751 def associate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

752 """Add existing datasets to a `~CollectionType.TAGGED` collection. 

753 

754 If a DatasetRef with the same exact ID is already in a collection 

755 nothing is changed. If a `DatasetRef` with the same `DatasetType` and 

756 data ID but with different ID exists in the collection, 

757 `~lsst.daf.butler.registry.ConflictingDefinitionError` is raised. 

758 

759 Parameters 

760 ---------- 

761 collection : `str` 

762 Indicates the collection the datasets should be associated with. 

763 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

764 An iterable of resolved `DatasetRef` instances that already exist 

765 in this `Registry`. 

766 

767 Raises 

768 ------ 

769 lsst.daf.butler.registry.ConflictingDefinitionError 

770 If a Dataset with the given `DatasetRef` already exists in the 

771 given collection. 

772 lsst.daf.butler.registry.MissingCollectionError 

773 Raised if ``collection`` does not exist in the registry. 

774 lsst.daf.butler.registry.CollectionTypeError 

775 Raise adding new datasets to the given ``collection`` is not 

776 allowed. 

777 """ 

778 raise NotImplementedError() 

779 

780 @abstractmethod 

781 def disassociate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

782 """Remove existing datasets from a `~CollectionType.TAGGED` collection. 

783 

784 ``collection`` and ``ref`` combinations that are not currently 

785 associated are silently ignored. 

786 

787 Parameters 

788 ---------- 

789 collection : `str` 

790 The collection the datasets should no longer be associated with. 

791 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

792 An iterable of resolved `DatasetRef` instances that already exist 

793 in this `Registry`. 

794 

795 Raises 

796 ------ 

797 lsst.daf.butler.AmbiguousDatasetError 

798 Raised if any of the given dataset references is unresolved. 

799 lsst.daf.butler.registry.MissingCollectionError 

800 Raised if ``collection`` does not exist in the registry. 

801 lsst.daf.butler.registry.CollectionTypeError 

802 Raise adding new datasets to the given ``collection`` is not 

803 allowed. 

804 """ 

805 raise NotImplementedError() 

806 

807 @abstractmethod 

808 def certify(self, collection: str, refs: Iterable[DatasetRef], timespan: Timespan) -> None: 

809 """Associate one or more datasets with a calibration collection and a 

810 validity range within it. 

811 

812 Parameters 

813 ---------- 

814 collection : `str` 

815 The name of an already-registered `~CollectionType.CALIBRATION` 

816 collection. 

817 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

818 Datasets to be associated. 

819 timespan : `Timespan` 

820 The validity range for these datasets within the collection. 

821 

822 Raises 

823 ------ 

824 lsst.daf.butler.AmbiguousDatasetError 

825 Raised if any of the given `DatasetRef` instances is unresolved. 

826 lsst.daf.butler.registry.ConflictingDefinitionError 

827 Raised if the collection already contains a different dataset with 

828 the same `DatasetType` and data ID and an overlapping validity 

829 range. 

830 lsst.daf.butler.registry.CollectionTypeError 

831 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

832 collection or if one or more datasets are of a dataset type for 

833 which `DatasetType.isCalibration` returns `False`. 

834 """ 

835 raise NotImplementedError() 

836 

837 @abstractmethod 

838 def decertify( 

839 self, 

840 collection: str, 

841 datasetType: str | DatasetType, 

842 timespan: Timespan, 

843 *, 

844 dataIds: Iterable[DataId] | None = None, 

845 ) -> None: 

846 """Remove or adjust datasets to clear a validity range within a 

847 calibration collection. 

848 

849 Parameters 

850 ---------- 

851 collection : `str` 

852 The name of an already-registered `~CollectionType.CALIBRATION` 

853 collection. 

854 datasetType : `str` or `DatasetType` 

855 Name or `DatasetType` instance for the datasets to be decertified. 

856 timespan : `Timespan`, optional 

857 The validity range to remove datasets from within the collection. 

858 Datasets that overlap this range but are not contained by it will 

859 have their validity ranges adjusted to not overlap it, which may 

860 split a single dataset validity range into two. 

861 dataIds : iterable [`dict` or `DataCoordinate`], optional 

862 Data IDs that should be decertified within the given validity range 

863 If `None`, all data IDs for ``self.datasetType`` will be 

864 decertified. 

865 

866 Raises 

867 ------ 

868 lsst.daf.butler.registry.CollectionTypeError 

869 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

870 collection or if ``datasetType.isCalibration() is False``. 

871 """ 

872 raise NotImplementedError() 

873 

874 @abstractmethod 

875 def getDatasetLocations(self, ref: DatasetRef) -> Iterable[str]: 

876 """Retrieve datastore locations for a given dataset. 

877 

878 Parameters 

879 ---------- 

880 ref : `DatasetRef` 

881 A reference to the dataset for which to retrieve storage 

882 information. 

883 

884 Returns 

885 ------- 

886 datastores : `~collections.abc.Iterable` [ `str` ] 

887 All the matching datastores holding this dataset. 

888 

889 Raises 

890 ------ 

891 lsst.daf.butler.AmbiguousDatasetError 

892 Raised if ``ref.id`` is `None`. 

893 """ 

894 raise NotImplementedError() 

895 

896 @abstractmethod 

897 def expandDataId( 

898 self, 

899 dataId: DataId | None = None, 

900 *, 

901 graph: DimensionGraph | None = None, 

902 records: NameLookupMapping[DimensionElement, DimensionRecord | None] | None = None, 

903 withDefaults: bool = True, 

904 **kwargs: Any, 

905 ) -> DataCoordinate: 

906 """Expand a dimension-based data ID to include additional information. 

907 

908 Parameters 

909 ---------- 

910 dataId : `DataCoordinate` or `dict`, optional 

911 Data ID to be expanded; augmented and overridden by ``kwargs``. 

912 graph : `DimensionGraph`, optional 

913 Set of dimensions for the expanded ID. If `None`, the dimensions 

914 will be inferred from the keys of ``dataId`` and ``kwargs``. 

915 Dimensions that are in ``dataId`` or ``kwargs`` but not in 

916 ``graph`` are silently ignored, providing a way to extract and 

917 ``graph`` expand a subset of a data ID. 

918 records : `~collections.abc.Mapping` [`str`, `DimensionRecord`], \ 

919 optional 

920 Dimension record data to use before querying the database for that 

921 data, keyed by element name. 

922 withDefaults : `bool`, optional 

923 Utilize ``self.defaults.dataId`` to fill in missing governor 

924 dimension key-value pairs. Defaults to `True` (i.e. defaults are 

925 used). 

926 **kwargs 

927 Additional keywords are treated like additional key-value pairs for 

928 ``dataId``, extending and overriding 

929 

930 Returns 

931 ------- 

932 expanded : `DataCoordinate` 

933 A data ID that includes full metadata for all of the dimensions it 

934 identifies, i.e. guarantees that ``expanded.hasRecords()`` and 

935 ``expanded.hasFull()`` both return `True`. 

936 

937 Raises 

938 ------ 

939 lsst.daf.butler.registry.DataIdError 

940 Raised when ``dataId`` or keyword arguments specify unknown 

941 dimensions or values, or when a resulting data ID contains 

942 contradictory key-value pairs, according to dimension 

943 relationships. 

944 

945 Notes 

946 ----- 

947 This method cannot be relied upon to reject invalid data ID values 

948 for dimensions that do actually not have any record columns. For 

949 efficiency reasons the records for these dimensions (which have only 

950 dimension key values that are given by the caller) may be constructed 

951 directly rather than obtained from the registry database. 

952 """ 

953 raise NotImplementedError() 

954 

955 @abstractmethod 

956 def insertDimensionData( 

957 self, 

958 element: DimensionElement | str, 

959 *data: Mapping[str, Any] | DimensionRecord, 

960 conform: bool = True, 

961 replace: bool = False, 

962 skip_existing: bool = False, 

963 ) -> None: 

964 """Insert one or more dimension records into the database. 

965 

966 Parameters 

967 ---------- 

968 element : `DimensionElement` or `str` 

969 The `DimensionElement` or name thereof that identifies the table 

970 records will be inserted into. 

971 *data : `dict` or `DimensionRecord` 

972 One or more records to insert. 

973 conform : `bool`, optional 

974 If `False` (`True` is default) perform no checking or conversions, 

975 and assume that ``element`` is a `DimensionElement` instance and 

976 ``data`` is a one or more `DimensionRecord` instances of the 

977 appropriate subclass. 

978 replace : `bool`, optional 

979 If `True` (`False` is default), replace existing records in the 

980 database if there is a conflict. 

981 skip_existing : `bool`, optional 

982 If `True` (`False` is default), skip insertion if a record with 

983 the same primary key values already exists. Unlike 

984 `syncDimensionData`, this will not detect when the given record 

985 differs from what is in the database, and should not be used when 

986 this is a concern. 

987 """ 

988 raise NotImplementedError() 

989 

990 @abstractmethod 

991 def syncDimensionData( 

992 self, 

993 element: DimensionElement | str, 

994 row: Mapping[str, Any] | DimensionRecord, 

995 conform: bool = True, 

996 update: bool = False, 

997 ) -> bool | dict[str, Any]: 

998 """Synchronize the given dimension record with the database, inserting 

999 if it does not already exist and comparing values if it does. 

1000 

1001 Parameters 

1002 ---------- 

1003 element : `DimensionElement` or `str` 

1004 The `DimensionElement` or name thereof that identifies the table 

1005 records will be inserted into. 

1006 row : `dict` or `DimensionRecord` 

1007 The record to insert. 

1008 conform : `bool`, optional 

1009 If `False` (`True` is default) perform no checking or conversions, 

1010 and assume that ``element`` is a `DimensionElement` instance and 

1011 ``data`` is a one or more `DimensionRecord` instances of the 

1012 appropriate subclass. 

1013 update : `bool`, optional 

1014 If `True` (`False` is default), update the existing record in the 

1015 database if there is a conflict. 

1016 

1017 Returns 

1018 ------- 

1019 inserted_or_updated : `bool` or `dict` 

1020 `True` if a new row was inserted, `False` if no changes were 

1021 needed, or a `dict` mapping updated column names to their old 

1022 values if an update was performed (only possible if 

1023 ``update=True``). 

1024 

1025 Raises 

1026 ------ 

1027 lsst.daf.butler.registry.ConflictingDefinitionError 

1028 Raised if the record exists in the database (according to primary 

1029 key lookup) but is inconsistent with the given one. 

1030 """ 

1031 raise NotImplementedError() 

1032 

1033 @abstractmethod 

1034 def queryDatasetTypes( 

1035 self, 

1036 expression: Any = ..., 

1037 *, 

1038 components: bool | None = False, 

1039 missing: list[str] | None = None, 

1040 ) -> Iterable[DatasetType]: 

1041 """Iterate over the dataset types whose names match an expression. 

1042 

1043 Parameters 

1044 ---------- 

1045 expression : dataset type expression, optional 

1046 An expression that fully or partially identifies the dataset types 

1047 to return, such as a `str`, `re.Pattern`, or iterable thereof. 

1048 ``...`` can be used to return all dataset types, and is the 

1049 default. See :ref:`daf_butler_dataset_type_expressions` for more 

1050 information. 

1051 components : `bool`, optional 

1052 If `True`, apply all expression patterns to component dataset type 

1053 names as well. If `False`, never apply patterns to components. 

1054 If `None`, apply patterns to components only if their 

1055 parent datasets were not matched by the expression. 

1056 Fully-specified component datasets (`str` or `DatasetType` 

1057 instances) are always included. 

1058 

1059 Values other than `False` are deprecated, and only `False` will be 

1060 supported after v26. After v27 this argument will be removed 

1061 entirely. 

1062 missing : `list` of `str`, optional 

1063 String dataset type names that were explicitly given (i.e. not 

1064 regular expression patterns) but not found will be appended to this 

1065 list, if it is provided. 

1066 

1067 Returns 

1068 ------- 

1069 dataset_types : `~collections.abc.Iterable` [ `DatasetType`] 

1070 An `~collections.abc.Iterable` of `DatasetType` instances whose 

1071 names match ``expression``. 

1072 

1073 Raises 

1074 ------ 

1075 lsst.daf.butler.registry.DatasetTypeExpressionError 

1076 Raised when ``expression`` is invalid. 

1077 """ 

1078 raise NotImplementedError() 

1079 

1080 @abstractmethod 

1081 def queryCollections( 

1082 self, 

1083 expression: Any = ..., 

1084 datasetType: DatasetType | None = None, 

1085 collectionTypes: Iterable[CollectionType] | CollectionType = CollectionType.all(), 

1086 flattenChains: bool = False, 

1087 includeChains: bool | None = None, 

1088 ) -> Sequence[str]: 

1089 """Iterate over the collections whose names match an expression. 

1090 

1091 Parameters 

1092 ---------- 

1093 expression : collection expression, optional 

1094 An expression that identifies the collections to return, such as 

1095 a `str` (for full matches or partial matches via globs), 

1096 `re.Pattern` (for partial matches), or iterable thereof. ``...`` 

1097 can be used to return all collections, and is the default. 

1098 See :ref:`daf_butler_collection_expressions` for more information. 

1099 datasetType : `DatasetType`, optional 

1100 If provided, only yield collections that may contain datasets of 

1101 this type. This is a conservative approximation in general; it may 

1102 yield collections that do not have any such datasets. 

1103 collectionTypes : `~collections.abc.Set` [`CollectionType`] or \ 

1104 `CollectionType`, optional 

1105 If provided, only yield collections of these types. 

1106 flattenChains : `bool`, optional 

1107 If `True` (`False` is default), recursively yield the child 

1108 collections of matching `~CollectionType.CHAINED` collections. 

1109 includeChains : `bool`, optional 

1110 If `True`, yield records for matching `~CollectionType.CHAINED` 

1111 collections. Default is the opposite of ``flattenChains``: include 

1112 either CHAINED collections or their children, but not both. 

1113 

1114 Returns 

1115 ------- 

1116 collections : `~collections.abc.Sequence` [ `str` ] 

1117 The names of collections that match ``expression``. 

1118 

1119 Raises 

1120 ------ 

1121 lsst.daf.butler.registry.CollectionExpressionError 

1122 Raised when ``expression`` is invalid. 

1123 

1124 Notes 

1125 ----- 

1126 The order in which collections are returned is unspecified, except that 

1127 the children of a `~CollectionType.CHAINED` collection are guaranteed 

1128 to be in the order in which they are searched. When multiple parent 

1129 `~CollectionType.CHAINED` collections match the same criteria, the 

1130 order in which the two lists appear is unspecified, and the lists of 

1131 children may be incomplete if a child has multiple parents. 

1132 """ 

1133 raise NotImplementedError() 

1134 

1135 @abstractmethod 

1136 def queryDatasets( 

1137 self, 

1138 datasetType: Any, 

1139 *, 

1140 collections: CollectionArgType | None = None, 

1141 dimensions: Iterable[Dimension | str] | None = None, 

1142 dataId: DataId | None = None, 

1143 where: str = "", 

1144 findFirst: bool = False, 

1145 components: bool | None = False, 

1146 bind: Mapping[str, Any] | None = None, 

1147 check: bool = True, 

1148 **kwargs: Any, 

1149 ) -> DatasetQueryResults: 

1150 """Query for and iterate over dataset references matching user-provided 

1151 criteria. 

1152 

1153 Parameters 

1154 ---------- 

1155 datasetType : dataset type expression 

1156 An expression that fully or partially identifies the dataset types 

1157 to be queried. Allowed types include `DatasetType`, `str`, 

1158 `re.Pattern`, and iterables thereof. The special value ``...`` can 

1159 be used to query all dataset types. See 

1160 :ref:`daf_butler_dataset_type_expressions` for more information. 

1161 collections : collection expression, optional 

1162 An expression that identifies the collections to search, such as a 

1163 `str` (for full matches or partial matches via globs), `re.Pattern` 

1164 (for partial matches), or iterable thereof. ``...`` can be used to 

1165 search all collections (actually just all `~CollectionType.RUN` 

1166 collections, because this will still find all datasets). 

1167 If not provided, ``self.default.collections`` is used. See 

1168 :ref:`daf_butler_collection_expressions` for more information. 

1169 dimensions : `~collections.abc.Iterable` of `Dimension` or `str` 

1170 Dimensions to include in the query (in addition to those used 

1171 to identify the queried dataset type(s)), either to constrain 

1172 the resulting datasets to those for which a matching dimension 

1173 exists, or to relate the dataset type's dimensions to dimensions 

1174 referenced by the ``dataId`` or ``where`` arguments. 

1175 dataId : `dict` or `DataCoordinate`, optional 

1176 A data ID whose key-value pairs are used as equality constraints 

1177 in the query. 

1178 where : `str`, optional 

1179 A string expression similar to a SQL WHERE clause. May involve 

1180 any column of a dimension table or (as a shortcut for the primary 

1181 key column of a dimension table) dimension name. See 

1182 :ref:`daf_butler_dimension_expressions` for more information. 

1183 findFirst : `bool`, optional 

1184 If `True` (`False` is default), for each result data ID, only 

1185 yield one `DatasetRef` of each `DatasetType`, from the first 

1186 collection in which a dataset of that dataset type appears 

1187 (according to the order of ``collections`` passed in). If `True`, 

1188 ``collections`` must not contain regular expressions and may not 

1189 be ``...``. 

1190 components : `bool`, optional 

1191 If `True`, apply all dataset expression patterns to component 

1192 dataset type names as well. If `False`, never apply patterns to 

1193 components. If `None`, apply patterns to components only 

1194 if their parent datasets were not matched by the expression. 

1195 Fully-specified component datasets (`str` or `DatasetType` 

1196 instances) are always included. 

1197 

1198 Values other than `False` are deprecated, and only `False` will be 

1199 supported after v26. After v27 this argument will be removed 

1200 entirely. 

1201 bind : `~collections.abc.Mapping`, optional 

1202 Mapping containing literal values that should be injected into the 

1203 ``where`` expression, keyed by the identifiers they replace. 

1204 Values of collection type can be expanded in some cases; see 

1205 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1206 information. 

1207 check : `bool`, optional 

1208 If `True` (default) check the query for consistency before 

1209 executing it. This may reject some valid queries that resemble 

1210 common mistakes (e.g. queries for visits without specifying an 

1211 instrument). 

1212 **kwargs 

1213 Additional keyword arguments are forwarded to 

1214 `DataCoordinate.standardize` when processing the ``dataId`` 

1215 argument (and may be used to provide a constraining data ID even 

1216 when the ``dataId`` argument is `None`). 

1217 

1218 Returns 

1219 ------- 

1220 refs : `.queries.DatasetQueryResults` 

1221 Dataset references matching the given query criteria. Nested data 

1222 IDs are guaranteed to include values for all implied dimensions 

1223 (i.e. `DataCoordinate.hasFull` will return `True`), but will not 

1224 include dimension records (`DataCoordinate.hasRecords` will be 

1225 `False`) unless `~.queries.DatasetQueryResults.expanded` is 

1226 called on the result object (which returns a new one). 

1227 

1228 Raises 

1229 ------ 

1230 lsst.daf.butler.registry.DatasetTypeExpressionError 

1231 Raised when ``datasetType`` expression is invalid. 

1232 TypeError 

1233 Raised when the arguments are incompatible, such as when a 

1234 collection wildcard is passed when ``findFirst`` is `True`, or 

1235 when ``collections`` is `None` and ``self.defaults.collections`` is 

1236 also `None`. 

1237 lsst.daf.butler.registry.DataIdError 

1238 Raised when ``dataId`` or keyword arguments specify unknown 

1239 dimensions or values, or when they contain inconsistent values. 

1240 lsst.daf.butler.registry.UserExpressionError 

1241 Raised when ``where`` expression is invalid. 

1242 

1243 Notes 

1244 ----- 

1245 When multiple dataset types are queried in a single call, the 

1246 results of this operation are equivalent to querying for each dataset 

1247 type separately in turn, and no information about the relationships 

1248 between datasets of different types is included. In contexts where 

1249 that kind of information is important, the recommended pattern is to 

1250 use `queryDataIds` to first obtain data IDs (possibly with the 

1251 desired dataset types and collections passed as constraints to the 

1252 query), and then use multiple (generally much simpler) calls to 

1253 `queryDatasets` with the returned data IDs passed as constraints. 

1254 """ 

1255 raise NotImplementedError() 

1256 

1257 @abstractmethod 

1258 def queryDataIds( 

1259 self, 

1260 dimensions: Iterable[Dimension | str] | Dimension | str, 

1261 *, 

1262 dataId: DataId | None = None, 

1263 datasets: Any = None, 

1264 collections: CollectionArgType | None = None, 

1265 where: str = "", 

1266 components: bool | None = False, 

1267 bind: Mapping[str, Any] | None = None, 

1268 check: bool = True, 

1269 **kwargs: Any, 

1270 ) -> DataCoordinateQueryResults: 

1271 """Query for data IDs matching user-provided criteria. 

1272 

1273 Parameters 

1274 ---------- 

1275 dimensions : `Dimension` or `str`, or iterable thereof 

1276 The dimensions of the data IDs to yield, as either `Dimension` 

1277 instances or `str`. Will be automatically expanded to a complete 

1278 `DimensionGraph`. 

1279 dataId : `dict` or `DataCoordinate`, optional 

1280 A data ID whose key-value pairs are used as equality constraints 

1281 in the query. 

1282 datasets : dataset type expression, optional 

1283 An expression that fully or partially identifies dataset types 

1284 that should constrain the yielded data IDs. For example, including 

1285 "raw" here would constrain the yielded ``instrument``, 

1286 ``exposure``, ``detector``, and ``physical_filter`` values to only 

1287 those for which at least one "raw" dataset exists in 

1288 ``collections``. Allowed types include `DatasetType`, `str`, 

1289 and iterables thereof. Regular expression objects (i.e. 

1290 `re.Pattern`) are deprecated and will be removed after the v26 

1291 release. See :ref:`daf_butler_dataset_type_expressions` for more 

1292 information. 

1293 collections : collection expression, optional 

1294 An expression that identifies the collections to search for 

1295 datasets, such as a `str` (for full matches or partial matches 

1296 via globs), `re.Pattern` (for partial matches), or iterable 

1297 thereof. ``...`` can be used to search all collections (actually 

1298 just all `~CollectionType.RUN` collections, because this will 

1299 still find all datasets). If not provided, 

1300 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1301 is also passed. See :ref:`daf_butler_collection_expressions` for 

1302 more information. 

1303 where : `str`, optional 

1304 A string expression similar to a SQL WHERE clause. May involve 

1305 any column of a dimension table or (as a shortcut for the primary 

1306 key column of a dimension table) dimension name. See 

1307 :ref:`daf_butler_dimension_expressions` for more information. 

1308 components : `bool`, optional 

1309 If `True`, apply all dataset expression patterns to component 

1310 dataset type names as well. If `False`, never apply patterns to 

1311 components. If `None`, apply patterns to components only 

1312 if their parent datasets were not matched by the expression. 

1313 Fully-specified component datasets (`str` or `DatasetType` 

1314 instances) are always included. 

1315 

1316 Values other than `False` are deprecated, and only `False` will be 

1317 supported after v26. After v27 this argument will be removed 

1318 entirely. 

1319 bind : `~collections.abc.Mapping`, optional 

1320 Mapping containing literal values that should be injected into the 

1321 ``where`` expression, keyed by the identifiers they replace. 

1322 Values of collection type can be expanded in some cases; see 

1323 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1324 information. 

1325 check : `bool`, optional 

1326 If `True` (default) check the query for consistency before 

1327 executing it. This may reject some valid queries that resemble 

1328 common mistakes (e.g. queries for visits without specifying an 

1329 instrument). 

1330 **kwargs 

1331 Additional keyword arguments are forwarded to 

1332 `DataCoordinate.standardize` when processing the ``dataId`` 

1333 argument (and may be used to provide a constraining data ID even 

1334 when the ``dataId`` argument is `None`). 

1335 

1336 Returns 

1337 ------- 

1338 dataIds : `.queries.DataCoordinateQueryResults` 

1339 Data IDs matching the given query parameters. These are guaranteed 

1340 to identify all dimensions (`DataCoordinate.hasFull` returns 

1341 `True`), but will not contain `DimensionRecord` objects 

1342 (`DataCoordinate.hasRecords` returns `False`). Call 

1343 `~.queries.DataCoordinateQueryResults.expanded` on the 

1344 returned object to fetch those (and consider using 

1345 `~.queries.DataCoordinateQueryResults.materialize` on the 

1346 returned object first if the expected number of rows is very 

1347 large). See documentation for those methods for additional 

1348 information. 

1349 

1350 Raises 

1351 ------ 

1352 lsst.daf.butler.registry.NoDefaultCollectionError 

1353 Raised if ``collections`` is `None` and 

1354 ``self.defaults.collections`` is `None`. 

1355 lsst.daf.butler.registry.CollectionExpressionError 

1356 Raised when ``collections`` expression is invalid. 

1357 lsst.daf.butler.registry.DataIdError 

1358 Raised when ``dataId`` or keyword arguments specify unknown 

1359 dimensions or values, or when they contain inconsistent values. 

1360 lsst.daf.butler.registry.DatasetTypeExpressionError 

1361 Raised when ``datasetType`` expression is invalid. 

1362 lsst.daf.butler.registry.UserExpressionError 

1363 Raised when ``where`` expression is invalid. 

1364 """ 

1365 raise NotImplementedError() 

1366 

1367 @abstractmethod 

1368 def queryDimensionRecords( 

1369 self, 

1370 element: DimensionElement | str, 

1371 *, 

1372 dataId: DataId | None = None, 

1373 datasets: Any = None, 

1374 collections: CollectionArgType | None = None, 

1375 where: str = "", 

1376 components: bool | None = False, 

1377 bind: Mapping[str, Any] | None = None, 

1378 check: bool = True, 

1379 **kwargs: Any, 

1380 ) -> DimensionRecordQueryResults: 

1381 """Query for dimension information matching user-provided criteria. 

1382 

1383 Parameters 

1384 ---------- 

1385 element : `DimensionElement` or `str` 

1386 The dimension element to obtain records for. 

1387 dataId : `dict` or `DataCoordinate`, optional 

1388 A data ID whose key-value pairs are used as equality constraints 

1389 in the query. 

1390 datasets : dataset type expression, optional 

1391 An expression that fully or partially identifies dataset types 

1392 that should constrain the yielded records. See `queryDataIds` and 

1393 :ref:`daf_butler_dataset_type_expressions` for more information. 

1394 collections : collection expression, optional 

1395 An expression that identifies the collections to search for 

1396 datasets, such as a `str` (for full matches or partial matches 

1397 via globs), `re.Pattern` (for partial matches), or iterable 

1398 thereof. ``...`` can be used to search all collections (actually 

1399 just all `~CollectionType.RUN` collections, because this will 

1400 still find all datasets). If not provided, 

1401 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1402 is also passed. See :ref:`daf_butler_collection_expressions` for 

1403 more information. 

1404 where : `str`, optional 

1405 A string expression similar to a SQL WHERE clause. See 

1406 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more 

1407 information. 

1408 components : `bool`, optional 

1409 Whether to apply dataset expressions to components as well. 

1410 See `queryDataIds` for more information. 

1411 

1412 Values other than `False` are deprecated, and only `False` will be 

1413 supported after v26. After v27 this argument will be removed 

1414 entirely. 

1415 bind : `~collections.abc.Mapping`, optional 

1416 Mapping containing literal values that should be injected into the 

1417 ``where`` expression, keyed by the identifiers they replace. 

1418 Values of collection type can be expanded in some cases; see 

1419 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1420 information. 

1421 check : `bool`, optional 

1422 If `True` (default) check the query for consistency before 

1423 executing it. This may reject some valid queries that resemble 

1424 common mistakes (e.g. queries for visits without specifying an 

1425 instrument). 

1426 **kwargs 

1427 Additional keyword arguments are forwarded to 

1428 `DataCoordinate.standardize` when processing the ``dataId`` 

1429 argument (and may be used to provide a constraining data ID even 

1430 when the ``dataId`` argument is `None`). 

1431 

1432 Returns 

1433 ------- 

1434 dataIds : `.queries.DimensionRecordQueryResults` 

1435 Data IDs matching the given query parameters. 

1436 

1437 Raises 

1438 ------ 

1439 lsst.daf.butler.registry.NoDefaultCollectionError 

1440 Raised if ``collections`` is `None` and 

1441 ``self.defaults.collections`` is `None`. 

1442 lsst.daf.butler.registry.CollectionExpressionError 

1443 Raised when ``collections`` expression is invalid. 

1444 lsst.daf.butler.registry.DataIdError 

1445 Raised when ``dataId`` or keyword arguments specify unknown 

1446 dimensions or values, or when they contain inconsistent values. 

1447 lsst.daf.butler.registry.DatasetTypeExpressionError 

1448 Raised when ``datasetType`` expression is invalid. 

1449 lsst.daf.butler.registry.UserExpressionError 

1450 Raised when ``where`` expression is invalid. 

1451 """ 

1452 raise NotImplementedError() 

1453 

1454 @abstractmethod 

1455 def queryDatasetAssociations( 

1456 self, 

1457 datasetType: str | DatasetType, 

1458 collections: CollectionArgType | None = ..., 

1459 *, 

1460 collectionTypes: Iterable[CollectionType] = CollectionType.all(), 

1461 flattenChains: bool = False, 

1462 ) -> Iterator[DatasetAssociation]: 

1463 """Iterate over dataset-collection combinations where the dataset is in 

1464 the collection. 

1465 

1466 This method is a temporary placeholder for better support for 

1467 association results in `queryDatasets`. It will probably be 

1468 removed in the future, and should be avoided in production code 

1469 whenever possible. 

1470 

1471 Parameters 

1472 ---------- 

1473 datasetType : `DatasetType` or `str` 

1474 A dataset type object or the name of one. 

1475 collections : collection expression, optional 

1476 An expression that identifies the collections to search for 

1477 datasets, such as a `str` (for full matches or partial matches 

1478 via globs), `re.Pattern` (for partial matches), or iterable 

1479 thereof. ``...`` can be used to search all collections (actually 

1480 just all `~CollectionType.RUN` collections, because this will still 

1481 find all datasets). If not provided, ``self.default.collections`` 

1482 is used. See :ref:`daf_butler_collection_expressions` for more 

1483 information. 

1484 collectionTypes : `~collections.abc.Set` [ `CollectionType` ], optional 

1485 If provided, only yield associations from collections of these 

1486 types. 

1487 flattenChains : `bool`, optional 

1488 If `True`, search in the children of `~CollectionType.CHAINED` 

1489 collections. If `False`, ``CHAINED`` collections are ignored. 

1490 

1491 Yields 

1492 ------ 

1493 association : `.DatasetAssociation` 

1494 Object representing the relationship between a single dataset and 

1495 a single collection. 

1496 

1497 Raises 

1498 ------ 

1499 lsst.daf.butler.registry.NoDefaultCollectionError 

1500 Raised if ``collections`` is `None` and 

1501 ``self.defaults.collections`` is `None`. 

1502 lsst.daf.butler.registry.CollectionExpressionError 

1503 Raised when ``collections`` expression is invalid. 

1504 """ 

1505 raise NotImplementedError() 

1506 

1507 @property 

1508 def obsCoreTableManager(self) -> ObsCoreTableManager | None: 

1509 """The ObsCore manager instance for this registry 

1510 (`~.interfaces.ObsCoreTableManager` 

1511 or `None`). 

1512 

1513 ObsCore manager may not be implemented for all registry backend, or 

1514 may not be enabled for many repositories. 

1515 """ 

1516 return None 

1517 

1518 storageClasses: StorageClassFactory 

1519 """All storage classes known to the registry (`StorageClassFactory`). 

1520 """