Coverage for python/lsst/daf/butler/registry/_registry.py: 95%

116 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-21 09:55 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("Registry",) 

25 

26import contextlib 

27import logging 

28import re 

29from abc import ABC, abstractmethod 

30from collections.abc import Iterable, Iterator, Mapping, Sequence 

31from types import EllipsisType 

32from typing import TYPE_CHECKING, Any 

33 

34from lsst.resources import ResourcePathExpression 

35 

36from ..core import ( 

37 DataCoordinate, 

38 DataId, 

39 DatasetAssociation, 

40 DatasetId, 

41 DatasetIdGenEnum, 

42 DatasetRef, 

43 DatasetType, 

44 Dimension, 

45 DimensionConfig, 

46 DimensionElement, 

47 DimensionGraph, 

48 DimensionRecord, 

49 DimensionUniverse, 

50 NameLookupMapping, 

51 StorageClassFactory, 

52 Timespan, 

53) 

54from ._collection_summary import CollectionSummary 

55from ._collectionType import CollectionType 

56from ._config import RegistryConfig 

57from ._defaults import RegistryDefaults 

58from .queries import DataCoordinateQueryResults, DatasetQueryResults, DimensionRecordQueryResults 

59from .wildcards import CollectionWildcard 

60 

61if TYPE_CHECKING: 

62 from .interfaces import ObsCoreTableManager 

63 

64_LOG = logging.getLogger(__name__) 

65 

66# TYpe alias for `collections` arguments. 

67CollectionArgType = str | re.Pattern | Iterable[str | re.Pattern] | EllipsisType | CollectionWildcard 

68 

69 

70class Registry(ABC): 

71 """Abstract Registry interface. 

72 

73 All subclasses should store `~lsst.daf.butler.registry.RegistryDefaults` in 

74 a ``_defaults`` property. No other properties are assumed shared between 

75 implementations. 

76 """ 

77 

78 @classmethod 

79 def createFromConfig( 

80 cls, 

81 config: RegistryConfig | str | None = None, 

82 dimensionConfig: DimensionConfig | str | None = None, 

83 butlerRoot: ResourcePathExpression | None = None, 

84 ) -> Registry: 

85 """Create registry database and return `Registry` instance. 

86 

87 This method initializes database contents, database must be empty 

88 prior to calling this method. 

89 

90 Parameters 

91 ---------- 

92 config : `RegistryConfig` or `str`, optional 

93 Registry configuration, if missing then default configuration will 

94 be loaded from registry.yaml. 

95 dimensionConfig : `DimensionConfig` or `str`, optional 

96 Dimensions configuration, if missing then default configuration 

97 will be loaded from dimensions.yaml. 

98 butlerRoot : convertible to `lsst.resources.ResourcePath`, optional 

99 Path to the repository root this `Registry` will manage. 

100 

101 Returns 

102 ------- 

103 registry : `Registry` 

104 A new `Registry` instance. 

105 

106 Notes 

107 ----- 

108 This method is for backward compatibility only, until all clients 

109 migrate to use new `~lsst.daf.butler.registry._RegistryFactory` factory 

110 class. Regular clients of registry class do not use this method, it is 

111 only used by tests in multiple packages. 

112 """ 

113 from ._registry_factory import _RegistryFactory 

114 

115 return _RegistryFactory(config).create_from_config(dimensionConfig, butlerRoot) 

116 

117 @abstractmethod 

118 def isWriteable(self) -> bool: 

119 """Return `True` if this registry allows write operations, and `False` 

120 otherwise. 

121 """ 

122 raise NotImplementedError() 

123 

124 @property 

125 @abstractmethod 

126 def dimensions(self) -> DimensionUniverse: 

127 """Definitions of all dimensions recognized by this `Registry` 

128 (`DimensionUniverse`). 

129 """ 

130 raise NotImplementedError() 

131 

132 @property 

133 def defaults(self) -> RegistryDefaults: 

134 """Default collection search path and/or output `~CollectionType.RUN` 

135 collection (`~lsst.daf.butler.registry.RegistryDefaults`). 

136 

137 This is an immutable struct whose components may not be set 

138 individually, but the entire struct can be set by assigning to this 

139 property. 

140 """ 

141 return self._defaults 

142 

143 @defaults.setter 

144 def defaults(self, value: RegistryDefaults) -> None: 

145 if value.run is not None: 

146 self.registerRun(value.run) 

147 value.finish(self) 

148 self._defaults = value 

149 

150 @abstractmethod 

151 def refresh(self) -> None: 

152 """Refresh all in-memory state by querying the database. 

153 

154 This may be necessary to enable querying for entities added by other 

155 registry instances after this one was constructed. 

156 """ 

157 raise NotImplementedError() 

158 

159 @contextlib.contextmanager 

160 @abstractmethod 

161 def transaction(self, *, savepoint: bool = False) -> Iterator[None]: 

162 """Return a context manager that represents a transaction.""" 

163 raise NotImplementedError() 

164 

165 def resetConnectionPool(self) -> None: 

166 """Reset connection pool for registry if relevant. 

167 

168 This operation can be used reset connections to servers when 

169 using registry with fork-based multiprocessing. This method should 

170 usually be called by the child process immediately 

171 after the fork. 

172 

173 The base class implementation is a no-op. 

174 """ 

175 pass 

176 

177 @abstractmethod 

178 def registerCollection( 

179 self, name: str, type: CollectionType = CollectionType.TAGGED, doc: str | None = None 

180 ) -> bool: 

181 """Add a new collection if one with the given name does not exist. 

182 

183 Parameters 

184 ---------- 

185 name : `str` 

186 The name of the collection to create. 

187 type : `CollectionType` 

188 Enum value indicating the type of collection to create. 

189 doc : `str`, optional 

190 Documentation string for the collection. 

191 

192 Returns 

193 ------- 

194 registered : `bool` 

195 Boolean indicating whether the collection was already registered 

196 or was created by this call. 

197 

198 Notes 

199 ----- 

200 This method cannot be called within transactions, as it needs to be 

201 able to perform its own transaction to be concurrent. 

202 """ 

203 raise NotImplementedError() 

204 

205 @abstractmethod 

206 def getCollectionType(self, name: str) -> CollectionType: 

207 """Return an enumeration value indicating the type of the given 

208 collection. 

209 

210 Parameters 

211 ---------- 

212 name : `str` 

213 The name of the collection. 

214 

215 Returns 

216 ------- 

217 type : `CollectionType` 

218 Enum value indicating the type of this collection. 

219 

220 Raises 

221 ------ 

222 lsst.daf.butler.registry.MissingCollectionError 

223 Raised if no collection with the given name exists. 

224 """ 

225 raise NotImplementedError() 

226 

227 @abstractmethod 

228 def registerRun(self, name: str, doc: str | None = None) -> bool: 

229 """Add a new run if one with the given name does not exist. 

230 

231 Parameters 

232 ---------- 

233 name : `str` 

234 The name of the run to create. 

235 doc : `str`, optional 

236 Documentation string for the collection. 

237 

238 Returns 

239 ------- 

240 registered : `bool` 

241 Boolean indicating whether a new run was registered. `False` 

242 if it already existed. 

243 

244 Notes 

245 ----- 

246 This method cannot be called within transactions, as it needs to be 

247 able to perform its own transaction to be concurrent. 

248 """ 

249 raise NotImplementedError() 

250 

251 @abstractmethod 

252 def removeCollection(self, name: str) -> None: 

253 """Remove the given collection from the registry. 

254 

255 Parameters 

256 ---------- 

257 name : `str` 

258 The name of the collection to remove. 

259 

260 Raises 

261 ------ 

262 lsst.daf.butler.registry.MissingCollectionError 

263 Raised if no collection with the given name exists. 

264 sqlalchemy.exc.IntegrityError 

265 Raised if the database rows associated with the collection are 

266 still referenced by some other table, such as a dataset in a 

267 datastore (for `~CollectionType.RUN` collections only) or a 

268 `~CollectionType.CHAINED` collection of which this collection is 

269 a child. 

270 

271 Notes 

272 ----- 

273 If this is a `~CollectionType.RUN` collection, all datasets and quanta 

274 in it will removed from the `Registry` database. This requires that 

275 those datasets be removed (or at least trashed) from any datastores 

276 that hold them first. 

277 

278 A collection may not be deleted as long as it is referenced by a 

279 `~CollectionType.CHAINED` collection; the ``CHAINED`` collection must 

280 be deleted or redefined first. 

281 """ 

282 raise NotImplementedError() 

283 

284 @abstractmethod 

285 def getCollectionChain(self, parent: str) -> Sequence[str]: 

286 """Return the child collections in a `~CollectionType.CHAINED` 

287 collection. 

288 

289 Parameters 

290 ---------- 

291 parent : `str` 

292 Name of the chained collection. Must have already been added via 

293 a call to `Registry.registerCollection`. 

294 

295 Returns 

296 ------- 

297 children : `~collections.abc.Sequence` [ `str` ] 

298 An ordered sequence of collection names that are searched when the 

299 given chained collection is searched. 

300 

301 Raises 

302 ------ 

303 lsst.daf.butler.registry.MissingCollectionError 

304 Raised if ``parent`` does not exist in the `Registry`. 

305 lsst.daf.butler.registry.CollectionTypeError 

306 Raised if ``parent`` does not correspond to a 

307 `~CollectionType.CHAINED` collection. 

308 """ 

309 raise NotImplementedError() 

310 

311 @abstractmethod 

312 def setCollectionChain(self, parent: str, children: Any, *, flatten: bool = False) -> None: 

313 """Define or redefine a `~CollectionType.CHAINED` collection. 

314 

315 Parameters 

316 ---------- 

317 parent : `str` 

318 Name of the chained collection. Must have already been added via 

319 a call to `Registry.registerCollection`. 

320 children : collection expression 

321 An expression defining an ordered search of child collections, 

322 generally an iterable of `str`; see 

323 :ref:`daf_butler_collection_expressions` for more information. 

324 flatten : `bool`, optional 

325 If `True` (`False` is default), recursively flatten out any nested 

326 `~CollectionType.CHAINED` collections in ``children`` first. 

327 

328 Raises 

329 ------ 

330 lsst.daf.butler.registry.MissingCollectionError 

331 Raised when any of the given collections do not exist in the 

332 `Registry`. 

333 lsst.daf.butler.registry.CollectionTypeError 

334 Raised if ``parent`` does not correspond to a 

335 `~CollectionType.CHAINED` collection. 

336 ValueError 

337 Raised if the given collections contains a cycle. 

338 """ 

339 raise NotImplementedError() 

340 

341 @abstractmethod 

342 def getCollectionParentChains(self, collection: str) -> set[str]: 

343 """Return the CHAINED collections that directly contain the given one. 

344 

345 Parameters 

346 ---------- 

347 name : `str` 

348 Name of the collection. 

349 

350 Returns 

351 ------- 

352 chains : `set` of `str` 

353 Set of `~CollectionType.CHAINED` collection names. 

354 """ 

355 raise NotImplementedError() 

356 

357 @abstractmethod 

358 def getCollectionDocumentation(self, collection: str) -> str | None: 

359 """Retrieve the documentation string for a collection. 

360 

361 Parameters 

362 ---------- 

363 name : `str` 

364 Name of the collection. 

365 

366 Returns 

367 ------- 

368 docs : `str` or `None` 

369 Docstring for the collection with the given name. 

370 """ 

371 raise NotImplementedError() 

372 

373 @abstractmethod 

374 def setCollectionDocumentation(self, collection: str, doc: str | None) -> None: 

375 """Set the documentation string for a collection. 

376 

377 Parameters 

378 ---------- 

379 name : `str` 

380 Name of the collection. 

381 docs : `str` or `None` 

382 Docstring for the collection with the given name; will replace any 

383 existing docstring. Passing `None` will remove any existing 

384 docstring. 

385 """ 

386 raise NotImplementedError() 

387 

388 @abstractmethod 

389 def getCollectionSummary(self, collection: str) -> CollectionSummary: 

390 """Return a summary for the given collection. 

391 

392 Parameters 

393 ---------- 

394 collection : `str` 

395 Name of the collection for which a summary is to be retrieved. 

396 

397 Returns 

398 ------- 

399 summary : `~lsst.daf.butler.registry.CollectionSummary` 

400 Summary of the dataset types and governor dimension values in 

401 this collection. 

402 """ 

403 raise NotImplementedError() 

404 

405 @abstractmethod 

406 def registerDatasetType(self, datasetType: DatasetType) -> bool: 

407 """Add a new `DatasetType` to the Registry. 

408 

409 It is not an error to register the same `DatasetType` twice. 

410 

411 Parameters 

412 ---------- 

413 datasetType : `DatasetType` 

414 The `DatasetType` to be added. 

415 

416 Returns 

417 ------- 

418 inserted : `bool` 

419 `True` if ``datasetType`` was inserted, `False` if an identical 

420 existing `DatasetType` was found. Note that in either case the 

421 DatasetType is guaranteed to be defined in the Registry 

422 consistently with the given definition. 

423 

424 Raises 

425 ------ 

426 ValueError 

427 Raised if the dimensions or storage class are invalid. 

428 lsst.daf.butler.registry.ConflictingDefinitionError 

429 Raised if this `DatasetType` is already registered with a different 

430 definition. 

431 

432 Notes 

433 ----- 

434 This method cannot be called within transactions, as it needs to be 

435 able to perform its own transaction to be concurrent. 

436 """ 

437 raise NotImplementedError() 

438 

439 @abstractmethod 

440 def removeDatasetType(self, name: str | tuple[str, ...]) -> None: 

441 """Remove the named `DatasetType` from the registry. 

442 

443 .. warning:: 

444 

445 Registry implementations can cache the dataset type definitions. 

446 This means that deleting the dataset type definition may result in 

447 unexpected behavior from other butler processes that are active 

448 that have not seen the deletion. 

449 

450 Parameters 

451 ---------- 

452 name : `str` or `tuple` [`str`] 

453 Name of the type to be removed or tuple containing a list of type 

454 names to be removed. Wildcards are allowed. 

455 

456 Raises 

457 ------ 

458 lsst.daf.butler.registry.OrphanedRecordError 

459 Raised if an attempt is made to remove the dataset type definition 

460 when there are already datasets associated with it. 

461 

462 Notes 

463 ----- 

464 If the dataset type is not registered the method will return without 

465 action. 

466 """ 

467 raise NotImplementedError() 

468 

469 @abstractmethod 

470 def getDatasetType(self, name: str) -> DatasetType: 

471 """Get the `DatasetType`. 

472 

473 Parameters 

474 ---------- 

475 name : `str` 

476 Name of the type. 

477 

478 Returns 

479 ------- 

480 type : `DatasetType` 

481 The `DatasetType` associated with the given name. 

482 

483 Raises 

484 ------ 

485 lsst.daf.butler.registry.MissingDatasetTypeError 

486 Raised if the requested dataset type has not been registered. 

487 

488 Notes 

489 ----- 

490 This method handles component dataset types automatically, though most 

491 other registry operations do not. 

492 """ 

493 raise NotImplementedError() 

494 

495 @abstractmethod 

496 def supportsIdGenerationMode(self, mode: DatasetIdGenEnum) -> bool: 

497 """Test whether the given dataset ID generation mode is supported by 

498 `insertDatasets`. 

499 

500 Parameters 

501 ---------- 

502 mode : `DatasetIdGenEnum` 

503 Enum value for the mode to test. 

504 

505 Returns 

506 ------- 

507 supported : `bool` 

508 Whether the given mode is supported. 

509 """ 

510 raise NotImplementedError() 

511 

512 @abstractmethod 

513 def findDataset( 

514 self, 

515 datasetType: DatasetType | str, 

516 dataId: DataId | None = None, 

517 *, 

518 collections: CollectionArgType | None = None, 

519 timespan: Timespan | None = None, 

520 **kwargs: Any, 

521 ) -> DatasetRef | None: 

522 """Find a dataset given its `DatasetType` and data ID. 

523 

524 This can be used to obtain a `DatasetRef` that permits the dataset to 

525 be read from a `Datastore`. If the dataset is a component and can not 

526 be found using the provided dataset type, a dataset ref for the parent 

527 will be returned instead but with the correct dataset type. 

528 

529 Parameters 

530 ---------- 

531 datasetType : `DatasetType` or `str` 

532 A `DatasetType` or the name of one. If this is a `DatasetType` 

533 instance, its storage class will be respected and propagated to 

534 the output, even if it differs from the dataset type definition 

535 in the registry, as long as the storage classes are convertible. 

536 dataId : `dict` or `DataCoordinate`, optional 

537 A `dict`-like object containing the `Dimension` links that identify 

538 the dataset within a collection. 

539 collections : collection expression, optional 

540 An expression that fully or partially identifies the collections to 

541 search for the dataset; see 

542 :ref:`daf_butler_collection_expressions` for more information. 

543 Defaults to ``self.defaults.collections``. 

544 timespan : `Timespan`, optional 

545 A timespan that the validity range of the dataset must overlap. 

546 If not provided, any `~CollectionType.CALIBRATION` collections 

547 matched by the ``collections`` argument will not be searched. 

548 **kwargs 

549 Additional keyword arguments passed to 

550 `DataCoordinate.standardize` to convert ``dataId`` to a true 

551 `DataCoordinate` or augment an existing one. 

552 

553 Returns 

554 ------- 

555 ref : `DatasetRef` 

556 A reference to the dataset, or `None` if no matching Dataset 

557 was found. 

558 

559 Raises 

560 ------ 

561 lsst.daf.butler.registry.NoDefaultCollectionError 

562 Raised if ``collections`` is `None` and 

563 ``self.defaults.collections`` is `None`. 

564 LookupError 

565 Raised if one or more data ID keys are missing. 

566 lsst.daf.butler.registry.MissingDatasetTypeError 

567 Raised if the dataset type does not exist. 

568 lsst.daf.butler.registry.MissingCollectionError 

569 Raised if any of ``collections`` does not exist in the registry. 

570 

571 Notes 

572 ----- 

573 This method simply returns `None` and does not raise an exception even 

574 when the set of collections searched is intrinsically incompatible with 

575 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but 

576 only `~CollectionType.CALIBRATION` collections are being searched. 

577 This may make it harder to debug some lookup failures, but the behavior 

578 is intentional; we consider it more important that failed searches are 

579 reported consistently, regardless of the reason, and that adding 

580 additional collections that do not contain a match to the search path 

581 never changes the behavior. 

582 

583 This method handles component dataset types automatically, though most 

584 other registry operations do not. 

585 """ 

586 raise NotImplementedError() 

587 

588 @abstractmethod 

589 def insertDatasets( 

590 self, 

591 datasetType: DatasetType | str, 

592 dataIds: Iterable[DataId], 

593 run: str | None = None, 

594 expand: bool = True, 

595 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE, 

596 ) -> list[DatasetRef]: 

597 """Insert one or more datasets into the `Registry`. 

598 

599 This always adds new datasets; to associate existing datasets with 

600 a new collection, use ``associate``. 

601 

602 Parameters 

603 ---------- 

604 datasetType : `DatasetType` or `str` 

605 A `DatasetType` or the name of one. 

606 dataIds : `~collections.abc.Iterable` of `dict` or `DataCoordinate` 

607 Dimension-based identifiers for the new datasets. 

608 run : `str`, optional 

609 The name of the run that produced the datasets. Defaults to 

610 ``self.defaults.run``. 

611 expand : `bool`, optional 

612 If `True` (default), expand data IDs as they are inserted. This is 

613 necessary in general to allow datastore to generate file templates, 

614 but it may be disabled if the caller can guarantee this is 

615 unnecessary. 

616 idGenerationMode : `DatasetIdGenEnum`, optional 

617 Specifies option for generating dataset IDs. By default unique IDs 

618 are generated for each inserted dataset. 

619 

620 Returns 

621 ------- 

622 refs : `list` of `DatasetRef` 

623 Resolved `DatasetRef` instances for all given data IDs (in the same 

624 order). 

625 

626 Raises 

627 ------ 

628 lsst.daf.butler.registry.DatasetTypeError 

629 Raised if ``datasetType`` is not known to registry. 

630 lsst.daf.butler.registry.CollectionTypeError 

631 Raised if ``run`` collection type is not `~CollectionType.RUN`. 

632 lsst.daf.butler.registry.NoDefaultCollectionError 

633 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

634 lsst.daf.butler.registry.ConflictingDefinitionError 

635 If a dataset with the same dataset type and data ID as one of those 

636 given already exists in ``run``. 

637 lsst.daf.butler.registry.MissingCollectionError 

638 Raised if ``run`` does not exist in the registry. 

639 """ 

640 raise NotImplementedError() 

641 

642 @abstractmethod 

643 def _importDatasets( 

644 self, 

645 datasets: Iterable[DatasetRef], 

646 expand: bool = True, 

647 ) -> list[DatasetRef]: 

648 """Import one or more datasets into the `Registry`. 

649 

650 Difference from `insertDatasets` method is that this method accepts 

651 `DatasetRef` instances which should already be resolved and have a 

652 dataset ID. If registry supports globally-unique dataset IDs (e.g. 

653 `uuid.UUID`) then datasets which already exist in the registry will be 

654 ignored if imported again. 

655 

656 Parameters 

657 ---------- 

658 datasets : `~collections.abc.Iterable` of `DatasetRef` 

659 Datasets to be inserted. All `DatasetRef` instances must have 

660 identical ``datasetType`` and ``run`` attributes. ``run`` 

661 attribute can be `None` and defaults to ``self.defaults.run``. 

662 Datasets can specify ``id`` attribute which will be used for 

663 inserted datasets. All dataset IDs must have the same type 

664 (`int` or `uuid.UUID`), if type of dataset IDs does not match 

665 configured backend then IDs will be ignored and new IDs will be 

666 generated by backend. 

667 expand : `bool`, optional 

668 If `True` (default), expand data IDs as they are inserted. This is 

669 necessary in general to allow datastore to generate file templates, 

670 but it may be disabled if the caller can guarantee this is 

671 unnecessary. 

672 

673 Returns 

674 ------- 

675 refs : `list` of `DatasetRef` 

676 Resolved `DatasetRef` instances for all given data IDs (in the same 

677 order). If any of ``datasets`` has an ID which already exists in 

678 the database then it will not be inserted or updated, but a 

679 resolved `DatasetRef` will be returned for it in any case. 

680 

681 Raises 

682 ------ 

683 lsst.daf.butler.registry.NoDefaultCollectionError 

684 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

685 lsst.daf.butler.registry.DatasetTypeError 

686 Raised if datasets correspond to more than one dataset type or 

687 dataset type is not known to registry. 

688 lsst.daf.butler.registry.ConflictingDefinitionError 

689 If a dataset with the same dataset type and data ID as one of those 

690 given already exists in ``run``. 

691 lsst.daf.butler.registry.MissingCollectionError 

692 Raised if ``run`` does not exist in the registry. 

693 

694 Notes 

695 ----- 

696 This method is considered package-private and internal to Butler 

697 implementation. Clients outside daf_butler package should not use this 

698 method. 

699 """ 

700 raise NotImplementedError() 

701 

702 @abstractmethod 

703 def getDataset(self, id: DatasetId) -> DatasetRef | None: 

704 """Retrieve a Dataset entry. 

705 

706 Parameters 

707 ---------- 

708 id : `DatasetId` 

709 The unique identifier for the dataset. 

710 

711 Returns 

712 ------- 

713 ref : `DatasetRef` or `None` 

714 A ref to the Dataset, or `None` if no matching Dataset 

715 was found. 

716 """ 

717 raise NotImplementedError() 

718 

719 @abstractmethod 

720 def removeDatasets(self, refs: Iterable[DatasetRef]) -> None: 

721 """Remove datasets from the Registry. 

722 

723 The datasets will be removed unconditionally from all collections, and 

724 any `Quantum` that consumed this dataset will instead be marked with 

725 having a NULL input. `Datastore` records will *not* be deleted; the 

726 caller is responsible for ensuring that the dataset has already been 

727 removed from all Datastores. 

728 

729 Parameters 

730 ---------- 

731 refs : `~collections.abc.Iterable` [`DatasetRef`] 

732 References to the datasets to be removed. Must include a valid 

733 ``id`` attribute, and should be considered invalidated upon return. 

734 

735 Raises 

736 ------ 

737 lsst.daf.butler.AmbiguousDatasetError 

738 Raised if any ``ref.id`` is `None`. 

739 lsst.daf.butler.registry.OrphanedRecordError 

740 Raised if any dataset is still present in any `Datastore`. 

741 """ 

742 raise NotImplementedError() 

743 

744 @abstractmethod 

745 def associate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

746 """Add existing datasets to a `~CollectionType.TAGGED` collection. 

747 

748 If a DatasetRef with the same exact ID is already in a collection 

749 nothing is changed. If a `DatasetRef` with the same `DatasetType` and 

750 data ID but with different ID exists in the collection, 

751 `~lsst.daf.butler.registry.ConflictingDefinitionError` is raised. 

752 

753 Parameters 

754 ---------- 

755 collection : `str` 

756 Indicates the collection the datasets should be associated with. 

757 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

758 An iterable of resolved `DatasetRef` instances that already exist 

759 in this `Registry`. 

760 

761 Raises 

762 ------ 

763 lsst.daf.butler.registry.ConflictingDefinitionError 

764 If a Dataset with the given `DatasetRef` already exists in the 

765 given collection. 

766 lsst.daf.butler.registry.MissingCollectionError 

767 Raised if ``collection`` does not exist in the registry. 

768 lsst.daf.butler.registry.CollectionTypeError 

769 Raise adding new datasets to the given ``collection`` is not 

770 allowed. 

771 """ 

772 raise NotImplementedError() 

773 

774 @abstractmethod 

775 def disassociate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

776 """Remove existing datasets from a `~CollectionType.TAGGED` collection. 

777 

778 ``collection`` and ``ref`` combinations that are not currently 

779 associated are silently ignored. 

780 

781 Parameters 

782 ---------- 

783 collection : `str` 

784 The collection the datasets should no longer be associated with. 

785 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

786 An iterable of resolved `DatasetRef` instances that already exist 

787 in this `Registry`. 

788 

789 Raises 

790 ------ 

791 lsst.daf.butler.AmbiguousDatasetError 

792 Raised if any of the given dataset references is unresolved. 

793 lsst.daf.butler.registry.MissingCollectionError 

794 Raised if ``collection`` does not exist in the registry. 

795 lsst.daf.butler.registry.CollectionTypeError 

796 Raise adding new datasets to the given ``collection`` is not 

797 allowed. 

798 """ 

799 raise NotImplementedError() 

800 

801 @abstractmethod 

802 def certify(self, collection: str, refs: Iterable[DatasetRef], timespan: Timespan) -> None: 

803 """Associate one or more datasets with a calibration collection and a 

804 validity range within it. 

805 

806 Parameters 

807 ---------- 

808 collection : `str` 

809 The name of an already-registered `~CollectionType.CALIBRATION` 

810 collection. 

811 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

812 Datasets to be associated. 

813 timespan : `Timespan` 

814 The validity range for these datasets within the collection. 

815 

816 Raises 

817 ------ 

818 lsst.daf.butler.AmbiguousDatasetError 

819 Raised if any of the given `DatasetRef` instances is unresolved. 

820 lsst.daf.butler.registry.ConflictingDefinitionError 

821 Raised if the collection already contains a different dataset with 

822 the same `DatasetType` and data ID and an overlapping validity 

823 range. 

824 lsst.daf.butler.registry.CollectionTypeError 

825 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

826 collection or if one or more datasets are of a dataset type for 

827 which `DatasetType.isCalibration` returns `False`. 

828 """ 

829 raise NotImplementedError() 

830 

831 @abstractmethod 

832 def decertify( 

833 self, 

834 collection: str, 

835 datasetType: str | DatasetType, 

836 timespan: Timespan, 

837 *, 

838 dataIds: Iterable[DataId] | None = None, 

839 ) -> None: 

840 """Remove or adjust datasets to clear a validity range within a 

841 calibration collection. 

842 

843 Parameters 

844 ---------- 

845 collection : `str` 

846 The name of an already-registered `~CollectionType.CALIBRATION` 

847 collection. 

848 datasetType : `str` or `DatasetType` 

849 Name or `DatasetType` instance for the datasets to be decertified. 

850 timespan : `Timespan`, optional 

851 The validity range to remove datasets from within the collection. 

852 Datasets that overlap this range but are not contained by it will 

853 have their validity ranges adjusted to not overlap it, which may 

854 split a single dataset validity range into two. 

855 dataIds : iterable [`dict` or `DataCoordinate`], optional 

856 Data IDs that should be decertified within the given validity range 

857 If `None`, all data IDs for ``self.datasetType`` will be 

858 decertified. 

859 

860 Raises 

861 ------ 

862 lsst.daf.butler.registry.CollectionTypeError 

863 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

864 collection or if ``datasetType.isCalibration() is False``. 

865 """ 

866 raise NotImplementedError() 

867 

868 @abstractmethod 

869 def getDatasetLocations(self, ref: DatasetRef) -> Iterable[str]: 

870 """Retrieve datastore locations for a given dataset. 

871 

872 Parameters 

873 ---------- 

874 ref : `DatasetRef` 

875 A reference to the dataset for which to retrieve storage 

876 information. 

877 

878 Returns 

879 ------- 

880 datastores : `~collections.abc.Iterable` [ `str` ] 

881 All the matching datastores holding this dataset. 

882 

883 Raises 

884 ------ 

885 lsst.daf.butler.AmbiguousDatasetError 

886 Raised if ``ref.id`` is `None`. 

887 """ 

888 raise NotImplementedError() 

889 

890 @abstractmethod 

891 def expandDataId( 

892 self, 

893 dataId: DataId | None = None, 

894 *, 

895 graph: DimensionGraph | None = None, 

896 records: NameLookupMapping[DimensionElement, DimensionRecord | None] | None = None, 

897 withDefaults: bool = True, 

898 **kwargs: Any, 

899 ) -> DataCoordinate: 

900 """Expand a dimension-based data ID to include additional information. 

901 

902 Parameters 

903 ---------- 

904 dataId : `DataCoordinate` or `dict`, optional 

905 Data ID to be expanded; augmented and overridden by ``kwargs``. 

906 graph : `DimensionGraph`, optional 

907 Set of dimensions for the expanded ID. If `None`, the dimensions 

908 will be inferred from the keys of ``dataId`` and ``kwargs``. 

909 Dimensions that are in ``dataId`` or ``kwargs`` but not in 

910 ``graph`` are silently ignored, providing a way to extract and 

911 ``graph`` expand a subset of a data ID. 

912 records : `~collections.abc.Mapping` [`str`, `DimensionRecord`], \ 

913 optional 

914 Dimension record data to use before querying the database for that 

915 data, keyed by element name. 

916 withDefaults : `bool`, optional 

917 Utilize ``self.defaults.dataId`` to fill in missing governor 

918 dimension key-value pairs. Defaults to `True` (i.e. defaults are 

919 used). 

920 **kwargs 

921 Additional keywords are treated like additional key-value pairs for 

922 ``dataId``, extending and overriding 

923 

924 Returns 

925 ------- 

926 expanded : `DataCoordinate` 

927 A data ID that includes full metadata for all of the dimensions it 

928 identifies, i.e. guarantees that ``expanded.hasRecords()`` and 

929 ``expanded.hasFull()`` both return `True`. 

930 

931 Raises 

932 ------ 

933 lsst.daf.butler.registry.DataIdError 

934 Raised when ``dataId`` or keyword arguments specify unknown 

935 dimensions or values, or when a resulting data ID contains 

936 contradictory key-value pairs, according to dimension 

937 relationships. 

938 

939 Notes 

940 ----- 

941 This method cannot be relied upon to reject invalid data ID values 

942 for dimensions that do actually not have any record columns. For 

943 efficiency reasons the records for these dimensions (which have only 

944 dimension key values that are given by the caller) may be constructed 

945 directly rather than obtained from the registry database. 

946 """ 

947 raise NotImplementedError() 

948 

949 @abstractmethod 

950 def insertDimensionData( 

951 self, 

952 element: DimensionElement | str, 

953 *data: Mapping[str, Any] | DimensionRecord, 

954 conform: bool = True, 

955 replace: bool = False, 

956 skip_existing: bool = False, 

957 ) -> None: 

958 """Insert one or more dimension records into the database. 

959 

960 Parameters 

961 ---------- 

962 element : `DimensionElement` or `str` 

963 The `DimensionElement` or name thereof that identifies the table 

964 records will be inserted into. 

965 *data : `dict` or `DimensionRecord` 

966 One or more records to insert. 

967 conform : `bool`, optional 

968 If `False` (`True` is default) perform no checking or conversions, 

969 and assume that ``element`` is a `DimensionElement` instance and 

970 ``data`` is a one or more `DimensionRecord` instances of the 

971 appropriate subclass. 

972 replace : `bool`, optional 

973 If `True` (`False` is default), replace existing records in the 

974 database if there is a conflict. 

975 skip_existing : `bool`, optional 

976 If `True` (`False` is default), skip insertion if a record with 

977 the same primary key values already exists. Unlike 

978 `syncDimensionData`, this will not detect when the given record 

979 differs from what is in the database, and should not be used when 

980 this is a concern. 

981 """ 

982 raise NotImplementedError() 

983 

984 @abstractmethod 

985 def syncDimensionData( 

986 self, 

987 element: DimensionElement | str, 

988 row: Mapping[str, Any] | DimensionRecord, 

989 conform: bool = True, 

990 update: bool = False, 

991 ) -> bool | dict[str, Any]: 

992 """Synchronize the given dimension record with the database, inserting 

993 if it does not already exist and comparing values if it does. 

994 

995 Parameters 

996 ---------- 

997 element : `DimensionElement` or `str` 

998 The `DimensionElement` or name thereof that identifies the table 

999 records will be inserted into. 

1000 row : `dict` or `DimensionRecord` 

1001 The record to insert. 

1002 conform : `bool`, optional 

1003 If `False` (`True` is default) perform no checking or conversions, 

1004 and assume that ``element`` is a `DimensionElement` instance and 

1005 ``data`` is a one or more `DimensionRecord` instances of the 

1006 appropriate subclass. 

1007 update : `bool`, optional 

1008 If `True` (`False` is default), update the existing record in the 

1009 database if there is a conflict. 

1010 

1011 Returns 

1012 ------- 

1013 inserted_or_updated : `bool` or `dict` 

1014 `True` if a new row was inserted, `False` if no changes were 

1015 needed, or a `dict` mapping updated column names to their old 

1016 values if an update was performed (only possible if 

1017 ``update=True``). 

1018 

1019 Raises 

1020 ------ 

1021 lsst.daf.butler.registry.ConflictingDefinitionError 

1022 Raised if the record exists in the database (according to primary 

1023 key lookup) but is inconsistent with the given one. 

1024 """ 

1025 raise NotImplementedError() 

1026 

1027 @abstractmethod 

1028 def queryDatasetTypes( 

1029 self, 

1030 expression: Any = ..., 

1031 *, 

1032 components: bool | None = None, 

1033 missing: list[str] | None = None, 

1034 ) -> Iterable[DatasetType]: 

1035 """Iterate over the dataset types whose names match an expression. 

1036 

1037 Parameters 

1038 ---------- 

1039 expression : dataset type expression, optional 

1040 An expression that fully or partially identifies the dataset types 

1041 to return, such as a `str`, `re.Pattern`, or iterable thereof. 

1042 ``...`` can be used to return all dataset types, and is the 

1043 default. See :ref:`daf_butler_dataset_type_expressions` for more 

1044 information. 

1045 components : `bool`, optional 

1046 If `True`, apply all expression patterns to component dataset type 

1047 names as well. If `False`, never apply patterns to components. 

1048 If `None` (default), apply patterns to components only if their 

1049 parent datasets were not matched by the expression. 

1050 Fully-specified component datasets (`str` or `DatasetType` 

1051 instances) are always included. 

1052 

1053 Values other than `False` are deprecated, and only `False` will be 

1054 supported after v26. After v27 this argument will be removed 

1055 entirely. 

1056 missing : `list` of `str`, optional 

1057 String dataset type names that were explicitly given (i.e. not 

1058 regular expression patterns) but not found will be appended to this 

1059 list, if it is provided. 

1060 

1061 Returns 

1062 ------- 

1063 dataset_types : `~collections.abc.Iterable` [ `DatasetType`] 

1064 An `~collections.abc.Iterable` of `DatasetType` instances whose 

1065 names match ``expression``. 

1066 

1067 Raises 

1068 ------ 

1069 lsst.daf.butler.registry.DatasetTypeExpressionError 

1070 Raised when ``expression`` is invalid. 

1071 """ 

1072 raise NotImplementedError() 

1073 

1074 @abstractmethod 

1075 def queryCollections( 

1076 self, 

1077 expression: Any = ..., 

1078 datasetType: DatasetType | None = None, 

1079 collectionTypes: Iterable[CollectionType] | CollectionType = CollectionType.all(), 

1080 flattenChains: bool = False, 

1081 includeChains: bool | None = None, 

1082 ) -> Sequence[str]: 

1083 """Iterate over the collections whose names match an expression. 

1084 

1085 Parameters 

1086 ---------- 

1087 expression : collection expression, optional 

1088 An expression that identifies the collections to return, such as 

1089 a `str` (for full matches or partial matches via globs), 

1090 `re.Pattern` (for partial matches), or iterable thereof. ``...`` 

1091 can be used to return all collections, and is the default. 

1092 See :ref:`daf_butler_collection_expressions` for more information. 

1093 datasetType : `DatasetType`, optional 

1094 If provided, only yield collections that may contain datasets of 

1095 this type. This is a conservative approximation in general; it may 

1096 yield collections that do not have any such datasets. 

1097 collectionTypes : `~collections.abc.Set` [`CollectionType`] or \ 

1098 `CollectionType`, optional 

1099 If provided, only yield collections of these types. 

1100 flattenChains : `bool`, optional 

1101 If `True` (`False` is default), recursively yield the child 

1102 collections of matching `~CollectionType.CHAINED` collections. 

1103 includeChains : `bool`, optional 

1104 If `True`, yield records for matching `~CollectionType.CHAINED` 

1105 collections. Default is the opposite of ``flattenChains``: include 

1106 either CHAINED collections or their children, but not both. 

1107 

1108 Returns 

1109 ------- 

1110 collections : `~collections.abc.Sequence` [ `str` ] 

1111 The names of collections that match ``expression``. 

1112 

1113 Raises 

1114 ------ 

1115 lsst.daf.butler.registry.CollectionExpressionError 

1116 Raised when ``expression`` is invalid. 

1117 

1118 Notes 

1119 ----- 

1120 The order in which collections are returned is unspecified, except that 

1121 the children of a `~CollectionType.CHAINED` collection are guaranteed 

1122 to be in the order in which they are searched. When multiple parent 

1123 `~CollectionType.CHAINED` collections match the same criteria, the 

1124 order in which the two lists appear is unspecified, and the lists of 

1125 children may be incomplete if a child has multiple parents. 

1126 """ 

1127 raise NotImplementedError() 

1128 

1129 @abstractmethod 

1130 def queryDatasets( 

1131 self, 

1132 datasetType: Any, 

1133 *, 

1134 collections: CollectionArgType | None = None, 

1135 dimensions: Iterable[Dimension | str] | None = None, 

1136 dataId: DataId | None = None, 

1137 where: str = "", 

1138 findFirst: bool = False, 

1139 components: bool | None = None, 

1140 bind: Mapping[str, Any] | None = None, 

1141 check: bool = True, 

1142 **kwargs: Any, 

1143 ) -> DatasetQueryResults: 

1144 """Query for and iterate over dataset references matching user-provided 

1145 criteria. 

1146 

1147 Parameters 

1148 ---------- 

1149 datasetType : dataset type expression 

1150 An expression that fully or partially identifies the dataset types 

1151 to be queried. Allowed types include `DatasetType`, `str`, 

1152 `re.Pattern`, and iterables thereof. The special value ``...`` can 

1153 be used to query all dataset types. See 

1154 :ref:`daf_butler_dataset_type_expressions` for more information. 

1155 collections : collection expression, optional 

1156 An expression that identifies the collections to search, such as a 

1157 `str` (for full matches or partial matches via globs), `re.Pattern` 

1158 (for partial matches), or iterable thereof. ``...`` can be used to 

1159 search all collections (actually just all `~CollectionType.RUN` 

1160 collections, because this will still find all datasets). 

1161 If not provided, ``self.default.collections`` is used. See 

1162 :ref:`daf_butler_collection_expressions` for more information. 

1163 dimensions : `~collections.abc.Iterable` of `Dimension` or `str` 

1164 Dimensions to include in the query (in addition to those used 

1165 to identify the queried dataset type(s)), either to constrain 

1166 the resulting datasets to those for which a matching dimension 

1167 exists, or to relate the dataset type's dimensions to dimensions 

1168 referenced by the ``dataId`` or ``where`` arguments. 

1169 dataId : `dict` or `DataCoordinate`, optional 

1170 A data ID whose key-value pairs are used as equality constraints 

1171 in the query. 

1172 where : `str`, optional 

1173 A string expression similar to a SQL WHERE clause. May involve 

1174 any column of a dimension table or (as a shortcut for the primary 

1175 key column of a dimension table) dimension name. See 

1176 :ref:`daf_butler_dimension_expressions` for more information. 

1177 findFirst : `bool`, optional 

1178 If `True` (`False` is default), for each result data ID, only 

1179 yield one `DatasetRef` of each `DatasetType`, from the first 

1180 collection in which a dataset of that dataset type appears 

1181 (according to the order of ``collections`` passed in). If `True`, 

1182 ``collections`` must not contain regular expressions and may not 

1183 be ``...``. 

1184 components : `bool`, optional 

1185 If `True`, apply all dataset expression patterns to component 

1186 dataset type names as well. If `False`, never apply patterns to 

1187 components. If `None` (default), apply patterns to components only 

1188 if their parent datasets were not matched by the expression. 

1189 Fully-specified component datasets (`str` or `DatasetType` 

1190 instances) are always included. 

1191 

1192 Values other than `False` are deprecated, and only `False` will be 

1193 supported after v26. After v27 this argument will be removed 

1194 entirely. 

1195 bind : `~collections.abc.Mapping`, optional 

1196 Mapping containing literal values that should be injected into the 

1197 ``where`` expression, keyed by the identifiers they replace. 

1198 Values of collection type can be expanded in some cases; see 

1199 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1200 information. 

1201 check : `bool`, optional 

1202 If `True` (default) check the query for consistency before 

1203 executing it. This may reject some valid queries that resemble 

1204 common mistakes (e.g. queries for visits without specifying an 

1205 instrument). 

1206 **kwargs 

1207 Additional keyword arguments are forwarded to 

1208 `DataCoordinate.standardize` when processing the ``dataId`` 

1209 argument (and may be used to provide a constraining data ID even 

1210 when the ``dataId`` argument is `None`). 

1211 

1212 Returns 

1213 ------- 

1214 refs : `.queries.DatasetQueryResults` 

1215 Dataset references matching the given query criteria. Nested data 

1216 IDs are guaranteed to include values for all implied dimensions 

1217 (i.e. `DataCoordinate.hasFull` will return `True`), but will not 

1218 include dimension records (`DataCoordinate.hasRecords` will be 

1219 `False`) unless `~.queries.DatasetQueryResults.expanded` is 

1220 called on the result object (which returns a new one). 

1221 

1222 Raises 

1223 ------ 

1224 lsst.daf.butler.registry.DatasetTypeExpressionError 

1225 Raised when ``datasetType`` expression is invalid. 

1226 TypeError 

1227 Raised when the arguments are incompatible, such as when a 

1228 collection wildcard is passed when ``findFirst`` is `True`, or 

1229 when ``collections`` is `None` and ``self.defaults.collections`` is 

1230 also `None`. 

1231 lsst.daf.butler.registry.DataIdError 

1232 Raised when ``dataId`` or keyword arguments specify unknown 

1233 dimensions or values, or when they contain inconsistent values. 

1234 lsst.daf.butler.registry.UserExpressionError 

1235 Raised when ``where`` expression is invalid. 

1236 

1237 Notes 

1238 ----- 

1239 When multiple dataset types are queried in a single call, the 

1240 results of this operation are equivalent to querying for each dataset 

1241 type separately in turn, and no information about the relationships 

1242 between datasets of different types is included. In contexts where 

1243 that kind of information is important, the recommended pattern is to 

1244 use `queryDataIds` to first obtain data IDs (possibly with the 

1245 desired dataset types and collections passed as constraints to the 

1246 query), and then use multiple (generally much simpler) calls to 

1247 `queryDatasets` with the returned data IDs passed as constraints. 

1248 """ 

1249 raise NotImplementedError() 

1250 

1251 @abstractmethod 

1252 def queryDataIds( 

1253 self, 

1254 dimensions: Iterable[Dimension | str] | Dimension | str, 

1255 *, 

1256 dataId: DataId | None = None, 

1257 datasets: Any = None, 

1258 collections: CollectionArgType | None = None, 

1259 where: str = "", 

1260 components: bool | None = None, 

1261 bind: Mapping[str, Any] | None = None, 

1262 check: bool = True, 

1263 **kwargs: Any, 

1264 ) -> DataCoordinateQueryResults: 

1265 """Query for data IDs matching user-provided criteria. 

1266 

1267 Parameters 

1268 ---------- 

1269 dimensions : `Dimension` or `str`, or iterable thereof 

1270 The dimensions of the data IDs to yield, as either `Dimension` 

1271 instances or `str`. Will be automatically expanded to a complete 

1272 `DimensionGraph`. 

1273 dataId : `dict` or `DataCoordinate`, optional 

1274 A data ID whose key-value pairs are used as equality constraints 

1275 in the query. 

1276 datasets : dataset type expression, optional 

1277 An expression that fully or partially identifies dataset types 

1278 that should constrain the yielded data IDs. For example, including 

1279 "raw" here would constrain the yielded ``instrument``, 

1280 ``exposure``, ``detector``, and ``physical_filter`` values to only 

1281 those for which at least one "raw" dataset exists in 

1282 ``collections``. Allowed types include `DatasetType`, `str`, 

1283 and iterables thereof. Regular expression objects (i.e. 

1284 `re.Pattern`) are deprecated and will be removed after the v26 

1285 release. See :ref:`daf_butler_dataset_type_expressions` for more 

1286 information. 

1287 collections : collection expression, optional 

1288 An expression that identifies the collections to search for 

1289 datasets, such as a `str` (for full matches or partial matches 

1290 via globs), `re.Pattern` (for partial matches), or iterable 

1291 thereof. ``...`` can be used to search all collections (actually 

1292 just all `~CollectionType.RUN` collections, because this will 

1293 still find all datasets). If not provided, 

1294 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1295 is also passed. See :ref:`daf_butler_collection_expressions` for 

1296 more information. 

1297 where : `str`, optional 

1298 A string expression similar to a SQL WHERE clause. May involve 

1299 any column of a dimension table or (as a shortcut for the primary 

1300 key column of a dimension table) dimension name. See 

1301 :ref:`daf_butler_dimension_expressions` for more information. 

1302 components : `bool`, optional 

1303 If `True`, apply all dataset expression patterns to component 

1304 dataset type names as well. If `False`, never apply patterns to 

1305 components. If `None` (default), apply patterns to components only 

1306 if their parent datasets were not matched by the expression. 

1307 Fully-specified component datasets (`str` or `DatasetType` 

1308 instances) are always included. 

1309 

1310 Values other than `False` are deprecated, and only `False` will be 

1311 supported after v26. After v27 this argument will be removed 

1312 entirely. 

1313 bind : `~collections.abc.Mapping`, optional 

1314 Mapping containing literal values that should be injected into the 

1315 ``where`` expression, keyed by the identifiers they replace. 

1316 Values of collection type can be expanded in some cases; see 

1317 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1318 information. 

1319 check : `bool`, optional 

1320 If `True` (default) check the query for consistency before 

1321 executing it. This may reject some valid queries that resemble 

1322 common mistakes (e.g. queries for visits without specifying an 

1323 instrument). 

1324 **kwargs 

1325 Additional keyword arguments are forwarded to 

1326 `DataCoordinate.standardize` when processing the ``dataId`` 

1327 argument (and may be used to provide a constraining data ID even 

1328 when the ``dataId`` argument is `None`). 

1329 

1330 Returns 

1331 ------- 

1332 dataIds : `.queries.DataCoordinateQueryResults` 

1333 Data IDs matching the given query parameters. These are guaranteed 

1334 to identify all dimensions (`DataCoordinate.hasFull` returns 

1335 `True`), but will not contain `DimensionRecord` objects 

1336 (`DataCoordinate.hasRecords` returns `False`). Call 

1337 `~.queries.DataCoordinateQueryResults.expanded` on the 

1338 returned object to fetch those (and consider using 

1339 `~.queries.DataCoordinateQueryResults.materialize` on the 

1340 returned object first if the expected number of rows is very 

1341 large). See documentation for those methods for additional 

1342 information. 

1343 

1344 Raises 

1345 ------ 

1346 lsst.daf.butler.registry.NoDefaultCollectionError 

1347 Raised if ``collections`` is `None` and 

1348 ``self.defaults.collections`` is `None`. 

1349 lsst.daf.butler.registry.CollectionExpressionError 

1350 Raised when ``collections`` expression is invalid. 

1351 lsst.daf.butler.registry.DataIdError 

1352 Raised when ``dataId`` or keyword arguments specify unknown 

1353 dimensions or values, or when they contain inconsistent values. 

1354 lsst.daf.butler.registry.DatasetTypeExpressionError 

1355 Raised when ``datasetType`` expression is invalid. 

1356 lsst.daf.butler.registry.UserExpressionError 

1357 Raised when ``where`` expression is invalid. 

1358 """ 

1359 raise NotImplementedError() 

1360 

1361 @abstractmethod 

1362 def queryDimensionRecords( 

1363 self, 

1364 element: DimensionElement | str, 

1365 *, 

1366 dataId: DataId | None = None, 

1367 datasets: Any = None, 

1368 collections: CollectionArgType | None = None, 

1369 where: str = "", 

1370 components: bool | None = None, 

1371 bind: Mapping[str, Any] | None = None, 

1372 check: bool = True, 

1373 **kwargs: Any, 

1374 ) -> DimensionRecordQueryResults: 

1375 """Query for dimension information matching user-provided criteria. 

1376 

1377 Parameters 

1378 ---------- 

1379 element : `DimensionElement` or `str` 

1380 The dimension element to obtain records for. 

1381 dataId : `dict` or `DataCoordinate`, optional 

1382 A data ID whose key-value pairs are used as equality constraints 

1383 in the query. 

1384 datasets : dataset type expression, optional 

1385 An expression that fully or partially identifies dataset types 

1386 that should constrain the yielded records. See `queryDataIds` and 

1387 :ref:`daf_butler_dataset_type_expressions` for more information. 

1388 collections : collection expression, optional 

1389 An expression that identifies the collections to search for 

1390 datasets, such as a `str` (for full matches or partial matches 

1391 via globs), `re.Pattern` (for partial matches), or iterable 

1392 thereof. ``...`` can be used to search all collections (actually 

1393 just all `~CollectionType.RUN` collections, because this will 

1394 still find all datasets). If not provided, 

1395 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1396 is also passed. See :ref:`daf_butler_collection_expressions` for 

1397 more information. 

1398 where : `str`, optional 

1399 A string expression similar to a SQL WHERE clause. See 

1400 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more 

1401 information. 

1402 components : `bool`, optional 

1403 Whether to apply dataset expressions to components as well. 

1404 See `queryDataIds` for more information. 

1405 

1406 Values other than `False` are deprecated, and only `False` will be 

1407 supported after v26. After v27 this argument will be removed 

1408 entirely. 

1409 bind : `~collections.abc.Mapping`, optional 

1410 Mapping containing literal values that should be injected into the 

1411 ``where`` expression, keyed by the identifiers they replace. 

1412 Values of collection type can be expanded in some cases; see 

1413 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1414 information. 

1415 check : `bool`, optional 

1416 If `True` (default) check the query for consistency before 

1417 executing it. This may reject some valid queries that resemble 

1418 common mistakes (e.g. queries for visits without specifying an 

1419 instrument). 

1420 **kwargs 

1421 Additional keyword arguments are forwarded to 

1422 `DataCoordinate.standardize` when processing the ``dataId`` 

1423 argument (and may be used to provide a constraining data ID even 

1424 when the ``dataId`` argument is `None`). 

1425 

1426 Returns 

1427 ------- 

1428 dataIds : `.queries.DimensionRecordQueryResults` 

1429 Data IDs matching the given query parameters. 

1430 

1431 Raises 

1432 ------ 

1433 lsst.daf.butler.registry.NoDefaultCollectionError 

1434 Raised if ``collections`` is `None` and 

1435 ``self.defaults.collections`` is `None`. 

1436 lsst.daf.butler.registry.CollectionExpressionError 

1437 Raised when ``collections`` expression is invalid. 

1438 lsst.daf.butler.registry.DataIdError 

1439 Raised when ``dataId`` or keyword arguments specify unknown 

1440 dimensions or values, or when they contain inconsistent values. 

1441 lsst.daf.butler.registry.DatasetTypeExpressionError 

1442 Raised when ``datasetType`` expression is invalid. 

1443 lsst.daf.butler.registry.UserExpressionError 

1444 Raised when ``where`` expression is invalid. 

1445 """ 

1446 raise NotImplementedError() 

1447 

1448 @abstractmethod 

1449 def queryDatasetAssociations( 

1450 self, 

1451 datasetType: str | DatasetType, 

1452 collections: CollectionArgType | None = ..., 

1453 *, 

1454 collectionTypes: Iterable[CollectionType] = CollectionType.all(), 

1455 flattenChains: bool = False, 

1456 ) -> Iterator[DatasetAssociation]: 

1457 """Iterate over dataset-collection combinations where the dataset is in 

1458 the collection. 

1459 

1460 This method is a temporary placeholder for better support for 

1461 association results in `queryDatasets`. It will probably be 

1462 removed in the future, and should be avoided in production code 

1463 whenever possible. 

1464 

1465 Parameters 

1466 ---------- 

1467 datasetType : `DatasetType` or `str` 

1468 A dataset type object or the name of one. 

1469 collections : collection expression, optional 

1470 An expression that identifies the collections to search for 

1471 datasets, such as a `str` (for full matches or partial matches 

1472 via globs), `re.Pattern` (for partial matches), or iterable 

1473 thereof. ``...`` can be used to search all collections (actually 

1474 just all `~CollectionType.RUN` collections, because this will still 

1475 find all datasets). If not provided, ``self.default.collections`` 

1476 is used. See :ref:`daf_butler_collection_expressions` for more 

1477 information. 

1478 collectionTypes : `~collections.abc.Set` [ `CollectionType` ], optional 

1479 If provided, only yield associations from collections of these 

1480 types. 

1481 flattenChains : `bool`, optional 

1482 If `True` (default) search in the children of 

1483 `~CollectionType.CHAINED` collections. If `False`, ``CHAINED`` 

1484 collections are ignored. 

1485 

1486 Yields 

1487 ------ 

1488 association : `.DatasetAssociation` 

1489 Object representing the relationship between a single dataset and 

1490 a single collection. 

1491 

1492 Raises 

1493 ------ 

1494 lsst.daf.butler.registry.NoDefaultCollectionError 

1495 Raised if ``collections`` is `None` and 

1496 ``self.defaults.collections`` is `None`. 

1497 lsst.daf.butler.registry.CollectionExpressionError 

1498 Raised when ``collections`` expression is invalid. 

1499 """ 

1500 raise NotImplementedError() 

1501 

1502 @property 

1503 def obsCoreTableManager(self) -> ObsCoreTableManager | None: 

1504 """The ObsCore manager instance for this registry 

1505 (`~.interfaces.ObsCoreTableManager` 

1506 or `None`). 

1507 

1508 ObsCore manager may not be implemented for all registry backend, or 

1509 may not be enabled for many repositories. 

1510 """ 

1511 return None 

1512 

1513 storageClasses: StorageClassFactory 

1514 """All storage classes known to the registry (`StorageClassFactory`). 

1515 """