Coverage for python/lsst/daf/butler/registry/_registry.py: 78%

147 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-08 05:05 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("Registry",) 

25 

26import contextlib 

27import logging 

28import re 

29from abc import ABC, abstractmethod 

30from types import EllipsisType 

31from typing import ( 

32 TYPE_CHECKING, 

33 Any, 

34 Dict, 

35 Iterable, 

36 Iterator, 

37 List, 

38 Mapping, 

39 Optional, 

40 Sequence, 

41 Set, 

42 Tuple, 

43 Type, 

44 Union, 

45) 

46 

47from lsst.resources import ResourcePathExpression 

48from lsst.utils import doImportType 

49 

50from ..core import ( 

51 Config, 

52 DataCoordinate, 

53 DataId, 

54 DatasetAssociation, 

55 DatasetId, 

56 DatasetIdFactory, 

57 DatasetIdGenEnum, 

58 DatasetRef, 

59 DatasetType, 

60 Dimension, 

61 DimensionConfig, 

62 DimensionElement, 

63 DimensionGraph, 

64 DimensionRecord, 

65 DimensionUniverse, 

66 NameLookupMapping, 

67 StorageClassFactory, 

68 Timespan, 

69) 

70from ._collection_summary import CollectionSummary 

71from ._collectionType import CollectionType 

72from ._config import RegistryConfig 

73from ._defaults import RegistryDefaults 

74from .queries import DataCoordinateQueryResults, DatasetQueryResults, DimensionRecordQueryResults 

75from .wildcards import CollectionWildcard 

76 

77if TYPE_CHECKING: 

78 from .._butlerConfig import ButlerConfig 

79 from .interfaces import CollectionRecord, DatastoreRegistryBridgeManager, ObsCoreTableManager 

80 

81_LOG = logging.getLogger(__name__) 

82 

83# TYpe alias for `collections` arguments. 

84CollectionArgType = str | re.Pattern | Iterable[str | re.Pattern] | EllipsisType | CollectionWildcard 

85 

86 

87class Registry(ABC): 

88 """Abstract Registry interface. 

89 

90 Each registry implementation can have its own constructor parameters. 

91 The assumption is that an instance of a specific subclass will be 

92 constructed from configuration using `Registry.fromConfig()`. 

93 The base class will look for a ``cls`` entry and call that specific 

94 `fromConfig()` method. 

95 

96 All subclasses should store `~lsst.daf.butler.registry.RegistryDefaults` in 

97 a ``_defaults`` property. No other properties are assumed shared between 

98 implementations. 

99 """ 

100 

101 defaultConfigFile: Optional[str] = None 

102 """Path to configuration defaults. Accessed within the ``configs`` resource 

103 or relative to a search path. Can be None if no defaults specified. 

104 """ 

105 

106 @classmethod 

107 def forceRegistryConfig( 

108 cls, config: Optional[Union[ButlerConfig, RegistryConfig, Config, str]] 

109 ) -> RegistryConfig: 

110 """Force the supplied config to a `RegistryConfig`. 

111 

112 Parameters 

113 ---------- 

114 config : `RegistryConfig`, `Config` or `str` or `None` 

115 Registry configuration, if missing then default configuration will 

116 be loaded from registry.yaml. 

117 

118 Returns 

119 ------- 

120 registry_config : `RegistryConfig` 

121 A registry config. 

122 """ 

123 if not isinstance(config, RegistryConfig): 

124 if isinstance(config, (str, Config)) or config is None: 

125 config = RegistryConfig(config) 

126 else: 

127 raise ValueError(f"Incompatible Registry configuration: {config}") 

128 return config 

129 

130 @classmethod 

131 def determineTrampoline( 

132 cls, config: Optional[Union[ButlerConfig, RegistryConfig, Config, str]] 

133 ) -> Tuple[Type[Registry], RegistryConfig]: 

134 """Return class to use to instantiate real registry. 

135 

136 Parameters 

137 ---------- 

138 config : `RegistryConfig` or `str`, optional 

139 Registry configuration, if missing then default configuration will 

140 be loaded from registry.yaml. 

141 

142 Returns 

143 ------- 

144 requested_cls : `type` of `Registry` 

145 The real registry class to use. 

146 registry_config : `RegistryConfig` 

147 The `RegistryConfig` to use. 

148 """ 

149 config = cls.forceRegistryConfig(config) 

150 

151 # Default to the standard registry 

152 registry_cls_name = config.get("cls", "lsst.daf.butler.registries.sql.SqlRegistry") 

153 registry_cls = doImportType(registry_cls_name) 

154 if registry_cls is cls: 

155 raise ValueError("Can not instantiate the abstract base Registry from config") 

156 if not issubclass(registry_cls, Registry): 

157 raise TypeError( 

158 f"Registry class obtained from config {registry_cls_name} is not a Registry class." 

159 ) 

160 return registry_cls, config 

161 

162 @classmethod 

163 def createFromConfig( 

164 cls, 

165 config: Optional[Union[RegistryConfig, str]] = None, 

166 dimensionConfig: Optional[Union[DimensionConfig, str]] = None, 

167 butlerRoot: Optional[ResourcePathExpression] = None, 

168 ) -> Registry: 

169 """Create registry database and return `Registry` instance. 

170 

171 This method initializes database contents, database must be empty 

172 prior to calling this method. 

173 

174 Parameters 

175 ---------- 

176 config : `RegistryConfig` or `str`, optional 

177 Registry configuration, if missing then default configuration will 

178 be loaded from registry.yaml. 

179 dimensionConfig : `DimensionConfig` or `str`, optional 

180 Dimensions configuration, if missing then default configuration 

181 will be loaded from dimensions.yaml. 

182 butlerRoot : convertible to `lsst.resources.ResourcePath`, optional 

183 Path to the repository root this `Registry` will manage. 

184 

185 Returns 

186 ------- 

187 registry : `Registry` 

188 A new `Registry` instance. 

189 

190 Notes 

191 ----- 

192 This class will determine the concrete `Registry` subclass to 

193 use from configuration. Each subclass should implement this method 

194 even if it can not create a registry. 

195 """ 

196 registry_cls, registry_config = cls.determineTrampoline(config) 

197 return registry_cls.createFromConfig(registry_config, dimensionConfig, butlerRoot) 

198 

199 @classmethod 

200 def fromConfig( 

201 cls, 

202 config: Union[ButlerConfig, RegistryConfig, Config, str], 

203 butlerRoot: Optional[ResourcePathExpression] = None, 

204 writeable: bool = True, 

205 defaults: Optional[RegistryDefaults] = None, 

206 ) -> Registry: 

207 """Create `Registry` subclass instance from ``config``. 

208 

209 Registry database must be initialized prior to calling this method. 

210 

211 Parameters 

212 ---------- 

213 config : `ButlerConfig`, `RegistryConfig`, `Config` or `str` 

214 Registry configuration 

215 butlerRoot : convertible to `lsst.resources.ResourcePath`, optional 

216 Path to the repository root this `Registry` will manage. 

217 writeable : `bool`, optional 

218 If `True` (default) create a read-write connection to the database. 

219 defaults : `~lsst.daf.butler.registry.RegistryDefaults`, optional 

220 Default collection search path and/or output `~CollectionType.RUN` 

221 collection. 

222 

223 Returns 

224 ------- 

225 registry : `Registry` (subclass) 

226 A new `Registry` subclass instance. 

227 

228 Notes 

229 ----- 

230 This class will determine the concrete `Registry` subclass to 

231 use from configuration. Each subclass should implement this method. 

232 """ 

233 # The base class implementation should trampoline to the correct 

234 # subclass. No implementation should ever use this implementation 

235 # directly. If no class is specified, default to the standard 

236 # registry. 

237 registry_cls, registry_config = cls.determineTrampoline(config) 

238 return registry_cls.fromConfig(config, butlerRoot, writeable, defaults) 

239 

240 @abstractmethod 

241 def isWriteable(self) -> bool: 

242 """Return `True` if this registry allows write operations, and `False` 

243 otherwise. 

244 """ 

245 raise NotImplementedError() 

246 

247 @abstractmethod 

248 def copy(self, defaults: Optional[RegistryDefaults] = None) -> Registry: 

249 """Create a new `Registry` backed by the same data repository and 

250 connection as this one, but independent defaults. 

251 

252 Parameters 

253 ---------- 

254 defaults : `~lsst.daf.butler.registry.RegistryDefaults`, optional 

255 Default collections and data ID values for the new registry. If 

256 not provided, ``self.defaults`` will be used (but future changes 

257 to either registry's defaults will not affect the other). 

258 

259 Returns 

260 ------- 

261 copy : `Registry` 

262 A new `Registry` instance with its own defaults. 

263 

264 Notes 

265 ----- 

266 Because the new registry shares a connection with the original, they 

267 also share transaction state (despite the fact that their `transaction` 

268 context manager methods do not reflect this), and must be used with 

269 care. 

270 """ 

271 raise NotImplementedError() 

272 

273 @property 

274 @abstractmethod 

275 def dimensions(self) -> DimensionUniverse: 

276 """Definitions of all dimensions recognized by this `Registry` 

277 (`DimensionUniverse`). 

278 """ 

279 raise NotImplementedError() 

280 

281 @property 

282 def defaults(self) -> RegistryDefaults: 

283 """Default collection search path and/or output `~CollectionType.RUN` 

284 collection (`~lsst.daf.butler.registry.RegistryDefaults`). 

285 

286 This is an immutable struct whose components may not be set 

287 individually, but the entire struct can be set by assigning to this 

288 property. 

289 """ 

290 return self._defaults 

291 

292 @defaults.setter 

293 def defaults(self, value: RegistryDefaults) -> None: 

294 if value.run is not None: 

295 self.registerRun(value.run) 

296 value.finish(self) 

297 self._defaults = value 

298 

299 @abstractmethod 

300 def refresh(self) -> None: 

301 """Refresh all in-memory state by querying the database. 

302 

303 This may be necessary to enable querying for entities added by other 

304 registry instances after this one was constructed. 

305 """ 

306 raise NotImplementedError() 

307 

308 @contextlib.contextmanager 

309 @abstractmethod 

310 def transaction(self, *, savepoint: bool = False) -> Iterator[None]: 

311 """Return a context manager that represents a transaction.""" 

312 raise NotImplementedError() 

313 

314 def resetConnectionPool(self) -> None: 

315 """Reset connection pool for registry if relevant. 

316 

317 This operation can be used reset connections to servers when 

318 using registry with fork-based multiprocessing. This method should 

319 usually be called by the child process immediately 

320 after the fork. 

321 

322 The base class implementation is a no-op. 

323 """ 

324 pass 

325 

326 @abstractmethod 

327 def registerCollection( 

328 self, name: str, type: CollectionType = CollectionType.TAGGED, doc: Optional[str] = None 

329 ) -> bool: 

330 """Add a new collection if one with the given name does not exist. 

331 

332 Parameters 

333 ---------- 

334 name : `str` 

335 The name of the collection to create. 

336 type : `CollectionType` 

337 Enum value indicating the type of collection to create. 

338 doc : `str`, optional 

339 Documentation string for the collection. 

340 

341 Returns 

342 ------- 

343 registered : `bool` 

344 Boolean indicating whether the collection was already registered 

345 or was created by this call. 

346 

347 Notes 

348 ----- 

349 This method cannot be called within transactions, as it needs to be 

350 able to perform its own transaction to be concurrent. 

351 """ 

352 raise NotImplementedError() 

353 

354 @abstractmethod 

355 def getCollectionType(self, name: str) -> CollectionType: 

356 """Return an enumeration value indicating the type of the given 

357 collection. 

358 

359 Parameters 

360 ---------- 

361 name : `str` 

362 The name of the collection. 

363 

364 Returns 

365 ------- 

366 type : `CollectionType` 

367 Enum value indicating the type of this collection. 

368 

369 Raises 

370 ------ 

371 lsst.daf.butler.registry.MissingCollectionError 

372 Raised if no collection with the given name exists. 

373 """ 

374 raise NotImplementedError() 

375 

376 @abstractmethod 

377 def _get_collection_record(self, name: str) -> CollectionRecord: 

378 """Return the record for this collection. 

379 

380 Parameters 

381 ---------- 

382 name : `str` 

383 Name of the collection for which the record is to be retrieved. 

384 

385 Returns 

386 ------- 

387 record : `CollectionRecord` 

388 The record for this collection. 

389 """ 

390 raise NotImplementedError() 

391 

392 @abstractmethod 

393 def registerRun(self, name: str, doc: Optional[str] = None) -> bool: 

394 """Add a new run if one with the given name does not exist. 

395 

396 Parameters 

397 ---------- 

398 name : `str` 

399 The name of the run to create. 

400 doc : `str`, optional 

401 Documentation string for the collection. 

402 

403 Returns 

404 ------- 

405 registered : `bool` 

406 Boolean indicating whether a new run was registered. `False` 

407 if it already existed. 

408 

409 Notes 

410 ----- 

411 This method cannot be called within transactions, as it needs to be 

412 able to perform its own transaction to be concurrent. 

413 """ 

414 raise NotImplementedError() 

415 

416 @abstractmethod 

417 def removeCollection(self, name: str) -> None: 

418 """Remove the given collection from the registry. 

419 

420 Parameters 

421 ---------- 

422 name : `str` 

423 The name of the collection to remove. 

424 

425 Raises 

426 ------ 

427 lsst.daf.butler.registry.MissingCollectionError 

428 Raised if no collection with the given name exists. 

429 sqlalchemy.exc.IntegrityError 

430 Raised if the database rows associated with the collection are 

431 still referenced by some other table, such as a dataset in a 

432 datastore (for `~CollectionType.RUN` collections only) or a 

433 `~CollectionType.CHAINED` collection of which this collection is 

434 a child. 

435 

436 Notes 

437 ----- 

438 If this is a `~CollectionType.RUN` collection, all datasets and quanta 

439 in it will removed from the `Registry` database. This requires that 

440 those datasets be removed (or at least trashed) from any datastores 

441 that hold them first. 

442 

443 A collection may not be deleted as long as it is referenced by a 

444 `~CollectionType.CHAINED` collection; the ``CHAINED`` collection must 

445 be deleted or redefined first. 

446 """ 

447 raise NotImplementedError() 

448 

449 @abstractmethod 

450 def getCollectionChain(self, parent: str) -> Sequence[str]: 

451 """Return the child collections in a `~CollectionType.CHAINED` 

452 collection. 

453 

454 Parameters 

455 ---------- 

456 parent : `str` 

457 Name of the chained collection. Must have already been added via 

458 a call to `Registry.registerCollection`. 

459 

460 Returns 

461 ------- 

462 children : `~collections.abc.Sequence` [ `str` ] 

463 An ordered sequence of collection names that are searched when the 

464 given chained collection is searched. 

465 

466 Raises 

467 ------ 

468 lsst.daf.butler.registry.MissingCollectionError 

469 Raised if ``parent`` does not exist in the `Registry`. 

470 lsst.daf.butler.registry.CollectionTypeError 

471 Raised if ``parent`` does not correspond to a 

472 `~CollectionType.CHAINED` collection. 

473 """ 

474 raise NotImplementedError() 

475 

476 @abstractmethod 

477 def setCollectionChain(self, parent: str, children: Any, *, flatten: bool = False) -> None: 

478 """Define or redefine a `~CollectionType.CHAINED` collection. 

479 

480 Parameters 

481 ---------- 

482 parent : `str` 

483 Name of the chained collection. Must have already been added via 

484 a call to `Registry.registerCollection`. 

485 children : collection expression 

486 An expression defining an ordered search of child collections, 

487 generally an iterable of `str`; see 

488 :ref:`daf_butler_collection_expressions` for more information. 

489 flatten : `bool`, optional 

490 If `True` (`False` is default), recursively flatten out any nested 

491 `~CollectionType.CHAINED` collections in ``children`` first. 

492 

493 Raises 

494 ------ 

495 lsst.daf.butler.registry.MissingCollectionError 

496 Raised when any of the given collections do not exist in the 

497 `Registry`. 

498 lsst.daf.butler.registry.CollectionTypeError 

499 Raised if ``parent`` does not correspond to a 

500 `~CollectionType.CHAINED` collection. 

501 ValueError 

502 Raised if the given collections contains a cycle. 

503 """ 

504 raise NotImplementedError() 

505 

506 @abstractmethod 

507 def getCollectionParentChains(self, collection: str) -> Set[str]: 

508 """Return the CHAINED collections that directly contain the given one. 

509 

510 Parameters 

511 ---------- 

512 name : `str` 

513 Name of the collection. 

514 

515 Returns 

516 ------- 

517 chains : `set` of `str` 

518 Set of `~CollectionType.CHAINED` collection names. 

519 """ 

520 raise NotImplementedError() 

521 

522 @abstractmethod 

523 def getCollectionDocumentation(self, collection: str) -> Optional[str]: 

524 """Retrieve the documentation string for a collection. 

525 

526 Parameters 

527 ---------- 

528 name : `str` 

529 Name of the collection. 

530 

531 Returns 

532 ------- 

533 docs : `str` or `None` 

534 Docstring for the collection with the given name. 

535 """ 

536 raise NotImplementedError() 

537 

538 @abstractmethod 

539 def setCollectionDocumentation(self, collection: str, doc: Optional[str]) -> None: 

540 """Set the documentation string for a collection. 

541 

542 Parameters 

543 ---------- 

544 name : `str` 

545 Name of the collection. 

546 docs : `str` or `None` 

547 Docstring for the collection with the given name; will replace any 

548 existing docstring. Passing `None` will remove any existing 

549 docstring. 

550 """ 

551 raise NotImplementedError() 

552 

553 @abstractmethod 

554 def getCollectionSummary(self, collection: str) -> CollectionSummary: 

555 """Return a summary for the given collection. 

556 

557 Parameters 

558 ---------- 

559 collection : `str` 

560 Name of the collection for which a summary is to be retrieved. 

561 

562 Returns 

563 ------- 

564 summary : `~lsst.daf.butler.registry.CollectionSummary` 

565 Summary of the dataset types and governor dimension values in 

566 this collection. 

567 """ 

568 raise NotImplementedError() 

569 

570 @abstractmethod 

571 def registerDatasetType(self, datasetType: DatasetType) -> bool: 

572 """Add a new `DatasetType` to the Registry. 

573 

574 It is not an error to register the same `DatasetType` twice. 

575 

576 Parameters 

577 ---------- 

578 datasetType : `DatasetType` 

579 The `DatasetType` to be added. 

580 

581 Returns 

582 ------- 

583 inserted : `bool` 

584 `True` if ``datasetType`` was inserted, `False` if an identical 

585 existing `DatasetType` was found. Note that in either case the 

586 DatasetType is guaranteed to be defined in the Registry 

587 consistently with the given definition. 

588 

589 Raises 

590 ------ 

591 ValueError 

592 Raised if the dimensions or storage class are invalid. 

593 lsst.daf.butler.registry.ConflictingDefinitionError 

594 Raised if this `DatasetType` is already registered with a different 

595 definition. 

596 

597 Notes 

598 ----- 

599 This method cannot be called within transactions, as it needs to be 

600 able to perform its own transaction to be concurrent. 

601 """ 

602 raise NotImplementedError() 

603 

604 @abstractmethod 

605 def removeDatasetType(self, name: str | tuple[str, ...]) -> None: 

606 """Remove the named `DatasetType` from the registry. 

607 

608 .. warning:: 

609 

610 Registry implementations can cache the dataset type definitions. 

611 This means that deleting the dataset type definition may result in 

612 unexpected behavior from other butler processes that are active 

613 that have not seen the deletion. 

614 

615 Parameters 

616 ---------- 

617 name : `str` or `tuple` [`str`] 

618 Name of the type to be removed or tuple containing a list of type 

619 names to be removed. Wildcards are allowed. 

620 

621 Raises 

622 ------ 

623 lsst.daf.butler.registry.OrphanedRecordError 

624 Raised if an attempt is made to remove the dataset type definition 

625 when there are already datasets associated with it. 

626 

627 Notes 

628 ----- 

629 If the dataset type is not registered the method will return without 

630 action. 

631 """ 

632 raise NotImplementedError() 

633 

634 @abstractmethod 

635 def getDatasetType(self, name: str) -> DatasetType: 

636 """Get the `DatasetType`. 

637 

638 Parameters 

639 ---------- 

640 name : `str` 

641 Name of the type. 

642 

643 Returns 

644 ------- 

645 type : `DatasetType` 

646 The `DatasetType` associated with the given name. 

647 

648 Raises 

649 ------ 

650 lsst.daf.butler.registry.MissingDatasetTypeError 

651 Raised if the requested dataset type has not been registered. 

652 

653 Notes 

654 ----- 

655 This method handles component dataset types automatically, though most 

656 other registry operations do not. 

657 """ 

658 raise NotImplementedError() 

659 

660 @abstractmethod 

661 def supportsIdGenerationMode(self, mode: DatasetIdGenEnum) -> bool: 

662 """Test whether the given dataset ID generation mode is supported by 

663 `insertDatasets`. 

664 

665 Parameters 

666 ---------- 

667 mode : `DatasetIdGenEnum` 

668 Enum value for the mode to test. 

669 

670 Returns 

671 ------- 

672 supported : `bool` 

673 Whether the given mode is supported. 

674 """ 

675 raise NotImplementedError() 

676 

677 @abstractmethod 

678 def findDataset( 

679 self, 

680 datasetType: Union[DatasetType, str], 

681 dataId: Optional[DataId] = None, 

682 *, 

683 collections: CollectionArgType | None = None, 

684 timespan: Optional[Timespan] = None, 

685 **kwargs: Any, 

686 ) -> Optional[DatasetRef]: 

687 """Find a dataset given its `DatasetType` and data ID. 

688 

689 This can be used to obtain a `DatasetRef` that permits the dataset to 

690 be read from a `Datastore`. If the dataset is a component and can not 

691 be found using the provided dataset type, a dataset ref for the parent 

692 will be returned instead but with the correct dataset type. 

693 

694 Parameters 

695 ---------- 

696 datasetType : `DatasetType` or `str` 

697 A `DatasetType` or the name of one. If this is a `DatasetType` 

698 instance, its storage class will be respected and propagated to 

699 the output, even if it differs from the dataset type definition 

700 in the registry, as long as the storage classes are convertible. 

701 dataId : `dict` or `DataCoordinate`, optional 

702 A `dict`-like object containing the `Dimension` links that identify 

703 the dataset within a collection. 

704 collections : collection expression, optional 

705 An expression that fully or partially identifies the collections to 

706 search for the dataset; see 

707 :ref:`daf_butler_collection_expressions` for more information. 

708 Defaults to ``self.defaults.collections``. 

709 timespan : `Timespan`, optional 

710 A timespan that the validity range of the dataset must overlap. 

711 If not provided, any `~CollectionType.CALIBRATION` collections 

712 matched by the ``collections`` argument will not be searched. 

713 **kwargs 

714 Additional keyword arguments passed to 

715 `DataCoordinate.standardize` to convert ``dataId`` to a true 

716 `DataCoordinate` or augment an existing one. 

717 

718 Returns 

719 ------- 

720 ref : `DatasetRef` 

721 A reference to the dataset, or `None` if no matching Dataset 

722 was found. 

723 

724 Raises 

725 ------ 

726 lsst.daf.butler.registry.NoDefaultCollectionError 

727 Raised if ``collections`` is `None` and 

728 ``self.defaults.collections`` is `None`. 

729 LookupError 

730 Raised if one or more data ID keys are missing. 

731 lsst.daf.butler.registry.MissingDatasetTypeError 

732 Raised if the dataset type does not exist. 

733 lsst.daf.butler.registry.MissingCollectionError 

734 Raised if any of ``collections`` does not exist in the registry. 

735 

736 Notes 

737 ----- 

738 This method simply returns `None` and does not raise an exception even 

739 when the set of collections searched is intrinsically incompatible with 

740 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but 

741 only `~CollectionType.CALIBRATION` collections are being searched. 

742 This may make it harder to debug some lookup failures, but the behavior 

743 is intentional; we consider it more important that failed searches are 

744 reported consistently, regardless of the reason, and that adding 

745 additional collections that do not contain a match to the search path 

746 never changes the behavior. 

747 

748 This method handles component dataset types automatically, though most 

749 other registry operations do not. 

750 """ 

751 raise NotImplementedError() 

752 

753 @abstractmethod 

754 def insertDatasets( 

755 self, 

756 datasetType: Union[DatasetType, str], 

757 dataIds: Iterable[DataId], 

758 run: Optional[str] = None, 

759 expand: bool = True, 

760 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE, 

761 ) -> List[DatasetRef]: 

762 """Insert one or more datasets into the `Registry` 

763 

764 This always adds new datasets; to associate existing datasets with 

765 a new collection, use ``associate``. 

766 

767 Parameters 

768 ---------- 

769 datasetType : `DatasetType` or `str` 

770 A `DatasetType` or the name of one. 

771 dataIds : `~collections.abc.Iterable` of `dict` or `DataCoordinate` 

772 Dimension-based identifiers for the new datasets. 

773 run : `str`, optional 

774 The name of the run that produced the datasets. Defaults to 

775 ``self.defaults.run``. 

776 expand : `bool`, optional 

777 If `True` (default), expand data IDs as they are inserted. This is 

778 necessary in general to allow datastore to generate file templates, 

779 but it may be disabled if the caller can guarantee this is 

780 unnecessary. 

781 idGenerationMode : `DatasetIdGenEnum`, optional 

782 Specifies option for generating dataset IDs. By default unique IDs 

783 are generated for each inserted dataset. 

784 

785 Returns 

786 ------- 

787 refs : `list` of `DatasetRef` 

788 Resolved `DatasetRef` instances for all given data IDs (in the same 

789 order). 

790 

791 Raises 

792 ------ 

793 lsst.daf.butler.registry.DatasetTypeError 

794 Raised if ``datasetType`` is not known to registry. 

795 lsst.daf.butler.registry.CollectionTypeError 

796 Raised if ``run`` collection type is not `~CollectionType.RUN`. 

797 lsst.daf.butler.registry.NoDefaultCollectionError 

798 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

799 lsst.daf.butler.registry.ConflictingDefinitionError 

800 If a dataset with the same dataset type and data ID as one of those 

801 given already exists in ``run``. 

802 lsst.daf.butler.registry.MissingCollectionError 

803 Raised if ``run`` does not exist in the registry. 

804 """ 

805 raise NotImplementedError() 

806 

807 @abstractmethod 

808 def _importDatasets( 

809 self, 

810 datasets: Iterable[DatasetRef], 

811 expand: bool = True, 

812 ) -> List[DatasetRef]: 

813 """Import one or more datasets into the `Registry`. 

814 

815 Difference from `insertDatasets` method is that this method accepts 

816 `DatasetRef` instances which should already be resolved and have a 

817 dataset ID. If registry supports globally-unique dataset IDs (e.g. 

818 `uuid.UUID`) then datasets which already exist in the registry will be 

819 ignored if imported again. 

820 

821 Parameters 

822 ---------- 

823 datasets : `~collections.abc.Iterable` of `DatasetRef` 

824 Datasets to be inserted. All `DatasetRef` instances must have 

825 identical ``datasetType`` and ``run`` attributes. ``run`` 

826 attribute can be `None` and defaults to ``self.defaults.run``. 

827 Datasets can specify ``id`` attribute which will be used for 

828 inserted datasets. All dataset IDs must have the same type 

829 (`int` or `uuid.UUID`), if type of dataset IDs does not match 

830 configured backend then IDs will be ignored and new IDs will be 

831 generated by backend. 

832 expand : `bool`, optional 

833 If `True` (default), expand data IDs as they are inserted. This is 

834 necessary in general to allow datastore to generate file templates, 

835 but it may be disabled if the caller can guarantee this is 

836 unnecessary. 

837 

838 Returns 

839 ------- 

840 refs : `list` of `DatasetRef` 

841 Resolved `DatasetRef` instances for all given data IDs (in the same 

842 order). If any of ``datasets`` has an ID which already exists in 

843 the database then it will not be inserted or updated, but a 

844 resolved `DatasetRef` will be returned for it in any case. 

845 

846 Raises 

847 ------ 

848 lsst.daf.butler.registry.NoDefaultCollectionError 

849 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

850 lsst.daf.butler.registry.DatasetTypeError 

851 Raised if datasets correspond to more than one dataset type or 

852 dataset type is not known to registry. 

853 lsst.daf.butler.registry.ConflictingDefinitionError 

854 If a dataset with the same dataset type and data ID as one of those 

855 given already exists in ``run``. 

856 lsst.daf.butler.registry.MissingCollectionError 

857 Raised if ``run`` does not exist in the registry. 

858 

859 Notes 

860 ----- 

861 This method is considered package-private and internal to Butler 

862 implementation. Clients outside daf_butler package should not use this 

863 method. 

864 """ 

865 raise NotImplementedError() 

866 

867 @abstractmethod 

868 def getDataset(self, id: DatasetId) -> Optional[DatasetRef]: 

869 """Retrieve a Dataset entry. 

870 

871 Parameters 

872 ---------- 

873 id : `DatasetId` 

874 The unique identifier for the dataset. 

875 

876 Returns 

877 ------- 

878 ref : `DatasetRef` or `None` 

879 A ref to the Dataset, or `None` if no matching Dataset 

880 was found. 

881 """ 

882 raise NotImplementedError() 

883 

884 @abstractmethod 

885 def removeDatasets(self, refs: Iterable[DatasetRef]) -> None: 

886 """Remove datasets from the Registry. 

887 

888 The datasets will be removed unconditionally from all collections, and 

889 any `Quantum` that consumed this dataset will instead be marked with 

890 having a NULL input. `Datastore` records will *not* be deleted; the 

891 caller is responsible for ensuring that the dataset has already been 

892 removed from all Datastores. 

893 

894 Parameters 

895 ---------- 

896 refs : `~collections.abc.Iterable` [`DatasetRef`] 

897 References to the datasets to be removed. Must include a valid 

898 ``id`` attribute, and should be considered invalidated upon return. 

899 

900 Raises 

901 ------ 

902 lsst.daf.butler.AmbiguousDatasetError 

903 Raised if any ``ref.id`` is `None`. 

904 lsst.daf.butler.registry.OrphanedRecordError 

905 Raised if any dataset is still present in any `Datastore`. 

906 """ 

907 raise NotImplementedError() 

908 

909 @abstractmethod 

910 def associate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

911 """Add existing datasets to a `~CollectionType.TAGGED` collection. 

912 

913 If a DatasetRef with the same exact ID is already in a collection 

914 nothing is changed. If a `DatasetRef` with the same `DatasetType` and 

915 data ID but with different ID exists in the collection, 

916 `~lsst.daf.butler.registry.ConflictingDefinitionError` is raised. 

917 

918 Parameters 

919 ---------- 

920 collection : `str` 

921 Indicates the collection the datasets should be associated with. 

922 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

923 An iterable of resolved `DatasetRef` instances that already exist 

924 in this `Registry`. 

925 

926 Raises 

927 ------ 

928 lsst.daf.butler.registry.ConflictingDefinitionError 

929 If a Dataset with the given `DatasetRef` already exists in the 

930 given collection. 

931 lsst.daf.butler.registry.MissingCollectionError 

932 Raised if ``collection`` does not exist in the registry. 

933 lsst.daf.butler.registry.CollectionTypeError 

934 Raise adding new datasets to the given ``collection`` is not 

935 allowed. 

936 """ 

937 raise NotImplementedError() 

938 

939 @abstractmethod 

940 def disassociate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

941 """Remove existing datasets from a `~CollectionType.TAGGED` collection. 

942 

943 ``collection`` and ``ref`` combinations that are not currently 

944 associated are silently ignored. 

945 

946 Parameters 

947 ---------- 

948 collection : `str` 

949 The collection the datasets should no longer be associated with. 

950 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

951 An iterable of resolved `DatasetRef` instances that already exist 

952 in this `Registry`. 

953 

954 Raises 

955 ------ 

956 lsst.daf.butler.AmbiguousDatasetError 

957 Raised if any of the given dataset references is unresolved. 

958 lsst.daf.butler.registry.MissingCollectionError 

959 Raised if ``collection`` does not exist in the registry. 

960 lsst.daf.butler.registry.CollectionTypeError 

961 Raise adding new datasets to the given ``collection`` is not 

962 allowed. 

963 """ 

964 raise NotImplementedError() 

965 

966 @abstractmethod 

967 def certify(self, collection: str, refs: Iterable[DatasetRef], timespan: Timespan) -> None: 

968 """Associate one or more datasets with a calibration collection and a 

969 validity range within it. 

970 

971 Parameters 

972 ---------- 

973 collection : `str` 

974 The name of an already-registered `~CollectionType.CALIBRATION` 

975 collection. 

976 refs : `Iterable` [ `DatasetRef` ] 

977 Datasets to be associated. 

978 timespan : `Timespan` 

979 The validity range for these datasets within the collection. 

980 

981 Raises 

982 ------ 

983 lsst.daf.butler.AmbiguousDatasetError 

984 Raised if any of the given `DatasetRef` instances is unresolved. 

985 lsst.daf.butler.registry.ConflictingDefinitionError 

986 Raised if the collection already contains a different dataset with 

987 the same `DatasetType` and data ID and an overlapping validity 

988 range. 

989 lsst.daf.butler.registry.CollectionTypeError 

990 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

991 collection or if one or more datasets are of a dataset type for 

992 which `DatasetType.isCalibration` returns `False`. 

993 """ 

994 raise NotImplementedError() 

995 

996 @abstractmethod 

997 def decertify( 

998 self, 

999 collection: str, 

1000 datasetType: Union[str, DatasetType], 

1001 timespan: Timespan, 

1002 *, 

1003 dataIds: Optional[Iterable[DataId]] = None, 

1004 ) -> None: 

1005 """Remove or adjust datasets to clear a validity range within a 

1006 calibration collection. 

1007 

1008 Parameters 

1009 ---------- 

1010 collection : `str` 

1011 The name of an already-registered `~CollectionType.CALIBRATION` 

1012 collection. 

1013 datasetType : `str` or `DatasetType` 

1014 Name or `DatasetType` instance for the datasets to be decertified. 

1015 timespan : `Timespan`, optional 

1016 The validity range to remove datasets from within the collection. 

1017 Datasets that overlap this range but are not contained by it will 

1018 have their validity ranges adjusted to not overlap it, which may 

1019 split a single dataset validity range into two. 

1020 dataIds : iterable [`dict` or `DataCoordinate`], optional 

1021 Data IDs that should be decertified within the given validity range 

1022 If `None`, all data IDs for ``self.datasetType`` will be 

1023 decertified. 

1024 

1025 Raises 

1026 ------ 

1027 lsst.daf.butler.registry.CollectionTypeError 

1028 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

1029 collection or if ``datasetType.isCalibration() is False``. 

1030 """ 

1031 raise NotImplementedError() 

1032 

1033 @abstractmethod 

1034 def getDatastoreBridgeManager(self) -> DatastoreRegistryBridgeManager: 

1035 """Return an object that allows a new `Datastore` instance to 

1036 communicate with this `Registry`. 

1037 

1038 Returns 

1039 ------- 

1040 manager : `~.interfaces.DatastoreRegistryBridgeManager` 

1041 Object that mediates communication between this `Registry` and its 

1042 associated datastores. 

1043 """ 

1044 raise NotImplementedError() 

1045 

1046 @abstractmethod 

1047 def getDatasetLocations(self, ref: DatasetRef) -> Iterable[str]: 

1048 """Retrieve datastore locations for a given dataset. 

1049 

1050 Parameters 

1051 ---------- 

1052 ref : `DatasetRef` 

1053 A reference to the dataset for which to retrieve storage 

1054 information. 

1055 

1056 Returns 

1057 ------- 

1058 datastores : `~collections.abc.Iterable` [ `str` ] 

1059 All the matching datastores holding this dataset. 

1060 

1061 Raises 

1062 ------ 

1063 lsst.daf.butler.AmbiguousDatasetError 

1064 Raised if ``ref.id`` is `None`. 

1065 """ 

1066 raise NotImplementedError() 

1067 

1068 @abstractmethod 

1069 def expandDataId( 

1070 self, 

1071 dataId: Optional[DataId] = None, 

1072 *, 

1073 graph: Optional[DimensionGraph] = None, 

1074 records: Optional[NameLookupMapping[DimensionElement, Optional[DimensionRecord]]] = None, 

1075 withDefaults: bool = True, 

1076 **kwargs: Any, 

1077 ) -> DataCoordinate: 

1078 """Expand a dimension-based data ID to include additional information. 

1079 

1080 Parameters 

1081 ---------- 

1082 dataId : `DataCoordinate` or `dict`, optional 

1083 Data ID to be expanded; augmented and overridden by ``kwargs``. 

1084 graph : `DimensionGraph`, optional 

1085 Set of dimensions for the expanded ID. If `None`, the dimensions 

1086 will be inferred from the keys of ``dataId`` and ``kwargs``. 

1087 Dimensions that are in ``dataId`` or ``kwargs`` but not in 

1088 ``graph`` are silently ignored, providing a way to extract and 

1089 ``graph`` expand a subset of a data ID. 

1090 records : `Mapping` [`str`, `DimensionRecord`], optional 

1091 Dimension record data to use before querying the database for that 

1092 data, keyed by element name. 

1093 withDefaults : `bool`, optional 

1094 Utilize ``self.defaults.dataId`` to fill in missing governor 

1095 dimension key-value pairs. Defaults to `True` (i.e. defaults are 

1096 used). 

1097 **kwargs 

1098 Additional keywords are treated like additional key-value pairs for 

1099 ``dataId``, extending and overriding 

1100 

1101 Returns 

1102 ------- 

1103 expanded : `DataCoordinate` 

1104 A data ID that includes full metadata for all of the dimensions it 

1105 identifies, i.e. guarantees that ``expanded.hasRecords()`` and 

1106 ``expanded.hasFull()`` both return `True`. 

1107 

1108 Raises 

1109 ------ 

1110 lsst.daf.butler.registry.DataIdError 

1111 Raised when ``dataId`` or keyword arguments specify unknown 

1112 dimensions or values, or when a resulting data ID contains 

1113 contradictory key-value pairs, according to dimension 

1114 relationships. 

1115 

1116 Notes 

1117 ----- 

1118 This method cannot be relied upon to reject invalid data ID values 

1119 for dimensions that do actually not have any record columns. For 

1120 efficiency reasons the records for these dimensions (which have only 

1121 dimension key values that are given by the caller) may be constructed 

1122 directly rather than obtained from the registry database. 

1123 """ 

1124 raise NotImplementedError() 

1125 

1126 @abstractmethod 

1127 def insertDimensionData( 

1128 self, 

1129 element: Union[DimensionElement, str], 

1130 *data: Union[Mapping[str, Any], DimensionRecord], 

1131 conform: bool = True, 

1132 replace: bool = False, 

1133 skip_existing: bool = False, 

1134 ) -> None: 

1135 """Insert one or more dimension records into the database. 

1136 

1137 Parameters 

1138 ---------- 

1139 element : `DimensionElement` or `str` 

1140 The `DimensionElement` or name thereof that identifies the table 

1141 records will be inserted into. 

1142 *data : `dict` or `DimensionRecord` 

1143 One or more records to insert. 

1144 conform : `bool`, optional 

1145 If `False` (`True` is default) perform no checking or conversions, 

1146 and assume that ``element`` is a `DimensionElement` instance and 

1147 ``data`` is a one or more `DimensionRecord` instances of the 

1148 appropriate subclass. 

1149 replace : `bool`, optional 

1150 If `True` (`False` is default), replace existing records in the 

1151 database if there is a conflict. 

1152 skip_existing : `bool`, optional 

1153 If `True` (`False` is default), skip insertion if a record with 

1154 the same primary key values already exists. Unlike 

1155 `syncDimensionData`, this will not detect when the given record 

1156 differs from what is in the database, and should not be used when 

1157 this is a concern. 

1158 """ 

1159 raise NotImplementedError() 

1160 

1161 @abstractmethod 

1162 def syncDimensionData( 

1163 self, 

1164 element: Union[DimensionElement, str], 

1165 row: Union[Mapping[str, Any], DimensionRecord], 

1166 conform: bool = True, 

1167 update: bool = False, 

1168 ) -> Union[bool, Dict[str, Any]]: 

1169 """Synchronize the given dimension record with the database, inserting 

1170 if it does not already exist and comparing values if it does. 

1171 

1172 Parameters 

1173 ---------- 

1174 element : `DimensionElement` or `str` 

1175 The `DimensionElement` or name thereof that identifies the table 

1176 records will be inserted into. 

1177 row : `dict` or `DimensionRecord` 

1178 The record to insert. 

1179 conform : `bool`, optional 

1180 If `False` (`True` is default) perform no checking or conversions, 

1181 and assume that ``element`` is a `DimensionElement` instance and 

1182 ``data`` is a one or more `DimensionRecord` instances of the 

1183 appropriate subclass. 

1184 update : `bool`, optional 

1185 If `True` (`False` is default), update the existing record in the 

1186 database if there is a conflict. 

1187 

1188 Returns 

1189 ------- 

1190 inserted_or_updated : `bool` or `dict` 

1191 `True` if a new row was inserted, `False` if no changes were 

1192 needed, or a `dict` mapping updated column names to their old 

1193 values if an update was performed (only possible if 

1194 ``update=True``). 

1195 

1196 Raises 

1197 ------ 

1198 lsst.daf.butler.registry.ConflictingDefinitionError 

1199 Raised if the record exists in the database (according to primary 

1200 key lookup) but is inconsistent with the given one. 

1201 """ 

1202 raise NotImplementedError() 

1203 

1204 @abstractmethod 

1205 def queryDatasetTypes( 

1206 self, 

1207 expression: Any = ..., 

1208 *, 

1209 components: Optional[bool] = None, 

1210 missing: Optional[List[str]] = None, 

1211 ) -> Iterable[DatasetType]: 

1212 """Iterate over the dataset types whose names match an expression. 

1213 

1214 Parameters 

1215 ---------- 

1216 expression : dataset type expression, optional 

1217 An expression that fully or partially identifies the dataset types 

1218 to return, such as a `str`, `re.Pattern`, or iterable thereof. 

1219 ``...`` can be used to return all dataset types, and is the 

1220 default. See :ref:`daf_butler_dataset_type_expressions` for more 

1221 information. 

1222 components : `bool`, optional 

1223 If `True`, apply all expression patterns to component dataset type 

1224 names as well. If `False`, never apply patterns to components. 

1225 If `None` (default), apply patterns to components only if their 

1226 parent datasets were not matched by the expression. 

1227 Fully-specified component datasets (`str` or `DatasetType` 

1228 instances) are always included. 

1229 

1230 Values other than `False` are deprecated, and only `False` will be 

1231 supported after v26. After v27 this argument will be removed 

1232 entirely. 

1233 missing : `list` of `str`, optional 

1234 String dataset type names that were explicitly given (i.e. not 

1235 regular expression patterns) but not found will be appended to this 

1236 list, if it is provided. 

1237 

1238 Returns 

1239 ------- 

1240 dataset_types : `~collections.abc.Iterable` [ `DatasetType`] 

1241 An `~collections.abc.Iterable` of `DatasetType` instances whose 

1242 names match ``expression``. 

1243 

1244 Raises 

1245 ------ 

1246 lsst.daf.butler.registry.DatasetTypeExpressionError 

1247 Raised when ``expression`` is invalid. 

1248 """ 

1249 raise NotImplementedError() 

1250 

1251 @abstractmethod 

1252 def queryCollections( 

1253 self, 

1254 expression: Any = ..., 

1255 datasetType: Optional[DatasetType] = None, 

1256 collectionTypes: Union[Iterable[CollectionType], CollectionType] = CollectionType.all(), 

1257 flattenChains: bool = False, 

1258 includeChains: Optional[bool] = None, 

1259 ) -> Sequence[str]: 

1260 """Iterate over the collections whose names match an expression. 

1261 

1262 Parameters 

1263 ---------- 

1264 expression : collection expression, optional 

1265 An expression that identifies the collections to return, such as 

1266 a `str` (for full matches or partial matches via globs), 

1267 `re.Pattern` (for partial matches), or iterable thereof. ``...`` 

1268 can be used to return all collections, and is the default. 

1269 See :ref:`daf_butler_collection_expressions` for more information. 

1270 datasetType : `DatasetType`, optional 

1271 If provided, only yield collections that may contain datasets of 

1272 this type. This is a conservative approximation in general; it may 

1273 yield collections that do not have any such datasets. 

1274 collectionTypes : `~collections.abc.Set` [`CollectionType`] or \ 

1275 `CollectionType`, optional 

1276 If provided, only yield collections of these types. 

1277 flattenChains : `bool`, optional 

1278 If `True` (`False` is default), recursively yield the child 

1279 collections of matching `~CollectionType.CHAINED` collections. 

1280 includeChains : `bool`, optional 

1281 If `True`, yield records for matching `~CollectionType.CHAINED` 

1282 collections. Default is the opposite of ``flattenChains``: include 

1283 either CHAINED collections or their children, but not both. 

1284 

1285 Returns 

1286 ------- 

1287 collections : `~collections.abc.Sequence` [ `str` ] 

1288 The names of collections that match ``expression``. 

1289 

1290 Raises 

1291 ------ 

1292 lsst.daf.butler.registry.CollectionExpressionError 

1293 Raised when ``expression`` is invalid. 

1294 

1295 Notes 

1296 ----- 

1297 The order in which collections are returned is unspecified, except that 

1298 the children of a `~CollectionType.CHAINED` collection are guaranteed 

1299 to be in the order in which they are searched. When multiple parent 

1300 `~CollectionType.CHAINED` collections match the same criteria, the 

1301 order in which the two lists appear is unspecified, and the lists of 

1302 children may be incomplete if a child has multiple parents. 

1303 """ 

1304 raise NotImplementedError() 

1305 

1306 @abstractmethod 

1307 def queryDatasets( 

1308 self, 

1309 datasetType: Any, 

1310 *, 

1311 collections: CollectionArgType | None = None, 

1312 dimensions: Optional[Iterable[Union[Dimension, str]]] = None, 

1313 dataId: Optional[DataId] = None, 

1314 where: str = "", 

1315 findFirst: bool = False, 

1316 components: Optional[bool] = None, 

1317 bind: Optional[Mapping[str, Any]] = None, 

1318 check: bool = True, 

1319 **kwargs: Any, 

1320 ) -> DatasetQueryResults: 

1321 """Query for and iterate over dataset references matching user-provided 

1322 criteria. 

1323 

1324 Parameters 

1325 ---------- 

1326 datasetType : dataset type expression 

1327 An expression that fully or partially identifies the dataset types 

1328 to be queried. Allowed types include `DatasetType`, `str`, 

1329 `re.Pattern`, and iterables thereof. The special value ``...`` can 

1330 be used to query all dataset types. See 

1331 :ref:`daf_butler_dataset_type_expressions` for more information. 

1332 collections : collection expression, optional 

1333 An expression that identifies the collections to search, such as a 

1334 `str` (for full matches or partial matches via globs), `re.Pattern` 

1335 (for partial matches), or iterable thereof. ``...`` can be used to 

1336 search all collections (actually just all `~CollectionType.RUN` 

1337 collections, because this will still find all datasets). 

1338 If not provided, ``self.default.collections`` is used. See 

1339 :ref:`daf_butler_collection_expressions` for more information. 

1340 dimensions : `~collections.abc.Iterable` of `Dimension` or `str` 

1341 Dimensions to include in the query (in addition to those used 

1342 to identify the queried dataset type(s)), either to constrain 

1343 the resulting datasets to those for which a matching dimension 

1344 exists, or to relate the dataset type's dimensions to dimensions 

1345 referenced by the ``dataId`` or ``where`` arguments. 

1346 dataId : `dict` or `DataCoordinate`, optional 

1347 A data ID whose key-value pairs are used as equality constraints 

1348 in the query. 

1349 where : `str`, optional 

1350 A string expression similar to a SQL WHERE clause. May involve 

1351 any column of a dimension table or (as a shortcut for the primary 

1352 key column of a dimension table) dimension name. See 

1353 :ref:`daf_butler_dimension_expressions` for more information. 

1354 findFirst : `bool`, optional 

1355 If `True` (`False` is default), for each result data ID, only 

1356 yield one `DatasetRef` of each `DatasetType`, from the first 

1357 collection in which a dataset of that dataset type appears 

1358 (according to the order of ``collections`` passed in). If `True`, 

1359 ``collections`` must not contain regular expressions and may not 

1360 be ``...``. 

1361 components : `bool`, optional 

1362 If `True`, apply all dataset expression patterns to component 

1363 dataset type names as well. If `False`, never apply patterns to 

1364 components. If `None` (default), apply patterns to components only 

1365 if their parent datasets were not matched by the expression. 

1366 Fully-specified component datasets (`str` or `DatasetType` 

1367 instances) are always included. 

1368 

1369 Values other than `False` are deprecated, and only `False` will be 

1370 supported after v26. After v27 this argument will be removed 

1371 entirely. 

1372 bind : `~collections.abc.Mapping`, optional 

1373 Mapping containing literal values that should be injected into the 

1374 ``where`` expression, keyed by the identifiers they replace. 

1375 Values of collection type can be expanded in some cases; see 

1376 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1377 information. 

1378 check : `bool`, optional 

1379 If `True` (default) check the query for consistency before 

1380 executing it. This may reject some valid queries that resemble 

1381 common mistakes (e.g. queries for visits without specifying an 

1382 instrument). 

1383 **kwargs 

1384 Additional keyword arguments are forwarded to 

1385 `DataCoordinate.standardize` when processing the ``dataId`` 

1386 argument (and may be used to provide a constraining data ID even 

1387 when the ``dataId`` argument is `None`). 

1388 

1389 Returns 

1390 ------- 

1391 refs : `.queries.DatasetQueryResults` 

1392 Dataset references matching the given query criteria. Nested data 

1393 IDs are guaranteed to include values for all implied dimensions 

1394 (i.e. `DataCoordinate.hasFull` will return `True`), but will not 

1395 include dimension records (`DataCoordinate.hasRecords` will be 

1396 `False`) unless `~.queries.DatasetQueryResults.expanded` is 

1397 called on the result object (which returns a new one). 

1398 

1399 Raises 

1400 ------ 

1401 lsst.daf.butler.registry.DatasetTypeExpressionError 

1402 Raised when ``datasetType`` expression is invalid. 

1403 TypeError 

1404 Raised when the arguments are incompatible, such as when a 

1405 collection wildcard is passed when ``findFirst`` is `True`, or 

1406 when ``collections`` is `None` and ``self.defaults.collections`` is 

1407 also `None`. 

1408 lsst.daf.butler.registry.DataIdError 

1409 Raised when ``dataId`` or keyword arguments specify unknown 

1410 dimensions or values, or when they contain inconsistent values. 

1411 lsst.daf.butler.registry.UserExpressionError 

1412 Raised when ``where`` expression is invalid. 

1413 

1414 Notes 

1415 ----- 

1416 When multiple dataset types are queried in a single call, the 

1417 results of this operation are equivalent to querying for each dataset 

1418 type separately in turn, and no information about the relationships 

1419 between datasets of different types is included. In contexts where 

1420 that kind of information is important, the recommended pattern is to 

1421 use `queryDataIds` to first obtain data IDs (possibly with the 

1422 desired dataset types and collections passed as constraints to the 

1423 query), and then use multiple (generally much simpler) calls to 

1424 `queryDatasets` with the returned data IDs passed as constraints. 

1425 """ 

1426 raise NotImplementedError() 

1427 

1428 @abstractmethod 

1429 def queryDataIds( 

1430 self, 

1431 dimensions: Union[Iterable[Union[Dimension, str]], Dimension, str], 

1432 *, 

1433 dataId: Optional[DataId] = None, 

1434 datasets: Any = None, 

1435 collections: CollectionArgType | None = None, 

1436 where: str = "", 

1437 components: Optional[bool] = None, 

1438 bind: Optional[Mapping[str, Any]] = None, 

1439 check: bool = True, 

1440 **kwargs: Any, 

1441 ) -> DataCoordinateQueryResults: 

1442 """Query for data IDs matching user-provided criteria. 

1443 

1444 Parameters 

1445 ---------- 

1446 dimensions : `Dimension` or `str`, or iterable thereof 

1447 The dimensions of the data IDs to yield, as either `Dimension` 

1448 instances or `str`. Will be automatically expanded to a complete 

1449 `DimensionGraph`. 

1450 dataId : `dict` or `DataCoordinate`, optional 

1451 A data ID whose key-value pairs are used as equality constraints 

1452 in the query. 

1453 datasets : dataset type expression, optional 

1454 An expression that fully or partially identifies dataset types 

1455 that should constrain the yielded data IDs. For example, including 

1456 "raw" here would constrain the yielded ``instrument``, 

1457 ``exposure``, ``detector``, and ``physical_filter`` values to only 

1458 those for which at least one "raw" dataset exists in 

1459 ``collections``. Allowed types include `DatasetType`, `str`, 

1460 and iterables thereof. Regular expression objects (i.e. 

1461 `re.Pattern`) are deprecated and will be removed after the v26 

1462 release. See :ref:`daf_butler_dataset_type_expressions` for more 

1463 information. 

1464 collections : collection expression, optional 

1465 An expression that identifies the collections to search for 

1466 datasets, such as a `str` (for full matches or partial matches 

1467 via globs), `re.Pattern` (for partial matches), or iterable 

1468 thereof. ``...`` can be used to search all collections (actually 

1469 just all `~CollectionType.RUN` collections, because this will 

1470 still find all datasets). If not provided, 

1471 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1472 is also passed. See :ref:`daf_butler_collection_expressions` for 

1473 more information. 

1474 where : `str`, optional 

1475 A string expression similar to a SQL WHERE clause. May involve 

1476 any column of a dimension table or (as a shortcut for the primary 

1477 key column of a dimension table) dimension name. See 

1478 :ref:`daf_butler_dimension_expressions` for more information. 

1479 components : `bool`, optional 

1480 If `True`, apply all dataset expression patterns to component 

1481 dataset type names as well. If `False`, never apply patterns to 

1482 components. If `None` (default), apply patterns to components only 

1483 if their parent datasets were not matched by the expression. 

1484 Fully-specified component datasets (`str` or `DatasetType` 

1485 instances) are always included. 

1486 

1487 Values other than `False` are deprecated, and only `False` will be 

1488 supported after v26. After v27 this argument will be removed 

1489 entirely. 

1490 bind : `~collections.abc.Mapping`, optional 

1491 Mapping containing literal values that should be injected into the 

1492 ``where`` expression, keyed by the identifiers they replace. 

1493 Values of collection type can be expanded in some cases; see 

1494 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1495 information. 

1496 check : `bool`, optional 

1497 If `True` (default) check the query for consistency before 

1498 executing it. This may reject some valid queries that resemble 

1499 common mistakes (e.g. queries for visits without specifying an 

1500 instrument). 

1501 **kwargs 

1502 Additional keyword arguments are forwarded to 

1503 `DataCoordinate.standardize` when processing the ``dataId`` 

1504 argument (and may be used to provide a constraining data ID even 

1505 when the ``dataId`` argument is `None`). 

1506 

1507 Returns 

1508 ------- 

1509 dataIds : `.queries.DataCoordinateQueryResults` 

1510 Data IDs matching the given query parameters. These are guaranteed 

1511 to identify all dimensions (`DataCoordinate.hasFull` returns 

1512 `True`), but will not contain `DimensionRecord` objects 

1513 (`DataCoordinate.hasRecords` returns `False`). Call 

1514 `~.queries.DataCoordinateQueryResults.expanded` on the 

1515 returned object to fetch those (and consider using 

1516 `~.queries.DataCoordinateQueryResults.materialize` on the 

1517 returned object first if the expected number of rows is very 

1518 large). See documentation for those methods for additional 

1519 information. 

1520 

1521 Raises 

1522 ------ 

1523 lsst.daf.butler.registry.NoDefaultCollectionError 

1524 Raised if ``collections`` is `None` and 

1525 ``self.defaults.collections`` is `None`. 

1526 lsst.daf.butler.registry.CollectionExpressionError 

1527 Raised when ``collections`` expression is invalid. 

1528 lsst.daf.butler.registry.DataIdError 

1529 Raised when ``dataId`` or keyword arguments specify unknown 

1530 dimensions or values, or when they contain inconsistent values. 

1531 lsst.daf.butler.registry.DatasetTypeExpressionError 

1532 Raised when ``datasetType`` expression is invalid. 

1533 lsst.daf.butler.registry.UserExpressionError 

1534 Raised when ``where`` expression is invalid. 

1535 """ 

1536 raise NotImplementedError() 

1537 

1538 @abstractmethod 

1539 def queryDimensionRecords( 

1540 self, 

1541 element: Union[DimensionElement, str], 

1542 *, 

1543 dataId: Optional[DataId] = None, 

1544 datasets: Any = None, 

1545 collections: CollectionArgType | None = None, 

1546 where: str = "", 

1547 components: Optional[bool] = None, 

1548 bind: Optional[Mapping[str, Any]] = None, 

1549 check: bool = True, 

1550 **kwargs: Any, 

1551 ) -> DimensionRecordQueryResults: 

1552 """Query for dimension information matching user-provided criteria. 

1553 

1554 Parameters 

1555 ---------- 

1556 element : `DimensionElement` or `str` 

1557 The dimension element to obtain records for. 

1558 dataId : `dict` or `DataCoordinate`, optional 

1559 A data ID whose key-value pairs are used as equality constraints 

1560 in the query. 

1561 datasets : dataset type expression, optional 

1562 An expression that fully or partially identifies dataset types 

1563 that should constrain the yielded records. See `queryDataIds` and 

1564 :ref:`daf_butler_dataset_type_expressions` for more information. 

1565 collections : collection expression, optional 

1566 An expression that identifies the collections to search for 

1567 datasets, such as a `str` (for full matches or partial matches 

1568 via globs), `re.Pattern` (for partial matches), or iterable 

1569 thereof. ``...`` can be used to search all collections (actually 

1570 just all `~CollectionType.RUN` collections, because this will 

1571 still find all datasets). If not provided, 

1572 ``self.default.collections`` is used. Ignored unless ``datasets`` 

1573 is also passed. See :ref:`daf_butler_collection_expressions` for 

1574 more information. 

1575 where : `str`, optional 

1576 A string expression similar to a SQL WHERE clause. See 

1577 `queryDataIds` and :ref:`daf_butler_dimension_expressions` for more 

1578 information. 

1579 components : `bool`, optional 

1580 Whether to apply dataset expressions to components as well. 

1581 See `queryDataIds` for more information. 

1582 

1583 Values other than `False` are deprecated, and only `False` will be 

1584 supported after v26. After v27 this argument will be removed 

1585 entirely. 

1586 bind : `~collections.abc.Mapping`, optional 

1587 Mapping containing literal values that should be injected into the 

1588 ``where`` expression, keyed by the identifiers they replace. 

1589 Values of collection type can be expanded in some cases; see 

1590 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1591 information. 

1592 check : `bool`, optional 

1593 If `True` (default) check the query for consistency before 

1594 executing it. This may reject some valid queries that resemble 

1595 common mistakes (e.g. queries for visits without specifying an 

1596 instrument). 

1597 **kwargs 

1598 Additional keyword arguments are forwarded to 

1599 `DataCoordinate.standardize` when processing the ``dataId`` 

1600 argument (and may be used to provide a constraining data ID even 

1601 when the ``dataId`` argument is `None`). 

1602 

1603 Returns 

1604 ------- 

1605 dataIds : `.queries.DimensionRecordQueryResults` 

1606 Data IDs matching the given query parameters. 

1607 

1608 Raises 

1609 ------ 

1610 lsst.daf.butler.registry.NoDefaultCollectionError 

1611 Raised if ``collections`` is `None` and 

1612 ``self.defaults.collections`` is `None`. 

1613 lsst.daf.butler.registry.CollectionExpressionError 

1614 Raised when ``collections`` expression is invalid. 

1615 lsst.daf.butler.registry.DataIdError 

1616 Raised when ``dataId`` or keyword arguments specify unknown 

1617 dimensions or values, or when they contain inconsistent values. 

1618 lsst.daf.butler.registry.DatasetTypeExpressionError 

1619 Raised when ``datasetType`` expression is invalid. 

1620 lsst.daf.butler.registry.UserExpressionError 

1621 Raised when ``where`` expression is invalid. 

1622 """ 

1623 raise NotImplementedError() 

1624 

1625 @abstractmethod 

1626 def queryDatasetAssociations( 

1627 self, 

1628 datasetType: Union[str, DatasetType], 

1629 collections: CollectionArgType | None = ..., 

1630 *, 

1631 collectionTypes: Iterable[CollectionType] = CollectionType.all(), 

1632 flattenChains: bool = False, 

1633 ) -> Iterator[DatasetAssociation]: 

1634 """Iterate over dataset-collection combinations where the dataset is in 

1635 the collection. 

1636 

1637 This method is a temporary placeholder for better support for 

1638 association results in `queryDatasets`. It will probably be 

1639 removed in the future, and should be avoided in production code 

1640 whenever possible. 

1641 

1642 Parameters 

1643 ---------- 

1644 datasetType : `DatasetType` or `str` 

1645 A dataset type object or the name of one. 

1646 collections : collection expression, optional 

1647 An expression that identifies the collections to search for 

1648 datasets, such as a `str` (for full matches or partial matches 

1649 via globs), `re.Pattern` (for partial matches), or iterable 

1650 thereof. ``...`` can be used to search all collections (actually 

1651 just all `~CollectionType.RUN` collections, because this will still 

1652 find all datasets). If not provided, ``self.default.collections`` 

1653 is used. See :ref:`daf_butler_collection_expressions` for more 

1654 information. 

1655 collectionTypes : `~collections.abc.Set` [ `CollectionType` ], optional 

1656 If provided, only yield associations from collections of these 

1657 types. 

1658 flattenChains : `bool`, optional 

1659 If `True` (default) search in the children of 

1660 `~CollectionType.CHAINED` collections. If `False`, ``CHAINED`` 

1661 collections are ignored. 

1662 

1663 Yields 

1664 ------ 

1665 association : `.DatasetAssociation` 

1666 Object representing the relationship between a single dataset and 

1667 a single collection. 

1668 

1669 Raises 

1670 ------ 

1671 lsst.daf.butler.registry.NoDefaultCollectionError 

1672 Raised if ``collections`` is `None` and 

1673 ``self.defaults.collections`` is `None`. 

1674 lsst.daf.butler.registry.CollectionExpressionError 

1675 Raised when ``collections`` expression is invalid. 

1676 """ 

1677 raise NotImplementedError() 

1678 

1679 @property 

1680 def obsCoreTableManager(self) -> ObsCoreTableManager | None: 

1681 """ObsCore manager instance for this registry 

1682 (`~.interfaces.ObsCoreTableManager` 

1683 or `None`). 

1684 

1685 ObsCore manager may not be implemented for all registry backend, or 

1686 may not be enabled for many repositories. 

1687 """ 

1688 return None 

1689 

1690 storageClasses: StorageClassFactory 

1691 """All storage classes known to the registry (`StorageClassFactory`). 

1692 """ 

1693 

1694 datasetIdFactory: DatasetIdFactory 

1695 """Factory for dataset IDs."""