Coverage for python/lsst/daf/butler/registry/_registry.py: 98%

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ("CollectionArgType", "Registry") 

31 

32import contextlib 

33import logging 

34import re 

35from abc import ABC, abstractmethod 

36from collections.abc import Iterable, Iterator, Mapping, Sequence 

37from types import EllipsisType 

38from typing import TYPE_CHECKING, Any, TypeAlias 

39 

40from .._collection_type import CollectionType 

41from .._dataset_association import DatasetAssociation 

42from .._dataset_ref import DatasetId, DatasetIdGenEnum, DatasetRef 

43from .._dataset_type import DatasetType 

44from .._storage_class import StorageClassFactory 

45from .._timespan import Timespan 

46from ..dimensions import ( 

47 DataCoordinate, 

48 DataId, 

49 DimensionElement, 

50 DimensionGroup, 

51 DimensionRecord, 

52 DimensionUniverse, 

53) 

54from ._collection_summary import CollectionSummary 

55from ._defaults import RegistryDefaults 

56from .queries import DataCoordinateQueryResults, DatasetQueryResults, DimensionRecordQueryResults 

57from .wildcards import CollectionWildcard 

58 

59if TYPE_CHECKING: 

60 from .interfaces import ObsCoreTableManager 

61 

62_LOG = logging.getLogger(__name__) 

63 

64# TYpe alias for `collections` arguments. 

65CollectionArgType: TypeAlias = ( 

66 str | re.Pattern | Iterable[str | re.Pattern] | EllipsisType | CollectionWildcard 

67) 

68 

69 

70class Registry(ABC): 

71 """Abstract Registry interface. 

72 

73 All subclasses should store `~lsst.daf.butler.registry.RegistryDefaults` in 

74 a ``_defaults`` property. No other properties are assumed shared between 

75 implementations. 

76 

77 See Also 

78 -------- 

79 lsst.daf.butler.Butler 

80 """ 

81 

82 @abstractmethod 

83 def isWriteable(self) -> bool: 

84 """Return `True` if this registry allows write operations, and `False` 

85 otherwise. 

86 """ 

87 raise NotImplementedError() 

88 

89 @property 

90 @abstractmethod 

91 def dimensions(self) -> DimensionUniverse: 

92 """Definitions of all dimensions recognized by this `Registry` 

93 (`DimensionUniverse`). 

94 """ 

95 raise NotImplementedError() 

96 

97 @property 

98 @abstractmethod 

99 def defaults(self) -> RegistryDefaults: 

100 """Default collection search path and/or output `~CollectionType.RUN` 

101 collection (`~lsst.daf.butler.registry.RegistryDefaults`). 

102 

103 This is an immutable struct whose components may not be set 

104 individually, but the entire struct can be set by assigning to this 

105 property. 

106 """ 

107 raise NotImplementedError() 

108 

109 @defaults.setter 

110 @abstractmethod 

111 def defaults(self, value: RegistryDefaults) -> None: 

112 raise NotImplementedError() 

113 

114 @abstractmethod 

115 def refresh(self) -> None: 

116 """Refresh all in-memory state by querying the database. 

117 

118 This may be necessary to enable querying for entities added by other 

119 registry instances after this one was constructed. 

120 """ 

121 raise NotImplementedError() 

122 

123 @abstractmethod 

124 def refresh_collection_summaries(self) -> None: 

125 """Refresh content of the collection summary tables in the database. 

126 

127 This method can be run to cleanup the contents of the summary tables 

128 after removing datasets from collections. 

129 """ 

130 raise NotImplementedError() 

131 

132 @abstractmethod 

133 def caching_context(self) -> contextlib.AbstractContextManager[None]: 

134 """Context manager that enables caching.""" 

135 raise NotImplementedError() 

136 

137 @contextlib.contextmanager 

138 @abstractmethod 

139 def transaction(self, *, savepoint: bool = False) -> Iterator[None]: 

140 """Return a context manager that represents a transaction. 

141 

142 Parameters 

143 ---------- 

144 savepoint : `bool` 

145 Whether to create a SAVEPOINT or not. 

146 """ 

147 raise NotImplementedError() 

148 

149 def resetConnectionPool(self) -> None: 

150 """Reset connection pool for registry if relevant. 

151 

152 This operation can be used reset connections to servers when 

153 using registry with fork-based multiprocessing. This method should 

154 usually be called by the child process immediately 

155 after the fork. 

156 

157 The base class implementation is a no-op. 

158 """ 

159 pass 

160 

161 @abstractmethod 

162 def registerCollection( 

163 self, name: str, type: CollectionType = CollectionType.TAGGED, doc: str | None = None 

164 ) -> bool: 

165 """Add a new collection if one with the given name does not exist. 

166 

167 Parameters 

168 ---------- 

169 name : `str` 

170 The name of the collection to create. 

171 type : `CollectionType` 

172 Enum value indicating the type of collection to create. 

173 doc : `str`, optional 

174 Documentation string for the collection. 

175 

176 Returns 

177 ------- 

178 registered : `bool` 

179 Boolean indicating whether the collection was already registered 

180 or was created by this call. 

181 

182 Notes 

183 ----- 

184 This method cannot be called within transactions, as it needs to be 

185 able to perform its own transaction to be concurrent. 

186 """ 

187 raise NotImplementedError() 

188 

189 @abstractmethod 

190 def getCollectionType(self, name: str) -> CollectionType: 

191 """Return an enumeration value indicating the type of the given 

192 collection. 

193 

194 Parameters 

195 ---------- 

196 name : `str` 

197 The name of the collection. 

198 

199 Returns 

200 ------- 

201 type : `CollectionType` 

202 Enum value indicating the type of this collection. 

203 

204 Raises 

205 ------ 

206 lsst.daf.butler.registry.MissingCollectionError 

207 Raised if no collection with the given name exists. 

208 """ 

209 raise NotImplementedError() 

210 

211 @abstractmethod 

212 def registerRun(self, name: str, doc: str | None = None) -> bool: 

213 """Add a new run if one with the given name does not exist. 

214 

215 Parameters 

216 ---------- 

217 name : `str` 

218 The name of the run to create. 

219 doc : `str`, optional 

220 Documentation string for the collection. 

221 

222 Returns 

223 ------- 

224 registered : `bool` 

225 Boolean indicating whether a new run was registered. `False` 

226 if it already existed. 

227 

228 Notes 

229 ----- 

230 This method cannot be called within transactions, as it needs to be 

231 able to perform its own transaction to be concurrent. 

232 """ 

233 raise NotImplementedError() 

234 

235 @abstractmethod 

236 def removeCollection(self, name: str) -> None: 

237 """Remove the given collection from the registry. 

238 

239 Parameters 

240 ---------- 

241 name : `str` 

242 The name of the collection to remove. 

243 

244 Raises 

245 ------ 

246 lsst.daf.butler.registry.MissingCollectionError 

247 Raised if no collection with the given name exists. 

248 sqlalchemy.exc.IntegrityError 

249 Raised if the database rows associated with the collection are 

250 still referenced by some other table, such as a dataset in a 

251 datastore (for `~CollectionType.RUN` collections only) or a 

252 `~CollectionType.CHAINED` collection of which this collection is 

253 a child. 

254 

255 Notes 

256 ----- 

257 If this is a `~CollectionType.RUN` collection, all datasets and quanta 

258 in it will removed from the `Registry` database. This requires that 

259 those datasets be removed (or at least trashed) from any datastores 

260 that hold them first. 

261 

262 A collection may not be deleted as long as it is referenced by a 

263 `~CollectionType.CHAINED` collection; the ``CHAINED`` collection must 

264 be deleted or redefined first. 

265 """ 

266 raise NotImplementedError() 

267 

268 @abstractmethod 

269 def getCollectionChain(self, parent: str) -> Sequence[str]: 

270 """Return the child collections in a `~CollectionType.CHAINED` 

271 collection. 

272 

273 Parameters 

274 ---------- 

275 parent : `str` 

276 Name of the chained collection. Must have already been added via 

277 a call to `Registry.registerCollection`. 

278 

279 Returns 

280 ------- 

281 children : `~collections.abc.Sequence` [ `str` ] 

282 An ordered sequence of collection names that are searched when the 

283 given chained collection is searched. 

284 

285 Raises 

286 ------ 

287 lsst.daf.butler.registry.MissingCollectionError 

288 Raised if ``parent`` does not exist in the `Registry`. 

289 lsst.daf.butler.registry.CollectionTypeError 

290 Raised if ``parent`` does not correspond to a 

291 `~CollectionType.CHAINED` collection. 

292 """ 

293 raise NotImplementedError() 

294 

295 @abstractmethod 

296 def setCollectionChain(self, parent: str, children: Any, *, flatten: bool = False) -> None: 

297 """Define or redefine a `~CollectionType.CHAINED` collection. 

298 

299 Parameters 

300 ---------- 

301 parent : `str` 

302 Name of the chained collection. Must have already been added via 

303 a call to `Registry.registerCollection`. 

304 children : collection expression 

305 An expression defining an ordered search of child collections, 

306 generally an iterable of `str`; see 

307 :ref:`daf_butler_collection_expressions` for more information. 

308 flatten : `bool`, optional 

309 If `True` (`False` is default), recursively flatten out any nested 

310 `~CollectionType.CHAINED` collections in ``children`` first. 

311 

312 Raises 

313 ------ 

314 lsst.daf.butler.registry.MissingCollectionError 

315 Raised when any of the given collections do not exist in the 

316 `Registry`. 

317 lsst.daf.butler.registry.CollectionTypeError 

318 Raised if ``parent`` does not correspond to a 

319 `~CollectionType.CHAINED` collection. 

320 ValueError 

321 Raised if the given collections contains a cycle. 

322 """ 

323 raise NotImplementedError() 

324 

325 @abstractmethod 

326 def getCollectionParentChains(self, collection: str) -> set[str]: 

327 """Return the CHAINED collections that directly contain the given one. 

328 

329 Parameters 

330 ---------- 

331 collection : `str` 

332 Name of the collection. 

333 

334 Returns 

335 ------- 

336 chains : `set` of `str` 

337 Set of `~CollectionType.CHAINED` collection names. 

338 """ 

339 raise NotImplementedError() 

340 

341 @abstractmethod 

342 def getCollectionDocumentation(self, collection: str) -> str | None: 

343 """Retrieve the documentation string for a collection. 

344 

345 Parameters 

346 ---------- 

347 collection : `str` 

348 Name of the collection. 

349 

350 Returns 

351 ------- 

352 docs : `str` or `None` 

353 Docstring for the collection with the given name. 

354 """ 

355 raise NotImplementedError() 

356 

357 @abstractmethod 

358 def setCollectionDocumentation(self, collection: str, doc: str | None) -> None: 

359 """Set the documentation string for a collection. 

360 

361 Parameters 

362 ---------- 

363 collection : `str` 

364 Name of the collection. 

365 doc : `str` or `None` 

366 Docstring for the collection with the given name; will replace any 

367 existing docstring. Passing `None` will remove any existing 

368 docstring. 

369 """ 

370 raise NotImplementedError() 

371 

372 @abstractmethod 

373 def getCollectionSummary(self, collection: str) -> CollectionSummary: 

374 """Return a summary for the given collection. 

375 

376 Parameters 

377 ---------- 

378 collection : `str` 

379 Name of the collection for which a summary is to be retrieved. 

380 

381 Returns 

382 ------- 

383 summary : `~lsst.daf.butler.registry.CollectionSummary` 

384 Summary of the dataset types and governor dimension values in 

385 this collection. 

386 """ 

387 raise NotImplementedError() 

388 

389 @abstractmethod 

390 def registerDatasetType(self, datasetType: DatasetType) -> bool: 

391 """Add a new `DatasetType` to the Registry. 

392 

393 It is not an error to register the same `DatasetType` twice. 

394 

395 Parameters 

396 ---------- 

397 datasetType : `DatasetType` 

398 The `DatasetType` to be added. 

399 

400 Returns 

401 ------- 

402 inserted : `bool` 

403 `True` if ``datasetType`` was inserted, `False` if an identical 

404 existing `DatasetType` was found. Note that in either case the 

405 DatasetType is guaranteed to be defined in the Registry 

406 consistently with the given definition. 

407 

408 Raises 

409 ------ 

410 ValueError 

411 Raised if the dimensions or storage class are invalid. 

412 lsst.daf.butler.registry.ConflictingDefinitionError 

413 Raised if this `DatasetType` is already registered with a different 

414 definition. 

415 

416 Notes 

417 ----- 

418 This method cannot be called within transactions, as it needs to be 

419 able to perform its own transaction to be concurrent. 

420 """ 

421 raise NotImplementedError() 

422 

423 @abstractmethod 

424 def removeDatasetType(self, name: str | tuple[str, ...]) -> None: 

425 """Remove the named `DatasetType` from the registry. 

426 

427 .. warning:: 

428 

429 Registry implementations can cache the dataset type definitions. 

430 This means that deleting the dataset type definition may result in 

431 unexpected behavior from other butler processes that are active 

432 that have not seen the deletion. 

433 

434 Parameters 

435 ---------- 

436 name : `str` or `tuple` [`str`] 

437 Name of the type to be removed or tuple containing a list of type 

438 names to be removed. Wildcards are allowed. 

439 

440 Raises 

441 ------ 

442 lsst.daf.butler.registry.OrphanedRecordError 

443 Raised if an attempt is made to remove the dataset type definition 

444 when there are already datasets associated with it. 

445 

446 Notes 

447 ----- 

448 If the dataset type is not registered the method will return without 

449 action. 

450 """ 

451 raise NotImplementedError() 

452 

453 @abstractmethod 

454 def getDatasetType(self, name: str) -> DatasetType: 

455 """Get the `DatasetType`. 

456 

457 Parameters 

458 ---------- 

459 name : `str` 

460 Name of the type. 

461 

462 Returns 

463 ------- 

464 type : `DatasetType` 

465 The `DatasetType` associated with the given name. 

466 

467 Raises 

468 ------ 

469 lsst.daf.butler.registry.MissingDatasetTypeError 

470 Raised if the requested dataset type has not been registered. 

471 

472 Notes 

473 ----- 

474 This method handles component dataset types automatically, though most 

475 other registry operations do not. 

476 """ 

477 raise NotImplementedError() 

478 

479 @abstractmethod 

480 def supportsIdGenerationMode(self, mode: DatasetIdGenEnum) -> bool: 

481 """Test whether the given dataset ID generation mode is supported by 

482 `insertDatasets`. 

483 

484 Parameters 

485 ---------- 

486 mode : `DatasetIdGenEnum` 

487 Enum value for the mode to test. 

488 

489 Returns 

490 ------- 

491 supported : `bool` 

492 Whether the given mode is supported. 

493 """ 

494 raise NotImplementedError() 

495 

496 @abstractmethod 

497 def findDataset( 

498 self, 

499 datasetType: DatasetType | str, 

500 dataId: DataId | None = None, 

501 *, 

502 collections: CollectionArgType | None = None, 

503 timespan: Timespan | None = None, 

504 datastore_records: bool = False, 

505 **kwargs: Any, 

506 ) -> DatasetRef | None: 

507 """Find a dataset given its `DatasetType` and data ID. 

508 

509 This can be used to obtain a `DatasetRef` that permits the dataset to 

510 be read from a `Datastore`. If the dataset is a component and can not 

511 be found using the provided dataset type, a dataset ref for the parent 

512 will be returned instead but with the correct dataset type. 

513 

514 Parameters 

515 ---------- 

516 datasetType : `DatasetType` or `str` 

517 A `DatasetType` or the name of one. If this is a `DatasetType` 

518 instance, its storage class will be respected and propagated to 

519 the output, even if it differs from the dataset type definition 

520 in the registry, as long as the storage classes are convertible. 

521 dataId : `dict` or `DataCoordinate`, optional 

522 A `dict`-like object containing the `Dimension` links that identify 

523 the dataset within a collection. 

524 collections : collection expression, optional 

525 An expression that fully or partially identifies the collections to 

526 search for the dataset; see 

527 :ref:`daf_butler_collection_expressions` for more information. 

528 Defaults to ``self.defaults.collections``. 

529 timespan : `Timespan`, optional 

530 A timespan that the validity range of the dataset must overlap. 

531 If not provided, any `~CollectionType.CALIBRATION` collections 

532 matched by the ``collections`` argument will not be searched. 

533 datastore_records : `bool`, optional 

534 Whether to attach datastore records to the `DatasetRef`. 

535 **kwargs 

536 Additional keyword arguments passed to 

537 `DataCoordinate.standardize` to convert ``dataId`` to a true 

538 `DataCoordinate` or augment an existing one. 

539 

540 Returns 

541 ------- 

542 ref : `DatasetRef` 

543 A reference to the dataset, or `None` if no matching Dataset 

544 was found. 

545 

546 Raises 

547 ------ 

548 lsst.daf.butler.registry.NoDefaultCollectionError 

549 Raised if ``collections`` is `None` and 

550 ``self.defaults.collections`` is `None`. 

551 LookupError 

552 Raised if one or more data ID keys are missing. 

553 lsst.daf.butler.registry.MissingDatasetTypeError 

554 Raised if the dataset type does not exist. 

555 lsst.daf.butler.registry.MissingCollectionError 

556 Raised if any of ``collections`` does not exist in the registry. 

557 

558 Notes 

559 ----- 

560 This method simply returns `None` and does not raise an exception even 

561 when the set of collections searched is intrinsically incompatible with 

562 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but 

563 only `~CollectionType.CALIBRATION` collections are being searched. 

564 This may make it harder to debug some lookup failures, but the behavior 

565 is intentional; we consider it more important that failed searches are 

566 reported consistently, regardless of the reason, and that adding 

567 additional collections that do not contain a match to the search path 

568 never changes the behavior. 

569 

570 This method handles component dataset types automatically, though most 

571 other registry operations do not. 

572 """ 

573 raise NotImplementedError() 

574 

575 @abstractmethod 

576 def insertDatasets( 

577 self, 

578 datasetType: DatasetType | str, 

579 dataIds: Iterable[DataId], 

580 run: str | None = None, 

581 expand: bool = True, 

582 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE, 

583 ) -> list[DatasetRef]: 

584 """Insert one or more datasets into the `Registry`. 

585 

586 This always adds new datasets; to associate existing datasets with 

587 a new collection, use ``associate``. 

588 

589 Parameters 

590 ---------- 

591 datasetType : `DatasetType` or `str` 

592 A `DatasetType` or the name of one. 

593 dataIds : `~collections.abc.Iterable` of `dict` or `DataCoordinate` 

594 Dimension-based identifiers for the new datasets. 

595 run : `str`, optional 

596 The name of the run that produced the datasets. Defaults to 

597 ``self.defaults.run``. 

598 expand : `bool`, optional 

599 If `True` (default), expand data IDs as they are inserted. This is 

600 necessary in general to allow datastore to generate file templates, 

601 but it may be disabled if the caller can guarantee this is 

602 unnecessary. 

603 idGenerationMode : `DatasetIdGenEnum`, optional 

604 Specifies option for generating dataset IDs. By default unique IDs 

605 are generated for each inserted dataset. 

606 

607 Returns 

608 ------- 

609 refs : `list` of `DatasetRef` 

610 Resolved `DatasetRef` instances for all given data IDs (in the same 

611 order). 

612 

613 Raises 

614 ------ 

615 lsst.daf.butler.registry.DatasetTypeError 

616 Raised if ``datasetType`` is not known to registry. 

617 lsst.daf.butler.registry.CollectionTypeError 

618 Raised if ``run`` collection type is not `~CollectionType.RUN`. 

619 lsst.daf.butler.registry.NoDefaultCollectionError 

620 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

621 lsst.daf.butler.registry.ConflictingDefinitionError 

622 If a dataset with the same dataset type and data ID as one of those 

623 given already exists in ``run``. 

624 lsst.daf.butler.registry.MissingCollectionError 

625 Raised if ``run`` does not exist in the registry. 

626 """ 

627 raise NotImplementedError() 

628 

629 @abstractmethod 

630 def _importDatasets( 

631 self, 

632 datasets: Iterable[DatasetRef], 

633 expand: bool = True, 

634 ) -> list[DatasetRef]: 

635 """Import one or more datasets into the `Registry`. 

636 

637 Difference from `insertDatasets` method is that this method accepts 

638 `DatasetRef` instances which should already be resolved and have a 

639 dataset ID. If registry supports globally-unique dataset IDs (e.g. 

640 `uuid.UUID`) then datasets which already exist in the registry will be 

641 ignored if imported again. 

642 

643 Parameters 

644 ---------- 

645 datasets : `~collections.abc.Iterable` of `DatasetRef` 

646 Datasets to be inserted. All `DatasetRef` instances must have 

647 identical ``datasetType`` and ``run`` attributes. ``run`` 

648 attribute can be `None` and defaults to ``self.defaults.run``. 

649 Datasets can specify ``id`` attribute which will be used for 

650 inserted datasets. All dataset IDs must have the same type 

651 (`int` or `uuid.UUID`), if type of dataset IDs does not match 

652 configured backend then IDs will be ignored and new IDs will be 

653 generated by backend. 

654 expand : `bool`, optional 

655 If `True` (default), expand data IDs as they are inserted. This is 

656 necessary in general, but it may be disabled if the caller can 

657 guarantee this is unnecessary. 

658 

659 Returns 

660 ------- 

661 refs : `list` of `DatasetRef` 

662 Resolved `DatasetRef` instances for all given data IDs (in the same 

663 order). If any of ``datasets`` has an ID which already exists in 

664 the database then it will not be inserted or updated, but a 

665 resolved `DatasetRef` will be returned for it in any case. 

666 

667 Raises 

668 ------ 

669 lsst.daf.butler.registry.NoDefaultCollectionError 

670 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

671 lsst.daf.butler.registry.DatasetTypeError 

672 Raised if datasets correspond to more than one dataset type or 

673 dataset type is not known to registry. 

674 lsst.daf.butler.registry.ConflictingDefinitionError 

675 If a dataset with the same dataset type and data ID as one of those 

676 given already exists in ``run``. 

677 lsst.daf.butler.registry.MissingCollectionError 

678 Raised if ``run`` does not exist in the registry. 

679 

680 Notes 

681 ----- 

682 This method is considered package-private and internal to Butler 

683 implementation. Clients outside daf_butler package should not use this 

684 method. 

685 """ 

686 raise NotImplementedError() 

687 

688 @abstractmethod 

689 def getDataset(self, id: DatasetId) -> DatasetRef | None: 

690 """Retrieve a Dataset entry. 

691 

692 Parameters 

693 ---------- 

694 id : `DatasetId` 

695 The unique identifier for the dataset. 

696 

697 Returns 

698 ------- 

699 ref : `DatasetRef` or `None` 

700 A ref to the Dataset, or `None` if no matching Dataset 

701 was found. 

702 """ 

703 raise NotImplementedError() 

704 

705 @abstractmethod 

706 def removeDatasets(self, refs: Iterable[DatasetRef]) -> None: 

707 """Remove datasets from the Registry. 

708 

709 The datasets will be removed unconditionally from all collections, and 

710 any `Quantum` that consumed this dataset will instead be marked with 

711 having a NULL input. `Datastore` records will *not* be deleted; the 

712 caller is responsible for ensuring that the dataset has already been 

713 removed from all Datastores. 

714 

715 Parameters 

716 ---------- 

717 refs : `~collections.abc.Iterable` [`DatasetRef`] 

718 References to the datasets to be removed. Must include a valid 

719 ``id`` attribute, and should be considered invalidated upon return. 

720 

721 Raises 

722 ------ 

723 lsst.daf.butler.AmbiguousDatasetError 

724 Raised if any ``ref.id`` is `None`. 

725 lsst.daf.butler.registry.OrphanedRecordError 

726 Raised if any dataset is still present in any `Datastore`. 

727 """ 

728 raise NotImplementedError() 

729 

730 @abstractmethod 

731 def associate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

732 """Add existing datasets to a `~CollectionType.TAGGED` collection. 

733 

734 If a DatasetRef with the same exact ID is already in a collection 

735 nothing is changed. If a `DatasetRef` with the same `DatasetType` and 

736 data ID but with different ID exists in the collection, 

737 `~lsst.daf.butler.registry.ConflictingDefinitionError` is raised. 

738 

739 Parameters 

740 ---------- 

741 collection : `str` 

742 Indicates the collection the datasets should be associated with. 

743 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

744 An iterable of resolved `DatasetRef` instances that already exist 

745 in this `Registry`. 

746 

747 Raises 

748 ------ 

749 lsst.daf.butler.registry.ConflictingDefinitionError 

750 If a Dataset with the given `DatasetRef` already exists in the 

751 given collection. 

752 lsst.daf.butler.registry.MissingCollectionError 

753 Raised if ``collection`` does not exist in the registry. 

754 lsst.daf.butler.registry.CollectionTypeError 

755 Raise adding new datasets to the given ``collection`` is not 

756 allowed. 

757 """ 

758 raise NotImplementedError() 

759 

760 @abstractmethod 

761 def disassociate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

762 """Remove existing datasets from a `~CollectionType.TAGGED` collection. 

763 

764 ``collection`` and ``ref`` combinations that are not currently 

765 associated are silently ignored. 

766 

767 Parameters 

768 ---------- 

769 collection : `str` 

770 The collection the datasets should no longer be associated with. 

771 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

772 An iterable of resolved `DatasetRef` instances that already exist 

773 in this `Registry`. 

774 

775 Raises 

776 ------ 

777 lsst.daf.butler.AmbiguousDatasetError 

778 Raised if any of the given dataset references is unresolved. 

779 lsst.daf.butler.registry.MissingCollectionError 

780 Raised if ``collection`` does not exist in the registry. 

781 lsst.daf.butler.registry.CollectionTypeError 

782 Raise adding new datasets to the given ``collection`` is not 

783 allowed. 

784 """ 

785 raise NotImplementedError() 

786 

787 @abstractmethod 

788 def certify(self, collection: str, refs: Iterable[DatasetRef], timespan: Timespan) -> None: 

789 """Associate one or more datasets with a calibration collection and a 

790 validity range within it. 

791 

792 Parameters 

793 ---------- 

794 collection : `str` 

795 The name of an already-registered `~CollectionType.CALIBRATION` 

796 collection. 

797 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

798 Datasets to be associated. 

799 timespan : `Timespan` 

800 The validity range for these datasets within the collection. 

801 

802 Raises 

803 ------ 

804 lsst.daf.butler.AmbiguousDatasetError 

805 Raised if any of the given `DatasetRef` instances is unresolved. 

806 lsst.daf.butler.registry.ConflictingDefinitionError 

807 Raised if the collection already contains a different dataset with 

808 the same `DatasetType` and data ID and an overlapping validity 

809 range. 

810 lsst.daf.butler.registry.CollectionTypeError 

811 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

812 collection or if one or more datasets are of a dataset type for 

813 which `DatasetType.isCalibration` returns `False`. 

814 """ 

815 raise NotImplementedError() 

816 

817 @abstractmethod 

818 def decertify( 

819 self, 

820 collection: str, 

821 datasetType: str | DatasetType, 

822 timespan: Timespan, 

823 *, 

824 dataIds: Iterable[DataId] | None = None, 

825 ) -> None: 

826 """Remove or adjust datasets to clear a validity range within a 

827 calibration collection. 

828 

829 Parameters 

830 ---------- 

831 collection : `str` 

832 The name of an already-registered `~CollectionType.CALIBRATION` 

833 collection. 

834 datasetType : `str` or `DatasetType` 

835 Name or `DatasetType` instance for the datasets to be decertified. 

836 timespan : `Timespan`, optional 

837 The validity range to remove datasets from within the collection. 

838 Datasets that overlap this range but are not contained by it will 

839 have their validity ranges adjusted to not overlap it, which may 

840 split a single dataset validity range into two. 

841 dataIds : iterable [`dict` or `DataCoordinate`], optional 

842 Data IDs that should be decertified within the given validity range 

843 If `None`, all data IDs for ``self.datasetType`` will be 

844 decertified. 

845 

846 Raises 

847 ------ 

848 lsst.daf.butler.registry.CollectionTypeError 

849 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

850 collection or if ``datasetType.isCalibration() is False``. 

851 """ 

852 raise NotImplementedError() 

853 

854 @abstractmethod 

855 def getDatasetLocations(self, ref: DatasetRef) -> Iterable[str]: 

856 """Retrieve datastore locations for a given dataset. 

857 

858 Parameters 

859 ---------- 

860 ref : `DatasetRef` 

861 A reference to the dataset for which to retrieve storage 

862 information. 

863 

864 Returns 

865 ------- 

866 datastores : `~collections.abc.Iterable` [ `str` ] 

867 All the matching datastores holding this dataset. 

868 

869 Raises 

870 ------ 

871 lsst.daf.butler.AmbiguousDatasetError 

872 Raised if ``ref.id`` is `None`. 

873 """ 

874 raise NotImplementedError() 

875 

876 @abstractmethod 

877 def expandDataId( 

878 self, 

879 dataId: DataId | None = None, 

880 *, 

881 dimensions: Iterable[str] | DimensionGroup | None = None, 

882 records: Mapping[str, DimensionRecord | None] | None = None, 

883 withDefaults: bool = True, 

884 **kwargs: Any, 

885 ) -> DataCoordinate: 

886 """Expand a dimension-based data ID to include additional information. 

887 

888 Parameters 

889 ---------- 

890 dataId : `DataCoordinate` or `dict`, optional 

891 Data ID to be expanded; augmented and overridden by ``kwargs``. 

892 dimensions : `~collections.abc.Iterable` [ `str` ] or \ 

893 `DimensionGroup` optional 

894 The dimensions to be identified by the new `DataCoordinate`. 

895 If not provided, will be inferred from the keys of ``dataId`` and 

896 ``**kwargs``, and ``universe`` must be provided unless ``dataId`` 

897 is already a `DataCoordinate`. 

898 records : `~collections.abc.Mapping` [`str`, `DimensionRecord`], \ 

899 optional 

900 Dimension record data to use before querying the database for that 

901 data, keyed by element name. 

902 withDefaults : `bool`, optional 

903 Utilize ``self.defaults.dataId`` to fill in missing governor 

904 dimension key-value pairs. Defaults to `True` (i.e. defaults are 

905 used). 

906 **kwargs 

907 Additional keywords are treated like additional key-value pairs for 

908 ``dataId``, extending and overriding. 

909 

910 Returns 

911 ------- 

912 expanded : `DataCoordinate` 

913 A data ID that includes full metadata for all of the dimensions it 

914 identifies, i.e. guarantees that ``expanded.hasRecords()`` and 

915 ``expanded.hasFull()`` both return `True`. 

916 

917 Raises 

918 ------ 

919 lsst.daf.butler.registry.DataIdError 

920 Raised when ``dataId`` or keyword arguments specify unknown 

921 dimensions or values, or when a resulting data ID contains 

922 contradictory key-value pairs, according to dimension 

923 relationships. 

924 

925 Notes 

926 ----- 

927 This method cannot be relied upon to reject invalid data ID values 

928 for dimensions that do actually not have any record columns. For 

929 efficiency reasons the records for these dimensions (which have only 

930 dimension key values that are given by the caller) may be constructed 

931 directly rather than obtained from the registry database. 

932 """ 

933 raise NotImplementedError() 

934 

935 @abstractmethod 

936 def insertDimensionData( 

937 self, 

938 element: DimensionElement | str, 

939 *data: Mapping[str, Any] | DimensionRecord, 

940 conform: bool = True, 

941 replace: bool = False, 

942 skip_existing: bool = False, 

943 ) -> None: 

944 """Insert one or more dimension records into the database. 

945 

946 Parameters 

947 ---------- 

948 element : `DimensionElement` or `str` 

949 The `DimensionElement` or name thereof that identifies the table 

950 records will be inserted into. 

951 *data : `dict` or `DimensionRecord` 

952 One or more records to insert. 

953 conform : `bool`, optional 

954 If `False` (`True` is default) perform no checking or conversions, 

955 and assume that ``element`` is a `DimensionElement` instance and 

956 ``data`` is a one or more `DimensionRecord` instances of the 

957 appropriate subclass. 

958 replace : `bool`, optional 

959 If `True` (`False` is default), replace existing records in the 

960 database if there is a conflict. 

961 skip_existing : `bool`, optional 

962 If `True` (`False` is default), skip insertion if a record with 

963 the same primary key values already exists. Unlike 

964 `syncDimensionData`, this will not detect when the given record 

965 differs from what is in the database, and should not be used when 

966 this is a concern. 

967 """ 

968 raise NotImplementedError() 

969 

970 @abstractmethod 

971 def syncDimensionData( 

972 self, 

973 element: DimensionElement | str, 

974 row: Mapping[str, Any] | DimensionRecord, 

975 conform: bool = True, 

976 update: bool = False, 

977 ) -> bool | dict[str, Any]: 

978 """Synchronize the given dimension record with the database, inserting 

979 if it does not already exist and comparing values if it does. 

980 

981 Parameters 

982 ---------- 

983 element : `DimensionElement` or `str` 

984 The `DimensionElement` or name thereof that identifies the table 

985 records will be inserted into. 

986 row : `dict` or `DimensionRecord` 

987 The record to insert. 

988 conform : `bool`, optional 

989 If `False` (`True` is default) perform no checking or conversions, 

990 and assume that ``element`` is a `DimensionElement` instance and 

991 ``data`` is a one or more `DimensionRecord` instances of the 

992 appropriate subclass. 

993 update : `bool`, optional 

994 If `True` (`False` is default), update the existing record in the 

995 database if there is a conflict. 

996 

997 Returns 

998 ------- 

999 inserted_or_updated : `bool` or `dict` 

1000 `True` if a new row was inserted, `False` if no changes were 

1001 needed, or a `dict` mapping updated column names to their old 

1002 values if an update was performed (only possible if 

1003 ``update=True``). 

1004 

1005 Raises 

1006 ------ 

1007 lsst.daf.butler.registry.ConflictingDefinitionError 

1008 Raised if the record exists in the database (according to primary 

1009 key lookup) but is inconsistent with the given one. 

1010 """ 

1011 raise NotImplementedError() 

1012 

1013 @abstractmethod 

1014 def queryDatasetTypes( 

1015 self, 

1016 expression: Any = ..., 

1017 *, 

1018 missing: list[str] | None = None, 

1019 ) -> Iterable[DatasetType]: 

1020 """Iterate over the dataset types whose names match an expression. 

1021 

1022 Parameters 

1023 ---------- 

1024 expression : dataset type expression, optional 

1025 An expression that fully or partially identifies the dataset types 

1026 to return, such as a `str`, `re.Pattern`, or iterable thereof. 

1027 ``...`` can be used to return all dataset types, and is the 

1028 default. See :ref:`daf_butler_dataset_type_expressions` for more 

1029 information. 

1030 missing : `list` of `str`, optional 

1031 String dataset type names that were explicitly given (i.e. not 

1032 regular expression patterns) but not found will be appended to this 

1033 list, if it is provided. 

1034 

1035 Returns 

1036 ------- 

1037 dataset_types : `~collections.abc.Iterable` [ `DatasetType`] 

1038 An `~collections.abc.Iterable` of `DatasetType` instances whose 

1039 names match ``expression``. 

1040 

1041 Raises 

1042 ------ 

1043 lsst.daf.butler.registry.DatasetTypeExpressionError 

1044 Raised when ``expression`` is invalid. 

1045 """ 

1046 raise NotImplementedError() 

1047 

1048 @abstractmethod 

1049 def queryCollections( 

1050 self, 

1051 expression: Any = ..., 

1052 datasetType: DatasetType | None = None, 

1053 collectionTypes: Iterable[CollectionType] | CollectionType = CollectionType.all(), 

1054 flattenChains: bool = False, 

1055 includeChains: bool | None = None, 

1056 ) -> Sequence[str]: 

1057 """Iterate over the collections whose names match an expression. 

1058 

1059 Parameters 

1060 ---------- 

1061 expression : collection expression, optional 

1062 An expression that identifies the collections to return, such as 

1063 a `str` (for full matches or partial matches via globs), 

1064 `re.Pattern` (for partial matches), or iterable thereof. ``...`` 

1065 can be used to return all collections, and is the default. 

1066 See :ref:`daf_butler_collection_expressions` for more information. 

1067 datasetType : `DatasetType`, optional 

1068 If provided, only yield collections that may contain datasets of 

1069 this type. This is a conservative approximation in general; it may 

1070 yield collections that do not have any such datasets. 

1071 collectionTypes : `~collections.abc.Set` [`CollectionType`] or \ 

1072 `CollectionType`, optional 

1073 If provided, only yield collections of these types. 

1074 flattenChains : `bool`, optional 

1075 If `True` (`False` is default), recursively yield the child 

1076 collections of matching `~CollectionType.CHAINED` collections. 

1077 includeChains : `bool`, optional 

1078 If `True`, yield records for matching `~CollectionType.CHAINED` 

1079 collections. Default is the opposite of ``flattenChains``: include 

1080 either CHAINED collections or their children, but not both. 

1081 

1082 Returns 

1083 ------- 

1084 collections : `~collections.abc.Sequence` [ `str` ] 

1085 The names of collections that match ``expression``. 

1086 

1087 Raises 

1088 ------ 

1089 lsst.daf.butler.registry.CollectionExpressionError 

1090 Raised when ``expression`` is invalid. 

1091 

1092 Notes 

1093 ----- 

1094 The order in which collections are returned is unspecified, except that 

1095 the children of a `~CollectionType.CHAINED` collection are guaranteed 

1096 to be in the order in which they are searched. When multiple parent 

1097 `~CollectionType.CHAINED` collections match the same criteria, the 

1098 order in which the two lists appear is unspecified, and the lists of 

1099 children may be incomplete if a child has multiple parents. 

1100 """ 

1101 raise NotImplementedError() 

1102 

1103 @abstractmethod 

1104 def queryDatasets( 

1105 self, 

1106 datasetType: Any, 

1107 *, 

1108 collections: CollectionArgType | None = None, 

1109 dimensions: Iterable[str] | None = None, 

1110 dataId: DataId | None = None, 

1111 where: str = "", 

1112 findFirst: bool = False, 

1113 bind: Mapping[str, Any] | None = None, 

1114 check: bool = True, 

1115 **kwargs: Any, 

1116 ) -> DatasetQueryResults: 

1117 """Query for and iterate over dataset references matching user-provided 

1118 criteria. 

1119 

1120 Parameters 

1121 ---------- 

1122 datasetType : dataset type expression 

1123 An expression that fully or partially identifies the dataset types 

1124 to be queried. Allowed types include `DatasetType`, `str`, 

1125 `re.Pattern`, and iterables thereof. The special value ``...`` can 

1126 be used to query all dataset types. See 

1127 :ref:`daf_butler_dataset_type_expressions` for more information. 

1128 collections : collection expression, optional 

1129 An expression that identifies the collections to search, such as a 

1130 `str` (for full matches or partial matches via globs), `re.Pattern` 

1131 (for partial matches), or iterable thereof. ``...`` can be used to 

1132 search all collections (actually just all `~CollectionType.RUN` 

1133 collections, because this will still find all datasets). 

1134 If not provided, ``self.default.collections`` is used. See 

1135 :ref:`daf_butler_collection_expressions` for more information. 

1136 dimensions : `~collections.abc.Iterable` [ `str` ] 

1137 Dimensions to include in the query (in addition to those used 

1138 to identify the queried dataset type(s)), either to constrain 

1139 the resulting datasets to those for which a matching dimension 

1140 exists, or to relate the dataset type's dimensions to dimensions 

1141 referenced by the ``dataId`` or ``where`` arguments. 

1142 dataId : `dict` or `DataCoordinate`, optional 

1143 A data ID whose key-value pairs are used as equality constraints 

1144 in the query. 

1145 where : `str`, optional 

1146 A string expression similar to a SQL WHERE clause. May involve 

1147 any column of a dimension table or (as a shortcut for the primary 

1148 key column of a dimension table) dimension name. See 

1149 :ref:`daf_butler_dimension_expressions` for more information. 

1150 findFirst : `bool`, optional 

1151 If `True` (`False` is default), for each result data ID, only 

1152 yield one `DatasetRef` of each `DatasetType`, from the first 

1153 collection in which a dataset of that dataset type appears 

1154 (according to the order of ``collections`` passed in). If `True`, 

1155 ``collections`` must not contain regular expressions and may not 

1156 be ``...``. 

1157 bind : `~collections.abc.Mapping`, optional 

1158 Mapping containing literal values that should be injected into the 

1159 ``where`` expression, keyed by the identifiers they replace. 

1160 Values of collection type can be expanded in some cases; see 

1161 :ref:`daf_butler_dimension_expressions_identifiers` for more 

1162 information. 

1163 check : `bool`, optional 

1164 If `True` (default) check the query for consistency before 

1165 executing it. This may reject some valid queries that resemble 

1166 common mistakes (e.g. queries for visits without specifying an 

1167 instrument). 

1168 **kwargs 

1169 Additional keyword arguments are forwarded to 

1170 `DataCoordinate.standardize` when processing the ``dataId`` 

1171 argument (and may be used to provide a constraining data ID even 

1172 when the ``dataId`` argument is `None`). 

1173 

1174 Returns 

1175 ------- 

1176 refs : `.queries.DatasetQueryResults` 

1177 Dataset references matching the given query criteria. Nested data 

1178 IDs are guaranteed to include values for all implied dimensions