Coverage for python / lsst / daf / butler / registry / _registry.py: 98%

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ("CollectionArgType", "Registry") 

31 

32import contextlib 

33import logging 

34import re 

35from abc import ABC, abstractmethod 

36from collections.abc import Iterable, Iterator, Mapping, Sequence 

37from types import EllipsisType 

38from typing import TYPE_CHECKING, Any, TypeAlias 

39 

40from .._collection_type import CollectionType 

41from .._dataset_association import DatasetAssociation 

42from .._dataset_ref import DatasetId, DatasetIdGenEnum, DatasetRef 

43from .._dataset_type import DatasetType 

44from .._storage_class import StorageClassFactory 

45from .._timespan import Timespan 

46from ..dimensions import ( 

47 DataCoordinate, 

48 DataId, 

49 DimensionElement, 

50 DimensionGroup, 

51 DimensionRecord, 

52 DimensionUniverse, 

53) 

54from ._collection_summary import CollectionSummary 

55from ._defaults import RegistryDefaults 

56from .queries import DataCoordinateQueryResults, DatasetQueryResults, DimensionRecordQueryResults 

57from .wildcards import CollectionWildcard 

58 

59if TYPE_CHECKING: 

60 from .interfaces import ObsCoreTableManager 

61 

62_LOG = logging.getLogger(__name__) 

63 

64# TYpe alias for `collections` arguments. 

65CollectionArgType: TypeAlias = ( 

66 str | re.Pattern | Iterable[str | re.Pattern] | EllipsisType | CollectionWildcard 

67) 

68 

69 

70class Registry(ABC): 

71 """Abstract Registry interface. 

72 

73 All subclasses should store `~lsst.daf.butler.registry.RegistryDefaults` in 

74 a ``_defaults`` property. No other properties are assumed shared between 

75 implementations. 

76 

77 See Also 

78 -------- 

79 lsst.daf.butler.Butler 

80 """ 

81 

82 @abstractmethod 

83 def isWriteable(self) -> bool: 

84 """Return `True` if this registry allows write operations, and `False` 

85 otherwise. 

86 """ 

87 raise NotImplementedError() 

88 

89 @property 

90 @abstractmethod 

91 def dimensions(self) -> DimensionUniverse: 

92 """Definitions of all dimensions recognized by this `Registry` 

93 (`DimensionUniverse`). 

94 """ 

95 raise NotImplementedError() 

96 

97 @property 

98 @abstractmethod 

99 def defaults(self) -> RegistryDefaults: 

100 """Default collection search path and/or output `~CollectionType.RUN` 

101 collection (`~lsst.daf.butler.registry.RegistryDefaults`). 

102 

103 This is an immutable struct whose components may not be set 

104 individually, but the entire struct can be set by assigning to this 

105 property. 

106 """ 

107 raise NotImplementedError() 

108 

109 @defaults.setter 

110 @abstractmethod 

111 def defaults(self, value: RegistryDefaults) -> None: 

112 raise NotImplementedError() 

113 

114 @abstractmethod 

115 def refresh(self) -> None: 

116 """Refresh all in-memory state by querying the database. 

117 

118 This may be necessary to enable querying for entities added by other 

119 registry instances after this one was constructed. 

120 """ 

121 raise NotImplementedError() 

122 

123 @abstractmethod 

124 def refresh_collection_summaries(self) -> None: 

125 """Refresh content of the collection summary tables in the database. 

126 

127 This method can be run to cleanup the contents of the summary tables 

128 after removing datasets from collections. 

129 """ 

130 raise NotImplementedError() 

131 

132 @abstractmethod 

133 def caching_context(self) -> contextlib.AbstractContextManager[None]: 

134 """Context manager that enables caching.""" 

135 raise NotImplementedError() 

136 

137 @contextlib.contextmanager 

138 @abstractmethod 

139 def transaction(self, *, savepoint: bool = False) -> Iterator[None]: 

140 """Return a context manager that represents a transaction. 

141 

142 Parameters 

143 ---------- 

144 savepoint : `bool` 

145 Whether to create a SAVEPOINT or not. 

146 """ 

147 raise NotImplementedError() 

148 

149 def resetConnectionPool(self) -> None: 

150 """Reset connection pool for registry if relevant. 

151 

152 This operation can be used reset connections to servers when 

153 using registry with fork-based multiprocessing. This method should 

154 usually be called by the child process immediately 

155 after the fork. 

156 

157 The base class implementation is a no-op. 

158 """ 

159 pass 

160 

161 @abstractmethod 

162 def registerCollection( 

163 self, name: str, type: CollectionType = CollectionType.TAGGED, doc: str | None = None 

164 ) -> bool: 

165 """Add a new collection if one with the given name does not exist. 

166 

167 Parameters 

168 ---------- 

169 name : `str` 

170 The name of the collection to create. 

171 type : `CollectionType` 

172 Enum value indicating the type of collection to create. 

173 doc : `str`, optional 

174 Documentation string for the collection. 

175 

176 Returns 

177 ------- 

178 registered : `bool` 

179 Boolean indicating whether the collection was already registered 

180 or was created by this call. 

181 

182 Notes 

183 ----- 

184 This method cannot be called within transactions, as it needs to be 

185 able to perform its own transaction to be concurrent. 

186 """ 

187 raise NotImplementedError() 

188 

189 @abstractmethod 

190 def getCollectionType(self, name: str) -> CollectionType: 

191 """Return an enumeration value indicating the type of the given 

192 collection. 

193 

194 Parameters 

195 ---------- 

196 name : `str` 

197 The name of the collection. 

198 

199 Returns 

200 ------- 

201 type : `CollectionType` 

202 Enum value indicating the type of this collection. 

203 

204 Raises 

205 ------ 

206 lsst.daf.butler.registry.MissingCollectionError 

207 Raised if no collection with the given name exists. 

208 """ 

209 raise NotImplementedError() 

210 

211 @abstractmethod 

212 def registerRun(self, name: str, doc: str | None = None) -> bool: 

213 """Add a new run if one with the given name does not exist. 

214 

215 Parameters 

216 ---------- 

217 name : `str` 

218 The name of the run to create. 

219 doc : `str`, optional 

220 Documentation string for the collection. 

221 

222 Returns 

223 ------- 

224 registered : `bool` 

225 Boolean indicating whether a new run was registered. `False` 

226 if it already existed. 

227 

228 Notes 

229 ----- 

230 This method cannot be called within transactions, as it needs to be 

231 able to perform its own transaction to be concurrent. 

232 """ 

233 raise NotImplementedError() 

234 

235 @abstractmethod 

236 def removeCollection(self, name: str) -> None: 

237 """Remove the given collection from the registry. 

238 

239 Parameters 

240 ---------- 

241 name : `str` 

242 The name of the collection to remove. 

243 

244 Raises 

245 ------ 

246 lsst.daf.butler.registry.MissingCollectionError 

247 Raised if no collection with the given name exists. 

248 sqlalchemy.exc.IntegrityError 

249 Raised if the database rows associated with the collection are 

250 still referenced by some other table, such as a dataset in a 

251 datastore (for `~CollectionType.RUN` collections only) or a 

252 `~CollectionType.CHAINED` collection of which this collection is 

253 a child. 

254 

255 Notes 

256 ----- 

257 If this is a `~CollectionType.RUN` collection, all datasets and quanta 

258 in it will removed from the `Registry` database. This requires that 

259 those datasets be removed (or at least trashed) from any datastores 

260 that hold them first. 

261 

262 A collection may not be deleted as long as it is referenced by a 

263 `~CollectionType.CHAINED` collection; the ``CHAINED`` collection must 

264 be deleted or redefined first. 

265 """ 

266 raise NotImplementedError() 

267 

268 @abstractmethod 

269 def getCollectionChain(self, parent: str) -> Sequence[str]: 

270 """Return the child collections in a `~CollectionType.CHAINED` 

271 collection. 

272 

273 Parameters 

274 ---------- 

275 parent : `str` 

276 Name of the chained collection. Must have already been added via 

277 a call to `Registry.registerCollection`. 

278 

279 Returns 

280 ------- 

281 children : `~collections.abc.Sequence` [ `str` ] 

282 An ordered sequence of collection names that are searched when the 

283 given chained collection is searched. 

284 

285 Raises 

286 ------ 

287 lsst.daf.butler.registry.MissingCollectionError 

288 Raised if ``parent`` does not exist in the `Registry`. 

289 lsst.daf.butler.registry.CollectionTypeError 

290 Raised if ``parent`` does not correspond to a 

291 `~CollectionType.CHAINED` collection. 

292 """ 

293 raise NotImplementedError() 

294 

295 @abstractmethod 

296 def setCollectionChain(self, parent: str, children: Any, *, flatten: bool = False) -> None: 

297 """Define or redefine a `~CollectionType.CHAINED` collection. 

298 

299 Parameters 

300 ---------- 

301 parent : `str` 

302 Name of the chained collection. Must have already been added via 

303 a call to `Registry.registerCollection`. 

304 children : collection expression 

305 An expression defining an ordered search of child collections, 

306 generally an iterable of `str`; see 

307 :ref:`daf_butler_collection_expressions` for more information. 

308 flatten : `bool`, optional 

309 If `True` (`False` is default), recursively flatten out any nested 

310 `~CollectionType.CHAINED` collections in ``children`` first. 

311 

312 Raises 

313 ------ 

314 lsst.daf.butler.registry.MissingCollectionError 

315 Raised when any of the given collections do not exist in the 

316 `Registry`. 

317 lsst.daf.butler.registry.CollectionTypeError 

318 Raised if ``parent`` does not correspond to a 

319 `~CollectionType.CHAINED` collection. 

320 ValueError 

321 Raised if the given collections contains a cycle. 

322 """ 

323 raise NotImplementedError() 

324 

325 @abstractmethod 

326 def getCollectionParentChains(self, collection: str) -> set[str]: 

327 """Return the CHAINED collections that directly contain the given one. 

328 

329 Parameters 

330 ---------- 

331 collection : `str` 

332 Name of the collection. 

333 

334 Returns 

335 ------- 

336 chains : `set` of `str` 

337 Set of `~CollectionType.CHAINED` collection names. 

338 """ 

339 raise NotImplementedError() 

340 

341 @abstractmethod 

342 def getCollectionDocumentation(self, collection: str) -> str | None: 

343 """Retrieve the documentation string for a collection. 

344 

345 Parameters 

346 ---------- 

347 collection : `str` 

348 Name of the collection. 

349 

350 Returns 

351 ------- 

352 docs : `str` or `None` 

353 Docstring for the collection with the given name. 

354 """ 

355 raise NotImplementedError() 

356 

357 @abstractmethod 

358 def setCollectionDocumentation(self, collection: str, doc: str | None) -> None: 

359 """Set the documentation string for a collection. 

360 

361 Parameters 

362 ---------- 

363 collection : `str` 

364 Name of the collection. 

365 doc : `str` or `None` 

366 Docstring for the collection with the given name; will replace any 

367 existing docstring. Passing `None` will remove any existing 

368 docstring. 

369 """ 

370 raise NotImplementedError() 

371 

372 @abstractmethod 

373 def getCollectionSummary(self, collection: str) -> CollectionSummary: 

374 """Return a summary for the given collection. 

375 

376 Parameters 

377 ---------- 

378 collection : `str` 

379 Name of the collection for which a summary is to be retrieved. 

380 

381 Returns 

382 ------- 

383 summary : `~lsst.daf.butler.registry.CollectionSummary` 

384 Summary of the dataset types and governor dimension values in 

385 this collection. 

386 """ 

387 raise NotImplementedError() 

388 

389 @abstractmethod 

390 def registerDatasetType(self, datasetType: DatasetType) -> bool: 

391 """Add a new `DatasetType` to the Registry. 

392 

393 It is not an error to register the same `DatasetType` twice. 

394 

395 Parameters 

396 ---------- 

397 datasetType : `DatasetType` 

398 The `DatasetType` to be added. 

399 

400 Returns 

401 ------- 

402 inserted : `bool` 

403 `True` if ``datasetType`` was inserted, `False` if an identical 

404 existing `DatasetType` was found. Note that in either case the 

405 DatasetType is guaranteed to be defined in the Registry 

406 consistently with the given definition. 

407 

408 Raises 

409 ------ 

410 ValueError 

411 Raised if the dimensions or storage class are invalid. 

412 lsst.daf.butler.registry.ConflictingDefinitionError 

413 Raised if this `DatasetType` is already registered with a different 

414 definition. 

415 

416 Notes 

417 ----- 

418 This method cannot be called within transactions, as it needs to be 

419 able to perform its own transaction to be concurrent. 

420 """ 

421 raise NotImplementedError() 

422 

423 @abstractmethod 

424 def removeDatasetType(self, name: str | tuple[str, ...]) -> None: 

425 """Remove the named `DatasetType` from the registry. 

426 

427 .. warning:: 

428 

429 Registry implementations can cache the dataset type definitions. 

430 This means that deleting the dataset type definition may result in 

431 unexpected behavior from other butler processes that are active 

432 that have not seen the deletion. 

433 

434 Parameters 

435 ---------- 

436 name : `str` or `tuple` [`str`] 

437 Name of the type to be removed or tuple containing a list of type 

438 names to be removed. Wildcards are allowed. 

439 

440 Raises 

441 ------ 

442 lsst.daf.butler.registry.OrphanedRecordError 

443 Raised if an attempt is made to remove the dataset type definition 

444 when there are already datasets associated with it. 

445 

446 Notes 

447 ----- 

448 If the dataset type is not registered the method will return without 

449 action. 

450 """ 

451 raise NotImplementedError() 

452 

453 @abstractmethod 

454 def getDatasetType(self, name: str) -> DatasetType: 

455 """Get the `DatasetType`. 

456 

457 Parameters 

458 ---------- 

459 name : `str` 

460 Name of the type. 

461 

462 Returns 

463 ------- 

464 type : `DatasetType` 

465 The `DatasetType` associated with the given name. 

466 

467 Raises 

468 ------ 

469 lsst.daf.butler.registry.MissingDatasetTypeError 

470 Raised if the requested dataset type has not been registered. 

471 

472 Notes 

473 ----- 

474 This method handles component dataset types automatically, though most 

475 other registry operations do not. 

476 """ 

477 raise NotImplementedError() 

478 

479 @abstractmethod 

480 def supportsIdGenerationMode(self, mode: DatasetIdGenEnum) -> bool: 

481 """Test whether the given dataset ID generation mode is supported by 

482 `insertDatasets`. 

483 

484 Parameters 

485 ---------- 

486 mode : `DatasetIdGenEnum` 

487 Enum value for the mode to test. 

488 

489 Returns 

490 ------- 

491 supported : `bool` 

492 Whether the given mode is supported. 

493 """ 

494 raise NotImplementedError() 

495 

496 @abstractmethod 

497 def findDataset( 

498 self, 

499 datasetType: DatasetType | str, 

500 dataId: DataId | None = None, 

501 *, 

502 collections: CollectionArgType | None = None, 

503 timespan: Timespan | None = None, 

504 datastore_records: bool = False, 

505 **kwargs: Any, 

506 ) -> DatasetRef | None: 

507 """Find a dataset given its `DatasetType` and data ID. 

508 

509 This can be used to obtain a `DatasetRef` that permits the dataset to 

510 be read from a `Datastore`. If the dataset is a component and can not 

511 be found using the provided dataset type, a dataset ref for the parent 

512 will be returned instead but with the correct dataset type. 

513 

514 Parameters 

515 ---------- 

516 datasetType : `DatasetType` or `str` 

517 A `DatasetType` or the name of one. If this is a `DatasetType` 

518 instance, its storage class will be respected and propagated to 

519 the output, even if it differs from the dataset type definition 

520 in the registry, as long as the storage classes are convertible. 

521 dataId : `dict` or `DataCoordinate`, optional 

522 A `dict`-like object containing the `Dimension` links that identify 

523 the dataset within a collection. 

524 collections : collection expression, optional 

525 An expression that fully or partially identifies the collections to 

526 search for the dataset; see 

527 :ref:`daf_butler_collection_expressions` for more information. 

528 Defaults to ``self.defaults.collections``. 

529 timespan : `Timespan`, optional 

530 A timespan that the validity range of the dataset must overlap. 

531 If not provided, any `~CollectionType.CALIBRATION` collections 

532 matched by the ``collections`` argument will not be searched. 

533 datastore_records : `bool`, optional 

534 Whether to attach datastore records to the `DatasetRef`. 

535 **kwargs 

536 Additional keyword arguments passed to 

537 `DataCoordinate.standardize` to convert ``dataId`` to a true 

538 `DataCoordinate` or augment an existing one. 

539 

540 Returns 

541 ------- 

542 ref : `DatasetRef` 

543 A reference to the dataset, or `None` if no matching Dataset 

544 was found. 

545 

546 Raises 

547 ------ 

548 lsst.daf.butler.registry.NoDefaultCollectionError 

549 Raised if ``collections`` is `None` and 

550 ``self.defaults.collections`` is `None`. 

551 LookupError 

552 Raised if one or more data ID keys are missing. 

553 lsst.daf.butler.registry.MissingDatasetTypeError 

554 Raised if the dataset type does not exist. 

555 lsst.daf.butler.registry.MissingCollectionError 

556 Raised if any of ``collections`` does not exist in the registry. 

557 

558 Notes 

559 ----- 

560 This method simply returns `None` and does not raise an exception even 

561 when the set of collections searched is intrinsically incompatible with 

562 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but 

563 only `~CollectionType.CALIBRATION` collections are being searched. 

564 This may make it harder to debug some lookup failures, but the behavior 

565 is intentional; we consider it more important that failed searches are 

566 reported consistently, regardless of the reason, and that adding 

567 additional collections that do not contain a match to the search path 

568 never changes the behavior. 

569 

570 This method handles component dataset types automatically, though most 

571 other registry operations do not. 

572 """ 

573 raise NotImplementedError() 

574 

575 @abstractmethod 

576 def insertDatasets( 

577 self, 

578 datasetType: DatasetType | str, 

579 dataIds: Iterable[DataId], 

580 run: str | None = None, 

581 expand: bool = True, 

582 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE, 

583 ) -> list[DatasetRef]: 

584 """Insert one or more datasets into the `Registry`. 

585 

586 This always adds new datasets; to associate existing datasets with 

587 a new collection, use ``associate``. 

588 

589 Parameters 

590 ---------- 

591 datasetType : `DatasetType` or `str` 

592 A `DatasetType` or the name of one. 

593 dataIds : `~collections.abc.Iterable` of `dict` or `DataCoordinate` 

594 Dimension-based identifiers for the new datasets. 

595 run : `str`, optional 

596 The name of the run that produced the datasets. Defaults to 

597 ``self.defaults.run``. 

598 expand : `bool`, optional 

599 If `True` (default), expand data IDs as they are inserted. This is 

600 necessary in general to allow datastore to generate file templates, 

601 but it may be disabled if the caller can guarantee this is 

602 unnecessary. 

603 idGenerationMode : `DatasetIdGenEnum`, optional 

604 Specifies option for generating dataset IDs. By default unique IDs 

605 are generated for each inserted dataset. 

606 

607 Returns 

608 ------- 

609 refs : `list` of `DatasetRef` 

610 Resolved `DatasetRef` instances for all given data IDs (in the same 

611 order). 

612 

613 Raises 

614 ------ 

615 lsst.daf.butler.registry.DatasetTypeError 

616 Raised if ``datasetType`` is not known to registry. 

617 lsst.daf.butler.registry.CollectionTypeError 

618 Raised if ``run`` collection type is not `~CollectionType.RUN`. 

619 lsst.daf.butler.registry.NoDefaultCollectionError 

620 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

621 lsst.daf.butler.registry.ConflictingDefinitionError 

622 If a dataset with the same dataset type and data ID as one of those 

623 given already exists in ``run``. 

624 lsst.daf.butler.registry.MissingCollectionError 

625 Raised if ``run`` does not exist in the registry. 

626 """ 

627 raise NotImplementedError() 

628 

629 @abstractmethod 

630 def _importDatasets( 

631 self, 

632 datasets: Iterable[DatasetRef], 

633 expand: bool = True, 

634 assume_new: bool = False, 

635 ) -> list[DatasetRef]: 

636 """Import one or more datasets into the `Registry`. 

637 

638 This differs from `insertDatasets` method in that this method accepts 

639 `DatasetRef` instances, which already have a dataset ID. 

640 

641 Parameters 

642 ---------- 

643 datasets : `~collections.abc.Iterable` of `DatasetRef` 

644 Datasets to be inserted. All `DatasetRef` instances must have 

645 identical ``run`` attributes. ``run`` 

646 attribute can be `None` and defaults to ``self.defaults.run``. 

647 Datasets can specify ``id`` attribute which will be used for 

648 inserted datasets. 

649 Datasets can be of multiple dataset types, but all the dataset 

650 types must have the same set of dimensions. 

651 expand : `bool`, optional 

652 If `True` (default), expand data IDs as they are inserted. This is 

653 necessary in general, but it may be disabled if the caller can 

654 guarantee this is unnecessary. 

655 assume_new : `bool`, optional 

656 If `True`, assume datasets are new. If `False`, datasets that are 

657 identical to an existing one are ignored. 

658 

659 Returns 

660 ------- 

661 refs : `list` of `DatasetRef` 

662 `DatasetRef` instances for all given data IDs (in the same order). 

663 If any of ``datasets`` has an ID which already exists in the 

664 database then it will not be inserted or updated, but a 

665 `DatasetRef` will be returned for it in any case. 

666 

667 Raises 

668 ------ 

669 lsst.daf.butler.registry.NoDefaultCollectionError 

670 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

671 lsst.daf.butler.registry.DatasetTypeError 

672 Raised if a dataset type is not known to registry. 

673 lsst.daf.butler.registry.ConflictingDefinitionError 

674 If a dataset with the same dataset type and data ID as one of those 

675 given already exists in ``run``, or if ``assume_new=True`` and at 

676 least one dataset is not new. 

677 lsst.daf.butler.registry.MissingCollectionError 

678 Raised if ``run`` does not exist in the registry. 

679 

680 Notes 

681 ----- 

682 This method is considered middleware-internal. 

683 """ 

684 raise NotImplementedError() 

685 

686 @abstractmethod 

687 def getDataset(self, id: DatasetId) -> DatasetRef | None: 

688 """Retrieve a Dataset entry. 

689 

690 Parameters 

691 ---------- 

692 id : `DatasetId` 

693 The unique identifier for the dataset. 

694 

695 Returns 

696 ------- 

697 ref : `DatasetRef` or `None` 

698 A ref to the Dataset, or `None` if no matching Dataset 

699 was found. 

700 """ 

701 raise NotImplementedError() 

702 

703 @abstractmethod 

704 def _fetch_run_dataset_ids(self, run: str) -> list[DatasetId]: 

705 """Return the IDs of all datasets in the given ``RUN`` 

706 collection. 

707 

708 Parameters 

709 ---------- 

710 run : `str` 

711 Name of the collection. 

712 

713 Returns 

714 ------- 

715 dataset_ids : `list` [`uuid.UUID`] 

716 List of dataset IDs. 

717 

718 Notes 

719 ----- 

720 This is a middleware-internal interface. 

721 """ 

722 raise NotImplementedError() 

723 

724 @abstractmethod 

725 def removeDatasets(self, refs: Iterable[DatasetRef]) -> None: 

726 """Remove datasets from the Registry. 

727 

728 The datasets will be removed unconditionally from all collections, and 

729 any `Quantum` that consumed this dataset will instead be marked with 

730 having a NULL input. `Datastore` records will *not* be deleted; the 

731 caller is responsible for ensuring that the dataset has already been 

732 removed from all Datastores. 

733 

734 Parameters 

735 ---------- 

736 refs : `~collections.abc.Iterable` [`DatasetRef`] 

737 References to the datasets to be removed. Must include a valid 

738 ``id`` attribute, and should be considered invalidated upon return. 

739 

740 Raises 

741 ------ 

742 lsst.daf.butler.AmbiguousDatasetError 

743 Raised if any ``ref.id`` is `None`. 

744 lsst.daf.butler.registry.OrphanedRecordError 

745 Raised if any dataset is still present in any `Datastore`. 

746 """ 

747 raise NotImplementedError() 

748 

749 @abstractmethod 

750 def associate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

751 """Add existing datasets to a `~CollectionType.TAGGED` collection. 

752 

753 If a DatasetRef with the same exact ID is already in a collection 

754 nothing is changed. If a `DatasetRef` with the same `DatasetType` and 

755 data ID but with different ID exists in the collection, 

756 `~lsst.daf.butler.registry.ConflictingDefinitionError` is raised. 

757 

758 Parameters 

759 ---------- 

760 collection : `str` 

761 Indicates the collection the datasets should be associated with. 

762 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

763 An iterable of resolved `DatasetRef` instances that already exist 

764 in this `Registry`. 

765 

766 Raises 

767 ------ 

768 lsst.daf.butler.registry.ConflictingDefinitionError 

769 If a Dataset with the given `DatasetRef` already exists in the 

770 given collection. 

771 lsst.daf.butler.registry.MissingCollectionError 

772 Raised if ``collection`` does not exist in the registry. 

773 lsst.daf.butler.registry.CollectionTypeError 

774 Raise adding new datasets to the given ``collection`` is not 

775 allowed. 

776 """ 

777 raise NotImplementedError() 

778 

779 @abstractmethod 

780 def disassociate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

781 """Remove existing datasets from a `~CollectionType.TAGGED` collection. 

782 

783 ``collection`` and ``ref`` combinations that are not currently 

784 associated are silently ignored. 

785 

786 Parameters 

787 ---------- 

788 collection : `str` 

789 The collection the datasets should no longer be associated with. 

790 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

791 An iterable of resolved `DatasetRef` instances that already exist 

792 in this `Registry`. 

793 

794 Raises 

795 ------ 

796 lsst.daf.butler.AmbiguousDatasetError 

797 Raised if any of the given dataset references is unresolved. 

798 lsst.daf.butler.registry.MissingCollectionError 

799 Raised if ``collection`` does not exist in the registry. 

800 lsst.daf.butler.registry.CollectionTypeError 

801 Raise adding new datasets to the given ``collection`` is not 

802 allowed. 

803 """ 

804 raise NotImplementedError() 

805 

806 @abstractmethod 

807 def certify(self, collection: str, refs: Iterable[DatasetRef], timespan: Timespan) -> None: 

808 """Associate one or more datasets with a calibration collection and a 

809 validity range within it. 

810 

811 Parameters 

812 ---------- 

813 collection : `str` 

814 The name of an already-registered `~CollectionType.CALIBRATION` 

815 collection. 

816 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

817 Datasets to be associated. 

818 timespan : `Timespan` 

819 The validity range for these datasets within the collection. 

820 

821 Raises 

822 ------ 

823 lsst.daf.butler.AmbiguousDatasetError 

824 Raised if any of the given `DatasetRef` instances is unresolved. 

825 lsst.daf.butler.registry.ConflictingDefinitionError 

826 Raised if the collection already contains a different dataset with 

827 the same `DatasetType` and data ID and an overlapping validity 

828 range. 

829 lsst.daf.butler.registry.CollectionTypeError 

830 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

831 collection or if one or more datasets are of a dataset type for 

832 which `DatasetType.isCalibration` returns `False`. 

833 """ 

834 raise NotImplementedError() 

835 

836 @abstractmethod 

837 def decertify( 

838 self, 

839 collection: str, 

840 datasetType: str | DatasetType, 

841 timespan: Timespan, 

842 *, 

843 dataIds: Iterable[DataId] | None = None, 

844 ) -> None: 

845 """Remove or adjust datasets to clear a validity range within a 

846 calibration collection. 

847 

848 Parameters 

849 ---------- 

850 collection : `str` 

851 The name of an already-registered `~CollectionType.CALIBRATION` 

852 collection. 

853 datasetType : `str` or `DatasetType` 

854 Name or `DatasetType` instance for the datasets to be decertified. 

855 timespan : `Timespan`, optional 

856 The validity range to remove datasets from within the collection. 

857 Datasets that overlap this range but are not contained by it will 

858 have their validity ranges adjusted to not overlap it, which may 

859 split a single dataset validity range into two. 

860 dataIds : iterable [`dict` or `DataCoordinate`], optional 

861 Data IDs that should be decertified within the given validity range 

862 If `None`, all data IDs for ``self.datasetType`` will be 

863 decertified. 

864 

865 Raises 

866 ------ 

867 lsst.daf.butler.registry.CollectionTypeError 

868 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

869 collection or if ``datasetType.isCalibration() is False``. 

870 """ 

871 raise NotImplementedError() 

872 

873 @abstractmethod 

874 def getDatasetLocations(self, ref: DatasetRef) -> Iterable[str]: 

875 """Retrieve datastore locations for a given dataset. 

876 

877 Parameters 

878 ---------- 

879 ref : `DatasetRef` 

880 A reference to the dataset for which to retrieve storage 

881 information. 

882 

883 Returns 

884 ------- 

885 datastores : `~collections.abc.Iterable` [ `str` ] 

886 All the matching datastores holding this dataset. 

887 

888 Raises 

889 ------ 

890 lsst.daf.butler.AmbiguousDatasetError 

891 Raised if ``ref.id`` is `None`. 

892 """ 

893 raise NotImplementedError() 

894 

895 @abstractmethod 

896 def expandDataId( 

897 self, 

898 dataId: DataId | None = None, 

899 *, 

900 dimensions: Iterable[str] | DimensionGroup | None = None, 

901 records: Mapping[str, DimensionRecord | None] | None = None, 

902 withDefaults: bool = True, 

903 **kwargs: Any, 

904 ) -> DataCoordinate: 

905 """Expand a dimension-based data ID to include additional information. 

906 

907 Parameters 

908 ---------- 

909 dataId : `DataCoordinate` or `dict`, optional 

910 Data ID to be expanded; augmented and overridden by ``kwargs``. 

911 dimensions : `~collections.abc.Iterable` [ `str` ] or \ 

912 `DimensionGroup` optional 

913 The dimensions to be identified by the new `DataCoordinate`. 

914 If not provided, will be inferred from the keys of ``dataId`` and 

915 ``**kwargs``, and ``universe`` must be provided unless ``dataId`` 

916 is already a `DataCoordinate`. 

917 records : `~collections.abc.Mapping` [`str`, `DimensionRecord`], \ 

918 optional 

919 Dimension record data to use before querying the database for that 

920 data, keyed by element name. 

921 withDefaults : `bool`, optional 

922 Utilize ``self.defaults.dataId`` to fill in missing governor 

923 dimension key-value pairs. Defaults to `True` (i.e. defaults are 

924 used). 

925 **kwargs 

926 Additional keywords are treated like additional key-value pairs for 

927 ``dataId``, extending and overriding. 

928 

929 Returns 

930 ------- 

931 expanded : `DataCoordinate` 

932 A data ID that includes full metadata for all of the dimensions it 

933 identifies, i.e. guarantees that ``expanded.hasRecords()`` and 

934 ``expanded.hasFull()`` both return `True`. 

935 

936 Raises 

937 ------ 

938 lsst.daf.butler.registry.DataIdError 

939 Raised when ``dataId`` or keyword arguments specify unknown 

940 dimensions or values, or when a resulting data ID contains 

941 contradictory key-value pairs, according to dimension 

942 relationships. 

943 

944 Notes 

945 ----- 

946 This method cannot be relied upon to reject invalid data ID values 

947 for dimensions that do actually not have any record columns. For 

948 efficiency reasons the records for these dimensions (which have only 

949 dimension key values that are given by the caller) may be constructed 

950 directly rather than obtained from the registry database. 

951 """ 

952 raise NotImplementedError() 

953 

954 @abstractmethod 

955 def insertDimensionData( 

956 self, 

957 element: DimensionElement | str, 

958 *data: Mapping[str, Any] | DimensionRecord, 

959 conform: bool = True, 

960 replace: bool = False, 

961 skip_existing: bool = False, 

962 ) -> None: 

963 """Insert one or more dimension records into the database. 

964 

965 Parameters 

966 ---------- 

967 element : `DimensionElement` or `str` 

968 The `DimensionElement` or name thereof that identifies the table 

969 records will be inserted into. 

970 *data : `dict` or `DimensionRecord` 

971 One or more records to insert. 

972 conform : `bool`, optional 

973 If `False` (`True` is default) perform no checking or conversions, 

974 and assume that ``element`` is a `DimensionElement` instance and 

975 ``data`` is a one or more `DimensionRecord` instances of the 

976 appropriate subclass. 

977 replace : `bool`, optional 

978 If `True` (`False` is default), replace existing records in the 

979 database if there is a conflict. 

980 skip_existing : `bool`, optional 

981 If `True` (`False` is default), skip insertion if a record with 

982 the same primary key values already exists. Unlike 

983 `syncDimensionData`, this will not detect when the given record 

984 differs from what is in the database, and should not be used when 

985 this is a concern. 

986 """ 

987 raise NotImplementedError() 

988 

989 @abstractmethod 

990 def syncDimensionData( 

991 self, 

992 element: DimensionElement | str, 

993 row: Mapping[str, Any] | DimensionRecord, 

994 conform: bool = True, 

995 update: bool = False, 

996 ) -> bool | dict[str, Any]: 

997 """Synchronize the given dimension record with the database, inserting 

998 if it does not already exist and comparing values if it does. 

999 

1000 Parameters 

1001 ---------- 

1002 element : `DimensionElement` or `str` 

1003 The `DimensionElement` or name thereof that identifies the table 

1004 records will be inserted into. 

1005 row : `dict` or `DimensionRecord` 

1006 The record to insert. 

1007 conform : `bool`, optional 

1008 If `False` (`True` is default) perform no checking or conversions, 

1009 and assume that ``element`` is a `DimensionElement` instance and 

1010 ``data`` is a one or more `DimensionRecord` instances of the 

1011 appropriate subclass. 

1012 update : `bool`, optional 

1013 If `True` (`False` is default), update the existing record in the 

1014 database if there is a conflict. 

1015 

1016 Returns 

1017 ------- 

1018 inserted_or_updated : `bool` or `dict` 

1019 `True` if a new row was inserted, `False` if no changes were 

1020 needed, or a `dict` mapping updated column names to their old 

1021 values if an update was performed (only possible if 

1022 ``update=True``). 

1023 

1024 Raises 

1025 ------ 

1026 lsst.daf.butler.registry.ConflictingDefinitionError 

1027 Raised if the record exists in the database (according to primary 

1028 key lookup) but is inconsistent with the given one. 

1029 """ 

1030 raise NotImplementedError() 

1031 

1032 @abstractmethod 

1033 def queryDatasetTypes( 

1034 self, 

1035 expression: Any = ..., 

1036 *, 

1037 missing: list[str] | None = None, 

1038 ) -> Iterable[DatasetType]: 

1039 """Iterate over the dataset types whose names match an expression. 

1040 

1041 Parameters 

1042 ---------- 

1043 expression : dataset type expression, optional 

1044 An expression that fully or partially identifies the dataset types 

1045 to return, such as a `str`, `re.Pattern`, or iterable thereof. 

1046 ``...`` can be used to return all dataset types, and is the 

1047 default. See :ref:`daf_butler_dataset_type_expressions` for more 

1048 information. 

1049 missing : `list` of `str`, optional 

1050 String dataset type names that were explicitly given (i.e. not 

1051 regular expression patterns) but not found will be appended to this 

1052 list, if it is provided. 

1053 

1054 Returns 

1055 ------- 

1056 dataset_types : `~collections.abc.Iterable` [ `DatasetType`] 

1057 An `~collections.abc.Iterable` of `DatasetType` instances whose 

1058 names match ``expression``. 

1059 

1060 Raises 

1061 ------ 

1062 lsst.daf.butler.registry.DatasetTypeExpressionError 

1063 Raised when ``expression`` is invalid. 

1064 """ 

1065 raise NotImplementedError() 

1066 

1067 @abstractmethod 

1068 def queryCollections( 

1069 self, 

1070 expression: Any = ..., 

1071 datasetType: DatasetType | None = None, 

1072 collectionTypes: Iterable[CollectionType] | CollectionType = CollectionType.all(), 

1073 flattenChains: bool = False, 

1074 includeChains: bool | None = None, 

1075 ) -> Sequence[str]: 

1076 """Iterate over the collections whose names match an expression. 

1077 

1078 Parameters 

1079 ---------- 

1080 expression : collection expression, optional 

1081 An expression that identifies the collections to return, such as 

1082 a `str` (for full matches or partial matches via globs), 

1083 `re.Pattern` (for partial matches), or iterable thereof. ``...`` 

1084 can be used to return all collections, and is the default. 

1085 See :ref:`daf_butler_collection_expressions` for more information. 

1086 datasetType : `DatasetType`, optional 

1087 If provided, only yield collections that may contain datasets of 

1088 this type. This is a conservative approximation in general; it may 

1089 yield collections that do not have any such datasets. 

1090 collectionTypes : `~collections.abc.Set` [`CollectionType`] or \ 

1091 `CollectionType`, optional 

1092 If provided, only yield collections of these types. 

1093 flattenChains : `bool`, optional 

1094 If `True` (`False` is default), recursively yield the child 

1095 collections of matching `~CollectionType.CHAINED` collections. 

1096 includeChains : `bool`, optional 

1097 If `True`, yield records for matching `~CollectionType.CHAINED` 

1098 collections. Default is the opposite of ``flattenChains``: include 

1099 either CHAINED collections or their children, but not both. 

1100 

1101 Returns 

1102 ------- 

1103 collections : `~collections.abc.Sequence` [ `str` ] 

1104 The names of collections that match ``expression``. 

1105 

1106 Raises 

1107 ------ 

1108 lsst.daf.butler.registry.CollectionExpressionError 

1109 Raised when ``expression`` is invalid. 

1110 

1111 Notes 

1112 ----- 

1113 The order in which collections are returned is unspecified, except that 

1114 the children of a `~CollectionType.CHAINED` collection are guaranteed 

1115 to be in the order in which they are searched. When multiple parent 

1116 `~CollectionType.CHAINED` collections match the same criteria, the 

1117 order in which the two lists appear is unspecified, and the lists of 

1118 children may be incomplete if a child has multiple parents. 

1119 """ 

1120 raise NotImplementedError() 

1121 

1122 @abstractmethod 

1123 def queryDatasets( 

1124 self, 

1125 datasetType: Any, 

1126 *, 

1127 collections: CollectionArgType | None = None, 

1128 dimensions: Iterable[str] | None = None, 

1129 dataId: DataId | None = None, 

1130 where: str = "", 

1131 findFirst: bool = False, 

1132 bind: Mapping[str, Any] | None = None, 

1133 check: bool = True, 

1134 **kwargs: Any, 

1135 ) -> DatasetQueryResults: 

1136 """Query for and iterate over dataset references matching user-provided 

1137 criteria. 

1138 

1139 Parameters 

1140 ---------- 

1141 datasetType : dataset type expression 

1142 An expression that fully or partially identifies the dataset types 

1143 to be queried. Allowed types include `DatasetType`, `str`, 

1144 `re.Pattern`, and iterables thereof. The special value ``...`` can 

1145 be used to query all dataset types. See 

1146 :ref:`daf_butler_dataset_type_expressions` for more information. 

1147 collections : collection expression, optional 

1148 An expression that identifies the collections to search, such as a 

1149 `str` (for full matches or partial matches via globs), `re.Pattern` 

1150 (for partial matches), or iterable thereof. ``...`` can be used to 

1151 search all collections (actually just all `~CollectionType.RUN` 

1152 collections, because this will still find all datasets). 

1153 If not provided, ``self.default.collections`` is used. See 

1154 :ref:`daf_butler_collection_expressions` for more information. 

1155 dimensions : `~collections.abc.Iterable` [ `str` ] 

1156 Dimensions to include in the query (in addition to those used 

1157 to identify the queried dataset type(s)), either to constrain 

1158 the resulting datasets to those for which a matching dimension 

1159 exists, or to relate the dataset type's dimensions to dimensions 

1160 referenced by the ``dataId`` or ``where`` arguments. 

1161 dataId : `dict` or `DataCoordinate`, optional 

1162 A data ID whose key-value pairs are used as equality constraints 

1163 in the query. 

1164 where : `str`, optional 

1165 A string expression similar to a SQL WHERE clause. May involve 

1166 any column of a dimension table or (as a shortcut for the primary 

1167 key column of a dimension table) dimension name. See 

1168 :ref:`daf_butler_dimension_expressions` for more information. 

1169 findFirst : `bool`, optional 

1170 If `True` (`False` is default), for each result data ID, only 

1171 yield one `DatasetRef` of each `DatasetType`, from the first 

1172 collection in which a dataset of that dataset type appears 

1173 (according to the order of ``collections`` passed in). If `True`, 

1174 ``collections`` must not contain regular expressions and may not