Coverage for python / lsst / daf / butler / registry / _registry.py: 98%

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ("CollectionArgType", "Registry") 

31 

32import contextlib 

33import logging 

34import re 

35from abc import ABC, abstractmethod 

36from collections.abc import Iterable, Iterator, Mapping, Sequence 

37from types import EllipsisType 

38from typing import TYPE_CHECKING, Any, TypeAlias 

39 

40from .._collection_type import CollectionType 

41from .._dataset_association import DatasetAssociation 

42from .._dataset_ref import DatasetId, DatasetIdGenEnum, DatasetRef 

43from .._dataset_type import DatasetType 

44from .._storage_class import StorageClassFactory 

45from .._timespan import Timespan 

46from ..dimensions import ( 

47 DataCoordinate, 

48 DataId, 

49 DimensionElement, 

50 DimensionGroup, 

51 DimensionRecord, 

52 DimensionUniverse, 

53) 

54from ._collection_summary import CollectionSummary 

55from ._defaults import RegistryDefaults 

56from .queries import DataCoordinateQueryResults, DatasetQueryResults, DimensionRecordQueryResults 

57from .wildcards import CollectionWildcard 

58 

59if TYPE_CHECKING: 

60 from .interfaces import ObsCoreTableManager 

61 

62_LOG = logging.getLogger(__name__) 

63 

64# TYpe alias for `collections` arguments. 

65CollectionArgType: TypeAlias = ( 

66 str | re.Pattern | Iterable[str | re.Pattern] | EllipsisType | CollectionWildcard 

67) 

68 

69 

70class Registry(ABC): 

71 """Abstract Registry interface. 

72 

73 All subclasses should store `~lsst.daf.butler.registry.RegistryDefaults` in 

74 a ``_defaults`` property. No other properties are assumed shared between 

75 implementations. 

76 

77 See Also 

78 -------- 

79 lsst.daf.butler.Butler 

80 """ 

81 

82 @abstractmethod 

83 def isWriteable(self) -> bool: 

84 """Return `True` if this registry allows write operations, and `False` 

85 otherwise. 

86 """ 

87 raise NotImplementedError() 

88 

89 @property 

90 @abstractmethod 

91 def dimensions(self) -> DimensionUniverse: 

92 """Definitions of all dimensions recognized by this `Registry` 

93 (`DimensionUniverse`). 

94 """ 

95 raise NotImplementedError() 

96 

97 @property 

98 @abstractmethod 

99 def defaults(self) -> RegistryDefaults: 

100 """Default collection search path and/or output `~CollectionType.RUN` 

101 collection (`~lsst.daf.butler.registry.RegistryDefaults`). 

102 

103 This is an immutable struct whose components may not be set 

104 individually, but the entire struct can be set by assigning to this 

105 property. 

106 """ 

107 raise NotImplementedError() 

108 

109 @defaults.setter 

110 @abstractmethod 

111 def defaults(self, value: RegistryDefaults) -> None: 

112 raise NotImplementedError() 

113 

114 @abstractmethod 

115 def refresh(self) -> None: 

116 """Refresh all in-memory state by querying the database. 

117 

118 This may be necessary to enable querying for entities added by other 

119 registry instances after this one was constructed. 

120 """ 

121 raise NotImplementedError() 

122 

123 @abstractmethod 

124 def refresh_collection_summaries(self) -> None: 

125 """Refresh content of the collection summary tables in the database. 

126 

127 This method can be run to cleanup the contents of the summary tables 

128 after removing datasets from collections. 

129 """ 

130 raise NotImplementedError() 

131 

132 @abstractmethod 

133 def caching_context(self) -> contextlib.AbstractContextManager[None]: 

134 """Context manager that enables caching.""" 

135 raise NotImplementedError() 

136 

137 @contextlib.contextmanager 

138 @abstractmethod 

139 def transaction(self, *, savepoint: bool = False) -> Iterator[None]: 

140 """Return a context manager that represents a transaction. 

141 

142 Parameters 

143 ---------- 

144 savepoint : `bool` 

145 Whether to create a SAVEPOINT or not. 

146 """ 

147 raise NotImplementedError() 

148 

149 def resetConnectionPool(self) -> None: 

150 """Reset connection pool for registry if relevant. 

151 

152 This operation can be used reset connections to servers when 

153 using registry with fork-based multiprocessing. This method should 

154 usually be called by the child process immediately 

155 after the fork. 

156 

157 The base class implementation is a no-op. 

158 """ 

159 pass 

160 

161 @abstractmethod 

162 def registerCollection( 

163 self, name: str, type: CollectionType = CollectionType.TAGGED, doc: str | None = None 

164 ) -> bool: 

165 """Add a new collection if one with the given name does not exist. 

166 

167 Parameters 

168 ---------- 

169 name : `str` 

170 The name of the collection to create. 

171 type : `CollectionType` 

172 Enum value indicating the type of collection to create. 

173 doc : `str`, optional 

174 Documentation string for the collection. 

175 

176 Returns 

177 ------- 

178 registered : `bool` 

179 Boolean indicating whether the collection was already registered 

180 or was created by this call. 

181 

182 Notes 

183 ----- 

184 This method cannot be called within transactions, as it needs to be 

185 able to perform its own transaction to be concurrent. 

186 """ 

187 raise NotImplementedError() 

188 

189 @abstractmethod 

190 def getCollectionType(self, name: str) -> CollectionType: 

191 """Return an enumeration value indicating the type of the given 

192 collection. 

193 

194 Parameters 

195 ---------- 

196 name : `str` 

197 The name of the collection. 

198 

199 Returns 

200 ------- 

201 type : `CollectionType` 

202 Enum value indicating the type of this collection. 

203 

204 Raises 

205 ------ 

206 lsst.daf.butler.registry.MissingCollectionError 

207 Raised if no collection with the given name exists. 

208 """ 

209 raise NotImplementedError() 

210 

211 @abstractmethod 

212 def registerRun(self, name: str, doc: str | None = None) -> bool: 

213 """Add a new run if one with the given name does not exist. 

214 

215 Parameters 

216 ---------- 

217 name : `str` 

218 The name of the run to create. 

219 doc : `str`, optional 

220 Documentation string for the collection. 

221 

222 Returns 

223 ------- 

224 registered : `bool` 

225 Boolean indicating whether a new run was registered. `False` 

226 if it already existed. 

227 

228 Notes 

229 ----- 

230 This method cannot be called within transactions, as it needs to be 

231 able to perform its own transaction to be concurrent. 

232 """ 

233 raise NotImplementedError() 

234 

235 @abstractmethod 

236 def removeCollection(self, name: str) -> None: 

237 """Remove the given collection from the registry. 

238 

239 Parameters 

240 ---------- 

241 name : `str` 

242 The name of the collection to remove. 

243 

244 Raises 

245 ------ 

246 lsst.daf.butler.registry.MissingCollectionError 

247 Raised if no collection with the given name exists. 

248 sqlalchemy.exc.IntegrityError 

249 Raised if the database rows associated with the collection are 

250 still referenced by some other table, such as a dataset in a 

251 datastore (for `~CollectionType.RUN` collections only) or a 

252 `~CollectionType.CHAINED` collection of which this collection is 

253 a child. 

254 

255 Notes 

256 ----- 

257 If this is a `~CollectionType.RUN` collection, all datasets and quanta 

258 in it will removed from the `Registry` database. This requires that 

259 those datasets be removed (or at least trashed) from any datastores 

260 that hold them first. 

261 

262 A collection may not be deleted as long as it is referenced by a 

263 `~CollectionType.CHAINED` collection; the ``CHAINED`` collection must 

264 be deleted or redefined first. 

265 """ 

266 raise NotImplementedError() 

267 

268 @abstractmethod 

269 def getCollectionChain(self, parent: str) -> Sequence[str]: 

270 """Return the child collections in a `~CollectionType.CHAINED` 

271 collection. 

272 

273 Parameters 

274 ---------- 

275 parent : `str` 

276 Name of the chained collection. Must have already been added via 

277 a call to `Registry.registerCollection`. 

278 

279 Returns 

280 ------- 

281 children : `~collections.abc.Sequence` [ `str` ] 

282 An ordered sequence of collection names that are searched when the 

283 given chained collection is searched. 

284 

285 Raises 

286 ------ 

287 lsst.daf.butler.registry.MissingCollectionError 

288 Raised if ``parent`` does not exist in the `Registry`. 

289 lsst.daf.butler.registry.CollectionTypeError 

290 Raised if ``parent`` does not correspond to a 

291 `~CollectionType.CHAINED` collection. 

292 """ 

293 raise NotImplementedError() 

294 

295 @abstractmethod 

296 def setCollectionChain(self, parent: str, children: Any, *, flatten: bool = False) -> None: 

297 """Define or redefine a `~CollectionType.CHAINED` collection. 

298 

299 Parameters 

300 ---------- 

301 parent : `str` 

302 Name of the chained collection. Must have already been added via 

303 a call to `Registry.registerCollection`. 

304 children : collection expression 

305 An expression defining an ordered search of child collections, 

306 generally an iterable of `str`; see 

307 :ref:`daf_butler_collection_expressions` for more information. 

308 flatten : `bool`, optional 

309 If `True` (`False` is default), recursively flatten out any nested 

310 `~CollectionType.CHAINED` collections in ``children`` first. 

311 

312 Raises 

313 ------ 

314 lsst.daf.butler.registry.MissingCollectionError 

315 Raised when any of the given collections do not exist in the 

316 `Registry`. 

317 lsst.daf.butler.registry.CollectionTypeError 

318 Raised if ``parent`` does not correspond to a 

319 `~CollectionType.CHAINED` collection. 

320 ValueError 

321 Raised if the given collections contains a cycle. 

322 """ 

323 raise NotImplementedError() 

324 

325 @abstractmethod 

326 def getCollectionParentChains(self, collection: str) -> set[str]: 

327 """Return the CHAINED collections that directly contain the given one. 

328 

329 Parameters 

330 ---------- 

331 collection : `str` 

332 Name of the collection. 

333 

334 Returns 

335 ------- 

336 chains : `set` of `str` 

337 Set of `~CollectionType.CHAINED` collection names. 

338 """ 

339 raise NotImplementedError() 

340 

341 @abstractmethod 

342 def getCollectionDocumentation(self, collection: str) -> str | None: 

343 """Retrieve the documentation string for a collection. 

344 

345 Parameters 

346 ---------- 

347 collection : `str` 

348 Name of the collection. 

349 

350 Returns 

351 ------- 

352 docs : `str` or `None` 

353 Docstring for the collection with the given name. 

354 """ 

355 raise NotImplementedError() 

356 

357 @abstractmethod 

358 def setCollectionDocumentation(self, collection: str, doc: str | None) -> None: 

359 """Set the documentation string for a collection. 

360 

361 Parameters 

362 ---------- 

363 collection : `str` 

364 Name of the collection. 

365 doc : `str` or `None` 

366 Docstring for the collection with the given name; will replace any 

367 existing docstring. Passing `None` will remove any existing 

368 docstring. 

369 """ 

370 raise NotImplementedError() 

371 

372 @abstractmethod 

373 def getCollectionSummary(self, collection: str) -> CollectionSummary: 

374 """Return a summary for the given collection. 

375 

376 Parameters 

377 ---------- 

378 collection : `str` 

379 Name of the collection for which a summary is to be retrieved. 

380 

381 Returns 

382 ------- 

383 summary : `~lsst.daf.butler.registry.CollectionSummary` 

384 Summary of the dataset types and governor dimension values in 

385 this collection. 

386 """ 

387 raise NotImplementedError() 

388 

389 @abstractmethod 

390 def registerDatasetType(self, datasetType: DatasetType) -> bool: 

391 """Add a new `DatasetType` to the Registry. 

392 

393 It is not an error to register the same `DatasetType` twice. 

394 

395 Parameters 

396 ---------- 

397 datasetType : `DatasetType` 

398 The `DatasetType` to be added. 

399 

400 Returns 

401 ------- 

402 inserted : `bool` 

403 `True` if ``datasetType`` was inserted, `False` if an identical 

404 existing `DatasetType` was found. Note that in either case the 

405 DatasetType is guaranteed to be defined in the Registry 

406 consistently with the given definition. 

407 

408 Raises 

409 ------ 

410 ValueError 

411 Raised if the dimensions or storage class are invalid. 

412 lsst.daf.butler.registry.ConflictingDefinitionError 

413 Raised if this `DatasetType` is already registered with a different 

414 definition. 

415 

416 Notes 

417 ----- 

418 This method cannot be called within transactions, as it needs to be 

419 able to perform its own transaction to be concurrent. 

420 """ 

421 raise NotImplementedError() 

422 

423 @abstractmethod 

424 def removeDatasetType(self, name: str | tuple[str, ...]) -> None: 

425 """Remove the named `DatasetType` from the registry. 

426 

427 .. warning:: 

428 

429 Registry implementations can cache the dataset type definitions. 

430 This means that deleting the dataset type definition may result in 

431 unexpected behavior from other butler processes that are active 

432 that have not seen the deletion. 

433 

434 Parameters 

435 ---------- 

436 name : `str` or `tuple` [`str`] 

437 Name of the type to be removed or tuple containing a list of type 

438 names to be removed. Wildcards are allowed. 

439 

440 Returns 

441 ------- 

442 None 

443 

444 Raises 

445 ------ 

446 lsst.daf.butler.registry.OrphanedRecordError 

447 Raised if an attempt is made to remove the dataset type definition 

448 when there are already datasets associated with it. 

449 

450 Notes 

451 ----- 

452 If the dataset type is not registered the method will return without 

453 action. 

454 """ 

455 raise NotImplementedError() 

456 

457 @abstractmethod 

458 def getDatasetType(self, name: str) -> DatasetType: 

459 """Get the `DatasetType`. 

460 

461 Parameters 

462 ---------- 

463 name : `str` 

464 Name of the type. 

465 

466 Returns 

467 ------- 

468 type : `DatasetType` 

469 The `DatasetType` associated with the given name. 

470 

471 Raises 

472 ------ 

473 lsst.daf.butler.registry.MissingDatasetTypeError 

474 Raised if the requested dataset type has not been registered. 

475 

476 Notes 

477 ----- 

478 This method handles component dataset types automatically, though most 

479 other registry operations do not. 

480 """ 

481 raise NotImplementedError() 

482 

483 @abstractmethod 

484 def supportsIdGenerationMode(self, mode: DatasetIdGenEnum) -> bool: 

485 """Test whether the given dataset ID generation mode is supported by 

486 `insertDatasets`. 

487 

488 Parameters 

489 ---------- 

490 mode : `DatasetIdGenEnum` 

491 Enum value for the mode to test. 

492 

493 Returns 

494 ------- 

495 supported : `bool` 

496 Whether the given mode is supported. 

497 """ 

498 raise NotImplementedError() 

499 

500 @abstractmethod 

501 def findDataset( 

502 self, 

503 datasetType: DatasetType | str, 

504 dataId: DataId | None = None, 

505 *, 

506 collections: CollectionArgType | None = None, 

507 timespan: Timespan | None = None, 

508 datastore_records: bool = False, 

509 **kwargs: Any, 

510 ) -> DatasetRef | None: 

511 """Find a dataset given its `DatasetType` and data ID. 

512 

513 This can be used to obtain a `DatasetRef` that permits the dataset to 

514 be read from a `Datastore`. If the dataset is a component and can not 

515 be found using the provided dataset type, a dataset ref for the parent 

516 will be returned instead but with the correct dataset type. 

517 

518 Parameters 

519 ---------- 

520 datasetType : `DatasetType` or `str` 

521 A `DatasetType` or the name of one. If this is a `DatasetType` 

522 instance, its storage class will be respected and propagated to 

523 the output, even if it differs from the dataset type definition 

524 in the registry, as long as the storage classes are convertible. 

525 dataId : `dict` or `DataCoordinate`, optional 

526 A `dict`-like object containing the `Dimension` links that identify 

527 the dataset within a collection. 

528 collections : collection expression, optional 

529 An expression that fully or partially identifies the collections to 

530 search for the dataset; see 

531 :ref:`daf_butler_collection_expressions` for more information. 

532 Defaults to ``self.defaults.collections``. 

533 timespan : `Timespan`, optional 

534 A timespan that the validity range of the dataset must overlap. 

535 If not provided, any `~CollectionType.CALIBRATION` collections 

536 matched by the ``collections`` argument will not be searched. 

537 datastore_records : `bool`, optional 

538 Whether to attach datastore records to the `DatasetRef`. 

539 **kwargs 

540 Additional keyword arguments passed to 

541 `DataCoordinate.standardize` to convert ``dataId`` to a true 

542 `DataCoordinate` or augment an existing one. 

543 

544 Returns 

545 ------- 

546 ref : `DatasetRef` 

547 A reference to the dataset, or `None` if no matching Dataset 

548 was found. 

549 

550 Raises 

551 ------ 

552 lsst.daf.butler.registry.NoDefaultCollectionError 

553 Raised if ``collections`` is `None` and 

554 ``self.defaults.collections`` is `None`. 

555 LookupError 

556 Raised if one or more data ID keys are missing. 

557 lsst.daf.butler.registry.MissingDatasetTypeError 

558 Raised if the dataset type does not exist. 

559 lsst.daf.butler.registry.MissingCollectionError 

560 Raised if any of ``collections`` does not exist in the registry. 

561 

562 Notes 

563 ----- 

564 This method simply returns `None` and does not raise an exception even 

565 when the set of collections searched is intrinsically incompatible with 

566 the dataset type, e.g. if ``datasetType.isCalibration() is False``, but 

567 only `~CollectionType.CALIBRATION` collections are being searched. 

568 This may make it harder to debug some lookup failures, but the behavior 

569 is intentional; we consider it more important that failed searches are 

570 reported consistently, regardless of the reason, and that adding 

571 additional collections that do not contain a match to the search path 

572 never changes the behavior. 

573 

574 This method handles component dataset types automatically, though most 

575 other registry operations do not. 

576 """ 

577 raise NotImplementedError() 

578 

579 @abstractmethod 

580 def insertDatasets( 

581 self, 

582 datasetType: DatasetType | str, 

583 dataIds: Iterable[DataId], 

584 run: str | None = None, 

585 expand: bool = True, 

586 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE, 

587 ) -> list[DatasetRef]: 

588 """Insert one or more datasets into the `Registry`. 

589 

590 This always adds new datasets; to associate existing datasets with 

591 a new collection, use ``associate``. 

592 

593 Parameters 

594 ---------- 

595 datasetType : `DatasetType` or `str` 

596 A `DatasetType` or the name of one. 

597 dataIds : `~collections.abc.Iterable` of `dict` or `DataCoordinate` 

598 Dimension-based identifiers for the new datasets. 

599 run : `str`, optional 

600 The name of the run that produced the datasets. Defaults to 

601 ``self.defaults.run``. 

602 expand : `bool`, optional 

603 If `True` (default), expand data IDs as they are inserted. This is 

604 necessary in general to allow datastore to generate file templates, 

605 but it may be disabled if the caller can guarantee this is 

606 unnecessary. 

607 idGenerationMode : `DatasetIdGenEnum`, optional 

608 Specifies option for generating dataset IDs. By default unique IDs 

609 are generated for each inserted dataset. 

610 

611 Returns 

612 ------- 

613 refs : `list` of `DatasetRef` 

614 Resolved `DatasetRef` instances for all given data IDs (in the same 

615 order). 

616 

617 Raises 

618 ------ 

619 lsst.daf.butler.registry.DatasetTypeError 

620 Raised if ``datasetType`` is not known to registry. 

621 lsst.daf.butler.registry.CollectionTypeError 

622 Raised if ``run`` collection type is not `~CollectionType.RUN`. 

623 lsst.daf.butler.registry.NoDefaultCollectionError 

624 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

625 lsst.daf.butler.registry.ConflictingDefinitionError 

626 If a dataset with the same dataset type and data ID as one of those 

627 given already exists in ``run``. 

628 lsst.daf.butler.registry.MissingCollectionError 

629 Raised if ``run`` does not exist in the registry. 

630 """ 

631 raise NotImplementedError() 

632 

633 @abstractmethod 

634 def _importDatasets( 

635 self, 

636 datasets: Iterable[DatasetRef], 

637 expand: bool = True, 

638 assume_new: bool = False, 

639 ) -> list[DatasetRef]: 

640 """Import one or more datasets into the `Registry`. 

641 

642 This differs from `insertDatasets` method in that this method accepts 

643 `DatasetRef` instances, which already have a dataset ID. 

644 

645 Parameters 

646 ---------- 

647 datasets : `~collections.abc.Iterable` of `DatasetRef` 

648 Datasets to be inserted. All `DatasetRef` instances must have 

649 identical ``run`` attributes. ``run`` 

650 attribute can be `None` and defaults to ``self.defaults.run``. 

651 Datasets can specify ``id`` attribute which will be used for 

652 inserted datasets. 

653 Datasets can be of multiple dataset types, but all the dataset 

654 types must have the same set of dimensions. 

655 expand : `bool`, optional 

656 If `True` (default), expand data IDs as they are inserted. This is 

657 necessary in general, but it may be disabled if the caller can 

658 guarantee this is unnecessary. 

659 assume_new : `bool`, optional 

660 If `True`, assume datasets are new. If `False`, datasets that are 

661 identical to an existing one are ignored. 

662 

663 Returns 

664 ------- 

665 refs : `list` of `DatasetRef` 

666 `DatasetRef` instances for all given data IDs (in the same order). 

667 If any of ``datasets`` has an ID which already exists in the 

668 database then it will not be inserted or updated, but a 

669 `DatasetRef` will be returned for it in any case. 

670 

671 Raises 

672 ------ 

673 lsst.daf.butler.registry.NoDefaultCollectionError 

674 Raised if ``run`` is `None` and ``self.defaults.run`` is `None`. 

675 lsst.daf.butler.registry.DatasetTypeError 

676 Raised if a dataset type is not known to registry. 

677 lsst.daf.butler.registry.ConflictingDefinitionError 

678 If a dataset with the same dataset type and data ID as one of those 

679 given already exists in ``run``, or if ``assume_new=True`` and at 

680 least one dataset is not new. 

681 lsst.daf.butler.registry.MissingCollectionError 

682 Raised if ``run`` does not exist in the registry. 

683 

684 Notes 

685 ----- 

686 This method is considered middleware-internal. 

687 """ 

688 raise NotImplementedError() 

689 

690 @abstractmethod 

691 def getDataset(self, id: DatasetId) -> DatasetRef | None: 

692 """Retrieve a Dataset entry. 

693 

694 Parameters 

695 ---------- 

696 id : `DatasetId` 

697 The unique identifier for the dataset. 

698 

699 Returns 

700 ------- 

701 ref : `DatasetRef` or `None` 

702 A ref to the Dataset, or `None` if no matching Dataset 

703 was found. 

704 """ 

705 raise NotImplementedError() 

706 

707 @abstractmethod 

708 def _fetch_run_dataset_ids(self, run: str) -> list[DatasetId]: 

709 """Return the IDs of all datasets in the given ``RUN`` 

710 collection. 

711 

712 Parameters 

713 ---------- 

714 run : `str` 

715 Name of the collection. 

716 

717 Returns 

718 ------- 

719 dataset_ids : `list` [`uuid.UUID`] 

720 List of dataset IDs. 

721 

722 Notes 

723 ----- 

724 This is a middleware-internal interface. 

725 """ 

726 raise NotImplementedError() 

727 

728 @abstractmethod 

729 def removeDatasets(self, refs: Iterable[DatasetRef]) -> None: 

730 """Remove datasets from the Registry. 

731 

732 The datasets will be removed unconditionally from all collections, and 

733 any `Quantum` that consumed this dataset will instead be marked with 

734 having a NULL input. `Datastore` records will *not* be deleted; the 

735 caller is responsible for ensuring that the dataset has already been 

736 removed from all Datastores. 

737 

738 Parameters 

739 ---------- 

740 refs : `~collections.abc.Iterable` [`DatasetRef`] 

741 References to the datasets to be removed. Must include a valid 

742 ``id`` attribute, and should be considered invalidated upon return. 

743 

744 Raises 

745 ------ 

746 lsst.daf.butler.AmbiguousDatasetError 

747 Raised if any ``ref.id`` is `None`. 

748 lsst.daf.butler.registry.OrphanedRecordError 

749 Raised if any dataset is still present in any `Datastore`. 

750 """ 

751 raise NotImplementedError() 

752 

753 @abstractmethod 

754 def associate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

755 """Add existing datasets to a `~CollectionType.TAGGED` collection. 

756 

757 If a DatasetRef with the same exact ID is already in a collection 

758 nothing is changed. If a `DatasetRef` with the same `DatasetType` and 

759 data ID but with different ID exists in the collection, 

760 `~lsst.daf.butler.registry.ConflictingDefinitionError` is raised. 

761 

762 Parameters 

763 ---------- 

764 collection : `str` 

765 Indicates the collection the datasets should be associated with. 

766 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

767 An iterable of resolved `DatasetRef` instances that already exist 

768 in this `Registry`. 

769 

770 Raises 

771 ------ 

772 lsst.daf.butler.registry.ConflictingDefinitionError 

773 If a Dataset with the given `DatasetRef` already exists in the 

774 given collection. 

775 lsst.daf.butler.registry.MissingCollectionError 

776 Raised if ``collection`` does not exist in the registry. 

777 lsst.daf.butler.registry.CollectionTypeError 

778 Raise adding new datasets to the given ``collection`` is not 

779 allowed. 

780 """ 

781 raise NotImplementedError() 

782 

783 @abstractmethod 

784 def disassociate(self, collection: str, refs: Iterable[DatasetRef]) -> None: 

785 """Remove existing datasets from a `~CollectionType.TAGGED` collection. 

786 

787 ``collection`` and ``ref`` combinations that are not currently 

788 associated are silently ignored. 

789 

790 Parameters 

791 ---------- 

792 collection : `str` 

793 The collection the datasets should no longer be associated with. 

794 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

795 An iterable of resolved `DatasetRef` instances that already exist 

796 in this `Registry`. 

797 

798 Raises 

799 ------ 

800 lsst.daf.butler.AmbiguousDatasetError 

801 Raised if any of the given dataset references is unresolved. 

802 lsst.daf.butler.registry.MissingCollectionError 

803 Raised if ``collection`` does not exist in the registry. 

804 lsst.daf.butler.registry.CollectionTypeError 

805 Raise adding new datasets to the given ``collection`` is not 

806 allowed. 

807 """ 

808 raise NotImplementedError() 

809 

810 @abstractmethod 

811 def certify(self, collection: str, refs: Iterable[DatasetRef], timespan: Timespan) -> None: 

812 """Associate one or more datasets with a calibration collection and a 

813 validity range within it. 

814 

815 Parameters 

816 ---------- 

817 collection : `str` 

818 The name of an already-registered `~CollectionType.CALIBRATION` 

819 collection. 

820 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

821 Datasets to be associated. 

822 timespan : `Timespan` 

823 The validity range for these datasets within the collection. 

824 

825 Raises 

826 ------ 

827 lsst.daf.butler.AmbiguousDatasetError 

828 Raised if any of the given `DatasetRef` instances is unresolved. 

829 lsst.daf.butler.registry.ConflictingDefinitionError 

830 Raised if the collection already contains a different dataset with 

831 the same `DatasetType` and data ID and an overlapping validity 

832 range. 

833 lsst.daf.butler.registry.CollectionTypeError 

834 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

835 collection or if one or more datasets are of a dataset type for 

836 which `DatasetType.isCalibration` returns `False`. 

837 """ 

838 raise NotImplementedError() 

839 

840 @abstractmethod 

841 def decertify( 

842 self, 

843 collection: str, 

844 datasetType: str | DatasetType, 

845 timespan: Timespan, 

846 *, 

847 dataIds: Iterable[DataId] | None = None, 

848 ) -> None: 

849 """Remove or adjust datasets to clear a validity range within a 

850 calibration collection. 

851 

852 Parameters 

853 ---------- 

854 collection : `str` 

855 The name of an already-registered `~CollectionType.CALIBRATION` 

856 collection. 

857 datasetType : `str` or `DatasetType` 

858 Name or `DatasetType` instance for the datasets to be decertified. 

859 timespan : `Timespan`, optional 

860 The validity range to remove datasets from within the collection. 

861 Datasets that overlap this range but are not contained by it will 

862 have their validity ranges adjusted to not overlap it, which may 

863 split a single dataset validity range into two. 

864 dataIds : iterable [`dict` or `DataCoordinate`], optional 

865 Data IDs that should be decertified within the given validity range 

866 If `None`, all data IDs for ``self.datasetType`` will be 

867 decertified. 

868 

869 Raises 

870 ------ 

871 lsst.daf.butler.registry.CollectionTypeError 

872 Raised if ``collection`` is not a `~CollectionType.CALIBRATION` 

873 collection or if ``datasetType.isCalibration() is False``. 

874 """ 

875 raise NotImplementedError() 

876 

877 @abstractmethod 

878 def getDatasetLocations(self, ref: DatasetRef) -> Iterable[str]: 

879 """Retrieve datastore locations for a given dataset. 

880 

881 Parameters 

882 ---------- 

883 ref : `DatasetRef` 

884 A reference to the dataset for which to retrieve storage 

885 information. 

886 

887 Returns 

888 ------- 

889 datastores : `~collections.abc.Iterable` [ `str` ] 

890 All the matching datastores holding this dataset. 

891 

892 Raises 

893 ------ 

894 lsst.daf.butler.AmbiguousDatasetError 

895 Raised if ``ref.id`` is `None`. 

896 """ 

897 raise NotImplementedError() 

898 

899 @abstractmethod 

900 def expandDataId( 

901 self, 

902 dataId: DataId | None = None, 

903 *, 

904 dimensions: Iterable[str] | DimensionGroup | None = None, 

905 records: Mapping[str, DimensionRecord | None] | None = None, 

906 withDefaults: bool = True, 

907 **kwargs: Any, 

908 ) -> DataCoordinate: 

909 """Expand a dimension-based data ID to include additional information. 

910 

911 Parameters 

912 ---------- 

913 dataId : `DataCoordinate` or `dict`, optional 

914 Data ID to be expanded; augmented and overridden by ``kwargs``. 

915 dimensions : `~collections.abc.Iterable` [ `str` ] or \ 

916 `DimensionGroup` optional 

917 The dimensions to be identified by the new `DataCoordinate`. 

918 If not provided, will be inferred from the keys of ``dataId`` and 

919 ``**kwargs``, and ``universe`` must be provided unless ``dataId`` 

920 is already a `DataCoordinate`. 

921 records : `~collections.abc.Mapping` [`str`, `DimensionRecord`], \ 

922 optional 

923 Dimension record data to use before querying the database for that 

924 data, keyed by element name. 

925 withDefaults : `bool`, optional 

926 Utilize ``self.defaults.dataId`` to fill in missing governor 

927 dimension key-value pairs. Defaults to `True` (i.e. defaults are 

928 used). 

929 **kwargs 

930 Additional keywords are treated like additional key-value pairs for 

931 ``dataId``, extending and overriding. 

932 

933 Returns 

934 ------- 

935 expanded : `DataCoordinate` 

936 A data ID that includes full metadata for all of the dimensions it 

937 identifies, i.e. guarantees that ``expanded.hasRecords()`` and 

938 ``expanded.hasFull()`` both return `True`. 

939 

940 Raises 

941 ------ 

942 lsst.daf.butler.registry.DataIdError 

943 Raised when ``dataId`` or keyword arguments specify unknown 

944 dimensions or values, or when a resulting data ID contains 

945 contradictory key-value pairs, according to dimension 

946 relationships. 

947 

948 Notes 

949 ----- 

950 This method cannot be relied upon to reject invalid data ID values 

951 for dimensions that do actually not have any record columns. For 

952 efficiency reasons the records for these dimensions (which have only 

953 dimension key values that are given by the caller) may be constructed 

954 directly rather than obtained from the registry database. 

955 """ 

956 raise NotImplementedError() 

957 

958 @abstractmethod 

959 def insertDimensionData( 

960 self, 

961 element: DimensionElement | str, 

962 *data: Mapping[str, Any] | DimensionRecord, 

963 conform: bool = True, 

964 replace: bool = False, 

965 skip_existing: bool = False, 

966 ) -> None: 

967 """Insert one or more dimension records into the database. 

968 

969 Parameters 

970 ---------- 

971 element : `DimensionElement` or `str` 

972 The `DimensionElement` or name thereof that identifies the table 

973 records will be inserted into. 

974 *data : `dict` or `DimensionRecord` 

975 One or more records to insert. 

976 conform : `bool`, optional 

977 If `False` (`True` is default) perform no checking or conversions, 

978 and assume that ``element`` is a `DimensionElement` instance and 

979 ``data`` is a one or more `DimensionRecord` instances of the 

980 appropriate subclass. 

981 replace : `bool`, optional 

982 If `True` (`False` is default), replace existing records in the 

983 database if there is a conflict. 

984 skip_existing : `bool`, optional 

985 If `True` (`False` is default), skip insertion if a record with 

986 the same primary key values already exists. Unlike 

987 `syncDimensionData`, this will not detect when the given record 

988 differs from what is in the database, and should not be used when 

989 this is a concern. 

990 """ 

991 raise NotImplementedError() 

992 

993 @abstractmethod 

994 def syncDimensionData( 

995 self, 

996 element: DimensionElement | str, 

997 row: Mapping[str, Any] | DimensionRecord, 

998 conform: bool = True, 

999 update: bool = False, 

1000 ) -> bool | dict[str, Any]: 

1001 """Synchronize the given dimension record with the database, inserting 

1002 if it does not already exist and comparing values if it does. 

1003 

1004 Parameters 

1005 ---------- 

1006 element : `DimensionElement` or `str` 

1007 The `DimensionElement` or name thereof that identifies the table 

1008 records will be inserted into. 

1009 row : `dict` or `DimensionRecord` 

1010 The record to insert. 

1011 conform : `bool`, optional 

1012 If `False` (`True` is default) perform no checking or conversions, 

1013 and assume that ``element`` is a `DimensionElement` instance and 

1014 ``data`` is a one or more `DimensionRecord` instances of the 

1015 appropriate subclass. 

1016 update : `bool`, optional 

1017 If `True` (`False` is default), update the existing record in the 

1018 database if there is a conflict. 

1019 

1020 Returns 

1021 ------- 

1022 inserted_or_updated : `bool` or `dict` 

1023 `True` if a new row was inserted, `False` if no changes were 

1024 needed, or a `dict` mapping updated column names to their old 

1025 values if an update was performed (only possible if 

1026 ``update=True``). 

1027 

1028 Raises 

1029 ------ 

1030 lsst.daf.butler.registry.ConflictingDefinitionError 

1031 Raised if the record exists in the database (according to primary 

1032 key lookup) but is inconsistent with the given one. 

1033 """ 

1034 raise NotImplementedError() 

1035 

1036 @abstractmethod 

1037 def queryDatasetTypes( 

1038 self, 

1039 expression: Any = ..., 

1040 *, 

1041 missing: list[str] | None = None, 

1042 ) -> Iterable[DatasetType]: 

1043 """Iterate over the dataset types whose names match an expression. 

1044 

1045 Parameters 

1046 ---------- 

1047 expression : dataset type expression, optional 

1048 An expression that fully or partially identifies the dataset types 

1049 to return, such as a `str`, `re.Pattern`, or iterable thereof. 

1050 ``...`` can be used to return all dataset types, and is the 

1051 default. See :ref:`daf_butler_dataset_type_expressions` for more 

1052 information. 

1053 missing : `list` of `str`, optional 

1054 String dataset type names that were explicitly given (i.e. not 

1055 regular expression patterns) but not found will be appended to this 

1056 list, if it is provided. 

1057 

1058 Returns 

1059 ------- 

1060 dataset_types : `~collections.abc.Iterable` [ `DatasetType`] 

1061 An `~collections.abc.Iterable` of `DatasetType` instances whose 

1062 names match ``expression``. 

1063 

1064 Raises 

1065 ------ 

1066 lsst.daf.butler.registry.DatasetTypeExpressionError 

1067 Raised when ``expression`` is invalid. 

1068 """ 

1069 raise NotImplementedError() 

1070 

1071 @abstractmethod 

1072 def queryCollections( 

1073 self, 

1074 expression: Any = ..., 

1075 datasetType: DatasetType | None = None, 

1076 collectionTypes: Iterable[CollectionType] | CollectionType = CollectionType.all(), 

1077 flattenChains: bool = False, 

1078 includeChains: bool | None = None, 

1079 ) -> Sequence[str]: 

1080 """Iterate over the collections whose names match an expression. 

1081 

1082 Parameters 

1083 ---------- 

1084 expression : collection expression, optional 

1085 An expression that identifies the collections to return, such as 

1086 a `str` (for full matches or partial matches via globs), 

1087 `re.Pattern` (for partial matches), or iterable thereof. ``...`` 

1088 can be used to return all collections, and is the default. 

1089 See :ref:`daf_butler_collection_expressions` for more information. 

1090 datasetType : `DatasetType`, optional 

1091 If provided, only yield collections that may contain datasets of 

1092 this type. This is a conservative approximation in general; it may 

1093 yield collections that do not have any such datasets. 

1094 collectionTypes : `~collections.abc.Set` [`CollectionType`] or \ 

1095 `CollectionType`, optional 

1096 If provided, only yield collections of these types. 

1097 flattenChains : `bool`, optional 

1098 If `True` (`False` is default), recursively yield the child 

1099 collections of matching `~CollectionType.CHAINED` collections. 

1100 includeChains : `bool`, optional 

1101 If `True`, yield records for matching `~CollectionType.CHAINED` 

1102 collections. Default is the opposite of ``flattenChains``: include 

1103 either CHAINED collections or their children, but not both. 

1104 

1105 Returns 

1106 ------- 

1107 collections : `~collections.abc.Sequence` [ `str` ] 

1108 The names of collections that match ``expression``. 

1109 

1110 Raises 

1111 ------ 

1112 lsst.daf.butler.registry.CollectionExpressionError 

1113 Raised when ``expression`` is invalid. 

1114 

1115 Notes 

1116 ----- 

1117 The order in which collections are returned is unspecified, except that 

1118 the children of a `~CollectionType.CHAINED` collection are guaranteed 

1119 to be in the order in which they are searched. When multiple parent 

1120 `~CollectionType.CHAINED` collections match the same criteria, the 

1121 order in which the two lists appear is unspecified, and the lists of 

1122 children may be incomplete if a child has multiple parents. 

1123 """ 

1124 raise NotImplementedError() 

1125 

1126 @abstractmethod 

1127 def queryDatasets( 

1128 self, 

1129 datasetType: Any, 

1130 *, 

1131 collections: CollectionArgType | None = None, 

1132 dimensions: Iterable[str] | None = None, 

1133 dataId: DataId | None = None, 

1134 where: str = "", 

1135 findFirst: bool = False, 

1136 bind: Mapping[str, Any] | None = None, 

1137 check: bool = True, 

1138 **kwargs: Any, 

1139 ) -> DatasetQueryResults: 

1140 """Query for and iterate over dataset references matching user-provided 

1141 criteria. 

1142 

1143 Parameters 

1144 ---------- 

1145 datasetType : dataset type expression 

1146 An expression that fully or partially identifies the dataset types 

1147 to be queried. Allowed types include `DatasetType`, `str`, 

1148 `re.Pattern`, and iterables thereof. The special value ``...`` can 

1149 be used to query all dataset types. See 

1150 :ref:`daf_butler_dataset_type_expressions` for more information. 

1151 collections : collection expression, optional 

1152 An expression that identifies the collections to search, such as a 

1153 `str` (for full matches or partial matches via globs), `re.Pattern` 

1154 (for partial matches), or iterable thereof. ``...`` can be used to 

1155 search all collections (actually just all `~CollectionType.RUN` 

1156 collections, because this will still find all datasets). 

1157 If not provided, ``self.default.collections`` is used. See 

1158 :ref:`daf_butler_collection_expressions` for more information. 

1159 dimensions : `~collections.abc.Iterable` [ `str` ] 

1160 Dimensions to include in the query (in addition to those used 

1161 to identify the queried dataset type(s)), either to constrain 

1162 the resulting datasets to those for which a matching dimension 

1163 exists, or to relate the dataset type's dimensions to dimensions 

1164 referenced by the ``dataId`` or ``where`` arguments. 

1165 dataId : `dict` or `DataCoordinate`, optional 

1166 A data ID whose key-value pairs are used as equality constraints 

1167 in the query. 

1168 where : `str`, optional 

1169 A string expression similar to a SQL WHERE clause. May involve 

1170 any column of a dimension table or (as a shortcut for the primary 

1171 key column of a dimension table) dimension name. See 

1172 :ref:`daf_butler_dimension_expressions` for more information. 

1173 findFirst : `bool`, optional 

1174 If `True` (`False` is default), for each result data ID, only 

1175 yield one `DatasetRef` of each `DatasetType`, from the first