Coverage for python/lsst/daf/butler/transfers/_interfaces.py: 100%

32 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-08 02:50 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ["RepoExportBackend", "RepoImportBackend", "RepoTransferFormatConfig"] 

31 

32from abc import ABC, abstractmethod 

33from collections.abc import Iterable 

34from typing import TYPE_CHECKING 

35 

36from .._config import ConfigSubset 

37from .._dataset_association import DatasetAssociation 

38from .._dataset_type import DatasetType 

39from .._file_dataset import FileDataset 

40from ..datastore import Datastore 

41from ..dimensions import DimensionElement, DimensionRecord 

42from ..registry import CollectionType 

43from ..registry.interfaces import CollectionRecord 

44 

45if TYPE_CHECKING: 

46 from lsst.resources import ResourcePathExpression 

47 

48 

49class RepoTransferFormatConfig(ConfigSubset): 

50 """The section of butler configuration that associates repo import/export 

51 backends with file formats. 

52 """ 

53 

54 component = "repo_transfer_formats" 

55 defaultConfigFile = "repo_transfer_formats.yaml" 

56 

57 

58class RepoExportBackend(ABC): 

59 """An abstract interface for data repository export implementations. 

60 

61 Methods are guaranteed to be called in ways that reflect foreign key 

62 dependencies. 

63 """ 

64 

65 @abstractmethod 

66 def saveDimensionData(self, element: DimensionElement, *data: DimensionRecord) -> None: 

67 """Export one or more dimension element records. 

68 

69 Parameters 

70 ---------- 

71 element : `DimensionElement` 

72 The `DimensionElement` whose elements are being exported. 

73 *data : `DimensionRecord` (variadic) 

74 One or more records to export. 

75 """ 

76 raise NotImplementedError() 

77 

78 @abstractmethod 

79 def saveCollection(self, record: CollectionRecord, doc: str | None) -> None: 

80 """Export a collection. 

81 

82 This only exports the collection's own state, not its associations with 

83 datasets. 

84 

85 Parameters 

86 ---------- 

87 record : `CollectionRecord` 

88 Object representing the collection to export. 

89 doc : `str` or `None` 

90 Documentation string for the collection. 

91 """ 

92 raise NotImplementedError() 

93 

94 @abstractmethod 

95 def saveDatasets(self, datasetType: DatasetType, run: str, *datasets: FileDataset) -> None: 

96 """Export one or more datasets, including their associated DatasetType 

97 and run information (but not including associated dimension 

98 information). 

99 

100 Parameters 

101 ---------- 

102 datasetType : `DatasetType` 

103 Type of all datasets being exported with this call. 

104 run : `str` 

105 Run associated with all datasets being exported with this call. 

106 *datasets : `FileDataset`, variadic 

107 Per-dataset information to be exported. `FileDataset.formatter` 

108 attributes should be strings, not `Formatter` instances or classes. 

109 """ 

110 raise NotImplementedError() 

111 

112 @abstractmethod 

113 def saveDatasetAssociations( 

114 self, collection: str, collectionType: CollectionType, associations: Iterable[DatasetAssociation] 

115 ) -> None: 

116 """Export the dataset-collection associations for a single collection. 

117 

118 Parameters 

119 ---------- 

120 collection : `str` 

121 The name of the collection. 

122 collectionType : `CollectionType` 

123 The type of the collection; either `CollectionType.TAGGED` or 

124 `CollectionType.CALIBRATION` (as other collection types are 

125 exported in other ways). 

126 associations : `~collections.abc.Iterable` [ `DatasetAssociation` ] 

127 Structs representing an association between this collection and 

128 this dataset. 

129 """ 

130 raise NotImplementedError() 

131 

132 @abstractmethod 

133 def finish(self) -> None: 

134 """Complete the export process.""" 

135 raise NotImplementedError() 

136 

137 

138class RepoImportBackend(ABC): 

139 """An abstract interface for data repository import implementations. 

140 

141 Import backends are expected to be constructed with a description of 

142 the objects that need to be imported (from, e.g., a file written by the 

143 corresponding export backend), along with a `Registry`. 

144 """ 

145 

146 @abstractmethod 

147 def register(self) -> None: 

148 """Register all runs and dataset types associated with the backend with 

149 the `Registry` the backend was constructed with. 

150 

151 These operations cannot be performed inside transactions, unlike those 

152 performed by `load`, and must in general be performed before `load`. 

153 """ 

154 

155 @abstractmethod 

156 def load( 

157 self, 

158 datastore: Datastore | None, 

159 *, 

160 directory: ResourcePathExpression | None = None, 

161 transfer: str | None = None, 

162 skip_dimensions: set | None = None, 

163 ) -> None: 

164 """Import information associated with the backend into the given 

165 registry and datastore. 

166 

167 This must be run after `register`, and may be performed inside a 

168 transaction. 

169 

170 Parameters 

171 ---------- 

172 datastore : `Datastore` 

173 Datastore to import into. If `None`, datasets will only be 

174 inserted into the `Registry` (primarily intended for tests). 

175 directory : `~lsst.resources.ResourcePathExpression`, optional 

176 Directory all dataset paths are relative to. 

177 transfer : `str`, optional 

178 Transfer mode forwarded to `Datastore.ingest`. 

179 skip_dimensions : `set`, optional 

180 Dimensions that should be skipped and not imported. This can 

181 be useful when importing into a registry that already knows 

182 about a specific instrument. 

183 """ 

184 raise NotImplementedError()