Coverage for python/lsst/daf/butler/transfers/_interfaces.py: 100%

26 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-17 02:30 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["RepoExportBackend", "RepoImportBackend", "RepoTransferFormatConfig"] 

25 

26from abc import ABC, abstractmethod 

27from typing import TYPE_CHECKING, Iterable, Optional, Set 

28 

29from ..core import ( 

30 ConfigSubset, 

31 DatasetAssociation, 

32 DatasetIdGenEnum, 

33 DatasetType, 

34 Datastore, 

35 DimensionElement, 

36 DimensionRecord, 

37 FileDataset, 

38) 

39from ..registry import CollectionType 

40from ..registry.interfaces import CollectionRecord 

41 

42if TYPE_CHECKING: 

43 from lsst.resources import ResourcePathExpression 

44 

45 

46class RepoTransferFormatConfig(ConfigSubset): 

47 """The section of butler configuration that associates repo import/export 

48 backends with file formats. 

49 """ 

50 

51 component = "repo_transfer_formats" 

52 defaultConfigFile = "repo_transfer_formats.yaml" 

53 

54 

55class RepoExportBackend(ABC): 

56 """An abstract interface for data repository export implementations. 

57 

58 Methods are guaranteed to be called in ways that reflect foreign key 

59 dependencies. 

60 """ 

61 

62 @abstractmethod 

63 def saveDimensionData(self, element: DimensionElement, *data: DimensionRecord) -> None: 

64 """Export one or more dimension element records. 

65 

66 Parameters 

67 ---------- 

68 element : `DimensionElement` 

69 The `DimensionElement` whose elements are being exported. 

70 data : `DimensionRecord` (variadic) 

71 One or more records to export. 

72 """ 

73 raise NotImplementedError() 

74 

75 @abstractmethod 

76 def saveCollection(self, record: CollectionRecord, doc: Optional[str]) -> None: 

77 """Export a collection. 

78 

79 This only exports the collection's own state, not its associations with 

80 datasets. 

81 

82 Parameters 

83 ---------- 

84 record: `CollectionRecord` 

85 Object representing the collection to export. 

86 doc : `str` or `None` 

87 Documentation string for the collection. 

88 """ 

89 raise NotImplementedError() 

90 

91 @abstractmethod 

92 def saveDatasets(self, datasetType: DatasetType, run: str, *datasets: FileDataset) -> None: 

93 """Export one or more datasets, including their associated DatasetType 

94 and run information (but not including associated dimension 

95 information). 

96 

97 Parameters 

98 ---------- 

99 datasetType : `DatasetType` 

100 Type of all datasets being exported with this call. 

101 run : `str` 

102 Run associated with all datasets being exported with this call. 

103 datasets : `FileDataset`, variadic 

104 Per-dataset information to be exported. `FileDataset.formatter` 

105 attributes should be strings, not `Formatter` instances or classes. 

106 """ 

107 raise NotImplementedError() 

108 

109 @abstractmethod 

110 def saveDatasetAssociations( 

111 self, collection: str, collectionType: CollectionType, associations: Iterable[DatasetAssociation] 

112 ) -> None: 

113 """Export the dataset-collection associations for a single collection. 

114 

115 Parameters 

116 ---------- 

117 collection : `str` 

118 The name of the collection. 

119 collectionType : `CollectionType` 

120 The type of the collection; either `CollectionType.TAGGED` or 

121 `CollectionType.CALIBRATION` (as other collection types are 

122 exported in other ways). 

123 associations : `Iterable` [ `DatasetAssociation` ] 

124 Structs representing an association between this collection and 

125 this dataset. 

126 """ 

127 raise NotImplementedError() 

128 

129 @abstractmethod 

130 def finish(self) -> None: 

131 """Complete the export process.""" 

132 raise NotImplementedError() 

133 

134 

135class RepoImportBackend(ABC): 

136 """An abstract interface for data repository import implementations. 

137 

138 Import backends are expected to be constructed with a description of 

139 the objects that need to be imported (from, e.g., a file written by the 

140 corresponding export backend), along with a `Registry`. 

141 """ 

142 

143 @abstractmethod 

144 def register(self) -> None: 

145 """Register all runs and dataset types associated with the backend with 

146 the `Registry` the backend was constructed with. 

147 

148 These operations cannot be performed inside transactions, unlike those 

149 performed by `load`, and must in general be performed before `load`. 

150 """ 

151 

152 @abstractmethod 

153 def load( 

154 self, 

155 datastore: Optional[Datastore], 

156 *, 

157 directory: ResourcePathExpression | None = None, 

158 transfer: Optional[str] = None, 

159 skip_dimensions: Optional[Set] = None, 

160 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE, 

161 reuseIds: bool = False, 

162 ) -> None: 

163 """Import information associated with the backend into the given 

164 registry and datastore. 

165 

166 This must be run after `register`, and may be performed inside a 

167 transaction. 

168 

169 Parameters 

170 ---------- 

171 datastore : `Datastore` 

172 Datastore to import into. If `None`, datasets will only be 

173 inserted into the `Registry` (primarily intended for tests). 

174 directory : `~lsst.resources.ResourcePathExpression`, optional 

175 Directory all dataset paths are relative to. 

176 transfer : `str`, optional 

177 Transfer mode forwarded to `Datastore.ingest`. 

178 skip_dimensions : `set`, optional 

179 Dimensions that should be skipped and not imported. This can 

180 be useful when importing into a registry that already knows 

181 about a specific instrument. 

182 idGenerationMode : `DatasetIdGenEnum`, optional 

183 Specifies option for generating dataset IDs when IDs are not 

184 provided or their type does not match backend type. By default 

185 unique IDs are generated for each inserted dataset. 

186 reuseIds : `bool`, optional 

187 If `True` then forces re-use of imported dataset IDs for integer 

188 IDs which are normally generated as auto-incremented. This option 

189 has no effect on the use of globally-unique IDs which are always 

190 re-used (or generated if integer IDs are being imported). 

191 """ 

192 raise NotImplementedError()