Coverage for python/lsst/daf/butler/transfers/_interfaces.py: 84%

32 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-09-27 01:59 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["RepoExportBackend", "RepoImportBackend", "RepoTransferFormatConfig"] 

25 

26from abc import ABC, abstractmethod 

27from typing import Iterable, Optional, Set 

28 

29from ..core import ( 

30 ConfigSubset, 

31 DatasetAssociation, 

32 DatasetType, 

33 Datastore, 

34 DimensionElement, 

35 DimensionRecord, 

36 FileDataset, 

37) 

38from ..registry import CollectionType 

39from ..registry.interfaces import CollectionRecord, DatasetIdGenEnum 

40 

41 

42class RepoTransferFormatConfig(ConfigSubset): 

43 """The section of butler configuration that associates repo import/export 

44 backends with file formats. 

45 """ 

46 

47 component = "repo_transfer_formats" 

48 defaultConfigFile = "repo_transfer_formats.yaml" 

49 

50 

51class RepoExportBackend(ABC): 

52 """An abstract interface for data repository export implementations. 

53 

54 Methods are guaranteed to be called in ways that reflect foreign key 

55 dependencies. 

56 """ 

57 

58 @abstractmethod 

59 def saveDimensionData(self, element: DimensionElement, *data: DimensionRecord) -> None: 

60 """Export one or more dimension element records. 

61 

62 Parameters 

63 ---------- 

64 element : `DimensionElement` 

65 The `DimensionElement` whose elements are being exported. 

66 data : `DimensionRecord` (variadic) 

67 One or more records to export. 

68 """ 

69 raise NotImplementedError() 

70 

71 @abstractmethod 

72 def saveCollection(self, record: CollectionRecord, doc: Optional[str]) -> None: 

73 """Export a collection. 

74 

75 This only exports the collection's own state, not its associations with 

76 datasets. 

77 

78 Parameters 

79 ---------- 

80 record: `CollectionRecord` 

81 Object representing the collection to export. 

82 doc : `str` or `None` 

83 Documentation string for the collection. 

84 """ 

85 raise NotImplementedError() 

86 

87 @abstractmethod 

88 def saveDatasets(self, datasetType: DatasetType, run: str, *datasets: FileDataset) -> None: 

89 """Export one or more datasets, including their associated DatasetType 

90 and run information (but not including associated dimension 

91 information). 

92 

93 Parameters 

94 ---------- 

95 datasetType : `DatasetType` 

96 Type of all datasets being exported with this call. 

97 run : `str` 

98 Run associated with all datasets being exported with this call. 

99 datasets : `FileDataset`, variadic 

100 Per-dataset information to be exported. `FileDataset.formatter` 

101 attributes should be strings, not `Formatter` instances or classes. 

102 """ 

103 raise NotImplementedError() 

104 

105 @abstractmethod 

106 def saveDatasetAssociations( 

107 self, collection: str, collectionType: CollectionType, associations: Iterable[DatasetAssociation] 

108 ) -> None: 

109 """Export the dataset-collection associations for a single collection. 

110 

111 Parameters 

112 ---------- 

113 collection : `str` 

114 The name of the collection. 

115 collectionType : `CollectionType` 

116 The type of the collection; either `CollectionType.TAGGED` or 

117 `CollectionType.CALIBRATION` (as other collection types are 

118 exported in other ways). 

119 associations : `Iterable` [ `DatasetAssociation` ] 

120 Structs representing an association between this collection and 

121 this dataset. 

122 """ 

123 raise NotImplementedError() 

124 

125 @abstractmethod 

126 def finish(self) -> None: 

127 """Complete the export process.""" 

128 raise NotImplementedError() 

129 

130 

131class RepoImportBackend(ABC): 

132 """An abstract interface for data repository import implementations. 

133 

134 Import backends are expected to be constructed with a description of 

135 the objects that need to be imported (from, e.g., a file written by the 

136 corresponding export backend), along with a `Registry`. 

137 """ 

138 

139 @abstractmethod 

140 def register(self) -> None: 

141 """Register all runs and dataset types associated with the backend with 

142 the `Registry` the backend was constructed with. 

143 

144 These operations cannot be performed inside transactions, unlike those 

145 performed by `load`, and must in general be performed before `load`. 

146 """ 

147 

148 @abstractmethod 

149 def load( 

150 self, 

151 datastore: Optional[Datastore], 

152 *, 

153 directory: Optional[str] = None, 

154 transfer: Optional[str] = None, 

155 skip_dimensions: Optional[Set] = None, 

156 idGenerationMode: DatasetIdGenEnum = DatasetIdGenEnum.UNIQUE, 

157 reuseIds: bool = False, 

158 ) -> None: 

159 """Import information associated with the backend into the given 

160 registry and datastore. 

161 

162 This must be run after `register`, and may be performed inside a 

163 transaction. 

164 

165 Parameters 

166 ---------- 

167 datastore : `Datastore` 

168 Datastore to import into. If `None`, datasets will only be 

169 inserted into the `Registry` (primarily intended for tests). 

170 directory : `str`, optional 

171 File all dataset paths are relative to. 

172 transfer : `str`, optional 

173 Transfer mode forwarded to `Datastore.ingest`. 

174 skip_dimensions : `set`, optional 

175 Dimensions that should be skipped and not imported. This can 

176 be useful when importing into a registry that already knows 

177 about a specific instrument. 

178 idGenerationMode : `DatasetIdGenEnum`, optional 

179 Specifies option for generating dataset IDs when IDs are not 

180 provided or their type does not match backend type. By default 

181 unique IDs are generated for each inserted dataset. 

182 reuseIds : `bool`, optional 

183 If `True` then forces re-use of imported dataset IDs for integer 

184 IDs which are normally generated as auto-incremented. This option 

185 has no effect on the use of globally-unique IDs which are always 

186 re-used (or generated if integer IDs are being imported). 

187 """ 

188 raise NotImplementedError()