Coverage for python/lsst/daf/butler/transfers/_interfaces.py: 100%

26 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-07 02:10 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["RepoExportBackend", "RepoImportBackend", "RepoTransferFormatConfig"] 

25 

26from abc import ABC, abstractmethod 

27from typing import TYPE_CHECKING, Iterable, Optional, Set 

28 

29from ..core import ( 

30 ConfigSubset, 

31 DatasetAssociation, 

32 DatasetType, 

33 Datastore, 

34 DimensionElement, 

35 DimensionRecord, 

36 FileDataset, 

37) 

38from ..registry import CollectionType 

39from ..registry.interfaces import CollectionRecord 

40 

41if TYPE_CHECKING: 

42 from lsst.resources import ResourcePathExpression 

43 

44 

45class RepoTransferFormatConfig(ConfigSubset): 

46 """The section of butler configuration that associates repo import/export 

47 backends with file formats. 

48 """ 

49 

50 component = "repo_transfer_formats" 

51 defaultConfigFile = "repo_transfer_formats.yaml" 

52 

53 

54class RepoExportBackend(ABC): 

55 """An abstract interface for data repository export implementations. 

56 

57 Methods are guaranteed to be called in ways that reflect foreign key 

58 dependencies. 

59 """ 

60 

61 @abstractmethod 

62 def saveDimensionData(self, element: DimensionElement, *data: DimensionRecord) -> None: 

63 """Export one or more dimension element records. 

64 

65 Parameters 

66 ---------- 

67 element : `DimensionElement` 

68 The `DimensionElement` whose elements are being exported. 

69 data : `DimensionRecord` (variadic) 

70 One or more records to export. 

71 """ 

72 raise NotImplementedError() 

73 

74 @abstractmethod 

75 def saveCollection(self, record: CollectionRecord, doc: Optional[str]) -> None: 

76 """Export a collection. 

77 

78 This only exports the collection's own state, not its associations with 

79 datasets. 

80 

81 Parameters 

82 ---------- 

83 record: `CollectionRecord` 

84 Object representing the collection to export. 

85 doc : `str` or `None` 

86 Documentation string for the collection. 

87 """ 

88 raise NotImplementedError() 

89 

90 @abstractmethod 

91 def saveDatasets(self, datasetType: DatasetType, run: str, *datasets: FileDataset) -> None: 

92 """Export one or more datasets, including their associated DatasetType 

93 and run information (but not including associated dimension 

94 information). 

95 

96 Parameters 

97 ---------- 

98 datasetType : `DatasetType` 

99 Type of all datasets being exported with this call. 

100 run : `str` 

101 Run associated with all datasets being exported with this call. 

102 datasets : `FileDataset`, variadic 

103 Per-dataset information to be exported. `FileDataset.formatter` 

104 attributes should be strings, not `Formatter` instances or classes. 

105 """ 

106 raise NotImplementedError() 

107 

108 @abstractmethod 

109 def saveDatasetAssociations( 

110 self, collection: str, collectionType: CollectionType, associations: Iterable[DatasetAssociation] 

111 ) -> None: 

112 """Export the dataset-collection associations for a single collection. 

113 

114 Parameters 

115 ---------- 

116 collection : `str` 

117 The name of the collection. 

118 collectionType : `CollectionType` 

119 The type of the collection; either `CollectionType.TAGGED` or 

120 `CollectionType.CALIBRATION` (as other collection types are 

121 exported in other ways). 

122 associations : `Iterable` [ `DatasetAssociation` ] 

123 Structs representing an association between this collection and 

124 this dataset. 

125 """ 

126 raise NotImplementedError() 

127 

128 @abstractmethod 

129 def finish(self) -> None: 

130 """Complete the export process.""" 

131 raise NotImplementedError() 

132 

133 

134class RepoImportBackend(ABC): 

135 """An abstract interface for data repository import implementations. 

136 

137 Import backends are expected to be constructed with a description of 

138 the objects that need to be imported (from, e.g., a file written by the 

139 corresponding export backend), along with a `Registry`. 

140 """ 

141 

142 @abstractmethod 

143 def register(self) -> None: 

144 """Register all runs and dataset types associated with the backend with 

145 the `Registry` the backend was constructed with. 

146 

147 These operations cannot be performed inside transactions, unlike those 

148 performed by `load`, and must in general be performed before `load`. 

149 """ 

150 

151 @abstractmethod 

152 def load( 

153 self, 

154 datastore: Optional[Datastore], 

155 *, 

156 directory: ResourcePathExpression | None = None, 

157 transfer: Optional[str] = None, 

158 skip_dimensions: Optional[Set] = None, 

159 ) -> None: 

160 """Import information associated with the backend into the given 

161 registry and datastore. 

162 

163 This must be run after `register`, and may be performed inside a 

164 transaction. 

165 

166 Parameters 

167 ---------- 

168 datastore : `Datastore` 

169 Datastore to import into. If `None`, datasets will only be 

170 inserted into the `Registry` (primarily intended for tests). 

171 directory : `~lsst.resources.ResourcePathExpression`, optional 

172 Directory all dataset paths are relative to. 

173 transfer : `str`, optional 

174 Transfer mode forwarded to `Datastore.ingest`. 

175 skip_dimensions : `set`, optional 

176 Dimensions that should be skipped and not imported. This can 

177 be useful when importing into a registry that already knows 

178 about a specific instrument. 

179 """ 

180 raise NotImplementedError()