Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ["RepoExportBackend", "RepoImportBackend", "RepoTransferFormatConfig"] 

25 

26from abc import ABC, abstractmethod 

27from typing import ( 

28 Optional, 

29 Set, 

30) 

31 

32from ..core import ( 

33 ConfigSubset, 

34 DatasetType, 

35 Datastore, 

36 DimensionElement, 

37 DimensionRecord, 

38 FileDataset, 

39) 

40 

41 

42class RepoTransferFormatConfig(ConfigSubset): 

43 """The section of butler configuration that associates repo import/export 

44 backends with file formats. 

45 """ 

46 component = "repo_transfer_formats" 

47 defaultConfigFile = "repo_transfer_formats.yaml" 

48 

49 

50class RepoExportBackend(ABC): 

51 """An abstract interface for data repository export implementations. 

52 """ 

53 

54 @abstractmethod 

55 def saveDimensionData(self, element: DimensionElement, *data: DimensionRecord) -> None: 

56 """Export one or more dimension element records. 

57 

58 Parameters 

59 ---------- 

60 element : `DimensionElement` 

61 The `DimensionElement` whose elements are being exported. 

62 data : `DimensionRecord` (variadic) 

63 One or more records to export. 

64 """ 

65 raise NotImplementedError() 

66 

67 @abstractmethod 

68 def saveDatasets(self, datasetType: DatasetType, run: str, *datasets: FileDataset) -> None: 

69 """Export one or more datasets, including their associated DatasetType 

70 and run information (but not including associated dimension 

71 information). 

72 

73 Parameters 

74 ---------- 

75 datasetType : `DatasetType` 

76 Type of all datasets being exported with this call. 

77 run : `str` 

78 Run associated with all datasets being exported with this call. 

79 datasets : `FileDataset`, variadic 

80 Per-dataset information to be exported. `FileDataset.formatter` 

81 attributes should be strings, not `Formatter` instances or classes. 

82 """ 

83 raise NotImplementedError() 

84 

85 @abstractmethod 

86 def finish(self) -> None: 

87 """Complete the export process. 

88 """ 

89 raise NotImplementedError() 

90 

91 

92class RepoImportBackend(ABC): 

93 """An abstract interface for data repository import implementations. 

94 

95 Import backends are expected to be constructed with a description of 

96 the objects that need to be imported (from, e.g., a file written by the 

97 corresponding export backend), along with a `Registry`. 

98 """ 

99 

100 @abstractmethod 

101 def register(self) -> None: 

102 """Register all runs and dataset types associated with the backend with 

103 the `Registry` the backend was constructed with. 

104 

105 These operations cannot be performed inside transactions, unlike those 

106 performed by `load`, and must in general be performed before `load`. 

107 """ 

108 

109 @abstractmethod 

110 def load(self, datastore: Optional[Datastore], *, 

111 directory: Optional[str] = None, transfer: Optional[str] = None, 

112 skip_dimensions: Optional[Set] = None) -> None: 

113 """Import information associated with the backend into the given 

114 registry and datastore. 

115 

116 This must be run after `register`, and may be performed inside a 

117 transaction. 

118 

119 Parameters 

120 ---------- 

121 datastore : `Datastore` 

122 Datastore to import into. If `None`, datasets will only be 

123 inserted into the `Registry` (primarily intended for tests). 

124 directory : `str`, optional 

125 File all dataset paths are relative to. 

126 transfer : `str`, optional 

127 Transfer mode forwarded to `Datastore.ingest`. 

128 skip_dimensions : `set`, optional 

129 Dimensions that should be skipped and not imported. This can 

130 be useful when importing into a registry that already knows 

131 about a specific instrument. 

132 """ 

133 raise NotImplementedError()