Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# This file is part of ap_verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24import os 

25import pathlib 

26import stat 

27 

28import lsst.daf.persistence as dafPersist 

29 

30 

31class Workspace: 

32 """A directory used by ``ap_verify`` to handle data. 

33 

34 Any object of this class represents a working directory containing 

35 (possibly empty) subdirectories for repositories. At present, constructing 

36 a Workspace does not *initialize* its repositories; for compatibility 

37 reasons, this is best deferred to individual tasks. 

38 

39 Parameters 

40 ---------- 

41 location : `str` 

42 The location on disk where the workspace will be set up. Will be 

43 created if it does not already exist. 

44 

45 Raises 

46 ------ 

47 EnvironmentError 

48 Raised if ``location`` is not readable or not writeable 

49 """ 

50 

51 def __init__(self, location): 

52 # Properties must be `str` for backwards compatibility 

53 self._location = str(pathlib.Path(location).resolve()) 

54 

55 mode = stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH # a+r, u+rwx 

56 kwargs = {"parents": True, "exist_ok": True, "mode": mode} 

57 pathlib.Path(self._location).mkdir(**kwargs) 

58 pathlib.Path(self.configDir).mkdir(**kwargs) 

59 pathlib.Path(self.dataRepo).mkdir(**kwargs) 

60 pathlib.Path(self.calibRepo).mkdir(**kwargs) 

61 pathlib.Path(self.templateRepo).mkdir(**kwargs) 

62 pathlib.Path(self.outputRepo).mkdir(**kwargs) 

63 

64 # Lazy evaluation to optimize workButler and analysisButler 

65 self._workButler = None 

66 self._analysisButler = None 

67 

68 @property 

69 def workDir(self): 

70 """The absolute location of the workspace as a whole 

71 (`str`, read-only). 

72 """ 

73 return self._location 

74 

75 @property 

76 def configDir(self): 

77 """The absolute location of a directory containing custom Task config 

78 files for use with the data (`str`, read-only). 

79 """ 

80 return os.path.join(self._location, 'config') 

81 

82 @property 

83 def dataRepo(self): 

84 """The absolute path/URI to a Butler repo for science data 

85 (`str`, read-only). 

86 """ 

87 return os.path.join(self._location, 'ingested') 

88 

89 @property 

90 def calibRepo(self): 

91 """The absolute path/URI to a Butler repo for calibration data 

92 (`str`, read-only). 

93 """ 

94 return os.path.join(self._location, 'calibingested') 

95 

96 @property 

97 def templateRepo(self): 

98 """The absolute path/URI to a Butler repo for precomputed templates 

99 (`str`, read-only). 

100 """ 

101 return self.dataRepo 

102 

103 @property 

104 def outputRepo(self): 

105 """The absolute path/URI to a Butler repo for AP pipeline products 

106 (`str`, read-only). 

107 """ 

108 return os.path.join(self._location, 'output') 

109 

110 @property 

111 def dbLocation(self): 

112 """The default absolute location of the source association database to 

113 be created or updated by the pipeline (`str`, read-only). 

114 

115 Shall be a filename to a database file suitable 

116 for the sqlite backend of `Apdb`. 

117 """ 

118 return os.path.join(self._location, 'association.db') 

119 

120 @property 

121 def workButler(self): 

122 """A Butler that can produce pipeline inputs and outputs 

123 (`lsst.daf.persistence.Butler`, read-only). 

124 """ 

125 if self._workButler is None: 

126 self._workButler = self._makeButler() 

127 return self._workButler 

128 

129 def _makeButler(self): 

130 """Create a butler for accessing the entire workspace. 

131 

132 Returns 

133 ------- 

134 butler : `lsst.daf.persistence.Butler` 

135 A butler accepting `dataRepo`, `calibRepo`, and `templateRepo` as 

136 inputs, and `outputRepo` as an output. 

137 

138 Notes 

139 ----- 

140 Assumes all `*Repo` properties have been initialized. 

141 """ 

142 # common arguments for butler elements 

143 mapperArgs = {"calibRoot": os.path.abspath(self.calibRepo)} 

144 

145 inputs = [{"root": self.dataRepo, "mapperArgs": mapperArgs}] 

146 outputs = [{"root": self.outputRepo, "mode": "rw", "mapperArgs": mapperArgs}] 

147 

148 if not os.path.samefile(self.dataRepo, self.templateRepo): 

149 inputs.append({'root': self.templateRepo, 'mode': 'r', 'mapperArgs': mapperArgs}) 

150 

151 return dafPersist.Butler(inputs=inputs, outputs=outputs) 

152 

153 @property 

154 def analysisButler(self): 

155 """A Butler that can read pipeline outputs (`lsst.daf.persistence.Butler`, read-only). 

156 """ 

157 if self._analysisButler is None: 

158 self._analysisButler = dafPersist.Butler(inputs={"root": self.outputRepo, "mode": "r"}) 

159 return self._analysisButler