Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# This file is part of ap_verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24import os 

25import pathlib 

26import stat 

27 

28import lsst.daf.persistence as dafPersist 

29 

30 

31class Workspace: 

32 """A directory used by ``ap_verify`` to handle data. 

33 

34 Any object of this class represents a working directory containing 

35 (possibly empty) subdirectories for repositories. At present, constructing 

36 a Workspace does not *initialize* its repositories; for compatibility 

37 reasons, this is best deferred to individual tasks. 

38 

39 Parameters 

40 ---------- 

41 location : `str` 

42 The location on disk where the workspace will be set up. Will be 

43 created if it does not already exist. 

44 

45 Raises 

46 ------ 

47 EnvironmentError 

48 Raised if ``location`` is not readable or not writeable 

49 """ 

50 

51 def __init__(self, location): 

52 self._location = location 

53 

54 mode = stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH # a+r, u+rwx 

55 kwargs = {"parents": True, "exist_ok": True, "mode": mode} 

56 pathlib.Path(self._location).mkdir(**kwargs) 

57 pathlib.Path(self.configDir).mkdir(**kwargs) 

58 pathlib.Path(self.dataRepo).mkdir(**kwargs) 

59 pathlib.Path(self.calibRepo).mkdir(**kwargs) 

60 pathlib.Path(self.templateRepo).mkdir(**kwargs) 

61 pathlib.Path(self.outputRepo).mkdir(**kwargs) 

62 

63 # Lazy evaluation to optimize workButler and analysisButler 

64 self._workButler = None 

65 self._analysisButler = None 

66 

67 @property 

68 def workDir(self): 

69 """The location of the workspace as a whole (`str`, read-only). 

70 """ 

71 return self._location 

72 

73 @property 

74 def configDir(self): 

75 """The location of a directory containing custom Task config files for 

76 use with the data (`str`, read-only). 

77 """ 

78 return os.path.join(self._location, 'config') 

79 

80 @property 

81 def dataRepo(self): 

82 """The URI to a Butler repo for science data (`str`, read-only). 

83 """ 

84 return os.path.join(self._location, 'ingested') 

85 

86 @property 

87 def calibRepo(self): 

88 """The URI to a Butler repo for calibration data (`str`, read-only). 

89 """ 

90 return os.path.join(self._location, 'calibingested') 

91 

92 @property 

93 def templateRepo(self): 

94 """The URI to a Butler repo for precomputed templates (`str`, read-only). 

95 """ 

96 return self.dataRepo 

97 

98 @property 

99 def outputRepo(self): 

100 """The URI to a Butler repo for AP pipeline products (`str`, read-only). 

101 """ 

102 return os.path.join(self._location, 'output') 

103 

104 @property 

105 def dbLocation(self): 

106 """The default location of the source association database to be 

107 created or updated by the pipeline (`str`, read-only). 

108 

109 Shall be a filename to a database file suitable 

110 for the sqlite backend of `Apdb`. 

111 """ 

112 return os.path.join(self._location, 'association.db') 

113 

114 @property 

115 def workButler(self): 

116 """A Butler that can produce pipeline inputs and outputs 

117 (`lsst.daf.persistence.Butler`, read-only). 

118 """ 

119 if self._workButler is None: 

120 self._workButler = self._makeButler() 

121 return self._workButler 

122 

123 def _makeButler(self): 

124 """Create a butler for accessing the entire workspace. 

125 

126 Returns 

127 ------- 

128 butler : `lsst.daf.persistence.Butler` 

129 A butler accepting `dataRepo`, `calibRepo`, and `templateRepo` as 

130 inputs, and `outputRepo` as an output. 

131 

132 Notes 

133 ----- 

134 Assumes all `*Repo` properties have been initialized. 

135 """ 

136 # common arguments for butler elements 

137 mapperArgs = {"calibRoot": os.path.abspath(self.calibRepo)} 

138 

139 inputs = [{"root": self.dataRepo, "mapperArgs": mapperArgs}] 

140 outputs = [{"root": self.outputRepo, "mode": "rw", "mapperArgs": mapperArgs}] 

141 

142 if not os.path.samefile(self.dataRepo, self.templateRepo): 

143 inputs.append({'root': self.templateRepo, 'mode': 'r', 'mapperArgs': mapperArgs}) 

144 

145 return dafPersist.Butler(inputs=inputs, outputs=outputs) 

146 

147 @property 

148 def analysisButler(self): 

149 """A Butler that can read pipeline outputs (`lsst.daf.persistence.Butler`, read-only). 

150 """ 

151 if self._analysisButler is None: 

152 self._analysisButler = dafPersist.Butler(inputs={"root": self.outputRepo, "mode": "r"}) 

153 return self._analysisButler