Coverage for tests/test_reposInButler.py: 30%

99 statements  

« prev     ^ index     » next       coverage.py v7.2.1, created at 2023-03-12 01:29 -0800

1# 

2# LSST Data Management System 

3# Copyright 2016 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23import pickle 

24import os 

25import shutil 

26import unittest 

27import tempfile 

28 

29import yaml 

30 

31import lsst.utils.tests 

32import lsst.daf.persistence as dp 

33from lsst.daf.persistence import Policy 

34 

35# Define the root of the tests relative to this file 

36ROOT = os.path.abspath(os.path.dirname(__file__)) 

37 

38 

39def setup_module(module): 

40 lsst.utils.tests.init() 

41 

42 

43class PosixPickleStringHanlder: 

44 

45 @staticmethod 

46 def get(butlerLocation): 

47 if butlerLocation.storageName != "PickleStorage": 

48 raise TypeError("PosixStoragePickleMapper only supports PickleStorage") 

49 location = butlerLocation.getLocations()[0] # should never be more than 1 location 

50 with open(location, 'rb') as f: 

51 ret = pickle.load(f) 

52 return ret 

53 

54 @staticmethod 

55 def put(obj, butlerLocation): 

56 if butlerLocation.storageName != "PickleStorage": 

57 raise TypeError("PosixStoragePickleMapper only supports PickleStorage") 

58 for location in butlerLocation.getLocations(): 

59 with open(location, 'wb') as f: 

60 pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL) 

61 

62 

63class MapperTestCfg(Policy, yaml.YAMLObject): 

64 yaml_tag = u"!MapperTestCfg" 

65 

66 def __init__(self, cls, root): 

67 super().__init__({'root': root, 'cls': cls}) 

68 

69 

70class MapperTest(dp.Mapper): 

71 

72 @classmethod 

73 def cfg(cls, root=None): 

74 return MapperTestCfg(cls=cls, root=root) 

75 

76 def __init__(self, cfg): 

77 super().__init__() 

78 # self.root = cfg['root'] 

79 self.storage = cfg['storage'] 

80 self.cfg = cfg 

81 

82 def __repr__(self): 

83 return 'MapperTest(cfg=%s)' % self.cfg 

84 

85 def map_str(self, dataId, write): 

86 template = "strfile_%(strId)s.pickle" 

87 path = template % dataId 

88 if not write: 

89 if not self.storage.exists(path): 

90 return None 

91 location = self.storage.locationWithRoot(path) 

92 return dp.ButlerLocation(pythonType=PosixPickleStringHanlder, cppType=None, 

93 storageName='PickleStorage', locationList=location, 

94 dataId=dataId, mapper=self, storage=self.storage) 

95 

96 

97class ReposInButler(unittest.TestCase): 

98 

99 def setUp(self): 

100 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix='ReposInButler-') 

101 

102 def tearDown(self): 

103 if os.path.exists(self.testDir): 

104 shutil.rmtree(self.testDir) 

105 

106 # disable this test until we work more on repo of repos; starting with DM-6227 

107 @unittest.expectedFailure 

108 def test(self): 

109 repoMapperPolicy = { 

110 'repositories': { 

111 'cfg': { 

112 'template': 'repos/repo_v%(version)s/repoCfg.yaml', 

113 'python': 'lsst.daf.persistence.RepositoryCfg', 

114 'storage': 'YamlStorage' 

115 } 

116 } 

117 } 

118 

119 # create a cfg of a repository for our repositories 

120 storageCfg = dp.PosixStorage.cfg(root=self.testDir) 

121 accessCfg = dp.Access.cfg(storageCfg=storageCfg) 

122 self.assertIsNotNone(accessCfg) 

123 mapperCfg = dp.RepositoryMapper.cfg(policy=repoMapperPolicy) 

124 self.assertIsNotNone(mapperCfg) 

125 # Note that right now a repo is either input OR output, there is no input-output repo, this design 

126 # is result of butler design conversations. Right now, if a user wants to write to and then read from 

127 # a repo, a repo can have a parent repo with the same access (and mapper) parameters as itself. 

128 repoOfRepoCfg = dp.Repository.cfg(mode='rw', 

129 storageCfg=dp.PosixStorage.cfg(root=self.testDir), 

130 mapper=dp.RepositoryMapper.cfg(policy=repoMapperPolicy)) 

131 

132 repoButler = dp.Butler(outputs=repoOfRepoCfg) 

133 # create a cfg of a repository we'd like to use. Note that we don't create the root of the cfg. 

134 # this will get populated by the repoOfRepos template. 

135 repoCfg = dp.Repository.cfg(mode='rw', storageCfg=dp.PosixStorage.cfg(), mapper=MapperTest.cfg()) 

136 # and put that config into the repoOfRep 

137 repoButler.put(repoCfg, 'cfg', dataId={'version': 123}) 

138 

139 # get the cfg back out of the butler. This will return a cfg with the root location populated. 

140 # i.e. repoCfg['accessCfg.storageCfg.root'] is populated. 

141 repoCfg = repoButler.get('cfg', dataId={'version': 123}, immediate=True) 

142 butler = dp.Butler(outputs=repoCfg) 

143 

144 obj = 'abc' 

145 butler.put(obj, 'str', {'strId': 'a'}) 

146 reloadedObj = butler.get('str', {'strId': 'a'}) 

147 self.assertEqual(obj, reloadedObj) 

148 self.assertIsNot(obj, reloadedObj) # reloaded object should be a new instance. 

149 

150 # explicitly release some objects; these names will be reused momentarily. 

151 del butler, repoCfg, obj, reloadedObj 

152 

153 # Create another repository, and put it in the repo of repos, with a new version number. 

154 repoCfg = dp.Repository.cfg(mode='rw', storageCfg=dp.PosixStorage.cfg(), mapper=MapperTest.cfg()) 

155 repoButler.put(repoCfg, 'cfg', dataId={'version': 124}) 

156 repoCfg = repoButler.get('cfg', dataId={'version': 124}, immediate=True) 

157 butler = dp.Butler(outputs=repoCfg) 

158 # create an object that is slightly different than the object in repo version 123, and give it the 

159 # same dataId as the object in repo 123. Put it, and get it to verify. 

160 obj = 'abcd' 

161 butler.put(obj, 'str', {'strId': 'a'}) 

162 reloadedObj = butler.get('str', {'strId': 'a'}) 

163 self.assertEqual(obj, reloadedObj) 

164 self.assertIsNot(obj, reloadedObj) 

165 

166 # release the objects for repo version 124 

167 del butler, repoCfg, obj, reloadedObj 

168 

169 # from the repo butler, get the cfgs for both repo versions, create butlers from each of them, get 

170 # the objects, and verify correct values. 

171 repo123Cfg = repoButler.get('cfg', dataId={'version': 123}, immediate=True) 

172 repo124Cfg = repoButler.get('cfg', dataId={'version': 124}, immediate=True) 

173 butler123 = dp.Butler(inputs=repo123Cfg) 

174 butler124 = dp.Butler(inputs=repo124Cfg) 

175 obj123 = butler123.get('str', {'strId': 'a'}) 

176 obj124 = butler124.get('str', {'strId': 'a'}) 

177 self.assertEqual(obj123, 'abc') 

178 self.assertEqual(obj124, 'abcd') 

179 

180 

181class MemoryTester(lsst.utils.tests.MemoryTestCase): 

182 pass 

183 

184 

185if __name__ == '__main__': 185 ↛ 186line 185 didn't jump to line 186, because the condition on line 185 was never true

186 lsst.utils.tests.init() 

187 unittest.main()