Coverage for tests/test_reposInButler.py: 36%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2016 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
23import pickle
24import os
25import shutil
26import unittest
27import tempfile
29import yaml
31import lsst.utils.tests
32import lsst.daf.persistence as dp
33from lsst.daf.persistence import Policy
35# Define the root of the tests relative to this file
36ROOT = os.path.abspath(os.path.dirname(__file__))
39def setup_module(module):
40 lsst.utils.tests.init()
43class PosixPickleStringHanlder:
45 @staticmethod
46 def get(butlerLocation):
47 if butlerLocation.storageName != "PickleStorage":
48 raise TypeError("PosixStoragePickleMapper only supports PickleStorage")
49 location = butlerLocation.getLocations()[0] # should never be more than 1 location
50 with open(location, 'rb') as f:
51 ret = pickle.load(f)
52 return ret
54 @staticmethod
55 def put(obj, butlerLocation):
56 if butlerLocation.storageName != "PickleStorage":
57 raise TypeError("PosixStoragePickleMapper only supports PickleStorage")
58 for location in butlerLocation.getLocations():
59 with open(location, 'wb') as f:
60 pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)
63class MapperTestCfg(Policy, yaml.YAMLObject):
64 yaml_tag = u"!MapperTestCfg"
66 def __init__(self, cls, root):
67 super().__init__({'root': root, 'cls': cls})
70class MapperTest(dp.Mapper):
72 @classmethod
73 def cfg(cls, root=None):
74 return MapperTestCfg(cls=cls, root=root)
76 def __init__(self, cfg):
77 super().__init__()
78 # self.root = cfg['root']
79 self.storage = cfg['storage']
80 self.cfg = cfg
82 def __repr__(self):
83 return 'MapperTest(cfg=%s)' % self.cfg
85 def map_str(self, dataId, write):
86 template = "strfile_%(strId)s.pickle"
87 path = template % dataId
88 if not write:
89 if not self.storage.exists(path):
90 return None
91 location = self.storage.locationWithRoot(path)
92 return dp.ButlerLocation(pythonType=PosixPickleStringHanlder, cppType=None,
93 storageName='PickleStorage', locationList=location,
94 dataId=dataId, mapper=self, storage=self.storage)
97class ReposInButler(unittest.TestCase):
99 def setUp(self):
100 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix='ReposInButler-')
102 def tearDown(self):
103 if os.path.exists(self.testDir):
104 shutil.rmtree(self.testDir)
106 # disable this test until we work more on repo of repos; starting with DM-6227
107 @unittest.expectedFailure
108 def test(self):
109 repoMapperPolicy = {
110 'repositories': {
111 'cfg': {
112 'template': 'repos/repo_v%(version)s/repoCfg.yaml',
113 'python': 'lsst.daf.persistence.RepositoryCfg',
114 'storage': 'YamlStorage'
115 }
116 }
117 }
119 # create a cfg of a repository for our repositories
120 storageCfg = dp.PosixStorage.cfg(root=self.testDir)
121 accessCfg = dp.Access.cfg(storageCfg=storageCfg)
122 self.assertIsNotNone(accessCfg)
123 mapperCfg = dp.RepositoryMapper.cfg(policy=repoMapperPolicy)
124 self.assertIsNotNone(mapperCfg)
125 # Note that right now a repo is either input OR output, there is no input-output repo, this design
126 # is result of butler design conversations. Right now, if a user wants to write to and then read from
127 # a repo, a repo can have a parent repo with the same access (and mapper) parameters as itself.
128 repoOfRepoCfg = dp.Repository.cfg(mode='rw',
129 storageCfg=dp.PosixStorage.cfg(root=self.testDir),
130 mapper=dp.RepositoryMapper.cfg(policy=repoMapperPolicy))
132 repoButler = dp.Butler(outputs=repoOfRepoCfg)
133 # create a cfg of a repository we'd like to use. Note that we don't create the root of the cfg.
134 # this will get populated by the repoOfRepos template.
135 repoCfg = dp.Repository.cfg(mode='rw', storageCfg=dp.PosixStorage.cfg(), mapper=MapperTest.cfg())
136 # and put that config into the repoOfRep
137 repoButler.put(repoCfg, 'cfg', dataId={'version': 123})
139 # get the cfg back out of the butler. This will return a cfg with the root location populated.
140 # i.e. repoCfg['accessCfg.storageCfg.root'] is populated.
141 repoCfg = repoButler.get('cfg', dataId={'version': 123}, immediate=True)
142 butler = dp.Butler(outputs=repoCfg)
144 obj = 'abc'
145 butler.put(obj, 'str', {'strId': 'a'})
146 reloadedObj = butler.get('str', {'strId': 'a'})
147 self.assertEqual(obj, reloadedObj)
148 self.assertIsNot(obj, reloadedObj) # reloaded object should be a new instance.
150 # explicitly release some objects; these names will be reused momentarily.
151 del butler, repoCfg, obj, reloadedObj
153 # Create another repository, and put it in the repo of repos, with a new version number.
154 repoCfg = dp.Repository.cfg(mode='rw', storageCfg=dp.PosixStorage.cfg(), mapper=MapperTest.cfg())
155 repoButler.put(repoCfg, 'cfg', dataId={'version': 124})
156 repoCfg = repoButler.get('cfg', dataId={'version': 124}, immediate=True)
157 butler = dp.Butler(outputs=repoCfg)
158 # create an object that is slightly different than the object in repo version 123, and give it the
159 # same dataId as the object in repo 123. Put it, and get it to verify.
160 obj = 'abcd'
161 butler.put(obj, 'str', {'strId': 'a'})
162 reloadedObj = butler.get('str', {'strId': 'a'})
163 self.assertEqual(obj, reloadedObj)
164 self.assertIsNot(obj, reloadedObj)
166 # release the objects for repo version 124
167 del butler, repoCfg, obj, reloadedObj
169 # from the repo butler, get the cfgs for both repo versions, create butlers from each of them, get
170 # the objects, and verify correct values.
171 repo123Cfg = repoButler.get('cfg', dataId={'version': 123}, immediate=True)
172 repo124Cfg = repoButler.get('cfg', dataId={'version': 124}, immediate=True)
173 butler123 = dp.Butler(inputs=repo123Cfg)
174 butler124 = dp.Butler(inputs=repo124Cfg)
175 obj123 = butler123.get('str', {'strId': 'a'})
176 obj124 = butler124.get('str', {'strId': 'a'})
177 self.assertEqual(obj123, 'abc')
178 self.assertEqual(obj124, 'abcd')
181class MemoryTester(lsst.utils.tests.MemoryTestCase):
182 pass
185if __name__ == '__main__': 185 ↛ 186line 185 didn't jump to line 186, because the condition on line 185 was never true
186 lsst.utils.tests.init()
187 unittest.main()