Coverage for tests/test_butlerSubset.py: 21%

140 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-06-11 02:48 -0700

1# 

2# LSST Data Management System 

3# Copyright 2008, 2009, 2010 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23import shutil 

24import unittest 

25 

26import os 

27import pickle 

28import tempfile 

29import lsst.daf.persistence as dafPersist 

30import lsst.utils.tests 

31from cameraMapper import CameraMapper 

32 

33# Define the root of the tests relative to this file 

34ROOT = os.path.abspath(os.path.dirname(__file__)) 

35 

36 

37def setup_module(module): 

38 lsst.utils.tests.init() 

39 

40 

41class Registry: 

42 

43 def __init__(self, dictList): 

44 self.dictList = dictList 

45 

46 def query(self, datasetType, format, dataId): 

47 result = set() 

48 for d in self.dictList: 

49 where = True 

50 for k in dataId.keys(): 

51 if k not in d: 

52 raise RuntimeError("%s not in %s" % (k, repr(d))) 

53 if d[k] != dataId[k]: 

54 where = False 

55 break 

56 if where: 

57 values = [] 

58 for k in format: 

59 values.append(d[k]) 

60 result.add(tuple(values)) 

61 return result 

62 

63 

64class ImgMapper(CameraMapper): 

65 

66 def __init__(self, root, **kwargs): 

67 CameraMapper.__init__(self, root, **kwargs) 

68 self.root = os.path.join(ROOT, root) 

69 self.registry = Registry([ 

70 dict(visit=123456, raft="1,1", sensor="2,2", amp="0,0", 

71 snap=0, skyTile=5), 

72 dict(visit=123456, raft="1,1", sensor="2,2", amp="0,0", 

73 snap=1, skyTile=5), 

74 dict(visit=123456, raft="1,1", sensor="2,2", amp="0,1", 

75 snap=0, skyTile=5), 

76 dict(visit=123456, raft="1,1", sensor="2,2", amp="1,0", 

77 snap=1, skyTile=5), 

78 dict(visit=123456, raft="1,1", sensor="2,2", amp="1,1", 

79 snap=0, skyTile=5), 

80 

81 dict(visit=123456, raft="1,2", sensor="2,1", amp="0,0", 

82 snap=1, skyTile=6), 

83 dict(visit=123456, raft="1,2", sensor="2,2", amp="0,0", 

84 snap=0, skyTile=6), 

85 

86 dict(visit=654321, raft="1,3", sensor="1,1", amp="0,0", 

87 snap=1, skyTile=6), 

88 dict(visit=654321, raft="1,3", sensor="1,2", amp="0,0", 

89 snap=0, skyTile=6) 

90 ]) 

91 

92 

93class ButlerSubsetTestCase(unittest.TestCase): 

94 """A test case for the subset capability of the data butler.""" 

95 

96 # create some random alias typenames 

97 calexpTypeName = "@goBears" 

98 rawTypeName = "@veggies" 

99 

100 def setUp(self): 

101 self.tmpRoot = tempfile.mkdtemp() 

102 

103 def tearDown(self): 

104 shutil.rmtree(self.tmpRoot, ignore_errors=True) 

105 

106 @staticmethod 

107 def registerAliases(butler): 

108 butler.defineAlias(ButlerSubsetTestCase.calexpTypeName, "calexp") 

109 butler.defineAlias(ButlerSubsetTestCase.rawTypeName, "raw") 

110 return butler 

111 

112 def testSingleIteration(self): 

113 butler = dafPersist.Butler( 

114 outputs={'mode': 'rw', 'root': self.tmpRoot, 'mapper': ImgMapper}) 

115 

116 ButlerSubsetTestCase.registerAliases(butler) 

117 

118 inputList = ["calexp_v123456_R1,2_S2,1.pickle", 

119 "calexp_v123456_R1,2_S2,2.pickle", 

120 "calexp_v654321_R1,3_S1,1.pickle", 

121 "calexp_v654321_R1,3_S1,2.pickle"] 

122 for fileName in inputList: 

123 with open(os.path.join(self.tmpRoot, fileName), "wb") as f: 

124 pickle.dump(inputList, f) 

125 

126 subset = butler.subset(self.calexpTypeName, skyTile=6) 

127 # all sensors overlapping that skyTile 

128 self.assertEqual(len(subset), 4) 

129 for iterator in subset: 

130 # calexp is by sensor, so get the data directly 

131 self.assertEqual(iterator.dataId["skyTile"], 6) 

132 if iterator.dataId["raft"] == "1,2": 

133 self.assertIn(iterator.dataId["sensor"], ["2,1", "2,2"]) 

134 self.assertEqual(iterator.dataId["visit"], 123456) 

135 elif iterator.dataId["raft"] == "1,3": 

136 self.assertIn(iterator.dataId["sensor"], ["1,1", "1,2"]) 

137 self.assertEqual(iterator.dataId["visit"], 654321) 

138 else: 

139 self.assertIn(iterator.dataId["raft"], ["1,2", "1,3"]) 

140 image = iterator.get(self.calexpTypeName, immediate=False) # succeeds since deferred 

141 self.assertEqual(type(image), dafPersist.readProxy.ReadProxy) 

142 image = iterator.get(self.calexpTypeName, immediate=True) # real test 

143 self.assertEqual(type(image), list) 

144 self.assertEqual(image, inputList) 

145 

146 # Tests for new getUri 

147 imageUri = iterator.getUri(self.calexpTypeName) 

148 fileName = 'calexp_v%(visit)d_R%(raft)s_S%(sensor)s.pickle' % iterator.dataId 

149 self.assertEqual(imageUri, os.path.join(self.tmpRoot, fileName)) 

150 self.assertEqual(os.path.isfile(imageUri), True) 

151 

152 for fileName in inputList: 

153 os.unlink(os.path.join(self.tmpRoot, fileName)) 

154 

155 def testNonexistentValue(self): 

156 butler = dafPersist.Butler( 

157 outputs={'mode': 'rw', 'root': self.tmpRoot, 'mapper': ImgMapper}) 

158 ButlerSubsetTestCase.registerAliases(butler) 

159 subset = butler.subset(self.calexpTypeName, skyTile=2349023905239) 

160 self.assertEqual(len(subset), 0) 

161 

162 def testInvalidValue(self): 

163 butler = dafPersist.Butler( 

164 outputs={'mode': 'rw', 'root': self.tmpRoot, 'mapper': ImgMapper}) 

165 ButlerSubsetTestCase.registerAliases(butler) 

166 subset = butler.subset(self.calexpTypeName, skyTile="foo") 

167 self.assertEqual(len(subset), 0) 

168 subset = butler.subset(self.calexpTypeName, visit=123456, sensor="1;2") 

169 self.assertEqual(len(subset), 0) 

170 

171 def testDoubleIteration(self): 

172 # create a bunch of files for testing 

173 inputList = ["flat_R1,1_S2,2_C0,0_E000.pickle", 

174 "flat_R1,1_S2,2_C0,0_E001.pickle", 

175 "flat_R1,1_S2,2_C0,1_E000.pickle", 

176 "flat_R1,1_S2,2_C1,0_E001.pickle", 

177 "flat_R1,1_S2,2_C1,1_E000.pickle", 

178 "flat_R1,2_S2,1_C0,0_E001.pickle", 

179 "flat_R1,2_S2,2_C0,0_E000.pickle"] 

180 for fileName in inputList: 

181 with open(os.path.join(self.tmpRoot, fileName), "wb") as f: 

182 pickle.dump(inputList, f) 

183 

184 butler = dafPersist.Butler( 

185 outputs={'mode': 'rw', 'root': self.tmpRoot, 'mapper': ImgMapper}) 

186 

187 ButlerSubsetTestCase.registerAliases(butler) 

188 

189 subset = butler.subset(self.rawTypeName, visit=123456) 

190 # Note: default level = "sensor" 

191 n = len(subset) 

192 self.assertEqual(n, 3) 

193 for iterator in subset: 

194 # this is a sensor iterator, but raw data is by amplifier, so 

195 # iterate over amplifiers 

196 self.assertEqual(set(iterator.subLevels()), set(["snap", "amp"])) 

197 if iterator.dataId["raft"] == "1,1": 

198 self.assertEqual(len(iterator.subItems()), 5) 

199 else: 

200 self.assertEqual(iterator.dataId["raft"], "1,2") 

201 self.assertEqual(len(iterator.subItems()), 1) 

202 for ampIterator in iterator.subItems(): # default level = "amp" 

203 if iterator.dataId["raft"] == "1,1": 

204 self.assertEqual(iterator.dataId["sensor"], "2,2") 

205 self.assertIn(ampIterator.dataId["amp"], ["0,0", "0,1", 

206 "1,0", "1,1"]) 

207 self.assertIn(ampIterator.dataId["snap"], [0, 1]) 

208 else: 

209 self.assertIn(iterator.dataId["sensor"], ["2,1", "2,2"]) 

210 # equivalent to butler.get(self.rawTypeName, ampIterator) 

211 self.assertTrue(ampIterator.datasetExists("flat")) 

212 flat = ampIterator.get("flat") 

213 self.assertEqual(flat, inputList) 

214 flat = ampIterator.get("flat", immediate=True) 

215 self.assertEqual(flat, inputList) 

216 # ...perform ISR, assemble and calibrate the CCD, then persist 

217 calexp = flat 

218 iterator.put(calexp, self.calexpTypeName) 

219 

220 for fileName in inputList: 

221 os.unlink(os.path.join(self.tmpRoot, fileName)) 

222 ref = butler.dataRef(self.rawTypeName, 

223 visit=123456, raft="1,1", sensor="2,2", amp="0,1", snap=0) 

224 self.assertFalse(ref.datasetExists("flat")) 

225 

226 for fileName in ["calexp_v123456_R1,1_S2,2.pickle", 

227 "calexp_v123456_R1,2_S2,1.pickle", 

228 "calexp_v123456_R1,2_S2,2.pickle"]: 

229 self.assertTrue(os.path.exists(os.path.join(self.tmpRoot, fileName))) 

230 os.unlink(os.path.join(self.tmpRoot, fileName)) 

231 

232 def testCompleteDataRef(self): 

233 butler = dafPersist.Butler( 

234 outputs={'mode': 'rw', 'root': self.tmpRoot, 'mapper': ImgMapper}) 

235 ButlerSubsetTestCase.registerAliases(butler) 

236 

237 # Test by using junk data 

238 tuples = butler.queryMetadata(self.rawTypeName, "amp", 

239 dict(visit=314159, raft="ab", sensor="cd", amp="ef", snap=9)) 

240 self.assertEqual(len(tuples), 0) 

241 subset = butler.subset(self.rawTypeName, "amp", visit=314159, raft="ab", sensor="cd") 

242 self.assertEqual(len(subset), 0) 

243 

244 # But allow user to fully specify dataId, even if it doesn't exist 

245 subset = butler.subset(self.rawTypeName, "sensor", visit=314159, raft="ab", sensor="cd") 

246 self.assertEqual(len(subset), 1) 

247 ref = butler.dataRef(self.rawTypeName, visit=314159, raft="ab", sensor="cd", amp="ef", snap=9) 

248 self.assertFalse(ref.datasetExists(self.rawTypeName)) 

249 

250 

251class MemoryTester(lsst.utils.tests.MemoryTestCase): 

252 pass 

253 

254 

255if __name__ == '__main__': 255 ↛ 256line 255 didn't jump to line 256, because the condition on line 255 was never true

256 lsst.utils.tests.init() 

257 unittest.main()