Coverage for tests/test_butlerSubset.py: 17%
140 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-05 02:50 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-01-05 02:50 -0800
1#
2# LSST Data Management System
3# Copyright 2008, 2009, 2010 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
23import shutil
24import unittest
26import os
27import pickle
28import tempfile
29import lsst.daf.persistence as dafPersist
30import lsst.utils.tests
31from cameraMapper import CameraMapper
33# Define the root of the tests relative to this file
34ROOT = os.path.abspath(os.path.dirname(__file__))
37def setup_module(module):
38 lsst.utils.tests.init()
41class Registry:
43 def __init__(self, dictList):
44 self.dictList = dictList
46 def query(self, datasetType, format, dataId):
47 result = set()
48 for d in self.dictList:
49 where = True
50 for k in dataId.keys():
51 if k not in d:
52 raise RuntimeError("%s not in %s" % (k, repr(d)))
53 if d[k] != dataId[k]:
54 where = False
55 break
56 if where:
57 values = []
58 for k in format:
59 values.append(d[k])
60 result.add(tuple(values))
61 return result
64class ImgMapper(CameraMapper):
66 def __init__(self, root, **kwargs):
67 CameraMapper.__init__(self, root, **kwargs)
68 self.root = os.path.join(ROOT, root)
69 self.registry = Registry([
70 dict(visit=123456, raft="1,1", sensor="2,2", amp="0,0",
71 snap=0, skyTile=5),
72 dict(visit=123456, raft="1,1", sensor="2,2", amp="0,0",
73 snap=1, skyTile=5),
74 dict(visit=123456, raft="1,1", sensor="2,2", amp="0,1",
75 snap=0, skyTile=5),
76 dict(visit=123456, raft="1,1", sensor="2,2", amp="1,0",
77 snap=1, skyTile=5),
78 dict(visit=123456, raft="1,1", sensor="2,2", amp="1,1",
79 snap=0, skyTile=5),
81 dict(visit=123456, raft="1,2", sensor="2,1", amp="0,0",
82 snap=1, skyTile=6),
83 dict(visit=123456, raft="1,2", sensor="2,2", amp="0,0",
84 snap=0, skyTile=6),
86 dict(visit=654321, raft="1,3", sensor="1,1", amp="0,0",
87 snap=1, skyTile=6),
88 dict(visit=654321, raft="1,3", sensor="1,2", amp="0,0",
89 snap=0, skyTile=6)
90 ])
93class ButlerSubsetTestCase(unittest.TestCase):
94 """A test case for the subset capability of the data butler."""
96 # create some random alias typenames
97 calexpTypeName = "@goBears"
98 rawTypeName = "@veggies"
100 def setUp(self):
101 self.tmpRoot = tempfile.mkdtemp()
103 def tearDown(self):
104 shutil.rmtree(self.tmpRoot, ignore_errors=True)
106 @staticmethod
107 def registerAliases(butler):
108 butler.defineAlias(ButlerSubsetTestCase.calexpTypeName, "calexp")
109 butler.defineAlias(ButlerSubsetTestCase.rawTypeName, "raw")
110 return butler
112 def testSingleIteration(self):
113 butler = dafPersist.Butler(
114 outputs={'mode': 'rw', 'root': self.tmpRoot, 'mapper': ImgMapper})
116 ButlerSubsetTestCase.registerAliases(butler)
118 inputList = ["calexp_v123456_R1,2_S2,1.pickle",
119 "calexp_v123456_R1,2_S2,2.pickle",
120 "calexp_v654321_R1,3_S1,1.pickle",
121 "calexp_v654321_R1,3_S1,2.pickle"]
122 for fileName in inputList:
123 with open(os.path.join(self.tmpRoot, fileName), "wb") as f:
124 pickle.dump(inputList, f)
126 subset = butler.subset(self.calexpTypeName, skyTile=6)
127 # all sensors overlapping that skyTile
128 self.assertEqual(len(subset), 4)
129 for iterator in subset:
130 # calexp is by sensor, so get the data directly
131 self.assertEqual(iterator.dataId["skyTile"], 6)
132 if iterator.dataId["raft"] == "1,2":
133 self.assertIn(iterator.dataId["sensor"], ["2,1", "2,2"])
134 self.assertEqual(iterator.dataId["visit"], 123456)
135 elif iterator.dataId["raft"] == "1,3":
136 self.assertIn(iterator.dataId["sensor"], ["1,1", "1,2"])
137 self.assertEqual(iterator.dataId["visit"], 654321)
138 else:
139 self.assertIn(iterator.dataId["raft"], ["1,2", "1,3"])
140 image = iterator.get(self.calexpTypeName, immediate=False) # succeeds since deferred
141 self.assertEqual(type(image), dafPersist.readProxy.ReadProxy)
142 image = iterator.get(self.calexpTypeName, immediate=True) # real test
143 self.assertEqual(type(image), list)
144 self.assertEqual(image, inputList)
146 # Tests for new getUri
147 imageUri = iterator.getUri(self.calexpTypeName)
148 fileName = 'calexp_v%(visit)d_R%(raft)s_S%(sensor)s.pickle' % iterator.dataId
149 self.assertEqual(imageUri, os.path.join(self.tmpRoot, fileName))
150 self.assertEqual(os.path.isfile(imageUri), True)
152 for fileName in inputList:
153 os.unlink(os.path.join(self.tmpRoot, fileName))
155 def testNonexistentValue(self):
156 butler = dafPersist.Butler(
157 outputs={'mode': 'rw', 'root': self.tmpRoot, 'mapper': ImgMapper})
158 ButlerSubsetTestCase.registerAliases(butler)
159 subset = butler.subset(self.calexpTypeName, skyTile=2349023905239)
160 self.assertEqual(len(subset), 0)
162 def testInvalidValue(self):
163 butler = dafPersist.Butler(
164 outputs={'mode': 'rw', 'root': self.tmpRoot, 'mapper': ImgMapper})
165 ButlerSubsetTestCase.registerAliases(butler)
166 subset = butler.subset(self.calexpTypeName, skyTile="foo")
167 self.assertEqual(len(subset), 0)
168 subset = butler.subset(self.calexpTypeName, visit=123456, sensor="1;2")
169 self.assertEqual(len(subset), 0)
171 def testDoubleIteration(self):
172 # create a bunch of files for testing
173 inputList = ["flat_R1,1_S2,2_C0,0_E000.pickle",
174 "flat_R1,1_S2,2_C0,0_E001.pickle",
175 "flat_R1,1_S2,2_C0,1_E000.pickle",
176 "flat_R1,1_S2,2_C1,0_E001.pickle",
177 "flat_R1,1_S2,2_C1,1_E000.pickle",
178 "flat_R1,2_S2,1_C0,0_E001.pickle",
179 "flat_R1,2_S2,2_C0,0_E000.pickle"]
180 for fileName in inputList:
181 with open(os.path.join(self.tmpRoot, fileName), "wb") as f:
182 pickle.dump(inputList, f)
184 butler = dafPersist.Butler(
185 outputs={'mode': 'rw', 'root': self.tmpRoot, 'mapper': ImgMapper})
187 ButlerSubsetTestCase.registerAliases(butler)
189 subset = butler.subset(self.rawTypeName, visit=123456)
190 # Note: default level = "sensor"
191 n = len(subset)
192 self.assertEqual(n, 3)
193 for iterator in subset:
194 # this is a sensor iterator, but raw data is by amplifier, so
195 # iterate over amplifiers
196 self.assertEqual(set(iterator.subLevels()), set(["snap", "amp"]))
197 if iterator.dataId["raft"] == "1,1":
198 self.assertEqual(len(iterator.subItems()), 5)
199 else:
200 self.assertEqual(iterator.dataId["raft"], "1,2")
201 self.assertEqual(len(iterator.subItems()), 1)
202 for ampIterator in iterator.subItems(): # default level = "amp"
203 if iterator.dataId["raft"] == "1,1":
204 self.assertEqual(iterator.dataId["sensor"], "2,2")
205 self.assertIn(ampIterator.dataId["amp"], ["0,0", "0,1",
206 "1,0", "1,1"])
207 self.assertIn(ampIterator.dataId["snap"], [0, 1])
208 else:
209 self.assertIn(iterator.dataId["sensor"], ["2,1", "2,2"])
210 # equivalent to butler.get(self.rawTypeName, ampIterator)
211 self.assertTrue(ampIterator.datasetExists("flat"))
212 flat = ampIterator.get("flat")
213 self.assertEqual(flat, inputList)
214 flat = ampIterator.get("flat", immediate=True)
215 self.assertEqual(flat, inputList)
216 # ...perform ISR, assemble and calibrate the CCD, then persist
217 calexp = flat
218 iterator.put(calexp, self.calexpTypeName)
220 for fileName in inputList:
221 os.unlink(os.path.join(self.tmpRoot, fileName))
222 ref = butler.dataRef(self.rawTypeName,
223 visit=123456, raft="1,1", sensor="2,2", amp="0,1", snap=0)
224 self.assertFalse(ref.datasetExists("flat"))
226 for fileName in ["calexp_v123456_R1,1_S2,2.pickle",
227 "calexp_v123456_R1,2_S2,1.pickle",
228 "calexp_v123456_R1,2_S2,2.pickle"]:
229 self.assertTrue(os.path.exists(os.path.join(self.tmpRoot, fileName)))
230 os.unlink(os.path.join(self.tmpRoot, fileName))
232 def testCompleteDataRef(self):
233 butler = dafPersist.Butler(
234 outputs={'mode': 'rw', 'root': self.tmpRoot, 'mapper': ImgMapper})
235 ButlerSubsetTestCase.registerAliases(butler)
237 # Test by using junk data
238 tuples = butler.queryMetadata(self.rawTypeName, "amp",
239 dict(visit=314159, raft="ab", sensor="cd", amp="ef", snap=9))
240 self.assertEqual(len(tuples), 0)
241 subset = butler.subset(self.rawTypeName, "amp", visit=314159, raft="ab", sensor="cd")
242 self.assertEqual(len(subset), 0)
244 # But allow user to fully specify dataId, even if it doesn't exist
245 subset = butler.subset(self.rawTypeName, "sensor", visit=314159, raft="ab", sensor="cd")
246 self.assertEqual(len(subset), 1)
247 ref = butler.dataRef(self.rawTypeName, visit=314159, raft="ab", sensor="cd", amp="ef", snap=9)
248 self.assertFalse(ref.datasetExists(self.rawTypeName))
251class MemoryTester(lsst.utils.tests.MemoryTestCase):
252 pass
255if __name__ == '__main__': 255 ↛ 256line 255 didn't jump to line 256, because the condition on line 255 was never true
256 lsst.utils.tests.init()
257 unittest.main()