Coverage for tests/test_composite.py : 17%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import os
23import shutil
24import unittest
25import tempfile
27import lsst.utils.tests
28import lsst.daf.persistence as dafPersist
29import lsst.daf.persistence.test as dpTest
31ROOT = os.path.abspath(os.path.dirname(__file__))
34class TestCompositeTestCase(unittest.TestCase):
35 """A test case for composite object i/o."""
37 def setUp(self):
38 self.testData = tempfile.mkdtemp(dir=ROOT, prefix='TestCompositeTestCase-')
39 self.firstRepoPath = os.path.join(self.testData, 'repo1')
40 self.objA = dpTest.TestObject("abc")
41 self.objB = dpTest.TestObject("def")
42 self.policy = dafPersist.Policy(
43 {'camera': 'lsst.afw.cameraGeom.Camera',
44 'datasets': {
45 'basicObject1': {
46 'python': 'lsst.daf.persistence.test.TestObject',
47 'template': 'basic/id%(id)s.pickle',
48 'storage': 'PickleStorage'},
49 'basicObject2': {
50 'python': 'lsst.daf.persistence.test.TestObject',
51 'template': 'basic/name%(name)s.pickle',
52 'storage': 'PickleStorage'},
53 'basicPair': {
54 'python': 'lsst.daf.persistence.test.TestObjectPair',
55 'composite': {
56 'a': {'datasetType': 'basicObject1'},
57 'b': {'datasetType': 'basicObject2'}
58 },
59 'assembler':
60 'lsst.daf.persistence.test.TestObjectPair.assembler',
61 'disassembler':
62 'lsst.daf.persistence.test.TestObjectPair.disassembler'},
63 'stdTestType': {
64 'python': 'lsst.daf.persistence.test.TestObjectPair',
65 'composite': {
66 'a': {'datasetType': 'basicObject1'},
67 'b': {'datasetType': 'basicObject2'}
68 }
69 },
70 'bypassTestType': {
71 'python': 'lsst.daf.persistence.test.TestObjectPair',
72 'composite': {
73 'a': {'datasetType': 'basicObject1'},
74 'b': {'datasetType': 'basicObject2'}
75 }
76 }
77 }})
79 repoArgs = dafPersist.RepositoryArgs(root=self.firstRepoPath, policy=self.policy,
80 mapper='lsst.obs.base.test.CompositeMapper')
81 butler = dafPersist.Butler(outputs=repoArgs)
82 butler.put(self.objA, 'basicObject1', dataId={'id': 'foo'})
83 butler.put(self.objB, 'basicObject2', dataId={'name': 'bar'})
84 del butler
85 del repoArgs
87 def tearDown(self):
88 if os.path.exists(self.testData):
89 shutil.rmtree(self.testData)
91 def testType3GetAndPut(self):
92 """1. Verify that a composite can be loaded and that its components are the same as when the type1
93 components are loaded individually (verifies correct lookup in this case).
94 2. Verify that when the individual components are put and when the composite is put (which
95 disassembles into individual components) that the objects that are written are the same.
96 """
97 secondRepoPath = os.path.join(self.testData, 'repo2')
98 repoArgs = dafPersist.RepositoryArgs(root=secondRepoPath, policy=self.policy)
99 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
100 verificationButler = dafPersist.Butler(inputs=secondRepoPath)
101 objABPair = butler.get('basicPair', dataId={'id': 'foo', 'name': 'bar'})
103 self.assertEqual(self.objA, objABPair.objA)
104 self.assertEqual(self.objB, objABPair.objB)
106 # For now also test that the type 1 and type 3 components are not the same object.
107 # These objects are not yet in the butler cache because they have not been gotten yet (they have only
108 # only been put)
109 self.assertIsNot(self.objA, objABPair.objA)
110 self.assertIsNot(self.objB, objABPair.objB)
112 # Now, get a type 1 copy of objA and objB, and they should be equivalent to the instance in the
113 # composite.
114 objA = butler.get('basicObject1', dataId={'id': 'foo'}, immediate=True)
115 objB = butler.get('basicObject2', dataId={'name': 'bar'}, immediate=True)
116 self.assertEqual(objA, objABPair.objA)
117 self.assertEqual(objB, objABPair.objB)
119 butler.put(objABPair, 'basicPair', dataId={'id': 'foo', 'name': 'bar'})
120 verObjA = verificationButler.get('basicObject1', {'id': 'foo'})
121 self.assertEqual(verObjA, objABPair.objA)
122 verObjB = verificationButler.get('basicObject2', {'name': 'bar'})
123 self.assertEqual(verObjB, objABPair.objB)
125 def testDottedDatasetType(self):
126 """Verify that components of a composite can be loaded by dotted name in the form
127 DatasetType.componentName
128 """
129 thirdRepoPath = os.path.join(self.testData, 'repo3')
130 # child repositories do not look up in-repo policies. We need to fix that.
131 repoArgs = dafPersist.RepositoryArgs(root=thirdRepoPath, policy=self.policy)
132 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
133 verificationButler = dafPersist.Butler(inputs=thirdRepoPath)
134 componentObjA = butler.get('basicPair.a', dataId={'id': 'foo', 'name': 'bar'})
135 componentObjB = butler.get('basicPair.b', dataId={'id': 'foo', 'name': 'bar'})
136 self.assertEqual(self.objA, componentObjA)
137 self.assertEqual(self.objB, componentObjB)
138 butler.put(componentObjA, 'basicPair.a', dataId={'id': 'foo', 'name': 'bar'})
139 butler.put(componentObjB, 'basicPair.b', dataId={'id': 'foo', 'name': 'bar'})
140 verObjA = verificationButler.get('basicObject1', {'id': 'foo'})
141 self.assertEqual(verObjA, componentObjA)
142 verObjB = verificationButler.get('basicObject2', {'name': 'bar'})
143 self.assertEqual(verObjB, componentObjB)
145 def testDatasetExists(self):
146 """Verify that Butler.datasetExists returns true for a composite dataset whose components exist."""
147 butler = dafPersist.Butler(inputs=self.firstRepoPath)
148 self.assertTrue(butler.datasetExists('basicPair', dataId={'id': 'foo', 'name': 'bar'}))
150 def testDatasetDoesNotExist(self):
151 """Verify that Butler.datasetExists returns false for a composite dataset where some of the
152 components do not exist."""
153 repoPath = os.path.join(self.testData, 'repo')
154 repoArgs = dafPersist.RepositoryArgs(root=repoPath, policy=self.policy,
155 mapper='lsst.obs.base.test.CompositeMapper')
157 butler = dafPersist.Butler(outputs=repoArgs)
158 self.objA = dpTest.TestObject("abc")
159 butler.put(self.objA, 'basicObject1', dataId={'id': 'foo'})
160 self.assertFalse(butler.datasetExists('basicPair', dataId={'id': 'foo', 'name': 'bar'}))
162 def testStd(self):
163 """Verify that composite dataset types with a std_ function are passed to the std_ function after
164 being instantiated."""
165 secondRepoPath = os.path.join(self.testData, 'repo2')
166 repoArgs = dafPersist.RepositoryArgs(root=secondRepoPath, policy=self.policy)
167 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
168 objABPair = butler.get('stdTestType', dataId={'id': 'foo', 'name': 'bar'})
169 self.assertTrue(hasattr(objABPair, 'standardized'))
170 self.assertTrue(objABPair.standardized)
172 def testBypass(self):
173 """Verify that composite dataset types with a bypass_ function are passed to the bypass function after
174 being instantiated."""
175 secondRepoPath = os.path.join(self.testData, 'repo2')
176 repoArgs = dafPersist.RepositoryArgs(root=secondRepoPath, policy=self.policy)
177 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
178 bypassObject = butler.get('bypassTestType', dataId={'id': 'foo', 'name': 'bar'})
179 self.assertEqual(bypassObject, set(['id', 'name']))
182class TestGenericAssembler(unittest.TestCase):
183 """A test case for the generic assembler feature of composite datasets."""
185 def setUp(self):
186 self.testData = tempfile.mkdtemp(dir=ROOT, prefix='TestGenericAssembler-')
187 self.firstRepoPath = os.path.join(self.testData, 'repo1')
188 self.secondRepoPath = os.path.join(self.testData, 'repo2')
189 self.objA = dpTest.TestObject("abc")
190 self.objB = dpTest.TestObject("def")
191 self.policy = dafPersist.Policy(
192 {'camera': 'lsst.afw.cameraGeom.Camera',
193 'datasets': {
194 'basicObject1': {
195 'python': 'lsst.daf.persistence.test.TestObject',
196 'template': 'basic/id%(id)s.pickle',
197 'storage': 'PickleStorage'
198 },
199 'basicObject2': {
200 'python': 'lsst.daf.persistence.test.TestObject',
201 'template': 'basic/name%(name)s.pickle',
202 'storage': 'PickleStorage'
203 },
204 'basicPair': {
205 'python': 'lsst.daf.persistence.test.TestObjectPair',
206 'composite': {
207 'a': {'datasetType': 'basicObject1'},
208 'b': {'datasetType': 'basicObject2'}
209 },
210 # note, no assembler or disassembler specified here, will use
211 # setter names inferred by component name.
212 },
213 # "generic assembler default constructor pair"
214 'gaDefCtorPair': { # dataset defition that uses the default ctor
215 'python': 'lsst.daf.persistence.test.TestObjectPair',
216 'composite': {
217 # note that the component names are the same as the argument
218 # names in the TestObjectPair.__init__ func.
219 'objA': {'datasetType': 'basicObject1',
220 'getter': 'get_a'},
221 'objB': {'datasetType': 'basicObject2',
222 'getter': 'get_b'}
223 },
224 # note, no assembler or disassembler specified here.
225 },
226 # "generic assembler default "
227 'gaPairWithSetter': {
228 'python': 'lsst.daf.persistence.test.TestObjectPair',
229 'composite': {
230 # note that the component names do not match argument names
231 # in the TestObjectPair.__init__ func or the set functions
232 # in the python object.
233 'z': {'datasetType': 'basicObject1',
234 'setter': 'set_a',
235 'getter': 'get_a'
236 },
237 'x': {'datasetType': 'basicObject2',
238 'setter': 'set_b',
239 'getter': 'get_b'
240 }
241 }
242 },
243 # simple object where setter and getter is named with underscore
244 # separator
245 'underscoreSetter': {
246 'python': 'lsst.daf.persistence.test.TestObjectUnderscoreSetter',
247 'composite': {
248 'foo': {'datasetType': 'basicObject1'}
249 }
250 },
251 # simple object where setter and getter is named with camelcase
252 'camelCaseSetter': {
253 'python': 'lsst.daf.persistence.test.TestObjectCamelCaseSetter',
254 'composite': {
255 'foo': {'datasetType': 'basicObject1'}
256 }
257 }
258 }})
260 repoArgs = dafPersist.RepositoryArgs(root=self.firstRepoPath, policy=self.policy,
261 mapper='lsst.obs.base.test.CompositeMapper')
262 butler = dafPersist.Butler(outputs=repoArgs)
263 butler.put(self.objA, 'basicObject1', dataId={'id': 'foo'})
264 butler.put(self.objB, 'basicObject2', dataId={'name': 'bar'})
265 del butler
266 del repoArgs
268 def tearDown(self):
269 if os.path.exists(self.testData):
270 shutil.rmtree(self.testData)
272 def testConstructor(self):
273 """Test the case where the arguments to the default constructor match the component names and so the
274 default constructor can be used by the generic assembler to assemble the object
275 Uses getters named by the policy to disassemble the object.
276 """
277 repoArgs = dafPersist.RepositoryArgs(root=self.secondRepoPath, policy=self.policy)
278 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
279 verificationButler = dafPersist.Butler(inputs=self.secondRepoPath)
281 objABPair = butler.get('basicPair', dataId={'id': 'foo', 'name': 'bar'})
282 self.assertEqual(self.objA, objABPair.objA)
283 self.assertEqual(self.objB, objABPair.objB)
285 butler.put(objABPair, 'basicPair', dataId={'id': 'foo', 'name': 'bar'})
286 # comparing the output files directly works so long as the storage is posix:
288 # put the composite object and verify it disassembled into the right locations by loading the
289 # components directly
290 objA = verificationButler.get('basicObject1', dataId={'id': 'foo'})
291 self.assertEqual(objA, objABPair.objA)
292 objB = verificationButler.get('basicObject2', dataId={'name': 'bar'})
293 self.assertEqual(objB, objABPair.objB)
295 del objABPair
297 objABPair = butler.get('gaDefCtorPair', dataId={'id': 'foo', 'name': 'bar'})
298 self.assertEqual(self.objA, objABPair.objA)
299 self.assertEqual(self.objB, objABPair.objB)
300 self.assertTrue(objABPair.usedInitSetter)
301 self.assertFalse(objABPair.usedASetter)
302 self.assertFalse(objABPair.usedBSetter)
304 # put the composite object and verify it disassembled into the right locations by loading the
305 # components directly
306 butler.put(objABPair, 'gaDefCtorPair', dataId={'id': 'baz', 'name': 'qux'})
307 verObjA = verificationButler.get('basicObject1', dataId={'id': 'baz'})
308 self.assertEqual(objABPair.objA, verObjA)
309 verObjB = verificationButler.get('basicObject2', dataId={'name': 'qux'})
310 self.assertEqual(objABPair.objB, verObjB)
312 def testGenericAssemblerPolicySpecifiedSetterGetter(self):
313 """Test the case where the component names do not have anything to do with the setter/getter names
314 or the init function parameter names, and instead the component policy entry specifies the setter and
315 getter names.
316 """
317 repoArgs = dafPersist.RepositoryArgs(root=self.secondRepoPath, policy=self.policy)
318 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
319 objABPair = butler.get('gaPairWithSetter', dataId={'id': 'foo', 'name': 'bar'})
320 self.assertEqual(self.objA, objABPair.objA)
321 self.assertEqual(self.objB, objABPair.objB)
322 self.assertFalse(objABPair.usedInitSetter)
323 self.assertTrue(objABPair.usedASetter)
324 self.assertTrue(objABPair.usedBSetter)
326 butler.put(objABPair, 'gaPairWithSetter', dataId={'id': 'foo', 'name': 'bar'})
327 # comparing the output files directly works so long as the storage is posix:
329 verificationButler = dafPersist.Butler(inputs=self.secondRepoPath)
330 verObjA = verificationButler.get('basicObject1', dataId={'id': 'foo'})
331 verObjB = verificationButler.get('basicObject2', dataId={'name': 'bar'})
332 self.assertEqual(objABPair.objA, verObjA)
333 self.assertEqual(objABPair.objB, verObjB)
335 def testInferredNameUnderscoreSeparator(self):
336 """Test the case where the name of the setter & getter is inferred by the policy name by prepending
337 'set_' and get_
338 """
339 repoArgs = dafPersist.RepositoryArgs(root=self.secondRepoPath, policy=self.policy)
340 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
341 obj = butler.get('underscoreSetter', dataId={'id': 'foo'})
342 self.assertEqual(self.objA, obj.get_foo())
343 butler.put(obj, 'underscoreSetter', dataId={'id': 'foo'})
345 verificationButler = dafPersist.Butler(inputs=self.secondRepoPath)
346 componentObj = verificationButler.get('basicObject1', dataId={'id': 'foo'})
347 self.assertEqual(componentObj, obj.get_foo())
349 def testInferredNameCamelcase(self):
350 """Test the case where the name of the setter & getter is inferred by the policy name by prepending
351 'set' or 'get', to the capitalized component name. E.g. for component name 'foo' the setter and getter
352 will be named setFoo and getFoo.
353 """
354 repoArgs = dafPersist.RepositoryArgs(root=self.secondRepoPath, policy=self.policy)
355 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
356 obj = butler.get('camelCaseSetter', dataId={'id': 'foo'})
357 self.assertEqual(self.objA, obj.getFoo())
358 butler.put(obj, 'camelCaseSetter', dataId={'id': 'foo'})
360 verificationButler = dafPersist.Butler(inputs=self.secondRepoPath)
361 componentObj = verificationButler.get('basicObject1', dataId={'id': 'foo'})
362 self.assertEqual(componentObj, obj.getFoo())
365def subsetAssembler(dataId, componentInfo, cls):
366 obj = cls()
367 obj.set_a(componentInfo['a'].obj)
368 return obj
371class TestSubset(unittest.TestCase):
372 """A test case for composite object subset keyword."""
374 def setUp(self):
375 self.testData = tempfile.mkdtemp(dir=ROOT, prefix='TestSubset-')
376 self.firstRepoPath = os.path.join(self.testData, 'repo1')
377 self.objA1 = dpTest.TestObject("abc")
378 self.objA2 = dpTest.TestObject("ABC")
379 self.objB = dpTest.TestObject("def")
380 self.policy = dafPersist.Policy(
381 {'camera': 'lsst.afw.cameraGeom.Camera',
382 'datasets': {
383 'basicObject1': {
384 'python': 'lsst.daf.persistence.test.TestObject',
385 'template': 'basic/id%(id)s.pickle',
386 'storage': 'PickleStorage'},
387 'basicObject2': {
388 'python': 'lsst.daf.persistence.test.TestObject',
389 'template': 'basic/name%(name)s.pickle',
390 'storage': 'PickleStorage'},
391 'basicPair': {
392 'python': 'lsst.daf.persistence.test.TestObjectPair',
393 'composite': {
394 'a': {
395 'datasetType': 'basicObject1',
396 'subset': True
397 }
398 },
399 'assembler': subsetAssembler
400 },
401 }})
403 repoArgs = dafPersist.RepositoryArgs(root=self.firstRepoPath, policy=self.policy,
404 mapper='lsst.obs.base.test.CompositeMapper')
405 butler = dafPersist.Butler(outputs=repoArgs)
406 butler.put(self.objA1, 'basicObject1', dataId={'id': 'foo1'})
407 butler.put(self.objA2, 'basicObject1', dataId={'id': 'foo2'})
408 butler.put(self.objB, 'basicObject2', dataId={'name': 'bar'})
409 del butler
410 del repoArgs
412 def tearDown(self):
413 if os.path.exists(self.testData):
414 shutil.rmtree(self.testData)
416 def test(self):
417 """Verify that the generic assembler and disassembler work for objects that conform to the generic
418 set/get API.
419 """
420 secondRepoPath = os.path.join(self.testData, 'repo2')
421 repoArgs = dafPersist.RepositoryArgs(root=secondRepoPath, policy=self.policy)
422 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
423 # the name 'bar' will find the obj that was put as obj b. It expects to find n objects of dataset
424 # type basicObject1. Since we don't specify any dataId that relates to basicObject1 (its only dataId
425 # key is 'id'), it will return everything it finds according to its policy. In this case that should
426 # be self.objA1 and self.objA2 that we put above. They will be in a list at objABPair.objA.
427 objABPair = butler.get('basicPair', dataId={'name': 'bar'})
428 objABPair.objA.sort()
429 self.assertEqual(self.objA2, objABPair.objA[0])
430 self.assertEqual(self.objA1, objABPair.objA[1])
431 # subset is a get-only operation. To put, the dataId must be specified, so there's no put to test
432 # here.
435class TestInputOnly(unittest.TestCase):
436 """A test case for composite input keyword."""
438 def setUp(self):
439 self.testData = tempfile.mkdtemp(dir=ROOT, prefix='TestInputOnly-')
440 self.firstRepoPath = os.path.join(self.testData, 'repo1')
441 self.objA = dpTest.TestObject("abc")
442 self.objB = dpTest.TestObject("def")
443 self.policy = dafPersist.Policy(
444 {'camera': 'lsst.afw.cameraGeom.Camera',
445 'datasets': {
446 'basicObject1': {
447 'python': 'lsst.daf.persistence.test.TestObject',
448 'template': 'basic/id%(id)s.pickle',
449 'storage': 'PickleStorage'},
450 'basicObject2': {
451 'python': 'lsst.daf.persistence.test.TestObject',
452 'template': 'basic/name%(name)s.pickle',
453 'storage': 'PickleStorage'},
454 'basicPair': {
455 'python': 'lsst.daf.persistence.test.TestObjectPair',
456 'composite': {
457 'a': {
458 'datasetType': 'basicObject1'
459 },
460 'b': {
461 'datasetType': 'basicObject2',
462 'inputOnly': True
463 }
464 },
465 'assembler': 'lsst.daf.persistence.test.TestObjectPair.assembler',
466 'disassembler': 'lsst.daf.persistence.test.TestObjectPair.disassembler'
468 }
469 }})
471 repoArgs = dafPersist.RepositoryArgs(root=self.firstRepoPath,
472 mapper='lsst.obs.base.test.CompositeMapper',
473 policy=self.policy)
474 butler = dafPersist.Butler(outputs=repoArgs)
475 butler.put(self.objA, 'basicObject1', dataId={'id': 'foo'})
476 butler.put(self.objB, 'basicObject2', dataId={'name': 'bar'})
477 del butler
478 del repoArgs
480 def tearDown(self):
481 if os.path.exists(self.testData):
482 shutil.rmtree(self.testData)
484 def test(self):
485 """ Verify that when a type 3 dataset is put and one of its components is marked 'inputOnly' by the
486 policy that the inputOnly comonent is not written.
487 """
488 secondRepoPath = os.path.join(self.testData, 'repo2')
489 repoArgs = dafPersist.RepositoryArgs(root=secondRepoPath, policy=self.policy)
490 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
491 objABPair = butler.get('basicPair', dataId={'id': 'foo', 'name': 'bar'})
493 verificationButler = dafPersist.Butler(outputs={'root': os.path.join(self.testData, 'repo3'),
494 'policy': self.policy,
495 'mapper': 'lsst.obs.base.test.CompositeMapper',
496 'mode': 'rw'})
497 verificationButler.put(objABPair, 'basicPair', dataId={'id': 'foo', 'name': 'bar'})
499 objA = verificationButler.get('basicObject1', {'id': 'foo'})
500 self.assertEqual(objA, objABPair.objA)
501 with self.assertRaises(RuntimeError):
502 verificationButler.get('basicObject2', {'name': 'bar'}, immediate=True)
505class MemoryTester(lsst.utils.tests.MemoryTestCase):
506 pass
509def setup_module(module):
510 lsst.utils.tests.init()
513if __name__ == "__main__": 513 ↛ 514line 513 didn't jump to line 514, because the condition on line 513 was never true
514 lsst.utils.tests.init()
515 unittest.main()