Coverage for tests/test_composite.py: 17%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import os
23import shutil
24import unittest
25import tempfile
27import lsst.utils.tests
28import lsst.daf.persistence as dafPersist
29import lsst.daf.persistence.test as dpTest
31ROOT = os.path.abspath(os.path.dirname(__file__))
34class TestCompositeTestCase(unittest.TestCase):
35 """A test case for composite object i/o."""
37 def setUp(self):
38 self.testData = tempfile.mkdtemp(dir=ROOT, prefix='TestCompositeTestCase-')
39 self.firstRepoPath = os.path.join(self.testData, 'repo1')
40 self.objA = dpTest.TestObject("abc")
41 self.objB = dpTest.TestObject("def")
42 self.policy = dafPersist.Policy(
43 {'camera': 'lsst.afw.cameraGeom.Camera',
44 'datasets': {
45 'basicObject1': {
46 'python': 'lsst.daf.persistence.test.TestObject',
47 'template': 'basic/id%(id)s.pickle',
48 'storage': 'PickleStorage'},
49 'basicObject2': {
50 'python': 'lsst.daf.persistence.test.TestObject',
51 'template': 'basic/name%(name)s.pickle',
52 'storage': 'PickleStorage'},
53 'basicPair': {
54 'python': 'lsst.daf.persistence.test.TestObjectPair',
55 'composite': {
56 'a': {'datasetType': 'basicObject1'},
57 'b': {'datasetType': 'basicObject2'}
58 },
59 'assembler':
60 'lsst.daf.persistence.test.TestObjectPair.assembler',
61 'disassembler':
62 'lsst.daf.persistence.test.TestObjectPair.disassembler'},
63 'stdTestType': {
64 'python': 'lsst.daf.persistence.test.TestObjectPair',
65 'composite': {
66 'a': {'datasetType': 'basicObject1'},
67 'b': {'datasetType': 'basicObject2'}
68 }
69 },
70 'bypassTestType': {
71 'python': 'lsst.daf.persistence.test.TestObjectPair',
72 'composite': {
73 'a': {'datasetType': 'basicObject1'},
74 'b': {'datasetType': 'basicObject2'}
75 }
76 }
77 }})
79 repoArgs = dafPersist.RepositoryArgs(root=self.firstRepoPath, policy=self.policy,
80 mapper='lsst.obs.base.test.CompositeMapper')
81 butler = dafPersist.Butler(outputs=repoArgs)
82 butler.put(self.objA, 'basicObject1', dataId={'id': 'foo'})
83 butler.put(self.objB, 'basicObject2', dataId={'name': 'bar'})
84 del butler
85 del repoArgs
87 def tearDown(self):
88 if os.path.exists(self.testData):
89 shutil.rmtree(self.testData)
91 def testType3GetAndPut(self):
92 """Verify get and put for composites.
94 1. Verify that a composite can be loaded and that its components are
95 the same as when the type1 components are loaded individually (verifies
96 correct lookup in this case).
97 2. Verify that when the individual components are put and when the
98 composite is put (which disassembles into individual components) that
99 the objects that are written are the same.
100 """
101 secondRepoPath = os.path.join(self.testData, 'repo2')
102 repoArgs = dafPersist.RepositoryArgs(root=secondRepoPath, policy=self.policy)
103 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
104 verificationButler = dafPersist.Butler(inputs=secondRepoPath)
105 objABPair = butler.get('basicPair', dataId={'id': 'foo', 'name': 'bar'})
107 self.assertEqual(self.objA, objABPair.objA)
108 self.assertEqual(self.objB, objABPair.objB)
110 # For now also test that the type 1 and type 3 components are not the
111 # same object. These objects are not yet in the butler cache because
112 # they have not been gotten yet (they have only only been put)
113 self.assertIsNot(self.objA, objABPair.objA)
114 self.assertIsNot(self.objB, objABPair.objB)
116 # Now, get a type 1 copy of objA and objB, and they should be
117 # equivalent to the instance in the composite.
118 objA = butler.get('basicObject1', dataId={'id': 'foo'}, immediate=True)
119 objB = butler.get('basicObject2', dataId={'name': 'bar'}, immediate=True)
120 self.assertEqual(objA, objABPair.objA)
121 self.assertEqual(objB, objABPair.objB)
123 butler.put(objABPair, 'basicPair', dataId={'id': 'foo', 'name': 'bar'})
124 verObjA = verificationButler.get('basicObject1', {'id': 'foo'})
125 self.assertEqual(verObjA, objABPair.objA)
126 verObjB = verificationButler.get('basicObject2', {'name': 'bar'})
127 self.assertEqual(verObjB, objABPair.objB)
129 def testDottedDatasetType(self):
130 """Verify that components of a composite can be loaded by dotted name
131 in the form DatasetType.componentName
132 """
133 thirdRepoPath = os.path.join(self.testData, 'repo3')
134 # child repositories do not look up in-repo policies. We need to fix
135 # that.
136 repoArgs = dafPersist.RepositoryArgs(root=thirdRepoPath, policy=self.policy)
137 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
138 verificationButler = dafPersist.Butler(inputs=thirdRepoPath)
139 componentObjA = butler.get('basicPair.a', dataId={'id': 'foo', 'name': 'bar'})
140 componentObjB = butler.get('basicPair.b', dataId={'id': 'foo', 'name': 'bar'})
141 self.assertEqual(self.objA, componentObjA)
142 self.assertEqual(self.objB, componentObjB)
143 butler.put(componentObjA, 'basicPair.a', dataId={'id': 'foo', 'name': 'bar'})
144 butler.put(componentObjB, 'basicPair.b', dataId={'id': 'foo', 'name': 'bar'})
145 verObjA = verificationButler.get('basicObject1', {'id': 'foo'})
146 self.assertEqual(verObjA, componentObjA)
147 verObjB = verificationButler.get('basicObject2', {'name': 'bar'})
148 self.assertEqual(verObjB, componentObjB)
150 def testDatasetExists(self):
151 """Verify that Butler.datasetExists returns true for a composite
152 dataset whose components exist."""
153 butler = dafPersist.Butler(inputs=self.firstRepoPath)
154 self.assertTrue(butler.datasetExists('basicPair', dataId={'id': 'foo', 'name': 'bar'}))
156 def testDatasetDoesNotExist(self):
157 """Verify that Butler.datasetExists returns false for a composite
158 dataset where some of the components do not exist."""
159 repoPath = os.path.join(self.testData, 'repo')
160 repoArgs = dafPersist.RepositoryArgs(root=repoPath, policy=self.policy,
161 mapper='lsst.obs.base.test.CompositeMapper')
163 butler = dafPersist.Butler(outputs=repoArgs)
164 self.objA = dpTest.TestObject("abc")
165 butler.put(self.objA, 'basicObject1', dataId={'id': 'foo'})
166 self.assertFalse(butler.datasetExists('basicPair', dataId={'id': 'foo', 'name': 'bar'}))
168 def testStd(self):
169 """Verify that composite dataset types with a std_ function are passed
170 to the std_ function after being instantiated."""
171 secondRepoPath = os.path.join(self.testData, 'repo2')
172 repoArgs = dafPersist.RepositoryArgs(root=secondRepoPath, policy=self.policy)
173 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
174 objABPair = butler.get('stdTestType', dataId={'id': 'foo', 'name': 'bar'})
175 self.assertTrue(hasattr(objABPair, 'standardized'))
176 self.assertTrue(objABPair.standardized)
178 def testBypass(self):
179 """Verify that composite dataset types with a bypass_ function are
180 passed to the bypass function after being instantiated."""
181 secondRepoPath = os.path.join(self.testData, 'repo2')
182 repoArgs = dafPersist.RepositoryArgs(root=secondRepoPath, policy=self.policy)
183 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
184 bypassObject = butler.get('bypassTestType', dataId={'id': 'foo', 'name': 'bar'})
185 self.assertEqual(bypassObject, set(['id', 'name']))
188class TestGenericAssembler(unittest.TestCase):
189 """A test case for the generic assembler feature of composite datasets."""
191 def setUp(self):
192 self.testData = tempfile.mkdtemp(dir=ROOT, prefix='TestGenericAssembler-')
193 self.firstRepoPath = os.path.join(self.testData, 'repo1')
194 self.secondRepoPath = os.path.join(self.testData, 'repo2')
195 self.objA = dpTest.TestObject("abc")
196 self.objB = dpTest.TestObject("def")
197 self.policy = dafPersist.Policy(
198 {'camera': 'lsst.afw.cameraGeom.Camera',
199 'datasets': {
200 'basicObject1': {
201 'python': 'lsst.daf.persistence.test.TestObject',
202 'template': 'basic/id%(id)s.pickle',
203 'storage': 'PickleStorage'
204 },
205 'basicObject2': {
206 'python': 'lsst.daf.persistence.test.TestObject',
207 'template': 'basic/name%(name)s.pickle',
208 'storage': 'PickleStorage'
209 },
210 'basicPair': {
211 'python': 'lsst.daf.persistence.test.TestObjectPair',
212 'composite': {
213 'a': {'datasetType': 'basicObject1'},
214 'b': {'datasetType': 'basicObject2'}
215 },
216 # note, no assembler or disassembler specified here, will
217 # use setter names inferred by component name.
218 },
219 # "generic assembler default constructor pair"
220 'gaDefCtorPair': { # dataset defition that uses the default ctor
221 'python': 'lsst.daf.persistence.test.TestObjectPair',
222 'composite': {
223 # note that the component names are the same as the
224 # argument names in the TestObjectPair.__init__ func.
225 'objA': {'datasetType': 'basicObject1',
226 'getter': 'get_a'},
227 'objB': {'datasetType': 'basicObject2',
228 'getter': 'get_b'}
229 },
230 # note, no assembler or disassembler specified here.
231 },
232 # "generic assembler default "
233 'gaPairWithSetter': {
234 'python': 'lsst.daf.persistence.test.TestObjectPair',
235 'composite': {
236 # note that the component names do not match argument
237 # names in the TestObjectPair.__init__ func or the set
238 # functions in the python object.
239 'z': {'datasetType': 'basicObject1',
240 'setter': 'set_a',
241 'getter': 'get_a'
242 },
243 'x': {'datasetType': 'basicObject2',
244 'setter': 'set_b',
245 'getter': 'get_b'
246 }
247 }
248 },
249 # simple object where setter and getter is named with
250 # underscore separator
251 'underscoreSetter': {
252 'python': 'lsst.daf.persistence.test.TestObjectUnderscoreSetter',
253 'composite': {
254 'foo': {'datasetType': 'basicObject1'}
255 }
256 },
257 # simple object where setter and getter is named with
258 # camelcase
259 'camelCaseSetter': {
260 'python': 'lsst.daf.persistence.test.TestObjectCamelCaseSetter',
261 'composite': {
262 'foo': {'datasetType': 'basicObject1'}
263 }
264 }
265 }})
267 repoArgs = dafPersist.RepositoryArgs(root=self.firstRepoPath, policy=self.policy,
268 mapper='lsst.obs.base.test.CompositeMapper')
269 butler = dafPersist.Butler(outputs=repoArgs)
270 butler.put(self.objA, 'basicObject1', dataId={'id': 'foo'})
271 butler.put(self.objB, 'basicObject2', dataId={'name': 'bar'})
272 del butler
273 del repoArgs
275 def tearDown(self):
276 if os.path.exists(self.testData):
277 shutil.rmtree(self.testData)
279 def testConstructor(self):
280 """Test the case where the arguments to the default constructor match
281 the component names and so the default constructor can be used by the
282 generic assembler to assemble the object.
284 Uses getters named by the policy to disassemble the object.
285 """
286 repoArgs = dafPersist.RepositoryArgs(root=self.secondRepoPath, policy=self.policy)
287 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
288 verificationButler = dafPersist.Butler(inputs=self.secondRepoPath)
290 objABPair = butler.get('basicPair', dataId={'id': 'foo', 'name': 'bar'})
291 self.assertEqual(self.objA, objABPair.objA)
292 self.assertEqual(self.objB, objABPair.objB)
294 butler.put(objABPair, 'basicPair', dataId={'id': 'foo', 'name': 'bar'})
295 # comparing the output files directly works so long as the storage is
296 # posix:
298 # put the composite object and verify it disassembled into the right
299 # locations by loading the components directly
300 objA = verificationButler.get('basicObject1', dataId={'id': 'foo'})
301 self.assertEqual(objA, objABPair.objA)
302 objB = verificationButler.get('basicObject2', dataId={'name': 'bar'})
303 self.assertEqual(objB, objABPair.objB)
305 del objABPair
307 objABPair = butler.get('gaDefCtorPair', dataId={'id': 'foo', 'name': 'bar'})
308 self.assertEqual(self.objA, objABPair.objA)
309 self.assertEqual(self.objB, objABPair.objB)
310 self.assertTrue(objABPair.usedInitSetter)
311 self.assertFalse(objABPair.usedASetter)
312 self.assertFalse(objABPair.usedBSetter)
314 # put the composite object and verify it disassembled into the right
315 # locations by loading the components directly
316 butler.put(objABPair, 'gaDefCtorPair', dataId={'id': 'baz', 'name': 'qux'})
317 verObjA = verificationButler.get('basicObject1', dataId={'id': 'baz'})
318 self.assertEqual(objABPair.objA, verObjA)
319 verObjB = verificationButler.get('basicObject2', dataId={'name': 'qux'})
320 self.assertEqual(objABPair.objB, verObjB)
322 def testGenericAssemblerPolicySpecifiedSetterGetter(self):
323 """Test the case where the component names do not have anything to do
324 with the setter/getter names or the init function parameter names,
325 and instead the component policy entry specifies the setter and
326 getter names.
327 """
328 repoArgs = dafPersist.RepositoryArgs(root=self.secondRepoPath, policy=self.policy)
329 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
330 objABPair = butler.get('gaPairWithSetter', dataId={'id': 'foo', 'name': 'bar'})
331 self.assertEqual(self.objA, objABPair.objA)
332 self.assertEqual(self.objB, objABPair.objB)
333 self.assertFalse(objABPair.usedInitSetter)
334 self.assertTrue(objABPair.usedASetter)
335 self.assertTrue(objABPair.usedBSetter)
337 butler.put(objABPair, 'gaPairWithSetter', dataId={'id': 'foo', 'name': 'bar'})
338 # comparing the output files directly works so long as the storage is
339 # posix:
341 verificationButler = dafPersist.Butler(inputs=self.secondRepoPath)
342 verObjA = verificationButler.get('basicObject1', dataId={'id': 'foo'})
343 verObjB = verificationButler.get('basicObject2', dataId={'name': 'bar'})
344 self.assertEqual(objABPair.objA, verObjA)
345 self.assertEqual(objABPair.objB, verObjB)
347 def testInferredNameUnderscoreSeparator(self):
348 """Test the case where the name of the setter & getter is inferred by
349 the policy name by prepending 'set_' and get_
350 """
351 repoArgs = dafPersist.RepositoryArgs(root=self.secondRepoPath, policy=self.policy)
352 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
353 obj = butler.get('underscoreSetter', dataId={'id': 'foo'})
354 self.assertEqual(self.objA, obj.get_foo())
355 butler.put(obj, 'underscoreSetter', dataId={'id': 'foo'})
357 verificationButler = dafPersist.Butler(inputs=self.secondRepoPath)
358 componentObj = verificationButler.get('basicObject1', dataId={'id': 'foo'})
359 self.assertEqual(componentObj, obj.get_foo())
361 def testInferredNameCamelcase(self):
362 """Test the case where the name of the setter & getter is inferred by
363 the policy name by prepending 'set' or 'get', to the capitalized
364 component name. E.g. for component name 'foo' the setter and getter
365 will be named setFoo and getFoo.
366 """
367 repoArgs = dafPersist.RepositoryArgs(root=self.secondRepoPath, policy=self.policy)
368 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
369 obj = butler.get('camelCaseSetter', dataId={'id': 'foo'})
370 self.assertEqual(self.objA, obj.getFoo())
371 butler.put(obj, 'camelCaseSetter', dataId={'id': 'foo'})
373 verificationButler = dafPersist.Butler(inputs=self.secondRepoPath)
374 componentObj = verificationButler.get('basicObject1', dataId={'id': 'foo'})
375 self.assertEqual(componentObj, obj.getFoo())
378def subsetAssembler(dataId, componentInfo, cls):
379 obj = cls()
380 obj.set_a(componentInfo['a'].obj)
381 return obj
384class TestSubset(unittest.TestCase):
385 """A test case for composite object subset keyword."""
387 def setUp(self):
388 self.testData = tempfile.mkdtemp(dir=ROOT, prefix='TestSubset-')
389 self.firstRepoPath = os.path.join(self.testData, 'repo1')
390 self.objA1 = dpTest.TestObject("abc")
391 self.objA2 = dpTest.TestObject("ABC")
392 self.objB = dpTest.TestObject("def")
393 self.policy = dafPersist.Policy(
394 {'camera': 'lsst.afw.cameraGeom.Camera',
395 'datasets': {
396 'basicObject1': {
397 'python': 'lsst.daf.persistence.test.TestObject',
398 'template': 'basic/id%(id)s.pickle',
399 'storage': 'PickleStorage'},
400 'basicObject2': {
401 'python': 'lsst.daf.persistence.test.TestObject',
402 'template': 'basic/name%(name)s.pickle',
403 'storage': 'PickleStorage'},
404 'basicPair': {
405 'python': 'lsst.daf.persistence.test.TestObjectPair',
406 'composite': {
407 'a': {
408 'datasetType': 'basicObject1',
409 'subset': True
410 }
411 },
412 'assembler': subsetAssembler
413 },
414 }})
416 repoArgs = dafPersist.RepositoryArgs(root=self.firstRepoPath, policy=self.policy,
417 mapper='lsst.obs.base.test.CompositeMapper')
418 butler = dafPersist.Butler(outputs=repoArgs)
419 butler.put(self.objA1, 'basicObject1', dataId={'id': 'foo1'})
420 butler.put(self.objA2, 'basicObject1', dataId={'id': 'foo2'})
421 butler.put(self.objB, 'basicObject2', dataId={'name': 'bar'})
422 del butler
423 del repoArgs
425 def tearDown(self):
426 if os.path.exists(self.testData):
427 shutil.rmtree(self.testData)
429 def test(self):
430 """Verify that the generic assembler and disassembler work for objects
431 that conform to the generic set/get API.
432 """
433 secondRepoPath = os.path.join(self.testData, 'repo2')
434 repoArgs = dafPersist.RepositoryArgs(root=secondRepoPath, policy=self.policy)
435 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
436 # the name 'bar' will find the obj that was put as obj b. It expects
437 # to find n objects of dataset type basicObject1. Since we don't
438 # specify any dataId that relates to basicObject1 (its only dataId
439 # key is 'id'), it will return everything it finds according to its
440 # policy. In this case that should be self.objA1 and self.objA2 that
441 # we put above. They will be in a list at objABPair.objA.
442 objABPair = butler.get('basicPair', dataId={'name': 'bar'})
443 objABPair.objA.sort()
444 self.assertEqual(self.objA2, objABPair.objA[0])
445 self.assertEqual(self.objA1, objABPair.objA[1])
446 # subset is a get-only operation. To put, the dataId must be specified,
447 # so there's no put to test here.
450class TestInputOnly(unittest.TestCase):
451 """A test case for composite input keyword."""
453 def setUp(self):
454 self.testData = tempfile.mkdtemp(dir=ROOT, prefix='TestInputOnly-')
455 self.firstRepoPath = os.path.join(self.testData, 'repo1')
456 self.objA = dpTest.TestObject("abc")
457 self.objB = dpTest.TestObject("def")
458 self.policy = dafPersist.Policy(
459 {'camera': 'lsst.afw.cameraGeom.Camera',
460 'datasets': {
461 'basicObject1': {
462 'python': 'lsst.daf.persistence.test.TestObject',
463 'template': 'basic/id%(id)s.pickle',
464 'storage': 'PickleStorage'},
465 'basicObject2': {
466 'python': 'lsst.daf.persistence.test.TestObject',
467 'template': 'basic/name%(name)s.pickle',
468 'storage': 'PickleStorage'},
469 'basicPair': {
470 'python': 'lsst.daf.persistence.test.TestObjectPair',
471 'composite': {
472 'a': {
473 'datasetType': 'basicObject1'
474 },
475 'b': {
476 'datasetType': 'basicObject2',
477 'inputOnly': True
478 }
479 },
480 'assembler': 'lsst.daf.persistence.test.TestObjectPair.assembler',
481 'disassembler': 'lsst.daf.persistence.test.TestObjectPair.disassembler'
483 }
484 }})
486 repoArgs = dafPersist.RepositoryArgs(root=self.firstRepoPath,
487 mapper='lsst.obs.base.test.CompositeMapper',
488 policy=self.policy)
489 butler = dafPersist.Butler(outputs=repoArgs)
490 butler.put(self.objA, 'basicObject1', dataId={'id': 'foo'})
491 butler.put(self.objB, 'basicObject2', dataId={'name': 'bar'})
492 del butler
493 del repoArgs
495 def tearDown(self):
496 if os.path.exists(self.testData):
497 shutil.rmtree(self.testData)
499 def test(self):
500 """ Verify that when a type 3 dataset is put and one of its components
501 is marked 'inputOnly' by the policy that the inputOnly comonent is not
502 written.
503 """
504 secondRepoPath = os.path.join(self.testData, 'repo2')
505 repoArgs = dafPersist.RepositoryArgs(root=secondRepoPath, policy=self.policy)
506 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
507 objABPair = butler.get('basicPair', dataId={'id': 'foo', 'name': 'bar'})
509 verificationButler = dafPersist.Butler(outputs={'root': os.path.join(self.testData, 'repo3'),
510 'policy': self.policy,
511 'mapper': 'lsst.obs.base.test.CompositeMapper',
512 'mode': 'rw'})
513 verificationButler.put(objABPair, 'basicPair', dataId={'id': 'foo', 'name': 'bar'})
515 objA = verificationButler.get('basicObject1', {'id': 'foo'})
516 self.assertEqual(objA, objABPair.objA)
517 with self.assertRaises(RuntimeError):
518 verificationButler.get('basicObject2', {'name': 'bar'}, immediate=True)
521class MemoryTester(lsst.utils.tests.MemoryTestCase):
522 pass
525def setup_module(module):
526 lsst.utils.tests.init()
529if __name__ == "__main__": 529 ↛ 530line 529 didn't jump to line 530, because the condition on line 529 was never true
530 lsst.utils.tests.init()
531 unittest.main()