Coverage for tests/test_composite.py: 20%
219 statements
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-11 11:06 +0000
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-11 11:06 +0000
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import os
23import shutil
24import tempfile
25import unittest
27import lsst.daf.persistence as dafPersist
28import lsst.daf.persistence.test as dpTest
29import lsst.utils.tests
31ROOT = os.path.abspath(os.path.dirname(__file__))
34class TestCompositeTestCase(unittest.TestCase):
35 """A test case for composite object i/o."""
37 def setUp(self):
38 self.testData = tempfile.mkdtemp(dir=ROOT, prefix="TestCompositeTestCase-")
39 self.firstRepoPath = os.path.join(self.testData, "repo1")
40 self.objA = dpTest.TestObject("abc")
41 self.objB = dpTest.TestObject("def")
42 self.policy = dafPersist.Policy(
43 {
44 "camera": "lsst.afw.cameraGeom.Camera",
45 "datasets": {
46 "basicObject1": {
47 "python": "lsst.daf.persistence.test.TestObject",
48 "template": "basic/id%(id)s.pickle",
49 "storage": "PickleStorage",
50 },
51 "basicObject2": {
52 "python": "lsst.daf.persistence.test.TestObject",
53 "template": "basic/name%(name)s.pickle",
54 "storage": "PickleStorage",
55 },
56 "basicPair": {
57 "python": "lsst.daf.persistence.test.TestObjectPair",
58 "composite": {
59 "a": {"datasetType": "basicObject1"},
60 "b": {"datasetType": "basicObject2"},
61 },
62 "assembler": "lsst.daf.persistence.test.TestObjectPair.assembler",
63 "disassembler": "lsst.daf.persistence.test.TestObjectPair.disassembler",
64 },
65 "stdTestType": {
66 "python": "lsst.daf.persistence.test.TestObjectPair",
67 "composite": {
68 "a": {"datasetType": "basicObject1"},
69 "b": {"datasetType": "basicObject2"},
70 },
71 },
72 "bypassTestType": {
73 "python": "lsst.daf.persistence.test.TestObjectPair",
74 "composite": {
75 "a": {"datasetType": "basicObject1"},
76 "b": {"datasetType": "basicObject2"},
77 },
78 },
79 },
80 }
81 )
83 repoArgs = dafPersist.RepositoryArgs(
84 root=self.firstRepoPath, policy=self.policy, mapper="lsst.obs.base.test.CompositeMapper"
85 )
86 butler = dafPersist.Butler(outputs=repoArgs)
87 butler.put(self.objA, "basicObject1", dataId={"id": "foo"})
88 butler.put(self.objB, "basicObject2", dataId={"name": "bar"})
89 del butler
90 del repoArgs
92 def tearDown(self):
93 if os.path.exists(self.testData):
94 shutil.rmtree(self.testData)
96 def testType3GetAndPut(self):
97 """Verify get and put for composites.
99 1. Verify that a composite can be loaded and that its components are
100 the same as when the type1 components are loaded individually (verifies
101 correct lookup in this case).
102 2. Verify that when the individual components are put and when the
103 composite is put (which disassembles into individual components) that
104 the objects that are written are the same.
105 """
106 secondRepoPath = os.path.join(self.testData, "repo2")
107 repoArgs = dafPersist.RepositoryArgs(root=secondRepoPath, policy=self.policy)
108 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
109 verificationButler = dafPersist.Butler(inputs=secondRepoPath)
110 objABPair = butler.get("basicPair", dataId={"id": "foo", "name": "bar"})
112 self.assertEqual(self.objA, objABPair.objA)
113 self.assertEqual(self.objB, objABPair.objB)
115 # For now also test that the type 1 and type 3 components are not the
116 # same object. These objects are not yet in the butler cache because
117 # they have not been gotten yet (they have only only been put)
118 self.assertIsNot(self.objA, objABPair.objA)
119 self.assertIsNot(self.objB, objABPair.objB)
121 # Now, get a type 1 copy of objA and objB, and they should be
122 # equivalent to the instance in the composite.
123 objA = butler.get("basicObject1", dataId={"id": "foo"}, immediate=True)
124 objB = butler.get("basicObject2", dataId={"name": "bar"}, immediate=True)
125 self.assertEqual(objA, objABPair.objA)
126 self.assertEqual(objB, objABPair.objB)
128 butler.put(objABPair, "basicPair", dataId={"id": "foo", "name": "bar"})
129 verObjA = verificationButler.get("basicObject1", {"id": "foo"})
130 self.assertEqual(verObjA, objABPair.objA)
131 verObjB = verificationButler.get("basicObject2", {"name": "bar"})
132 self.assertEqual(verObjB, objABPair.objB)
134 def testDottedDatasetType(self):
135 """Verify that components of a composite can be loaded by dotted name
136 in the form DatasetType.componentName
137 """
138 thirdRepoPath = os.path.join(self.testData, "repo3")
139 # child repositories do not look up in-repo policies. We need to fix
140 # that.
141 repoArgs = dafPersist.RepositoryArgs(root=thirdRepoPath, policy=self.policy)
142 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
143 verificationButler = dafPersist.Butler(inputs=thirdRepoPath)
144 componentObjA = butler.get("basicPair.a", dataId={"id": "foo", "name": "bar"})
145 componentObjB = butler.get("basicPair.b", dataId={"id": "foo", "name": "bar"})
146 self.assertEqual(self.objA, componentObjA)
147 self.assertEqual(self.objB, componentObjB)
148 butler.put(componentObjA, "basicPair.a", dataId={"id": "foo", "name": "bar"})
149 butler.put(componentObjB, "basicPair.b", dataId={"id": "foo", "name": "bar"})
150 verObjA = verificationButler.get("basicObject1", {"id": "foo"})
151 self.assertEqual(verObjA, componentObjA)
152 verObjB = verificationButler.get("basicObject2", {"name": "bar"})
153 self.assertEqual(verObjB, componentObjB)
155 def testDatasetExists(self):
156 """Verify that Butler.datasetExists returns true for a composite
157 dataset whose components exist."""
158 butler = dafPersist.Butler(inputs=self.firstRepoPath)
159 self.assertTrue(butler.datasetExists("basicPair", dataId={"id": "foo", "name": "bar"}))
161 def testDatasetDoesNotExist(self):
162 """Verify that Butler.datasetExists returns false for a composite
163 dataset where some of the components do not exist."""
164 repoPath = os.path.join(self.testData, "repo")
165 repoArgs = dafPersist.RepositoryArgs(
166 root=repoPath, policy=self.policy, mapper="lsst.obs.base.test.CompositeMapper"
167 )
169 butler = dafPersist.Butler(outputs=repoArgs)
170 self.objA = dpTest.TestObject("abc")
171 butler.put(self.objA, "basicObject1", dataId={"id": "foo"})
172 self.assertFalse(butler.datasetExists("basicPair", dataId={"id": "foo", "name": "bar"}))
174 def testStd(self):
175 """Verify that composite dataset types with a std_ function are passed
176 to the std_ function after being instantiated."""
177 secondRepoPath = os.path.join(self.testData, "repo2")
178 repoArgs = dafPersist.RepositoryArgs(root=secondRepoPath, policy=self.policy)
179 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
180 objABPair = butler.get("stdTestType", dataId={"id": "foo", "name": "bar"})
181 self.assertTrue(hasattr(objABPair, "standardized"))
182 self.assertTrue(objABPair.standardized)
184 def testBypass(self):
185 """Verify that composite dataset types with a bypass_ function are
186 passed to the bypass function after being instantiated."""
187 secondRepoPath = os.path.join(self.testData, "repo2")
188 repoArgs = dafPersist.RepositoryArgs(root=secondRepoPath, policy=self.policy)
189 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
190 bypassObject = butler.get("bypassTestType", dataId={"id": "foo", "name": "bar"})
191 self.assertEqual(bypassObject, set(["id", "name"]))
194class TestGenericAssembler(unittest.TestCase):
195 """A test case for the generic assembler feature of composite datasets."""
197 def setUp(self):
198 self.testData = tempfile.mkdtemp(dir=ROOT, prefix="TestGenericAssembler-")
199 self.firstRepoPath = os.path.join(self.testData, "repo1")
200 self.secondRepoPath = os.path.join(self.testData, "repo2")
201 self.objA = dpTest.TestObject("abc")
202 self.objB = dpTest.TestObject("def")
203 self.policy = dafPersist.Policy(
204 {
205 "camera": "lsst.afw.cameraGeom.Camera",
206 "datasets": {
207 "basicObject1": {
208 "python": "lsst.daf.persistence.test.TestObject",
209 "template": "basic/id%(id)s.pickle",
210 "storage": "PickleStorage",
211 },
212 "basicObject2": {
213 "python": "lsst.daf.persistence.test.TestObject",
214 "template": "basic/name%(name)s.pickle",
215 "storage": "PickleStorage",
216 },
217 "basicPair": {
218 "python": "lsst.daf.persistence.test.TestObjectPair",
219 "composite": {
220 "a": {"datasetType": "basicObject1"},
221 "b": {"datasetType": "basicObject2"},
222 },
223 # note, no assembler or disassembler specified here,
224 # will use setter names inferred by component name.
225 },
226 # "generic assembler default constructor pair"
227 "gaDefCtorPair": { # dataset defition that uses the default ctor
228 "python": "lsst.daf.persistence.test.TestObjectPair",
229 "composite": {
230 # note that the component names are the same as the
231 # argument names in the TestObjectPair.__init__
232 # func.
233 "objA": {"datasetType": "basicObject1", "getter": "get_a"},
234 "objB": {"datasetType": "basicObject2", "getter": "get_b"},
235 },
236 # note, no assembler or disassembler specified here.
237 },
238 # "generic assembler default "
239 "gaPairWithSetter": {
240 "python": "lsst.daf.persistence.test.TestObjectPair",
241 "composite": {
242 # note that the component names do not match
243 # argument names in the TestObjectPair.__init__
244 # func or the set functions in the python object.
245 "z": {"datasetType": "basicObject1", "setter": "set_a", "getter": "get_a"},
246 "x": {"datasetType": "basicObject2", "setter": "set_b", "getter": "get_b"},
247 },
248 },
249 # simple object where setter and getter is named with
250 # underscore separator
251 "underscoreSetter": {
252 "python": "lsst.daf.persistence.test.TestObjectUnderscoreSetter",
253 "composite": {"foo": {"datasetType": "basicObject1"}},
254 },
255 # simple object where setter and getter is named with
256 # camelcase
257 "camelCaseSetter": {
258 "python": "lsst.daf.persistence.test.TestObjectCamelCaseSetter",
259 "composite": {"foo": {"datasetType": "basicObject1"}},
260 },
261 },
262 }
263 )
265 repoArgs = dafPersist.RepositoryArgs(
266 root=self.firstRepoPath, policy=self.policy, mapper="lsst.obs.base.test.CompositeMapper"
267 )
268 butler = dafPersist.Butler(outputs=repoArgs)
269 butler.put(self.objA, "basicObject1", dataId={"id": "foo"})
270 butler.put(self.objB, "basicObject2", dataId={"name": "bar"})
271 del butler
272 del repoArgs
274 def tearDown(self):
275 if os.path.exists(self.testData):
276 shutil.rmtree(self.testData)
278 def testConstructor(self):
279 """Test the case where the arguments to the default constructor match
280 the component names and so the default constructor can be used by the
281 generic assembler to assemble the object.
283 Uses getters named by the policy to disassemble the object.
284 """
285 repoArgs = dafPersist.RepositoryArgs(root=self.secondRepoPath, policy=self.policy)
286 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
287 verificationButler = dafPersist.Butler(inputs=self.secondRepoPath)
289 objABPair = butler.get("basicPair", dataId={"id": "foo", "name": "bar"})
290 self.assertEqual(self.objA, objABPair.objA)
291 self.assertEqual(self.objB, objABPair.objB)
293 butler.put(objABPair, "basicPair", dataId={"id": "foo", "name": "bar"})
294 # comparing the output files directly works so long as the storage is
295 # posix:
297 # put the composite object and verify it disassembled into the right
298 # locations by loading the components directly
299 objA = verificationButler.get("basicObject1", dataId={"id": "foo"})
300 self.assertEqual(objA, objABPair.objA)
301 objB = verificationButler.get("basicObject2", dataId={"name": "bar"})
302 self.assertEqual(objB, objABPair.objB)
304 del objABPair
306 objABPair = butler.get("gaDefCtorPair", dataId={"id": "foo", "name": "bar"})
307 self.assertEqual(self.objA, objABPair.objA)
308 self.assertEqual(self.objB, objABPair.objB)
309 self.assertTrue(objABPair.usedInitSetter)
310 self.assertFalse(objABPair.usedASetter)
311 self.assertFalse(objABPair.usedBSetter)
313 # put the composite object and verify it disassembled into the right
314 # locations by loading the components directly
315 butler.put(objABPair, "gaDefCtorPair", dataId={"id": "baz", "name": "qux"})
316 verObjA = verificationButler.get("basicObject1", dataId={"id": "baz"})
317 self.assertEqual(objABPair.objA, verObjA)
318 verObjB = verificationButler.get("basicObject2", dataId={"name": "qux"})
319 self.assertEqual(objABPair.objB, verObjB)
321 def testGenericAssemblerPolicySpecifiedSetterGetter(self):
322 """Test the case where the component names do not have anything to do
323 with the setter/getter names or the init function parameter names,
324 and instead the component policy entry specifies the setter and
325 getter names.
326 """
327 repoArgs = dafPersist.RepositoryArgs(root=self.secondRepoPath, policy=self.policy)
328 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
329 objABPair = butler.get("gaPairWithSetter", dataId={"id": "foo", "name": "bar"})
330 self.assertEqual(self.objA, objABPair.objA)
331 self.assertEqual(self.objB, objABPair.objB)
332 self.assertFalse(objABPair.usedInitSetter)
333 self.assertTrue(objABPair.usedASetter)
334 self.assertTrue(objABPair.usedBSetter)
336 butler.put(objABPair, "gaPairWithSetter", dataId={"id": "foo", "name": "bar"})
337 # comparing the output files directly works so long as the storage is
338 # posix:
340 verificationButler = dafPersist.Butler(inputs=self.secondRepoPath)
341 verObjA = verificationButler.get("basicObject1", dataId={"id": "foo"})
342 verObjB = verificationButler.get("basicObject2", dataId={"name": "bar"})
343 self.assertEqual(objABPair.objA, verObjA)
344 self.assertEqual(objABPair.objB, verObjB)
346 def testInferredNameUnderscoreSeparator(self):
347 """Test the case where the name of the setter & getter is inferred by
348 the policy name by prepending 'set_' and get_
349 """
350 repoArgs = dafPersist.RepositoryArgs(root=self.secondRepoPath, policy=self.policy)
351 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
352 obj = butler.get("underscoreSetter", dataId={"id": "foo"})
353 self.assertEqual(self.objA, obj.get_foo())
354 butler.put(obj, "underscoreSetter", dataId={"id": "foo"})
356 verificationButler = dafPersist.Butler(inputs=self.secondRepoPath)
357 componentObj = verificationButler.get("basicObject1", dataId={"id": "foo"})
358 self.assertEqual(componentObj, obj.get_foo())
360 def testInferredNameCamelcase(self):
361 """Test the case where the name of the setter & getter is inferred by
362 the policy name by prepending 'set' or 'get', to the capitalized
363 component name. E.g. for component name 'foo' the setter and getter
364 will be named setFoo and getFoo.
365 """
366 repoArgs = dafPersist.RepositoryArgs(root=self.secondRepoPath, policy=self.policy)
367 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
368 obj = butler.get("camelCaseSetter", dataId={"id": "foo"})
369 self.assertEqual(self.objA, obj.getFoo())
370 butler.put(obj, "camelCaseSetter", dataId={"id": "foo"})
372 verificationButler = dafPersist.Butler(inputs=self.secondRepoPath)
373 componentObj = verificationButler.get("basicObject1", dataId={"id": "foo"})
374 self.assertEqual(componentObj, obj.getFoo())
377def subsetAssembler(dataId, componentInfo, cls):
378 obj = cls()
379 obj.set_a(componentInfo["a"].obj)
380 return obj
383class TestSubset(unittest.TestCase):
384 """A test case for composite object subset keyword."""
386 def setUp(self):
387 self.testData = tempfile.mkdtemp(dir=ROOT, prefix="TestSubset-")
388 self.firstRepoPath = os.path.join(self.testData, "repo1")
389 self.objA1 = dpTest.TestObject("abc")
390 self.objA2 = dpTest.TestObject("ABC")
391 self.objB = dpTest.TestObject("def")
392 self.policy = dafPersist.Policy(
393 {
394 "camera": "lsst.afw.cameraGeom.Camera",
395 "datasets": {
396 "basicObject1": {
397 "python": "lsst.daf.persistence.test.TestObject",
398 "template": "basic/id%(id)s.pickle",
399 "storage": "PickleStorage",
400 },
401 "basicObject2": {
402 "python": "lsst.daf.persistence.test.TestObject",
403 "template": "basic/name%(name)s.pickle",
404 "storage": "PickleStorage",
405 },
406 "basicPair": {
407 "python": "lsst.daf.persistence.test.TestObjectPair",
408 "composite": {"a": {"datasetType": "basicObject1", "subset": True}},
409 "assembler": subsetAssembler,
410 },
411 },
412 }
413 )
415 repoArgs = dafPersist.RepositoryArgs(
416 root=self.firstRepoPath, policy=self.policy, mapper="lsst.obs.base.test.CompositeMapper"
417 )
418 butler = dafPersist.Butler(outputs=repoArgs)
419 butler.put(self.objA1, "basicObject1", dataId={"id": "foo1"})
420 butler.put(self.objA2, "basicObject1", dataId={"id": "foo2"})
421 butler.put(self.objB, "basicObject2", dataId={"name": "bar"})
422 del butler
423 del repoArgs
425 def tearDown(self):
426 if os.path.exists(self.testData):
427 shutil.rmtree(self.testData)
429 def test(self):
430 """Verify that the generic assembler and disassembler work for objects
431 that conform to the generic set/get API.
432 """
433 secondRepoPath = os.path.join(self.testData, "repo2")
434 repoArgs = dafPersist.RepositoryArgs(root=secondRepoPath, policy=self.policy)
435 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
436 # the name 'bar' will find the obj that was put as obj b. It expects
437 # to find n objects of dataset type basicObject1. Since we don't
438 # specify any dataId that relates to basicObject1 (its only dataId
439 # key is 'id'), it will return everything it finds according to its
440 # policy. In this case that should be self.objA1 and self.objA2 that
441 # we put above. They will be in a list at objABPair.objA.
442 objABPair = butler.get("basicPair", dataId={"name": "bar"})
443 objABPair.objA.sort()
444 self.assertEqual(self.objA2, objABPair.objA[0])
445 self.assertEqual(self.objA1, objABPair.objA[1])
446 # subset is a get-only operation. To put, the dataId must be specified,
447 # so there's no put to test here.
450class TestInputOnly(unittest.TestCase):
451 """A test case for composite input keyword."""
453 def setUp(self):
454 self.testData = tempfile.mkdtemp(dir=ROOT, prefix="TestInputOnly-")
455 self.firstRepoPath = os.path.join(self.testData, "repo1")
456 self.objA = dpTest.TestObject("abc")
457 self.objB = dpTest.TestObject("def")
458 self.policy = dafPersist.Policy(
459 {
460 "camera": "lsst.afw.cameraGeom.Camera",
461 "datasets": {
462 "basicObject1": {
463 "python": "lsst.daf.persistence.test.TestObject",
464 "template": "basic/id%(id)s.pickle",
465 "storage": "PickleStorage",
466 },
467 "basicObject2": {
468 "python": "lsst.daf.persistence.test.TestObject",
469 "template": "basic/name%(name)s.pickle",
470 "storage": "PickleStorage",
471 },
472 "basicPair": {
473 "python": "lsst.daf.persistence.test.TestObjectPair",
474 "composite": {
475 "a": {"datasetType": "basicObject1"},
476 "b": {"datasetType": "basicObject2", "inputOnly": True},
477 },
478 "assembler": "lsst.daf.persistence.test.TestObjectPair.assembler",
479 "disassembler": "lsst.daf.persistence.test.TestObjectPair.disassembler",
480 },
481 },
482 }
483 )
485 repoArgs = dafPersist.RepositoryArgs(
486 root=self.firstRepoPath, mapper="lsst.obs.base.test.CompositeMapper", policy=self.policy
487 )
488 butler = dafPersist.Butler(outputs=repoArgs)
489 butler.put(self.objA, "basicObject1", dataId={"id": "foo"})
490 butler.put(self.objB, "basicObject2", dataId={"name": "bar"})
491 del butler
492 del repoArgs
494 def tearDown(self):
495 if os.path.exists(self.testData):
496 shutil.rmtree(self.testData)
498 def test(self):
499 """Verify that when a type 3 dataset is put and one of its components
500 is marked 'inputOnly' by the policy that the inputOnly comonent is not
501 written.
502 """
503 secondRepoPath = os.path.join(self.testData, "repo2")
504 repoArgs = dafPersist.RepositoryArgs(root=secondRepoPath, policy=self.policy)
505 butler = dafPersist.Butler(inputs=self.firstRepoPath, outputs=repoArgs)
506 objABPair = butler.get("basicPair", dataId={"id": "foo", "name": "bar"})
508 verificationButler = dafPersist.Butler(
509 outputs={
510 "root": os.path.join(self.testData, "repo3"),
511 "policy": self.policy,
512 "mapper": "lsst.obs.base.test.CompositeMapper",
513 "mode": "rw",
514 }
515 )
516 verificationButler.put(objABPair, "basicPair", dataId={"id": "foo", "name": "bar"})
518 objA = verificationButler.get("basicObject1", {"id": "foo"})
519 self.assertEqual(objA, objABPair.objA)
520 with self.assertRaises(RuntimeError):
521 verificationButler.get("basicObject2", {"name": "bar"}, immediate=True)
524class MemoryTester(lsst.utils.tests.MemoryTestCase):
525 pass
528def setup_module(module):
529 lsst.utils.tests.init()
532if __name__ == "__main__": 532 ↛ 533line 532 didn't jump to line 533, because the condition on line 532 was never true
533 lsst.utils.tests.init()
534 unittest.main()