Coverage for tests/test_repository.py: 16%
572 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-11 02:39 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-11 02:39 -0800
1# -*- coding: UTF-8 -*-
3#
4# LSST Data Management System
5# Copyright 2016 LSST Corporation.
6#
7# This product includes software developed by the
8# LSST Project (http://www.lsst.org/).
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the LSST License Statement and
21# the GNU General Public License along with this program. If not,
22# see <http://www.lsstcorp.org/LegalNotices/>.
23#
25import os
26import astropy.io.fits
27import shutil
28import sqlite3
29import unittest
30import tempfile
32import lsst.daf.persistence as dp
33# can't use name TestObject, becuase it messes Pytest up. Alias it to tstObj
34from lsst.daf.persistence.test import TestObject as tstObj
35from lsst.daf.persistence.test import MapperForTestWriting
36import lsst.utils.tests
38# Define the root of the tests relative to this file
39ROOT = os.path.abspath(os.path.dirname(__file__))
42def setup_module(module):
43 lsst.utils.tests.init()
46class ParentMapper(dp.Mapper):
48 def __init__(self, root, **kwargs):
49 self.root = root
50 self.storage = dp.Storage.makeFromURI(self.root)
52 def __repr__(self):
53 return 'ParentMapper(root=%s)' % self.root
55 def map_raw(self, dataId, write):
56 python = 'astropy.io.fits.HDUList'
57 persistable = None
58 storage = 'PickleStorage'
59 path = os.path.join(self.root, 'raw')
60 path = os.path.join(path, 'raw_v' + str(dataId['visit']) + '_f' + dataId['filter'] + '.fits.gz')
61 if os.path.exists(path):
62 return dp.ButlerLocation(python, persistable, storage, path,
63 dataId, self, self.storage)
64 return None
66 def bypass_raw(self, datasetType, pythonType, location, dataId):
67 return astropy.io.fits.open(location.getLocations()[0])
69 def query_raw(self, format, dataId):
70 values = [{'visit': 1, 'filter': 'g'}, {'visit': 2, 'filter': 'g'}, {'visit': 3, 'filter': 'r'}]
71 matches = []
72 for value in values:
73 match = True
74 for item in dataId:
75 if value[item] != dataId[item]:
76 match = False
77 break
78 if match:
79 matches.append(value)
80 results = set()
81 for match in matches:
82 tempTup = []
83 for word in format:
84 tempTup.append(match[word])
85 results.add(tuple(tempTup))
86 return results
88 def getDefaultLevel(self):
89 return 'visit'
91 def getKeys(self, datasetType, level):
92 return {'filter': str, 'visit': int}
94 def map_str(self, dataId, write):
95 path = os.path.join(self.root, 'raw')
96 path = os.path.join(path, 'raw_v' + str(dataId['str']) + '_f' + dataId['filter'] + '.fits.gz')
97 if os.path.exists(path):
98 return dp.ButlerLocation(str, None, 'PickleStorage', path, dataId,
99 self, self.storage)
100 return None
103class ChildrenMapper(dp.Mapper):
105 def __init__(self, root, **kwargs):
106 self.root = root
107 self.storage = dp.Storage.makeFromURI(self.root)
109 def map_raw(self, dataId, write):
110 python = 'astropy.io.fits.HDUList'
111 persistable = None
112 storage = 'FitsStorage'
113 path = os.path.join(self.root, 'raw')
114 path = os.path.join(path, 'raw_v' + str(dataId['visit']) + '_f' + dataId['filter'] + '.fits.gz')
115 if write or os.path.exists(path):
116 return dp.ButlerLocation(python, persistable, storage, path,
117 dataId, self, self.storage)
118 return None
120 def map_pickled(self, dataId, write):
121 python = 'dict'
122 persistable = None
123 storage = 'PickleStorage'
124 path = os.path.join(self.root, 'raw')
125 path = os.path.join(path, 'pickled_v' + str(dataId['visit']) + '_f' + dataId['filter'] + '.fits.gz')
126 if write or os.path.exists(path):
127 return dp.ButlerLocation(python, persistable, storage, path,
128 dataId, self, self.storage)
130 def bypass_raw(self, datasetType, pythonType, location, dataId):
131 return astropy.io.fits.open(location.getLocations()[0])
133 def query_raw(self, format, dataId):
134 return None
135 # results = set()
136 # return results
138 def getDefaultLevel(self):
139 return 'visit'
141 def getKeys(self, datasetType, level):
142 return {'filter': str, 'visit': int}
145class TestBasics(unittest.TestCase):
146 """Test case for basic functions of the repository classes."""
148 def setUp(self):
149 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestBasics-")
150 inputRepoArgs = dp.RepositoryArgs(root=os.path.join(ROOT, 'butlerAlias', 'data', 'input'),
151 mapper=ParentMapper,
152 tags='baArgs')
153 # mode of output repos is write-only by default
154 outputRepoArgs = dp.RepositoryArgs(root=os.path.join(self.testDir, 'repoA'),
155 mapper=ChildrenMapper,
156 mode='rw')
157 self.butler = dp.Butler(inputs=inputRepoArgs, outputs=outputRepoArgs)
159 def tearDown(self):
160 if os.path.exists(self.testDir):
161 shutil.rmtree(self.testDir)
162 if os.path.exists(os.path.join(ROOT, 'butlerAlias/repositoryCfg.yaml')):
163 os.remove(os.path.join(ROOT, 'butlerAlias/repositoryCfg.yaml'))
164 if hasattr(self, "butler"):
165 del self.butler
167 def testGet(self):
168 raw_image = self.butler.get('raw', {'visit': '2', 'filter': 'g'})
169 # in this case the width is known to be 1026:
170 self.assertEqual(raw_image[1].header["NAXIS1"], 1026)
172 def testGetUri(self):
173 raw_uri = self.butler.getUri('raw', {'visit': '2', 'filter': 'g'})
174 self.assertEqual(raw_uri,
175 os.path.join(ROOT, 'butlerAlias', 'data', 'input', 'raw', 'raw_v2_fg.fits.gz'))
176 self.assertEqual(os.path.isfile(raw_uri), True)
178 def testGetUriWrite(self):
179 raw_uri = self.butler.getUri('raw', {'visit': '9', 'filter': 'g'}, write=True)
180 self.assertEqual(raw_uri,
181 os.path.join(self.testDir, 'repoA', 'raw', 'raw_v9_fg.fits.gz'))
182 self.assertEqual(os.path.isfile(raw_uri), False)
184 def testSubset(self):
185 subset = self.butler.subset('raw')
186 self.assertEqual(len(subset), 3)
188 def testGetKeys(self):
189 keys = self.butler.getKeys('raw')
190 self.assertIn('filter', keys)
191 self.assertIn('visit', keys)
192 self.assertEqual(keys['filter'], str)
193 self.assertEqual(keys['visit'], int)
195 keys = self.butler.getKeys('raw', tag='baArgs')
196 self.assertIn('filter', keys)
197 self.assertIn('visit', keys)
198 self.assertEqual(keys['filter'], str)
199 self.assertEqual(keys['visit'], int)
201 keys = self.butler.getKeys('raw', tag=('baArgs', 'foo'))
202 self.assertIn('filter', keys)
203 self.assertIn('visit', keys)
204 self.assertEqual(keys['filter'], str)
205 self.assertEqual(keys['visit'], int)
207 keys = self.butler.getKeys('raw', tag='foo')
208 self.assertIsNone(keys)
210 keys = self.butler.getKeys('raw', tag=set(['baArgs', 'foo']))
211 self.assertIn('filter', keys)
212 self.assertIn('visit', keys)
213 self.assertEqual(keys['filter'], str)
214 self.assertEqual(keys['visit'], int)
216 def testGetDatasetTypes(self):
217 self.assertEqual(self.butler.getDatasetTypes(), {"raw", "str", "pickled"})
219 def testQueryMetadata(self):
220 val = self.butler.queryMetadata('raw', ('filter',))
221 val.sort()
222 self.assertEqual(val, ['g', 'r'])
224 val = self.butler.queryMetadata('raw', ('visit',))
225 val.sort()
226 self.assertEqual(val, [1, 2, 3])
228 val = self.butler.queryMetadata('raw', ('visit',), dataId={'filter': 'g'})
229 val.sort()
230 self.assertEqual(val, [1, 2])
232 val = self.butler.queryMetadata('raw', ('visit',), dataId={'filter': 'r'})
233 self.assertEqual(val, [3])
235 val = self.butler.queryMetadata('raw', ('filter',), dataId={'visit': 1})
236 self.assertEqual(val, ['g'])
238 val = self.butler.queryMetadata('raw', ('filter',), dataId={'visit': 2})
239 self.assertEqual(val, ['g'])
241 val = self.butler.queryMetadata('raw', ('filter',), dataId={'visit': 3})
242 self.assertEqual(val, ['r'])
244 val = self.butler.queryMetadata('raw', ('visit',), dataId={'filter': 'h'})
245 self.assertEqual(val, [])
247 dataId = dp.DataId({'filter': 'g'}, tag='baArgs')
248 val = self.butler.queryMetadata('raw', ('visit',), dataId={'filter': 'g'})
249 val.sort()
250 self.assertEqual(val, [1, 2])
252 dataId = dp.DataId({'filter': 'g'}, tag='foo')
253 val = self.butler.queryMetadata('raw', ('visit',), dataId=dataId)
254 self.assertEqual(val, [])
256 def testDatasetExists(self):
257 # test the values that are expected to be true:
258 self.assertEqual(self.butler.datasetExists('raw', {'filter': 'g', 'visit': 1}), True)
259 self.assertEqual(self.butler.datasetExists('raw', {'filter': 'g', 'visit': 2}), True)
260 self.assertEqual(self.butler.datasetExists('raw', {'filter': 'r', 'visit': 3}), True)
262 # test a few values that are expected to be false:
263 self.assertEqual(self.butler.datasetExists('raw', {'filter': 'f', 'visit': 1}), False)
264 self.assertEqual(self.butler.datasetExists('raw', {'filter': 'r', 'visit': 1}), False)
265 self.assertEqual(self.butler.datasetExists('raw', {'filter': 'g', 'visit': 3}), False)
267 # test that we don't see datasets in the input repo as being in the output repo
268 self.assertEqual(self.butler.datasetExists('raw', {'filter': 'g', 'visit': 1}, write=True), False)
269 self.assertEqual(self.butler.datasetExists('raw', {'filter': 'g', 'visit': 2}, write=True), False)
270 self.assertEqual(self.butler.datasetExists('raw', {'filter': 'r', 'visit': 3}, write=True), False)
272 # write a dataset to the output repo and check that we see it with both write=True and write=False
273 dataId = {'visit': '2', 'filter': 'g'}
274 self.butler.put({"number": 3}, 'pickled', dataId)
275 self.assertEqual(self.butler.datasetExists('pickled', dataId, write=True), True)
276 self.assertEqual(self.butler.datasetExists('pickled', dataId, write=False), True)
279##############################################################################################################
280##############################################################################################################
281##############################################################################################################
284class NoResultsMapper(dp.Mapper):
286 def __init__(self, root, **kwargs):
287 self.root = root
288 self.storage = dp.Storage.makeFromURI(self.root)
290 def __repr__(self):
291 return 'ParentMapper(root=%s)' % self.root
293 def map_list(self, dataId, write):
294 return []
296 def map_None(self, dataId, write):
297 return None
300class TestWriting(unittest.TestCase):
301 """A test case for the repository classes.
303 A test that
304 1. creates repo with a peer repo, writes to those repos.
305 2. reloads those output repos as a parents of new repos
306 * does a read from from the repo (verifies parent search)
307 3. writes to the new output repo and reloads it as a parent of a new repo
308 * verifies masking
309 4. reloads the repo from its persisted cfg
310 * verifies reload from cfg
311 """
312 def setUp(self):
313 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestWriting-")
315 def tearDown(self):
316 if os.path.exists(self.testDir):
317 shutil.rmtree(self.testDir)
318 # del self.butler
320 def testCreateAggregateAndLoadingAChild(self):
321 """Tests putting a very basic pickled object in a variety of Repository configuration settings
322 :return:
323 """
325 repoAArgs = dp.RepositoryArgs(mode='w',
326 root=os.path.join(self.testDir, 'repoA'),
327 mapper=MapperForTestWriting)
328 repoBArgs = dp.RepositoryArgs(mode='w',
329 root=os.path.join(self.testDir, 'repoB'),
330 mapper=MapperForTestWriting)
331 butlerAB = dp.Butler(outputs=[repoAArgs, repoBArgs])
333 objA = tstObj('abc')
334 butlerAB.put(objA, 'foo', {'val': 1})
335 objB = tstObj('def')
336 butlerAB.put(objB, 'foo', {'val': 2})
338 # create butlers where the output repos are now input repos
339 butlerC = dp.Butler(inputs=dp.RepositoryArgs(root=os.path.join(self.testDir, 'repoA')))
340 butlerD = dp.Butler(inputs=dp.RepositoryArgs(root=os.path.join(self.testDir, 'repoB')))
342 # # verify the objects exist by getting them
343 self.assertEqual(objA, butlerC.get('foo', {'val': 1}))
344 self.assertEqual(objA, butlerC.get('foo', {'val': 1}))
345 self.assertEqual(objB, butlerD.get('foo', {'val': 2}))
346 self.assertEqual(objB, butlerD.get('foo', {'val': 2}))
348 def testPutWithIncompleteDataId(self):
349 repoAArgs = dp.RepositoryArgs(mode='rw',
350 root=os.path.join(self.testDir, 'repoA'),
351 mapper=NoResultsMapper)
352 butler = dp.Butler(outputs=repoAArgs)
353 obj = tstObj('abc')
354 with self.assertRaises(RuntimeError):
355 butler.put(obj, 'list', {'visit': '2'})
356 with self.assertRaises(RuntimeError):
357 butler.put(obj, 'None', {'visit': '2'})
360class TestDiamondPattern(unittest.TestCase):
361 """A test case for when a butler is created with an output repository and two input repositories that have
362 a common parent; verifies that the parent's registry is loaded as the parentRegistry of the output
363 repository.
364 """
365 @staticmethod
366 def makeRegistry(location, name):
367 conn = sqlite3.connect(location)
368 conn.execute("CREATE TABLE {name} (real)".format(name=name))
369 conn.commit()
370 conn.close()
372 def setUp(self):
373 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestDiamondPattern-")
375 def tearDown(self):
376 if os.path.exists(self.testDir):
377 shutil.rmtree(self.testDir)
379 def test(self):
380 repoARoot = os.path.join(self.testDir, 'repoA')
381 butler = dp.Butler(outputs={'root': repoARoot, 'mapper': ParentRepoTestMapper})
382 self.makeRegistry(os.path.join(repoARoot, 'registry.sqlite3'), 'repoA')
383 del butler
385 repoBRoot = os.path.join(self.testDir, 'repoB')
386 butlerB = dp.Butler(inputs=repoARoot, outputs=repoBRoot)
387 del butlerB
389 repoCRoot = os.path.join(self.testDir, 'repoC')
390 butlerC = dp.Butler(inputs=repoARoot, outputs=repoCRoot)
391 del butlerC
393 repoDRoot = os.path.join(self.testDir, 'repoD')
394 butlerD = dp.Butler(inputs=(repoBRoot, repoCRoot), outputs=repoDRoot)
396 self.assertEqual(1, len(butlerD._repos.outputs()))
397 self.assertEqual(os.path.dirname(butlerD._repos.outputs()[0].parentRegistry.root), repoARoot)
400class TestMasking(unittest.TestCase):
401 """A test case for the repository classes.
403 A test that
404 1. creates a repo, does a put
405 2. creates a new butler, uses that repo as an input and creates a new read-write output repo
406 3. gets from the input repo, modifies the dataset, and puts into the output repo
407 4. does a get and verifies that the changed dataset is returned.
408 """
409 def setUp(self):
410 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestMasking-")
412 def tearDown(self):
413 if os.path.exists(self.testDir):
414 shutil.rmtree(self.testDir)
416 def test(self):
417 repoAArgs = dp.RepositoryArgs(mode='w',
418 root=os.path.join(self.testDir, 'repoA'),
419 mapper=MapperForTestWriting)
420 butler = dp.Butler(outputs=repoAArgs)
421 obj0 = tstObj('abc')
422 butler.put(obj0, 'foo', {'bar': 1})
423 del butler
424 del repoAArgs
426 repoBArgs = dp.RepositoryArgs(mode='rw',
427 root=os.path.join(self.testDir, 'repoB'),
428 mapper=MapperForTestWriting)
429 butler = dp.Butler(inputs=os.path.join(self.testDir, 'repoA'), outputs=repoBArgs)
430 obj1 = butler.get('foo', {'bar': 1})
431 self.assertEqual(obj0, obj1)
432 obj1.data = "def"
433 butler.put(obj1, 'foo', {'bar': 1})
434 obj2 = butler.get('foo', {'bar': 1})
435 self.assertEqual(obj1, obj2)
438class TestMultipleOutputsPut(unittest.TestCase):
439 """A test case for the repository classes.
441 A test that
442 1. creates 3 peer repositories and readers for them
443 2. does a single put
444 3. verifies that all repos received the put
445 """
446 def setUp(self):
447 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestMultipleOutputsPut-")
449 def tearDown(self):
450 if os.path.exists(self.testDir):
451 shutil.rmtree(self.testDir)
453 def test(self):
454 repoAArgs = dp.RepositoryArgs(root=os.path.join(self.testDir, 'repoA'),
455 mapper=MapperForTestWriting)
456 repoBArgs = dp.RepositoryArgs(root=os.path.join(self.testDir, 'repoB'),
457 mapper=MapperForTestWriting)
459 butler = dp.Butler(outputs=(repoAArgs, repoBArgs))
460 obj0 = tstObj('abc')
461 butler.put(obj0, 'foo', {'bar': 1})
463 for root in (os.path.join(self.testDir, 'repoA'),
464 os.path.join(self.testDir, 'repoB')):
465 butler = dp.Butler(inputs=root)
466 self.assertEqual(butler.get('foo', {'bar': 1}), obj0)
469class TestMultipleInputs(unittest.TestCase):
470 """A test case for the repository classes.
472 A test that
473 - create output 2 repos, write same & different data to them & close them
474 - create a new butler with those repos as inputs
475 - read data that was written to both repos:
476 - verify data that was written to only one of each repo
477 - verify dissimilar datasets with same id that were written to the repos
479 """
480 def setUp(self):
481 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestMapperInference-")
483 def tearDown(self):
484 if os.path.exists(self.testDir):
485 shutil.rmtree(self.testDir)
487 def test(self):
488 objAbc = tstObj('abc')
489 objDef = tstObj('def')
490 objGhi = tstObj('ghi')
491 objJkl = tstObj('jkl')
493 repoAArgs = dp.RepositoryArgs(mode='w',
494 root=os.path.join(self.testDir, 'repoA'),
495 mapper=MapperForTestWriting)
496 butler = dp.Butler(outputs=repoAArgs)
497 butler.put(objAbc, 'foo', {'bar': 1})
498 butler.put(objDef, 'foo', {'bar': 2})
500 repoBArgs = dp.RepositoryArgs(mode='w',
501 root=os.path.join(self.testDir, 'repoB'),
502 mapper=MapperForTestWriting)
503 butler = dp.Butler(outputs=repoBArgs)
504 butler.put(objGhi, 'foo', {'bar': 1}) # note different object with overlapping dataId with repoA
505 butler.put(objJkl, 'foo', {'bar': 3})
507 # note repo order: A, B
508 butler = dp.Butler(inputs=(os.path.join(self.testDir, 'repoA'),
509 os.path.join(self.testDir, 'repoB')))
510 self.assertEqual(butler.get('foo', {'bar': 1}), objAbc)
511 self.assertEqual(butler.get('foo', {'bar': 2}), objDef)
512 self.assertEqual(butler.get('foo', {'bar': 3}), objJkl)
514 # note reversed repo order: B, A
515 butler = dp.Butler(inputs=(os.path.join(self.testDir, 'repoB'),
516 os.path.join(self.testDir, 'repoA')))
517 self.assertEqual(butler.get('foo', {'bar': 1}), objGhi)
518 self.assertEqual(butler.get('foo', {'bar': 2}), objDef)
519 self.assertEqual(butler.get('foo', {'bar': 3}), objJkl)
522class TestTagging(unittest.TestCase):
523 """A test case for the tagging of repository classes.
524 """
526 def setUp(self):
527 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestTagging-")
529 def tearDown(self):
530 if os.path.exists(self.testDir):
531 shutil.rmtree(self.testDir)
533 def testOneLevelInputs(self):
534 """
535 1. put an object with the same ID but slightly different value into 2 repositories.
536 2. use those repositories as inputs to a butler, and tag them
537 3. make sure that the correct object is gotten for each of
538 a. one tag
539 b. the other tag
540 c. no tag
541 4. repeat step 3 but reverse the order of input cfgs to a new butler.
542 5. use the butler from step 4 and write an output. The inputs will get recorded as parents of the
543 output repo.
544 6. create a new butler with a new overlapping repo, and verify that objects can be gotten from the
545 other's parent repos via tagging.
546 """
547 objA = tstObj('a')
548 objB = tstObj('b')
550 # put objA in repo1:
551 repo1Args = dp.RepositoryArgs(mode='rw',
552 root=os.path.join(self.testDir, 'repo1'),
553 mapper=MapperForTestWriting)
554 butler = dp.Butler(outputs=repo1Args)
555 butler.put(objA, 'foo', {'bar': 1})
556 del butler
558 # put objB in repo2:
559 repo2Args = dp.RepositoryArgs(mode='rw',
560 root=os.path.join(self.testDir, 'repo2'),
561 mapper=MapperForTestWriting)
562 butler = dp.Butler(outputs=repo2Args)
563 butler.put(objB, 'foo', {'bar': 1})
564 del butler
565 del repo1Args
566 del repo2Args
568 # make the objects inputs of repos
569 # and verify the correct object can ge fetched using the tag and not using the tag
571 repo1Args = dp.RepositoryArgs(root=os.path.join(self.testDir, 'repo1'), tags='one')
572 repo2Args = dp.RepositoryArgs(root=os.path.join(self.testDir, 'repo2'), tags='two')
574 butler = dp.Butler(inputs=(repo1Args, repo2Args))
575 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='one')), objA)
576 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='two')), objB)
577 self.assertEqual(butler.get('foo', {'bar': 1}), objA)
579 butler = dp.Butler(inputs=(repo2Args, repo1Args))
580 self.assertEqual(butler.get('foo', dp.DataId(bar=1, tag='one')), objA)
581 self.assertEqual(butler.get('foo', dp.DataId(bar=1, tag='two')), objB)
582 self.assertEqual(butler.get('foo', dp.DataId(bar=1)), objB)
584 # create butler with repo1 and repo2 as parents, and an output repo3.
585 repo3Args = dp.RepositoryArgs(mode='rw',
586 root=os.path.join(self.testDir, 'repo3'),
587 mapper=MapperForTestWriting)
588 butler = dp.Butler(inputs=(repo1Args, repo2Args), outputs=repo3Args)
589 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='one')), objA)
590 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='two')), objB)
591 self.assertEqual(butler.get('foo', {'bar': 1}), objA)
592 # add an object to the output repo. note since the output repo mode is 'rw' that object is gettable
593 # and it has first priority in search order. Other repos should be searchable by tagging.
594 objC = tstObj('c')
595 butler.put(objC, 'foo', {'bar': 1})
596 self.assertEqual(butler.get('foo', {'bar': 1}), objC)
597 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='one')), objA)
598 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='two')), objB)
599 del butler
601 repo3Cfg = dp.Storage().getRepositoryCfg(os.path.join(self.testDir, 'repo3'))
602 self.assertEqual(repo3Cfg.parents, [os.path.join(self.testDir, 'repo1'),
603 os.path.join(self.testDir, 'repo2')])
605 # expand the structure to look like this:
606 # ┌────────────────────────┐ ┌────────────────────────┐
607 # │repo1 │ │repo2 │
608 # │ tag:"one" │ │ tag:"two" │
609 # │ tstObj('a') │ │ tstObj('b') │
610 # │ at ('foo', {'bar:1'})│ │ at ('foo', {'bar:1'})│
611 # └───────────┬────────────┘ └───────────┬────────────┘
612 # └─────────────┬────────────┘
613 # ┌────────────┴───────────┐ ┌────────────────────────┐
614 # │repo4 │ │repo5 │
615 # │ tag:"four" │ │ tag:"five" │
616 # │ tstObj('d') │ │ tstObj('e') │
617 # │ at ('foo', {'bar:2'})│ │ at ('foo', {'bar:1'})│
618 # └───────────┬────────────┘ └───────────┬────────────┘
619 # └─────────────┬────────────┘
620 # ┌──┴───┐
621 # │butler│
622 # └──────┘
624 repo4Args = dp.RepositoryArgs(mode='rw',
625 root=os.path.join(self.testDir, 'repo4'),
626 mapper=MapperForTestWriting)
627 butler = dp.Butler(inputs=(os.path.join(self.testDir, 'repo1'),
628 os.path.join(self.testDir, 'repo2')), outputs=repo4Args)
629 objD = tstObj('d')
630 butler.put(objD, 'foo', {'bar': 2})
631 del butler
633 repo5Cfg = dp.RepositoryArgs(mode='rw',
634 root=os.path.join(self.testDir, 'repo5'),
635 mapper=MapperForTestWriting)
636 butler = dp.Butler(outputs=repo5Cfg)
637 objE = tstObj('e')
638 butler.put(objE, 'foo', {'bar': 1})
639 del butler
641 repo4Args = dp.RepositoryArgs(cfgRoot=os.path.join(self.testDir, 'repo4'), tags='four')
642 repo5Args = dp.RepositoryArgs(cfgRoot=os.path.join(self.testDir, 'repo5'), tags='five')
643 butler = dp.Butler(inputs=(repo4Args, repo5Args))
644 self.assertEqual(butler.get('foo', {'bar': 1}), objA)
645 self.assertEqual(butler.get('foo', {'bar': 2}), objD)
646 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='four')), objA)
647 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='five')), objE)
648 del butler
650 butler = dp.Butler(inputs=(repo5Args, repo4Args))
651 self.assertEqual(butler.get('foo', {'bar': 1}), objE)
652 self.assertEqual(butler.get('foo', {'bar': 2}), objD)
653 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='four')), objA)
654 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='five')), objE)
655 del butler
658class TestMapperInference(unittest.TestCase):
659 """A test for inferring mapper in the cfg from parent cfgs"""
661 def setUp(self):
662 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestMapperInference-")
664 def tearDown(self):
665 if os.path.exists(self.testDir):
666 shutil.rmtree(self.testDir)
668 def testSingleParent(self):
669 """
670 creates a repo that:
671 a. does not have a mapper specified in the cfg
672 b. has a parent that does have a mapper specified in the cfg
673 verifies that the child repo inherits the parent's mapper via inference.
674 """
675 repoACfg = dp.RepositoryCfg(root=os.path.join(self.testDir, 'repoA'),
676 mapper=MapperForTestWriting,
677 mapperArgs=None,
678 parents=None,
679 policy=None)
680 dp.Storage.putRepositoryCfg(repoACfg, os.path.join(self.testDir, 'repoA'))
682 repoBArgs = dp.RepositoryArgs(mode='rw',
683 root=os.path.join(self.testDir, 'repoB'))
684 butler = dp.Butler(inputs=os.path.join(self.testDir, 'repoA'), outputs=repoBArgs)
685 self.assertIsInstance(butler._repos.outputs()[0].repo._mapper, MapperForTestWriting)
687 def testMultipleParentsSameMapper(self):
688 """
689 creates a repo that:
690 a. does not have a mapper specified in the cfg
691 b. has 2 parents that do have the same mapper specified in the cfg
692 verifies that the child repo inherits the parent's mapper via inference.
694 """
695 repoACfg = dp.RepositoryCfg(root=os.path.join(self.testDir, 'repoA'),
696 mapper=MapperForTestWriting,
697 mapperArgs=None,
698 parents=None,
699 policy=None,)
700 repoBCfg = dp.RepositoryCfg(root=os.path.join(self.testDir, 'repoB'),
701 mapper=MapperForTestWriting,
702 mapperArgs=None,
703 parents=None,
704 policy=None)
705 dp.Storage.putRepositoryCfg(repoACfg, os.path.join(self.testDir, 'repoA'))
706 dp.Storage.putRepositoryCfg(repoBCfg, os.path.join(self.testDir, 'repoB'))
708 repoCArgs = dp.RepositoryArgs(mode='w',
709 root=os.path.join(self.testDir, 'repoC'))
711 butler = dp.Butler(inputs=(os.path.join(self.testDir, 'repoA'),
712 os.path.join(self.testDir, 'repoB')),
713 outputs=repoCArgs)
714 self.assertIsInstance(butler._repos.outputs()[0].repo._mapper, MapperForTestWriting)
716 def testMultipleParentsDifferentMappers(self):
717 """
718 creates a repo that:
719 a. does not have a mapper specified in the cfg
720 b. has 2 parent repos that have different mappers
721 verifies that the constructor raises a RuntimeError because the mapper can not be inferred.
722 """
723 repoACfg = dp.RepositoryCfg(root=os.path.join(self.testDir, 'repoA'),
724 mapper=MapperForTestWriting,
725 mapperArgs=None,
726 parents=None,
727 policy=None)
728 repoBCfg = dp.RepositoryCfg(root=os.path.join(self.testDir, 'repoB'),
729 mapper='lsst.daf.persistence.test.EmptyTestMapper',
730 mapperArgs=None,
731 parents=None,
732 policy=None)
733 dp.Storage.putRepositoryCfg(repoACfg, os.path.join(self.testDir, 'repoA'))
734 dp.Storage.putRepositoryCfg(repoBCfg, os.path.join(self.testDir, 'repoB'))
736 repoCArgs = dp.RepositoryArgs(mode='w',
737 root=os.path.join(self.testDir, 'repoC'))
738 self.assertRaises(RuntimeError,
739 dp.Butler,
740 inputs=(os.path.join(self.testDir, 'repoA'),
741 os.path.join(self.testDir, 'repoB')),
742 outputs=repoCArgs)
745class TestMovedRepositoryCfg(unittest.TestCase):
746 """Test if a repository cfg is in-place (resides at root of the repository) and the cfg is moved, the
747 deserialized cfg root should be the new location if the repository is moved.
748 """
750 def setUp(self):
751 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestMovedRepositoryCfg-")
753 def tearDown(self):
754 if os.path.exists(self.testDir):
755 shutil.rmtree(self.testDir)
757 def test(self):
758 butler = dp.Butler(outputs=dp.RepositoryArgs(root=os.path.join(self.testDir, 'a'),
759 mapper=MapperForTestWriting))
760 del butler
761 os.makedirs(os.path.join(self.testDir, 'b'))
762 os.rename(os.path.join(self.testDir, 'a/repositoryCfg.yaml'),
763 os.path.join(self.testDir, 'b/repositoryCfg.yaml'))
764 butler = dp.Butler(inputs=os.path.join(self.testDir, 'b'))
765 self.assertEqual(list(butler._repos.inputs())[0].cfg,
766 dp.RepositoryCfg(root=os.path.join(self.testDir, 'b'),
767 mapper=MapperForTestWriting,
768 mapperArgs=None,
769 parents=None,
770 policy=None))
773class TestOutputAlreadyHasParent(unittest.TestCase):
775 def setUp(self):
776 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestOutputAlreadyHasParent-")
778 def tearDown(self):
779 if os.path.exists(self.testDir):
780 shutil.rmtree(self.testDir)
782 def test(self):
783 # create a repo where repo 'a' is a parent of repo 'b'
784 butler = dp.Butler(outputs=dp.RepositoryArgs(root=os.path.join(self.testDir, 'a'),
785 mapper=MapperForTestWriting))
786 del butler
787 butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'),
788 outputs=os.path.join(self.testDir, 'b'))
789 self.assertEqual(len(butler._repos.inputs()), 1)
790 self.assertEqual(butler._repos.inputs()[0].cfg.root,
791 os.path.join(self.testDir, 'a'))
792 self.assertEqual(len(butler._repos.outputs()), 1)
793 self.assertEqual(butler._repos.outputs()[0].cfg.root,
794 os.path.join(self.testDir, 'b'))
795 del butler
797 # load that repo a few times, include 'a' as an input.
798 for i in range(4):
799 butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'),
800 outputs=dp.RepositoryArgs(root=os.path.join(self.testDir, 'b'),
801 mode='rw'))
802 self.assertEqual(len(butler._repos.inputs()), 2)
803 self.assertEqual(butler._repos.inputs()[0].cfg.root,
804 os.path.join(self.testDir, 'b'))
805 self.assertEqual(butler._repos.inputs()[1].cfg.root,
806 os.path.join(self.testDir, 'a'))
807 self.assertEqual(len(butler._repos.outputs()), 1)
808 self.assertEqual(butler._repos.outputs()[0].cfg.root,
809 os.path.join(self.testDir, 'b'))
810 cfg = dp.Storage().getRepositoryCfg(os.path.join(self.testDir, 'b'))
811 self.assertEqual(cfg, dp.RepositoryCfg(root=os.path.join(self.testDir, 'b'),
812 mapper=MapperForTestWriting,
813 mapperArgs=None,
814 parents=[os.path.join(self.testDir, 'a')],
815 policy=None))
817 # load the repo a few times and don't explicitly list 'a' as an input
818 for i in range(4):
819 butler = dp.Butler(outputs=dp.RepositoryArgs(root=os.path.join(self.testDir, 'b'),
820 mode='rw'))
821 self.assertEqual(len(butler._repos.inputs()), 2)
822 self.assertEqual(butler._repos.inputs()[0].cfg.root,
823 os.path.join(self.testDir, 'b'))
824 self.assertEqual(butler._repos.inputs()[1].cfg.root,
825 os.path.join(self.testDir, 'a'))
826 self.assertEqual(len(butler._repos.outputs()), 1)
827 self.assertEqual(butler._repos.outputs()[0].cfg.root,
828 os.path.join(self.testDir, 'b'))
829 cfg = dp.Storage().getRepositoryCfg(os.path.join(self.testDir, 'b'))
830 self.assertEqual(cfg, dp.RepositoryCfg(root=os.path.join(self.testDir, 'b'),
831 mapper=MapperForTestWriting,
832 mapperArgs=None,
833 parents=[os.path.join(self.testDir, 'a')],
834 policy=None))
836 # load 'b' as 'write only' and don't list 'a' as an input. This should raise, because inputs must
837 # match readable outputs parents.
838 with self.assertRaises(RuntimeError):
839 butler = dp.Butler(outputs=os.path.join(self.testDir, 'b'))
841 # load 'b' as 'write only' and explicitly list 'a' as an input.
842 butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'),
843 outputs=os.path.join(self.testDir, 'b'))
844 self.assertEqual(len(butler._repos.inputs()), 1)
845 self.assertEqual(len(butler._repos.outputs()), 1)
846 self.assertEqual(butler._repos.inputs()[0].cfg.root,
847 os.path.join(self.testDir, 'a'))
848 self.assertEqual(butler._repos.outputs()[0].cfg.root,
849 os.path.join(self.testDir, 'b'))
850 cfg = dp.Storage().getRepositoryCfg(os.path.join(self.testDir, 'b'))
853class ParentRepoTestMapper(dp.Mapper):
854 def __init__(self, parentRegistry, repositoryCfg, **kwargs):
855 self.parentRegistry = parentRegistry
856 root = repositoryCfg.root
857 if os.path.exists(os.path.join(root, 'registry.sqlite3')):
858 self.registry = dp.Registry.create(os.path.join(root, 'registry.sqlite3'))
859 else:
860 self.registry = None
862 def getRegistry(self):
863 return self.registry
866class TestParentRepository(unittest.TestCase):
867 """A test to verify that when a parent repository is used the correct one is found."""
869 def setUp(self):
870 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestParentRepository-")
872 def tearDown(self):
873 if os.path.exists(self.testDir):
874 shutil.rmtree(self.testDir, True)
876 def test(self):
878 """Tests 1. That an sqlite registry in a parent repo is used as the
879 registry in a child repo that does not have its own sqlite registry.
880 2. That when a repo has a parent and grandparent and only the
881 grandparent has an sqlite registry that the grandparent's registry is
882 used.
883 3. That when the parent and grandparent both have sqlite registries
884 that the parent's sqlite registry is used."""
885 # setup; create the parent repo
886 def makeRegistry(location, name):
887 conn = sqlite3.connect(location)
888 conn.execute("CREATE TABLE {name} (real)".format(name=name))
889 conn.commit()
890 conn.close()
892 repoADir = os.path.join(self.testDir, 'repoA')
893 repoBDir = os.path.join(self.testDir, 'repoB')
894 repoCDir = os.path.join(self.testDir, 'repoC')
895 butler = dp.Butler(outputs={'root': repoADir, 'mapper': ParentRepoTestMapper})
896 del butler
897 makeRegistry(os.path.join(repoADir, 'registry.sqlite3'), 'repoA')
898 # test 1:
899 butler = dp.Butler(inputs=repoADir, outputs=repoBDir)
900 registry = butler._repos.outputs()[0].repo._mapper.parentRegistry
901 self.assertIsInstance(registry, dp.SqliteRegistry)
902 tables = registry.conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()
903 self.assertEqual(tables, [('repoA', )])
904 del butler
905 # test 2:
906 butler = dp.Butler(inputs=repoBDir, outputs=repoCDir)
907 registry = butler._repos.outputs()[0].repo._mapper.parentRegistry
908 self.assertIsInstance(registry, dp.SqliteRegistry)
909 tables = registry.conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()
910 self.assertEqual(tables, [('repoA', )])
911 del butler
912 # test 3:
913 makeRegistry(os.path.join(repoBDir, 'registry.sqlite3'), 'repoB')
914 butler = dp.Butler(inputs=repoBDir, outputs=repoCDir)
915 registry = butler._repos.outputs()[0].repo._mapper.parentRegistry
916 self.assertIsInstance(registry, dp.SqliteRegistry)
917 tables = registry.conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall()
918 self.assertEqual(tables, [('repoB', )])
921class TestOldButlerParent(unittest.TestCase):
922 """A test to verify that when a parent is an old butler repo that it still gets loaded correctly,
923 including mapperArgs."""
925 def setUp(self):
926 """Create temp test dir and create an OldButler repo at 'repoA'"""
927 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestOldButlerParent-")
928 self.repoADir = os.path.join(self.testDir, 'repoA')
929 os.makedirs(self.repoADir)
930 with open(os.path.join(self.repoADir, '_mapper'), 'w') as mapperFile:
931 mapperFile.write('lsst.daf.persistence.test.EmptyTestMapper')
933 def tearDown(self):
934 if os.path.exists(self.testDir):
935 shutil.rmtree(self.testDir, True)
937 def testOldButlerCfgRecordedInOutputs(self):
938 """Verify that when an Old Butler is used as an input with parameters such as mapperArgs, that those
939 parameters get recalled correctly (in the RepositoryCfg nested in the parents list of the output repo)
940 """
941 repoBDir = os.path.join(self.testDir, 'repoB')
942 # create a 'repoB' with the Old Butler 'repoA' as a parent, specify mapperArgs for 'repoA', and
943 # verify that the mapperArgs got passed to the repoA mapper.
944 butler = dp.Butler(inputs={'root': self.repoADir,
945 'mapperArgs': {'foo': 1}},
946 outputs={'root': repoBDir,
947 'mapperArgs': {'foo': 1},
948 'mode': 'rw'})
949 self.assertEqual(butler._repos.inputs()[0].repo._mapper.kwargs, {'foo': 1})
950 # init a Butler again, with exactly the same configuration as before.
951 butler = dp.Butler(inputs={'root': self.repoADir,
952 'mapperArgs': {'foo': 1}},
953 outputs={'root': repoBDir,
954 'mapperArgs': {'foo': 1},
955 'mode': 'rw'})
956 self.assertEqual(butler._repos.inputs()[0].repo._mapper.kwargs, {'foo': 1})
957 # try initializing the butler again with the 'repoB' output, but do not specify the mapperArgs in the
958 # repoA args. this should raise a runtime error because the input speficied this way does not match
959 # the cfg recorded in 'repoB'.
960 with self.assertRaises(RuntimeError):
961 butler = dp.Butler(inputs=self.repoADir,
962 outputs=repoBDir)
963 # initializing the butler, but only specifiying the root of 'repoB' (the rest of the cfg will be
964 # loaded from its `repositoryCfg` file), but fully specify the input as required.
965 butler = dp.Butler(inputs={'root': self.repoADir,
966 'mapperArgs': {'foo': 1}},
967 outputs=repoBDir)
968 self.assertEqual(butler._repos.inputs()[0].repo._mapper.kwargs, {'foo': 1})
969 # use 'repoB' as an input, and verify that 'repoA' is loaded, including the mapperArgs.
970 butler = dp.Butler(inputs=repoBDir)
971 self.assertEqual(butler._repos.inputs()[1].repo._mapper.kwargs, {'foo': 1})
974class TestOldButlerParentTagging(unittest.TestCase):
975 """A test to verify that when a parent is an old butler repo that its tagging gets loaded correctly"""
977 def setUp(self):
978 """Remove testDir from any previous runs and create an OldButler repo at repoA and an Old Butler repoB
979 with a _parent symlink to repoA.
980 """
981 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestOldButlerParentTagging-")
982 self.repoADir = os.path.join(self.testDir, 'repoA')
983 os.makedirs(self.repoADir)
984 with open(os.path.join(self.repoADir, '_mapper'), 'w') as mapperFile:
985 mapperFile.write('lsst.daf.persistence.test.MapperForTestWriting')
986 self.repoBDir = os.path.join(self.testDir, 'repoB')
987 os.makedirs(self.repoBDir)
988 os.symlink(self.repoADir, os.path.join(self.repoBDir, '_parent'))
990 def tearDown(self):
991 if os.path.exists(self.testDir):
992 shutil.rmtree(self.testDir, True)
994 def test(self):
995 """Verify that the tags on a repository with an Old Butler repository parent are applied to that
996 parent
997 """
998 # put objA in repoA:
999 objA = tstObj('a')
1000 butler = dp.Butler(outputs=self.repoADir)
1001 butler.put(objA, 'foo', {'bar': 1})
1002 del butler
1004 # create repoB and put objB in it:
1005 objB = tstObj('b')
1006 butler = dp.Butler(inputs=self.repoADir, outputs=self.repoBDir)
1007 butler.put(objB, 'foo', {'bar': 2})
1008 del butler
1010 # verify that repoB can be used as an input, and objA and objB can be gotten from it:
1011 butler = dp.Butler(inputs=self.repoBDir)
1012 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1})), objA)
1013 self.assertEqual(butler.get('foo', dp.DataId({'bar': 2})), objB)
1014 del butler
1016 # apply a tag and verify that repoB can still be used as an input, and both objA (in repoA) and objB
1017 # can be gotten from it:
1018 butler = dp.Butler(inputs={'root': self.repoBDir, 'tags': 'baz'})
1019 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='baz')), objA)
1020 self.assertEqual(butler.get('foo', dp.DataId({'bar': 2}, tag='baz')), objB)
1021 del butler
1023 # create a New Butler repoC and put objC in it:
1024 objC = tstObj('c')
1025 repoCDir = os.path.join(self.testDir, 'repoC')
1026 butler = dp.Butler(inputs=self.repoBDir, outputs=repoCDir)
1027 butler.put(objC, 'foo', {'bar': 3})
1028 del butler
1030 # verify that repoC can be used as an input, and objA, objB, and objC can be gotten from it:
1031 butler = dp.Butler(inputs=repoCDir)
1032 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1})), objA)
1033 self.assertEqual(butler.get('foo', dp.DataId({'bar': 2})), objB)
1034 self.assertEqual(butler.get('foo', dp.DataId({'bar': 3})), objC)
1035 del butler
1037 # apply a tag and verify that repoC can be used as an input, and objA, objB, and objC can be gotten
1038 # from it:
1039 butler = dp.Butler(inputs={'root': repoCDir, 'tags': 'baz'})
1040 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='baz')), objA)
1041 self.assertEqual(butler.get('foo', dp.DataId({'bar': 2}, tag='baz')), objB)
1042 self.assertEqual(butler.get('foo', dp.DataId({'bar': 3}, tag='baz')), objC)
1043 del butler
1046class MemoryTester(lsst.utils.tests.MemoryTestCase):
1047 pass
1050if __name__ == '__main__': 1050 ↛ 1051line 1050 didn't jump to line 1051, because the condition on line 1050 was never true
1051 lsst.utils.tests.init()
1052 unittest.main()