lsst.obs.base  18.1.0-16-g069e110+1
ingest_tests.py
Go to the documentation of this file.
1 # This file is part of obs_subaru.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """Base class for writing Gen3 raw data ingest tests.
23 """
24 
25 __all__ = ("IngestTestBase",)
26 
27 import abc
28 import tempfile
29 import os
30 import shutil
31 
32 from lsst.daf.butler import Butler
33 from lsst.obs.base.gen3 import RawIngestTask
34 
35 
36 class IngestTestBase(metaclass=abc.ABCMeta):
37  """Base class for tests of gen3 ingest. Subclass from this, then
38  `unittest.TestCase` to get a working test suite.
39  """
40 
41  ingestDir = ""
42  """Root path to ingest files into. Typically `obs_package/tests/`; the
43  actual directory will be a tempdir under this one.
44  """
45 
46  instrument = None
47  """The instrument to be registered and tested."""
48 
49  dataId = {}
50  """Butler data ID of a file to ingest when testing."""
51 
52  file = ""
53  """Full path to a file to ingest in tests."""
54 
55  def setUp(self):
56  # Use a temporary working directory
57  self.root = tempfile.mkdtemp(dir=self.ingestDir)
58  Butler.makeRepo(self.root)
59  self.butler = Butler(self.root, run="raw")
60 
61  # Register the instrument and its static metadata
62  self.instrument.register(self.butler.registry)
63 
64  # Make a default config for test methods to play with
65  self.config = RawIngestTask.ConfigClass()
66  self.config.onError = "break"
67 
68  def tearDown(self):
69  if os.path.exists(self.root):
70  shutil.rmtree(self.root, ignore_errors=True)
71 
72  def runIngest(self, files=None):
73  """
74  Initialize and run RawIngestTask on a list of files.
75 
76  Parameters
77  ----------
78  files : `list`, [`str`], or None
79  List of files to be ingested, or None to use ``self.file``
80  """
81  if files is None:
82  files = [self.file]
83  task = RawIngestTask(config=self.config, butler=self.butler)
84  task.log.setLevel(task.log.FATAL) # silence logs, since we expect a lot of warnings
85  task.run(files)
86 
87  def runIngestTest(self, files=None):
88  """
89  Test that RawIngestTask ingested the expected files.
90 
91  Parameters
92  ----------
93  files : `list`, [`str`], or None
94  List of files to be ingested, or None to use ``self.file``
95  """
96  self.runIngest(files)
97  exposure = self.butler.get("raw", self.dataId)
98  metadata = self.butler.get("raw.metadata", self.dataId)
99  image = self.butler.get("raw.image", self.dataId)
100  self.assertImagesEqual(exposure.image, image)
101  self.assertEqual(metadata.toDict(), exposure.getMetadata().toDict())
102 
103  def testSymLink(self):
104  self.config.transfer = "symlink"
105  self.runIngestTest()
106 
107  def testCopy(self):
108  self.config.transfer = "copy"
109  self.runIngestTest()
110 
111  def testHardLink(self):
112  self.config.transfer = "hardlink"
113  self.runIngestTest()
114 
115  def testInPlace(self):
116  """Test that files already in the directory can be added to the
117  registry in-place.
118  """
119  # hardlink into repo root manually
120  newPath = os.path.join(self.butler.datastore.root, os.path.basename(self.file))
121  os.link(self.file, newPath)
122  self.config.transfer = None
123  self.runIngestTest([newPath])
124 
126  """Re-ingesting the same data into the repository should fail, if
127  configured to do so.
128  """
129  self.config.transfer = "symlink"
130  self.config.conflict = "fail"
131  self.runIngest()
132  with self.assertRaises(Exception):
133  self.runIngest()
134 
136  """Re-ingesting the same data into the repository does not fail, if
137  configured to ignore conflict errors.
138  """
139  self.config.transfer = "symlink"
140  self.config.conflict = "ignore"
141  self.runIngest() # this one should succeed
142  n1, = self.butler.registry.query("SELECT COUNT(*) FROM Dataset")
143  self.runIngest() # this one should silently fail
144  n2, = self.butler.registry.query("SELECT COUNT(*) FROM Dataset")
145  self.assertEqual(n1, n2)
146 
148  """Re-ingesting the same data will be put into a different collection,
149  if configured to do so.
150  """
151  self.config.transfer = "symlink"
152  self.config.conflict = "ignore"
153  self.config.stash = "stash"
154  self.runIngest() # this one should write to 'raw'
155  self.runIngest() # this one should write to 'stash'
156  dt = self.butler.registry.getDatasetType("raw.metadata")
157  ref1 = self.butler.registry.find(self.butler.collection, dt, self.dataId)
158  ref2 = self.butler.registry.find("stash", dt, self.dataId)
159  self.assertNotEqual(ref1.id, ref2.id)
160  self.assertEqual(self.butler.get(ref1).toDict(), self.butler.getDirect(ref2).toDict())
161 
162  def testOnErrorBreak(self):
163  """Test that errors do not roll back success, when configured to do so.
164 
165  Failing to ingest a nonexistent file after ingesting the valid one should
166  leave the valid one in the registry, despite raising an exception.
167  """
168  self.config.transfer = "symlink"
169  self.config.onError = "break"
170  with self.assertRaises(Exception):
171  self.runIngest(files=[self.file, "nonexistent.fits"])
172  dt = self.butler.registry.getDatasetType("raw.metadata")
173  self.assertIsNotNone(self.butler.registry.find(self.butler.collection, dt, self.dataId))
174 
176  """Failing to ingest nonexistent files before and after ingesting the
177  valid one should leave the valid one in the registry and not raise
178  an exception.
179  """
180  self.config.transfer = "symlink"
181  self.config.onError = "continue"
182  self.runIngest(files=["nonexistent.fits", self.file, "still-not-here.fits"])
183  dt = self.butler.registry.getDatasetType("raw.metadata")
184  self.assertIsNotNone(self.butler.registry.find(self.butler.collection, dt, self.dataId))
185 
187  """Failing to ingest nonexistent files after ingesting the
188  valid one should leave the registry unchanged.
189  """
190  self.config.transfer = "symlink"
191  self.config.onError = "rollback"
192  with self.assertRaises(Exception):
193  self.runIngest(file=[self.file, "nonexistent.fits"])
194  try:
195  dt = self.butler.registry.getDatasetType("raw.metadata")
196  except KeyError:
197  # If we also rollback registering the DatasetType, that's fine,
198  # but not required.
199  pass
200  else:
201  self.assertIsNotNone(self.butler.registry.find(self.butler.collection, dt, self.dataId))