22 """Base class for writing Gen3 raw data ingest tests. 25 __all__ = (
"IngestTestBase",)
32 from lsst.daf.butler
import Butler
37 """Base class for tests of gen3 ingest. Subclass from this, then 38 `unittest.TestCase` to get a working test suite. 42 """Root path to ingest files into. Typically `obs_package/tests/`; the 43 actual directory will be a tempdir under this one. 47 """The instrument to be registered and tested.""" 50 """Butler data ID of a file to ingest when testing.""" 53 """Full path to a file to ingest in tests.""" 58 Butler.makeRepo(self.
root)
65 self.
config = RawIngestTask.ConfigClass()
66 self.
config.onError =
"break" 69 if os.path.exists(self.
root):
70 shutil.rmtree(self.
root, ignore_errors=
True)
74 Initialize and run RawIngestTask on a list of files. 78 files : `list`, [`str`], or None 79 List of files to be ingested, or None to use ``self.file`` 84 task.log.setLevel(task.log.FATAL)
89 Test that RawIngestTask ingested the expected files. 93 files : `list`, [`str`], or None 94 List of files to be ingested, or None to use ``self.file`` 100 self.assertImagesEqual(exposure.image, image)
101 self.assertEqual(metadata.toDict(), exposure.getMetadata().toDict())
104 self.
config.transfer =
"symlink" 108 self.
config.transfer =
"copy" 112 self.
config.transfer =
"hardlink" 116 """Test that files already in the directory can be added to the 120 newPath = os.path.join(self.
butler.datastore.root, os.path.basename(self.
file))
121 os.link(self.
file, newPath)
122 self.
config.transfer =
None 126 """Re-ingesting the same data into the repository should fail, if 129 self.
config.transfer =
"symlink" 130 self.
config.conflict =
"fail" 132 with self.assertRaises(Exception):
136 """Re-ingesting the same data into the repository does not fail, if 137 configured to ignore conflict errors. 139 self.
config.transfer =
"symlink" 140 self.
config.conflict =
"ignore" 142 n1, = self.
butler.registry.query(
"SELECT COUNT(*) FROM Dataset")
144 n2, = self.
butler.registry.query(
"SELECT COUNT(*) FROM Dataset")
145 self.assertEqual(n1, n2)
148 """Re-ingesting the same data will be put into a different collection, 149 if configured to do so. 151 self.
config.transfer =
"symlink" 152 self.
config.conflict =
"ignore" 153 self.
config.stash =
"stash" 156 dt = self.
butler.registry.getDatasetType(
"raw.metadata")
158 ref2 = self.
butler.registry.find(
"stash", dt, self.
dataId)
159 self.assertNotEqual(ref1.id, ref2.id)
160 self.assertEqual(self.
butler.get(ref1).toDict(), self.
butler.getDirect(ref2).toDict())
163 """Test that errors do not roll back success, when configured to do so. 165 Failing to ingest a nonexistent file after ingesting the valid one should 166 leave the valid one in the registry, despite raising an exception. 168 self.
config.transfer =
"symlink" 169 self.
config.onError =
"break" 170 with self.assertRaises(Exception):
172 dt = self.
butler.registry.getDatasetType(
"raw.metadata")
173 self.assertIsNotNone(self.
butler.registry.find(self.
butler.collection, dt, self.
dataId))
176 """Failing to ingest nonexistent files before and after ingesting the 177 valid one should leave the valid one in the registry and not raise 180 self.
config.transfer =
"symlink" 181 self.
config.onError =
"continue" 182 self.
runIngest(files=[
"nonexistent.fits", self.
file,
"still-not-here.fits"])
183 dt = self.
butler.registry.getDatasetType(
"raw.metadata")
184 self.assertIsNotNone(self.
butler.registry.find(self.
butler.collection, dt, self.
dataId))
187 """Failing to ingest nonexistent files after ingesting the 188 valid one should leave the registry unchanged. 190 self.
config.transfer =
"symlink" 191 self.
config.onError =
"rollback" 192 with self.assertRaises(Exception):
195 dt = self.
butler.registry.getDatasetType(
"raw.metadata")
201 self.assertIsNotNone(self.
butler.registry.find(self.
butler.collection, dt, self.
dataId))
def testOnErrorRollback(self)
def runIngestTest(self, files=None)
def testOnConflictStash(self)
def testOnErrorContinue(self)
def testOnConflictFail(self)
def testOnConflictIgnore(self)
def runIngest(self, files=None)
def testOnErrorBreak(self)