22 """Base class for writing Gen3 raw data ingest tests. 25 __all__ = (
"IngestTestBase",)
33 from lsst.daf.butler
import Butler
38 """Base class for tests of gen3 ingest. Subclass from this, then 39 `unittest.TestCase` to get a working test suite. 43 """Root path to ingest files into. Typically `obs_package/tests/`; the 44 actual directory will be a tempdir under this one. 48 """The instrument to be registered and tested.""" 51 """list of butler data IDs of files that should have been ingested.""" 54 """Full path to a file to ingest in tests.""" 57 """The task to use in the Ingest test.""" 59 curatedCalibrationDatasetTypes =
None 60 """List or tuple of Datasets types that should be present after calling 61 writeCuratedCalibrations. If `None` writeCuratedCalibrations will 62 not be called and the test will be skipped.""" 67 Butler.makeRepo(self.
root)
76 f
"{self.instrument.__class__.__module__}.{self.instrument.__class__.__name__}" 79 if os.path.exists(self.
root):
80 shutil.rmtree(self.
root, ignore_errors=
True)
84 Initialize and run RawIngestTask on a list of files. 88 files : `list` [`str`], or None 89 List of files to be ingested, or None to use ``self.file`` 94 task.log.setLevel(task.log.FATAL)
99 Test that RawIngestTask ingested the expected files. 103 files : `list` [`str`], or None 104 List of files to be ingested, or None to use ``self.file`` 107 datasets = self.
butler.registry.queryDatasets(
'raw', collections=...)
108 self.assertEqual(len(list(datasets)), len(self.
dataIds))
110 exposure = self.
butler.get(
"raw", dataId)
111 metadata = self.
butler.get(
"raw.metadata", dataId)
113 self.assertEqual(metadata.toDict(), exposure.getMetadata().toDict())
117 """Check the state of the repository after ingest. 119 This is an optional hook provided for subclasses; by default it does 124 files : `list` [`str`], or None 125 List of files to be ingested, or None to use ``self.file`` 130 self.config.transfer =
"symlink" 134 self.
config.transfer =
"copy" 138 self.
config.transfer =
"hardlink" 141 except PermissionError
as err:
142 raise unittest.SkipTest(
"Skipping hard-link test because input data" 143 " is on a different filesystem.")
from err
146 """Test that files already in the directory can be added to the 150 newPath = os.path.join(self.
butler.datastore.root, os.path.basename(self.
file))
151 os.symlink(os.path.abspath(self.
file), newPath)
152 self.
config.transfer =
None 156 """Re-ingesting the same data into the repository should fail. 158 self.
config.transfer =
"symlink" 160 with self.assertRaises(Exception):
164 """Test that we can ingest the curated calibrations""" 166 raise unittest.SkipTest(
"Class requests disabling of writeCuratedCalibrations test")
170 dataId = {
"instrument": self.
instrument.getName()}
172 with self.subTest(dtype=datasetTypeName, dataId=dataId):
173 datasets = list(self.
butler.registry.queryDatasets(datasetTypeName, collections=...,
175 self.assertGreater(len(datasets), 0, f
"Checking {datasetTypeName}")
def runIngest(self, files=None)
def runIngestTest(self, files=None)
def checkRepo(self, files=None)
def testWriteCuratedCalibrations(self)
def testFailOnConflict(self)
curatedCalibrationDatasetTypes