22 """Base class for writing Gen3 raw data ingest tests. 25 __all__ = (
"IngestTestBase",)
33 from lsst.daf.butler
import Butler
38 """Base class for tests of gen3 ingest. Subclass from this, then 39 `unittest.TestCase` to get a working test suite. 43 """Root path to ingest files into. Typically `obs_package/tests/`; the 44 actual directory will be a tempdir under this one. 48 """The instrument to be registered and tested.""" 51 """list of butler data IDs of files that should have been ingested.""" 54 """Full path to a file to ingest in tests.""" 57 """The task to use in the Ingest test.""" 59 curatedCalibrationDatasetTypes =
None 60 """List or tuple of Datasets types that should be present after calling 61 writeCuratedCalibrations. If `None` writeCuratedCalibrations will 62 not be called and the test will be skipped.""" 67 Butler.makeRepo(self.
root)
76 f
"{self.instrument.__class__.__module__}.{self.instrument.__class__.__name__}" 79 if os.path.exists(self.
root):
80 shutil.rmtree(self.
root, ignore_errors=
True)
84 Initialize and run RawIngestTask on a list of files. 88 files : `list` [`str`], or None 89 List of files to be ingested, or None to use ``self.file`` 94 task.log.setLevel(task.log.FATAL)
99 Test that RawIngestTask ingested the expected files. 103 files : `list` [`str`], or None 104 List of files to be ingested, or None to use ``self.file`` 107 datasets = self.
butler.registry.queryDatasets(
'raw', collections=...)
108 self.assertEqual(len(list(datasets)), len(self.
dataIds))
110 exposure = self.
butler.get(
"raw", dataId)
111 metadata = self.
butler.get(
"raw.metadata", dataId)
112 self.assertEqual(metadata.toDict(), exposure.getMetadata().toDict())
118 obsType = self.
butler.registry.expandDataId(dataId).records[
"exposure"].observation_type
119 if obsType ==
"science":
120 wcs = self.
butler.get(
"raw.wcs", dataId)
121 self.assertEqual(wcs, exposure.getWcs())
123 rawImage = self.
butler.get(
"raw.image", dataId)
124 self.assertEqual(rawImage.getBBox(), exposure.getBBox())
129 """Check the state of the repository after ingest. 131 This is an optional hook provided for subclasses; by default it does 136 files : `list` [`str`], or None 137 List of files to be ingested, or None to use ``self.file`` 142 self.config.transfer =
"symlink" 146 self.
config.transfer =
"copy" 150 self.
config.transfer =
"hardlink" 153 except PermissionError
as err:
154 raise unittest.SkipTest(
"Skipping hard-link test because input data" 155 " is on a different filesystem.")
from err
158 """Test that files already in the directory can be added to the 162 newPath = os.path.join(self.
butler.datastore.root, os.path.basename(self.
file))
163 os.symlink(os.path.abspath(self.
file), newPath)
164 self.
config.transfer =
None 168 """Re-ingesting the same data into the repository should fail. 170 self.
config.transfer =
"symlink" 172 with self.assertRaises(Exception):
176 """Test that we can ingest the curated calibrations""" 178 raise unittest.SkipTest(
"Class requests disabling of writeCuratedCalibrations test")
182 dataId = {
"instrument": self.
instrument.getName()}
184 with self.subTest(dtype=datasetTypeName, dataId=dataId):
185 datasets = list(self.
butler.registry.queryDatasets(datasetTypeName, collections=...,
187 self.assertGreater(len(datasets), 0, f
"Checking {datasetTypeName}")
def runIngest(self, files=None)
def runIngestTest(self, files=None)
def checkRepo(self, files=None)
def testWriteCuratedCalibrations(self)
def testFailOnConflict(self)
curatedCalibrationDatasetTypes