22 """Base class for writing Gen3 raw data ingest tests.
25 __all__ = (
"IngestTestBase",)
33 from lsst.daf.butler
import Butler
38 """Base class for tests of gen3 ingest. Subclass from this, then
39 `unittest.TestCase` to get a working test suite.
43 """Root path to ingest files into. Typically `obs_package/tests/`; the
44 actual directory will be a tempdir under this one.
48 """The instrument to be registered and tested."""
51 """list of butler data IDs of files that should have been ingested."""
54 """Full path to a file to ingest in tests."""
57 """The task to use in the Ingest test."""
59 curatedCalibrationDatasetTypes =
None
60 """List or tuple of Datasets types that should be present after calling
61 writeCuratedCalibrations. If `None` writeCuratedCalibrations will
62 not be called and the test will be skipped."""
65 """The task to use to define visits from groups of exposures.
67 This is ignored if ``visits`` is `None`.
71 """A dictionary mapping visit data IDs the lists of exposure data IDs that
72 are associated with them.
74 If this is empty (but not `None`), visit definition will be run but no
75 visits will be expected (e.g. because no exposures are on-sky
82 Butler.makeRepo(self.
root)
92 if os.path.exists(self.
root):
93 shutil.rmtree(self.
root, ignore_errors=
True)
97 Initialize and run RawIngestTask on a list of files.
101 files : `list` [`str`], or None
102 List of files to be ingested, or None to use ``self.file``
107 task.log.setLevel(task.log.FATAL)
112 Test that RawIngestTask ingested the expected files.
116 files : `list` [`str`], or None
117 List of files to be ingested, or None to use ``self.file``
120 datasets = self.
butler.registry.queryDatasets(
'raw', collections=...)
121 self.assertEqual(len(list(datasets)), len(self.
dataIds))
123 exposure = self.
butler.get(
"raw", dataId)
124 metadata = self.
butler.get(
"raw.metadata", dataId)
125 self.assertEqual(metadata.toDict(), exposure.getMetadata().toDict())
130 wcs = self.
butler.get(
"raw.wcs", dataId)
131 self.assertEqual(wcs, exposure.getWcs())
133 rawImage = self.
butler.get(
"raw.image", dataId)
134 self.assertEqual(rawImage.getBBox(), exposure.getBBox())
139 """Check the state of the repository after ingest.
141 This is an optional hook provided for subclasses; by default it does
146 files : `list` [`str`], or None
147 List of files to be ingested, or None to use ``self.file``
152 self.config.transfer =
"link"
156 self.
config.transfer =
"symlink"
160 self.
config.transfer =
"copy"
164 self.
config.transfer =
"hardlink"
167 except PermissionError
as err:
168 raise unittest.SkipTest(
"Skipping hard-link test because input data"
169 " is on a different filesystem.")
from err
172 """Test that files already in the directory can be added to the
176 newPath = os.path.join(self.
butler.datastore.root, os.path.basename(self.
file))
177 os.symlink(os.path.abspath(self.
file), newPath)
178 self.
config.transfer =
None
182 """Re-ingesting the same data into the repository should fail.
184 self.
config.transfer =
"symlink"
186 with self.assertRaises(Exception):
190 """Test that we can ingest the curated calibrations"""
192 raise unittest.SkipTest(
"Class requests disabling of writeCuratedCalibrations test")
196 dataId = {
"instrument": self.
instrument.getName()}
198 with self.subTest(dtype=datasetTypeName, dataId=dataId):
199 datasets = list(self.
butler.registry.queryDatasets(datasetTypeName, collections=...,
201 self.assertGreater(len(datasets), 0, f
"Checking {datasetTypeName}")
205 self.skipTest(
"Expected visits were not defined.")
206 self.
config.transfer =
"link"
213 visits = set(self.
butler.registry.queryDimensions([
"visit"], expand=
True))
214 self.assertCountEqual(visits, self.
visits.keys())
216 for foundVisit, (expectedVisit, expectedExposures)
in zip(visits, self.
visits.items()):
218 foundExposures = set(self.
butler.registry.queryDimensions([
"exposure"], dataId=expectedVisit,
220 self.assertCountEqual(foundExposures, expectedExposures)
223 self.assertIsNotNone(foundVisit.region)
224 detectorVisitDataIds = set(self.
butler.registry.queryDimensions([
"visit",
"detector"],
225 dataId=expectedVisit,
227 self.assertEqual(len(detectorVisitDataIds), len(camera))
228 for dataId
in detectorVisitDataIds:
229 self.assertTrue(foundVisit.region.contains(dataId.region))