22 """Base class for writing Gen3 raw data ingest tests.
25 __all__ = (
"IngestTestBase",)
33 from lsst.daf.butler
import Butler
34 from lsst.daf.butler.script
import createRepo
36 from .utils
import getInstrument
37 from .script
import ingestRaws, registerInstrument, writeCuratedCalibrations
41 """Base class for tests of gen3 ingest. Subclass from this, then
42 `unittest.TestCase` to get a working test suite.
46 """Root path to ingest files into. Typically `obs_package/tests/`; the
47 actual directory will be a tempdir under this one.
51 """The instrument to be registered and tested."""
54 """list of butler data IDs of files that should have been ingested."""
57 """Full path to a file to ingest in tests."""
59 RawIngestTask =
"lsst.obs.base.RawIngestTask"
60 """The task to use in the Ingest test."""
62 curatedCalibrationDatasetTypes =
None
63 """List or tuple of Datasets types that should be present after calling
64 writeCuratedCalibrations. If `None` writeCuratedCalibrations will
65 not be called and the test will be skipped."""
68 """The task to use to define visits from groups of exposures.
70 This is ignored if ``visits`` is `None`.
74 """A dictionary mapping visit data IDs the lists of exposure data IDs that
75 are associated with them.
77 If this is empty (but not `None`), visit definition will be run but no
78 visits will be expected (e.g. because no exposures are on-sky
83 """The fully qualified name of the instrument.
87 """The name of the instrument.
91 """The name of the output run to use in tests.
103 if os.path.exists(self.
root):
104 shutil.rmtree(self.
root, ignore_errors=
True)
108 Test that RawIngestTask ingested the expected files.
112 files : `list` [`str`], or None
113 List of files to be ingested, or None to use ``self.file``
116 datasets = butler.registry.queryDatasets(self.
outputRun, collections=...)
117 self.assertEqual(len(list(datasets)), len(self.
dataIds))
119 exposure = butler.get(self.
outputRun, dataId)
120 metadata = butler.get(
"raw.metadata", dataId)
121 self.assertEqual(metadata.toDict(), exposure.getMetadata().toDict())
126 wcs = butler.get(
"raw.wcs", dataId)
127 self.assertEqual(wcs, exposure.getWcs())
129 rawImage = butler.get(
"raw.image", dataId)
130 self.assertEqual(rawImage.getBBox(), exposure.getBBox())
135 """Check the state of the repository after ingest.
137 This is an optional hook provided for subclasses; by default it does
142 files : `list` [`str`], or None
143 List of files to be ingested, or None to use ``self.file``
148 ingestRaws(self.root, self.outputRun, file=self.file, transfer=
"link", ingest_task=self.RawIngestTask)
165 except PermissionError
as err:
166 raise unittest.SkipTest(
"Skipping hard-link test because input data"
167 " is on a different filesystem.")
from err
170 """Test that files already in the directory can be added to the
175 newPath = os.path.join(butler.datastore.root, os.path.basename(self.
file))
176 os.symlink(os.path.abspath(self.
file), newPath)
181 """Re-ingesting the same data into the repository should fail.
185 with self.assertRaises(Exception):
190 """Test that we can ingest the curated calibrations"""
192 raise unittest.SkipTest(
"Class requests disabling of writeCuratedCalibrations test")
199 with self.subTest(dtype=datasetTypeName, dataId=dataId):
200 datasets = list(butler.registry.queryDatasets(datasetTypeName, collections=...,
202 self.assertGreater(len(datasets), 0, f
"Checking {datasetTypeName}")
206 self.skipTest(
"Expected visits were not defined.")
217 visits = set(butler.registry.queryDimensions([
"visit"], expand=
True))
218 self.assertCountEqual(visits, self.
visits.keys())
219 camera = instrument.getCamera()
220 for foundVisit, (expectedVisit, expectedExposures)
in zip(visits, self.
visits.items()):
222 foundExposures = set(butler.registry.queryDimensions([
"exposure"], dataId=expectedVisit,
224 self.assertCountEqual(foundExposures, expectedExposures)
227 self.assertIsNotNone(foundVisit.region)
228 detectorVisitDataIds = set(butler.registry.queryDimensions([
"visit",
"detector"],
229 dataId=expectedVisit,
231 self.assertEqual(len(detectorVisitDataIds), len(camera))
232 for dataId
in detectorVisitDataIds:
233 self.assertTrue(foundVisit.region.contains(dataId.region))