22 """Base class for writing Gen3 raw data ingest tests.
25 __all__ = (
"IngestTestBase",)
34 from lsst.daf.butler
import Butler
35 from lsst.daf.butler.cli.butler
import cli
as butlerCli
38 from .utils
import getInstrument
43 """Base class for tests of gen3 ingest. Subclass from this, then
44 `unittest.TestCase` to get a working test suite.
48 """Root path to ingest files into. Typically `obs_package/tests/`; the
49 actual directory will be a tempdir under this one.
53 """list of butler data IDs of files that should have been ingested."""
56 """Full path to a file to ingest in tests."""
58 rawIngestTask =
"lsst.obs.base.RawIngestTask"
59 """The task to use in the Ingest test."""
61 curatedCalibrationDatasetTypes =
None
62 """List or tuple of Datasets types that should be present after calling
63 writeCuratedCalibrations. If `None` writeCuratedCalibrations will
64 not be called and the test will be skipped."""
67 """The task to use to define visits from groups of exposures.
68 This is ignored if ``visits`` is `None`.
72 """A dictionary mapping visit data IDs the lists of exposure data IDs that
73 are associated with them.
74 If this is empty (but not `None`), visit definition will be run but no
75 visits will be expected (e.g. because no exposures are on-sky
80 """The name of the output run to use in tests.
86 """The fully qualified instrument class name.
91 The fully qualified instrument class name.
97 """The instrument class."""
102 """The name of the instrument.
107 The name of the instrument.
120 if os.path.exists(self.
root):
121 shutil.rmtree(self.
root, ignore_errors=
True)
125 Test that RawIngestTask ingested the expected files.
129 files : `list` [`str`], or None
130 List of files to be ingested, or None to use ``self.file``
133 datasets = butler.registry.queryDatasets(self.
outputRun, collections=...)
134 self.assertEqual(len(list(datasets)), len(self.
dataIds))
136 exposure = butler.get(self.
outputRun, dataId)
137 metadata = butler.get(
"raw.metadata", dataId)
138 self.assertEqual(metadata.toDict(), exposure.getMetadata().toDict())
143 wcs = butler.get(
"raw.wcs", dataId)
144 self.assertEqual(wcs, exposure.getWcs())
146 rawImage = butler.get(
"raw.image", dataId)
147 self.assertEqual(rawImage.getBBox(), exposure.getBBox())
152 """Check the state of the repository after ingest.
154 This is an optional hook provided for subclasses; by default it does
159 files : `list` [`str`], or None
160 List of files to be ingested, or None to use ``self.file``
164 def _createRepo(self):
165 """Use the Click `testing` module to call the butler command line api
166 to create a repository."""
167 runner = click.testing.CliRunner()
168 result = runner.invoke(butlerCli, [
"create", self.root])
169 self.assertEqual(result.exit_code, 0, f
"output: {result.output} exception: {result.exception}")
171 def _ingestRaws(self, transfer):
172 """Use the Click `testing` module to call the butler command line api
178 The external data transfer type.
180 runner = click.testing.CliRunner()
181 result = runner.invoke(butlerCli, [
"ingest-raws", self.root,
182 "--output-run", self.outputRun,
184 "--transfer", transfer,
185 "--ingest-task", self.rawIngestTask])
186 self.assertEqual(result.exit_code, 0, f
"output: {result.output} exception: {result.exception}")
188 def _registerInstrument(self):
189 """Use the Click `testing` module to call the butler command line api
190 to register the instrument."""
191 runner = click.testing.CliRunner()
192 result = runner.invoke(butlerCli, [
"register-instrument", self.root,
193 "--instrument", self.instrumentClassName])
194 self.assertEqual(result.exit_code, 0, f
"output: {result.output} exception: {result.exception}")
196 def _writeCuratedCalibrations(self):
197 """Use the Click `testing` module to call the butler command line api
198 to write curated calibrations."""
199 runner = click.testing.CliRunner()
200 result = runner.invoke(butlerCli, [
"write-curated-calibrations", self.root,
201 "--instrument", self.instrumentName,
202 "--output-run", self.outputRun])
203 self.assertEqual(result.exit_code, 0, f
"output: {result.output} exception: {result.exception}")
221 except PermissionError
as err:
222 raise unittest.SkipTest(
"Skipping hard-link test because input data"
223 " is on a different filesystem.")
from err
226 """Test that files already in the directory can be added to the
231 newPath = os.path.join(butler.datastore.root, os.path.basename(self.
file))
232 os.symlink(os.path.abspath(self.
file), newPath)
237 """Re-ingesting the same data into the repository should fail.
240 with self.assertRaises(Exception):
244 """Test that we can ingest the curated calibrations"""
246 raise unittest.SkipTest(
"Class requests disabling of writeCuratedCalibrations test")
253 with self.subTest(dtype=datasetTypeName, dataId=dataId):
254 datasets = list(butler.registry.queryDatasets(datasetTypeName, collections=...,
256 self.assertGreater(len(datasets), 0, f
"Checking {datasetTypeName}")
260 self.skipTest(
"Expected visits were not defined.")
266 script.defineVisits(self.
root, config_file=
None, collections=self.
outputRun,
271 visits = set(butler.registry.queryDimensions([
"visit"], expand=
True))
272 self.assertCountEqual(visits, self.
visits.keys())
274 camera = instr.getCamera()
275 for foundVisit, (expectedVisit, expectedExposures)
in zip(visits, self.
visits.items()):
277 foundExposures = set(butler.registry.queryDimensions([
"exposure"], dataId=expectedVisit,
279 self.assertCountEqual(foundExposures, expectedExposures)
282 self.assertIsNotNone(foundVisit.region)
283 detectorVisitDataIds = set(butler.registry.queryDimensions([
"visit",
"detector"],
284 dataId=expectedVisit,
286 self.assertEqual(len(detectorVisitDataIds), len(camera))
287 for dataId
in detectorVisitDataIds:
288 self.assertTrue(foundVisit.region.contains(dataId.region))