22 """Base class for writing Gen3 raw data ingest tests.
25 __all__ = (
"IngestTestBase",)
33 import lsst.afw.cameraGeom
34 from lsst.daf.butler
import Butler, ButlerURI
35 from lsst.daf.butler.cli.butler
import cli
as butlerCli
36 from lsst.daf.butler.cli.utils
import LogCliRunner
39 from .utils
import getInstrument
44 """Base class for tests of gen3 ingest. Subclass from this, then
45 `unittest.TestCase` to get a working test suite.
49 """Root path to ingest files into. Typically `obs_package/tests/`; the
50 actual directory will be a tempdir under this one.
54 """list of butler data IDs of files that should have been ingested."""
57 """Full path to a file to ingest in tests."""
59 rawIngestTask =
"lsst.obs.base.RawIngestTask"
60 """The task to use in the Ingest test."""
62 curatedCalibrationDatasetTypes =
None
63 """List or tuple of Datasets types that should be present after calling
64 writeCuratedCalibrations. If `None` writeCuratedCalibrations will
65 not be called and the test will be skipped."""
68 """The task to use to define visits from groups of exposures.
69 This is ignored if ``visits`` is `None`.
73 """A dictionary mapping visit data IDs the lists of exposure data IDs that
74 are associated with them.
75 If this is empty (but not `None`), visit definition will be run but no
76 visits will be expected (e.g. because no exposures are on-sky
83 """The fully qualified instrument class name.
88 The fully qualified instrument class name.
94 """The instrument class."""
99 """The name of the instrument.
104 The name of the instrument.
123 if os.path.exists(cls.
root):
124 shutil.rmtree(cls.
root, ignore_errors=
True)
128 Test that RawIngestTask ingested the expected files.
132 files : `list` [`str`], or None
133 List of files to be ingested, or None to use ``self.file``
134 fullCheck : `bool`, optional
135 If `True`, read the full raw dataset and check component
136 consistency. If `False` check that a component can be read
137 but do not read the entire raw exposure.
141 Reading all the ingested test data can be expensive. The code paths
142 for reading the second raw are the same as reading the first so
143 we do not gain anything by doing full checks of everything.
144 Only read full pixel data for first dataset from file.
145 Don't even do that if we are requested not to by the caller.
146 This only really affects files that contain multiple datasets.
149 datasets = list(butler.registry.queryDatasets(
"raw", collections=self.
outputRun))
150 self.assertEqual(len(datasets), len(self.
dataIds))
154 datasetUri = butler.getURI(datasets[0])
155 self.assertIsNotNone(datasetUri.relative_to(butler.datastore.root))
159 metadata = butler.get(
"raw.metadata", dataId)
163 exposure = butler.get(
"raw", dataId)
164 self.assertEqual(metadata.toDict(), exposure.getMetadata().toDict())
169 wcs = butler.get(
"raw.wcs", dataId)
170 self.assertEqual(wcs, exposure.getWcs())
172 rawImage = butler.get(
"raw.image", dataId)
173 self.assertEqual(rawImage.getBBox(), exposure.getBBox())
178 """Check the state of the repository after ingest.
180 This is an optional hook provided for subclasses; by default it does
185 files : `list` [`str`], or None
186 List of files to be ingested, or None to use ``self.file``
191 def _createRepo(cls):
192 """Use the Click `testing` module to call the butler command line api
193 to create a repository."""
194 runner = LogCliRunner()
195 result = runner.invoke(butlerCli, [
"create", cls.
root])
197 assert result.exit_code == 0, f
"output: {result.output} exception: {result.exception}"
199 def _ingestRaws(self, transfer, file=None):
200 """Use the Click `testing` module to call the butler command line api
206 The external data transfer type.
208 Path to a file to ingest instead of the default associated with
213 runner = LogCliRunner()
214 result = runner.invoke(butlerCli, [
"ingest-raws", self.
root, file,
216 "--transfer", transfer,
218 self.assertEqual(result.exit_code, 0, f
"output: {result.output} exception: {result.exception}")
221 def _registerInstrument(cls):
222 """Use the Click `testing` module to call the butler command line api
223 to register the instrument."""
224 runner = LogCliRunner()
227 assert result.exit_code == 0, f
"output: {result.output} exception: {result.exception}"
229 def _writeCuratedCalibrations(self):
230 """Use the Click `testing` module to call the butler command line api
231 to write curated calibrations."""
232 runner = LogCliRunner()
233 result = runner.invoke(butlerCli, [
"write-curated-calibrations", self.
root, self.
instrumentName])
234 self.assertEqual(result.exit_code, 0, f
"output: {result.output} exception: {result.exception}")
248 srcUri = ButlerURI(self.
file)
250 datasets = list(butler.registry.queryDatasets(
"raw", collections=self.
outputRun))
251 datastoreUri = butler.getURI(datasets[0])
252 self.assertEqual(datastoreUri, srcUri)
268 except (AssertionError, PermissionError)
as err:
269 raise unittest.SkipTest(
"Skipping hard-link test because input data"
270 " is on a different filesystem.")
from err
274 """Test that files already in the directory can be added to the
279 pathInStore =
"prefix-" + os.path.basename(self.
file)
280 newPath = butler.datastore.root.join(pathInStore)
281 os.symlink(os.path.abspath(self.
file), newPath.ospath)
282 self.
_ingestRaws(transfer=
"auto", file=newPath.ospath)
290 uri = butler.getURI(
"raw", self.
dataIds[0])
291 self.assertEqual(uri.relative_to(butler.datastore.root), pathInStore)
294 """Re-ingesting the same data into the repository should fail.
297 with self.assertRaises(Exception):
301 """Test that we can ingest the curated calibrations, and read them
302 with `loadCamera` both before and after.
305 raise unittest.SkipTest(
"Class requests disabling of writeCuratedCalibrations test")
307 butler = Butler(self.
root, writeable=
False)
312 with self.assertRaises(LookupError):
313 lsst.obs.base.loadCamera(butler, {
"exposure": 0}, collections=collection)
320 camera, isVersioned = lsst.obs.base.loadCamera(butler, self.
dataIds[0], collections=collection)
321 self.assertFalse(isVersioned)
322 self.assertIsInstance(camera, lsst.afw.cameraGeom.Camera)
330 butler = Butler(self.
root, writeable=
False)
333 with self.subTest(dtype=datasetTypeName):
335 butler.registry.queryDatasetAssociations(
337 collections=collection,
340 self.assertGreater(len(found), 0, f
"Checking {datasetTypeName}")
343 camera, isVersioned = lsst.obs.base.loadCamera(butler, self.
dataIds[0], collections=collection)
344 self.assertTrue(isVersioned)
345 self.assertIsInstance(camera, lsst.afw.cameraGeom.Camera)
349 self.skipTest(
"Expected visits were not defined.")
356 script.defineVisits(self.
root, config_file=
None, collections=self.
outputRun,
361 visits = butler.registry.queryDataIds([
"visit"]).expanded().toSet()
362 self.assertCountEqual(visits, self.
visits.keys())
364 camera = instr.getCamera()
365 for foundVisit, (expectedVisit, expectedExposures)
in zip(visits, self.
visits.items()):
367 foundExposures = butler.registry.queryDataIds([
"exposure"], dataId=expectedVisit
369 self.assertCountEqual(foundExposures, expectedExposures)
372 self.assertIsNotNone(foundVisit.region)
373 detectorVisitDataIds = butler.registry.queryDataIds([
"visit",
"detector"], dataId=expectedVisit
375 self.assertEqual(len(detectorVisitDataIds), len(camera))
376 for dataId
in detectorVisitDataIds:
377 self.assertTrue(foundVisit.region.contains(dataId.region))