22 """Base class for writing Gen3 raw data ingest tests.
25 __all__ = (
"IngestTestBase",)
33 import lsst.afw.cameraGeom
34 from lsst.daf.butler
import Butler, ButlerURI
35 from lsst.daf.butler.cli.butler
import cli
as butlerCli
36 from lsst.daf.butler.cli.utils
import LogCliRunner
39 from .utils
import getInstrument
44 """Base class for tests of gen3 ingest. Subclass from this, then
45 `unittest.TestCase` to get a working test suite.
49 """Root path to ingest files into. Typically `obs_package/tests/`; the
50 actual directory will be a tempdir under this one.
54 """list of butler data IDs of files that should have been ingested."""
57 """Full path to a file to ingest in tests."""
59 rawIngestTask =
"lsst.obs.base.RawIngestTask"
60 """The task to use in the Ingest test."""
62 curatedCalibrationDatasetTypes =
None
63 """List or tuple of Datasets types that should be present after calling
64 writeCuratedCalibrations. If `None` writeCuratedCalibrations will
65 not be called and the test will be skipped."""
68 """The task to use to define visits from groups of exposures.
69 This is ignored if ``visits`` is `None`.
73 """A dictionary mapping visit data IDs the lists of exposure data IDs that
74 are associated with them.
75 If this is empty (but not `None`), visit definition will be run but no
76 visits will be expected (e.g. because no exposures are on-sky
83 """The fully qualified instrument class name.
88 The fully qualified instrument class name.
94 """The instrument class."""
99 """The name of the instrument.
104 The name of the instrument.
123 if os.path.exists(cls.
root):
124 shutil.rmtree(cls.
root, ignore_errors=
True)
128 Test that RawIngestTask ingested the expected files.
132 files : `list` [`str`], or None
133 List of files to be ingested, or None to use ``self.file``
134 fullCheck : `bool`, optional
135 If `True`, read the full raw dataset and check component
136 consistency. If `False` check that a component can be read
137 but do not read the entire raw exposure.
141 Reading all the ingested test data can be expensive. The code paths
142 for reading the second raw are the same as reading the first so
143 we do not gain anything by doing full checks of everything.
144 Only read full pixel data for first dataset from file.
145 Don't even do that if we are requested not to by the caller.
146 This only really affects files that contain multiple datasets.
149 datasets = list(butler.registry.queryDatasets(
"raw", collections=self.
outputRun))
150 self.assertEqual(len(datasets), len(self.
dataIds))
154 datasetUri = butler.getURI(datasets[0])
155 self.assertIsNotNone(datasetUri.relative_to(butler.datastore.root))
159 metadata = butler.get(
"raw.metadata", dataId)
163 exposure = butler.get(
"raw", dataId)
164 self.assertEqual(metadata.toDict(), exposure.getMetadata().toDict())
169 wcs = butler.get(
"raw.wcs", dataId)
170 self.assertEqual(wcs, exposure.getWcs())
172 rawImage = butler.get(
"raw.image", dataId)
173 self.assertEqual(rawImage.getBBox(), exposure.getBBox())
178 """Check the state of the repository after ingest.
180 This is an optional hook provided for subclasses; by default it does
185 files : `list` [`str`], or None
186 List of files to be ingested, or None to use ``self.file``
191 def _createRepo(cls):
192 """Use the Click `testing` module to call the butler command line api
193 to create a repository."""
194 runner = LogCliRunner()
195 result = runner.invoke(butlerCli, [
"create", cls.
root])
197 assert result.exit_code == 0, f
"output: {result.output} exception: {result.exception}"
199 def _ingestRaws(self, transfer, file=None):
200 """Use the Click `testing` module to call the butler command line api
206 The external data transfer type.
208 Path to a file to ingest instead of the default associated with
213 runner = LogCliRunner()
214 result = runner.invoke(butlerCli, [
"ingest-raws", self.
root, file,
216 "--transfer", transfer,
218 self.assertEqual(result.exit_code, 0, f
"output: {result.output} exception: {result.exception}")
221 def _registerInstrument(cls):
222 """Use the Click `testing` module to call the butler command line api
223 to register the instrument."""
224 runner = LogCliRunner()
227 assert result.exit_code == 0, f
"output: {result.output} exception: {result.exception}"
229 def _writeCuratedCalibrations(self):
230 """Use the Click `testing` module to call the butler command line api
231 to write curated calibrations."""
232 runner = LogCliRunner()
233 result = runner.invoke(butlerCli, [
"write-curated-calibrations", self.
root, self.
instrumentName])
234 self.assertEqual(result.exit_code, 0, f
"output: {result.output} exception: {result.exception}")
248 srcUri = ButlerURI(self.
file)
250 datasets = list(butler.registry.queryDatasets(
"raw", collections=self.
outputRun))
251 datastoreUri = butler.getURI(datasets[0])
252 self.assertEqual(datastoreUri, srcUri)
265 except PermissionError
as err:
266 raise unittest.SkipTest(
"Skipping hard-link test because input data"
267 " is on a different filesystem.")
from err
270 """Test that files already in the directory can be added to the
275 pathInStore =
"prefix-" + os.path.basename(self.
file)
276 newPath = butler.datastore.root.join(pathInStore)
277 os.symlink(os.path.abspath(self.
file), newPath.ospath)
278 self.
_ingestRaws(transfer=
"auto", file=newPath.ospath)
286 uri = butler.getURI(
"raw", self.
dataIds[0])
287 self.assertEqual(uri.relative_to(butler.datastore.root), pathInStore)
290 """Re-ingesting the same data into the repository should fail.
293 with self.assertRaises(Exception):
297 """Test that we can ingest the curated calibrations, and read them
298 with `loadCamera` both before and after.
301 raise unittest.SkipTest(
"Class requests disabling of writeCuratedCalibrations test")
303 butler = Butler(self.
root, writeable=
False)
308 with self.assertRaises(LookupError):
309 lsst.obs.base.loadCamera(butler, {
"exposure": 0}, collections=collection)
316 camera, isVersioned = lsst.obs.base.loadCamera(butler, self.
dataIds[0], collections=collection)
317 self.assertFalse(isVersioned)
318 self.assertIsInstance(camera, lsst.afw.cameraGeom.Camera)
326 butler = Butler(self.
root, writeable=
False)
329 with self.subTest(dtype=datasetTypeName):
331 butler.registry.queryDatasetAssociations(
333 collections=collection,
336 self.assertGreater(len(found), 0, f
"Checking {datasetTypeName}")
339 camera, isVersioned = lsst.obs.base.loadCamera(butler, self.
dataIds[0], collections=collection)
340 self.assertTrue(isVersioned)
341 self.assertIsInstance(camera, lsst.afw.cameraGeom.Camera)
345 self.skipTest(
"Expected visits were not defined.")
352 script.defineVisits(self.
root, config_file=
None, collections=self.
outputRun,
357 visits = butler.registry.queryDataIds([
"visit"]).expanded().toSet()
358 self.assertCountEqual(visits, self.
visits.keys())
360 camera = instr.getCamera()
361 for foundVisit, (expectedVisit, expectedExposures)
in zip(visits, self.
visits.items()):
363 foundExposures = butler.registry.queryDataIds([
"exposure"], dataId=expectedVisit
365 self.assertCountEqual(foundExposures, expectedExposures)
368 self.assertIsNotNone(foundVisit.region)
369 detectorVisitDataIds = butler.registry.queryDataIds([
"visit",
"detector"], dataId=expectedVisit
371 self.assertEqual(len(detectorVisitDataIds), len(camera))
372 for dataId
in detectorVisitDataIds:
373 self.assertTrue(foundVisit.region.contains(dataId.region))