22 """Base class for writing Gen3 raw data ingest tests.
25 __all__ = (
"IngestTestBase",)
33 import lsst.afw.cameraGeom
34 from lsst.daf.butler
import Butler
35 from lsst.daf.butler.cli.butler
import cli
as butlerCli
36 from lsst.daf.butler.cli.utils
import LogCliRunner
39 from .utils
import getInstrument
44 """Base class for tests of gen3 ingest. Subclass from this, then
45 `unittest.TestCase` to get a working test suite.
49 """Root path to ingest files into. Typically `obs_package/tests/`; the
50 actual directory will be a tempdir under this one.
54 """list of butler data IDs of files that should have been ingested."""
57 """Full path to a file to ingest in tests."""
59 rawIngestTask =
"lsst.obs.base.RawIngestTask"
60 """The task to use in the Ingest test."""
62 curatedCalibrationDatasetTypes =
None
63 """List or tuple of Datasets types that should be present after calling
64 writeCuratedCalibrations. If `None` writeCuratedCalibrations will
65 not be called and the test will be skipped."""
68 """The task to use to define visits from groups of exposures.
69 This is ignored if ``visits`` is `None`.
73 """A dictionary mapping visit data IDs the lists of exposure data IDs that
74 are associated with them.
75 If this is empty (but not `None`), visit definition will be run but no
76 visits will be expected (e.g. because no exposures are on-sky
81 """The name of the output run to use in tests.
87 """The fully qualified instrument class name.
92 The fully qualified instrument class name.
98 """The instrument class."""
103 """The name of the instrument.
108 The name of the instrument.
121 if os.path.exists(self.
root):
122 shutil.rmtree(self.
root, ignore_errors=
True)
126 Test that RawIngestTask ingested the expected files.
130 files : `list` [`str`], or None
131 List of files to be ingested, or None to use ``self.file``
134 datasets = butler.registry.queryDatasets(self.
outputRun, collections=...)
135 self.assertEqual(len(list(datasets)), len(self.
dataIds))
137 exposure = butler.get(self.
outputRun, dataId)
138 metadata = butler.get(
"raw.metadata", dataId)
139 self.assertEqual(metadata.toDict(), exposure.getMetadata().toDict())
144 wcs = butler.get(
"raw.wcs", dataId)
145 self.assertEqual(wcs, exposure.getWcs())
147 rawImage = butler.get(
"raw.image", dataId)
148 self.assertEqual(rawImage.getBBox(), exposure.getBBox())
153 """Check the state of the repository after ingest.
155 This is an optional hook provided for subclasses; by default it does
160 files : `list` [`str`], or None
161 List of files to be ingested, or None to use ``self.file``
165 def _createRepo(self):
166 """Use the Click `testing` module to call the butler command line api
167 to create a repository."""
168 runner = LogCliRunner()
169 result = runner.invoke(butlerCli, [
"create", self.root])
170 self.assertEqual(result.exit_code, 0, f
"output: {result.output} exception: {result.exception}")
172 def _ingestRaws(self, transfer):
173 """Use the Click `testing` module to call the butler command line api
179 The external data transfer type.
181 runner = LogCliRunner()
182 result = runner.invoke(butlerCli, [
"ingest-raws", self.root, self.file,
183 "--output-run", self.outputRun,
184 "--transfer", transfer,
185 "--ingest-task", self.rawIngestTask])
186 self.assertEqual(result.exit_code, 0, f
"output: {result.output} exception: {result.exception}")
188 def _registerInstrument(self):
189 """Use the Click `testing` module to call the butler command line api
190 to register the instrument."""
191 runner = LogCliRunner()
192 result = runner.invoke(butlerCli, [
"register-instrument", self.root, self.instrumentClassName])
193 self.assertEqual(result.exit_code, 0, f
"output: {result.output} exception: {result.exception}")
195 def _writeCuratedCalibrations(self):
196 """Use the Click `testing` module to call the butler command line api
197 to write curated calibrations."""
198 runner = LogCliRunner()
199 result = runner.invoke(butlerCli, [
"write-curated-calibrations", self.root, self.instrumentName])
200 self.assertEqual(result.exit_code, 0, f
"output: {result.output} exception: {result.exception}")
218 except PermissionError
as err:
219 raise unittest.SkipTest(
"Skipping hard-link test because input data"
220 " is on a different filesystem.")
from err
223 """Test that files already in the directory can be added to the
228 newPath = butler.datastore.root.join(os.path.basename(self.
file))
229 os.symlink(os.path.abspath(self.
file), newPath.ospath)
234 """Re-ingesting the same data into the repository should fail.
237 with self.assertRaises(Exception):
241 """Test that we can ingest the curated calibrations, and read them
242 with `loadCamera` both before and after.
245 raise unittest.SkipTest(
"Class requests disabling of writeCuratedCalibrations test")
247 butler = Butler(self.
root, writeable=
False)
252 with self.assertRaises(LookupError):
253 lsst.obs.base.loadCamera(butler, self.
dataIds[0], collections=collection)
260 camera, isVersioned = lsst.obs.base.loadCamera(butler, self.
dataIds[0], collections=collection)
261 self.assertFalse(isVersioned)
262 self.assertIsInstance(camera, lsst.afw.cameraGeom.Camera)
270 butler = Butler(self.
root, writeable=
False)
273 with self.subTest(dtype=datasetTypeName):
275 butler.registry.queryDatasetAssociations(
277 collections=collection,
280 self.assertGreater(len(found), 0, f
"Checking {datasetTypeName}")
283 camera, isVersioned = lsst.obs.base.loadCamera(butler, self.
dataIds[0], collections=collection)
284 self.assertTrue(isVersioned)
285 self.assertIsInstance(camera, lsst.afw.cameraGeom.Camera)
289 self.skipTest(
"Expected visits were not defined.")
296 script.defineVisits(self.
root, config_file=
None, collections=self.
outputRun,
301 visits = butler.registry.queryDataIds([
"visit"]).expanded().toSet()
302 self.assertCountEqual(visits, self.
visits.keys())
304 camera = instr.getCamera()
305 for foundVisit, (expectedVisit, expectedExposures)
in zip(visits, self.
visits.items()):
307 foundExposures = butler.registry.queryDataIds([
"exposure"], dataId=expectedVisit
309 self.assertCountEqual(foundExposures, expectedExposures)
312 self.assertIsNotNone(foundVisit.region)
313 detectorVisitDataIds = butler.registry.queryDataIds([
"visit",
"detector"], dataId=expectedVisit
315 self.assertEqual(len(detectorVisitDataIds), len(camera))
316 for dataId
in detectorVisitDataIds:
317 self.assertTrue(foundVisit.region.contains(dataId.region))