22 """Base class for writing Gen3 raw data ingest tests. 25 __all__ = (
"IngestTestBase",)
33 from lsst.daf.butler
import Butler
38 """Base class for tests of gen3 ingest. Subclass from this, then 39 `unittest.TestCase` to get a working test suite. 43 """Root path to ingest files into. Typically `obs_package/tests/`; the 44 actual directory will be a tempdir under this one. 48 """The instrument to be registered and tested.""" 51 """list of butler data IDs of files that should have been ingested.""" 54 """Full path to a file to ingest in tests.""" 57 """The task to use in the Ingest test.""" 59 curatedCalibrationDatasetTypes =
None 60 """List or tuple of Datasets types that should be present after calling 61 writeCuratedCalibrations. If `None` writeCuratedCalibrations will 62 not be called and the test will be skipped.""" 65 """The task to use to define visits from groups of exposures. 67 This is ignored if ``visits`` is `None`. 71 """A dictionary mapping visit data IDs the lists of exposure data IDs that 72 are associated with them. 74 If this is empty (but not `None`), visit definition will be run but no 75 visits will be expected (e.g. because no exposures are on-sky 82 Butler.makeRepo(self.
root)
92 if os.path.exists(self.
root):
93 shutil.rmtree(self.
root, ignore_errors=
True)
97 Initialize and run RawIngestTask on a list of files. 101 files : `list` [`str`], or None 102 List of files to be ingested, or None to use ``self.file`` 107 task.log.setLevel(task.log.FATAL)
112 Test that RawIngestTask ingested the expected files. 116 files : `list` [`str`], or None 117 List of files to be ingested, or None to use ``self.file`` 120 datasets = self.
butler.registry.queryDatasets(
'raw', collections=...)
121 self.assertEqual(len(list(datasets)), len(self.
dataIds))
123 exposure = self.
butler.get(
"raw", dataId)
124 metadata = self.
butler.get(
"raw.metadata", dataId)
125 self.assertEqual(metadata.toDict(), exposure.getMetadata().toDict())
130 wcs = self.
butler.get(
"raw.wcs", dataId)
131 self.assertEqual(wcs, exposure.getWcs())
133 rawImage = self.
butler.get(
"raw.image", dataId)
134 self.assertEqual(rawImage.getBBox(), exposure.getBBox())
139 """Check the state of the repository after ingest. 141 This is an optional hook provided for subclasses; by default it does 146 files : `list` [`str`], or None 147 List of files to be ingested, or None to use ``self.file`` 152 self.config.transfer =
"link" 156 self.
config.transfer =
"symlink" 160 self.
config.transfer =
"copy" 164 self.
config.transfer =
"hardlink" 167 except PermissionError
as err:
168 raise unittest.SkipTest(
"Skipping hard-link test because input data" 169 " is on a different filesystem.")
from err
172 """Test that files already in the directory can be added to the 176 newPath = os.path.join(self.
butler.datastore.root, os.path.basename(self.
file))
177 os.symlink(os.path.abspath(self.
file), newPath)
178 self.
config.transfer =
None 182 """Re-ingesting the same data into the repository should fail. 184 self.
config.transfer =
"symlink" 186 with self.assertRaises(Exception):
190 """Test that we can ingest the curated calibrations""" 192 raise unittest.SkipTest(
"Class requests disabling of writeCuratedCalibrations test")
196 dataId = {
"instrument": self.
instrument.getName()}
198 with self.subTest(dtype=datasetTypeName, dataId=dataId):
199 datasets = list(self.
butler.registry.queryDatasets(datasetTypeName, collections=...,
201 self.assertGreater(len(datasets), 0, f
"Checking {datasetTypeName}")
205 self.skipTest(
"Expected visits were not defined.")
206 self.
config.transfer =
"link" 213 visits = set(self.
butler.registry.queryDimensions([
"visit"], expand=
True))
214 self.assertCountEqual(visits, self.
visits.keys())
216 for foundVisit, (expectedVisit, expectedExposures)
in zip(visits, self.
visits.items()):
218 foundExposures = set(self.
butler.registry.queryDimensions([
"exposure"], dataId=expectedVisit,
220 self.assertCountEqual(foundExposures, expectedExposures)
223 self.assertIsNotNone(foundVisit.region)
224 detectorVisitDataIds = set(self.
butler.registry.queryDimensions([
"visit",
"detector"],
225 dataId=expectedVisit,
227 self.assertEqual(len(detectorVisitDataIds), len(camera))
228 for dataId
in detectorVisitDataIds:
229 self.assertTrue(foundVisit.region.contains(dataId.region))
def runIngest(self, files=None)
def runIngestTest(self, files=None)
def checkRepo(self, files=None)
def testWriteCuratedCalibrations(self)
def testDefineVisits(self)
def testFailOnConflict(self)
curatedCalibrationDatasetTypes