lsst.obs.base  19.0.0-53-gf4ac060
ingest_tests.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 """Base class for writing Gen3 raw data ingest tests.
23 """
24 
25 __all__ = ("IngestTestBase",)
26 
27 import abc
28 import tempfile
29 import unittest
30 import os
31 import shutil
32 
33 from lsst.daf.butler import Butler
34 import lsst.obs.base
35 
36 
37 class IngestTestBase(metaclass=abc.ABCMeta):
38  """Base class for tests of gen3 ingest. Subclass from this, then
39  `unittest.TestCase` to get a working test suite.
40  """
41 
42  ingestDir = ""
43  """Root path to ingest files into. Typically `obs_package/tests/`; the
44  actual directory will be a tempdir under this one.
45  """
46 
47  instrument = None
48  """The instrument to be registered and tested."""
49 
50  dataIds = []
51  """list of butler data IDs of files that should have been ingested."""
52 
53  file = ""
54  """Full path to a file to ingest in tests."""
55 
57  """The task to use in the Ingest test."""
58 
59  curatedCalibrationDatasetTypes = None
60  """List or tuple of Datasets types that should be present after calling
61  writeCuratedCalibrations. If `None` writeCuratedCalibrations will
62  not be called and the test will be skipped."""
63 
64  DefineVisitsTask = lsst.obs.base.DefineVisitsTask
65  """The task to use to define visits from groups of exposures.
66 
67  This is ignored if ``visits`` is `None`.
68  """
69 
70  visits = {}
71  """A dictionary mapping visit data IDs the lists of exposure data IDs that
72  are associated with them.
73 
74  If this is empty (but not `None`), visit definition will be run but no
75  visits will be expected (e.g. because no exposures are on-sky
76  observations).
77  """
78 
79  def setUp(self):
80  # Use a temporary working directory
81  self.root = tempfile.mkdtemp(dir=self.ingestDir)
82  Butler.makeRepo(self.root)
83  self.butler = Butler(self.root, run="raw")
84 
85  # Register the instrument and its static metadata
86  self.instrument.register(self.butler.registry)
87 
88  # Make a default config for test methods to play with
89  self.config = self.RawIngestTask.ConfigClass()
90 
91  def tearDown(self):
92  if os.path.exists(self.root):
93  shutil.rmtree(self.root, ignore_errors=True)
94 
95  def runIngest(self, files=None):
96  """
97  Initialize and run RawIngestTask on a list of files.
98 
99  Parameters
100  ----------
101  files : `list` [`str`], or None
102  List of files to be ingested, or None to use ``self.file``
103  """
104  if files is None:
105  files = [self.file]
106  task = self.RawIngestTask(config=self.config, butler=self.butler)
107  task.log.setLevel(task.log.FATAL) # silence logs, since we expect a lot of warnings
108  task.run(files)
109 
110  def runIngestTest(self, files=None):
111  """
112  Test that RawIngestTask ingested the expected files.
113 
114  Parameters
115  ----------
116  files : `list` [`str`], or None
117  List of files to be ingested, or None to use ``self.file``
118  """
119  self.runIngest(files)
120  datasets = self.butler.registry.queryDatasets('raw', collections=...)
121  self.assertEqual(len(list(datasets)), len(self.dataIds))
122  for dataId in self.dataIds:
123  exposure = self.butler.get("raw", dataId)
124  metadata = self.butler.get("raw.metadata", dataId)
125  self.assertEqual(metadata.toDict(), exposure.getMetadata().toDict())
126 
127  # Since components follow a different code path we check that
128  # WCS match and also we check that at least the shape
129  # of the image is the same (rather than doing per-pixel equality)
130  wcs = self.butler.get("raw.wcs", dataId)
131  self.assertEqual(wcs, exposure.getWcs())
132 
133  rawImage = self.butler.get("raw.image", dataId)
134  self.assertEqual(rawImage.getBBox(), exposure.getBBox())
135 
136  self.checkRepo(files=files)
137 
138  def checkRepo(self, files=None):
139  """Check the state of the repository after ingest.
140 
141  This is an optional hook provided for subclasses; by default it does
142  nothing.
143 
144  Parameters
145  ----------
146  files : `list` [`str`], or None
147  List of files to be ingested, or None to use ``self.file``
148  """
149  pass
150 
151  def testLink(self):
152  self.config.transfer = "link"
153  self.runIngestTest()
154 
155  def testSymLink(self):
156  self.config.transfer = "symlink"
157  self.runIngestTest()
158 
159  def testCopy(self):
160  self.config.transfer = "copy"
161  self.runIngestTest()
162 
163  def testHardLink(self):
164  self.config.transfer = "hardlink"
165  try:
166  self.runIngestTest()
167  except PermissionError as err:
168  raise unittest.SkipTest("Skipping hard-link test because input data"
169  " is on a different filesystem.") from err
170 
171  def testInPlace(self):
172  """Test that files already in the directory can be added to the
173  registry in-place.
174  """
175  # symlink into repo root manually
176  newPath = os.path.join(self.butler.datastore.root, os.path.basename(self.file))
177  os.symlink(os.path.abspath(self.file), newPath)
178  self.config.transfer = None
179  self.runIngestTest([newPath])
180 
182  """Re-ingesting the same data into the repository should fail.
183  """
184  self.config.transfer = "symlink"
185  self.runIngest()
186  with self.assertRaises(Exception):
187  self.runIngest()
188 
190  """Test that we can ingest the curated calibrations"""
191  if self.curatedCalibrationDatasetTypes is None:
192  raise unittest.SkipTest("Class requests disabling of writeCuratedCalibrations test")
193 
194  self.instrument.writeCuratedCalibrations(self.butler)
195 
196  dataId = {"instrument": self.instrument.getName()}
197  for datasetTypeName in self.curatedCalibrationDatasetTypes:
198  with self.subTest(dtype=datasetTypeName, dataId=dataId):
199  datasets = list(self.butler.registry.queryDatasets(datasetTypeName, collections=...,
200  dataId=dataId))
201  self.assertGreater(len(datasets), 0, f"Checking {datasetTypeName}")
202 
203  def testDefineVisits(self):
204  if self.visits is None:
205  self.skipTest("Expected visits were not defined.")
206  self.config.transfer = "link"
207  self.runIngest()
208  config = self.DefineVisitsTask.ConfigClass()
209  self.instrument.applyConfigOverrides(self.DefineVisitsTask._DefaultName, config)
210  task = self.DefineVisitsTask(config=config, butler=self.butler)
211  task.run(self.dataIds)
212  # Test that we got the visits we expected.
213  visits = set(self.butler.registry.queryDimensions(["visit"], expand=True))
214  self.assertCountEqual(visits, self.visits.keys())
215  camera = self.instrument.getCamera()
216  for foundVisit, (expectedVisit, expectedExposures) in zip(visits, self.visits.items()):
217  # Test that this visit is associated with the expected exposures.
218  foundExposures = set(self.butler.registry.queryDimensions(["exposure"], dataId=expectedVisit,
219  expand=True))
220  self.assertCountEqual(foundExposures, expectedExposures)
221  # Test that we have a visit region, and that it contains all of the
222  # detector+visit regions.
223  self.assertIsNotNone(foundVisit.region)
224  detectorVisitDataIds = set(self.butler.registry.queryDimensions(["visit", "detector"],
225  dataId=expectedVisit,
226  expand=True))
227  self.assertEqual(len(detectorVisitDataIds), len(camera))
228  for dataId in detectorVisitDataIds:
229  self.assertTrue(foundVisit.region.contains(dataId.region))
lsst.obs.base.ingest_tests.IngestTestBase.setUp
def setUp(self)
Definition: ingest_tests.py:79
lsst.obs.base.ingest_tests.IngestTestBase.testHardLink
def testHardLink(self)
Definition: ingest_tests.py:163
lsst.obs.base.ingest_tests.IngestTestBase.testInPlace
def testInPlace(self)
Definition: ingest_tests.py:171
lsst.obs.base.ingest_tests.IngestTestBase.config
config
Definition: ingest_tests.py:89
lsst.obs.base.ingest_tests.IngestTestBase.testCopy
def testCopy(self)
Definition: ingest_tests.py:159
lsst.obs.base.ingest_tests.IngestTestBase.visits
dictionary visits
Definition: ingest_tests.py:70
lsst.obs.base.ingest_tests.IngestTestBase.ingestDir
string ingestDir
Definition: ingest_tests.py:42
lsst.obs.base.ingest_tests.IngestTestBase.dataIds
list dataIds
Definition: ingest_tests.py:50
lsst.obs.base.ingest_tests.IngestTestBase.tearDown
def tearDown(self)
Definition: ingest_tests.py:91
lsst.obs.base.ingest_tests.IngestTestBase.testSymLink
def testSymLink(self)
Definition: ingest_tests.py:155
lsst.obs.base.ingest_tests.IngestTestBase.runIngest
def runIngest(self, files=None)
Definition: ingest_tests.py:95
lsst.obs.base.ingest_tests.IngestTestBase.instrument
instrument
Definition: ingest_tests.py:47
lsst.obs.base.ingest_tests.IngestTestBase.testWriteCuratedCalibrations
def testWriteCuratedCalibrations(self)
Definition: ingest_tests.py:189
lsst.obs.base.ingest_tests.IngestTestBase.runIngestTest
def runIngestTest(self, files=None)
Definition: ingest_tests.py:110
lsst.obs.base.ingest_tests.IngestTestBase.DefineVisitsTask
DefineVisitsTask
Definition: ingest_tests.py:64
lsst.obs.base.defineVisits.DefineVisitsTask
Definition: defineVisits.py:250
lsst.obs.base.ingest_tests.IngestTestBase.butler
butler
Definition: ingest_tests.py:83
lsst.obs.base.ingest_tests.IngestTestBase.curatedCalibrationDatasetTypes
curatedCalibrationDatasetTypes
Definition: ingest_tests.py:59
lsst.obs.base.ingest_tests.IngestTestBase.root
root
Definition: ingest_tests.py:81
lsst.obs.base.ingest_tests.IngestTestBase.testLink
def testLink(self)
Definition: ingest_tests.py:151
lsst.obs.base.ingest_tests.IngestTestBase.file
string file
Definition: ingest_tests.py:53
lsst.obs.base.ingest_tests.IngestTestBase.RawIngestTask
RawIngestTask
Definition: ingest_tests.py:56
lsst.obs.base.ingest_tests.IngestTestBase.testDefineVisits
def testDefineVisits(self)
Definition: ingest_tests.py:203
lsst.obs.base.ingest_tests.IngestTestBase.checkRepo
def checkRepo(self, files=None)
Definition: ingest_tests.py:138
lsst.obs.base.ingest_tests.IngestTestBase.testFailOnConflict
def testFailOnConflict(self)
Definition: ingest_tests.py:181
lsst.obs.base.ingest_tests.IngestTestBase
Definition: ingest_tests.py:37
lsst.obs.base.ingest.RawIngestTask
Definition: ingest.py:160
lsst.obs.base
Definition: __init__.py:1