lsst.obs.base  18.1.0-21-gde80ed3+2
ingest_tests.py
Go to the documentation of this file.
1 # This file is part of obs_subaru.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """Base class for writing Gen3 raw data ingest tests.
23 """
24 
25 __all__ = ("IngestTestBase",)
26 
27 import abc
28 import tempfile
29 import unittest
30 import os
31 import shutil
32 
33 from lsst.daf.butler import Butler
34 from lsst.obs.base import RawIngestTask
35 
36 
37 class IngestTestBase(metaclass=abc.ABCMeta):
38  """Base class for tests of gen3 ingest. Subclass from this, then
39  `unittest.TestCase` to get a working test suite.
40  """
41 
42  ingestDir = ""
43  """Root path to ingest files into. Typically `obs_package/tests/`; the
44  actual directory will be a tempdir under this one.
45  """
46 
47  instrument = None
48  """The instrument to be registered and tested."""
49 
50  dataId = {}
51  """Butler data ID of a file to ingest when testing."""
52 
53  file = ""
54  """Full path to a file to ingest in tests."""
55 
56  def setUp(self):
57  # Use a temporary working directory
58  self.root = tempfile.mkdtemp(dir=self.ingestDir)
59  Butler.makeRepo(self.root)
60  self.butler = Butler(self.root, run="raw")
61 
62  # Register the instrument and its static metadata
63  self.instrument.register(self.butler.registry)
64 
65  # Make a default config for test methods to play with
66  self.config = RawIngestTask.ConfigClass()
67  self.config.instrument = \
68  f"{self.instrument.__class__.__module__}.{self.instrument.__class__.__name__}"
69 
70  def tearDown(self):
71  if os.path.exists(self.root):
72  shutil.rmtree(self.root, ignore_errors=True)
73 
74  def runIngest(self, files=None):
75  """
76  Initialize and run RawIngestTask on a list of files.
77 
78  Parameters
79  ----------
80  files : `list`, [`str`], or None
81  List of files to be ingested, or None to use ``self.file``
82  """
83  if files is None:
84  files = [self.file]
85  task = RawIngestTask(config=self.config, butler=self.butler)
86  task.log.setLevel(task.log.FATAL) # silence logs, since we expect a lot of warnings
87  task.run(files)
88 
89  def runIngestTest(self, files=None):
90  """
91  Test that RawIngestTask ingested the expected files.
92 
93  Parameters
94  ----------
95  files : `list`, [`str`], or None
96  List of files to be ingested, or None to use ``self.file``
97  """
98  self.runIngest(files)
99  exposure = self.butler.get("raw", self.dataId)
100  metadata = self.butler.get("raw.metadata", self.dataId)
101  image = self.butler.get("raw.image", self.dataId)
102  self.assertImagesEqual(exposure.image, image)
103  self.assertEqual(metadata.toDict(), exposure.getMetadata().toDict())
104 
105  def testSymLink(self):
106  self.config.transfer = "symlink"
107  self.runIngestTest()
108 
109  def testCopy(self):
110  self.config.transfer = "copy"
111  self.runIngestTest()
112 
113  def testHardLink(self):
114  self.config.transfer = "hardlink"
115  try:
116  self.runIngestTest()
117  except PermissionError as err:
118  raise unittest.SkipTest("Skipping hard-link test because input data"
119  " is on a different filesystem.") from err
120 
121  def testInPlace(self):
122  """Test that files already in the directory can be added to the
123  registry in-place.
124  """
125  # symlink into repo root manually
126  newPath = os.path.join(self.butler.datastore.root, os.path.basename(self.file))
127  os.symlink(self.file, newPath)
128  self.config.transfer = None
129  self.runIngestTest([newPath])
130 
132  """Re-ingesting the same data into the repository should fail.
133  """
134  self.config.transfer = "symlink"
135  self.runIngest()
136  with self.assertRaises(Exception):
137  self.runIngest()