lsst.obs.base  19.0.0-22-g282de62+1
ingest_tests.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 """Base class for writing Gen3 raw data ingest tests.
23 """
24 
25 __all__ = ("IngestTestBase",)
26 
27 import abc
28 import tempfile
29 import unittest
30 import os
31 import shutil
32 
33 from lsst.daf.butler import Butler
34 import lsst.obs.base
35 
36 
37 class IngestTestBase(metaclass=abc.ABCMeta):
38  """Base class for tests of gen3 ingest. Subclass from this, then
39  `unittest.TestCase` to get a working test suite.
40  """
41 
42  ingestDir = ""
43  """Root path to ingest files into. Typically `obs_package/tests/`; the
44  actual directory will be a tempdir under this one.
45  """
46 
47  instrument = None
48  """The instrument to be registered and tested."""
49 
50  dataIds = []
51  """list of butler data IDs of files that should have been ingested."""
52 
53  file = ""
54  """Full path to a file to ingest in tests."""
55 
57  """The task to use in the Ingest test."""
58 
59  def setUp(self):
60  # Use a temporary working directory
61  self.root = tempfile.mkdtemp(dir=self.ingestDir)
62  Butler.makeRepo(self.root)
63  self.butler = Butler(self.root, run="raw")
64 
65  # Register the instrument and its static metadata
66  self.instrument.register(self.butler.registry)
67 
68  # Make a default config for test methods to play with
69  self.config = self.RawIngestTask.ConfigClass()
70  self.config.instrument = \
71  f"{self.instrument.__class__.__module__}.{self.instrument.__class__.__name__}"
72 
73  def tearDown(self):
74  if os.path.exists(self.root):
75  shutil.rmtree(self.root, ignore_errors=True)
76 
77  def runIngest(self, files=None):
78  """
79  Initialize and run RawIngestTask on a list of files.
80 
81  Parameters
82  ----------
83  files : `list` [`str`], or None
84  List of files to be ingested, or None to use ``self.file``
85  """
86  if files is None:
87  files = [self.file]
88  task = self.RawIngestTask(config=self.config, butler=self.butler)
89  task.log.setLevel(task.log.FATAL) # silence logs, since we expect a lot of warnings
90  task.run(files)
91 
92  def runIngestTest(self, files=None):
93  """
94  Test that RawIngestTask ingested the expected files.
95 
96  Parameters
97  ----------
98  files : `list` [`str`], or None
99  List of files to be ingested, or None to use ``self.file``
100  """
101  self.runIngest(files)
102  datasets = self.butler.registry.queryDatasets('raw', collections=...)
103  self.assertEqual(len(list(datasets)), len(self.dataIds))
104  for dataId in self.dataIds:
105  exposure = self.butler.get("raw", dataId)
106  metadata = self.butler.get("raw.metadata", dataId)
107  # only check the metadata, not the images, to speed up tests
108  self.assertEqual(metadata.toDict(), exposure.getMetadata().toDict())
109  self.checkRepo(files=files)
110 
111  def checkRepo(self, files=None):
112  """Check the state of the repository after ingest.
113 
114  This is an optional hook provided for subclasses; by default it does
115  nothing.
116 
117  Parameters
118  ----------
119  files : `list` [`str`], or None
120  List of files to be ingested, or None to use ``self.file``
121  """
122  pass
123 
124  def testSymLink(self):
125  self.config.transfer = "symlink"
126  self.runIngestTest()
127 
128  def testCopy(self):
129  self.config.transfer = "copy"
130  self.runIngestTest()
131 
132  def testHardLink(self):
133  self.config.transfer = "hardlink"
134  try:
135  self.runIngestTest()
136  except PermissionError as err:
137  raise unittest.SkipTest("Skipping hard-link test because input data"
138  " is on a different filesystem.") from err
139 
140  def testInPlace(self):
141  """Test that files already in the directory can be added to the
142  registry in-place.
143  """
144  # symlink into repo root manually
145  newPath = os.path.join(self.butler.datastore.root, os.path.basename(self.file))
146  os.symlink(os.path.abspath(self.file), newPath)
147  self.config.transfer = None
148  self.runIngestTest([newPath])
149 
151  """Re-ingesting the same data into the repository should fail.
152  """
153  self.config.transfer = "symlink"
154  self.runIngest()
155  with self.assertRaises(Exception):
156  self.runIngest()