lsst.obs.base  19.0.0-43-gbcf6a3c+2
ingest_tests.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 """Base class for writing Gen3 raw data ingest tests.
23 """
24 
25 __all__ = ("IngestTestBase",)
26 
27 import abc
28 import tempfile
29 import unittest
30 import os
31 import shutil
32 
33 from lsst.daf.butler import Butler
34 import lsst.obs.base
35 
36 
37 class IngestTestBase(metaclass=abc.ABCMeta):
38  """Base class for tests of gen3 ingest. Subclass from this, then
39  `unittest.TestCase` to get a working test suite.
40  """
41 
42  ingestDir = ""
43  """Root path to ingest files into. Typically `obs_package/tests/`; the
44  actual directory will be a tempdir under this one.
45  """
46 
47  instrument = None
48  """The instrument to be registered and tested."""
49 
50  dataIds = []
51  """list of butler data IDs of files that should have been ingested."""
52 
53  file = ""
54  """Full path to a file to ingest in tests."""
55 
57  """The task to use in the Ingest test."""
58 
59  curatedCalibrationDatasetTypes = None
60  """List or tuple of Datasets types that should be present after calling
61  writeCuratedCalibrations. If `None` writeCuratedCalibrations will
62  not be called and the test will be skipped."""
63 
64  def setUp(self):
65  # Use a temporary working directory
66  self.root = tempfile.mkdtemp(dir=self.ingestDir)
67  Butler.makeRepo(self.root)
68  self.butler = Butler(self.root, run="raw")
69 
70  # Register the instrument and its static metadata
71  self.instrument.register(self.butler.registry)
72 
73  # Make a default config for test methods to play with
74  self.config = self.RawIngestTask.ConfigClass()
75  self.config.instrument = \
76  f"{self.instrument.__class__.__module__}.{self.instrument.__class__.__name__}"
77 
78  def tearDown(self):
79  if os.path.exists(self.root):
80  shutil.rmtree(self.root, ignore_errors=True)
81 
82  def runIngest(self, files=None):
83  """
84  Initialize and run RawIngestTask on a list of files.
85 
86  Parameters
87  ----------
88  files : `list` [`str`], or None
89  List of files to be ingested, or None to use ``self.file``
90  """
91  if files is None:
92  files = [self.file]
93  task = self.RawIngestTask(config=self.config, butler=self.butler)
94  task.log.setLevel(task.log.FATAL) # silence logs, since we expect a lot of warnings
95  task.run(files)
96 
97  def runIngestTest(self, files=None):
98  """
99  Test that RawIngestTask ingested the expected files.
100 
101  Parameters
102  ----------
103  files : `list` [`str`], or None
104  List of files to be ingested, or None to use ``self.file``
105  """
106  self.runIngest(files)
107  datasets = self.butler.registry.queryDatasets('raw', collections=...)
108  self.assertEqual(len(list(datasets)), len(self.dataIds))
109  for dataId in self.dataIds:
110  exposure = self.butler.get("raw", dataId)
111  metadata = self.butler.get("raw.metadata", dataId)
112  self.assertEqual(metadata.toDict(), exposure.getMetadata().toDict())
113 
114  # Since components follow a different code path we check that
115  # WCS match and also we check that at least the shape
116  # of the image is the same (rather than doing per-pixel equality)
117  # Check the observation type before trying to check WCS
118  obsType = self.butler.registry.expandDataId(dataId).records["exposure"].observation_type
119  if obsType == "science":
120  wcs = self.butler.get("raw.wcs", dataId)
121  self.assertEqual(wcs, exposure.getWcs())
122 
123  rawImage = self.butler.get("raw.image", dataId)
124  self.assertEqual(rawImage.getBBox(), exposure.getBBox())
125 
126  self.checkRepo(files=files)
127 
128  def checkRepo(self, files=None):
129  """Check the state of the repository after ingest.
130 
131  This is an optional hook provided for subclasses; by default it does
132  nothing.
133 
134  Parameters
135  ----------
136  files : `list` [`str`], or None
137  List of files to be ingested, or None to use ``self.file``
138  """
139  pass
140 
141  def testLink(self):
142  self.config.transfer = "link"
143  self.runIngestTest()
144 
145  def testSymLink(self):
146  self.config.transfer = "symlink"
147  self.runIngestTest()
148 
149  def testCopy(self):
150  self.config.transfer = "copy"
151  self.runIngestTest()
152 
153  def testHardLink(self):
154  self.config.transfer = "hardlink"
155  try:
156  self.runIngestTest()
157  except PermissionError as err:
158  raise unittest.SkipTest("Skipping hard-link test because input data"
159  " is on a different filesystem.") from err
160 
161  def testInPlace(self):
162  """Test that files already in the directory can be added to the
163  registry in-place.
164  """
165  # symlink into repo root manually
166  newPath = os.path.join(self.butler.datastore.root, os.path.basename(self.file))
167  os.symlink(os.path.abspath(self.file), newPath)
168  self.config.transfer = None
169  self.runIngestTest([newPath])
170 
172  """Re-ingesting the same data into the repository should fail.
173  """
174  self.config.transfer = "symlink"
175  self.runIngest()
176  with self.assertRaises(Exception):
177  self.runIngest()
178 
180  """Test that we can ingest the curated calibrations"""
181  if self.curatedCalibrationDatasetTypes is None:
182  raise unittest.SkipTest("Class requests disabling of writeCuratedCalibrations test")
183 
184  self.instrument.writeCuratedCalibrations(self.butler)
185 
186  dataId = {"instrument": self.instrument.getName()}
187  for datasetTypeName in self.curatedCalibrationDatasetTypes:
188  with self.subTest(dtype=datasetTypeName, dataId=dataId):
189  datasets = list(self.butler.registry.queryDatasets(datasetTypeName, collections=...,
190  dataId=dataId))
191  self.assertGreater(len(datasets), 0, f"Checking {datasetTypeName}")