lsst.obs.base  19.0.0-59-gee00874
ingest_tests.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 """Base class for writing Gen3 raw data ingest tests.
23 """
24 
25 __all__ = ("IngestTestBase",)
26 
27 import abc
28 import tempfile
29 import unittest
30 import os
31 import shutil
32 
33 from lsst.daf.butler import Butler
34 from lsst.daf.butler.script import createRepo
35 import lsst.obs.base
36 from .utils import getInstrument
37 from .script import ingestRaws, registerInstrument, writeCuratedCalibrations
38 
39 
40 class IngestTestBase(metaclass=abc.ABCMeta):
41  """Base class for tests of gen3 ingest. Subclass from this, then
42  `unittest.TestCase` to get a working test suite.
43  """
44 
45  ingestDir = ""
46  """Root path to ingest files into. Typically `obs_package/tests/`; the
47  actual directory will be a tempdir under this one.
48  """
49 
50  instrument = None
51  """The instrument to be registered and tested."""
52 
53  dataIds = []
54  """list of butler data IDs of files that should have been ingested."""
55 
56  file = ""
57  """Full path to a file to ingest in tests."""
58 
59  RawIngestTask = "lsst.obs.base.RawIngestTask"
60  """The task to use in the Ingest test."""
61 
62  curatedCalibrationDatasetTypes = None
63  """List or tuple of Datasets types that should be present after calling
64  writeCuratedCalibrations. If `None` writeCuratedCalibrations will
65  not be called and the test will be skipped."""
66 
67  DefineVisitsTask = lsst.obs.base.DefineVisitsTask
68  """The task to use to define visits from groups of exposures.
69 
70  This is ignored if ``visits`` is `None`.
71  """
72 
73  visits = {}
74  """A dictionary mapping visit data IDs the lists of exposure data IDs that
75  are associated with them.
76 
77  If this is empty (but not `None`), visit definition will be run but no
78  visits will be expected (e.g. because no exposures are on-sky
79  observations).
80  """
81 
82  instrument = ""
83  """The fully qualified name of the instrument.
84  """
85 
86  instrumentName = ""
87  """The name of the instrument.
88  """
89 
90  outputRun = "raw"
91  """The name of the output run to use in tests.
92  """
93 
94  def setUp(self):
95  # Use a temporary working directory
96  self.root = tempfile.mkdtemp(dir=self.ingestDir)
97  createRepo(self.root)
98 
99  # Register the instrument and its static metadata
101 
102  def tearDown(self):
103  if os.path.exists(self.root):
104  shutil.rmtree(self.root, ignore_errors=True)
105 
106  def verifyIngest(self, files=None, cli=False):
107  """
108  Test that RawIngestTask ingested the expected files.
109 
110  Parameters
111  ----------
112  files : `list` [`str`], or None
113  List of files to be ingested, or None to use ``self.file``
114  """
115  butler = Butler(self.root, run=self.outputRun)
116  datasets = butler.registry.queryDatasets(self.outputRun, collections=...)
117  self.assertEqual(len(list(datasets)), len(self.dataIds))
118  for dataId in self.dataIds:
119  exposure = butler.get(self.outputRun, dataId)
120  metadata = butler.get("raw.metadata", dataId)
121  self.assertEqual(metadata.toDict(), exposure.getMetadata().toDict())
122 
123  # Since components follow a different code path we check that
124  # WCS match and also we check that at least the shape
125  # of the image is the same (rather than doing per-pixel equality)
126  wcs = butler.get("raw.wcs", dataId)
127  self.assertEqual(wcs, exposure.getWcs())
128 
129  rawImage = butler.get("raw.image", dataId)
130  self.assertEqual(rawImage.getBBox(), exposure.getBBox())
131 
132  self.checkRepo(files=files)
133 
134  def checkRepo(self, files=None):
135  """Check the state of the repository after ingest.
136 
137  This is an optional hook provided for subclasses; by default it does
138  nothing.
139 
140  Parameters
141  ----------
142  files : `list` [`str`], or None
143  List of files to be ingested, or None to use ``self.file``
144  """
145  pass
146 
147  def testLink(self):
148  ingestRaws(self.root, self.outputRun, file=self.file, transfer="link", ingest_task=self.RawIngestTask)
149  self.verifyIngest()
150 
151  def testSymLink(self):
152  ingestRaws(self.root, self.outputRun, file=self.file, transfer="symlink",
153  ingest_task=self.RawIngestTask)
154  self.verifyIngest()
155 
156  def testCopy(self):
157  ingestRaws(self.root, self.outputRun, file=self.file, transfer="copy", ingest_task=self.RawIngestTask)
158  self.verifyIngest()
159 
160  def testHardLink(self):
161  try:
162  ingestRaws(self.root, self.outputRun, file=self.file, transfer="hardlink",
163  ingest_task=self.RawIngestTask)
164  self.verifyIngest()
165  except PermissionError as err:
166  raise unittest.SkipTest("Skipping hard-link test because input data"
167  " is on a different filesystem.") from err
168 
169  def testInPlace(self):
170  """Test that files already in the directory can be added to the
171  registry in-place.
172  """
173  # symlink into repo root manually
174  butler = Butler(self.root, run=self.outputRun)
175  newPath = os.path.join(butler.datastore.root, os.path.basename(self.file))
176  os.symlink(os.path.abspath(self.file), newPath)
177  ingestRaws(self.root, self.outputRun, file=newPath, transfer=None, ingest_task=self.RawIngestTask)
178  self.verifyIngest()
179 
181  """Re-ingesting the same data into the repository should fail.
182  """
183  ingestRaws(self.root, self.outputRun, file=self.file, transfer="symlink",
184  ingest_task=self.RawIngestTask)
185  with self.assertRaises(Exception):
186  ingestRaws(self.root, self.outputRun, file=self.file, transfer="symlink",
187  ingest_task=self.RawIngestTask)
188 
190  """Test that we can ingest the curated calibrations"""
191  if self.curatedCalibrationDatasetTypes is None:
192  raise unittest.SkipTest("Class requests disabling of writeCuratedCalibrations test")
193 
195 
196  dataId = {"instrument": self.instrumentName}
197  butler = Butler(self.root, run=self.outputRun)
198  for datasetTypeName in self.curatedCalibrationDatasetTypes:
199  with self.subTest(dtype=datasetTypeName, dataId=dataId):
200  datasets = list(butler.registry.queryDatasets(datasetTypeName, collections=...,
201  dataId=dataId))
202  self.assertGreater(len(datasets), 0, f"Checking {datasetTypeName}")
203 
204  def testDefineVisits(self):
205  if self.visits is None:
206  self.skipTest("Expected visits were not defined.")
207  ingestRaws(self.root, self.outputRun, file=self.file, transfer="link", ingest_task=self.RawIngestTask)
208 
209  config = self.DefineVisitsTask.ConfigClass()
210  butler = Butler(self.root, run=self.outputRun)
211  instrument = getInstrument(self.instrumentName, butler.registry)
212  instrument.applyConfigOverrides(self.DefineVisitsTask._DefaultName, config)
213  task = self.DefineVisitsTask(config=config, butler=butler)
214  task.run(self.dataIds)
215 
216  # Test that we got the visits we expected.
217  visits = set(butler.registry.queryDimensions(["visit"], expand=True))
218  self.assertCountEqual(visits, self.visits.keys())
219  camera = instrument.getCamera()
220  for foundVisit, (expectedVisit, expectedExposures) in zip(visits, self.visits.items()):
221  # Test that this visit is associated with the expected exposures.
222  foundExposures = set(butler.registry.queryDimensions(["exposure"], dataId=expectedVisit,
223  expand=True))
224  self.assertCountEqual(foundExposures, expectedExposures)
225  # Test that we have a visit region, and that it contains all of the
226  # detector+visit regions.
227  self.assertIsNotNone(foundVisit.region)
228  detectorVisitDataIds = set(butler.registry.queryDimensions(["visit", "detector"],
229  dataId=expectedVisit,
230  expand=True))
231  self.assertEqual(len(detectorVisitDataIds), len(camera))
232  for dataId in detectorVisitDataIds:
233  self.assertTrue(foundVisit.region.contains(dataId.region))
lsst.obs.base.script.registerInstrument.registerInstrument
def registerInstrument(repo, instrument)
Definition: registerInstrument.py:26
lsst.obs.base.ingest_tests.IngestTestBase.verifyIngest
def verifyIngest(self, files=None, cli=False)
Definition: ingest_tests.py:106
lsst.obs.base.ingest_tests.IngestTestBase.setUp
def setUp(self)
Definition: ingest_tests.py:94
lsst.obs.base.utils.getInstrument
def getInstrument(instrumentName, registry=None)
Definition: utils.py:100
lsst.obs.base.ingest_tests.IngestTestBase.instrumentName
string instrumentName
Definition: ingest_tests.py:86
lsst.obs.base.ingest_tests.IngestTestBase.testHardLink
def testHardLink(self)
Definition: ingest_tests.py:160
lsst.obs.base.ingest_tests.IngestTestBase.testInPlace
def testInPlace(self)
Definition: ingest_tests.py:169
lsst.obs.base.ingest_tests.IngestTestBase.testCopy
def testCopy(self)
Definition: ingest_tests.py:156
lsst.obs.base.ingest_tests.IngestTestBase.visits
dictionary visits
Definition: ingest_tests.py:73
lsst.obs.base.ingest_tests.IngestTestBase.ingestDir
string ingestDir
Definition: ingest_tests.py:45
lsst.obs.base.ingest_tests.IngestTestBase.dataIds
list dataIds
Definition: ingest_tests.py:53
lsst.obs.base.ingest_tests.IngestTestBase.tearDown
def tearDown(self)
Definition: ingest_tests.py:102
lsst.obs.base.ingest_tests.IngestTestBase.testSymLink
def testSymLink(self)
Definition: ingest_tests.py:151
lsst.obs.base.ingest_tests.IngestTestBase.instrument
instrument
Definition: ingest_tests.py:50
lsst.obs.base.ingest_tests.IngestTestBase.testWriteCuratedCalibrations
def testWriteCuratedCalibrations(self)
Definition: ingest_tests.py:189
lsst.obs.base.script.ingestRaws.ingestRaws
def ingestRaws(repo, output_run, config=None, config_file=None, directory=None, file=None, transfer="auto", ingest_task="lsst.obs.base.RawIngestTask")
Definition: ingestRaws.py:29
lsst.obs.base.ingest_tests.IngestTestBase.DefineVisitsTask
DefineVisitsTask
Definition: ingest_tests.py:67
lsst.obs.base.defineVisits.DefineVisitsTask
Definition: defineVisits.py:250
lsst.obs.base.ingest_tests.IngestTestBase.curatedCalibrationDatasetTypes
curatedCalibrationDatasetTypes
Definition: ingest_tests.py:62
lsst.obs.base.ingest_tests.IngestTestBase.root
root
Definition: ingest_tests.py:96
lsst.obs.base.script.writeCuratedCalibrations.writeCuratedCalibrations
def writeCuratedCalibrations(repo, instrument, output_run)
Definition: writeCuratedCalibrations.py:30
lsst.obs.base.ingest_tests.IngestTestBase.testLink
def testLink(self)
Definition: ingest_tests.py:147
lsst.obs.base.ingest_tests.IngestTestBase.file
string file
Definition: ingest_tests.py:56
lsst.obs.base.ingest_tests.IngestTestBase.testDefineVisits
def testDefineVisits(self)
Definition: ingest_tests.py:204
lsst.obs.base.ingest_tests.IngestTestBase.RawIngestTask
string RawIngestTask
Definition: ingest_tests.py:59
lsst.obs.base.ingest_tests.IngestTestBase.outputRun
string outputRun
Definition: ingest_tests.py:90
lsst.obs.base.ingest_tests.IngestTestBase.checkRepo
def checkRepo(self, files=None)
Definition: ingest_tests.py:134
lsst.obs.base.ingest_tests.IngestTestBase.testFailOnConflict
def testFailOnConflict(self)
Definition: ingest_tests.py:180
lsst.obs.base.ingest_tests.IngestTestBase
Definition: ingest_tests.py:40
lsst.obs.base
Definition: __init__.py:1