Coverage for tests/test_ingestion.py : 18%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# This file is part of ap_verify.
3#
4# Developed for the LSST Data Management System.
5# This product includes software developed by the LSST Project
6# (http://www.lsst.org).
7# See the COPYRIGHT file at the top-level directory of this distribution
8# for details of code ownership.
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program. If not, see <http://www.gnu.org/licenses/>.
22#
24import os
25import shutil
26import tempfile
27import unittest.mock
29from lsst.utils import getPackageDir
30import lsst.utils.tests
31import lsst.pipe.tasks as pipeTasks
32import lsst.pex.exceptions as pexExcept
33import lsst.obs.test
34from lsst.pipe.tasks.read_curated_calibs import read_all
35from lsst.ap.verify import ingestion
36from lsst.ap.verify.dataset import Dataset
37from lsst.ap.verify.workspace import Workspace
40class MockDetector(object):
41 def getName(self):
42 return '0'
44 def getId(self):
45 return 0
48class MockCamera(object):
49 def __init__(self, detector):
50 self.det_list = [detector, ]
51 self.det_dict = {'0': detector}
53 def __getitem__(self, item):
54 if type(item) is int:
55 return self.det_list[item]
56 else:
57 return self.det_dict[item]
60class IngestionTestSuite(lsst.utils.tests.TestCase):
62 @classmethod
63 def setUpClass(cls):
64 try:
65 cls.testData = os.path.join(lsst.utils.getPackageDir("obs_test"), 'data', 'input')
66 except pexExcept.NotFoundError:
67 message = "obs_test not setup. Skipping."
68 raise unittest.SkipTest(message)
70 cls.mockCamera = MockCamera(MockDetector())
71 cls.config = cls.makeTestConfig()
72 cls.config.freeze()
74 cls.testApVerifyData = os.path.join('tests', 'ingestion')
75 cls.rawDataId = {'visit': 229388, 'ccdnum': 1}
77 cls.rawData = [{'file': 'raw_v1_fg.fits.gz', 'visit': 890104911, 'filter': 'g', 'exptime': 15.0},
78 {'file': 'raw_v2_fg.fits.gz', 'visit': 890106021, 'filter': 'g', 'exptime': 15.0},
79 {'file': 'raw_v3_fr.fits.gz', 'visit': 890880321, 'filter': 'r', 'exptime': 15.0},
80 ]
81 cls.calibData = [{'type': 'bias', 'file': 'bias.fits.gz', 'filter': '_unknown_',
82 'date': '1999-01-17'},
83 {'type': 'flat', 'file': 'flat_fg.fits.gz', 'filter': 'g', 'date': '1999-01-17'},
84 {'type': 'flat', 'file': 'flat_fr.fits.gz', 'filter': 'r', 'date': '1999-01-17'},
85 ]
87 @staticmethod
88 def makeTestConfig():
89 obsDir = os.path.join(getPackageDir('obs_test'), 'config')
90 config = ingestion.DatasetIngestConfig()
91 config.textDefectPath = os.path.join(getPackageDir('obs_test_data'), 'test', 'defects')
92 config.dataIngester.load(os.path.join(obsDir, 'ingest.py'))
93 config.calibIngester.load(os.path.join(obsDir, 'ingestCalibs.py'))
94 config.defectIngester.load(os.path.join(obsDir, 'ingestCuratedCalibs.py'))
95 return config
97 def setUp(self):
98 # Repositories still get used by IngestTask despite Butler being a mock object
99 self._repo = self._calibRepo = tempfile.mkdtemp()
100 self.addCleanup(shutil.rmtree, self._repo, ignore_errors=True)
102 # Fake Butler and RegisterTask to avoid initialization or DB overhead
103 def mockGet(datasetType, dataId=None):
104 """Minimally fake a butler.get().
105 """
106 if "raw_filename" in datasetType:
107 matchingFiles = [datum['file'] for datum in IngestionTestSuite.rawData
108 if datum['visit'] == dataId['visit']]
109 return [os.path.join(self._repo, file) for file in matchingFiles]
110 elif "bias_filename" in datasetType:
111 matchingFiles = [datum['file'] for datum in IngestionTestSuite.calibData
112 if datum['type'] == 'bias']
113 return [os.path.join(self._repo, file) for file in matchingFiles]
114 elif "flat_filename" in datasetType:
115 matchingFiles = [datum['file'] for datum in IngestionTestSuite.calibData
116 if datum['type'] == 'flat' and datum['filter'] == dataId['filter']]
117 return [os.path.join(self._repo, file) for file in matchingFiles]
118 elif "defects_filename" in datasetType:
119 return [os.path.join(self._repo, 'defects', 'defects.fits'), ]
120 elif "camera" in datasetType:
121 return IngestionTestSuite.mockCamera
122 else:
123 return None
125 butlerPatcher = unittest.mock.patch("lsst.daf.persistence.Butler", autospec=True)
126 self._butler = butlerPatcher.start()
127 self._butler.getMapperClass.return_value = lsst.obs.test.TestMapper
128 self._butler.return_value.get = mockGet
129 self.addCleanup(butlerPatcher.stop)
131 # Fake Dataset and Workspace because it's too hard to make real ones
132 self._dataset = unittest.mock.NonCallableMock(
133 spec=Dataset,
134 rawLocation=os.path.join(IngestionTestSuite.testData, 'raw'),
135 )
136 self._workspace = unittest.mock.NonCallableMock(
137 spec=Workspace,
138 dataRepo=self._repo,
139 calibRepo=self._calibRepo,
140 )
142 self._task = ingestion.DatasetIngestTask(config=IngestionTestSuite.config)
144 def setUpRawRegistry(self):
145 """Mock up the RegisterTask used for ingesting raw data.
147 This method initializes ``self._registerTask``. It should be
148 called at the start of any test case that attempts raw ingestion.
150 Behavior is undefined if more than one of `setUpRawRegistry`, `setUpCalibRegistry`,
151 or `setupDefectRegistry` is called.
152 """
153 patcherRegister = unittest.mock.patch.object(self._task.dataIngester, "register",
154 spec=pipeTasks.ingest.RegisterTask,
155 new_callable=unittest.mock.NonCallableMagicMock)
156 self._registerTask = patcherRegister.start()
157 self.addCleanup(patcherRegister.stop)
159 def setUpCalibRegistry(self):
160 """Mock up the RegisterTask used for ingesting calib data.
162 This method initializes ``self._registerTask``. It should be
163 called at the start of any test case that attempts calib ingestion.
165 Behavior is undefined if more than one of `setUpRawRegistry`, `setUpCalibRegistry`,
166 or `setupDefectRegistry` is called.
167 """
168 patcherRegister = unittest.mock.patch.object(self._task.calibIngester, "register",
169 spec=pipeTasks.ingestCalibs.CalibsRegisterTask,
170 new_callable=unittest.mock.NonCallableMagicMock)
171 self._registerTask = patcherRegister.start()
172 self._registerTask.config = self._task.config.calibIngester.register
173 self.addCleanup(patcherRegister.stop)
175 def assertRawRegistryCalls(self, registryMock, expectedData):
176 """Test that a particular set of science data is registered correctly.
178 Parameters
179 ----------
180 registryMock : `unittest.mock.Mock`
181 a mock object representing the repository's registry. Must have a
182 mock for the `~lsst.pipe.tasks.ingest.RegisterTask.addRow` method.
183 expectedData : iterable of `dict`
184 a collection of dictionaries, each representing one item that
185 should have been ingested. Each dictionary must contain the
186 following keys:
187 - ``file``: file name to be ingested (`str`).
188 - ``filter``: the filter of the file, or "_unknown_" if not applicable (`str`).
189 - ``visit``: visit ID of the file (`int`).
190 - ``exptime``: the exposure time of the file (`float`).
191 calib : `bool`
192 `True` if ``expectedData`` represents calibration data, `False` if
193 it represents science data
194 """
195 kwargs = {'create': False, 'dryrun': False}
196 for datum in expectedData:
197 # TODO: find a way to avoid having to know exact data ID expansion
198 dataId = {'visit': datum['visit'], 'expTime': datum['exptime'], 'filter': datum['filter']}
199 # TODO: I don't think we actually care about the keywords -- especially since they're defaults
200 registryMock.addRow.assert_any_call(registryMock.openRegistry().__enter__(), dataId,
201 **kwargs)
203 self.assertEqual(registryMock.addRow.call_count, len(expectedData))
205 def assertCalibRegistryCalls(self, registryMock, expectedData):
206 """Test that a particular set of calibration data is registered correctly.
208 Parameters
209 ----------
210 registryMock : `unittest.mock.Mock`
211 a mock object representing the repository's registry. Must have a
212 mock for the `~lsst.pipe.tasks.ingest.CalibsRegisterTask.addRow` method.
213 expectedData : iterable of `dict`
214 a collection of dictionaries, each representing one item that
215 should have been ingested. Each dictionary must contain the
216 following keys:
217 - ``file``: file name to be ingested (`str`).
218 - ``filter``: the filter of the file, or "_unknown_" if not applicable (`str`).
219 - ``type``: a valid calibration dataset type (`str`).
220 - ``date``: the calibration date in YYY-MM-DD format (`str`).
221 calib : `bool`
222 `True` if ``expectedData`` represents calibration data, `False` if
223 it represents science data
224 """
225 kwargs = {'create': False, 'dryrun': False}
226 for datum in expectedData:
227 # TODO: find a way to avoid having to know exact data ID expansion
228 dataId = {'calibDate': datum['date'], 'filter': datum['filter']}
229 kwargs['table'] = datum['type']
230 # TODO: I don't think we actually care about the keywords -- especially since they're defaults
231 registryMock.addRow.assert_any_call(registryMock.openRegistry().__enter__(), dataId,
232 **kwargs)
234 self.assertEqual(registryMock.addRow.call_count, len(expectedData))
236 def testDataIngest(self):
237 """Test that ingesting science images given specific files adds them to a repository.
238 """
239 self.setUpRawRegistry()
240 files = [os.path.join(self._dataset.rawLocation, datum['file'])
241 for datum in IngestionTestSuite.rawData]
242 self._task._doIngestRaws(self._repo, self._calibRepo, files, [])
244 self.assertRawRegistryCalls(self._registerTask, IngestionTestSuite.rawData)
246 def testDataIngestDriver(self):
247 """Test that ingesting science images starting from an abstract dataset adds them to a repository.
248 """
249 self.setUpRawRegistry()
250 self._task._ingestRaws(self._dataset, self._workspace)
252 self.assertRawRegistryCalls(self._registerTask, IngestionTestSuite.rawData)
254 def testCalibIngest(self):
255 """Test that ingesting calibrations given specific files adds them to a repository.
256 """
257 files = [os.path.join(IngestionTestSuite.testData, datum['type'], datum['file'])
258 for datum in IngestionTestSuite.calibData]
259 self.setUpCalibRegistry()
261 self._task._doIngestCalibs(self._repo, self._calibRepo, files)
263 self.assertCalibRegistryCalls(self._registerTask, IngestionTestSuite.calibData)
265 def testCalibIngestDriver(self):
266 """Test that ingesting calibrations starting from an abstract dataset adds them to a repository.
267 """
268 self.setUpCalibRegistry()
269 # obs_test doesn't store calibs together; emulate normal behavior with two calls
270 self._dataset.calibLocation = os.path.join(IngestionTestSuite.testData, 'bias')
271 self._task._ingestCalibs(self._dataset, self._workspace)
272 self._dataset.calibLocation = os.path.join(IngestionTestSuite.testData, 'flat')
273 self._task._ingestCalibs(self._dataset, self._workspace)
275 self.assertCalibRegistryCalls(self._registerTask, IngestionTestSuite.calibData)
277 def testDefectIngest(self):
278 """Test that ingesting defects starting from a concrete file adds them to a repository.
279 """
280 self.setUpCalibRegistry()
282 # Second return is calib type
283 defects = read_all(self._task.config.textDefectPath, IngestionTestSuite.mockCamera)[0]
284 numDefects = 0
285 # These are keyes on sensor and validity date
286 for s in defects:
287 for d in defects[s]:
288 numDefects += len(defects[s][d])
289 self._task._doIngestDefects(self._repo, self._calibRepo, self._task.config.textDefectPath)
291 self.assertEqual(504, numDefects) # Update if the number of defects in obs_test_data changes
293 def testDefectIngestDriver(self):
294 """Test that ingesting defects starting from an abstract dataset adds them to a repository.
295 """
296 self.setUpCalibRegistry()
298 # second return is calib type
299 defects = read_all(self._task.config.textDefectPath, IngestionTestSuite.mockCamera)[0]
300 numDefects = 0
301 # These are keyes on sensor and validity date
302 for s in defects:
303 for d in defects[s]:
304 numDefects += len(defects[s][d])
306 self._task._ingestDefects(self._dataset, self._workspace)
308 self.assertEqual(504, numDefects) # Update if the number of defects in obs_test_data changes
310 def testNoFileIngest(self):
311 """Test that attempts to ingest nothing raise an exception.
312 """
313 files = []
314 self.setUpRawRegistry()
316 with self.assertRaises(RuntimeError):
317 self._task._doIngestRaws(self._repo, self._calibRepo, files, [])
318 with self.assertRaises(RuntimeError):
319 self._task._doIngestCalibs(self._repo, self._calibRepo, files)
321 self._registerTask.addRow.assert_not_called()
323 def testNoFileIngestDriver(self):
324 """Test that attempts to ingest nothing using high-level methods raise an exception.
325 """
326 emptyDir = tempfile.mkdtemp()
327 self.addCleanup(shutil.rmtree, emptyDir, ignore_errors=True)
328 self._dataset.rawLocation = self._dataset.calibLocation = emptyDir
329 with self.assertRaises(RuntimeError):
330 self._task._ingestRaws(self._dataset, self._workspace)
331 with self.assertRaises(RuntimeError):
332 self._task._ingestCalibs(self._dataset, self._workspace)
334 def testBadFileIngest(self):
335 """Test that ingestion of raw data ignores blacklisted files.
336 """
337 badFiles = ['raw_v2_fg.fits.gz']
338 self.setUpRawRegistry()
340 files = [os.path.join(self._dataset.rawLocation, datum['file'])
341 for datum in IngestionTestSuite.rawData]
342 self._task._doIngestRaws(self._repo, self._calibRepo, files, badFiles)
344 filteredData = [datum for datum in IngestionTestSuite.rawData if datum['file'] not in badFiles]
345 self.assertRawRegistryCalls(self._registerTask, filteredData)
347 for datum in IngestionTestSuite.rawData:
348 if datum['file'] in badFiles:
349 dataId = {'visit': datum['visit'], 'expTime': datum['exptime'], 'filter': datum['filter']}
350 # This call should never happen for badFiles
351 call = unittest.mock.call(self._registerTask.openRegistry().__enter__(), dataId,
352 create=False, dryrun=False)
353 self.assertNotIn(call, self._registerTask.addRow.mock_calls)
355 def testFindMatchingFiles(self):
356 """Test that _findMatchingFiles finds the desired files.
357 """
358 testDir = os.path.join(IngestionTestSuite.testData)
360 self.assertSetEqual(
361 ingestion._findMatchingFiles(testDir, ['raw_*.fits.gz']),
362 {os.path.join(testDir, 'raw', f) for f in
363 {'raw_v1_fg.fits.gz', 'raw_v2_fg.fits.gz', 'raw_v3_fr.fits.gz'}}
364 )
365 self.assertSetEqual(
366 ingestion._findMatchingFiles(testDir, ['raw_*.fits.gz'], exclude=['*fr*']),
367 {os.path.join(testDir, 'raw', f) for f in {'raw_v1_fg.fits.gz', 'raw_v2_fg.fits.gz'}}
368 )
369 self.assertSetEqual(
370 ingestion._findMatchingFiles(testDir, ['raw_*.fits.gz'], exclude=['*_v?_f?.fits.gz']),
371 set()
372 )
373 self.assertSetEqual(
374 ingestion._findMatchingFiles(testDir, ['raw_*.fits.gz'], exclude=['obs_test']),
375 {os.path.join(testDir, 'raw', f) for f in
376 {'raw_v1_fg.fits.gz', 'raw_v2_fg.fits.gz', 'raw_v3_fr.fits.gz'}}
377 )
380class MemoryTester(lsst.utils.tests.MemoryTestCase):
381 pass
384def setup_module(module):
385 lsst.utils.tests.init()
388if __name__ == "__main__": 388 ↛ 389line 388 didn't jump to line 389, because the condition on line 388 was never true
389 lsst.utils.tests.init()
390 unittest.main()