Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# This file is part of ap_verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24import os 

25import shutil 

26import tempfile 

27import unittest.mock 

28 

29from lsst.utils import getPackageDir 

30import lsst.utils.tests 

31import lsst.pipe.tasks as pipeTasks 

32import lsst.pex.exceptions as pexExcept 

33import lsst.obs.test 

34from lsst.pipe.tasks.read_curated_calibs import read_all 

35from lsst.ap.verify import ingestion 

36from lsst.ap.verify.dataset import Dataset 

37from lsst.ap.verify.workspace import Workspace 

38 

39 

40class MockDetector(object): 

41 def getName(self): 

42 return '0' 

43 

44 def getId(self): 

45 return 0 

46 

47 

48class MockCamera(object): 

49 def __init__(self, detector): 

50 self.det_list = [detector, ] 

51 self.det_dict = {'0': detector} 

52 

53 def __getitem__(self, item): 

54 if type(item) is int: 

55 return self.det_list[item] 

56 else: 

57 return self.det_dict[item] 

58 

59 

60class IngestionTestSuite(lsst.utils.tests.TestCase): 

61 

62 @classmethod 

63 def setUpClass(cls): 

64 try: 

65 cls.testData = os.path.join(lsst.utils.getPackageDir("obs_test"), 'data', 'input') 

66 except pexExcept.NotFoundError: 

67 message = "obs_test not setup. Skipping." 

68 raise unittest.SkipTest(message) 

69 

70 cls.mockCamera = MockCamera(MockDetector()) 

71 cls.config = cls.makeTestConfig() 

72 cls.config.freeze() 

73 

74 cls.testApVerifyData = os.path.join('tests', 'ingestion') 

75 cls.rawDataId = {'visit': 229388, 'ccdnum': 1} 

76 

77 cls.rawData = [{'file': 'raw_v1_fg.fits.gz', 'visit': 890104911, 'filter': 'g', 'exptime': 15.0}, 

78 {'file': 'raw_v2_fg.fits.gz', 'visit': 890106021, 'filter': 'g', 'exptime': 15.0}, 

79 {'file': 'raw_v3_fr.fits.gz', 'visit': 890880321, 'filter': 'r', 'exptime': 15.0}, 

80 ] 

81 cls.calibData = [{'type': 'bias', 'file': 'bias.fits.gz', 'filter': '_unknown_', 

82 'date': '1999-01-17'}, 

83 {'type': 'flat', 'file': 'flat_fg.fits.gz', 'filter': 'g', 'date': '1999-01-17'}, 

84 {'type': 'flat', 'file': 'flat_fr.fits.gz', 'filter': 'r', 'date': '1999-01-17'}, 

85 ] 

86 

87 @staticmethod 

88 def makeTestConfig(): 

89 obsDir = os.path.join(getPackageDir('obs_test'), 'config') 

90 config = ingestion.DatasetIngestConfig() 

91 config.textDefectPath = os.path.join(getPackageDir('obs_test_data'), 'test', 'defects') 

92 config.dataIngester.load(os.path.join(obsDir, 'ingest.py')) 

93 config.calibIngester.load(os.path.join(obsDir, 'ingestCalibs.py')) 

94 config.defectIngester.load(os.path.join(obsDir, 'ingestCuratedCalibs.py')) 

95 return config 

96 

97 def setUp(self): 

98 # Repositories still get used by IngestTask despite Butler being a mock object 

99 self._repo = self._calibRepo = tempfile.mkdtemp() 

100 self.addCleanup(shutil.rmtree, self._repo, ignore_errors=True) 

101 

102 # Fake Butler and RegisterTask to avoid initialization or DB overhead 

103 def mockGet(datasetType, dataId=None): 

104 """Minimally fake a butler.get(). 

105 """ 

106 if "raw_filename" in datasetType: 

107 matchingFiles = [datum['file'] for datum in IngestionTestSuite.rawData 

108 if datum['visit'] == dataId['visit']] 

109 return [os.path.join(self._repo, file) for file in matchingFiles] 

110 elif "bias_filename" in datasetType: 

111 matchingFiles = [datum['file'] for datum in IngestionTestSuite.calibData 

112 if datum['type'] == 'bias'] 

113 return [os.path.join(self._repo, file) for file in matchingFiles] 

114 elif "flat_filename" in datasetType: 

115 matchingFiles = [datum['file'] for datum in IngestionTestSuite.calibData 

116 if datum['type'] == 'flat' and datum['filter'] == dataId['filter']] 

117 return [os.path.join(self._repo, file) for file in matchingFiles] 

118 elif "defects_filename" in datasetType: 

119 return [os.path.join(self._repo, 'defects', 'defects.fits'), ] 

120 elif "camera" in datasetType: 

121 return IngestionTestSuite.mockCamera 

122 else: 

123 return None 

124 

125 butlerPatcher = unittest.mock.patch("lsst.daf.persistence.Butler", autospec=True) 

126 self._butler = butlerPatcher.start() 

127 self._butler.getMapperClass.return_value = lsst.obs.test.TestMapper 

128 self._butler.return_value.get = mockGet 

129 self.addCleanup(butlerPatcher.stop) 

130 

131 # Fake Dataset and Workspace because it's too hard to make real ones 

132 self._dataset = unittest.mock.NonCallableMock( 

133 spec=Dataset, 

134 rawLocation=os.path.join(IngestionTestSuite.testData, 'raw'), 

135 ) 

136 self._workspace = unittest.mock.NonCallableMock( 

137 spec=Workspace, 

138 dataRepo=self._repo, 

139 calibRepo=self._calibRepo, 

140 ) 

141 

142 self._task = ingestion.DatasetIngestTask(config=IngestionTestSuite.config) 

143 

144 def setUpRawRegistry(self): 

145 """Mock up the RegisterTask used for ingesting raw data. 

146 

147 This method initializes ``self._registerTask``. It should be 

148 called at the start of any test case that attempts raw ingestion. 

149 

150 Behavior is undefined if more than one of `setUpRawRegistry`, `setUpCalibRegistry`, 

151 or `setupDefectRegistry` is called. 

152 """ 

153 patcherRegister = unittest.mock.patch.object(self._task.dataIngester, "register", 

154 spec=pipeTasks.ingest.RegisterTask, 

155 new_callable=unittest.mock.NonCallableMagicMock) 

156 self._registerTask = patcherRegister.start() 

157 self.addCleanup(patcherRegister.stop) 

158 

159 def setUpCalibRegistry(self): 

160 """Mock up the RegisterTask used for ingesting calib data. 

161 

162 This method initializes ``self._registerTask``. It should be 

163 called at the start of any test case that attempts calib ingestion. 

164 

165 Behavior is undefined if more than one of `setUpRawRegistry`, `setUpCalibRegistry`, 

166 or `setupDefectRegistry` is called. 

167 """ 

168 patcherRegister = unittest.mock.patch.object(self._task.calibIngester, "register", 

169 spec=pipeTasks.ingestCalibs.CalibsRegisterTask, 

170 new_callable=unittest.mock.NonCallableMagicMock) 

171 self._registerTask = patcherRegister.start() 

172 self._registerTask.config = self._task.config.calibIngester.register 

173 self.addCleanup(patcherRegister.stop) 

174 

175 def assertRawRegistryCalls(self, registryMock, expectedData): 

176 """Test that a particular set of science data is registered correctly. 

177 

178 Parameters 

179 ---------- 

180 registryMock : `unittest.mock.Mock` 

181 a mock object representing the repository's registry. Must have a 

182 mock for the `~lsst.pipe.tasks.ingest.RegisterTask.addRow` method. 

183 expectedData : iterable of `dict` 

184 a collection of dictionaries, each representing one item that 

185 should have been ingested. Each dictionary must contain the 

186 following keys: 

187 - ``file``: file name to be ingested (`str`). 

188 - ``filter``: the filter of the file, or "_unknown_" if not applicable (`str`). 

189 - ``visit``: visit ID of the file (`int`). 

190 - ``exptime``: the exposure time of the file (`float`). 

191 calib : `bool` 

192 `True` if ``expectedData`` represents calibration data, `False` if 

193 it represents science data 

194 """ 

195 kwargs = {'create': False, 'dryrun': False} 

196 for datum in expectedData: 

197 # TODO: find a way to avoid having to know exact data ID expansion 

198 dataId = {'visit': datum['visit'], 'expTime': datum['exptime'], 'filter': datum['filter']} 

199 # TODO: I don't think we actually care about the keywords -- especially since they're defaults 

200 registryMock.addRow.assert_any_call(registryMock.openRegistry().__enter__(), dataId, 

201 **kwargs) 

202 

203 self.assertEqual(registryMock.addRow.call_count, len(expectedData)) 

204 

205 def assertCalibRegistryCalls(self, registryMock, expectedData): 

206 """Test that a particular set of calibration data is registered correctly. 

207 

208 Parameters 

209 ---------- 

210 registryMock : `unittest.mock.Mock` 

211 a mock object representing the repository's registry. Must have a 

212 mock for the `~lsst.pipe.tasks.ingest.CalibsRegisterTask.addRow` method. 

213 expectedData : iterable of `dict` 

214 a collection of dictionaries, each representing one item that 

215 should have been ingested. Each dictionary must contain the 

216 following keys: 

217 - ``file``: file name to be ingested (`str`). 

218 - ``filter``: the filter of the file, or "_unknown_" if not applicable (`str`). 

219 - ``type``: a valid calibration dataset type (`str`). 

220 - ``date``: the calibration date in YYY-MM-DD format (`str`). 

221 calib : `bool` 

222 `True` if ``expectedData`` represents calibration data, `False` if 

223 it represents science data 

224 """ 

225 kwargs = {'create': False, 'dryrun': False} 

226 for datum in expectedData: 

227 # TODO: find a way to avoid having to know exact data ID expansion 

228 dataId = {'calibDate': datum['date'], 'filter': datum['filter']} 

229 kwargs['table'] = datum['type'] 

230 # TODO: I don't think we actually care about the keywords -- especially since they're defaults 

231 registryMock.addRow.assert_any_call(registryMock.openRegistry().__enter__(), dataId, 

232 **kwargs) 

233 

234 self.assertEqual(registryMock.addRow.call_count, len(expectedData)) 

235 

236 def testDataIngest(self): 

237 """Test that ingesting science images given specific files adds them to a repository. 

238 """ 

239 self.setUpRawRegistry() 

240 files = [os.path.join(self._dataset.rawLocation, datum['file']) 

241 for datum in IngestionTestSuite.rawData] 

242 self._task._doIngestRaws(self._repo, self._calibRepo, files, []) 

243 

244 self.assertRawRegistryCalls(self._registerTask, IngestionTestSuite.rawData) 

245 

246 def testDataIngestDriver(self): 

247 """Test that ingesting science images starting from an abstract dataset adds them to a repository. 

248 """ 

249 self.setUpRawRegistry() 

250 self._task._ingestRaws(self._dataset, self._workspace) 

251 

252 self.assertRawRegistryCalls(self._registerTask, IngestionTestSuite.rawData) 

253 

254 def testCalibIngest(self): 

255 """Test that ingesting calibrations given specific files adds them to a repository. 

256 """ 

257 files = [os.path.join(IngestionTestSuite.testData, datum['type'], datum['file']) 

258 for datum in IngestionTestSuite.calibData] 

259 self.setUpCalibRegistry() 

260 

261 self._task._doIngestCalibs(self._repo, self._calibRepo, files) 

262 

263 self.assertCalibRegistryCalls(self._registerTask, IngestionTestSuite.calibData) 

264 

265 def testCalibIngestDriver(self): 

266 """Test that ingesting calibrations starting from an abstract dataset adds them to a repository. 

267 """ 

268 self.setUpCalibRegistry() 

269 # obs_test doesn't store calibs together; emulate normal behavior with two calls 

270 self._dataset.calibLocation = os.path.join(IngestionTestSuite.testData, 'bias') 

271 self._task._ingestCalibs(self._dataset, self._workspace) 

272 self._dataset.calibLocation = os.path.join(IngestionTestSuite.testData, 'flat') 

273 self._task._ingestCalibs(self._dataset, self._workspace) 

274 

275 self.assertCalibRegistryCalls(self._registerTask, IngestionTestSuite.calibData) 

276 

277 def testDefectIngest(self): 

278 """Test that ingesting defects starting from a concrete file adds them to a repository. 

279 """ 

280 self.setUpCalibRegistry() 

281 

282 # Second return is calib type 

283 defects = read_all(self._task.config.textDefectPath, IngestionTestSuite.mockCamera)[0] 

284 numDefects = 0 

285 # These are keyes on sensor and validity date 

286 for s in defects: 

287 for d in defects[s]: 

288 numDefects += len(defects[s][d]) 

289 self._task._doIngestDefects(self._repo, self._calibRepo, self._task.config.textDefectPath) 

290 

291 self.assertEqual(504, numDefects) # Update if the number of defects in obs_test_data changes 

292 

293 def testDefectIngestDriver(self): 

294 """Test that ingesting defects starting from an abstract dataset adds them to a repository. 

295 """ 

296 self.setUpCalibRegistry() 

297 

298 # second return is calib type 

299 defects = read_all(self._task.config.textDefectPath, IngestionTestSuite.mockCamera)[0] 

300 numDefects = 0 

301 # These are keyes on sensor and validity date 

302 for s in defects: 

303 for d in defects[s]: 

304 numDefects += len(defects[s][d]) 

305 

306 self._task._ingestDefects(self._dataset, self._workspace) 

307 

308 self.assertEqual(504, numDefects) # Update if the number of defects in obs_test_data changes 

309 

310 def testNoFileIngest(self): 

311 """Test that attempts to ingest nothing raise an exception. 

312 """ 

313 files = [] 

314 self.setUpRawRegistry() 

315 

316 with self.assertRaises(RuntimeError): 

317 self._task._doIngestRaws(self._repo, self._calibRepo, files, []) 

318 with self.assertRaises(RuntimeError): 

319 self._task._doIngestCalibs(self._repo, self._calibRepo, files) 

320 

321 self._registerTask.addRow.assert_not_called() 

322 

323 def testNoFileIngestDriver(self): 

324 """Test that attempts to ingest nothing using high-level methods raise an exception. 

325 """ 

326 emptyDir = tempfile.mkdtemp() 

327 self.addCleanup(shutil.rmtree, emptyDir, ignore_errors=True) 

328 self._dataset.rawLocation = self._dataset.calibLocation = emptyDir 

329 with self.assertRaises(RuntimeError): 

330 self._task._ingestRaws(self._dataset, self._workspace) 

331 with self.assertRaises(RuntimeError): 

332 self._task._ingestCalibs(self._dataset, self._workspace) 

333 

334 def testBadFileIngest(self): 

335 """Test that ingestion of raw data ignores blacklisted files. 

336 """ 

337 badFiles = ['raw_v2_fg.fits.gz'] 

338 self.setUpRawRegistry() 

339 

340 files = [os.path.join(self._dataset.rawLocation, datum['file']) 

341 for datum in IngestionTestSuite.rawData] 

342 self._task._doIngestRaws(self._repo, self._calibRepo, files, badFiles) 

343 

344 filteredData = [datum for datum in IngestionTestSuite.rawData if datum['file'] not in badFiles] 

345 self.assertRawRegistryCalls(self._registerTask, filteredData) 

346 

347 for datum in IngestionTestSuite.rawData: 

348 if datum['file'] in badFiles: 

349 dataId = {'visit': datum['visit'], 'expTime': datum['exptime'], 'filter': datum['filter']} 

350 # This call should never happen for badFiles 

351 call = unittest.mock.call(self._registerTask.openRegistry().__enter__(), dataId, 

352 create=False, dryrun=False) 

353 self.assertNotIn(call, self._registerTask.addRow.mock_calls) 

354 

355 def testFindMatchingFiles(self): 

356 """Test that _findMatchingFiles finds the desired files. 

357 """ 

358 testDir = os.path.join(IngestionTestSuite.testData) 

359 

360 self.assertSetEqual( 

361 ingestion._findMatchingFiles(testDir, ['raw_*.fits.gz']), 

362 {os.path.join(testDir, 'raw', f) for f in 

363 {'raw_v1_fg.fits.gz', 'raw_v2_fg.fits.gz', 'raw_v3_fr.fits.gz'}} 

364 ) 

365 self.assertSetEqual( 

366 ingestion._findMatchingFiles(testDir, ['raw_*.fits.gz'], exclude=['*fr*']), 

367 {os.path.join(testDir, 'raw', f) for f in {'raw_v1_fg.fits.gz', 'raw_v2_fg.fits.gz'}} 

368 ) 

369 self.assertSetEqual( 

370 ingestion._findMatchingFiles(testDir, ['raw_*.fits.gz'], exclude=['*_v?_f?.fits.gz']), 

371 set() 

372 ) 

373 self.assertSetEqual( 

374 ingestion._findMatchingFiles(testDir, ['raw_*.fits.gz'], exclude=['obs_test']), 

375 {os.path.join(testDir, 'raw', f) for f in 

376 {'raw_v1_fg.fits.gz', 'raw_v2_fg.fits.gz', 'raw_v3_fr.fits.gz'}} 

377 ) 

378 

379 

380class MemoryTester(lsst.utils.tests.MemoryTestCase): 

381 pass 

382 

383 

384def setup_module(module): 

385 lsst.utils.tests.init() 

386 

387 

388if __name__ == "__main__": 388 ↛ 389line 388 didn't jump to line 389, because the condition on line 388 was never true

389 lsst.utils.tests.init() 

390 unittest.main()