Coverage for tests/test_parsetask.py: 28%

111 statements  

« prev     ^ index     » next       coverage.py v7.1.0, created at 2023-02-05 18:52 -0800

1# This file is part of obs_lsst. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import os.path 

23import unittest 

24 

25from lsst.pipe.tasks.ingest import IngestConfig 

26import lsst.daf.base 

27import lsst.log 

28 

29import lsst.obs.lsst.translators # noqa: F401 -- register the translators 

30from lsst.obs.lsst.latiss import LatissParseTask 

31from lsst.obs.lsst.ts8 import Ts8ParseTask 

32from lsst.obs.lsst.ts3 import Ts3ParseTask 

33from lsst.obs.lsst.phosim import PhosimParseTask, PhosimEimgParseTask 

34from lsst.obs.lsst.imsim import ImsimParseTask 

35from lsst.obs.lsst.ucd import UcdParseTask 

36from lsst.obs.lsst.comCam import LsstComCamParseTask 

37from lsst.obs.lsst.ingest import LsstCamParseTask, LsstCamEimgParseTask 

38 

39TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

40ROOTDIR = os.path.normpath(os.path.join(TESTDIR, os.path.pardir)) 

41DATADIR = os.path.join(ROOTDIR, "data", "input") 

42CONFIGDIR = os.path.join(ROOTDIR, "config") 

43 

44 

45class LsstCamParseTaskTestCase(unittest.TestCase): 

46 

47 def _constructParseTask(self, configdir, name, parseTaskClass): 

48 """Construct a parser task suitable for testing translation methods. 

49 

50 Parameters 

51 ---------- 

52 configdir : `str` 

53 Root of the config directory. This directory must include a 

54 directory of name ``name``. 

55 name : `str` 

56 Name of instrument within data directory and config directory. 

57 parseTaskClass : `lsst.pipe.tasks.ParseTask` 

58 Class, not instance, to use to extract information from header. 

59 

60 Returns 

61 ------- 

62 parseTask : `lsst.pipe.tasks.ParseTask` 

63 Instance of a ``parseTaskClass`` class. 

64 """ 

65 ingestConfig = IngestConfig() 

66 ingestConfig.load(os.path.join(configdir, "ingest.py")) 

67 specificPath = os.path.join(configdir, name, "ingest.py") 

68 if os.path.exists(specificPath): 

69 ingestConfig.load(specificPath) 

70 parser = parseTaskClass(ingestConfig.parse, name=name) 

71 return parser 

72 

73 def assertParseCompare(self, datadir, configdir, name, parseTask, testData): 

74 """Compare parsed output from headers with expected output. 

75 

76 Parameters 

77 ---------- 

78 datadir : `str` 

79 Name of the data directory to use. This directory must include 

80 a directory of name ``name``. 

81 configdir : `str` 

82 Root of the config directory. This directory must include a 

83 directory of name ``name``. 

84 name : `str` 

85 Name of instrument within data directory and config directory. 

86 parseTask : `lsst.pipe.tasks.ParseTask` 

87 Class, not instance, to use to extract information from header. 

88 testData : `tuple` of `tuple` (`str`, `dict`) pairs. 

89 Each pair in the tuple defines the file name to read from the 

90 constructed data directory and a `dict` defining the expected 

91 results from metadata extraction. 

92 

93 Raises 

94 ------ 

95 AssertionError 

96 If the results differ from the expected values. 

97 

98 """ 

99 self.maxDiff = None # Give useful diffs on failure 

100 parser = self._constructParseTask(configdir, name, parseTask) 

101 for fileFragment, expected in testData: 

102 file = os.path.join(DATADIR, name, fileFragment) 

103 with self.subTest(f"Testing {file}"): 

104 phuInfo, infoList = parser.getInfo(file) 

105 print(f"Name: {file}") 

106 for k, v in phuInfo.items(): 

107 print(f"{k}: {v!r}") 

108 self.assertEqual(phuInfo, expected) 

109 

110 def test_parsetask_latiss_translator(self): 

111 """Run the gen 2 metadata extraction code for LATISS""" 

112 test_data = (("raw/2018-09-20/3018092000065-det000.fits", 

113 dict( 

114 expTime=27.0, 

115 object='UNKNOWN', 

116 imageType='UNKNOWN', 

117 detectorName='S00', 

118 dateObs='2018-09-21T06:12:18.210', 

119 date='2018-09-21T06:12:18.210', 

120 dayObs='2018-09-20', 

121 detector=0, 

122 filter='unknown~unknown', 

123 seqNum=65, 

124 visit=3018092000065, 

125 expId=3018092000065, 

126 wavelength=-666, 

127 controller="C", 

128 obsid="AT_C_20180920_000065", 

129 expGroup="3018092000065", 

130 )), 

131 ) 

132 self.assertParseCompare(DATADIR, CONFIGDIR, "latiss", LatissParseTask, test_data) 

133 

134 # Need to test some code paths for translations where we don't have 

135 # example headers. 

136 parseTask = self._constructParseTask(CONFIGDIR, "latiss", LatissParseTask) 

137 

138 md = lsst.daf.base.PropertyList() 

139 md["IMGNAME"] = "AT-O-20180816-00008" 

140 seqNum = parseTask.translate_seqNum(md) 

141 self.assertEqual(seqNum, 8) 

142 

143 del md["IMGNAME"] 

144 md["FILENAME"] = "ats_exp_27_AT_C_20180920_000065.fits" 

145 seqNum = parseTask.translate_seqNum(md) 

146 self.assertEqual(seqNum, 65) 

147 

148 # This will issue a warning 

149 md["FILENAME"] = "ats_exp_27_AT_C.fits" 

150 with self.assertLogs(level="WARNING"): 

151 with lsst.log.UsePythonLogging(): 

152 seqNum = parseTask.translate_seqNum(md) 

153 self.assertEqual(seqNum, 0) 

154 

155 # Test the wavelength code path for non integer wavelength 

156 md["MONOWL"] = 600.4 

157 with self.assertLogs(level="WARNING"): 

158 with lsst.log.UsePythonLogging(): 

159 wl = parseTask.translate_wavelength(md) 

160 self.assertEqual(wl, int(md.getScalar("MONOWL"))) 

161 

162 def test_parsetask_ts8_translator(self): 

163 """Run the gen 2 metadata extraction code for TS8""" 

164 test_data = (("raw/6006D/201807241028453-RTM-010-S11-det067.fits", 

165 dict( 

166 expTime=21.913, 

167 object='UNKNOWN', 

168 imageType='FLAT', 

169 testType='LAMBDA', 

170 lsstSerial='E2V-CCD250-179', 

171 date='2018-07-24T10:28:45.342', 

172 dateObs='2018-07-24T10:28:45.342', 

173 run='6006D', 

174 wavelength=700, 

175 detectorName='S11', 

176 raftName="RTM-010", 

177 detector=67, 

178 dayObs='2018-07-24', 

179 filter='z', 

180 visit=201807241028453, 

181 expId=201807241028453, 

182 testSeqNum=17, 

183 controller="C", 

184 obsid="E2V-CCD250-179_lambda_flat_0700_6006D_20180724102845", 

185 expGroup="201807241028453", 

186 )), 

187 ) 

188 self.assertParseCompare(DATADIR, CONFIGDIR, "ts8", Ts8ParseTask, test_data) 

189 

190 # Need to test some code paths for translations where we don't have 

191 # example headers. 

192 parseTask = self._constructParseTask(CONFIGDIR, "ts8", Ts8ParseTask) 

193 

194 md = lsst.daf.base.PropertyList() 

195 wl = parseTask.translate_testSeqNum(md) 

196 self.assertEqual(wl, 0) 

197 md["SEQNUM"] = 5 

198 wl = parseTask.translate_testSeqNum(md) 

199 self.assertEqual(wl, 5) 

200 

201 def test_parsetask_ts3_translator(self): 

202 """Run the gen 2 metadata extraction code for TS3""" 

203 test_data = (("raw/2016-07-22/201607220607067-R071-S00-det071.fits", 

204 dict( 

205 expTime=30.611, 

206 object='UNKNOWN', 

207 imageType='FLAT', 

208 testType='LAMBDA', 

209 lsstSerial='ITL-3800C-098', 

210 date='2016-07-22T06:07:06.784', 

211 dateObs='2016-07-22T06:07:06.784', 

212 run='2016-07-22', 

213 wavelength=1000, 

214 detectorName='S00', 

215 raftName="R071", 

216 detector=71, 

217 dayObs='2016-07-21', 

218 filter='550CutOn', 

219 visit=201607220607067, 

220 expId=201607220607067, 

221 testSeqNum=67, 

222 controller="C", 

223 obsid="ITL-3800C-098_lambda_flat_1000_067_20160722020740", 

224 expGroup="201607220607067", 

225 )), 

226 ("raw/2018-11-15/201811151255111-R433-S00-det433.fits", 

227 dict( 

228 expTime=44.631, 

229 object='UNKNOWN', 

230 imageType='FLAT', 

231 testType='LAMBDA', 

232 lsstSerial='E2V-CCD250-411', 

233 date='2018-11-15T12:55:11.149', 

234 dateObs='2018-11-15T12:55:11.149', 

235 run='2018-11-15', 

236 wavelength=1000, 

237 detectorName='S00', 

238 raftName="R433", 

239 detector=433, 

240 dayObs='2018-11-15', 

241 filter='550CutOn', 

242 visit=201811151255111, 

243 expId=201811151255111, 

244 testSeqNum=25, 

245 controller="C", 

246 obsid="E2V-CCD250-411_lambda_flat_1000_025_20181115075559", 

247 expGroup="201811151255111", 

248 )), 

249 ) 

250 self.assertParseCompare(DATADIR, CONFIGDIR, "ts3", Ts3ParseTask, test_data) 

251 

252 # Need to test some code paths for translations where we don't have 

253 # example headers. 

254 parseTask = self._constructParseTask(CONFIGDIR, "ts3", Ts8ParseTask) 

255 

256 md = lsst.daf.base.PropertyList() 

257 wl = parseTask.translate_testSeqNum(md) 

258 self.assertEqual(wl, 0) 

259 md["SEQNUM"] = 5 

260 wl = parseTask.translate_testSeqNum(md) 

261 self.assertEqual(wl, 5) 

262 

263 def test_parsetask_imsim_translator(self): 

264 """Run the gen 2 metadata extraction code for Imsim""" 

265 test_data = (("raw/204595/R11/00204595-R11-S20-det042.fits", 

266 dict( 

267 expTime=30.0, 

268 object='UNKNOWN', 

269 imageType='SKYEXP', 

270 testType='IMSIM', 

271 filter='i_sim_1.4', 

272 lsstSerial='LCA-11021_RTM-000', 

273 date='2022-10-05T06:53:26.357', 

274 dateObs='2022-10-05T06:53:26.357', 

275 run='204595', 

276 visit=204595, 

277 expId=204595, 

278 wavelength=-666, 

279 raftName='R11', 

280 detectorName='S20', 

281 detector=42, 

282 snap=0, 

283 controller="S", 

284 obsid="204595", 

285 expGroup="204595", 

286 )), 

287 ) 

288 self.assertParseCompare(DATADIR, CONFIGDIR, "imsim", ImsimParseTask, test_data) 

289 

290 def test_parsetask_phosim_translator(self): 

291 """Run the gen 2 metadata extraction code for Phosim""" 

292 test_data = (("raw/204595/R11/00204595-R11-S20-det042.fits", 

293 dict( 

294 expTime=30.0, 

295 object='UNKNOWN', 

296 imageType='SKYEXP', 

297 testType='PHOSIM', 

298 lsstSerial='R11_S20', 

299 date='2022-10-05T06:53:11.357', 

300 dateObs='2022-10-05T06:53:11.357', 

301 run='204595', 

302 wavelength=-666, 

303 raftName='R11', 

304 detectorName='S20', 

305 detector=42, 

306 snap=0, 

307 filter='i', 

308 visit=204595, 

309 expId=204595, 

310 controller="S", 

311 obsid="204595", 

312 expGroup="204595", 

313 )), 

314 ) 

315 self.assertParseCompare(DATADIR, CONFIGDIR, "phosim", PhosimParseTask, test_data) 

316 

317 def test_parsetask_phosim_eimg_translator(self): 

318 """Run the gen 2 metadata extraction code for Phosim""" 

319 test_data = (("eimage/9006002/E000/R22/eimage_09006002_R22_S00_E000.fits.gz", 

320 dict( 

321 expTime=15.0, 

322 object='UNKNOWN', 

323 imageType='SKYEXP', 

324 testType='PHOSIM', 

325 lsstSerial='R22_S00', 

326 date='2021-12-31T23:59:52.500', 

327 dateObs='2021-12-31T23:59:52.500', 

328 run='9006002', 

329 wavelength=-666, 

330 raftName='R22', 

331 detectorName='S00', 

332 detector=90, 

333 snap=0, 

334 filter='g', 

335 visit=9006002, 

336 expId=9006002, 

337 controller="S", 

338 obsid="9006002", 

339 expGroup="9006002", 

340 )), 

341 ) 

342 self.assertParseCompare(DATADIR, CONFIGDIR, "phosim", PhosimEimgParseTask, test_data) 

343 

344 def test_parsetask_ucd_translator(self): 

345 """Run the gen 2 metadata extraction code for UCDCam""" 

346 self.maxDiff = None 

347 test_data = (("raw/2018-12-05/20181205233148-S00-det000.fits", 

348 dict( 

349 expTime=0.5, 

350 object='UNKNOWN', 

351 imageType='FLAT', 

352 testType='FLAT', 

353 lsstSerial='E2V-CCD250-112-04', 

354 date='2018-12-05T23:31:48.288', 

355 dateObs='2018-12-05T23:31:48.288', 

356 run='2018-12-05', 

357 wavelength=-666, 

358 raftName='R00', 

359 detectorName='S00', 

360 detector=0, 

361 dayObs='2018-12-05', 

362 filter='r', 

363 visit=20181205233148, 

364 expId=20181205233148, 

365 testSeqNum=100, 

366 controller="C", 

367 obsid="E2V-CCD250-112-04_flat_flat_100_20181205153143", 

368 expGroup="20181205233148", 

369 )), 

370 ("raw/2018-05-30/20180530150355-S00-det002.fits", 

371 dict( 

372 expTime=0.5, 

373 object='UNKNOWN', 

374 imageType='FLAT', 

375 testType='FLAT', 

376 lsstSerial='ITL-3800C-002', 

377 date='2018-05-30T15:03:55.872', 

378 dateObs='2018-05-30T15:03:55.872', 

379 run='2018-05-30', 

380 wavelength=-666, 

381 raftName='R02', 

382 detectorName='S00', 

383 detector=2, 

384 dayObs='2018-05-30', 

385 filter='r', 

386 visit=20180530150355, 

387 expId=20180530150355, 

388 testSeqNum=100, 

389 controller="C", 

390 obsid="ITL-3800C-002_flat_flat_100_20180530080354", 

391 expGroup="20180530150355", 

392 )), 

393 ) 

394 self.assertParseCompare(DATADIR, CONFIGDIR, "ucd", UcdParseTask, test_data) 

395 

396 # Need to test some code paths for translations where we don't have 

397 # example headers. 

398 parseTask = self._constructParseTask(CONFIGDIR, "ucd", UcdParseTask) 

399 

400 md = lsst.daf.base.PropertyList() 

401 wl = parseTask.translate_testSeqNum(md) 

402 self.assertEqual(wl, 0) 

403 md["SEQNUM"] = 5 

404 wl = parseTask.translate_testSeqNum(md) 

405 self.assertEqual(wl, 5) 

406 

407 def test_parsetask_lsstCam_translator(self): 

408 """Run the gen 2 metadata extraction code for lsstCam""" 

409 test_data = (("raw/2019-03-19/3019031900001/3019031900001-R10-S02-det029.fits", 

410 dict( 

411 expTime=0.0, 

412 object='UNKNOWN', 

413 imageType='BIAS', 

414 testType='BIAS', 

415 seqNum=1, 

416 dayObs="2019-03-19", 

417 filter='unknown', 

418 lsstSerial='ITL-3800C-041', 

419 date='2019-03-19T15:50:28.145', 

420 dateObs='2019-03-19T15:50:28.145', 

421 run='unknown', 

422 visit=3019031900001, 

423 expId=3019031900001, 

424 wavelength=-666, 

425 raftName='R10', 

426 detectorName='S02', 

427 detector=29, 

428 snap=0, 

429 controller="C", 

430 obsid="MC_C_20190319_000001", 

431 expGroup="3019031900001", 

432 )), 

433 ("raw/2019-03-22/3019032200002/3019032200002-R10-S22-det035.fits", 

434 dict( 

435 expTime=1.0, 

436 object='UNKNOWN', 

437 imageType='FLAT', 

438 testType='FLAT', 

439 seqNum=2, 

440 dayObs="2019-03-22", 

441 filter='SDSSi~ND_OD0.5', 

442 lsstSerial='ITL-3800C-103', 

443 date='2019-03-22T15:31:01.904', 

444 dateObs='2019-03-22T15:31:01.904', 

445 run='6489D', 

446 visit=3019032200002, 

447 expId=3019032200002, 

448 wavelength=-666, 

449 raftName='R10', 

450 detectorName='S22', 

451 detector=35, 

452 snap=0, 

453 controller="C", 

454 obsid="MC_C_20190322_000002", 

455 expGroup="3019032200002", 

456 )), 

457 ) 

458 self.assertParseCompare(DATADIR, CONFIGDIR, "lsstCam", LsstCamParseTask, test_data) 

459 

460 def test_parsetask_comCam_translator(self): 

461 """Run the gen 2 metadata extraction code for comCam""" 

462 test_data = (("raw/2019-05-30/3019053000001/3019053000001-R22-S00-det000.fits", 

463 dict( 

464 expTime=0.0, 

465 object='UNKNOWN', 

466 imageType='BIAS', 

467 testType='BIAS', 

468 seqNum=1, 

469 dayObs="2019-05-30", 

470 filter='unknown', 

471 lsstSerial='ITL-3800C-229', 

472 date='2019-05-31T02:38:37.384', 

473 dateObs='2019-05-31T02:38:37.384', 

474 run='unknown', 

475 visit=3019053000001, 

476 expId=3019053000001, 

477 wavelength=-666, 

478 raftName='R22', 

479 detectorName='S00', 

480 detector=0, 

481 snap=0, 

482 controller="C", 

483 obsid="CC_C_20190530_000001", 

484 expGroup="3019053000001", 

485 )), 

486 ) 

487 self.assertParseCompare(DATADIR, CONFIGDIR, "comCam", LsstComCamParseTask, test_data) 

488 

489 def test_parsetask_lsstCamEimg_translator(self): 

490 """Run the gen2 metadata extraction code for lsstCam e-image""" 

491 test_data = (("eimage/4000021706001/E000/R22/eimage_4000021706001_R22_S00_E000.fits.gz", 

492 dict( 

493 controller='H', 

494 date='2000-02-17T02:14:40.319', 

495 dateObs='2000-02-17T02:14:40.319', 

496 dayObs='2000-02-17', 

497 detector=90, 

498 detectorName='S00', 

499 expGroup='4000021706001', 

500 expId=4000021706001, 

501 expTime=1.0, 

502 filter='g', 

503 imageType='SKYEXP', 

504 lsstSerial='E2V-CCD250-369', 

505 object='UNKNOWN', 

506 obsid='MC_H_20000217_006001', 

507 raftName='R22', 

508 run='9006001', 

509 seqNum=6001, 

510 snap=0, 

511 testType='PHOSIM', 

512 visit=4000021706001, 

513 wavelength=-666, 

514 )), 

515 ) 

516 self.assertParseCompare(DATADIR, CONFIGDIR, "lsstCam", LsstCamEimgParseTask, test_data) 

517 

518 

519if __name__ == "__main__": 519 ↛ 520line 519 didn't jump to line 520, because the condition on line 519 was never true

520 unittest.main()