Coverage for tests / test_ingest.py: 53%

1# This file is part of obs_lsst. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for LSST raw data ingest. 

23""" 

24 

25import unittest 

26import os 

27import tempfile 

28import lsst.utils.tests 

29 

30from lsst.afw.math import flipImage 

31from lsst.afw.cameraGeom import AmplifierGeometryComparison 

32from lsst.daf.butler import Butler, DataCoordinate 

33from lsst.daf.butler.cli.butler import cli as butlerCli 

34from lsst.daf.butler.cli.utils import LogCliRunner 

35from lsst.obs.base.ingest_tests import IngestTestBase 

36from lsst.ip.isr import PhotodiodeCalib, ShutterMotionProfile 

37import lsst.afw.cameraGeom.testUtils # for injected test asserts 

38import lsst.obs.lsst 

39 

40TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

41DATAROOT = os.path.join(TESTDIR, "data", "input") 

42 

43 

44class LatissIngestTestCase(IngestTestBase, lsst.utils.tests.TestCase): 

45 

46 curatedCalibrationDatasetTypes = ("camera", "defects") 

47 instrumentClassName = "lsst.obs.lsst.Latiss" 

48 ingestDir = TESTDIR 

49 file = os.path.join(DATAROOT, "latiss", "raw", "2018-09-20", "3018092000065-det000.fits") 

50 dataIds = [dict(instrument="LATISS", exposure=3018092000065, detector=0)] 

51 filterLabel = lsst.afw.image.FilterLabel(band="unknown", physical="unknown~unknown") 

52 

53 def checkRepo(self, files=None): 

54 # Test amp parameter implementation for the LSST raw formatter. This 

55 # is the same for all instruments, so repeating it in other test cases 

56 # is wasteful. 

57 with Butler.from_config(self.root, run=self.outputRun) as butler: 

58 ref = butler.find_dataset("raw", self.dataIds[0]) 

59 full_assembled = butler.get(ref) 

60 unassembled_detector = self.instrumentClass().getCamera()[ref.dataId["detector"]] 

61 assembled_detector = full_assembled.getDetector() 

62 for unassembled_amp, assembled_amp in zip(unassembled_detector, assembled_detector): 

63 # Check that we're testing what we think we're testing: these 

64 # amps should differ in assembly state (offsets, flips), and 

65 # they _may_ differ in fundamental geometry if we had to patch 

66 # the overscan region sizes. 

67 comparison = unassembled_amp.compareGeometry(assembled_amp) 

68 self.assertTrue(comparison & AmplifierGeometryComparison.ASSEMBLY_DIFFERS) 

69 assembled_subimage = butler.get(ref, parameters={"amp": assembled_amp}) 

70 unassembled_subimage = butler.get(ref, parameters={"amp": unassembled_amp.getName()}) 

71 self.assertEqual(len(assembled_subimage.getDetector()), 1) 

72 self.assertEqual(len(unassembled_subimage.getDetector()), 1) 

73 self.assertEqual(len(assembled_subimage.getDetector()), 1) 

74 self.assertEqual(len(unassembled_subimage.getDetector()), 1) 

75 self.assertImagesEqual( 

76 assembled_subimage.image, full_assembled.image[assembled_amp.getRawBBox()] 

77 ) 

78 self.assertImagesEqual( 

79 unassembled_subimage.image, 

80 flipImage( 

81 full_assembled.image[assembled_amp.getRawBBox()], 

82 flipLR=unassembled_amp.getRawFlipX(), 

83 flipTB=unassembled_amp.getRawFlipY(), 

84 ), 

85 ) 

86 self.assertAmplifiersEqual(assembled_subimage.getDetector()[0], assembled_amp) 

87 if comparison & comparison.REGIONS_DIFFER: 

88 # We needed to patch overscans, but unassembled_amp (which 

89 # comes straight from the camera) won't have those patches, 

90 # so we can't compare it to the amp attached to 

91 # unassembled_subimage (which does have those patches). 

92 comparison2 = unassembled_subimage.getDetector()[0].compareGeometry(unassembled_amp) 

93 

94 self.assertTrue(comparison2 & AmplifierGeometryComparison.REGIONS_DIFFER) 

95 # ...and that unassembled_subimage's amp has the same 

96 # regions (after accounting for assembly/orientation) as 

97 # assembled_amp. 

98 comparison3 = unassembled_subimage.getDetector()[0].compareGeometry(assembled_amp) 

99 self.assertTrue(comparison3 & AmplifierGeometryComparison.ASSEMBLY_DIFFERS) 

100 self.assertFalse(comparison3 & AmplifierGeometryComparison.REGIONS_DIFFER) 

101 else: 

102 self.assertAmplifiersEqual(unassembled_subimage.getDetector()[0], unassembled_amp) 

103 

104 

105class Ts3IngestTestCase(IngestTestBase, lsst.utils.tests.TestCase): 

106 

107 curatedCalibrationDatasetTypes = ("camera",) 

108 instrumentClassName = "lsst.obs.lsst.LsstTS3" 

109 ingestDir = TESTDIR 

110 file = os.path.join(DATAROOT, "ts3", "raw", "2018-11-15", "201811151255111-R433-S00-det433.fits") 

111 dataIds = [dict(instrument="LSST-TS3", exposure=201811151255111, detector=433)] 

112 filterLabel = lsst.afw.image.FilterLabel(physical="550CutOn") 

113 

114 

115class ComCamIngestTestCase(IngestTestBase, lsst.utils.tests.TestCase): 

116 

117 curatedCalibrationDatasetTypes = ("camera",) 

118 instrumentClassName = "lsst.obs.lsst.LsstComCam" 

119 ingestDir = TESTDIR 

120 file = os.path.join(DATAROOT, "comCam", "raw", "2019-05-30", 

121 "3019053000001", "3019053000001-R22-S00-det000.fits") 

122 dataIds = [dict(instrument="LSSTComCam", exposure=3019053000001, detector=0)] 

123 filterLabel = lsst.afw.image.FilterLabel(physical="unknown", band="unknown") 

124 

125 

126class ComCamSimIngestTestCase(IngestTestBase, lsst.utils.tests.TestCase): 

127 

128 curatedCalibrationDatasetTypes = ("camera",) 

129 instrumentClassName = "lsst.obs.lsst.LsstComCamSim" 

130 ingestDir = TESTDIR 

131 file = os.path.join(DATAROOT, "comCamSim", "raw", "2024-04-04", 

132 "7024040400780", "CC_S_20240404_000780_R22_S01.fits") 

133 dataIds = [dict(instrument="LSSTComCamSim", exposure=7024040400780, detector=1)] 

134 filterLabel = lsst.afw.image.FilterLabel(physical="r_03", band="r") 

135 

136 @property 

137 def visits(self): 

138 with Butler.from_config(self.root, collections=[self.outputRun]) as butler: 

139 return { 

140 DataCoordinate.standardize( 

141 instrument="LSSTComCamSim", 

142 visit=7024040400780, 

143 universe=butler.dimensions 

144 ): [ 

145 DataCoordinate.standardize( 

146 instrument="LSSTComCamSim", 

147 exposure=7024040400780, 

148 universe=butler.dimensions 

149 ) 

150 ] 

151 } 

152 

153 

154class LSSTCamIngestTestCase(IngestTestBase, lsst.utils.tests.TestCase): 

155 

156 curatedCalibrationDatasetTypes = ("camera",) 

157 instrumentClassName = "lsst.obs.lsst.LsstCam" 

158 ingestDir = TESTDIR 

159 file = os.path.join(DATAROOT, "lsstCam", "raw", "2019-03-19", 

160 "3019031900001", "3019031900001-R10-S02-det029.fits") 

161 dataIds = [dict(instrument="LSSTCam", exposure=3019031900001, detector=29)] 

162 filterLabel = lsst.afw.image.FilterLabel(physical="unknown", band="unknown") 

163 

164 

165class LSSTCamSimIngestTestCase(IngestTestBase, lsst.utils.tests.TestCase): 

166 

167 curatedCalibrationDatasetTypes = ("camera",) 

168 instrumentClassName = "lsst.obs.lsst.LsstCamSim" 

169 ingestDir = TESTDIR 

170 file = os.path.join(DATAROOT, "lsstCamSim", "raw", "2024-03-21", 

171 "7024032100720", "7024032100720-R22-S11-det094.fits.fz") 

172 dataIds = [dict(instrument="LSSTCamSim", exposure=7024032100720, detector=94)] 

173 filterLabel = lsst.afw.image.FilterLabel(physical="r_57", band="r") 

174 

175 @property 

176 def visits(self): 

177 with Butler.from_config(self.root, collections=[self.outputRun]) as butler: 

178 return { 

179 DataCoordinate.standardize( 

180 instrument="LSSTCamSim", 

181 visit=7024032100720, 

182 universe=butler.dimensions 

183 ): [ 

184 DataCoordinate.standardize( 

185 instrument="LSSTCamSim", 

186 exposure=7024032100720, 

187 universe=butler.dimensions 

188 ) 

189 ] 

190 } 

191 

192 

193class LSSTCamPhotodiodeIngestTestCase(lsst.utils.tests.TestCase): 

194 instrumentClassName = "lsst.obs.lsst.LsstCam" 

195 rawIngestTask = "lsst.obs.base.RawIngestTask" 

196 ingestDir = TESTDIR 

197 file = os.path.join(DATAROOT, "lsstCam", "raw", "2021-12-12", 

198 "30211212000310", "30211212000310-R22-S22-det098.fits") 

199 dataIds = [dict(instrument="LSSTCam", exposure=3021121200310, detector=98)] 

200 filterLabel = lsst.afw.image.FilterLabel(physical="SDSSi", band="i") 

201 pdPath = os.path.join(DATAROOT, "lsstCam", "raw") 

202 

203 def setUp(self): 

204 """Setup for lightweight photodiode ingest task. 

205 

206 This will create the repo and register the instrument. 

207 """ 

208 self.root = tempfile.mkdtemp(dir=self.ingestDir) 

209 

210 # Create Repo 

211 runner = LogCliRunner() 

212 result = runner.invoke(butlerCli, ["create", self.root]) 

213 self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}") 

214 

215 # Register Instrument 

216 runner = LogCliRunner() 

217 result = runner.invoke(butlerCli, ["register-instrument", self.root, self.instrumentClassName]) 

218 self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}") 

219 

220 def testPhotodiodeFailure(self): 

221 """Test ingest to a repo missing exposure information will raise. 

222 """ 

223 runner = LogCliRunner() 

224 result = runner.invoke( 

225 butlerCli, 

226 [ 

227 "ingest-photodiode", 

228 self.root, 

229 self.instrumentClassName, 

230 self.pdPath, 

231 ], 

232 ) 

233 self.assertEqual(result.exit_code, 1, f"output: {result.output} exception: {result.exception}") 

234 

235 def testPhotodiode(self): 

236 """Test ingest to a repo with the exposure information will not raise. 

237 """ 

238 # Ingest raw to provide exposure information. 

239 outputRun = "raw_ingest_" + self.id() 

240 runner = LogCliRunner() 

241 result = runner.invoke( 

242 butlerCli, 

243 [ 

244 "ingest-raws", 

245 self.root, 

246 self.file, 

247 "--output-run", 

248 outputRun, 

249 "--ingest-task", 

250 self.rawIngestTask, 

251 ], 

252 ) 

253 self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}") 

254 

255 # Ingest photodiode matching this exposure. 

256 runner = LogCliRunner() 

257 result = runner.invoke( 

258 butlerCli, 

259 [ 

260 "ingest-photodiode", 

261 self.root, 

262 self.instrumentClassName, 

263 self.pdPath, 

264 ], 

265 ) 

266 self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}") 

267 

268 # Confirm that we can retrieve the ingested photodiode, and 

269 # that it has the correct type. 

270 with Butler.from_config(self.root, run="LSSTCam/calib/photodiode") as butler: 

271 getResult = butler.get('photodiode', dataId=self.dataIds[0]) 

272 self.assertIsInstance(getResult, PhotodiodeCalib) 

273 

274 

275class LSSTCamShutterMotionIngestTestCase(lsst.utils.tests.TestCase): 

276 instrumentClassName = "lsst.obs.lsst.LsstCam" 

277 rawIngestTask = "lsst.obs.base.RawIngestTask" 

278 ingestDir = TESTDIR 

279 file = os.path.join(DATAROOT, "lsstCam", "raw", "2021-12-12", 

280 "30211212000310", "30211212000310-R22-S22-det098.fits") 

281 dataIds = [dict(instrument="LSSTCam", exposure=3021121200310, detector=98)] 

282 filterLabel = lsst.afw.image.FilterLabel(physical="SDSSi", band="i") 

283 smpPath = os.path.join(DATAROOT, "lsstCam", "raw") 

284 

285 def setUp(self): 

286 """Setup for lightweight shutter motion ingest task. 

287 

288 This will create the repo and register the instrument. 

289 """ 

290 self.root = tempfile.mkdtemp(dir=self.ingestDir) 

291 

292 # Create Repo 

293 runner = LogCliRunner() 

294 result = runner.invoke(butlerCli, ["create", self.root]) 

295 self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}") 

296 

297 # Register Instrument 

298 runner = LogCliRunner() 

299 result = runner.invoke(butlerCli, ["register-instrument", self.root, self.instrumentClassName]) 

300 self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}") 

301 

302 def testShutterMotionFailure(self): 

303 """Test ingest to a repo missing exposure information will raise. 

304 """ 

305 runner = LogCliRunner() 

306 result = runner.invoke( 

307 butlerCli, 

308 [ 

309 "ingest-shuttermotion", 

310 self.root, 

311 self.instrumentClassName, 

312 self.smpPath, 

313 "-c", 

314 "doRaiseOnMissingExposure=True", 

315 ], 

316 ) 

317 self.assertEqual(result.exit_code, 1, f"output: {result.output} exception: {result.exception}") 

318 

319 def testShutterMotion(self): 

320 """Test ingest to a repo with the exposure information will not raise. 

321 """ 

322 # Ingest raw to provide exposure information. 

323 outputRun = "raw_ingest_" + self.id() 

324 runner = LogCliRunner() 

325 result = runner.invoke( 

326 butlerCli, 

327 [ 

328 "ingest-raws", 

329 self.root, 

330 self.file, 

331 "--output-run", 

332 outputRun, 

333 "--ingest-task", 

334 self.rawIngestTask, 

335 ], 

336 ) 

337 self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}") 

338 

339 # Ingest photodiode matching this exposure. 

340 runner = LogCliRunner() 

341 result = runner.invoke( 

342 butlerCli, 

343 [ 

344 "ingest-shuttermotion", 

345 self.root, 

346 self.instrumentClassName, 

347 self.smpPath, 

348 "-c", 

349 "doRaiseOnMissingExposure=True", 

350 ], 

351 ) 

352 self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}") 

353 

354 # Confirm that we can retrieve the ingested photodiode, and 

355 # that it has the correct type. 

356 with Butler.from_config(self.root, run="LSSTCam/calib/shutterMotion") as butler: 

357 getResult = butler.get('shutterMotionProfileOpen', dataId=self.dataIds[0]) 

358 self.assertIsInstance(getResult, ShutterMotionProfile) 

359 

360 

361def setup_module(module): 

362 lsst.utils.tests.init() 

363 

364 

365if __name__ == "__main__": 365 ↛ 366line 365 didn't jump to line 366 because the condition on line 365 was never true

366 lsst.utils.tests.init() 

367 unittest.main()