Coverage for tests/test_ingest.py: 59%

127 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-12 02:12 -0700

1# This file is part of obs_lsst. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for LSST raw data ingest. 

23""" 

24 

25import unittest 

26import os 

27import tempfile 

28import lsst.utils.tests 

29 

30from lsst.afw.math import flipImage 

31from lsst.afw.cameraGeom import AmplifierGeometryComparison 

32from lsst.daf.butler import Butler, DataCoordinate 

33from lsst.daf.butler.cli.butler import cli as butlerCli 

34from lsst.daf.butler.cli.utils import LogCliRunner 

35from lsst.obs.base.ingest_tests import IngestTestBase 

36from lsst.ip.isr import PhotodiodeCalib 

37import lsst.afw.cameraGeom.testUtils # for injected test asserts 

38import lsst.obs.lsst 

39 

40TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

41DATAROOT = os.path.join(TESTDIR, os.path.pardir, "data", "input") 

42 

43 

44class LatissIngestTestCase(IngestTestBase, lsst.utils.tests.TestCase): 

45 

46 curatedCalibrationDatasetTypes = ("camera", "defects") 

47 instrumentClassName = "lsst.obs.lsst.Latiss" 

48 ingestDir = TESTDIR 

49 file = os.path.join(DATAROOT, "latiss", "raw", "2018-09-20", "3018092000065-det000.fits") 

50 dataIds = [dict(instrument="LATISS", exposure=3018092000065, detector=0)] 

51 filterLabel = lsst.afw.image.FilterLabel(band="unknown", physical="unknown~unknown") 

52 

53 def checkRepo(self, files=None): 

54 # Test amp parameter implementation for the LSST raw formatter. This 

55 # is the same for all instruments, so repeating it in other test cases 

56 # is wasteful. 

57 butler = Butler(self.root, run=self.outputRun) 

58 ref = butler.find_dataset("raw", self.dataIds[0]) 

59 full_assembled = butler.get(ref) 

60 unassembled_detector = self.instrumentClass().getCamera()[ref.dataId["detector"]] 

61 assembled_detector = full_assembled.getDetector() 

62 for unassembled_amp, assembled_amp in zip(unassembled_detector, assembled_detector): 

63 # Check that we're testing what we think we're testing: these 

64 # amps should differ in assembly state (offsets, flips), and they 

65 # _may_ differ in fundamental geometry if we had to patch the 

66 # overscan region sizes. 

67 comparison = unassembled_amp.compareGeometry(assembled_amp) 

68 self.assertTrue(comparison & AmplifierGeometryComparison.ASSEMBLY_DIFFERS) 

69 assembled_subimage = butler.get(ref, parameters={"amp": assembled_amp}) 

70 unassembled_subimage = butler.get(ref, parameters={"amp": unassembled_amp.getName()}) 

71 self.assertEqual(len(assembled_subimage.getDetector()), 1) 

72 self.assertEqual(len(unassembled_subimage.getDetector()), 1) 

73 self.assertEqual(len(assembled_subimage.getDetector()), 1) 

74 self.assertEqual(len(unassembled_subimage.getDetector()), 1) 

75 self.assertImagesEqual(assembled_subimage.image, full_assembled.image[assembled_amp.getRawBBox()]) 

76 self.assertImagesEqual( 

77 unassembled_subimage.image, 

78 flipImage( 

79 full_assembled.image[assembled_amp.getRawBBox()], 

80 flipLR=unassembled_amp.getRawFlipX(), 

81 flipTB=unassembled_amp.getRawFlipY(), 

82 ), 

83 ) 

84 self.assertAmplifiersEqual(assembled_subimage.getDetector()[0], assembled_amp) 

85 if comparison & comparison.REGIONS_DIFFER: 

86 # We needed to patch overscans, but unassembled_amp (which 

87 # comes straight from the camera) won't have those patches, so 

88 # we can't compare it to the amp attached to 

89 # unassembled_subimage (which does have those patches). 

90 comparison2 = unassembled_subimage.getDetector()[0].compareGeometry(unassembled_amp) 

91 

92 self.assertTrue(comparison2 & AmplifierGeometryComparison.REGIONS_DIFFER) 

93 # ...and that unassembled_subimage's amp has the same regions 

94 # (after accounting for assembly/orientation) as assembled_amp. 

95 comparison3 = unassembled_subimage.getDetector()[0].compareGeometry(assembled_amp) 

96 self.assertTrue(comparison3 & AmplifierGeometryComparison.ASSEMBLY_DIFFERS) 

97 self.assertFalse(comparison3 & AmplifierGeometryComparison.REGIONS_DIFFER) 

98 else: 

99 self.assertAmplifiersEqual(unassembled_subimage.getDetector()[0], unassembled_amp) 

100 

101 

102class Ts3IngestTestCase(IngestTestBase, lsst.utils.tests.TestCase): 

103 

104 curatedCalibrationDatasetTypes = ("camera",) 

105 instrumentClassName = "lsst.obs.lsst.LsstTS3" 

106 ingestDir = TESTDIR 

107 file = os.path.join(DATAROOT, "ts3", "raw", "2018-11-15", "201811151255111-R433-S00-det433.fits") 

108 dataIds = [dict(instrument="LSST-TS3", exposure=201811151255111, detector=433)] 

109 filterLabel = lsst.afw.image.FilterLabel(physical="550CutOn") 

110 

111 

112class ComCamIngestTestCase(IngestTestBase, lsst.utils.tests.TestCase): 

113 

114 curatedCalibrationDatasetTypes = ("camera",) 

115 instrumentClassName = "lsst.obs.lsst.LsstComCam" 

116 ingestDir = TESTDIR 

117 file = os.path.join(DATAROOT, "comCam", "raw", "2019-05-30", 

118 "3019053000001", "3019053000001-R22-S00-det000.fits") 

119 dataIds = [dict(instrument="LSSTComCam", exposure=3019053000001, detector=0)] 

120 filterLabel = lsst.afw.image.FilterLabel(physical="unknown", band="unknown") 

121 

122 

123class ComCamSimIngestTestCase(IngestTestBase, lsst.utils.tests.TestCase): 

124 

125 curatedCalibrationDatasetTypes = ("camera",) 

126 instrumentClassName = "lsst.obs.lsst.LsstComCamSim" 

127 ingestDir = TESTDIR 

128 file = os.path.join(DATAROOT, "comCamSim", "raw", "2024-04-04", 

129 "7024040400780", "CC_S_20240404_000780_R22_S01.fits") 

130 dataIds = [dict(instrument="LSSTComCamSim", exposure=7024040400780, detector=1)] 

131 filterLabel = lsst.afw.image.FilterLabel(physical="r_03", band="r") 

132 

133 @property 

134 def visits(self): 

135 butler = Butler(self.root, collections=[self.outputRun]) 

136 return { 

137 DataCoordinate.standardize( 

138 instrument="LSSTComCamSim", 

139 visit=7024040400780, 

140 universe=butler.dimensions 

141 ): [ 

142 DataCoordinate.standardize( 

143 instrument="LSSTComCamSim", 

144 exposure=7024040400780, 

145 universe=butler.dimensions 

146 ) 

147 ] 

148 } 

149 

150 

151class LSSTCamIngestTestCase(IngestTestBase, lsst.utils.tests.TestCase): 

152 

153 curatedCalibrationDatasetTypes = ("camera",) 

154 instrumentClassName = "lsst.obs.lsst.LsstCam" 

155 ingestDir = TESTDIR 

156 file = os.path.join(DATAROOT, "lsstCam", "raw", "2019-03-19", 

157 "3019031900001", "3019031900001-R10-S02-det029.fits") 

158 dataIds = [dict(instrument="LSSTCam", exposure=3019031900001, detector=29)] 

159 filterLabel = lsst.afw.image.FilterLabel(physical="unknown", band="unknown") 

160 

161 

162class LSSTCamSimIngestTestCase(IngestTestBase, lsst.utils.tests.TestCase): 

163 

164 curatedCalibrationDatasetTypes = ("camera",) 

165 instrumentClassName = "lsst.obs.lsst.LsstCamSim" 

166 ingestDir = TESTDIR 

167 file = os.path.join(DATAROOT, "lsstCamSim", "raw", "2024-03-21", 

168 "7024032100720", "7024032100720-R22-S11-det094.fits.fz") 

169 dataIds = [dict(instrument="LSSTCamSim", exposure=7024032100720, detector=94)] 

170 filterLabel = lsst.afw.image.FilterLabel(physical="r_57", band="r") 

171 

172 @property 

173 def visits(self): 

174 butler = Butler(self.root, collections=[self.outputRun]) 

175 return { 

176 DataCoordinate.standardize( 

177 instrument="LSSTCamSim", 

178 visit=7024032100720, 

179 universe=butler.dimensions 

180 ): [ 

181 DataCoordinate.standardize( 

182 instrument="LSSTCamSim", 

183 exposure=7024032100720, 

184 universe=butler.dimensions 

185 ) 

186 ] 

187 } 

188 

189 

190class LSSTCamPhotodiodeIngestTestCase(lsst.utils.tests.TestCase): 

191 instrumentClassName = "lsst.obs.lsst.LsstCam" 

192 rawIngestTask = "lsst.obs.base.RawIngestTask" 

193 ingestDir = TESTDIR 

194 file = os.path.join(DATAROOT, "lsstCam", "raw", "2021-12-12", 

195 "30211212000310", "30211212000310-R22-S22-det098.fits") 

196 dataIds = [dict(instrument="LSSTCam", exposure=3021121200310, detector=98)] 

197 filterLabel = lsst.afw.image.FilterLabel(physical="SDSSi", band="i") 

198 pdPath = os.path.join(DATAROOT, "lsstCam", "raw") 

199 

200 def setUp(self): 

201 """Setup for lightweight photodiode ingest task. 

202 

203 This will create the repo and register the instrument. 

204 """ 

205 self.root = tempfile.mkdtemp(dir=self.ingestDir) 

206 

207 # Create Repo 

208 runner = LogCliRunner() 

209 result = runner.invoke(butlerCli, ["create", self.root]) 

210 self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}") 

211 

212 # Register Instrument 

213 runner = LogCliRunner() 

214 result = runner.invoke(butlerCli, ["register-instrument", self.root, self.instrumentClassName]) 

215 self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}") 

216 

217 def testPhotodiodeFailure(self): 

218 """Test ingest to a repo missing exposure information will raise. 

219 """ 

220 runner = LogCliRunner() 

221 result = runner.invoke( 

222 butlerCli, 

223 [ 

224 "ingest-photodiode", 

225 self.root, 

226 self.instrumentClassName, 

227 self.pdPath, 

228 ], 

229 ) 

230 self.assertEqual(result.exit_code, 1, f"output: {result.output} exception: {result.exception}") 

231 

232 def testPhotodiode(self): 

233 """Test ingest to a repo with the exposure information will not raise. 

234 """ 

235 # Ingest raw to provide exposure information. 

236 outputRun = "raw_ingest_" + self.id() 

237 runner = LogCliRunner() 

238 result = runner.invoke( 

239 butlerCli, 

240 [ 

241 "ingest-raws", 

242 self.root, 

243 self.file, 

244 "--output-run", 

245 outputRun, 

246 "--ingest-task", 

247 self.rawIngestTask, 

248 ], 

249 ) 

250 self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}") 

251 

252 # Ingest photodiode matching this exposure. 

253 runner = LogCliRunner() 

254 result = runner.invoke( 

255 butlerCli, 

256 [ 

257 "ingest-photodiode", 

258 self.root, 

259 self.instrumentClassName, 

260 self.pdPath, 

261 ], 

262 ) 

263 self.assertEqual(result.exit_code, 0, f"output: {result.output} exception: {result.exception}") 

264 

265 # Confirm that we can retrieve the ingested photodiode, and 

266 # that it has the correct type. 

267 butler = Butler(self.root, run="LSSTCam/calib/photodiode") 

268 getResult = butler.get('photodiode', dataId=self.dataIds[0]) 

269 self.assertIsInstance(getResult, PhotodiodeCalib) 

270 

271 

272def setup_module(module): 

273 lsst.utils.tests.init() 

274 

275 

276if __name__ == "__main__": 276 ↛ 277line 276 didn't jump to line 277, because the condition on line 276 was never true

277 lsst.utils.tests.init() 

278 unittest.main()