Coverage for tests/test_instrument.py: 21%

143 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-19 04:46 -0700

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Tests of the Instrument class. 

23""" 

24 

25import datetime 

26import math 

27import unittest 

28 

29from lsst.daf.butler import DataCoordinate, DimensionPacker, DimensionUniverse, Registry, RegistryConfig 

30from lsst.daf.butler.formatters.json import JsonFormatter 

31from lsst.pex.config import Config 

32from lsst.pipe.base import Instrument 

33from lsst.utils.introspection import get_full_type_name 

34 

35 

36class DummyInstrument(Instrument): 

37 @classmethod 

38 def getName(cls): 

39 return "DummyInstrument" 

40 

41 def register(self, registry, update=False): 

42 detector_max = 2 

43 visit_max = 10 

44 exposure_max = 8 

45 record = { 

46 "instrument": self.getName(), 

47 "class_name": get_full_type_name(DummyInstrument), 

48 "detector_max": detector_max, 

49 "visit_max": visit_max, 

50 "exposure_max": exposure_max, 

51 } 

52 with registry.transaction(): 

53 registry.syncDimensionData("instrument", record, update=update) 

54 

55 def getRawFormatter(self, dataId): 

56 return JsonFormatter 

57 

58 

59class BadInstrument(DummyInstrument): 

60 """Instrument with wrong class name.""" 

61 

62 raw_definition = ("raw2", ("instrument", "detector", "exposure"), "StructuredDataDict") 

63 

64 @classmethod 

65 def getName(cls): 

66 return "BadInstrument" 

67 

68 def register(self, registry, update=False): 

69 # Register a bad class name 

70 record = { 

71 "instrument": self.getName(), 

72 "class_name": "builtins.str", 

73 "detector_max": 1, 

74 } 

75 registry.syncDimensionData("instrument", record, update=update) 

76 

77 

78class UnimportableInstrument(DummyInstrument): 

79 """Instrument with class name that does not exist.""" 

80 

81 @classmethod 

82 def getName(cls): 

83 return "NoImportInstr" 

84 

85 def register(self, registry, update=False): 

86 # Register a bad class name 

87 record = { 

88 "instrument": self.getName(), 

89 "class_name": "not.importable", 

90 "detector_max": 1, 

91 } 

92 registry.syncDimensionData("instrument", record, update=update) 

93 

94 

95class DimensionPackerTestConfig(Config): 

96 packer = Instrument.make_dimension_packer_config_field() 

97 

98 

99class InstrumentTestCase(unittest.TestCase): 

100 """Test for Instrument.""" 

101 

102 def setUp(self): 

103 self.instrument = DummyInstrument() 

104 self.name = "DummyInstrument" 

105 

106 def test_basics(self): 

107 self.assertEqual(self.instrument.getName(), self.name) 

108 self.assertEqual(self.instrument.getRawFormatter({}), JsonFormatter) 

109 self.assertIsNone(DummyInstrument.raw_definition) 

110 raw = BadInstrument.raw_definition 

111 self.assertEqual(raw[2], "StructuredDataDict") 

112 

113 def test_register(self): 

114 """Test that register() sets appropriate Dimensions.""" 

115 registryConfig = RegistryConfig() 

116 registryConfig["db"] = "sqlite://" 

117 registry = Registry.createFromConfig(registryConfig) 

118 # Check that the registry starts out empty. 

119 self.instrument.importAll(registry) 

120 self.assertFalse(list(registry.queryDimensionRecords("instrument"))) 

121 

122 # Register and check again. 

123 self.instrument.register(registry) 

124 instruments = list(registry.queryDimensionRecords("instrument")) 

125 self.assertEqual(len(instruments), 1) 

126 self.assertEqual(instruments[0].name, self.name) 

127 self.assertEqual(instruments[0].detector_max, 2) 

128 self.assertIn("DummyInstrument", instruments[0].class_name) 

129 

130 self.instrument.importAll(registry) 

131 from_registry = DummyInstrument.fromName("DummyInstrument", registry) 

132 self.assertIsInstance(from_registry, Instrument) 

133 with self.assertRaises(LookupError): 

134 Instrument.fromName("NotThrere", registry) 

135 

136 # Register a bad instrument. 

137 BadInstrument().register(registry) 

138 with self.assertRaises(TypeError): 

139 Instrument.fromName("BadInstrument", registry) 

140 

141 UnimportableInstrument().register(registry) 

142 with self.assertRaises(ImportError): 

143 Instrument.fromName("NoImportInstr", registry) 

144 

145 # This should work even with the bad class name. 

146 self.instrument.importAll(registry) 

147 

148 def test_defaults(self): 

149 self.assertEqual(self.instrument.makeDefaultRawIngestRunName(), "DummyInstrument/raw/all") 

150 self.assertEqual( 

151 self.instrument.makeUnboundedCalibrationRunName("a", "b"), "DummyInstrument/calib/a/b/unbounded" 

152 ) 

153 self.assertEqual( 

154 self.instrument.makeCuratedCalibrationRunName("2018-05-04", "a"), 

155 "DummyInstrument/calib/a/curated/20180504T000000Z", 

156 ) 

157 self.assertEqual(self.instrument.makeCalibrationCollectionName("c"), "DummyInstrument/calib/c") 

158 self.assertEqual(self.instrument.makeRefCatCollectionName(), "refcats") 

159 self.assertEqual(self.instrument.makeRefCatCollectionName("a"), "refcats/a") 

160 self.assertEqual(self.instrument.makeUmbrellaCollectionName(), "DummyInstrument/defaults") 

161 

162 instrument = DummyInstrument(collection_prefix="Different") 

163 self.assertEqual(instrument.makeCollectionName("a"), "Different/a") 

164 self.assertEqual(self.instrument.makeCollectionName("a"), "DummyInstrument/a") 

165 

166 def test_collection_timestamps(self): 

167 self.assertEqual( 

168 Instrument.formatCollectionTimestamp("2018-05-03"), 

169 "20180503T000000Z", 

170 ) 

171 self.assertEqual( 

172 Instrument.formatCollectionTimestamp("2018-05-03T14:32:16"), 

173 "20180503T143216Z", 

174 ) 

175 self.assertEqual( 

176 Instrument.formatCollectionTimestamp("20180503T143216Z"), 

177 "20180503T143216Z", 

178 ) 

179 self.assertEqual( 

180 Instrument.formatCollectionTimestamp(datetime.datetime(2018, 5, 3, 14, 32, 16)), 

181 "20180503T143216Z", 

182 ) 

183 formattedNow = Instrument.makeCollectionTimestamp() 

184 self.assertIsInstance(formattedNow, str) 

185 datetimeThen1 = datetime.datetime.strptime(formattedNow, "%Y%m%dT%H%M%S%z") 

186 self.assertEqual(datetimeThen1.tzinfo, datetime.timezone.utc) 

187 

188 with self.assertRaises(TypeError): 

189 Instrument.formatCollectionTimestamp(0) 

190 

191 def test_dimension_packer_config_defaults(self): 

192 """Test the ObservationDimensionPacker class and the Instrument-based 

193 systems for constructing it, in the case where the Instrument-defined 

194 default is used. 

195 """ 

196 registry_config = RegistryConfig() 

197 registry_config["db"] = "sqlite://" 

198 registry = Registry.createFromConfig(registry_config) 

199 self.instrument.register(registry) 

200 config = DimensionPackerTestConfig() 

201 instrument_data_id = registry.expandDataId(instrument=self.name) 

202 record = instrument_data_id.records["instrument"] 

203 self.check_dimension_packers( 

204 registry.dimensions, 

205 # Test just one packer-construction signature here as that saves us 

206 # from having to insert dimension records for visits, exposures, 

207 # and detectors for this test. 

208 # Note that we don't need to pass any more than the instrument in 

209 # the data ID yet, because we're just constructing packers, not 

210 # calling their pack method. 

211 visit_packers=[config.packer.apply(instrument_data_id, is_exposure=False)], 

212 exposure_packers=[config.packer.apply(instrument_data_id, is_exposure=True)], 

213 n_detectors=record.detector_max, 

214 n_visits=record.visit_max, 

215 n_exposures=record.exposure_max, 

216 ) 

217 

218 def test_dimension_packer_config_override(self): 

219 """Test the ObservationDimensionPacker class and the Instrument-based 

220 systems for constructing it, in the case where configuration overrides 

221 the Instrument's default. 

222 """ 

223 registry_config = RegistryConfig() 

224 registry_config["db"] = "sqlite://" 

225 registry = Registry.createFromConfig(registry_config) 

226 # Intentionally do not register instrument or insert any other 

227 # dimension records to ensure we don't need them in this mode. 

228 config = DimensionPackerTestConfig() 

229 config.packer["observation"].n_observations = 16 

230 config.packer["observation"].n_detectors = 4 

231 config.packer.name = "observation" 

232 instrument_data_id = DataCoordinate.standardize(instrument=self.name, universe=registry.dimensions) 

233 full_data_id = DataCoordinate.standardize(instrument_data_id, detector=1, visit=3, exposure=7) 

234 visit_data_id = full_data_id.subset(full_data_id.universe.extract(["visit", "detector"])) 

235 exposure_data_id = full_data_id.subset(full_data_id.universe.extract(["exposure", "detector"])) 

236 self.check_dimension_packers( 

237 registry.dimensions, 

238 # Note that we don't need to pass any more than the instrument in 

239 # the data ID yet, because we're just constructing packers, not 

240 # calling their pack method. 

241 visit_packers=[ 

242 config.packer.apply(visit_data_id), 

243 config.packer.apply(full_data_id), 

244 config.packer.apply(instrument_data_id, is_exposure=False), 

245 ], 

246 exposure_packers=[ 

247 config.packer.apply(instrument_data_id, is_exposure=True), 

248 config.packer.apply(exposure_data_id), 

249 ], 

250 n_detectors=config.packer["observation"].n_detectors, 

251 n_visits=config.packer["observation"].n_observations, 

252 n_exposures=config.packer["observation"].n_observations, 

253 ) 

254 

255 def check_dimension_packers( 

256 self, 

257 universe: DimensionUniverse, 

258 visit_packers: list[DimensionPacker], 

259 exposure_packers: list[DimensionPacker], 

260 n_detectors: int, 

261 n_visits: int, 

262 n_exposures: int, 

263 ) -> None: 

264 """Test the behavior of one or more dimension packers constructed by 

265 an instrument. 

266 

267 Parameters 

268 ---------- 

269 universe : `lsst.daf.butler.DimensionUniverse` 

270 Data model for butler data IDs. 

271 visit_packers : `list` [ `DimensionPacker` ] 

272 Packers with ``{visit, detector}`` dimensions to test. 

273 exposure_packers : `list` [ `DimensionPacker` ] 

274 Packers with ``{exposure, detector}`` dimensions to test. 

275 n_detectors : `int` 

276 Number of detectors all packers have been configured to reserve 

277 space for. 

278 n_visits : `int` 

279 Number of visits all packers in ``visit_packers`` have been 

280 configured to reserve space for. 

281 n_exposures : `int` 

282 Number of exposures all packers in ``exposure_packers`` have been 

283 configured to reserve space for. 

284 """ 

285 full_data_id = DataCoordinate.standardize( 

286 instrument=self.name, detector=1, visit=3, exposure=7, universe=universe 

287 ) 

288 visit_data_id = full_data_id.subset(full_data_id.universe.extract(["visit", "detector"])) 

289 exposure_data_id = full_data_id.subset(full_data_id.universe.extract(["exposure", "detector"])) 

290 for n, packer in enumerate(visit_packers): 

291 with self.subTest(n=n): 

292 packed_id, max_bits = packer.pack(full_data_id, returnMaxBits=True) 

293 self.assertEqual(packed_id, full_data_id["detector"] + full_data_id["visit"] * n_detectors) 

294 self.assertEqual(max_bits, math.ceil(math.log2(n_detectors * n_visits))) 

295 self.assertEqual(visit_data_id, packer.unpack(packed_id)) 

296 with self.assertRaisesRegex(ValueError, f"Detector ID {n_detectors} is out of bounds"): 

297 packer.pack(instrument=self.name, detector=n_detectors, visit=3) 

298 with self.assertRaisesRegex(ValueError, f"Visit ID {n_visits} is out of bounds"): 

299 packer.pack(instrument=self.name, detector=1, visit=n_visits) 

300 for n, packer in enumerate(exposure_packers): 

301 with self.subTest(n=n): 

302 packed_id, max_bits = packer.pack(full_data_id, returnMaxBits=True) 

303 self.assertEqual(packed_id, full_data_id["detector"] + full_data_id["exposure"] * n_detectors) 

304 self.assertEqual(max_bits, math.ceil(math.log2(n_detectors * n_exposures))) 

305 self.assertEqual(exposure_data_id, packer.unpack(packed_id)) 

306 with self.assertRaisesRegex(ValueError, f"Detector ID {n_detectors} is out of bounds"): 

307 packer.pack(instrument=self.name, detector=n_detectors, exposure=3) 

308 with self.assertRaisesRegex(ValueError, f"Exposure ID {n_exposures} is out of bounds"): 

309 packer.pack(instrument=self.name, detector=1, exposure=n_exposures) 

310 

311 

312if __name__ == "__main__": 312 ↛ 313line 312 didn't jump to line 313, because the condition on line 312 was never true

313 unittest.main()