Coverage for tests/test_instrument.py: 20%

141 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-16 09:02 +0000

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Tests of the Instrument class. 

23""" 

24 

25import datetime 

26import math 

27import unittest 

28 

29from lsst.daf.butler import DataCoordinate, DimensionPacker, DimensionUniverse, Registry, RegistryConfig 

30from lsst.daf.butler.formatters.json import JsonFormatter 

31from lsst.pex.config import Config 

32from lsst.pipe.base import Instrument 

33from lsst.utils.introspection import get_full_type_name 

34 

35 

36class DummyInstrument(Instrument): 

37 @classmethod 

38 def getName(cls): 

39 return "DummyInstrument" 

40 

41 def register(self, registry, update=False): 

42 detector_max = 2 

43 visit_max = 10 

44 exposure_max = 8 

45 record = { 

46 "instrument": self.getName(), 

47 "class_name": get_full_type_name(DummyInstrument), 

48 "detector_max": detector_max, 

49 "visit_max": visit_max, 

50 "exposure_max": exposure_max, 

51 } 

52 with registry.transaction(): 

53 registry.syncDimensionData("instrument", record, update=update) 

54 

55 def getRawFormatter(self, dataId): 

56 return JsonFormatter 

57 

58 

59class BadInstrument(DummyInstrument): 

60 """Instrument with wrong class name.""" 

61 

62 raw_definition = ("raw2", ("instrument", "detector", "exposure"), "StructuredDataDict") 

63 

64 @classmethod 

65 def getName(cls): 

66 return "BadInstrument" 

67 

68 def register(self, registry, update=False): 

69 # Register a bad class name 

70 record = { 

71 "instrument": self.getName(), 

72 "class_name": "builtins.str", 

73 "detector_max": 1, 

74 } 

75 registry.syncDimensionData("instrument", record, update=update) 

76 

77 

78class UnimportableInstrument(DummyInstrument): 

79 """Instrument with class name that does not exist.""" 

80 

81 @classmethod 

82 def getName(cls): 

83 return "NoImportInstr" 

84 

85 def register(self, registry, update=False): 

86 # Register a bad class name 

87 record = { 

88 "instrument": self.getName(), 

89 "class_name": "not.importable", 

90 "detector_max": 1, 

91 } 

92 registry.syncDimensionData("instrument", record, update=update) 

93 

94 

95class DimensionPackerTestConfig(Config): 

96 packer = Instrument.make_dimension_packer_config_field() 

97 

98 

99class InstrumentTestCase(unittest.TestCase): 

100 """Test for Instrument.""" 

101 

102 def setUp(self): 

103 self.instrument = DummyInstrument() 

104 self.name = "DummyInstrument" 

105 

106 def test_basics(self): 

107 self.assertEqual(self.instrument.getName(), self.name) 

108 self.assertEqual(self.instrument.getRawFormatter({}), JsonFormatter) 

109 self.assertIsNone(DummyInstrument.raw_definition) 

110 raw = BadInstrument.raw_definition 

111 self.assertEqual(raw[2], "StructuredDataDict") 

112 

113 def test_register(self): 

114 """Test that register() sets appropriate Dimensions.""" 

115 registryConfig = RegistryConfig() 

116 registryConfig["db"] = "sqlite://" 

117 registry = Registry.createFromConfig(registryConfig) 

118 # Check that the registry starts out empty. 

119 self.instrument.importAll(registry) 

120 self.assertFalse(list(registry.queryDimensionRecords("instrument"))) 

121 

122 # Register and check again. 

123 self.instrument.register(registry) 

124 instruments = list(registry.queryDimensionRecords("instrument")) 

125 self.assertEqual(len(instruments), 1) 

126 self.assertEqual(instruments[0].name, self.name) 

127 self.assertEqual(instruments[0].detector_max, 2) 

128 self.assertIn("DummyInstrument", instruments[0].class_name) 

129 

130 self.instrument.importAll(registry) 

131 from_registry = DummyInstrument.fromName("DummyInstrument", registry) 

132 self.assertIsInstance(from_registry, Instrument) 

133 with self.assertRaises(LookupError): 

134 Instrument.fromName("NotThrere", registry) 

135 

136 # Register a bad instrument. 

137 BadInstrument().register(registry) 

138 with self.assertRaises(TypeError): 

139 Instrument.fromName("BadInstrument", registry) 

140 

141 UnimportableInstrument().register(registry) 

142 with self.assertRaises(ImportError): 

143 Instrument.fromName("NoImportInstr", registry) 

144 

145 # This should work even with the bad class name. 

146 self.instrument.importAll(registry) 

147 

148 def test_defaults(self): 

149 self.assertEqual(self.instrument.makeDefaultRawIngestRunName(), "DummyInstrument/raw/all") 

150 self.assertEqual( 

151 self.instrument.makeUnboundedCalibrationRunName("a", "b"), "DummyInstrument/calib/a/b/unbounded" 

152 ) 

153 self.assertEqual( 

154 self.instrument.makeCuratedCalibrationRunName("2018-05-04", "a"), 

155 "DummyInstrument/calib/a/curated/20180504T000000Z", 

156 ) 

157 self.assertEqual(self.instrument.makeCalibrationCollectionName("c"), "DummyInstrument/calib/c") 

158 self.assertEqual(self.instrument.makeRefCatCollectionName(), "refcats") 

159 self.assertEqual(self.instrument.makeRefCatCollectionName("a"), "refcats/a") 

160 self.assertEqual(self.instrument.makeUmbrellaCollectionName(), "DummyInstrument/defaults") 

161 

162 instrument = DummyInstrument(collection_prefix="Different") 

163 self.assertEqual(instrument.makeCollectionName("a"), "Different/a") 

164 self.assertEqual(self.instrument.makeCollectionName("a"), "DummyInstrument/a") 

165 

166 def test_collection_timestamps(self): 

167 self.assertEqual( 

168 Instrument.formatCollectionTimestamp("2018-05-03"), 

169 "20180503T000000Z", 

170 ) 

171 self.assertEqual( 

172 Instrument.formatCollectionTimestamp("2018-05-03T14:32:16"), 

173 "20180503T143216Z", 

174 ) 

175 self.assertEqual( 

176 Instrument.formatCollectionTimestamp("20180503T143216Z"), 

177 "20180503T143216Z", 

178 ) 

179 self.assertEqual( 

180 Instrument.formatCollectionTimestamp(datetime.datetime(2018, 5, 3, 14, 32, 16)), 

181 "20180503T143216Z", 

182 ) 

183 formattedNow = Instrument.makeCollectionTimestamp() 

184 self.assertIsInstance(formattedNow, str) 

185 datetimeThen1 = datetime.datetime.strptime(formattedNow, "%Y%m%dT%H%M%S%z") 

186 self.assertEqual(datetimeThen1.tzinfo, datetime.timezone.utc) 

187 

188 with self.assertRaises(TypeError): 

189 Instrument.formatCollectionTimestamp(0) 

190 

191 def test_dimension_packer_config_defaults(self): 

192 """Test the ObservationDimensionPacker class and the Instrument-based 

193 systems for constructing it, in the case where the Instrument-defined 

194 default is used. 

195 """ 

196 registry_config = RegistryConfig() 

197 registry_config["db"] = "sqlite://" 

198 registry = Registry.createFromConfig(registry_config) 

199 self.instrument.register(registry) 

200 config = DimensionPackerTestConfig() 

201 instrument_data_id = registry.expandDataId(instrument=self.name) 

202 record = instrument_data_id.records["instrument"] 

203 self.check_dimension_packers( 

204 registry.dimensions, 

205 # Test just one packer-construction signature here as that saves us 

206 # from having to insert dimension records for visits, exposures, 

207 # and detectors for this test. 

208 # Note that we don't need to pass any more than the instrument in 

209 # the data ID yet, because we're just constructing packers, not 

210 # calling their pack method. 

211 visit_packers=[ 

212 config.packer.apply(instrument_data_id, is_exposure=False), 

213 Instrument.make_default_dimension_packer(instrument_data_id, is_exposure=False), 

214 ], 

215 exposure_packers=[ 

216 config.packer.apply(instrument_data_id, is_exposure=True), 

217 Instrument.make_default_dimension_packer(instrument_data_id, is_exposure=True), 

218 ], 

219 n_detectors=record.detector_max, 

220 n_visits=record.visit_max, 

221 n_exposures=record.exposure_max, 

222 ) 

223 

224 def test_dimension_packer_config_override(self): 

225 """Test the ObservationDimensionPacker class and the Instrument-based 

226 systems for constructing it, in the case where configuration overrides 

227 the Instrument's default. 

228 """ 

229 registry_config = RegistryConfig() 

230 registry_config["db"] = "sqlite://" 

231 registry = Registry.createFromConfig(registry_config) 

232 # Intentionally do not register instrument or insert any other 

233 # dimension records to ensure we don't need them in this mode. 

234 config = DimensionPackerTestConfig() 

235 config.packer["observation"].n_observations = 16 

236 config.packer["observation"].n_detectors = 4 

237 config.packer.name = "observation" 

238 instrument_data_id = DataCoordinate.standardize(instrument=self.name, universe=registry.dimensions) 

239 full_data_id = DataCoordinate.standardize(instrument_data_id, detector=1, visit=3, exposure=7) 

240 visit_data_id = full_data_id.subset(full_data_id.universe.extract(["visit", "detector"])) 

241 exposure_data_id = full_data_id.subset(full_data_id.universe.extract(["exposure", "detector"])) 

242 self.check_dimension_packers( 

243 registry.dimensions, 

244 # Note that we don't need to pass any more than the instrument in 

245 # the data ID yet, because we're just constructing packers, not 

246 # calling their pack method. 

247 visit_packers=[ 

248 config.packer.apply(visit_data_id), 

249 config.packer.apply(full_data_id), 

250 config.packer.apply(instrument_data_id, is_exposure=False), 

251 ], 

252 exposure_packers=[ 

253 config.packer.apply(instrument_data_id, is_exposure=True), 

254 config.packer.apply(exposure_data_id), 

255 ], 

256 n_detectors=config.packer["observation"].n_detectors, 

257 n_visits=config.packer["observation"].n_observations, 

258 n_exposures=config.packer["observation"].n_observations, 

259 ) 

260 

261 def check_dimension_packers( 

262 self, 

263 universe: DimensionUniverse, 

264 visit_packers: list[DimensionPacker], 

265 exposure_packers: list[DimensionPacker], 

266 n_detectors: int, 

267 n_visits: int, 

268 n_exposures: int, 

269 ) -> None: 

270 """Test the behavior of one or more dimension packers constructed by 

271 an instrument. 

272 

273 Parameters 

274 ---------- 

275 universe : `lsst.daf.butler.DimensionUniverse` 

276 Data model for butler data IDs. 

277 visit_packers : `list` [ `DimensionPacker` ] 

278 Packers with ``{visit, detector}`` dimensions to test. 

279 exposure_packers : `list` [ `DimensionPacker` ] 

280 Packers with ``{exposure, detector}`` dimensions to test. 

281 n_detectors : `int` 

282 Number of detectors all packers have been configured to reserve 

283 space for. 

284 n_visits : `int` 

285 Number of visits all packers in ``visit_packers`` have been 

286 configured to reserve space for. 

287 n_exposures : `int` 

288 Number of exposures all packers in ``exposure_packers`` have been 

289 configured to reserve space for. 

290 """ 

291 full_data_id = DataCoordinate.standardize( 

292 instrument=self.name, detector=1, visit=3, exposure=7, universe=universe 

293 ) 

294 visit_data_id = full_data_id.subset(full_data_id.universe.extract(["visit", "detector"])) 

295 exposure_data_id = full_data_id.subset(full_data_id.universe.extract(["exposure", "detector"])) 

296 for n, packer in enumerate(visit_packers): 

297 with self.subTest(n=n): 

298 packed_id, max_bits = packer.pack(full_data_id, returnMaxBits=True) 

299 self.assertEqual(packed_id, full_data_id["detector"] + full_data_id["visit"] * n_detectors) 

300 self.assertEqual(max_bits, math.ceil(math.log2(n_detectors * n_visits))) 

301 self.assertEqual(visit_data_id, packer.unpack(packed_id)) 

302 with self.assertRaisesRegex(ValueError, f"Detector ID {n_detectors} is out of bounds"): 

303 packer.pack(instrument=self.name, detector=n_detectors, visit=3) 

304 with self.assertRaisesRegex(ValueError, f"Visit ID {n_visits} is out of bounds"): 

305 packer.pack(instrument=self.name, detector=1, visit=n_visits) 

306 for n, packer in enumerate(exposure_packers): 

307 with self.subTest(n=n): 

308 packed_id, max_bits = packer.pack(full_data_id, returnMaxBits=True) 

309 self.assertEqual(packed_id, full_data_id["detector"] + full_data_id["exposure"] * n_detectors) 

310 self.assertEqual(max_bits, math.ceil(math.log2(n_detectors * n_exposures))) 

311 self.assertEqual(exposure_data_id, packer.unpack(packed_id)) 

312 with self.assertRaisesRegex(ValueError, f"Detector ID {n_detectors} is out of bounds"): 

313 packer.pack(instrument=self.name, detector=n_detectors, exposure=3) 

314 with self.assertRaisesRegex(ValueError, f"Exposure ID {n_exposures} is out of bounds"): 

315 packer.pack(instrument=self.name, detector=1, exposure=n_exposures) 

316 

317 

318if __name__ == "__main__": 

319 unittest.main()