Coverage for tests/test_instrument.py: 24%

142 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-12-06 10:56 +0000

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Tests of the Instrument class. 

29""" 

30 

31import datetime 

32import math 

33import unittest 

34 

35from lsst.daf.butler import DataCoordinate, DimensionPacker, DimensionUniverse, RegistryConfig 

36from lsst.daf.butler.formatters.json import JsonFormatter 

37from lsst.daf.butler.registry.sql_registry import SqlRegistry 

38from lsst.pex.config import Config 

39from lsst.pipe.base import Instrument 

40from lsst.utils.introspection import get_full_type_name 

41 

42 

43class DummyInstrument(Instrument): 

44 """Test instrument.""" 

45 

46 @classmethod 

47 def getName(cls): 

48 return "DummyInstrument" 

49 

50 def register(self, registry, update=False): 

51 detector_max = 2 

52 visit_max = 10 

53 exposure_max = 8 

54 record = { 

55 "instrument": self.getName(), 

56 "class_name": get_full_type_name(DummyInstrument), 

57 "detector_max": detector_max, 

58 "visit_max": visit_max, 

59 "exposure_max": exposure_max, 

60 } 

61 with registry.transaction(): 

62 registry.syncDimensionData("instrument", record, update=update) 

63 

64 def getRawFormatter(self, dataId): 

65 return JsonFormatter 

66 

67 

68class BadInstrument(DummyInstrument): 

69 """Instrument with wrong class name.""" 

70 

71 raw_definition = ("raw2", ("instrument", "detector", "exposure"), "StructuredDataDict") 

72 

73 @classmethod 

74 def getName(cls): 

75 return "BadInstrument" 

76 

77 def register(self, registry, update=False): 

78 # Register a bad class name 

79 record = { 

80 "instrument": self.getName(), 

81 "class_name": "builtins.str", 

82 "detector_max": 1, 

83 } 

84 registry.syncDimensionData("instrument", record, update=update) 

85 

86 

87class UnimportableInstrument(DummyInstrument): 

88 """Instrument with class name that does not exist.""" 

89 

90 @classmethod 

91 def getName(cls): 

92 return "NoImportInstr" 

93 

94 def register(self, registry, update=False): 

95 # Register a bad class name 

96 record = { 

97 "instrument": self.getName(), 

98 "class_name": "not.importable", 

99 "detector_max": 1, 

100 } 

101 registry.syncDimensionData("instrument", record, update=update) 

102 

103 

104class DimensionPackerTestConfig(Config): 

105 """Configuration for the dimension packer.""" 

106 

107 packer = Instrument.make_dimension_packer_config_field() 

108 

109 

110class InstrumentTestCase(unittest.TestCase): 

111 """Test for Instrument.""" 

112 

113 def setUp(self): 

114 self.instrument = DummyInstrument() 

115 self.name = "DummyInstrument" 

116 

117 def test_basics(self): 

118 self.assertEqual(self.instrument.getName(), self.name) 

119 self.assertEqual(self.instrument.getRawFormatter({}), JsonFormatter) 

120 self.assertIsNone(DummyInstrument.raw_definition) 

121 raw = BadInstrument.raw_definition 

122 self.assertEqual(raw[2], "StructuredDataDict") 

123 

124 def test_register(self): 

125 """Test that register() sets appropriate Dimensions.""" 

126 registryConfig = RegistryConfig() 

127 registryConfig["db"] = "sqlite://" 

128 registry = SqlRegistry.createFromConfig(registryConfig) 

129 # Check that the registry starts out empty. 

130 self.instrument.importAll(registry) 

131 self.assertFalse(list(registry.queryDimensionRecords("instrument"))) 

132 

133 # Register and check again. 

134 self.instrument.register(registry) 

135 instruments = list(registry.queryDimensionRecords("instrument")) 

136 self.assertEqual(len(instruments), 1) 

137 self.assertEqual(instruments[0].name, self.name) 

138 self.assertEqual(instruments[0].detector_max, 2) 

139 self.assertIn("DummyInstrument", instruments[0].class_name) 

140 

141 self.instrument.importAll(registry) 

142 from_registry = DummyInstrument.fromName("DummyInstrument", registry) 

143 self.assertIsInstance(from_registry, Instrument) 

144 with self.assertRaises(LookupError): 

145 Instrument.fromName("NotThrere", registry) 

146 

147 # Register a bad instrument. 

148 BadInstrument().register(registry) 

149 with self.assertRaises(TypeError): 

150 Instrument.fromName("BadInstrument", registry) 

151 

152 UnimportableInstrument().register(registry) 

153 with self.assertRaises(ImportError): 

154 Instrument.fromName("NoImportInstr", registry) 

155 

156 # This should work even with the bad class name. 

157 self.instrument.importAll(registry) 

158 

159 def test_defaults(self): 

160 self.assertEqual(self.instrument.makeDefaultRawIngestRunName(), "DummyInstrument/raw/all") 

161 self.assertEqual( 

162 self.instrument.makeUnboundedCalibrationRunName("a", "b"), "DummyInstrument/calib/a/b/unbounded" 

163 ) 

164 self.assertEqual( 

165 self.instrument.makeCuratedCalibrationRunName("2018-05-04", "a"), 

166 "DummyInstrument/calib/a/curated/20180504T000000Z", 

167 ) 

168 self.assertEqual(self.instrument.makeCalibrationCollectionName("c"), "DummyInstrument/calib/c") 

169 self.assertEqual(self.instrument.makeRefCatCollectionName(), "refcats") 

170 self.assertEqual(self.instrument.makeRefCatCollectionName("a"), "refcats/a") 

171 self.assertEqual(self.instrument.makeUmbrellaCollectionName(), "DummyInstrument/defaults") 

172 

173 instrument = DummyInstrument(collection_prefix="Different") 

174 self.assertEqual(instrument.makeCollectionName("a"), "Different/a") 

175 self.assertEqual(self.instrument.makeCollectionName("a"), "DummyInstrument/a") 

176 

177 def test_collection_timestamps(self): 

178 self.assertEqual( 

179 Instrument.formatCollectionTimestamp("2018-05-03"), 

180 "20180503T000000Z", 

181 ) 

182 self.assertEqual( 

183 Instrument.formatCollectionTimestamp("2018-05-03T14:32:16"), 

184 "20180503T143216Z", 

185 ) 

186 self.assertEqual( 

187 Instrument.formatCollectionTimestamp("20180503T143216Z"), 

188 "20180503T143216Z", 

189 ) 

190 self.assertEqual( 

191 Instrument.formatCollectionTimestamp(datetime.datetime(2018, 5, 3, 14, 32, 16)), 

192 "20180503T143216Z", 

193 ) 

194 formattedNow = Instrument.makeCollectionTimestamp() 

195 self.assertIsInstance(formattedNow, str) 

196 datetimeThen1 = datetime.datetime.strptime(formattedNow, "%Y%m%dT%H%M%S%z") 

197 self.assertEqual(datetimeThen1.tzinfo, datetime.timezone.utc) 

198 

199 with self.assertRaises(TypeError): 

200 Instrument.formatCollectionTimestamp(0) 

201 

202 def test_dimension_packer_config_defaults(self): 

203 """Test the ObservationDimensionPacker class and the Instrument-based 

204 systems for constructing it, in the case where the Instrument-defined 

205 default is used. 

206 """ 

207 registry_config = RegistryConfig() 

208 registry_config["db"] = "sqlite://" 

209 registry = SqlRegistry.createFromConfig(registry_config) 

210 self.instrument.register(registry) 

211 config = DimensionPackerTestConfig() 

212 instrument_data_id = registry.expandDataId(instrument=self.name) 

213 record = instrument_data_id.records["instrument"] 

214 self.check_dimension_packers( 

215 registry.dimensions, 

216 # Test just one packer-construction signature here as that saves us 

217 # from having to insert dimension records for visits, exposures, 

218 # and detectors for this test. 

219 # Note that we don't need to pass any more than the instrument in 

220 # the data ID yet, because we're just constructing packers, not 

221 # calling their pack method. 

222 visit_packers=[ 

223 config.packer.apply(instrument_data_id, is_exposure=False), 

224 Instrument.make_default_dimension_packer(instrument_data_id, is_exposure=False), 

225 ], 

226 exposure_packers=[ 

227 config.packer.apply(instrument_data_id, is_exposure=True), 

228 Instrument.make_default_dimension_packer(instrument_data_id, is_exposure=True), 

229 ], 

230 n_detectors=record.detector_max, 

231 n_visits=record.visit_max, 

232 n_exposures=record.exposure_max, 

233 ) 

234 

235 def test_dimension_packer_config_override(self): 

236 """Test the ObservationDimensionPacker class and the Instrument-based 

237 systems for constructing it, in the case where configuration overrides 

238 the Instrument's default. 

239 """ 

240 registry_config = RegistryConfig() 

241 registry_config["db"] = "sqlite://" 

242 registry = SqlRegistry.createFromConfig(registry_config) 

243 # Intentionally do not register instrument or insert any other 

244 # dimension records to ensure we don't need them in this mode. 

245 config = DimensionPackerTestConfig() 

246 config.packer["observation"].n_observations = 16 

247 config.packer["observation"].n_detectors = 4 

248 config.packer.name = "observation" 

249 instrument_data_id = DataCoordinate.standardize(instrument=self.name, universe=registry.dimensions) 

250 full_data_id = DataCoordinate.standardize(instrument_data_id, detector=1, visit=3, exposure=7) 

251 visit_data_id = full_data_id.subset(full_data_id.universe.conform(["visit", "detector"])) 

252 exposure_data_id = full_data_id.subset(full_data_id.universe.conform(["exposure", "detector"])) 

253 self.check_dimension_packers( 

254 registry.dimensions, 

255 # Note that we don't need to pass any more than the instrument in 

256 # the data ID yet, because we're just constructing packers, not 

257 # calling their pack method. 

258 visit_packers=[ 

259 config.packer.apply(visit_data_id), 

260 config.packer.apply(full_data_id), 

261 config.packer.apply(instrument_data_id, is_exposure=False), 

262 ], 

263 exposure_packers=[ 

264 config.packer.apply(instrument_data_id, is_exposure=True), 

265 config.packer.apply(exposure_data_id), 

266 ], 

267 n_detectors=config.packer["observation"].n_detectors, 

268 n_visits=config.packer["observation"].n_observations, 

269 n_exposures=config.packer["observation"].n_observations, 

270 ) 

271 

272 def check_dimension_packers( 

273 self, 

274 universe: DimensionUniverse, 

275 visit_packers: list[DimensionPacker], 

276 exposure_packers: list[DimensionPacker], 

277 n_detectors: int, 

278 n_visits: int, 

279 n_exposures: int, 

280 ) -> None: 

281 """Test the behavior of one or more dimension packers constructed by 

282 an instrument. 

283 

284 Parameters 

285 ---------- 

286 universe : `lsst.daf.butler.DimensionUniverse` 

287 Data model for butler data IDs. 

288 visit_packers : `list` [ `DimensionPacker` ] 

289 Packers with ``{visit, detector}`` dimensions to test. 

290 exposure_packers : `list` [ `DimensionPacker` ] 

291 Packers with ``{exposure, detector}`` dimensions to test. 

292 n_detectors : `int` 

293 Number of detectors all packers have been configured to reserve 

294 space for. 

295 n_visits : `int` 

296 Number of visits all packers in ``visit_packers`` have been 

297 configured to reserve space for. 

298 n_exposures : `int` 

299 Number of exposures all packers in ``exposure_packers`` have been 

300 configured to reserve space for. 

301 """ 

302 full_data_id = DataCoordinate.standardize( 

303 instrument=self.name, detector=1, visit=3, exposure=7, universe=universe 

304 ) 

305 visit_data_id = full_data_id.subset(full_data_id.universe.conform(["visit", "detector"])) 

306 exposure_data_id = full_data_id.subset(full_data_id.universe.conform(["exposure", "detector"])) 

307 for n, packer in enumerate(visit_packers): 

308 with self.subTest(n=n): 

309 packed_id, max_bits = packer.pack(full_data_id, returnMaxBits=True) 

310 self.assertEqual(packed_id, full_data_id["detector"] + full_data_id["visit"] * n_detectors) 

311 self.assertEqual(max_bits, math.ceil(math.log2(n_detectors * n_visits))) 

312 self.assertEqual(visit_data_id, packer.unpack(packed_id)) 

313 with self.assertRaisesRegex(ValueError, f"Detector ID {n_detectors} is out of bounds"): 

314 packer.pack(instrument=self.name, detector=n_detectors, visit=3) 

315 with self.assertRaisesRegex(ValueError, f"Visit ID {n_visits} is out of bounds"): 

316 packer.pack(instrument=self.name, detector=1, visit=n_visits) 

317 for n, packer in enumerate(exposure_packers): 

318 with self.subTest(n=n): 

319 packed_id, max_bits = packer.pack(full_data_id, returnMaxBits=True) 

320 self.assertEqual(packed_id, full_data_id["detector"] + full_data_id["exposure"] * n_detectors) 

321 self.assertEqual(max_bits, math.ceil(math.log2(n_detectors * n_exposures))) 

322 self.assertEqual(exposure_data_id, packer.unpack(packed_id)) 

323 with self.assertRaisesRegex(ValueError, f"Detector ID {n_detectors} is out of bounds"): 

324 packer.pack(instrument=self.name, detector=n_detectors, exposure=3) 

325 with self.assertRaisesRegex(ValueError, f"Exposure ID {n_exposures} is out of bounds"): 

326 packer.pack(instrument=self.name, detector=1, exposure=n_exposures) 

327 

328 

329if __name__ == "__main__": 

330 unittest.main()