Coverage for tests/test_instrument.py: 23%

141 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-09-28 09:35 +0000

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Tests of the Instrument class. 

29""" 

30 

31import datetime 

32import math 

33import unittest 

34 

35from lsst.daf.butler import DataCoordinate, DimensionPacker, DimensionUniverse, Registry, RegistryConfig 

36from lsst.daf.butler.formatters.json import JsonFormatter 

37from lsst.pex.config import Config 

38from lsst.pipe.base import Instrument 

39from lsst.utils.introspection import get_full_type_name 

40 

41 

42class DummyInstrument(Instrument): 

43 """Test instrument.""" 

44 

45 @classmethod 

46 def getName(cls): 

47 return "DummyInstrument" 

48 

49 def register(self, registry, update=False): 

50 detector_max = 2 

51 visit_max = 10 

52 exposure_max = 8 

53 record = { 

54 "instrument": self.getName(), 

55 "class_name": get_full_type_name(DummyInstrument), 

56 "detector_max": detector_max, 

57 "visit_max": visit_max, 

58 "exposure_max": exposure_max, 

59 } 

60 with registry.transaction(): 

61 registry.syncDimensionData("instrument", record, update=update) 

62 

63 def getRawFormatter(self, dataId): 

64 return JsonFormatter 

65 

66 

67class BadInstrument(DummyInstrument): 

68 """Instrument with wrong class name.""" 

69 

70 raw_definition = ("raw2", ("instrument", "detector", "exposure"), "StructuredDataDict") 

71 

72 @classmethod 

73 def getName(cls): 

74 return "BadInstrument" 

75 

76 def register(self, registry, update=False): 

77 # Register a bad class name 

78 record = { 

79 "instrument": self.getName(), 

80 "class_name": "builtins.str", 

81 "detector_max": 1, 

82 } 

83 registry.syncDimensionData("instrument", record, update=update) 

84 

85 

86class UnimportableInstrument(DummyInstrument): 

87 """Instrument with class name that does not exist.""" 

88 

89 @classmethod 

90 def getName(cls): 

91 return "NoImportInstr" 

92 

93 def register(self, registry, update=False): 

94 # Register a bad class name 

95 record = { 

96 "instrument": self.getName(), 

97 "class_name": "not.importable", 

98 "detector_max": 1, 

99 } 

100 registry.syncDimensionData("instrument", record, update=update) 

101 

102 

103class DimensionPackerTestConfig(Config): 

104 """Configuration for the dimension packer.""" 

105 

106 packer = Instrument.make_dimension_packer_config_field() 

107 

108 

109class InstrumentTestCase(unittest.TestCase): 

110 """Test for Instrument.""" 

111 

112 def setUp(self): 

113 self.instrument = DummyInstrument() 

114 self.name = "DummyInstrument" 

115 

116 def test_basics(self): 

117 self.assertEqual(self.instrument.getName(), self.name) 

118 self.assertEqual(self.instrument.getRawFormatter({}), JsonFormatter) 

119 self.assertIsNone(DummyInstrument.raw_definition) 

120 raw = BadInstrument.raw_definition 

121 self.assertEqual(raw[2], "StructuredDataDict") 

122 

123 def test_register(self): 

124 """Test that register() sets appropriate Dimensions.""" 

125 registryConfig = RegistryConfig() 

126 registryConfig["db"] = "sqlite://" 

127 registry = Registry.createFromConfig(registryConfig) 

128 # Check that the registry starts out empty. 

129 self.instrument.importAll(registry) 

130 self.assertFalse(list(registry.queryDimensionRecords("instrument"))) 

131 

132 # Register and check again. 

133 self.instrument.register(registry) 

134 instruments = list(registry.queryDimensionRecords("instrument")) 

135 self.assertEqual(len(instruments), 1) 

136 self.assertEqual(instruments[0].name, self.name) 

137 self.assertEqual(instruments[0].detector_max, 2) 

138 self.assertIn("DummyInstrument", instruments[0].class_name) 

139 

140 self.instrument.importAll(registry) 

141 from_registry = DummyInstrument.fromName("DummyInstrument", registry) 

142 self.assertIsInstance(from_registry, Instrument) 

143 with self.assertRaises(LookupError): 

144 Instrument.fromName("NotThrere", registry) 

145 

146 # Register a bad instrument. 

147 BadInstrument().register(registry) 

148 with self.assertRaises(TypeError): 

149 Instrument.fromName("BadInstrument", registry) 

150 

151 UnimportableInstrument().register(registry) 

152 with self.assertRaises(ImportError): 

153 Instrument.fromName("NoImportInstr", registry) 

154 

155 # This should work even with the bad class name. 

156 self.instrument.importAll(registry) 

157 

158 def test_defaults(self): 

159 self.assertEqual(self.instrument.makeDefaultRawIngestRunName(), "DummyInstrument/raw/all") 

160 self.assertEqual( 

161 self.instrument.makeUnboundedCalibrationRunName("a", "b"), "DummyInstrument/calib/a/b/unbounded" 

162 ) 

163 self.assertEqual( 

164 self.instrument.makeCuratedCalibrationRunName("2018-05-04", "a"), 

165 "DummyInstrument/calib/a/curated/20180504T000000Z", 

166 ) 

167 self.assertEqual(self.instrument.makeCalibrationCollectionName("c"), "DummyInstrument/calib/c") 

168 self.assertEqual(self.instrument.makeRefCatCollectionName(), "refcats") 

169 self.assertEqual(self.instrument.makeRefCatCollectionName("a"), "refcats/a") 

170 self.assertEqual(self.instrument.makeUmbrellaCollectionName(), "DummyInstrument/defaults") 

171 

172 instrument = DummyInstrument(collection_prefix="Different") 

173 self.assertEqual(instrument.makeCollectionName("a"), "Different/a") 

174 self.assertEqual(self.instrument.makeCollectionName("a"), "DummyInstrument/a") 

175 

176 def test_collection_timestamps(self): 

177 self.assertEqual( 

178 Instrument.formatCollectionTimestamp("2018-05-03"), 

179 "20180503T000000Z", 

180 ) 

181 self.assertEqual( 

182 Instrument.formatCollectionTimestamp("2018-05-03T14:32:16"), 

183 "20180503T143216Z", 

184 ) 

185 self.assertEqual( 

186 Instrument.formatCollectionTimestamp("20180503T143216Z"), 

187 "20180503T143216Z", 

188 ) 

189 self.assertEqual( 

190 Instrument.formatCollectionTimestamp(datetime.datetime(2018, 5, 3, 14, 32, 16)), 

191 "20180503T143216Z", 

192 ) 

193 formattedNow = Instrument.makeCollectionTimestamp() 

194 self.assertIsInstance(formattedNow, str) 

195 datetimeThen1 = datetime.datetime.strptime(formattedNow, "%Y%m%dT%H%M%S%z") 

196 self.assertEqual(datetimeThen1.tzinfo, datetime.timezone.utc) 

197 

198 with self.assertRaises(TypeError): 

199 Instrument.formatCollectionTimestamp(0) 

200 

201 def test_dimension_packer_config_defaults(self): 

202 """Test the ObservationDimensionPacker class and the Instrument-based 

203 systems for constructing it, in the case where the Instrument-defined 

204 default is used. 

205 """ 

206 registry_config = RegistryConfig() 

207 registry_config["db"] = "sqlite://" 

208 registry = Registry.createFromConfig(registry_config) 

209 self.instrument.register(registry) 

210 config = DimensionPackerTestConfig() 

211 instrument_data_id = registry.expandDataId(instrument=self.name) 

212 record = instrument_data_id.records["instrument"] 

213 self.check_dimension_packers( 

214 registry.dimensions, 

215 # Test just one packer-construction signature here as that saves us 

216 # from having to insert dimension records for visits, exposures, 

217 # and detectors for this test. 

218 # Note that we don't need to pass any more than the instrument in 

219 # the data ID yet, because we're just constructing packers, not 

220 # calling their pack method. 

221 visit_packers=[ 

222 config.packer.apply(instrument_data_id, is_exposure=False), 

223 Instrument.make_default_dimension_packer(instrument_data_id, is_exposure=False), 

224 ], 

225 exposure_packers=[ 

226 config.packer.apply(instrument_data_id, is_exposure=True), 

227 Instrument.make_default_dimension_packer(instrument_data_id, is_exposure=True), 

228 ], 

229 n_detectors=record.detector_max, 

230 n_visits=record.visit_max, 

231 n_exposures=record.exposure_max, 

232 ) 

233 

234 def test_dimension_packer_config_override(self): 

235 """Test the ObservationDimensionPacker class and the Instrument-based 

236 systems for constructing it, in the case where configuration overrides 

237 the Instrument's default. 

238 """ 

239 registry_config = RegistryConfig() 

240 registry_config["db"] = "sqlite://" 

241 registry = Registry.createFromConfig(registry_config) 

242 # Intentionally do not register instrument or insert any other 

243 # dimension records to ensure we don't need them in this mode. 

244 config = DimensionPackerTestConfig() 

245 config.packer["observation"].n_observations = 16 

246 config.packer["observation"].n_detectors = 4 

247 config.packer.name = "observation" 

248 instrument_data_id = DataCoordinate.standardize(instrument=self.name, universe=registry.dimensions) 

249 full_data_id = DataCoordinate.standardize(instrument_data_id, detector=1, visit=3, exposure=7) 

250 visit_data_id = full_data_id.subset(full_data_id.universe.extract(["visit", "detector"])) 

251 exposure_data_id = full_data_id.subset(full_data_id.universe.extract(["exposure", "detector"])) 

252 self.check_dimension_packers( 

253 registry.dimensions, 

254 # Note that we don't need to pass any more than the instrument in 

255 # the data ID yet, because we're just constructing packers, not 

256 # calling their pack method. 

257 visit_packers=[ 

258 config.packer.apply(visit_data_id), 

259 config.packer.apply(full_data_id), 

260 config.packer.apply(instrument_data_id, is_exposure=False), 

261 ], 

262 exposure_packers=[ 

263 config.packer.apply(instrument_data_id, is_exposure=True), 

264 config.packer.apply(exposure_data_id), 

265 ], 

266 n_detectors=config.packer["observation"].n_detectors, 

267 n_visits=config.packer["observation"].n_observations, 

268 n_exposures=config.packer["observation"].n_observations, 

269 ) 

270 

271 def check_dimension_packers( 

272 self, 

273 universe: DimensionUniverse, 

274 visit_packers: list[DimensionPacker], 

275 exposure_packers: list[DimensionPacker], 

276 n_detectors: int, 

277 n_visits: int, 

278 n_exposures: int, 

279 ) -> None: 

280 """Test the behavior of one or more dimension packers constructed by 

281 an instrument. 

282 

283 Parameters 

284 ---------- 

285 universe : `lsst.daf.butler.DimensionUniverse` 

286 Data model for butler data IDs. 

287 visit_packers : `list` [ `DimensionPacker` ] 

288 Packers with ``{visit, detector}`` dimensions to test. 

289 exposure_packers : `list` [ `DimensionPacker` ] 

290 Packers with ``{exposure, detector}`` dimensions to test. 

291 n_detectors : `int` 

292 Number of detectors all packers have been configured to reserve 

293 space for. 

294 n_visits : `int` 

295 Number of visits all packers in ``visit_packers`` have been 

296 configured to reserve space for. 

297 n_exposures : `int` 

298 Number of exposures all packers in ``exposure_packers`` have been 

299 configured to reserve space for. 

300 """ 

301 full_data_id = DataCoordinate.standardize( 

302 instrument=self.name, detector=1, visit=3, exposure=7, universe=universe 

303 ) 

304 visit_data_id = full_data_id.subset(full_data_id.universe.extract(["visit", "detector"])) 

305 exposure_data_id = full_data_id.subset(full_data_id.universe.extract(["exposure", "detector"])) 

306 for n, packer in enumerate(visit_packers): 

307 with self.subTest(n=n): 

308 packed_id, max_bits = packer.pack(full_data_id, returnMaxBits=True) 

309 self.assertEqual(packed_id, full_data_id["detector"] + full_data_id["visit"] * n_detectors) 

310 self.assertEqual(max_bits, math.ceil(math.log2(n_detectors * n_visits))) 

311 self.assertEqual(visit_data_id, packer.unpack(packed_id)) 

312 with self.assertRaisesRegex(ValueError, f"Detector ID {n_detectors} is out of bounds"): 

313 packer.pack(instrument=self.name, detector=n_detectors, visit=3) 

314 with self.assertRaisesRegex(ValueError, f"Visit ID {n_visits} is out of bounds"): 

315 packer.pack(instrument=self.name, detector=1, visit=n_visits) 

316 for n, packer in enumerate(exposure_packers): 

317 with self.subTest(n=n): 

318 packed_id, max_bits = packer.pack(full_data_id, returnMaxBits=True) 

319 self.assertEqual(packed_id, full_data_id["detector"] + full_data_id["exposure"] * n_detectors) 

320 self.assertEqual(max_bits, math.ceil(math.log2(n_detectors * n_exposures))) 

321 self.assertEqual(exposure_data_id, packer.unpack(packed_id)) 

322 with self.assertRaisesRegex(ValueError, f"Detector ID {n_detectors} is out of bounds"): 

323 packer.pack(instrument=self.name, detector=n_detectors, exposure=3) 

324 with self.assertRaisesRegex(ValueError, f"Exposure ID {n_exposures} is out of bounds"): 

325 packer.pack(instrument=self.name, detector=1, exposure=n_exposures) 

326 

327 

328if __name__ == "__main__": 

329 unittest.main()