Coverage for tests/test_instrument.py: 23%

159 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-10 03:25 -0700

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Tests of the Instrument class. 

29""" 

30 

31import datetime 

32import math 

33import unittest 

34 

35from lsst.daf.butler import DataCoordinate, DimensionPacker, DimensionUniverse, RegistryConfig 

36from lsst.daf.butler.formatters.json import JsonFormatter 

37from lsst.daf.butler.registry.sql_registry import SqlRegistry 

38from lsst.pex.config import Config 

39from lsst.pipe.base import Instrument 

40from lsst.utils.introspection import get_full_type_name 

41 

42 

43class BaseDummyInstrument(Instrument): 

44 """Test instrument base class.""" 

45 

46 @classmethod 

47 def getName(cls): 

48 return "DummyInstrument" 

49 

50 def register(self, registry, update=False): 

51 detector_max = 2 

52 visit_max = 10 

53 exposure_max = 8 

54 record = { 

55 "instrument": self.getName(), 

56 "class_name": get_full_type_name(DummyInstrument), 

57 "detector_max": detector_max, 

58 "visit_max": visit_max, 

59 "exposure_max": exposure_max, 

60 } 

61 with registry.transaction(): 

62 registry.syncDimensionData("instrument", record, update=update) 

63 

64 def getRawFormatter(self, dataId): 

65 return JsonFormatter 

66 

67 

68class DummyInstrument(BaseDummyInstrument): 

69 """Test instrument.""" 

70 

71 

72class NotInstrument: 

73 """Not an instrument class at all.""" 

74 

75 def __init__(self, collection_prefix: str = ""): 

76 self.collection_prefix = collection_prefix 

77 

78 

79class BadInstrument(DummyInstrument): 

80 """Instrument with wrong class name.""" 

81 

82 raw_definition = ("raw2", ("instrument", "detector", "exposure"), "StructuredDataDict") 

83 

84 @classmethod 

85 def getName(cls): 

86 return "BadInstrument" 

87 

88 def register(self, registry, update=False): 

89 # Register a bad class name 

90 record = { 

91 "instrument": self.getName(), 

92 "class_name": "builtins.str", 

93 "detector_max": 1, 

94 } 

95 registry.syncDimensionData("instrument", record, update=update) 

96 

97 

98class UnimportableInstrument(DummyInstrument): 

99 """Instrument with class name that does not exist.""" 

100 

101 @classmethod 

102 def getName(cls): 

103 return "NoImportInstr" 

104 

105 def register(self, registry, update=False): 

106 # Register a bad class name 

107 record = { 

108 "instrument": self.getName(), 

109 "class_name": "not.importable", 

110 "detector_max": 1, 

111 } 

112 registry.syncDimensionData("instrument", record, update=update) 

113 

114 

115class DimensionPackerTestConfig(Config): 

116 """Configuration for the dimension packer.""" 

117 

118 packer = Instrument.make_dimension_packer_config_field() 

119 

120 

121class InstrumentTestCase(unittest.TestCase): 

122 """Test for Instrument.""" 

123 

124 def setUp(self): 

125 self.instrument = DummyInstrument() 

126 self.name = "DummyInstrument" 

127 

128 def test_basics(self): 

129 self.assertEqual(self.instrument.getName(), self.name) 

130 self.assertEqual(self.instrument.getRawFormatter({}), JsonFormatter) 

131 self.assertIsNone(DummyInstrument.raw_definition) 

132 raw = BadInstrument.raw_definition 

133 self.assertEqual(raw[2], "StructuredDataDict") 

134 

135 def test_register(self): 

136 """Test that register() sets appropriate Dimensions.""" 

137 registryConfig = RegistryConfig() 

138 registryConfig["db"] = "sqlite://" 

139 registry = SqlRegistry.createFromConfig(registryConfig) 

140 # Check that the registry starts out empty. 

141 self.instrument.importAll(registry) 

142 self.assertFalse(list(registry.queryDimensionRecords("instrument"))) 

143 

144 # Register and check again. 

145 self.instrument.register(registry) 

146 instruments = list(registry.queryDimensionRecords("instrument")) 

147 self.assertEqual(len(instruments), 1) 

148 self.assertEqual(instruments[0].name, self.name) 

149 self.assertEqual(instruments[0].detector_max, 2) 

150 self.assertIn("DummyInstrument", instruments[0].class_name) 

151 

152 self.instrument.importAll(registry) 

153 from_registry = DummyInstrument.fromName("DummyInstrument", registry) 

154 self.assertIsInstance(from_registry, Instrument) 

155 with self.assertRaises(LookupError): 

156 Instrument.fromName("NotThrere", registry) 

157 

158 # Register a bad instrument. 

159 BadInstrument().register(registry) 

160 with self.assertRaises(TypeError): 

161 Instrument.fromName("BadInstrument", registry) 

162 

163 UnimportableInstrument().register(registry) 

164 with self.assertRaises(ImportError): 

165 Instrument.fromName("NoImportInstr", registry) 

166 

167 # This should work even with the bad class name. 

168 self.instrument.importAll(registry) 

169 

170 # Check that from_string falls back to fromName. 

171 from_string = DummyInstrument.from_string("DummyInstrument", registry) 

172 self.assertIsInstance(from_string, type(from_registry)) 

173 

174 def test_from_string(self): 

175 """Test Instrument.from_string works.""" 

176 with self.assertRaises(RuntimeError): 

177 # No registry. 

178 DummyInstrument.from_string("DummyInstrument") 

179 

180 with self.assertRaises(TypeError): 

181 # This will raise because collection_prefix is not understood. 

182 DummyInstrument.from_string("lsst.pipe.base.Task") 

183 

184 with self.assertRaises(TypeError): 

185 # Not an instrument but does have collection_prefix. 

186 DummyInstrument.from_string(get_full_type_name(NotInstrument)) 

187 

188 with self.assertRaises(TypeError): 

189 # This will raise because BaseDummyInstrument is not a subclass 

190 # of DummyInstrument. 

191 DummyInstrument.from_string(get_full_type_name(BaseDummyInstrument)) 

192 

193 instrument = DummyInstrument.from_string( 

194 get_full_type_name(DummyInstrument), collection_prefix="test" 

195 ) 

196 self.assertEqual(instrument.collection_prefix, "test") 

197 

198 def test_defaults(self): 

199 self.assertEqual(self.instrument.makeDefaultRawIngestRunName(), "DummyInstrument/raw/all") 

200 self.assertEqual( 

201 self.instrument.makeUnboundedCalibrationRunName("a", "b"), "DummyInstrument/calib/a/b/unbounded" 

202 ) 

203 self.assertEqual( 

204 self.instrument.makeCuratedCalibrationRunName("2018-05-04", "a"), 

205 "DummyInstrument/calib/a/curated/20180504T000000Z", 

206 ) 

207 self.assertEqual(self.instrument.makeCalibrationCollectionName("c"), "DummyInstrument/calib/c") 

208 self.assertEqual(self.instrument.makeRefCatCollectionName(), "refcats") 

209 self.assertEqual(self.instrument.makeRefCatCollectionName("a"), "refcats/a") 

210 self.assertEqual(self.instrument.makeUmbrellaCollectionName(), "DummyInstrument/defaults") 

211 

212 instrument = DummyInstrument(collection_prefix="Different") 

213 self.assertEqual(instrument.makeCollectionName("a"), "Different/a") 

214 self.assertEqual(self.instrument.makeCollectionName("a"), "DummyInstrument/a") 

215 

216 def test_collection_timestamps(self): 

217 self.assertEqual( 

218 Instrument.formatCollectionTimestamp("2018-05-03"), 

219 "20180503T000000Z", 

220 ) 

221 self.assertEqual( 

222 Instrument.formatCollectionTimestamp("2018-05-03T14:32:16"), 

223 "20180503T143216Z", 

224 ) 

225 self.assertEqual( 

226 Instrument.formatCollectionTimestamp("20180503T143216Z"), 

227 "20180503T143216Z", 

228 ) 

229 self.assertEqual( 

230 Instrument.formatCollectionTimestamp(datetime.datetime(2018, 5, 3, 14, 32, 16)), 

231 "20180503T143216Z", 

232 ) 

233 formattedNow = Instrument.makeCollectionTimestamp() 

234 self.assertIsInstance(formattedNow, str) 

235 datetimeThen1 = datetime.datetime.strptime(formattedNow, "%Y%m%dT%H%M%S%z") 

236 self.assertEqual(datetimeThen1.tzinfo, datetime.timezone.utc) 

237 

238 with self.assertRaises(TypeError): 

239 Instrument.formatCollectionTimestamp(0) 

240 

241 def test_dimension_packer_config_defaults(self): 

242 """Test the ObservationDimensionPacker class and the Instrument-based 

243 systems for constructing it, in the case where the Instrument-defined 

244 default is used. 

245 """ 

246 registry_config = RegistryConfig() 

247 registry_config["db"] = "sqlite://" 

248 registry = SqlRegistry.createFromConfig(registry_config) 

249 self.instrument.register(registry) 

250 config = DimensionPackerTestConfig() 

251 instrument_data_id = registry.expandDataId(instrument=self.name) 

252 record = instrument_data_id.records["instrument"] 

253 self.check_dimension_packers( 

254 registry.dimensions, 

255 # Test just one packer-construction signature here as that saves us 

256 # from having to insert dimension records for visits, exposures, 

257 # and detectors for this test. 

258 # Note that we don't need to pass any more than the instrument in 

259 # the data ID yet, because we're just constructing packers, not 

260 # calling their pack method. 

261 visit_packers=[ 

262 config.packer.apply(instrument_data_id, is_exposure=False), 

263 Instrument.make_default_dimension_packer(instrument_data_id, is_exposure=False), 

264 ], 

265 exposure_packers=[ 

266 config.packer.apply(instrument_data_id, is_exposure=True), 

267 Instrument.make_default_dimension_packer(instrument_data_id, is_exposure=True), 

268 ], 

269 n_detectors=record.detector_max, 

270 n_visits=record.visit_max, 

271 n_exposures=record.exposure_max, 

272 ) 

273 

274 def test_dimension_packer_config_override(self): 

275 """Test the ObservationDimensionPacker class and the Instrument-based 

276 systems for constructing it, in the case where configuration overrides 

277 the Instrument's default. 

278 """ 

279 registry_config = RegistryConfig() 

280 registry_config["db"] = "sqlite://" 

281 registry = SqlRegistry.createFromConfig(registry_config) 

282 # Intentionally do not register instrument or insert any other 

283 # dimension records to ensure we don't need them in this mode. 

284 config = DimensionPackerTestConfig() 

285 config.packer["observation"].n_observations = 16 

286 config.packer["observation"].n_detectors = 4 

287 config.packer.name = "observation" 

288 instrument_data_id = DataCoordinate.standardize(instrument=self.name, universe=registry.dimensions) 

289 full_data_id = DataCoordinate.standardize(instrument_data_id, detector=1, visit=3, exposure=7) 

290 visit_data_id = full_data_id.subset(full_data_id.universe.conform(["visit", "detector"])) 

291 exposure_data_id = full_data_id.subset(full_data_id.universe.conform(["exposure", "detector"])) 

292 self.check_dimension_packers( 

293 registry.dimensions, 

294 # Note that we don't need to pass any more than the instrument in 

295 # the data ID yet, because we're just constructing packers, not 

296 # calling their pack method. 

297 visit_packers=[ 

298 config.packer.apply(visit_data_id), 

299 config.packer.apply(full_data_id), 

300 config.packer.apply(instrument_data_id, is_exposure=False), 

301 ], 

302 exposure_packers=[ 

303 config.packer.apply(instrument_data_id, is_exposure=True), 

304 config.packer.apply(exposure_data_id), 

305 ], 

306 n_detectors=config.packer["observation"].n_detectors, 

307 n_visits=config.packer["observation"].n_observations, 

308 n_exposures=config.packer["observation"].n_observations, 

309 ) 

310 

311 def check_dimension_packers( 

312 self, 

313 universe: DimensionUniverse, 

314 visit_packers: list[DimensionPacker], 

315 exposure_packers: list[DimensionPacker], 

316 n_detectors: int, 

317 n_visits: int, 

318 n_exposures: int, 

319 ) -> None: 

320 """Test the behavior of one or more dimension packers constructed by 

321 an instrument. 

322 

323 Parameters 

324 ---------- 

325 universe : `lsst.daf.butler.DimensionUniverse` 

326 Data model for butler data IDs. 

327 visit_packers : `list` [ `DimensionPacker` ] 

328 Packers with ``{visit, detector}`` dimensions to test. 

329 exposure_packers : `list` [ `DimensionPacker` ] 

330 Packers with ``{exposure, detector}`` dimensions to test. 

331 n_detectors : `int` 

332 Number of detectors all packers have been configured to reserve 

333 space for. 

334 n_visits : `int` 

335 Number of visits all packers in ``visit_packers`` have been 

336 configured to reserve space for. 

337 n_exposures : `int` 

338 Number of exposures all packers in ``exposure_packers`` have been 

339 configured to reserve space for. 

340 """ 

341 full_data_id = DataCoordinate.standardize( 

342 instrument=self.name, detector=1, visit=3, exposure=7, universe=universe 

343 ) 

344 visit_data_id = full_data_id.subset(full_data_id.universe.conform(["visit", "detector"])) 

345 exposure_data_id = full_data_id.subset(full_data_id.universe.conform(["exposure", "detector"])) 

346 for n, packer in enumerate(visit_packers): 

347 with self.subTest(n=n): 

348 packed_id, max_bits = packer.pack(full_data_id, returnMaxBits=True) 

349 self.assertEqual(packed_id, full_data_id["detector"] + full_data_id["visit"] * n_detectors) 

350 self.assertEqual(max_bits, math.ceil(math.log2(n_detectors * n_visits))) 

351 self.assertEqual(visit_data_id, packer.unpack(packed_id)) 

352 with self.assertRaisesRegex(ValueError, f"Detector ID {n_detectors} is out of bounds"): 

353 packer.pack(instrument=self.name, detector=n_detectors, visit=3) 

354 with self.assertRaisesRegex(ValueError, f"Visit ID {n_visits} is out of bounds"): 

355 packer.pack(instrument=self.name, detector=1, visit=n_visits) 

356 for n, packer in enumerate(exposure_packers): 

357 with self.subTest(n=n): 

358 packed_id, max_bits = packer.pack(full_data_id, returnMaxBits=True) 

359 self.assertEqual(packed_id, full_data_id["detector"] + full_data_id["exposure"] * n_detectors) 

360 self.assertEqual(max_bits, math.ceil(math.log2(n_detectors * n_exposures))) 

361 self.assertEqual(exposure_data_id, packer.unpack(packed_id)) 

362 with self.assertRaisesRegex(ValueError, f"Detector ID {n_detectors} is out of bounds"): 

363 packer.pack(instrument=self.name, detector=n_detectors, exposure=3) 

364 with self.assertRaisesRegex(ValueError, f"Exposure ID {n_exposures} is out of bounds"): 

365 packer.pack(instrument=self.name, detector=1, exposure=n_exposures) 

366 

367 

368if __name__ == "__main__": 

369 unittest.main()