Coverage for tests/test_instrument.py: 21%
143 statements
« prev ^ index » next coverage.py v7.2.3, created at 2023-04-27 02:47 -0700
« prev ^ index » next coverage.py v7.2.3, created at 2023-04-27 02:47 -0700
1# This file is part of pipe_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Tests of the Instrument class.
23"""
25import datetime
26import math
27import unittest
29from lsst.daf.butler import DataCoordinate, DimensionPacker, DimensionUniverse, Registry, RegistryConfig
30from lsst.daf.butler.formatters.json import JsonFormatter
31from lsst.pex.config import Config
32from lsst.pipe.base import Instrument
33from lsst.utils.introspection import get_full_type_name
36class DummyInstrument(Instrument):
37 @classmethod
38 def getName(cls):
39 return "DummyInstrument"
41 def register(self, registry, update=False):
42 detector_max = 2
43 visit_max = 10
44 exposure_max = 8
45 record = {
46 "instrument": self.getName(),
47 "class_name": get_full_type_name(DummyInstrument),
48 "detector_max": detector_max,
49 "visit_max": visit_max,
50 "exposure_max": exposure_max,
51 }
52 with registry.transaction():
53 registry.syncDimensionData("instrument", record, update=update)
55 def getRawFormatter(self, dataId):
56 return JsonFormatter
59class BadInstrument(DummyInstrument):
60 """Instrument with wrong class name."""
62 raw_definition = ("raw2", ("instrument", "detector", "exposure"), "StructuredDataDict")
64 @classmethod
65 def getName(cls):
66 return "BadInstrument"
68 def register(self, registry, update=False):
69 # Register a bad class name
70 record = {
71 "instrument": self.getName(),
72 "class_name": "builtins.str",
73 "detector_max": 1,
74 }
75 registry.syncDimensionData("instrument", record, update=update)
78class UnimportableInstrument(DummyInstrument):
79 """Instrument with class name that does not exist."""
81 @classmethod
82 def getName(cls):
83 return "NoImportInstr"
85 def register(self, registry, update=False):
86 # Register a bad class name
87 record = {
88 "instrument": self.getName(),
89 "class_name": "not.importable",
90 "detector_max": 1,
91 }
92 registry.syncDimensionData("instrument", record, update=update)
95class DimensionPackerTestConfig(Config):
96 packer = Instrument.make_dimension_packer_config_field()
99class InstrumentTestCase(unittest.TestCase):
100 """Test for Instrument."""
102 def setUp(self):
103 self.instrument = DummyInstrument()
104 self.name = "DummyInstrument"
106 def test_basics(self):
107 self.assertEqual(self.instrument.getName(), self.name)
108 self.assertEqual(self.instrument.getRawFormatter({}), JsonFormatter)
109 self.assertIsNone(DummyInstrument.raw_definition)
110 raw = BadInstrument.raw_definition
111 self.assertEqual(raw[2], "StructuredDataDict")
113 def test_register(self):
114 """Test that register() sets appropriate Dimensions."""
115 registryConfig = RegistryConfig()
116 registryConfig["db"] = "sqlite://"
117 registry = Registry.createFromConfig(registryConfig)
118 # Check that the registry starts out empty.
119 self.instrument.importAll(registry)
120 self.assertFalse(list(registry.queryDimensionRecords("instrument")))
122 # Register and check again.
123 self.instrument.register(registry)
124 instruments = list(registry.queryDimensionRecords("instrument"))
125 self.assertEqual(len(instruments), 1)
126 self.assertEqual(instruments[0].name, self.name)
127 self.assertEqual(instruments[0].detector_max, 2)
128 self.assertIn("DummyInstrument", instruments[0].class_name)
130 self.instrument.importAll(registry)
131 from_registry = DummyInstrument.fromName("DummyInstrument", registry)
132 self.assertIsInstance(from_registry, Instrument)
133 with self.assertRaises(LookupError):
134 Instrument.fromName("NotThrere", registry)
136 # Register a bad instrument.
137 BadInstrument().register(registry)
138 with self.assertRaises(TypeError):
139 Instrument.fromName("BadInstrument", registry)
141 UnimportableInstrument().register(registry)
142 with self.assertRaises(ImportError):
143 Instrument.fromName("NoImportInstr", registry)
145 # This should work even with the bad class name.
146 self.instrument.importAll(registry)
148 def test_defaults(self):
149 self.assertEqual(self.instrument.makeDefaultRawIngestRunName(), "DummyInstrument/raw/all")
150 self.assertEqual(
151 self.instrument.makeUnboundedCalibrationRunName("a", "b"), "DummyInstrument/calib/a/b/unbounded"
152 )
153 self.assertEqual(
154 self.instrument.makeCuratedCalibrationRunName("2018-05-04", "a"),
155 "DummyInstrument/calib/a/curated/20180504T000000Z",
156 )
157 self.assertEqual(self.instrument.makeCalibrationCollectionName("c"), "DummyInstrument/calib/c")
158 self.assertEqual(self.instrument.makeRefCatCollectionName(), "refcats")
159 self.assertEqual(self.instrument.makeRefCatCollectionName("a"), "refcats/a")
160 self.assertEqual(self.instrument.makeUmbrellaCollectionName(), "DummyInstrument/defaults")
162 instrument = DummyInstrument(collection_prefix="Different")
163 self.assertEqual(instrument.makeCollectionName("a"), "Different/a")
164 self.assertEqual(self.instrument.makeCollectionName("a"), "DummyInstrument/a")
166 def test_collection_timestamps(self):
167 self.assertEqual(
168 Instrument.formatCollectionTimestamp("2018-05-03"),
169 "20180503T000000Z",
170 )
171 self.assertEqual(
172 Instrument.formatCollectionTimestamp("2018-05-03T14:32:16"),
173 "20180503T143216Z",
174 )
175 self.assertEqual(
176 Instrument.formatCollectionTimestamp("20180503T143216Z"),
177 "20180503T143216Z",
178 )
179 self.assertEqual(
180 Instrument.formatCollectionTimestamp(datetime.datetime(2018, 5, 3, 14, 32, 16)),
181 "20180503T143216Z",
182 )
183 formattedNow = Instrument.makeCollectionTimestamp()
184 self.assertIsInstance(formattedNow, str)
185 datetimeThen1 = datetime.datetime.strptime(formattedNow, "%Y%m%dT%H%M%S%z")
186 self.assertEqual(datetimeThen1.tzinfo, datetime.timezone.utc)
188 with self.assertRaises(TypeError):
189 Instrument.formatCollectionTimestamp(0)
191 def test_dimension_packer_config_defaults(self):
192 """Test the ObservationDimensionPacker class and the Instrument-based
193 systems for constructing it, in the case where the Instrument-defined
194 default is used.
195 """
196 registry_config = RegistryConfig()
197 registry_config["db"] = "sqlite://"
198 registry = Registry.createFromConfig(registry_config)
199 self.instrument.register(registry)
200 config = DimensionPackerTestConfig()
201 instrument_data_id = registry.expandDataId(instrument=self.name)
202 record = instrument_data_id.records["instrument"]
203 self.check_dimension_packers(
204 registry.dimensions,
205 # Test just one packer-construction signature here as that saves us
206 # from having to insert dimension records for visits, exposures,
207 # and detectors for this test.
208 # Note that we don't need to pass any more than the instrument in
209 # the data ID yet, because we're just constructing packers, not
210 # calling their pack method.
211 visit_packers=[config.packer.apply(instrument_data_id, is_exposure=False)],
212 exposure_packers=[config.packer.apply(instrument_data_id, is_exposure=True)],
213 n_detectors=record.detector_max,
214 n_visits=record.visit_max,
215 n_exposures=record.exposure_max,
216 )
218 def test_dimension_packer_config_override(self):
219 """Test the ObservationDimensionPacker class and the Instrument-based
220 systems for constructing it, in the case where configuration overrides
221 the Instrument's default.
222 """
223 registry_config = RegistryConfig()
224 registry_config["db"] = "sqlite://"
225 registry = Registry.createFromConfig(registry_config)
226 # Intentionally do not register instrument or insert any other
227 # dimension records to ensure we don't need them in this mode.
228 config = DimensionPackerTestConfig()
229 config.packer["observation"].n_observations = 16
230 config.packer["observation"].n_detectors = 4
231 config.packer.name = "observation"
232 instrument_data_id = DataCoordinate.standardize(instrument=self.name, universe=registry.dimensions)
233 full_data_id = DataCoordinate.standardize(instrument_data_id, detector=1, visit=3, exposure=7)
234 visit_data_id = full_data_id.subset(full_data_id.universe.extract(["visit", "detector"]))
235 exposure_data_id = full_data_id.subset(full_data_id.universe.extract(["exposure", "detector"]))
236 self.check_dimension_packers(
237 registry.dimensions,
238 # Note that we don't need to pass any more than the instrument in
239 # the data ID yet, because we're just constructing packers, not
240 # calling their pack method.
241 visit_packers=[
242 config.packer.apply(visit_data_id),
243 config.packer.apply(full_data_id),
244 config.packer.apply(instrument_data_id, is_exposure=False),
245 ],
246 exposure_packers=[
247 config.packer.apply(instrument_data_id, is_exposure=True),
248 config.packer.apply(exposure_data_id),
249 ],
250 n_detectors=config.packer["observation"].n_detectors,
251 n_visits=config.packer["observation"].n_observations,
252 n_exposures=config.packer["observation"].n_observations,
253 )
255 def check_dimension_packers(
256 self,
257 universe: DimensionUniverse,
258 visit_packers: list[DimensionPacker],
259 exposure_packers: list[DimensionPacker],
260 n_detectors: int,
261 n_visits: int,
262 n_exposures: int,
263 ) -> None:
264 """Test the behavior of one or more dimension packers constructed by
265 an instrument.
267 Parameters
268 ----------
269 universe : `lsst.daf.butler.DimensionUniverse`
270 Data model for butler data IDs.
271 visit_packers : `list` [ `DimensionPacker` ]
272 Packers with ``{visit, detector}`` dimensions to test.
273 exposure_packers : `list` [ `DimensionPacker` ]
274 Packers with ``{exposure, detector}`` dimensions to test.
275 n_detectors : `int`
276 Number of detectors all packers have been configured to reserve
277 space for.
278 n_visits : `int`
279 Number of visits all packers in ``visit_packers`` have been
280 configured to reserve space for.
281 n_exposures : `int`
282 Number of exposures all packers in ``exposure_packers`` have been
283 configured to reserve space for.
284 """
285 full_data_id = DataCoordinate.standardize(
286 instrument=self.name, detector=1, visit=3, exposure=7, universe=universe
287 )
288 visit_data_id = full_data_id.subset(full_data_id.universe.extract(["visit", "detector"]))
289 exposure_data_id = full_data_id.subset(full_data_id.universe.extract(["exposure", "detector"]))
290 for n, packer in enumerate(visit_packers):
291 with self.subTest(n=n):
292 packed_id, max_bits = packer.pack(full_data_id, returnMaxBits=True)
293 self.assertEqual(packed_id, full_data_id["detector"] + full_data_id["visit"] * n_detectors)
294 self.assertEqual(max_bits, math.ceil(math.log2(n_detectors * n_visits)))
295 self.assertEqual(visit_data_id, packer.unpack(packed_id))
296 with self.assertRaisesRegex(ValueError, f"Detector ID {n_detectors} is out of bounds"):
297 packer.pack(instrument=self.name, detector=n_detectors, visit=3)
298 with self.assertRaisesRegex(ValueError, f"Visit ID {n_visits} is out of bounds"):
299 packer.pack(instrument=self.name, detector=1, visit=n_visits)
300 for n, packer in enumerate(exposure_packers):
301 with self.subTest(n=n):
302 packed_id, max_bits = packer.pack(full_data_id, returnMaxBits=True)
303 self.assertEqual(packed_id, full_data_id["detector"] + full_data_id["exposure"] * n_detectors)
304 self.assertEqual(max_bits, math.ceil(math.log2(n_detectors * n_exposures)))
305 self.assertEqual(exposure_data_id, packer.unpack(packed_id))
306 with self.assertRaisesRegex(ValueError, f"Detector ID {n_detectors} is out of bounds"):
307 packer.pack(instrument=self.name, detector=n_detectors, exposure=3)
308 with self.assertRaisesRegex(ValueError, f"Exposure ID {n_exposures} is out of bounds"):
309 packer.pack(instrument=self.name, detector=1, exposure=n_exposures)
312if __name__ == "__main__": 312 ↛ 313line 312 didn't jump to line 313, because the condition on line 312 was never true
313 unittest.main()