Coverage for tests/test_instrument.py: 23%
141 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-23 08:14 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-23 08:14 +0000
1# This file is part of pipe_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Tests of the Instrument class.
23"""
25import datetime
26import math
27import unittest
29from lsst.daf.butler import DataCoordinate, DimensionPacker, DimensionUniverse, Registry, RegistryConfig
30from lsst.daf.butler.formatters.json import JsonFormatter
31from lsst.pex.config import Config
32from lsst.pipe.base import Instrument
33from lsst.utils.introspection import get_full_type_name
36class DummyInstrument(Instrument):
37 """Test instrument."""
39 @classmethod
40 def getName(cls):
41 return "DummyInstrument"
43 def register(self, registry, update=False):
44 detector_max = 2
45 visit_max = 10
46 exposure_max = 8
47 record = {
48 "instrument": self.getName(),
49 "class_name": get_full_type_name(DummyInstrument),
50 "detector_max": detector_max,
51 "visit_max": visit_max,
52 "exposure_max": exposure_max,
53 }
54 with registry.transaction():
55 registry.syncDimensionData("instrument", record, update=update)
57 def getRawFormatter(self, dataId):
58 return JsonFormatter
61class BadInstrument(DummyInstrument):
62 """Instrument with wrong class name."""
64 raw_definition = ("raw2", ("instrument", "detector", "exposure"), "StructuredDataDict")
66 @classmethod
67 def getName(cls):
68 return "BadInstrument"
70 def register(self, registry, update=False):
71 # Register a bad class name
72 record = {
73 "instrument": self.getName(),
74 "class_name": "builtins.str",
75 "detector_max": 1,
76 }
77 registry.syncDimensionData("instrument", record, update=update)
80class UnimportableInstrument(DummyInstrument):
81 """Instrument with class name that does not exist."""
83 @classmethod
84 def getName(cls):
85 return "NoImportInstr"
87 def register(self, registry, update=False):
88 # Register a bad class name
89 record = {
90 "instrument": self.getName(),
91 "class_name": "not.importable",
92 "detector_max": 1,
93 }
94 registry.syncDimensionData("instrument", record, update=update)
97class DimensionPackerTestConfig(Config):
98 """Configuration for the dimension packer."""
100 packer = Instrument.make_dimension_packer_config_field()
103class InstrumentTestCase(unittest.TestCase):
104 """Test for Instrument."""
106 def setUp(self):
107 self.instrument = DummyInstrument()
108 self.name = "DummyInstrument"
110 def test_basics(self):
111 self.assertEqual(self.instrument.getName(), self.name)
112 self.assertEqual(self.instrument.getRawFormatter({}), JsonFormatter)
113 self.assertIsNone(DummyInstrument.raw_definition)
114 raw = BadInstrument.raw_definition
115 self.assertEqual(raw[2], "StructuredDataDict")
117 def test_register(self):
118 """Test that register() sets appropriate Dimensions."""
119 registryConfig = RegistryConfig()
120 registryConfig["db"] = "sqlite://"
121 registry = Registry.createFromConfig(registryConfig)
122 # Check that the registry starts out empty.
123 self.instrument.importAll(registry)
124 self.assertFalse(list(registry.queryDimensionRecords("instrument")))
126 # Register and check again.
127 self.instrument.register(registry)
128 instruments = list(registry.queryDimensionRecords("instrument"))
129 self.assertEqual(len(instruments), 1)
130 self.assertEqual(instruments[0].name, self.name)
131 self.assertEqual(instruments[0].detector_max, 2)
132 self.assertIn("DummyInstrument", instruments[0].class_name)
134 self.instrument.importAll(registry)
135 from_registry = DummyInstrument.fromName("DummyInstrument", registry)
136 self.assertIsInstance(from_registry, Instrument)
137 with self.assertRaises(LookupError):
138 Instrument.fromName("NotThrere", registry)
140 # Register a bad instrument.
141 BadInstrument().register(registry)
142 with self.assertRaises(TypeError):
143 Instrument.fromName("BadInstrument", registry)
145 UnimportableInstrument().register(registry)
146 with self.assertRaises(ImportError):
147 Instrument.fromName("NoImportInstr", registry)
149 # This should work even with the bad class name.
150 self.instrument.importAll(registry)
152 def test_defaults(self):
153 self.assertEqual(self.instrument.makeDefaultRawIngestRunName(), "DummyInstrument/raw/all")
154 self.assertEqual(
155 self.instrument.makeUnboundedCalibrationRunName("a", "b"), "DummyInstrument/calib/a/b/unbounded"
156 )
157 self.assertEqual(
158 self.instrument.makeCuratedCalibrationRunName("2018-05-04", "a"),
159 "DummyInstrument/calib/a/curated/20180504T000000Z",
160 )
161 self.assertEqual(self.instrument.makeCalibrationCollectionName("c"), "DummyInstrument/calib/c")
162 self.assertEqual(self.instrument.makeRefCatCollectionName(), "refcats")
163 self.assertEqual(self.instrument.makeRefCatCollectionName("a"), "refcats/a")
164 self.assertEqual(self.instrument.makeUmbrellaCollectionName(), "DummyInstrument/defaults")
166 instrument = DummyInstrument(collection_prefix="Different")
167 self.assertEqual(instrument.makeCollectionName("a"), "Different/a")
168 self.assertEqual(self.instrument.makeCollectionName("a"), "DummyInstrument/a")
170 def test_collection_timestamps(self):
171 self.assertEqual(
172 Instrument.formatCollectionTimestamp("2018-05-03"),
173 "20180503T000000Z",
174 )
175 self.assertEqual(
176 Instrument.formatCollectionTimestamp("2018-05-03T14:32:16"),
177 "20180503T143216Z",
178 )
179 self.assertEqual(
180 Instrument.formatCollectionTimestamp("20180503T143216Z"),
181 "20180503T143216Z",
182 )
183 self.assertEqual(
184 Instrument.formatCollectionTimestamp(datetime.datetime(2018, 5, 3, 14, 32, 16)),
185 "20180503T143216Z",
186 )
187 formattedNow = Instrument.makeCollectionTimestamp()
188 self.assertIsInstance(formattedNow, str)
189 datetimeThen1 = datetime.datetime.strptime(formattedNow, "%Y%m%dT%H%M%S%z")
190 self.assertEqual(datetimeThen1.tzinfo, datetime.timezone.utc)
192 with self.assertRaises(TypeError):
193 Instrument.formatCollectionTimestamp(0)
195 def test_dimension_packer_config_defaults(self):
196 """Test the ObservationDimensionPacker class and the Instrument-based
197 systems for constructing it, in the case where the Instrument-defined
198 default is used.
199 """
200 registry_config = RegistryConfig()
201 registry_config["db"] = "sqlite://"
202 registry = Registry.createFromConfig(registry_config)
203 self.instrument.register(registry)
204 config = DimensionPackerTestConfig()
205 instrument_data_id = registry.expandDataId(instrument=self.name)
206 record = instrument_data_id.records["instrument"]
207 self.check_dimension_packers(
208 registry.dimensions,
209 # Test just one packer-construction signature here as that saves us
210 # from having to insert dimension records for visits, exposures,
211 # and detectors for this test.
212 # Note that we don't need to pass any more than the instrument in
213 # the data ID yet, because we're just constructing packers, not
214 # calling their pack method.
215 visit_packers=[
216 config.packer.apply(instrument_data_id, is_exposure=False),
217 Instrument.make_default_dimension_packer(instrument_data_id, is_exposure=False),
218 ],
219 exposure_packers=[
220 config.packer.apply(instrument_data_id, is_exposure=True),
221 Instrument.make_default_dimension_packer(instrument_data_id, is_exposure=True),
222 ],
223 n_detectors=record.detector_max,
224 n_visits=record.visit_max,
225 n_exposures=record.exposure_max,
226 )
228 def test_dimension_packer_config_override(self):
229 """Test the ObservationDimensionPacker class and the Instrument-based
230 systems for constructing it, in the case where configuration overrides
231 the Instrument's default.
232 """
233 registry_config = RegistryConfig()
234 registry_config["db"] = "sqlite://"
235 registry = Registry.createFromConfig(registry_config)
236 # Intentionally do not register instrument or insert any other
237 # dimension records to ensure we don't need them in this mode.
238 config = DimensionPackerTestConfig()
239 config.packer["observation"].n_observations = 16
240 config.packer["observation"].n_detectors = 4
241 config.packer.name = "observation"
242 instrument_data_id = DataCoordinate.standardize(instrument=self.name, universe=registry.dimensions)
243 full_data_id = DataCoordinate.standardize(instrument_data_id, detector=1, visit=3, exposure=7)
244 visit_data_id = full_data_id.subset(full_data_id.universe.extract(["visit", "detector"]))
245 exposure_data_id = full_data_id.subset(full_data_id.universe.extract(["exposure", "detector"]))
246 self.check_dimension_packers(
247 registry.dimensions,
248 # Note that we don't need to pass any more than the instrument in
249 # the data ID yet, because we're just constructing packers, not
250 # calling their pack method.
251 visit_packers=[
252 config.packer.apply(visit_data_id),
253 config.packer.apply(full_data_id),
254 config.packer.apply(instrument_data_id, is_exposure=False),
255 ],
256 exposure_packers=[
257 config.packer.apply(instrument_data_id, is_exposure=True),
258 config.packer.apply(exposure_data_id),
259 ],
260 n_detectors=config.packer["observation"].n_detectors,
261 n_visits=config.packer["observation"].n_observations,
262 n_exposures=config.packer["observation"].n_observations,
263 )
265 def check_dimension_packers(
266 self,
267 universe: DimensionUniverse,
268 visit_packers: list[DimensionPacker],
269 exposure_packers: list[DimensionPacker],
270 n_detectors: int,
271 n_visits: int,
272 n_exposures: int,
273 ) -> None:
274 """Test the behavior of one or more dimension packers constructed by
275 an instrument.
277 Parameters
278 ----------
279 universe : `lsst.daf.butler.DimensionUniverse`
280 Data model for butler data IDs.
281 visit_packers : `list` [ `DimensionPacker` ]
282 Packers with ``{visit, detector}`` dimensions to test.
283 exposure_packers : `list` [ `DimensionPacker` ]
284 Packers with ``{exposure, detector}`` dimensions to test.
285 n_detectors : `int`
286 Number of detectors all packers have been configured to reserve
287 space for.
288 n_visits : `int`
289 Number of visits all packers in ``visit_packers`` have been
290 configured to reserve space for.
291 n_exposures : `int`
292 Number of exposures all packers in ``exposure_packers`` have been
293 configured to reserve space for.
294 """
295 full_data_id = DataCoordinate.standardize(
296 instrument=self.name, detector=1, visit=3, exposure=7, universe=universe
297 )
298 visit_data_id = full_data_id.subset(full_data_id.universe.extract(["visit", "detector"]))
299 exposure_data_id = full_data_id.subset(full_data_id.universe.extract(["exposure", "detector"]))
300 for n, packer in enumerate(visit_packers):
301 with self.subTest(n=n):
302 packed_id, max_bits = packer.pack(full_data_id, returnMaxBits=True)
303 self.assertEqual(packed_id, full_data_id["detector"] + full_data_id["visit"] * n_detectors)
304 self.assertEqual(max_bits, math.ceil(math.log2(n_detectors * n_visits)))
305 self.assertEqual(visit_data_id, packer.unpack(packed_id))
306 with self.assertRaisesRegex(ValueError, f"Detector ID {n_detectors} is out of bounds"):
307 packer.pack(instrument=self.name, detector=n_detectors, visit=3)
308 with self.assertRaisesRegex(ValueError, f"Visit ID {n_visits} is out of bounds"):
309 packer.pack(instrument=self.name, detector=1, visit=n_visits)
310 for n, packer in enumerate(exposure_packers):
311 with self.subTest(n=n):
312 packed_id, max_bits = packer.pack(full_data_id, returnMaxBits=True)
313 self.assertEqual(packed_id, full_data_id["detector"] + full_data_id["exposure"] * n_detectors)
314 self.assertEqual(max_bits, math.ceil(math.log2(n_detectors * n_exposures)))
315 self.assertEqual(exposure_data_id, packer.unpack(packed_id))
316 with self.assertRaisesRegex(ValueError, f"Detector ID {n_detectors} is out of bounds"):
317 packer.pack(instrument=self.name, detector=n_detectors, exposure=3)
318 with self.assertRaisesRegex(ValueError, f"Exposure ID {n_exposures} is out of bounds"):
319 packer.pack(instrument=self.name, detector=1, exposure=n_exposures)
322if __name__ == "__main__":
323 unittest.main()