Coverage for tests/test_instrument.py: 23%
159 statements
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-11 03:31 -0700
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-11 03:31 -0700
1# This file is part of pipe_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28"""Tests of the Instrument class.
29"""
31import datetime
32import math
33import unittest
35from lsst.daf.butler import DataCoordinate, DimensionPacker, DimensionUniverse, RegistryConfig
36from lsst.daf.butler.formatters.json import JsonFormatter
37from lsst.daf.butler.registry.sql_registry import SqlRegistry
38from lsst.pex.config import Config
39from lsst.pipe.base import Instrument
40from lsst.utils.introspection import get_full_type_name
43class BaseDummyInstrument(Instrument):
44 """Test instrument base class."""
46 @classmethod
47 def getName(cls):
48 return "DummyInstrument"
50 def register(self, registry, update=False):
51 detector_max = 2
52 visit_max = 10
53 exposure_max = 8
54 record = {
55 "instrument": self.getName(),
56 "class_name": get_full_type_name(DummyInstrument),
57 "detector_max": detector_max,
58 "visit_max": visit_max,
59 "exposure_max": exposure_max,
60 }
61 with registry.transaction():
62 registry.syncDimensionData("instrument", record, update=update)
64 def getRawFormatter(self, dataId):
65 return JsonFormatter
68class DummyInstrument(BaseDummyInstrument):
69 """Test instrument."""
72class NotInstrument:
73 """Not an instrument class at all."""
75 def __init__(self, collection_prefix: str = ""):
76 self.collection_prefix = collection_prefix
79class BadInstrument(DummyInstrument):
80 """Instrument with wrong class name."""
82 raw_definition = ("raw2", ("instrument", "detector", "exposure"), "StructuredDataDict")
84 @classmethod
85 def getName(cls):
86 return "BadInstrument"
88 def register(self, registry, update=False):
89 # Register a bad class name
90 record = {
91 "instrument": self.getName(),
92 "class_name": "builtins.str",
93 "detector_max": 1,
94 }
95 registry.syncDimensionData("instrument", record, update=update)
98class UnimportableInstrument(DummyInstrument):
99 """Instrument with class name that does not exist."""
101 @classmethod
102 def getName(cls):
103 return "NoImportInstr"
105 def register(self, registry, update=False):
106 # Register a bad class name
107 record = {
108 "instrument": self.getName(),
109 "class_name": "not.importable",
110 "detector_max": 1,
111 }
112 registry.syncDimensionData("instrument", record, update=update)
115class DimensionPackerTestConfig(Config):
116 """Configuration for the dimension packer."""
118 packer = Instrument.make_dimension_packer_config_field()
121class InstrumentTestCase(unittest.TestCase):
122 """Test for Instrument."""
124 def setUp(self):
125 self.instrument = DummyInstrument()
126 self.name = "DummyInstrument"
128 def test_basics(self):
129 self.assertEqual(self.instrument.getName(), self.name)
130 self.assertEqual(self.instrument.getRawFormatter({}), JsonFormatter)
131 self.assertIsNone(DummyInstrument.raw_definition)
132 raw = BadInstrument.raw_definition
133 self.assertEqual(raw[2], "StructuredDataDict")
135 def test_register(self):
136 """Test that register() sets appropriate Dimensions."""
137 registryConfig = RegistryConfig()
138 registryConfig["db"] = "sqlite://"
139 registry = SqlRegistry.createFromConfig(registryConfig)
140 # Check that the registry starts out empty.
141 self.instrument.importAll(registry)
142 self.assertFalse(list(registry.queryDimensionRecords("instrument")))
144 # Register and check again.
145 self.instrument.register(registry)
146 instruments = list(registry.queryDimensionRecords("instrument"))
147 self.assertEqual(len(instruments), 1)
148 self.assertEqual(instruments[0].name, self.name)
149 self.assertEqual(instruments[0].detector_max, 2)
150 self.assertIn("DummyInstrument", instruments[0].class_name)
152 self.instrument.importAll(registry)
153 from_registry = DummyInstrument.fromName("DummyInstrument", registry)
154 self.assertIsInstance(from_registry, Instrument)
155 with self.assertRaises(LookupError):
156 Instrument.fromName("NotThrere", registry)
158 # Register a bad instrument.
159 BadInstrument().register(registry)
160 with self.assertRaises(TypeError):
161 Instrument.fromName("BadInstrument", registry)
163 UnimportableInstrument().register(registry)
164 with self.assertRaises(ImportError):
165 Instrument.fromName("NoImportInstr", registry)
167 # This should work even with the bad class name.
168 self.instrument.importAll(registry)
170 # Check that from_string falls back to fromName.
171 from_string = DummyInstrument.from_string("DummyInstrument", registry)
172 self.assertIsInstance(from_string, type(from_registry))
174 def test_from_string(self):
175 """Test Instrument.from_string works."""
176 with self.assertRaises(RuntimeError):
177 # No registry.
178 DummyInstrument.from_string("DummyInstrument")
180 with self.assertRaises(TypeError):
181 # This will raise because collection_prefix is not understood.
182 DummyInstrument.from_string("lsst.pipe.base.Task")
184 with self.assertRaises(TypeError):
185 # Not an instrument but does have collection_prefix.
186 DummyInstrument.from_string(get_full_type_name(NotInstrument))
188 with self.assertRaises(TypeError):
189 # This will raise because BaseDummyInstrument is not a subclass
190 # of DummyInstrument.
191 DummyInstrument.from_string(get_full_type_name(BaseDummyInstrument))
193 instrument = DummyInstrument.from_string(
194 get_full_type_name(DummyInstrument), collection_prefix="test"
195 )
196 self.assertEqual(instrument.collection_prefix, "test")
198 def test_defaults(self):
199 self.assertEqual(self.instrument.makeDefaultRawIngestRunName(), "DummyInstrument/raw/all")
200 self.assertEqual(
201 self.instrument.makeUnboundedCalibrationRunName("a", "b"), "DummyInstrument/calib/a/b/unbounded"
202 )
203 self.assertEqual(
204 self.instrument.makeCuratedCalibrationRunName("2018-05-04", "a"),
205 "DummyInstrument/calib/a/curated/20180504T000000Z",
206 )
207 self.assertEqual(self.instrument.makeCalibrationCollectionName("c"), "DummyInstrument/calib/c")
208 self.assertEqual(self.instrument.makeRefCatCollectionName(), "refcats")
209 self.assertEqual(self.instrument.makeRefCatCollectionName("a"), "refcats/a")
210 self.assertEqual(self.instrument.makeUmbrellaCollectionName(), "DummyInstrument/defaults")
212 instrument = DummyInstrument(collection_prefix="Different")
213 self.assertEqual(instrument.makeCollectionName("a"), "Different/a")
214 self.assertEqual(self.instrument.makeCollectionName("a"), "DummyInstrument/a")
216 def test_collection_timestamps(self):
217 self.assertEqual(
218 Instrument.formatCollectionTimestamp("2018-05-03"),
219 "20180503T000000Z",
220 )
221 self.assertEqual(
222 Instrument.formatCollectionTimestamp("2018-05-03T14:32:16"),
223 "20180503T143216Z",
224 )
225 self.assertEqual(
226 Instrument.formatCollectionTimestamp("20180503T143216Z"),
227 "20180503T143216Z",
228 )
229 self.assertEqual(
230 Instrument.formatCollectionTimestamp(datetime.datetime(2018, 5, 3, 14, 32, 16)),
231 "20180503T143216Z",
232 )
233 formattedNow = Instrument.makeCollectionTimestamp()
234 self.assertIsInstance(formattedNow, str)
235 datetimeThen1 = datetime.datetime.strptime(formattedNow, "%Y%m%dT%H%M%S%z")
236 self.assertEqual(datetimeThen1.tzinfo, datetime.timezone.utc)
238 with self.assertRaises(TypeError):
239 Instrument.formatCollectionTimestamp(0)
241 def test_dimension_packer_config_defaults(self):
242 """Test the ObservationDimensionPacker class and the Instrument-based
243 systems for constructing it, in the case where the Instrument-defined
244 default is used.
245 """
246 registry_config = RegistryConfig()
247 registry_config["db"] = "sqlite://"
248 registry = SqlRegistry.createFromConfig(registry_config)
249 self.instrument.register(registry)
250 config = DimensionPackerTestConfig()
251 instrument_data_id = registry.expandDataId(instrument=self.name)
252 record = instrument_data_id.records["instrument"]
253 self.check_dimension_packers(
254 registry.dimensions,
255 # Test just one packer-construction signature here as that saves us
256 # from having to insert dimension records for visits, exposures,
257 # and detectors for this test.
258 # Note that we don't need to pass any more than the instrument in
259 # the data ID yet, because we're just constructing packers, not
260 # calling their pack method.
261 visit_packers=[
262 config.packer.apply(instrument_data_id, is_exposure=False),
263 Instrument.make_default_dimension_packer(instrument_data_id, is_exposure=False),
264 ],
265 exposure_packers=[
266 config.packer.apply(instrument_data_id, is_exposure=True),
267 Instrument.make_default_dimension_packer(instrument_data_id, is_exposure=True),
268 ],
269 n_detectors=record.detector_max,
270 n_visits=record.visit_max,
271 n_exposures=record.exposure_max,
272 )
274 def test_dimension_packer_config_override(self):
275 """Test the ObservationDimensionPacker class and the Instrument-based
276 systems for constructing it, in the case where configuration overrides
277 the Instrument's default.
278 """
279 registry_config = RegistryConfig()
280 registry_config["db"] = "sqlite://"
281 registry = SqlRegistry.createFromConfig(registry_config)
282 # Intentionally do not register instrument or insert any other
283 # dimension records to ensure we don't need them in this mode.
284 config = DimensionPackerTestConfig()
285 config.packer["observation"].n_observations = 16
286 config.packer["observation"].n_detectors = 4
287 config.packer.name = "observation"
288 instrument_data_id = DataCoordinate.standardize(instrument=self.name, universe=registry.dimensions)
289 full_data_id = DataCoordinate.standardize(instrument_data_id, detector=1, visit=3, exposure=7)
290 visit_data_id = full_data_id.subset(full_data_id.universe.conform(["visit", "detector"]))
291 exposure_data_id = full_data_id.subset(full_data_id.universe.conform(["exposure", "detector"]))
292 self.check_dimension_packers(
293 registry.dimensions,
294 # Note that we don't need to pass any more than the instrument in
295 # the data ID yet, because we're just constructing packers, not
296 # calling their pack method.
297 visit_packers=[
298 config.packer.apply(visit_data_id),
299 config.packer.apply(full_data_id),
300 config.packer.apply(instrument_data_id, is_exposure=False),
301 ],
302 exposure_packers=[
303 config.packer.apply(instrument_data_id, is_exposure=True),
304 config.packer.apply(exposure_data_id),
305 ],
306 n_detectors=config.packer["observation"].n_detectors,
307 n_visits=config.packer["observation"].n_observations,
308 n_exposures=config.packer["observation"].n_observations,
309 )
311 def check_dimension_packers(
312 self,
313 universe: DimensionUniverse,
314 visit_packers: list[DimensionPacker],
315 exposure_packers: list[DimensionPacker],
316 n_detectors: int,
317 n_visits: int,
318 n_exposures: int,
319 ) -> None:
320 """Test the behavior of one or more dimension packers constructed by
321 an instrument.
323 Parameters
324 ----------
325 universe : `lsst.daf.butler.DimensionUniverse`
326 Data model for butler data IDs.
327 visit_packers : `list` [ `DimensionPacker` ]
328 Packers with ``{visit, detector}`` dimensions to test.
329 exposure_packers : `list` [ `DimensionPacker` ]
330 Packers with ``{exposure, detector}`` dimensions to test.
331 n_detectors : `int`
332 Number of detectors all packers have been configured to reserve
333 space for.
334 n_visits : `int`
335 Number of visits all packers in ``visit_packers`` have been
336 configured to reserve space for.
337 n_exposures : `int`
338 Number of exposures all packers in ``exposure_packers`` have been
339 configured to reserve space for.
340 """
341 full_data_id = DataCoordinate.standardize(
342 instrument=self.name, detector=1, visit=3, exposure=7, universe=universe
343 )
344 visit_data_id = full_data_id.subset(full_data_id.universe.conform(["visit", "detector"]))
345 exposure_data_id = full_data_id.subset(full_data_id.universe.conform(["exposure", "detector"]))
346 for n, packer in enumerate(visit_packers):
347 with self.subTest(n=n):
348 packed_id, max_bits = packer.pack(full_data_id, returnMaxBits=True)
349 self.assertEqual(packed_id, full_data_id["detector"] + full_data_id["visit"] * n_detectors)
350 self.assertEqual(max_bits, math.ceil(math.log2(n_detectors * n_visits)))
351 self.assertEqual(visit_data_id, packer.unpack(packed_id))
352 with self.assertRaisesRegex(ValueError, f"Detector ID {n_detectors} is out of bounds"):
353 packer.pack(instrument=self.name, detector=n_detectors, visit=3)
354 with self.assertRaisesRegex(ValueError, f"Visit ID {n_visits} is out of bounds"):
355 packer.pack(instrument=self.name, detector=1, visit=n_visits)
356 for n, packer in enumerate(exposure_packers):
357 with self.subTest(n=n):
358 packed_id, max_bits = packer.pack(full_data_id, returnMaxBits=True)
359 self.assertEqual(packed_id, full_data_id["detector"] + full_data_id["exposure"] * n_detectors)
360 self.assertEqual(max_bits, math.ceil(math.log2(n_detectors * n_exposures)))
361 self.assertEqual(exposure_data_id, packer.unpack(packed_id))
362 with self.assertRaisesRegex(ValueError, f"Detector ID {n_detectors} is out of bounds"):
363 packer.pack(instrument=self.name, detector=n_detectors, exposure=3)
364 with self.assertRaisesRegex(ValueError, f"Exposure ID {n_exposures} is out of bounds"):
365 packer.pack(instrument=self.name, detector=1, exposure=n_exposures)
368if __name__ == "__main__":
369 unittest.main()