Coverage for tests/test_instrument.py: 23%
141 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-28 09:35 +0000
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-28 09:35 +0000
1# This file is part of pipe_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28"""Tests of the Instrument class.
29"""
31import datetime
32import math
33import unittest
35from lsst.daf.butler import DataCoordinate, DimensionPacker, DimensionUniverse, Registry, RegistryConfig
36from lsst.daf.butler.formatters.json import JsonFormatter
37from lsst.pex.config import Config
38from lsst.pipe.base import Instrument
39from lsst.utils.introspection import get_full_type_name
42class DummyInstrument(Instrument):
43 """Test instrument."""
45 @classmethod
46 def getName(cls):
47 return "DummyInstrument"
49 def register(self, registry, update=False):
50 detector_max = 2
51 visit_max = 10
52 exposure_max = 8
53 record = {
54 "instrument": self.getName(),
55 "class_name": get_full_type_name(DummyInstrument),
56 "detector_max": detector_max,
57 "visit_max": visit_max,
58 "exposure_max": exposure_max,
59 }
60 with registry.transaction():
61 registry.syncDimensionData("instrument", record, update=update)
63 def getRawFormatter(self, dataId):
64 return JsonFormatter
67class BadInstrument(DummyInstrument):
68 """Instrument with wrong class name."""
70 raw_definition = ("raw2", ("instrument", "detector", "exposure"), "StructuredDataDict")
72 @classmethod
73 def getName(cls):
74 return "BadInstrument"
76 def register(self, registry, update=False):
77 # Register a bad class name
78 record = {
79 "instrument": self.getName(),
80 "class_name": "builtins.str",
81 "detector_max": 1,
82 }
83 registry.syncDimensionData("instrument", record, update=update)
86class UnimportableInstrument(DummyInstrument):
87 """Instrument with class name that does not exist."""
89 @classmethod
90 def getName(cls):
91 return "NoImportInstr"
93 def register(self, registry, update=False):
94 # Register a bad class name
95 record = {
96 "instrument": self.getName(),
97 "class_name": "not.importable",
98 "detector_max": 1,
99 }
100 registry.syncDimensionData("instrument", record, update=update)
103class DimensionPackerTestConfig(Config):
104 """Configuration for the dimension packer."""
106 packer = Instrument.make_dimension_packer_config_field()
109class InstrumentTestCase(unittest.TestCase):
110 """Test for Instrument."""
112 def setUp(self):
113 self.instrument = DummyInstrument()
114 self.name = "DummyInstrument"
116 def test_basics(self):
117 self.assertEqual(self.instrument.getName(), self.name)
118 self.assertEqual(self.instrument.getRawFormatter({}), JsonFormatter)
119 self.assertIsNone(DummyInstrument.raw_definition)
120 raw = BadInstrument.raw_definition
121 self.assertEqual(raw[2], "StructuredDataDict")
123 def test_register(self):
124 """Test that register() sets appropriate Dimensions."""
125 registryConfig = RegistryConfig()
126 registryConfig["db"] = "sqlite://"
127 registry = Registry.createFromConfig(registryConfig)
128 # Check that the registry starts out empty.
129 self.instrument.importAll(registry)
130 self.assertFalse(list(registry.queryDimensionRecords("instrument")))
132 # Register and check again.
133 self.instrument.register(registry)
134 instruments = list(registry.queryDimensionRecords("instrument"))
135 self.assertEqual(len(instruments), 1)
136 self.assertEqual(instruments[0].name, self.name)
137 self.assertEqual(instruments[0].detector_max, 2)
138 self.assertIn("DummyInstrument", instruments[0].class_name)
140 self.instrument.importAll(registry)
141 from_registry = DummyInstrument.fromName("DummyInstrument", registry)
142 self.assertIsInstance(from_registry, Instrument)
143 with self.assertRaises(LookupError):
144 Instrument.fromName("NotThrere", registry)
146 # Register a bad instrument.
147 BadInstrument().register(registry)
148 with self.assertRaises(TypeError):
149 Instrument.fromName("BadInstrument", registry)
151 UnimportableInstrument().register(registry)
152 with self.assertRaises(ImportError):
153 Instrument.fromName("NoImportInstr", registry)
155 # This should work even with the bad class name.
156 self.instrument.importAll(registry)
158 def test_defaults(self):
159 self.assertEqual(self.instrument.makeDefaultRawIngestRunName(), "DummyInstrument/raw/all")
160 self.assertEqual(
161 self.instrument.makeUnboundedCalibrationRunName("a", "b"), "DummyInstrument/calib/a/b/unbounded"
162 )
163 self.assertEqual(
164 self.instrument.makeCuratedCalibrationRunName("2018-05-04", "a"),
165 "DummyInstrument/calib/a/curated/20180504T000000Z",
166 )
167 self.assertEqual(self.instrument.makeCalibrationCollectionName("c"), "DummyInstrument/calib/c")
168 self.assertEqual(self.instrument.makeRefCatCollectionName(), "refcats")
169 self.assertEqual(self.instrument.makeRefCatCollectionName("a"), "refcats/a")
170 self.assertEqual(self.instrument.makeUmbrellaCollectionName(), "DummyInstrument/defaults")
172 instrument = DummyInstrument(collection_prefix="Different")
173 self.assertEqual(instrument.makeCollectionName("a"), "Different/a")
174 self.assertEqual(self.instrument.makeCollectionName("a"), "DummyInstrument/a")
176 def test_collection_timestamps(self):
177 self.assertEqual(
178 Instrument.formatCollectionTimestamp("2018-05-03"),
179 "20180503T000000Z",
180 )
181 self.assertEqual(
182 Instrument.formatCollectionTimestamp("2018-05-03T14:32:16"),
183 "20180503T143216Z",
184 )
185 self.assertEqual(
186 Instrument.formatCollectionTimestamp("20180503T143216Z"),
187 "20180503T143216Z",
188 )
189 self.assertEqual(
190 Instrument.formatCollectionTimestamp(datetime.datetime(2018, 5, 3, 14, 32, 16)),
191 "20180503T143216Z",
192 )
193 formattedNow = Instrument.makeCollectionTimestamp()
194 self.assertIsInstance(formattedNow, str)
195 datetimeThen1 = datetime.datetime.strptime(formattedNow, "%Y%m%dT%H%M%S%z")
196 self.assertEqual(datetimeThen1.tzinfo, datetime.timezone.utc)
198 with self.assertRaises(TypeError):
199 Instrument.formatCollectionTimestamp(0)
201 def test_dimension_packer_config_defaults(self):
202 """Test the ObservationDimensionPacker class and the Instrument-based
203 systems for constructing it, in the case where the Instrument-defined
204 default is used.
205 """
206 registry_config = RegistryConfig()
207 registry_config["db"] = "sqlite://"
208 registry = Registry.createFromConfig(registry_config)
209 self.instrument.register(registry)
210 config = DimensionPackerTestConfig()
211 instrument_data_id = registry.expandDataId(instrument=self.name)
212 record = instrument_data_id.records["instrument"]
213 self.check_dimension_packers(
214 registry.dimensions,
215 # Test just one packer-construction signature here as that saves us
216 # from having to insert dimension records for visits, exposures,
217 # and detectors for this test.
218 # Note that we don't need to pass any more than the instrument in
219 # the data ID yet, because we're just constructing packers, not
220 # calling their pack method.
221 visit_packers=[
222 config.packer.apply(instrument_data_id, is_exposure=False),
223 Instrument.make_default_dimension_packer(instrument_data_id, is_exposure=False),
224 ],
225 exposure_packers=[
226 config.packer.apply(instrument_data_id, is_exposure=True),
227 Instrument.make_default_dimension_packer(instrument_data_id, is_exposure=True),
228 ],
229 n_detectors=record.detector_max,
230 n_visits=record.visit_max,
231 n_exposures=record.exposure_max,
232 )
234 def test_dimension_packer_config_override(self):
235 """Test the ObservationDimensionPacker class and the Instrument-based
236 systems for constructing it, in the case where configuration overrides
237 the Instrument's default.
238 """
239 registry_config = RegistryConfig()
240 registry_config["db"] = "sqlite://"
241 registry = Registry.createFromConfig(registry_config)
242 # Intentionally do not register instrument or insert any other
243 # dimension records to ensure we don't need them in this mode.
244 config = DimensionPackerTestConfig()
245 config.packer["observation"].n_observations = 16
246 config.packer["observation"].n_detectors = 4
247 config.packer.name = "observation"
248 instrument_data_id = DataCoordinate.standardize(instrument=self.name, universe=registry.dimensions)
249 full_data_id = DataCoordinate.standardize(instrument_data_id, detector=1, visit=3, exposure=7)
250 visit_data_id = full_data_id.subset(full_data_id.universe.extract(["visit", "detector"]))
251 exposure_data_id = full_data_id.subset(full_data_id.universe.extract(["exposure", "detector"]))
252 self.check_dimension_packers(
253 registry.dimensions,
254 # Note that we don't need to pass any more than the instrument in
255 # the data ID yet, because we're just constructing packers, not
256 # calling their pack method.
257 visit_packers=[
258 config.packer.apply(visit_data_id),
259 config.packer.apply(full_data_id),
260 config.packer.apply(instrument_data_id, is_exposure=False),
261 ],
262 exposure_packers=[
263 config.packer.apply(instrument_data_id, is_exposure=True),
264 config.packer.apply(exposure_data_id),
265 ],
266 n_detectors=config.packer["observation"].n_detectors,
267 n_visits=config.packer["observation"].n_observations,
268 n_exposures=config.packer["observation"].n_observations,
269 )
271 def check_dimension_packers(
272 self,
273 universe: DimensionUniverse,
274 visit_packers: list[DimensionPacker],
275 exposure_packers: list[DimensionPacker],
276 n_detectors: int,
277 n_visits: int,
278 n_exposures: int,
279 ) -> None:
280 """Test the behavior of one or more dimension packers constructed by
281 an instrument.
283 Parameters
284 ----------
285 universe : `lsst.daf.butler.DimensionUniverse`
286 Data model for butler data IDs.
287 visit_packers : `list` [ `DimensionPacker` ]
288 Packers with ``{visit, detector}`` dimensions to test.
289 exposure_packers : `list` [ `DimensionPacker` ]
290 Packers with ``{exposure, detector}`` dimensions to test.
291 n_detectors : `int`
292 Number of detectors all packers have been configured to reserve
293 space for.
294 n_visits : `int`
295 Number of visits all packers in ``visit_packers`` have been
296 configured to reserve space for.
297 n_exposures : `int`
298 Number of exposures all packers in ``exposure_packers`` have been
299 configured to reserve space for.
300 """
301 full_data_id = DataCoordinate.standardize(
302 instrument=self.name, detector=1, visit=3, exposure=7, universe=universe
303 )
304 visit_data_id = full_data_id.subset(full_data_id.universe.extract(["visit", "detector"]))
305 exposure_data_id = full_data_id.subset(full_data_id.universe.extract(["exposure", "detector"]))
306 for n, packer in enumerate(visit_packers):
307 with self.subTest(n=n):
308 packed_id, max_bits = packer.pack(full_data_id, returnMaxBits=True)
309 self.assertEqual(packed_id, full_data_id["detector"] + full_data_id["visit"] * n_detectors)
310 self.assertEqual(max_bits, math.ceil(math.log2(n_detectors * n_visits)))
311 self.assertEqual(visit_data_id, packer.unpack(packed_id))
312 with self.assertRaisesRegex(ValueError, f"Detector ID {n_detectors} is out of bounds"):
313 packer.pack(instrument=self.name, detector=n_detectors, visit=3)
314 with self.assertRaisesRegex(ValueError, f"Visit ID {n_visits} is out of bounds"):
315 packer.pack(instrument=self.name, detector=1, visit=n_visits)
316 for n, packer in enumerate(exposure_packers):
317 with self.subTest(n=n):
318 packed_id, max_bits = packer.pack(full_data_id, returnMaxBits=True)
319 self.assertEqual(packed_id, full_data_id["detector"] + full_data_id["exposure"] * n_detectors)
320 self.assertEqual(max_bits, math.ceil(math.log2(n_detectors * n_exposures)))
321 self.assertEqual(exposure_data_id, packer.unpack(packed_id))
322 with self.assertRaisesRegex(ValueError, f"Detector ID {n_detectors} is out of bounds"):
323 packer.pack(instrument=self.name, detector=n_detectors, exposure=3)
324 with self.assertRaisesRegex(ValueError, f"Exposure ID {n_exposures} is out of bounds"):
325 packer.pack(instrument=self.name, detector=1, exposure=n_exposures)
328if __name__ == "__main__":
329 unittest.main()