Coverage for tests/test_instrument.py: 24%
142 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-11-18 10:50 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-11-18 10:50 +0000
1# This file is part of pipe_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28"""Tests of the Instrument class.
29"""
31import datetime
32import math
33import unittest
35from lsst.daf.butler import DataCoordinate, DimensionPacker, DimensionUniverse, RegistryConfig
36from lsst.daf.butler.formatters.json import JsonFormatter
37from lsst.daf.butler.registry.sql_registry import SqlRegistry
38from lsst.pex.config import Config
39from lsst.pipe.base import Instrument
40from lsst.utils.introspection import get_full_type_name
43class DummyInstrument(Instrument):
44 """Test instrument."""
46 @classmethod
47 def getName(cls):
48 return "DummyInstrument"
50 def register(self, registry, update=False):
51 detector_max = 2
52 visit_max = 10
53 exposure_max = 8
54 record = {
55 "instrument": self.getName(),
56 "class_name": get_full_type_name(DummyInstrument),
57 "detector_max": detector_max,
58 "visit_max": visit_max,
59 "exposure_max": exposure_max,
60 }
61 with registry.transaction():
62 registry.syncDimensionData("instrument", record, update=update)
64 def getRawFormatter(self, dataId):
65 return JsonFormatter
68class BadInstrument(DummyInstrument):
69 """Instrument with wrong class name."""
71 raw_definition = ("raw2", ("instrument", "detector", "exposure"), "StructuredDataDict")
73 @classmethod
74 def getName(cls):
75 return "BadInstrument"
77 def register(self, registry, update=False):
78 # Register a bad class name
79 record = {
80 "instrument": self.getName(),
81 "class_name": "builtins.str",
82 "detector_max": 1,
83 }
84 registry.syncDimensionData("instrument", record, update=update)
87class UnimportableInstrument(DummyInstrument):
88 """Instrument with class name that does not exist."""
90 @classmethod
91 def getName(cls):
92 return "NoImportInstr"
94 def register(self, registry, update=False):
95 # Register a bad class name
96 record = {
97 "instrument": self.getName(),
98 "class_name": "not.importable",
99 "detector_max": 1,
100 }
101 registry.syncDimensionData("instrument", record, update=update)
104class DimensionPackerTestConfig(Config):
105 """Configuration for the dimension packer."""
107 packer = Instrument.make_dimension_packer_config_field()
110class InstrumentTestCase(unittest.TestCase):
111 """Test for Instrument."""
113 def setUp(self):
114 self.instrument = DummyInstrument()
115 self.name = "DummyInstrument"
117 def test_basics(self):
118 self.assertEqual(self.instrument.getName(), self.name)
119 self.assertEqual(self.instrument.getRawFormatter({}), JsonFormatter)
120 self.assertIsNone(DummyInstrument.raw_definition)
121 raw = BadInstrument.raw_definition
122 self.assertEqual(raw[2], "StructuredDataDict")
124 def test_register(self):
125 """Test that register() sets appropriate Dimensions."""
126 registryConfig = RegistryConfig()
127 registryConfig["db"] = "sqlite://"
128 registry = SqlRegistry.createFromConfig(registryConfig)
129 # Check that the registry starts out empty.
130 self.instrument.importAll(registry)
131 self.assertFalse(list(registry.queryDimensionRecords("instrument")))
133 # Register and check again.
134 self.instrument.register(registry)
135 instruments = list(registry.queryDimensionRecords("instrument"))
136 self.assertEqual(len(instruments), 1)
137 self.assertEqual(instruments[0].name, self.name)
138 self.assertEqual(instruments[0].detector_max, 2)
139 self.assertIn("DummyInstrument", instruments[0].class_name)
141 self.instrument.importAll(registry)
142 from_registry = DummyInstrument.fromName("DummyInstrument", registry)
143 self.assertIsInstance(from_registry, Instrument)
144 with self.assertRaises(LookupError):
145 Instrument.fromName("NotThrere", registry)
147 # Register a bad instrument.
148 BadInstrument().register(registry)
149 with self.assertRaises(TypeError):
150 Instrument.fromName("BadInstrument", registry)
152 UnimportableInstrument().register(registry)
153 with self.assertRaises(ImportError):
154 Instrument.fromName("NoImportInstr", registry)
156 # This should work even with the bad class name.
157 self.instrument.importAll(registry)
159 def test_defaults(self):
160 self.assertEqual(self.instrument.makeDefaultRawIngestRunName(), "DummyInstrument/raw/all")
161 self.assertEqual(
162 self.instrument.makeUnboundedCalibrationRunName("a", "b"), "DummyInstrument/calib/a/b/unbounded"
163 )
164 self.assertEqual(
165 self.instrument.makeCuratedCalibrationRunName("2018-05-04", "a"),
166 "DummyInstrument/calib/a/curated/20180504T000000Z",
167 )
168 self.assertEqual(self.instrument.makeCalibrationCollectionName("c"), "DummyInstrument/calib/c")
169 self.assertEqual(self.instrument.makeRefCatCollectionName(), "refcats")
170 self.assertEqual(self.instrument.makeRefCatCollectionName("a"), "refcats/a")
171 self.assertEqual(self.instrument.makeUmbrellaCollectionName(), "DummyInstrument/defaults")
173 instrument = DummyInstrument(collection_prefix="Different")
174 self.assertEqual(instrument.makeCollectionName("a"), "Different/a")
175 self.assertEqual(self.instrument.makeCollectionName("a"), "DummyInstrument/a")
177 def test_collection_timestamps(self):
178 self.assertEqual(
179 Instrument.formatCollectionTimestamp("2018-05-03"),
180 "20180503T000000Z",
181 )
182 self.assertEqual(
183 Instrument.formatCollectionTimestamp("2018-05-03T14:32:16"),
184 "20180503T143216Z",
185 )
186 self.assertEqual(
187 Instrument.formatCollectionTimestamp("20180503T143216Z"),
188 "20180503T143216Z",
189 )
190 self.assertEqual(
191 Instrument.formatCollectionTimestamp(datetime.datetime(2018, 5, 3, 14, 32, 16)),
192 "20180503T143216Z",
193 )
194 formattedNow = Instrument.makeCollectionTimestamp()
195 self.assertIsInstance(formattedNow, str)
196 datetimeThen1 = datetime.datetime.strptime(formattedNow, "%Y%m%dT%H%M%S%z")
197 self.assertEqual(datetimeThen1.tzinfo, datetime.timezone.utc)
199 with self.assertRaises(TypeError):
200 Instrument.formatCollectionTimestamp(0)
202 def test_dimension_packer_config_defaults(self):
203 """Test the ObservationDimensionPacker class and the Instrument-based
204 systems for constructing it, in the case where the Instrument-defined
205 default is used.
206 """
207 registry_config = RegistryConfig()
208 registry_config["db"] = "sqlite://"
209 registry = SqlRegistry.createFromConfig(registry_config)
210 self.instrument.register(registry)
211 config = DimensionPackerTestConfig()
212 instrument_data_id = registry.expandDataId(instrument=self.name)
213 record = instrument_data_id.records["instrument"]
214 self.check_dimension_packers(
215 registry.dimensions,
216 # Test just one packer-construction signature here as that saves us
217 # from having to insert dimension records for visits, exposures,
218 # and detectors for this test.
219 # Note that we don't need to pass any more than the instrument in
220 # the data ID yet, because we're just constructing packers, not
221 # calling their pack method.
222 visit_packers=[
223 config.packer.apply(instrument_data_id, is_exposure=False),
224 Instrument.make_default_dimension_packer(instrument_data_id, is_exposure=False),
225 ],
226 exposure_packers=[
227 config.packer.apply(instrument_data_id, is_exposure=True),
228 Instrument.make_default_dimension_packer(instrument_data_id, is_exposure=True),
229 ],
230 n_detectors=record.detector_max,
231 n_visits=record.visit_max,
232 n_exposures=record.exposure_max,
233 )
235 def test_dimension_packer_config_override(self):
236 """Test the ObservationDimensionPacker class and the Instrument-based
237 systems for constructing it, in the case where configuration overrides
238 the Instrument's default.
239 """
240 registry_config = RegistryConfig()
241 registry_config["db"] = "sqlite://"
242 registry = SqlRegistry.createFromConfig(registry_config)
243 # Intentionally do not register instrument or insert any other
244 # dimension records to ensure we don't need them in this mode.
245 config = DimensionPackerTestConfig()
246 config.packer["observation"].n_observations = 16
247 config.packer["observation"].n_detectors = 4
248 config.packer.name = "observation"
249 instrument_data_id = DataCoordinate.standardize(instrument=self.name, universe=registry.dimensions)
250 full_data_id = DataCoordinate.standardize(instrument_data_id, detector=1, visit=3, exposure=7)
251 visit_data_id = full_data_id.subset(full_data_id.universe.extract(["visit", "detector"]))
252 exposure_data_id = full_data_id.subset(full_data_id.universe.extract(["exposure", "detector"]))
253 self.check_dimension_packers(
254 registry.dimensions,
255 # Note that we don't need to pass any more than the instrument in
256 # the data ID yet, because we're just constructing packers, not
257 # calling their pack method.
258 visit_packers=[
259 config.packer.apply(visit_data_id),
260 config.packer.apply(full_data_id),
261 config.packer.apply(instrument_data_id, is_exposure=False),
262 ],
263 exposure_packers=[
264 config.packer.apply(instrument_data_id, is_exposure=True),
265 config.packer.apply(exposure_data_id),
266 ],
267 n_detectors=config.packer["observation"].n_detectors,
268 n_visits=config.packer["observation"].n_observations,
269 n_exposures=config.packer["observation"].n_observations,
270 )
272 def check_dimension_packers(
273 self,
274 universe: DimensionUniverse,
275 visit_packers: list[DimensionPacker],
276 exposure_packers: list[DimensionPacker],
277 n_detectors: int,
278 n_visits: int,
279 n_exposures: int,
280 ) -> None:
281 """Test the behavior of one or more dimension packers constructed by
282 an instrument.
284 Parameters
285 ----------
286 universe : `lsst.daf.butler.DimensionUniverse`
287 Data model for butler data IDs.
288 visit_packers : `list` [ `DimensionPacker` ]
289 Packers with ``{visit, detector}`` dimensions to test.
290 exposure_packers : `list` [ `DimensionPacker` ]
291 Packers with ``{exposure, detector}`` dimensions to test.
292 n_detectors : `int`
293 Number of detectors all packers have been configured to reserve
294 space for.
295 n_visits : `int`
296 Number of visits all packers in ``visit_packers`` have been
297 configured to reserve space for.
298 n_exposures : `int`
299 Number of exposures all packers in ``exposure_packers`` have been
300 configured to reserve space for.
301 """
302 full_data_id = DataCoordinate.standardize(
303 instrument=self.name, detector=1, visit=3, exposure=7, universe=universe
304 )
305 visit_data_id = full_data_id.subset(full_data_id.universe.extract(["visit", "detector"]))
306 exposure_data_id = full_data_id.subset(full_data_id.universe.extract(["exposure", "detector"]))
307 for n, packer in enumerate(visit_packers):
308 with self.subTest(n=n):
309 packed_id, max_bits = packer.pack(full_data_id, returnMaxBits=True)
310 self.assertEqual(packed_id, full_data_id["detector"] + full_data_id["visit"] * n_detectors)
311 self.assertEqual(max_bits, math.ceil(math.log2(n_detectors * n_visits)))
312 self.assertEqual(visit_data_id, packer.unpack(packed_id))
313 with self.assertRaisesRegex(ValueError, f"Detector ID {n_detectors} is out of bounds"):
314 packer.pack(instrument=self.name, detector=n_detectors, visit=3)
315 with self.assertRaisesRegex(ValueError, f"Visit ID {n_visits} is out of bounds"):
316 packer.pack(instrument=self.name, detector=1, visit=n_visits)
317 for n, packer in enumerate(exposure_packers):
318 with self.subTest(n=n):
319 packed_id, max_bits = packer.pack(full_data_id, returnMaxBits=True)
320 self.assertEqual(packed_id, full_data_id["detector"] + full_data_id["exposure"] * n_detectors)
321 self.assertEqual(max_bits, math.ceil(math.log2(n_detectors * n_exposures)))
322 self.assertEqual(exposure_data_id, packer.unpack(packed_id))
323 with self.assertRaisesRegex(ValueError, f"Detector ID {n_detectors} is out of bounds"):
324 packer.pack(instrument=self.name, detector=n_detectors, exposure=3)
325 with self.assertRaisesRegex(ValueError, f"Exposure ID {n_exposures} is out of bounds"):
326 packer.pack(instrument=self.name, detector=1, exposure=n_exposures)
329if __name__ == "__main__":
330 unittest.main()