Coverage for tests/test_defineVisits.py: 21%
131 statements
« prev ^ index » next coverage.py v7.5.0, created at 2024-05-03 10:02 +0000
« prev ^ index » next coverage.py v7.5.0, created at 2024-05-03 10:02 +0000
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import os
23import pickle
24import shutil
25import tempfile
26import unittest
27import warnings
28from collections import defaultdict
30import lsst.daf.butler.tests as butlerTests
31from lsst.daf.butler import DataCoordinate, DimensionRecord, SerializedDimensionRecord
32from lsst.obs.base import DefineVisitsConfig, DefineVisitsTask
33from lsst.obs.base.instrument_tests import DummyCam
34from lsst.utils.iteration import ensure_iterable
36TESTDIR = os.path.dirname(__file__)
37DATADIR = os.path.join(TESTDIR, "data", "visits")
40class DefineVisitsBase:
41 """General set up that can be shared."""
43 def setUpExposures(self):
44 """Create a new butler for each test since we are changing dimension
45 records.
46 """
47 self.root = tempfile.mkdtemp(dir=TESTDIR)
48 self.creatorButler = butlerTests.makeTestRepo(self.root, [])
49 self.butler = butlerTests.makeTestCollection(self.creatorButler, uniqueId=self.id())
51 self.config = self.get_config()
52 self.task = DefineVisitsTask(config=self.config, butler=self.butler)
54 # Need to register the instrument.
55 DummyCam().register(self.butler.registry)
57 # Choose serializations based on universe.
58 universe = self.butler.dimensions
59 uversion = universe.version
60 # Not all universe changes result in visible changes.
61 match uversion:
62 case uversion if uversion < 2:
63 raise unittest.SkipTest(f"Universe {uversion} is not compatible with these test files.")
64 case 2 | 3 | 4 | 5:
65 # has_simulated, azimuth, seq_start, seq_end.
66 v = 2
67 case 6:
68 # group not group_name, group_id dropped.
69 v = 6
70 case 7:
71 # can_see_sky.
72 v = 7
73 case _:
74 # Might work.
75 warnings.warn(f"Universe {uversion} has not been validated.")
76 v = 7
78 # Read the exposure records.
79 self.records: dict[int, DimensionRecord] = {}
80 for i in (347, 348, 349):
81 with open(os.path.join(DATADIR, f"exp_v{v}_{i}.json")) as fh:
82 simple = SerializedDimensionRecord.model_validate_json(fh.read())
83 self.records[i] = DimensionRecord.from_simple(simple, registry=self.butler.registry)
85 def define_visits(
86 self,
87 exposures: list[DimensionRecord | list[DimensionRecord]],
88 incremental: bool,
89 ) -> None:
90 for records in exposures:
91 records = list(ensure_iterable(records))
92 if "group" in self.butler.dimensions["exposure"].implied:
93 # This is a group + day_obs universe.
94 for rec in records:
95 self.butler.registry.syncDimensionData(
96 "group", dict(instrument=rec.instrument, name=rec.group)
97 )
98 self.butler.registry.syncDimensionData(
99 "day_obs", dict(instrument=rec.instrument, id=rec.day_obs)
100 )
102 self.butler.registry.insertDimensionData("exposure", *records)
103 # Include all records so far in definition.
104 dataIds = list(self.butler.registry.queryDataIds("exposure", instrument="DummyCam"))
105 self.task.run(dataIds, incremental=incremental)
108class DefineVisitsTestCase(unittest.TestCase, DefineVisitsBase):
109 """Test visit definition."""
111 def setUp(self):
112 self.setUpExposures()
114 def tearDown(self):
115 if self.root is not None:
116 shutil.rmtree(self.root, ignore_errors=True)
118 def get_config(self) -> DefineVisitsConfig:
119 return DefineVisitsTask.ConfigClass()
121 def assertVisits(self):
122 """Check that the visits were registered as expected."""
123 visits = list(self.butler.registry.queryDimensionRecords("visit"))
124 self.assertEqual(len(visits), 4)
125 self.assertEqual(
126 {visit.id for visit in visits}, {2022040500347, 2022040500348, 2022040500349, 92022040500348}
127 )
129 # Ensure that the definitions are correct (ignoring order).
130 defmap = defaultdict(set)
131 definitions = list(self.butler.registry.queryDimensionRecords("visit_definition"))
132 for defn in definitions:
133 defmap[defn.visit].add(defn.exposure)
135 self.assertEqual(
136 dict(defmap),
137 {
138 92022040500348: {2022040500348},
139 2022040500347: {2022040500347},
140 2022040500348: {2022040500348, 2022040500349},
141 2022040500349: {2022040500349},
142 },
143 )
145 def test_defineVisits(self):
146 # Test visit definition with all the records.
147 self.define_visits([list(self.records.values())], incremental=False) # list inside a list
148 self.assertVisits()
150 def test_incremental_cumulative(self):
151 # Define the visits after each exposure.
152 self.define_visits(list(self.records.values()), incremental=True)
153 self.assertVisits()
155 def test_incremental_cumulative_reverse(self):
156 # In reverse order we should still eventually end up with the right
157 # answer.
158 with self.assertLogs("lsst.defineVisits.groupExposures", level="WARNING") as cm:
159 self.define_visits(list(reversed(self.records.values())), incremental=True)
160 self.assertIn("Skipping the multi-snap definition", "\n".join(cm.output))
161 self.assertVisits()
163 def define_visits_incrementally(self, exposure: DimensionRecord) -> None:
164 if "group" in self.butler.dimensions["exposure"].implied:
165 self.butler.registry.syncDimensionData(
166 "group", dict(instrument=exposure.instrument, name=exposure.group)
167 )
168 self.butler.registry.syncDimensionData(
169 "day_obs",
170 dict(
171 instrument=exposure.instrument,
172 id=exposure.day_obs,
173 ),
174 )
175 self.butler.registry.insertDimensionData("exposure", exposure)
176 dataIds = [
177 DataCoordinate.standardize(
178 instrument="DummyCam", exposure=exposure.id, universe=self.butler.dimensions
179 )
180 ]
181 self.task.run(dataIds, incremental=True)
183 def test_incremental(self):
184 for record in self.records.values():
185 self.define_visits_incrementally(record)
186 self.assertVisits()
188 def test_incremental_reverse(self):
189 for record in reversed(self.records.values()):
190 self.define_visits_incrementally(record)
191 self.assertVisits()
193 def testPickleTask(self):
194 stream = pickle.dumps(self.task)
195 copy = pickle.loads(stream)
196 self.assertEqual(self.task.getFullName(), copy.getFullName())
197 self.assertEqual(self.task.log.name, copy.log.name)
198 self.assertEqual(self.task.config, copy.config)
199 self.assertEqual(self.task.butler._config, copy.butler._config)
200 self.assertEqual(self.task.butler.collections, copy.butler.collections)
201 self.assertEqual(self.task.butler.run, copy.butler.run)
202 self.assertEqual(self.task.universe, copy.universe)
205class DefineVisitsGroupingTestCase(unittest.TestCase, DefineVisitsBase):
206 """Test visit grouping by group metadata."""
208 def setUp(self):
209 self.setUpExposures()
211 def tearDown(self):
212 if self.root is not None:
213 shutil.rmtree(self.root, ignore_errors=True)
215 def get_config(self) -> DefineVisitsConfig:
216 config = DefineVisitsTask.ConfigClass()
217 config.groupExposures.name = "by-group-metadata"
218 return config
220 def test_defineVisits(self):
221 # Test visit definition with all the records.
222 self.define_visits([list(self.records.values())], incremental=False) # list inside a list
223 self.assertVisits()
225 def assertVisits(self):
226 """Check that the visits were registered as expected."""
227 visits = list(self.butler.registry.queryDimensionRecords("visit"))
228 self.assertEqual(len(visits), 2)
230 # The visit ID itself depends on which universe we are using.
231 # It is either calculated or comes from the JSON record.
232 if "group" in self.butler.dimensions["exposure"].implied:
233 visit_ids = [20220406025653255, 20220406025807181]
234 else:
235 visit_ids = [2291434132550000, 2291434871810000]
236 self.assertEqual({visit.id for visit in visits}, set(visit_ids))
238 # Ensure that the definitions are correct (ignoring order).
239 defmap = defaultdict(set)
240 definitions = list(self.butler.registry.queryDimensionRecords("visit_definition"))
241 for defn in definitions:
242 defmap[defn.visit].add(defn.exposure)
244 self.assertEqual(
245 dict(defmap),
246 {
247 visit_ids[0]: {2022040500347},
248 visit_ids[1]: {2022040500348, 2022040500349},
249 },
250 )
253if __name__ == "__main__": 253 ↛ 254line 253 didn't jump to line 254, because the condition on line 253 was never true
254 unittest.main()