Coverage for tests/test_defineVisits.py: 23%
119 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-03-27 02:44 -0700
« prev ^ index » next coverage.py v7.4.4, created at 2024-03-27 02:44 -0700
1# This file is part of obs_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import os
23import pickle
24import shutil
25import tempfile
26import unittest
27from collections import defaultdict
29import lsst.daf.butler.tests as butlerTests
30from lsst.daf.butler import DataCoordinate, DimensionRecord, SerializedDimensionRecord
31from lsst.obs.base import DefineVisitsConfig, DefineVisitsTask
32from lsst.obs.base.instrument_tests import DummyCam
33from lsst.utils.iteration import ensure_iterable
35TESTDIR = os.path.dirname(__file__)
36DATADIR = os.path.join(TESTDIR, "data", "visits")
39class DefineVisitsBase:
40 """General set up that can be shared."""
42 def setUpExposures(self):
43 """Create a new butler for each test since we are changing dimension
44 records.
45 """
46 self.root = tempfile.mkdtemp(dir=TESTDIR)
47 self.creatorButler = butlerTests.makeTestRepo(self.root, [])
48 self.butler = butlerTests.makeTestCollection(self.creatorButler, uniqueId=self.id())
50 self.config = self.get_config()
51 self.task = DefineVisitsTask(config=self.config, butler=self.butler)
53 # Need to register the instrument.
54 DummyCam().register(self.butler.registry)
56 # Choose serializations based on universe.
57 if "group" in self.butler.dimensions["exposure"].implied:
58 v = "_v6"
59 else:
60 v = ""
62 # Read the exposure records.
63 self.records: dict[int, DimensionRecord] = {}
64 for i in (347, 348, 349):
65 with open(os.path.join(DATADIR, f"exp{v}_{i}.json")) as fh:
66 simple = SerializedDimensionRecord.model_validate_json(fh.read())
67 self.records[i] = DimensionRecord.from_simple(simple, registry=self.butler.registry)
69 def define_visits(
70 self,
71 exposures: list[DimensionRecord | list[DimensionRecord]],
72 incremental: bool,
73 ) -> None:
74 for records in exposures:
75 records = list(ensure_iterable(records))
76 if "group" in self.butler.dimensions["exposure"].implied:
77 # This is a group + day_obs universe.
78 for rec in records:
79 self.butler.registry.syncDimensionData(
80 "group", dict(instrument=rec.instrument, name=rec.group)
81 )
82 self.butler.registry.syncDimensionData(
83 "day_obs", dict(instrument=rec.instrument, id=rec.day_obs)
84 )
86 self.butler.registry.insertDimensionData("exposure", *records)
87 # Include all records so far in definition.
88 dataIds = list(self.butler.registry.queryDataIds("exposure", instrument="DummyCam"))
89 self.task.run(dataIds, incremental=incremental)
92class DefineVisitsTestCase(unittest.TestCase, DefineVisitsBase):
93 """Test visit definition."""
95 def setUp(self):
96 self.setUpExposures()
98 def tearDown(self):
99 if self.root is not None:
100 shutil.rmtree(self.root, ignore_errors=True)
102 def get_config(self) -> DefineVisitsConfig:
103 return DefineVisitsTask.ConfigClass()
105 def assertVisits(self):
106 """Check that the visits were registered as expected."""
107 visits = list(self.butler.registry.queryDimensionRecords("visit"))
108 self.assertEqual(len(visits), 4)
109 self.assertEqual(
110 {visit.id for visit in visits}, {2022040500347, 2022040500348, 2022040500349, 92022040500348}
111 )
113 # Ensure that the definitions are correct (ignoring order).
114 defmap = defaultdict(set)
115 definitions = list(self.butler.registry.queryDimensionRecords("visit_definition"))
116 for defn in definitions:
117 defmap[defn.visit].add(defn.exposure)
119 self.assertEqual(
120 dict(defmap),
121 {
122 92022040500348: {2022040500348},
123 2022040500347: {2022040500347},
124 2022040500348: {2022040500348, 2022040500349},
125 2022040500349: {2022040500349},
126 },
127 )
129 def test_defineVisits(self):
130 # Test visit definition with all the records.
131 self.define_visits([list(self.records.values())], incremental=False) # list inside a list
132 self.assertVisits()
134 def test_incremental_cumulative(self):
135 # Define the visits after each exposure.
136 self.define_visits(list(self.records.values()), incremental=True)
137 self.assertVisits()
139 def test_incremental_cumulative_reverse(self):
140 # In reverse order we should still eventually end up with the right
141 # answer.
142 with self.assertLogs("lsst.defineVisits.groupExposures", level="WARNING") as cm:
143 self.define_visits(list(reversed(self.records.values())), incremental=True)
144 self.assertIn("Skipping the multi-snap definition", "\n".join(cm.output))
145 self.assertVisits()
147 def define_visits_incrementally(self, exposure: DimensionRecord) -> None:
148 if "group" in self.butler.dimensions["exposure"].implied:
149 self.butler.registry.syncDimensionData(
150 "group", dict(instrument=exposure.instrument, name=exposure.group)
151 )
152 self.butler.registry.syncDimensionData(
153 "day_obs",
154 dict(
155 instrument=exposure.instrument,
156 id=exposure.day_obs,
157 ),
158 )
159 self.butler.registry.insertDimensionData("exposure", exposure)
160 dataIds = [
161 DataCoordinate.standardize(
162 instrument="DummyCam", exposure=exposure.id, universe=self.butler.dimensions
163 )
164 ]
165 self.task.run(dataIds, incremental=True)
167 def test_incremental(self):
168 for record in self.records.values():
169 self.define_visits_incrementally(record)
170 self.assertVisits()
172 def test_incremental_reverse(self):
173 for record in reversed(self.records.values()):
174 self.define_visits_incrementally(record)
175 self.assertVisits()
177 def testPickleTask(self):
178 stream = pickle.dumps(self.task)
179 copy = pickle.loads(stream)
180 self.assertEqual(self.task.getFullName(), copy.getFullName())
181 self.assertEqual(self.task.log.name, copy.log.name)
182 self.assertEqual(self.task.config, copy.config)
183 self.assertEqual(self.task.butler._config, copy.butler._config)
184 self.assertEqual(self.task.butler.collections, copy.butler.collections)
185 self.assertEqual(self.task.butler.run, copy.butler.run)
186 self.assertEqual(self.task.universe, copy.universe)
189class DefineVisitsGroupingTestCase(unittest.TestCase, DefineVisitsBase):
190 """Test visit grouping by group metadata."""
192 def setUp(self):
193 self.setUpExposures()
195 def tearDown(self):
196 if self.root is not None:
197 shutil.rmtree(self.root, ignore_errors=True)
199 def get_config(self) -> DefineVisitsConfig:
200 config = DefineVisitsTask.ConfigClass()
201 config.groupExposures.name = "by-group-metadata"
202 return config
204 def test_defineVisits(self):
205 # Test visit definition with all the records.
206 self.define_visits([list(self.records.values())], incremental=False) # list inside a list
207 self.assertVisits()
209 def assertVisits(self):
210 """Check that the visits were registered as expected."""
211 visits = list(self.butler.registry.queryDimensionRecords("visit"))
212 self.assertEqual(len(visits), 2)
214 # The visit ID itself depends on which universe we are using.
215 # It is either calculated or comes from the JSON record.
216 if "group" in self.butler.dimensions["exposure"].implied:
217 visit_ids = [20220406025653255, 20220406025807181]
218 else:
219 visit_ids = [2291434132550000, 2291434871810000]
220 self.assertEqual({visit.id for visit in visits}, set(visit_ids))
222 # Ensure that the definitions are correct (ignoring order).
223 defmap = defaultdict(set)
224 definitions = list(self.butler.registry.queryDimensionRecords("visit_definition"))
225 for defn in definitions:
226 defmap[defn.visit].add(defn.exposure)
228 self.assertEqual(
229 dict(defmap),
230 {
231 visit_ids[0]: {2022040500347},
232 visit_ids[1]: {2022040500348, 2022040500349},
233 },
234 )
237if __name__ == "__main__": 237 ↛ 238line 237 didn't jump to line 238, because the condition on line 237 was never true
238 unittest.main()