Coverage for tests/test_defineVisits.py: 21%

131 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-04 10:09 +0000

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import os 

23import pickle 

24import shutil 

25import tempfile 

26import unittest 

27import warnings 

28from collections import defaultdict 

29 

30import lsst.daf.butler.tests as butlerTests 

31from lsst.daf.butler import DataCoordinate, DimensionRecord, SerializedDimensionRecord 

32from lsst.obs.base import DefineVisitsConfig, DefineVisitsTask 

33from lsst.obs.base.instrument_tests import DummyCam 

34from lsst.utils.iteration import ensure_iterable 

35 

36TESTDIR = os.path.dirname(__file__) 

37DATADIR = os.path.join(TESTDIR, "data", "visits") 

38 

39 

40class DefineVisitsBase: 

41 """General set up that can be shared.""" 

42 

43 def setUpExposures(self): 

44 """Create a new butler for each test since we are changing dimension 

45 records. 

46 """ 

47 self.root = tempfile.mkdtemp(dir=TESTDIR) 

48 self.creatorButler = butlerTests.makeTestRepo(self.root, []) 

49 self.butler = butlerTests.makeTestCollection(self.creatorButler, uniqueId=self.id()) 

50 

51 self.config = self.get_config() 

52 self.task = DefineVisitsTask(config=self.config, butler=self.butler) 

53 

54 # Need to register the instrument. 

55 DummyCam().register(self.butler.registry) 

56 

57 # Choose serializations based on universe. 

58 universe = self.butler.dimensions 

59 uversion = universe.version 

60 # Not all universe changes result in visible changes. 

61 match uversion: 

62 case uversion if uversion < 2: 

63 raise unittest.SkipTest(f"Universe {uversion} is not compatible with these test files.") 

64 case 2 | 3 | 4 | 5: 

65 # has_simulated, azimuth, seq_start, seq_end. 

66 v = 2 

67 case 6: 

68 # group not group_name, group_id dropped. 

69 v = 6 

70 case 7: 

71 # can_see_sky. 

72 v = 7 

73 case _: 

74 # Might work. 

75 warnings.warn(f"Universe {uversion} has not been validated.") 

76 v = 7 

77 

78 # Read the exposure records. 

79 self.records: dict[int, DimensionRecord] = {} 

80 for i in (347, 348, 349): 

81 with open(os.path.join(DATADIR, f"exp_v{v}_{i}.json")) as fh: 

82 simple = SerializedDimensionRecord.model_validate_json(fh.read()) 

83 self.records[i] = DimensionRecord.from_simple(simple, registry=self.butler.registry) 

84 

85 def define_visits( 

86 self, 

87 exposures: list[DimensionRecord | list[DimensionRecord]], 

88 incremental: bool, 

89 ) -> None: 

90 for records in exposures: 

91 records = list(ensure_iterable(records)) 

92 if "group" in self.butler.dimensions["exposure"].implied: 

93 # This is a group + day_obs universe. 

94 for rec in records: 

95 self.butler.registry.syncDimensionData( 

96 "group", dict(instrument=rec.instrument, name=rec.group) 

97 ) 

98 self.butler.registry.syncDimensionData( 

99 "day_obs", dict(instrument=rec.instrument, id=rec.day_obs) 

100 ) 

101 

102 self.butler.registry.insertDimensionData("exposure", *records) 

103 # Include all records so far in definition. 

104 dataIds = list(self.butler.registry.queryDataIds("exposure", instrument="DummyCam")) 

105 self.task.run(dataIds, incremental=incremental) 

106 

107 

108class DefineVisitsTestCase(unittest.TestCase, DefineVisitsBase): 

109 """Test visit definition.""" 

110 

111 def setUp(self): 

112 self.setUpExposures() 

113 

114 def tearDown(self): 

115 if self.root is not None: 

116 shutil.rmtree(self.root, ignore_errors=True) 

117 

118 def get_config(self) -> DefineVisitsConfig: 

119 return DefineVisitsTask.ConfigClass() 

120 

121 def assertVisits(self): 

122 """Check that the visits were registered as expected.""" 

123 visits = list(self.butler.registry.queryDimensionRecords("visit")) 

124 self.assertEqual(len(visits), 4) 

125 self.assertEqual( 

126 {visit.id for visit in visits}, {2022040500347, 2022040500348, 2022040500349, 92022040500348} 

127 ) 

128 

129 # Ensure that the definitions are correct (ignoring order). 

130 defmap = defaultdict(set) 

131 definitions = list(self.butler.registry.queryDimensionRecords("visit_definition")) 

132 for defn in definitions: 

133 defmap[defn.visit].add(defn.exposure) 

134 

135 self.assertEqual( 

136 dict(defmap), 

137 { 

138 92022040500348: {2022040500348}, 

139 2022040500347: {2022040500347}, 

140 2022040500348: {2022040500348, 2022040500349}, 

141 2022040500349: {2022040500349}, 

142 }, 

143 ) 

144 

145 def test_defineVisits(self): 

146 # Test visit definition with all the records. 

147 self.define_visits([list(self.records.values())], incremental=False) # list inside a list 

148 self.assertVisits() 

149 

150 def test_incremental_cumulative(self): 

151 # Define the visits after each exposure. 

152 self.define_visits(list(self.records.values()), incremental=True) 

153 self.assertVisits() 

154 

155 def test_incremental_cumulative_reverse(self): 

156 # In reverse order we should still eventually end up with the right 

157 # answer. 

158 with self.assertLogs("lsst.defineVisits.groupExposures", level="WARNING") as cm: 

159 self.define_visits(list(reversed(self.records.values())), incremental=True) 

160 self.assertIn("Skipping the multi-snap definition", "\n".join(cm.output)) 

161 self.assertVisits() 

162 

163 def define_visits_incrementally(self, exposure: DimensionRecord) -> None: 

164 if "group" in self.butler.dimensions["exposure"].implied: 

165 self.butler.registry.syncDimensionData( 

166 "group", dict(instrument=exposure.instrument, name=exposure.group) 

167 ) 

168 self.butler.registry.syncDimensionData( 

169 "day_obs", 

170 dict( 

171 instrument=exposure.instrument, 

172 id=exposure.day_obs, 

173 ), 

174 ) 

175 self.butler.registry.insertDimensionData("exposure", exposure) 

176 dataIds = [ 

177 DataCoordinate.standardize( 

178 instrument="DummyCam", exposure=exposure.id, universe=self.butler.dimensions 

179 ) 

180 ] 

181 self.task.run(dataIds, incremental=True) 

182 

183 def test_incremental(self): 

184 for record in self.records.values(): 

185 self.define_visits_incrementally(record) 

186 self.assertVisits() 

187 

188 def test_incremental_reverse(self): 

189 for record in reversed(self.records.values()): 

190 self.define_visits_incrementally(record) 

191 self.assertVisits() 

192 

193 def testPickleTask(self): 

194 stream = pickle.dumps(self.task) 

195 copy = pickle.loads(stream) 

196 self.assertEqual(self.task.getFullName(), copy.getFullName()) 

197 self.assertEqual(self.task.log.name, copy.log.name) 

198 self.assertEqual(self.task.config, copy.config) 

199 self.assertEqual(self.task.butler._config, copy.butler._config) 

200 self.assertEqual(self.task.butler.collections, copy.butler.collections) 

201 self.assertEqual(self.task.butler.run, copy.butler.run) 

202 self.assertEqual(self.task.universe, copy.universe) 

203 

204 

205class DefineVisitsGroupingTestCase(unittest.TestCase, DefineVisitsBase): 

206 """Test visit grouping by group metadata.""" 

207 

208 def setUp(self): 

209 self.setUpExposures() 

210 

211 def tearDown(self): 

212 if self.root is not None: 

213 shutil.rmtree(self.root, ignore_errors=True) 

214 

215 def get_config(self) -> DefineVisitsConfig: 

216 config = DefineVisitsTask.ConfigClass() 

217 config.groupExposures.name = "by-group-metadata" 

218 return config 

219 

220 def test_defineVisits(self): 

221 # Test visit definition with all the records. 

222 self.define_visits([list(self.records.values())], incremental=False) # list inside a list 

223 self.assertVisits() 

224 

225 def assertVisits(self): 

226 """Check that the visits were registered as expected.""" 

227 visits = list(self.butler.registry.queryDimensionRecords("visit")) 

228 self.assertEqual(len(visits), 2) 

229 

230 # The visit ID itself depends on which universe we are using. 

231 # It is either calculated or comes from the JSON record. 

232 if "group" in self.butler.dimensions["exposure"].implied: 

233 visit_ids = [20220406025653255, 20220406025807181] 

234 else: 

235 visit_ids = [2291434132550000, 2291434871810000] 

236 self.assertEqual({visit.id for visit in visits}, set(visit_ids)) 

237 

238 # Ensure that the definitions are correct (ignoring order). 

239 defmap = defaultdict(set) 

240 definitions = list(self.butler.registry.queryDimensionRecords("visit_definition")) 

241 for defn in definitions: 

242 defmap[defn.visit].add(defn.exposure) 

243 

244 self.assertEqual( 

245 dict(defmap), 

246 { 

247 visit_ids[0]: {2022040500347}, 

248 visit_ids[1]: {2022040500348, 2022040500349}, 

249 }, 

250 ) 

251 

252 

253if __name__ == "__main__": 253 ↛ 254line 253 didn't jump to line 254, because the condition on line 253 was never true

254 unittest.main()