Coverage for tests/test_defineVisits.py: 23%

119 statements  

« prev     ^ index     » next       coverage.py v7.4.3, created at 2024-03-05 12:09 +0000

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import os 

23import pickle 

24import shutil 

25import tempfile 

26import unittest 

27from collections import defaultdict 

28 

29import lsst.daf.butler.tests as butlerTests 

30from lsst.daf.butler import DataCoordinate, DimensionRecord, SerializedDimensionRecord 

31from lsst.obs.base import DefineVisitsConfig, DefineVisitsTask 

32from lsst.obs.base.instrument_tests import DummyCam 

33from lsst.utils.iteration import ensure_iterable 

34 

35TESTDIR = os.path.dirname(__file__) 

36DATADIR = os.path.join(TESTDIR, "data", "visits") 

37 

38 

39class DefineVisitsBase: 

40 """General set up that can be shared.""" 

41 

42 def setUpExposures(self): 

43 """Create a new butler for each test since we are changing dimension 

44 records. 

45 """ 

46 self.root = tempfile.mkdtemp(dir=TESTDIR) 

47 self.creatorButler = butlerTests.makeTestRepo(self.root, []) 

48 self.butler = butlerTests.makeTestCollection(self.creatorButler, uniqueId=self.id()) 

49 

50 self.config = self.get_config() 

51 self.task = DefineVisitsTask(config=self.config, butler=self.butler) 

52 

53 # Need to register the instrument. 

54 DummyCam().register(self.butler.registry) 

55 

56 # Choose serializations based on universe. 

57 if "group" in self.butler.dimensions["exposure"].implied: 

58 v = "_v6" 

59 else: 

60 v = "" 

61 

62 # Read the exposure records. 

63 self.records: dict[int, DimensionRecord] = {} 

64 for i in (347, 348, 349): 

65 with open(os.path.join(DATADIR, f"exp{v}_{i}.json")) as fh: 

66 simple = SerializedDimensionRecord.model_validate_json(fh.read()) 

67 self.records[i] = DimensionRecord.from_simple(simple, registry=self.butler.registry) 

68 

69 def define_visits( 

70 self, 

71 exposures: list[DimensionRecord | list[DimensionRecord]], 

72 incremental: bool, 

73 ) -> None: 

74 for records in exposures: 

75 records = list(ensure_iterable(records)) 

76 if "group" in self.butler.dimensions["exposure"].implied: 

77 # This is a group + day_obs universe. 

78 for rec in records: 

79 self.butler.registry.syncDimensionData( 

80 "group", dict(instrument=rec.instrument, name=rec.group) 

81 ) 

82 self.butler.registry.syncDimensionData( 

83 "day_obs", dict(instrument=rec.instrument, id=rec.day_obs) 

84 ) 

85 

86 self.butler.registry.insertDimensionData("exposure", *records) 

87 # Include all records so far in definition. 

88 dataIds = list(self.butler.registry.queryDataIds("exposure", instrument="DummyCam")) 

89 self.task.run(dataIds, incremental=incremental) 

90 

91 

92class DefineVisitsTestCase(unittest.TestCase, DefineVisitsBase): 

93 """Test visit definition.""" 

94 

95 def setUp(self): 

96 self.setUpExposures() 

97 

98 def tearDown(self): 

99 if self.root is not None: 

100 shutil.rmtree(self.root, ignore_errors=True) 

101 

102 def get_config(self) -> DefineVisitsConfig: 

103 return DefineVisitsTask.ConfigClass() 

104 

105 def assertVisits(self): 

106 """Check that the visits were registered as expected.""" 

107 visits = list(self.butler.registry.queryDimensionRecords("visit")) 

108 self.assertEqual(len(visits), 4) 

109 self.assertEqual( 

110 {visit.id for visit in visits}, {2022040500347, 2022040500348, 2022040500349, 92022040500348} 

111 ) 

112 

113 # Ensure that the definitions are correct (ignoring order). 

114 defmap = defaultdict(set) 

115 definitions = list(self.butler.registry.queryDimensionRecords("visit_definition")) 

116 for defn in definitions: 

117 defmap[defn.visit].add(defn.exposure) 

118 

119 self.assertEqual( 

120 dict(defmap), 

121 { 

122 92022040500348: {2022040500348}, 

123 2022040500347: {2022040500347}, 

124 2022040500348: {2022040500348, 2022040500349}, 

125 2022040500349: {2022040500349}, 

126 }, 

127 ) 

128 

129 def test_defineVisits(self): 

130 # Test visit definition with all the records. 

131 self.define_visits([list(self.records.values())], incremental=False) # list inside a list 

132 self.assertVisits() 

133 

134 def test_incremental_cumulative(self): 

135 # Define the visits after each exposure. 

136 self.define_visits(list(self.records.values()), incremental=True) 

137 self.assertVisits() 

138 

139 def test_incremental_cumulative_reverse(self): 

140 # In reverse order we should still eventually end up with the right 

141 # answer. 

142 with self.assertLogs("lsst.defineVisits.groupExposures", level="WARNING") as cm: 

143 self.define_visits(list(reversed(self.records.values())), incremental=True) 

144 self.assertIn("Skipping the multi-snap definition", "\n".join(cm.output)) 

145 self.assertVisits() 

146 

147 def define_visits_incrementally(self, exposure: DimensionRecord) -> None: 

148 if "group" in self.butler.dimensions["exposure"].implied: 

149 self.butler.registry.syncDimensionData( 

150 "group", dict(instrument=exposure.instrument, name=exposure.group) 

151 ) 

152 self.butler.registry.syncDimensionData( 

153 "day_obs", 

154 dict( 

155 instrument=exposure.instrument, 

156 id=exposure.day_obs, 

157 ), 

158 ) 

159 self.butler.registry.insertDimensionData("exposure", exposure) 

160 dataIds = [ 

161 DataCoordinate.standardize( 

162 instrument="DummyCam", exposure=exposure.id, universe=self.butler.dimensions 

163 ) 

164 ] 

165 self.task.run(dataIds, incremental=True) 

166 

167 def test_incremental(self): 

168 for record in self.records.values(): 

169 self.define_visits_incrementally(record) 

170 self.assertVisits() 

171 

172 def test_incremental_reverse(self): 

173 for record in reversed(self.records.values()): 

174 self.define_visits_incrementally(record) 

175 self.assertVisits() 

176 

177 def testPickleTask(self): 

178 stream = pickle.dumps(self.task) 

179 copy = pickle.loads(stream) 

180 self.assertEqual(self.task.getFullName(), copy.getFullName()) 

181 self.assertEqual(self.task.log.name, copy.log.name) 

182 self.assertEqual(self.task.config, copy.config) 

183 self.assertEqual(self.task.butler._config, copy.butler._config) 

184 self.assertEqual(self.task.butler.collections, copy.butler.collections) 

185 self.assertEqual(self.task.butler.run, copy.butler.run) 

186 self.assertEqual(self.task.universe, copy.universe) 

187 

188 

189class DefineVisitsGroupingTestCase(unittest.TestCase, DefineVisitsBase): 

190 """Test visit grouping by group metadata.""" 

191 

192 def setUp(self): 

193 self.setUpExposures() 

194 

195 def tearDown(self): 

196 if self.root is not None: 

197 shutil.rmtree(self.root, ignore_errors=True) 

198 

199 def get_config(self) -> DefineVisitsConfig: 

200 config = DefineVisitsTask.ConfigClass() 

201 config.groupExposures.name = "by-group-metadata" 

202 return config 

203 

204 def test_defineVisits(self): 

205 # Test visit definition with all the records. 

206 self.define_visits([list(self.records.values())], incremental=False) # list inside a list 

207 self.assertVisits() 

208 

209 def assertVisits(self): 

210 """Check that the visits were registered as expected.""" 

211 visits = list(self.butler.registry.queryDimensionRecords("visit")) 

212 self.assertEqual(len(visits), 2) 

213 

214 # The visit ID itself depends on which universe we are using. 

215 # It is either calculated or comes from the JSON record. 

216 if "group" in self.butler.dimensions["exposure"].implied: 

217 visit_ids = [20220406025653255, 20220406025807181] 

218 else: 

219 visit_ids = [2291434132550000, 2291434871810000] 

220 self.assertEqual({visit.id for visit in visits}, set(visit_ids)) 

221 

222 # Ensure that the definitions are correct (ignoring order). 

223 defmap = defaultdict(set) 

224 definitions = list(self.butler.registry.queryDimensionRecords("visit_definition")) 

225 for defn in definitions: 

226 defmap[defn.visit].add(defn.exposure) 

227 

228 self.assertEqual( 

229 dict(defmap), 

230 { 

231 visit_ids[0]: {2022040500347}, 

232 visit_ids[1]: {2022040500348, 2022040500349}, 

233 }, 

234 ) 

235 

236 

237if __name__ == "__main__": 237 ↛ 238line 237 didn't jump to line 238, because the condition on line 237 was never true

238 unittest.main()