Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import unittest 

23import copy 

24import pickle 

25import itertools 

26 

27from lsst.daf.butler import ( 

28 Dimension, 

29 DimensionGraph, 

30 DimensionUniverse, 

31 makeDimensionElementTableSpec, 

32 NamedKeyDict, 

33 NamedValueSet, 

34) 

35 

36 

37class DimensionTestCase(unittest.TestCase): 

38 """Tests for dimensions. 

39 

40 All tests here rely on the content of ``config/dimensions.yaml``, either 

41 to test that the definitions there are read in properly or just as generic 

42 data for testing various operations. 

43 """ 

44 

45 def setUp(self): 

46 self.universe = DimensionUniverse() 

47 

48 def checkGraphInvariants(self, graph): 

49 elements = list(graph.elements) 

50 for n, element in enumerate(elements): 

51 # Ordered comparisons on graphs behave like sets. 

52 self.assertLessEqual(element.graph, graph) 

53 # Ordered comparisons on elements correspond to the ordering within 

54 # a DimensionUniverse (topological, with deterministic 

55 # tiebreakers). 

56 for other in elements[:n]: 

57 self.assertLess(other, element) 

58 self.assertLessEqual(other, element) 

59 for other in elements[n + 1:]: 

60 self.assertGreater(other, element) 

61 self.assertGreaterEqual(other, element) 

62 if isinstance(element, Dimension): 

63 self.assertEqual(element.graph.required, element.required) 

64 self.assertEqual(DimensionGraph(self.universe, graph.required), graph) 

65 self.assertCountEqual(graph.required, 

66 [dimension for dimension in graph.dimensions 

67 if not any(dimension in other.graph.implied for other in graph.elements)]) 

68 self.assertCountEqual(graph.implied, graph.dimensions - graph.required) 

69 self.assertCountEqual(graph.dimensions, 

70 [element for element in graph.elements 

71 if isinstance(element, Dimension)]) 

72 self.assertCountEqual(graph.dimensions, itertools.chain(graph.required, graph.implied)) 

73 # Check primary key traversal order: each element should follow any it 

74 # requires, and element that is implied by any other in the graph 

75 # follow at least one of those. 

76 seen = NamedValueSet() 

77 for element in graph.primaryKeyTraversalOrder: 

78 with self.subTest(required=graph.required, implied=graph.implied, element=element): 

79 seen.add(element) 

80 self.assertLessEqual(element.graph.required, seen) 

81 if element in graph.implied: 

82 self.assertTrue(any(element in s.implied for s in seen)) 

83 self.assertCountEqual(seen, graph.elements) 

84 # Test encoding and decoding of DimensionGraphs to bytes. 

85 encoded = graph.encode() 

86 self.assertEqual(len(encoded), self.universe.getEncodeLength()) 

87 self.assertEqual(DimensionGraph.decode(encoded, universe=self.universe), graph) 

88 

89 def testConfigRead(self): 

90 self.assertEqual(self.universe.dimensions.names, 

91 {"instrument", "visit", "visit_system", "exposure", "detector", 

92 "physical_filter", "abstract_filter", "subfilter", "calibration_label", 

93 "skymap", "tract", "patch", "htm7", "htm9"}) 

94 

95 def testGraphs(self): 

96 self.checkGraphInvariants(self.universe.empty) 

97 self.checkGraphInvariants(self.universe) 

98 for element in self.universe.elements: 

99 self.checkGraphInvariants(element.graph) 

100 

101 def testInstrumentDimensions(self): 

102 graph = DimensionGraph(self.universe, names=("exposure", "detector", "visit", "calibration_label")) 

103 self.assertCountEqual(graph.dimensions.names, 

104 ("instrument", "exposure", "detector", "calibration_label", 

105 "visit", "physical_filter", "abstract_filter", "visit_system")) 

106 self.assertCountEqual(graph.required.names, ("instrument", "exposure", "detector", 

107 "calibration_label", "visit")) 

108 self.assertCountEqual(graph.implied.names, ("physical_filter", "abstract_filter", "visit_system")) 

109 self.assertCountEqual(graph.elements.names - graph.dimensions.names, 

110 ("visit_detector_region", "visit_definition")) 

111 

112 def testCalibrationDimensions(self): 

113 graph = DimensionGraph(self.universe, names=("calibration_label", "physical_filter", "detector")) 

114 self.assertCountEqual(graph.dimensions.names, 

115 ("instrument", "detector", "calibration_label", 

116 "physical_filter", "abstract_filter")) 

117 self.assertCountEqual(graph.required.names, ("instrument", "detector", "calibration_label", 

118 "physical_filter")) 

119 self.assertCountEqual(graph.implied.names, ("abstract_filter",)) 

120 self.assertCountEqual(graph.elements.names, graph.dimensions.names) 

121 

122 def testObservationDimensions(self): 

123 graph = DimensionGraph(self.universe, names=("exposure", "detector", "visit")) 

124 self.assertCountEqual(graph.dimensions.names, ("instrument", "detector", "visit", "exposure", 

125 "physical_filter", "abstract_filter", "visit_system")) 

126 self.assertCountEqual(graph.required.names, ("instrument", "detector", "exposure", "visit")) 

127 self.assertCountEqual(graph.implied.names, ("physical_filter", "abstract_filter", "visit_system")) 

128 self.assertCountEqual(graph.elements.names - graph.dimensions.names, 

129 ("visit_detector_region", "visit_definition")) 

130 self.assertCountEqual(graph.spatial.names, ("visit_detector_region",)) 

131 self.assertCountEqual(graph.temporal.names, ("exposure",)) 

132 

133 def testSkyMapDimensions(self): 

134 graph = DimensionGraph(self.universe, names=("patch",)) 

135 self.assertCountEqual(graph.dimensions.names, ("skymap", "tract", "patch")) 

136 self.assertCountEqual(graph.required.names, ("skymap", "tract", "patch")) 

137 self.assertCountEqual(graph.implied.names, ()) 

138 self.assertCountEqual(graph.elements.names, graph.dimensions.names) 

139 self.assertCountEqual(graph.spatial.names, ("patch",)) 

140 

141 def testSubsetCalculation(self): 

142 """Test that independent spatial and temporal options are computed 

143 correctly. 

144 """ 

145 graph = DimensionGraph(self.universe, names=("visit", "detector", "tract", "patch", "htm7", 

146 "exposure", "calibration_label")) 

147 self.assertCountEqual(graph.spatial.names, 

148 ("visit_detector_region", "patch", "htm7")) 

149 self.assertCountEqual(graph.temporal.names, 

150 ("exposure", "calibration_label")) 

151 

152 def testSchemaGeneration(self): 

153 tableSpecs = NamedKeyDict({}) 

154 for element in self.universe.elements: 

155 if element.hasTable and element.viewOf is None: 

156 tableSpecs[element] = makeDimensionElementTableSpec(element) 

157 for element, tableSpec in tableSpecs.items(): 

158 for dep in element.required: 

159 with self.subTest(element=element.name, dep=dep.name): 

160 if dep != element: 

161 self.assertIn(dep.name, tableSpec.fields) 

162 self.assertEqual(tableSpec.fields[dep.name].dtype, dep.primaryKey.dtype) 

163 self.assertEqual(tableSpec.fields[dep.name].length, dep.primaryKey.length) 

164 self.assertEqual(tableSpec.fields[dep.name].nbytes, dep.primaryKey.nbytes) 

165 self.assertFalse(tableSpec.fields[dep.name].nullable) 

166 self.assertTrue(tableSpec.fields[dep.name].primaryKey) 

167 else: 

168 self.assertIn(element.primaryKey.name, tableSpec.fields) 

169 self.assertEqual(tableSpec.fields[element.primaryKey.name].dtype, 

170 dep.primaryKey.dtype) 

171 self.assertEqual(tableSpec.fields[element.primaryKey.name].length, 

172 dep.primaryKey.length) 

173 self.assertEqual(tableSpec.fields[element.primaryKey.name].nbytes, 

174 dep.primaryKey.nbytes) 

175 self.assertFalse(tableSpec.fields[element.primaryKey.name].nullable) 

176 self.assertTrue(tableSpec.fields[element.primaryKey.name].primaryKey) 

177 for dep in element.implied: 

178 with self.subTest(element=element.name, dep=dep.name): 

179 self.assertIn(dep.name, tableSpec.fields) 

180 self.assertEqual(tableSpec.fields[dep.name].dtype, dep.primaryKey.dtype) 

181 self.assertFalse(tableSpec.fields[dep.name].primaryKey) 

182 for foreignKey in tableSpec.foreignKeys: 

183 self.assertIn(foreignKey.table, tableSpecs) 

184 self.assertIn(foreignKey.table, element.graph.dimensions.names) 

185 self.assertEqual(len(foreignKey.source), len(foreignKey.target)) 

186 for source, target in zip(foreignKey.source, foreignKey.target): 

187 self.assertIn(source, tableSpec.fields.names) 

188 self.assertIn(target, tableSpecs[foreignKey.table].fields.names) 

189 self.assertEqual(tableSpec.fields[source].dtype, 

190 tableSpecs[foreignKey.table].fields[target].dtype) 

191 self.assertEqual(tableSpec.fields[source].length, 

192 tableSpecs[foreignKey.table].fields[target].length) 

193 self.assertEqual(tableSpec.fields[source].nbytes, 

194 tableSpecs[foreignKey.table].fields[target].nbytes) 

195 self.assertEqual(tuple(tableSpec.fields.names), element.RecordClass.__slots__) 

196 

197 def testPickling(self): 

198 # Pickling and copying should always yield the exact same object within 

199 # a single process (cross-process is impossible to test here). 

200 universe1 = DimensionUniverse() 

201 universe2 = pickle.loads(pickle.dumps(universe1)) 

202 universe3 = copy.copy(universe1) 

203 universe4 = copy.deepcopy(universe1) 

204 self.assertIs(universe1, universe2) 

205 self.assertIs(universe1, universe3) 

206 self.assertIs(universe1, universe4) 

207 for element1 in universe1.elements: 

208 element2 = pickle.loads(pickle.dumps(element1)) 

209 self.assertIs(element1, element2) 

210 graph1 = element1.graph 

211 graph2 = pickle.loads(pickle.dumps(graph1)) 

212 self.assertIs(graph1, graph2) 

213 

214 

215if __name__ == "__main__": 215 ↛ 216line 215 didn't jump to line 216, because the condition on line 215 was never true

216 unittest.main()