Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import unittest 

23import copy 

24import pickle 

25import itertools 

26 

27from lsst.daf.butler.core.utils import NamedKeyDict, NamedValueSet 

28from lsst.daf.butler import Dimension, DimensionGraph, DimensionUniverse, makeDimensionElementTableSpec 

29 

30 

31class DimensionTestCase(unittest.TestCase): 

32 """Tests for dimensions. 

33 

34 All tests here rely on the content of ``config/dimensions.yaml``, either 

35 to test that the definitions there are read in properly or just as generic 

36 data for testing various operations. 

37 """ 

38 

39 def setUp(self): 

40 self.universe = DimensionUniverse() 

41 

42 def checkGraphInvariants(self, graph): 

43 elements = list(graph.elements) 

44 for n, element in enumerate(elements): 

45 # Ordered comparisons on graphs behave like sets. 

46 self.assertLessEqual(element.graph, graph) 

47 # Ordered comparisons on elements correspond to the ordering within 

48 # a DimensionUniverse (topological, with deterministic 

49 # tiebreakers). 

50 for other in elements[:n]: 

51 self.assertLess(other, element) 

52 self.assertLessEqual(other, element) 

53 for other in elements[n + 1:]: 

54 self.assertGreater(other, element) 

55 self.assertGreaterEqual(other, element) 

56 if isinstance(element, Dimension): 

57 self.assertEqual(element.graph.required, element.required) 

58 self.assertEqual(DimensionGraph(self.universe, graph.required), graph) 

59 self.assertCountEqual(graph.required, 

60 [dimension for dimension in graph.dimensions 

61 if not any(dimension in other.graph.implied for other in graph.elements)]) 

62 self.assertCountEqual(graph.implied, graph.dimensions - graph.required) 

63 self.assertCountEqual(graph.dimensions, 

64 [element for element in graph.elements 

65 if isinstance(element, Dimension)]) 

66 self.assertCountEqual(graph.dimensions, itertools.chain(graph.required, graph.implied)) 

67 # Check primary key traversal order: each element should follow any it 

68 # requires, and element that is implied by any other in the graph 

69 # follow at least one of those. 

70 seen = NamedValueSet() 

71 for element in graph.primaryKeyTraversalOrder: 

72 with self.subTest(required=graph.required, implied=graph.implied, element=element): 

73 seen.add(element) 

74 self.assertLessEqual(element.graph.required, seen) 

75 if element in graph.implied: 

76 self.assertTrue(any(element in s.implied for s in seen)) 

77 self.assertCountEqual(seen, graph.elements) 

78 # Test encoding and decoding of DimensionGraphs to bytes. 

79 encoded = graph.encode() 

80 self.assertEqual(len(encoded), self.universe.getEncodeLength()) 

81 self.assertEqual(DimensionGraph.decode(encoded, universe=self.universe), graph) 

82 

83 def testConfigRead(self): 

84 self.assertEqual(self.universe.dimensions.names, 

85 {"instrument", "visit", "visit_system", "exposure", "detector", 

86 "physical_filter", "abstract_filter", "subfilter", "calibration_label", 

87 "skymap", "tract", "patch", "htm7", "htm9"}) 

88 

89 def testGraphs(self): 

90 self.checkGraphInvariants(self.universe.empty) 

91 self.checkGraphInvariants(self.universe) 

92 for element in self.universe.elements: 

93 self.checkGraphInvariants(element.graph) 

94 

95 def testInstrumentDimensions(self): 

96 graph = DimensionGraph(self.universe, names=("exposure", "detector", "visit", "calibration_label")) 

97 self.assertCountEqual(graph.dimensions.names, 

98 ("instrument", "exposure", "detector", "calibration_label", 

99 "visit", "physical_filter", "abstract_filter", "visit_system")) 

100 self.assertCountEqual(graph.required.names, ("instrument", "exposure", "detector", 

101 "calibration_label", "visit")) 

102 self.assertCountEqual(graph.implied.names, ("physical_filter", "abstract_filter", "visit_system")) 

103 self.assertCountEqual(graph.elements.names - graph.dimensions.names, 

104 ("visit_detector_region", "visit_definition")) 

105 

106 def testCalibrationDimensions(self): 

107 graph = DimensionGraph(self.universe, names=("calibration_label", "physical_filter", "detector")) 

108 self.assertCountEqual(graph.dimensions.names, 

109 ("instrument", "detector", "calibration_label", 

110 "physical_filter", "abstract_filter")) 

111 self.assertCountEqual(graph.required.names, ("instrument", "detector", "calibration_label", 

112 "physical_filter")) 

113 self.assertCountEqual(graph.implied.names, ("abstract_filter",)) 

114 self.assertCountEqual(graph.elements.names, graph.dimensions.names) 

115 

116 def testObservationDimensions(self): 

117 graph = DimensionGraph(self.universe, names=("exposure", "detector", "visit")) 

118 self.assertCountEqual(graph.dimensions.names, ("instrument", "detector", "visit", "exposure", 

119 "physical_filter", "abstract_filter", "visit_system")) 

120 self.assertCountEqual(graph.required.names, ("instrument", "detector", "exposure", "visit")) 

121 self.assertCountEqual(graph.implied.names, ("physical_filter", "abstract_filter", "visit_system")) 

122 self.assertCountEqual(graph.elements.names - graph.dimensions.names, 

123 ("visit_detector_region", "visit_definition")) 

124 self.assertCountEqual(graph.spatial.names, ("visit_detector_region",)) 

125 self.assertCountEqual(graph.temporal.names, ("exposure",)) 

126 

127 def testSkyMapDimensions(self): 

128 graph = DimensionGraph(self.universe, names=("patch",)) 

129 self.assertCountEqual(graph.dimensions.names, ("skymap", "tract", "patch")) 

130 self.assertCountEqual(graph.required.names, ("skymap", "tract", "patch")) 

131 self.assertCountEqual(graph.implied.names, ()) 

132 self.assertCountEqual(graph.elements.names, graph.dimensions.names) 

133 self.assertCountEqual(graph.spatial.names, ("patch",)) 

134 

135 def testSubsetCalculation(self): 

136 """Test that independent spatial and temporal options are computed 

137 correctly. 

138 """ 

139 graph = DimensionGraph(self.universe, names=("visit", "detector", "tract", "patch", "htm7", 

140 "exposure", "calibration_label")) 

141 self.assertCountEqual(graph.spatial.names, 

142 ("visit_detector_region", "patch", "htm7")) 

143 self.assertCountEqual(graph.temporal.names, 

144 ("exposure", "calibration_label")) 

145 

146 def testSchemaGeneration(self): 

147 tableSpecs = NamedKeyDict({}) 

148 for element in self.universe.elements: 

149 if element.hasTable and element.viewOf is None: 

150 tableSpecs[element] = makeDimensionElementTableSpec(element) 

151 for element, tableSpec in tableSpecs.items(): 

152 for dep in element.required: 

153 with self.subTest(element=element.name, dep=dep.name): 

154 if dep != element: 

155 self.assertIn(dep.name, tableSpec.fields) 

156 self.assertEqual(tableSpec.fields[dep.name].dtype, dep.primaryKey.dtype) 

157 self.assertEqual(tableSpec.fields[dep.name].length, dep.primaryKey.length) 

158 self.assertEqual(tableSpec.fields[dep.name].nbytes, dep.primaryKey.nbytes) 

159 self.assertFalse(tableSpec.fields[dep.name].nullable) 

160 self.assertTrue(tableSpec.fields[dep.name].primaryKey) 

161 else: 

162 self.assertIn(element.primaryKey.name, tableSpec.fields) 

163 self.assertEqual(tableSpec.fields[element.primaryKey.name].dtype, 

164 dep.primaryKey.dtype) 

165 self.assertEqual(tableSpec.fields[element.primaryKey.name].length, 

166 dep.primaryKey.length) 

167 self.assertEqual(tableSpec.fields[element.primaryKey.name].nbytes, 

168 dep.primaryKey.nbytes) 

169 self.assertFalse(tableSpec.fields[element.primaryKey.name].nullable) 

170 self.assertTrue(tableSpec.fields[element.primaryKey.name].primaryKey) 

171 for dep in element.implied: 

172 with self.subTest(element=element.name, dep=dep.name): 

173 self.assertIn(dep.name, tableSpec.fields) 

174 self.assertEqual(tableSpec.fields[dep.name].dtype, dep.primaryKey.dtype) 

175 self.assertFalse(tableSpec.fields[dep.name].primaryKey) 

176 for foreignKey in tableSpec.foreignKeys: 

177 self.assertIn(foreignKey.table, tableSpecs) 

178 self.assertIn(foreignKey.table, element.graph.dimensions.names) 

179 self.assertEqual(len(foreignKey.source), len(foreignKey.target)) 

180 for source, target in zip(foreignKey.source, foreignKey.target): 

181 self.assertIn(source, tableSpec.fields.names) 

182 self.assertIn(target, tableSpecs[foreignKey.table].fields.names) 

183 self.assertEqual(tableSpec.fields[source].dtype, 

184 tableSpecs[foreignKey.table].fields[target].dtype) 

185 self.assertEqual(tableSpec.fields[source].length, 

186 tableSpecs[foreignKey.table].fields[target].length) 

187 self.assertEqual(tableSpec.fields[source].nbytes, 

188 tableSpecs[foreignKey.table].fields[target].nbytes) 

189 self.assertEqual(tuple(tableSpec.fields.names), element.RecordClass.__slots__) 

190 

191 def testPickling(self): 

192 # Pickling and copying should always yield the exact same object within 

193 # a single process (cross-process is impossible to test here). 

194 universe1 = DimensionUniverse() 

195 universe2 = pickle.loads(pickle.dumps(universe1)) 

196 universe3 = copy.copy(universe1) 

197 universe4 = copy.deepcopy(universe1) 

198 self.assertIs(universe1, universe2) 

199 self.assertIs(universe1, universe3) 

200 self.assertIs(universe1, universe4) 

201 for element1 in universe1.elements: 

202 element2 = pickle.loads(pickle.dumps(element1)) 

203 self.assertIs(element1, element2) 

204 graph1 = element1.graph 

205 graph2 = pickle.loads(pickle.dumps(graph1)) 

206 self.assertIs(graph1, graph2) 

207 

208 

209if __name__ == "__main__": 209 ↛ 210line 209 didn't jump to line 210, because the condition on line 209 was never true

210 unittest.main()