Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import unittest 

23import copy 

24import pickle 

25import itertools 

26 

27from lsst.daf.butler.core.utils import NamedKeyDict, NamedValueSet 

28from lsst.daf.butler.core.dimensions import DimensionUniverse, DimensionGraph, Dimension 

29from lsst.daf.butler.core.dimensions.schema import makeElementTableSpec 

30 

31 

32class DimensionTestCase(unittest.TestCase): 

33 """Tests for dimensions. 

34 

35 All tests here rely on the content of ``config/dimensions.yaml``, either 

36 to test that the definitions there are read in properly or just as generic 

37 data for testing various operations. 

38 """ 

39 

40 def setUp(self): 

41 self.universe = DimensionUniverse() 

42 

43 def checkGraphInvariants(self, graph): 

44 elements = list(graph.elements) 

45 for n, element in enumerate(elements): 

46 # Ordered comparisons on graphs behave like sets. 

47 self.assertLessEqual(element.graph, graph) 

48 # Ordered comparisons on elements correspond to the ordering within 

49 # a DimensionUniverse (topological, with deterministic 

50 # tiebreakers). 

51 for other in elements[:n]: 

52 self.assertLess(other, element) 

53 self.assertLessEqual(other, element) 

54 for other in elements[n + 1:]: 

55 self.assertGreater(other, element) 

56 self.assertGreaterEqual(other, element) 

57 self.assertEqual(DimensionGraph(self.universe, graph.required), graph) 

58 self.assertCountEqual(graph.required, 

59 [dimension for dimension in graph.dimensions 

60 if not any(dimension in other.graph.implied for other in graph.elements)]) 

61 self.assertCountEqual(graph.implied, graph.dimensions - graph.required) 

62 self.assertCountEqual(graph.dimensions, 

63 [element for element in graph.elements 

64 if isinstance(element, Dimension)]) 

65 self.assertCountEqual(graph.dimensions, itertools.chain(graph.required, graph.implied)) 

66 # Check primary key traversal order: each element should follow any it 

67 # requires, and element that is implied by any other in the graph 

68 # follow at least one of those. 

69 seen = NamedValueSet() 

70 for element in graph.primaryKeyTraversalOrder: 

71 with self.subTest(required=graph.required, implied=graph.implied, element=element): 

72 seen.add(element) 

73 self.assertLessEqual(element.graph.required, seen) 

74 if element in graph.implied: 

75 self.assertTrue(any(element in s.implied for s in seen)) 

76 self.assertCountEqual(seen, graph.elements) 

77 

78 def testConfigRead(self): 

79 self.assertEqual(self.universe.dimensions.names, 

80 {"instrument", "visit", "exposure", "detector", "physical_filter", 

81 "abstract_filter", "subfilter", "calibration_label", 

82 "skymap", "tract", "patch", "htm7", "htm9"}) 

83 

84 def testGraphs(self): 

85 self.checkGraphInvariants(self.universe.empty) 

86 self.checkGraphInvariants(self.universe) 

87 for element in self.universe.elements: 

88 self.checkGraphInvariants(element.graph) 

89 

90 def testInstrumentDimensions(self): 

91 graph = DimensionGraph(self.universe, names=("exposure", "detector", "calibration_label")) 

92 self.assertCountEqual(graph.dimensions.names, 

93 ("instrument", "exposure", "detector", "calibration_label", 

94 "visit", "physical_filter", "abstract_filter")) 

95 self.assertCountEqual(graph.required.names, ("instrument", "exposure", "detector", 

96 "calibration_label")) 

97 self.assertCountEqual(graph.implied.names, ("visit", "physical_filter", "abstract_filter")) 

98 self.assertCountEqual(graph.elements.names - graph.dimensions.names, ("visit_detector_region",)) 

99 

100 def testCalibrationDimensions(self): 

101 graph = DimensionGraph(self.universe, names=("calibration_label", "physical_filter", "detector")) 

102 self.assertCountEqual(graph.dimensions.names, 

103 ("instrument", "detector", "calibration_label", 

104 "physical_filter", "abstract_filter")) 

105 self.assertCountEqual(graph.required.names, ("instrument", "detector", "calibration_label", 

106 "physical_filter")) 

107 self.assertCountEqual(graph.implied.names, ("abstract_filter",)) 

108 self.assertCountEqual(graph.elements.names, graph.dimensions.names) 

109 

110 def testObservationDimensions(self): 

111 graph = DimensionGraph(self.universe, names=("exposure", "detector")) 

112 self.assertCountEqual(graph.dimensions.names, ("instrument", "detector", "visit", "exposure", 

113 "physical_filter", "abstract_filter")) 

114 self.assertCountEqual(graph.required.names, ("instrument", "detector", "exposure")) 

115 self.assertCountEqual(graph.implied.names, ("physical_filter", "abstract_filter", "visit")) 

116 self.assertCountEqual(graph.elements.names - graph.dimensions.names, ("visit_detector_region",)) 

117 self.assertCountEqual(graph.spatial.names, ("visit_detector_region",)) 

118 self.assertCountEqual(graph.getSpatial(independent=False).names, 

119 ("visit", "visit_detector_region",)) 

120 self.assertCountEqual(graph.getSpatial(prefer=("visit",)).names, ("visit",)) 

121 self.assertCountEqual(graph.getTemporal(independent=False).names, ("visit", "exposure")) 

122 self.assertCountEqual(graph.temporal.names, ("exposure",)) 

123 self.assertCountEqual(graph.getTemporal(prefer=("visit",)).names, ("visit",)) 

124 

125 def testSkyMapDimensions(self): 

126 graph = DimensionGraph(self.universe, names=("patch",)) 

127 self.assertCountEqual(graph.dimensions.names, ("skymap", "tract", "patch")) 

128 self.assertCountEqual(graph.required.names, ("skymap", "tract", "patch")) 

129 self.assertCountEqual(graph.implied.names, ()) 

130 self.assertCountEqual(graph.elements.names, graph.dimensions.names) 

131 self.assertCountEqual(graph.spatial.names, ("patch",)) 

132 self.assertCountEqual(graph.getSpatial(independent=False).names, ("patch", "tract")) 

133 self.assertCountEqual(graph.getSpatial(prefer=("tract",)).names, ("tract",)) 

134 

135 def testSubsetCalculation(self): 

136 """Test that independent spatial and temporal options are computed 

137 correctly. 

138 """ 

139 graph = DimensionGraph(self.universe, names=("visit", "detector", "tract", "patch", "htm7", 

140 "exposure", "calibration_label")) 

141 self.assertCountEqual(graph.spatial.names, 

142 ("visit_detector_region", "patch", "htm7")) 

143 self.assertCountEqual(graph.getSpatial(independent=False).names, 

144 ("visit_detector_region", "patch", "htm7", "visit", "tract")) 

145 self.assertCountEqual(graph.getSpatial(prefer=["tract"]).names, 

146 ("visit_detector_region", "tract", "htm7")) 

147 self.assertCountEqual(graph.temporal.names, 

148 ("exposure", "calibration_label")) 

149 self.assertCountEqual(graph.getTemporal(independent=False).names, 

150 ("visit", "exposure", "calibration_label")) 

151 self.assertCountEqual(graph.getTemporal(prefer=["visit"]).names, 

152 ("visit", "calibration_label")) 

153 

154 def testSchemaGeneration(self): 

155 tableSpecs = NamedKeyDict({}) 

156 for element in self.universe.elements: 

157 if element.hasTable and element.viewOf is None: 

158 tableSpecs[element] = makeElementTableSpec(element) 

159 for element, tableSpec in tableSpecs.items(): 

160 for dep in element.graph.required: 

161 with self.subTest(element=element.name, dep=dep.name): 

162 if dep != element: 

163 self.assertIn(dep.name, tableSpec.fields) 

164 self.assertEqual(tableSpec.fields[dep.name].dtype, dep.primaryKey.dtype) 

165 self.assertEqual(tableSpec.fields[dep.name].length, dep.primaryKey.length) 

166 self.assertEqual(tableSpec.fields[dep.name].nbytes, dep.primaryKey.nbytes) 

167 self.assertFalse(tableSpec.fields[dep.name].nullable) 

168 self.assertTrue(tableSpec.fields[dep.name].primaryKey) 

169 else: 

170 self.assertIn(element.primaryKey.name, tableSpec.fields) 

171 self.assertEqual(tableSpec.fields[element.primaryKey.name].dtype, 

172 dep.primaryKey.dtype) 

173 self.assertEqual(tableSpec.fields[element.primaryKey.name].length, 

174 dep.primaryKey.length) 

175 self.assertEqual(tableSpec.fields[element.primaryKey.name].nbytes, 

176 dep.primaryKey.nbytes) 

177 self.assertFalse(tableSpec.fields[element.primaryKey.name].nullable) 

178 self.assertTrue(tableSpec.fields[element.primaryKey.name].primaryKey) 

179 for dep in element.implied: 

180 with self.subTest(element=element.name, dep=dep.name): 

181 self.assertIn(dep.name, tableSpec.fields) 

182 self.assertEqual(tableSpec.fields[dep.name].dtype, dep.primaryKey.dtype) 

183 self.assertFalse(tableSpec.fields[dep.name].primaryKey) 

184 for foreignKey in tableSpec.foreignKeys: 

185 self.assertIn(foreignKey.table, tableSpecs) 

186 self.assertIn(foreignKey.table, element.graph.dimensions.names) 

187 self.assertEqual(len(foreignKey.source), len(foreignKey.target)) 

188 for source, target in zip(foreignKey.source, foreignKey.target): 

189 self.assertIn(source, tableSpec.fields.names) 

190 self.assertIn(target, tableSpecs[foreignKey.table].fields.names) 

191 self.assertEqual(tableSpec.fields[source].dtype, 

192 tableSpecs[foreignKey.table].fields[target].dtype) 

193 self.assertEqual(tableSpec.fields[source].length, 

194 tableSpecs[foreignKey.table].fields[target].length) 

195 self.assertEqual(tableSpec.fields[source].nbytes, 

196 tableSpecs[foreignKey.table].fields[target].nbytes) 

197 self.assertEqual(tuple(tableSpec.fields.names), element.RecordClass.__slots__) 

198 

199 def testPickling(self): 

200 # Pickling and copying should always yield the exact same object within 

201 # a single process (cross-process is impossible to test here). 

202 universe1 = DimensionUniverse() 

203 universe2 = pickle.loads(pickle.dumps(universe1)) 

204 universe3 = copy.copy(universe1) 

205 universe4 = copy.deepcopy(universe1) 

206 self.assertIs(universe1, universe2) 

207 self.assertIs(universe1, universe3) 

208 self.assertIs(universe1, universe4) 

209 for element1 in universe1.elements: 

210 element2 = pickle.loads(pickle.dumps(element1)) 

211 self.assertIs(element1, element2) 

212 graph1 = element1.graph 

213 graph2 = pickle.loads(pickle.dumps(graph1)) 

214 self.assertIs(graph1, graph2) 

215 

216 

217if __name__ == "__main__": 217 ↛ 218line 217 didn't jump to line 218, because the condition on line 217 was never true

218 unittest.main()