Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22 

23import astropy 

24from astropy.table import Table as AstropyTable 

25from astropy.utils.diff import report_diff_values 

26import io 

27import os 

28 

29 

30from .. import ( 

31 Butler, 

32 Config, 

33 StorageClassFactory, 

34) 

35from ..tests import addDatasetType, MetricsExample 

36from ..registry import CollectionType 

37 

38 

39class ButlerTestHelper: 

40 """Mixin with helpers for unit tests.""" 

41 

42 def assertAstropyTablesEqual(self, tables, expectedTables): 

43 """Verify that a list of astropy tables matches a list of expected 

44 astropy tables. 

45 

46 Parameters 

47 ---------- 

48 tables : `astropy.table.Table` or iterable [`astropy.table.Table`] 

49 The table or tables that should match the expected tables. 

50 expectedTables : `astropy.table.Table` 

51 or iterable [`astropy.table.Table`] 

52 The tables with expected values to which the tables under test will 

53 be compared. 

54 """ 

55 # If a single table is passed in for tables or expectedTables, put it 

56 # in a list. 

57 if isinstance(tables, AstropyTable): 

58 tables = [tables] 

59 if isinstance(expectedTables, AstropyTable): 

60 expectedTables = [expectedTables] 

61 diff = io.StringIO() 

62 self.assertEqual(len(tables), len(expectedTables)) 

63 for table, expected in zip(tables, expectedTables): 

64 # Assert that we are testing what we think we are testing: 

65 self.assertIsInstance(table, AstropyTable) 

66 self.assertIsInstance(expected, AstropyTable) 

67 # Assert that they match: 

68 self.assertTrue(report_diff_values(table, expected, fileobj=diff), msg="\n" + diff.getvalue()) 

69 

70 

71def readTable(textTable): 

72 """Read an astropy table from formatted text. 

73 

74 Contains formatting that causes the astropy table to print an empty string 

75 instead of "--" for missing/unpopulated values in the text table. 

76 

77 

78 Parameters 

79 ---------- 

80 textTable : `str` 

81 The text version of the table to read. 

82 

83 Returns 

84 ------- 

85 table : `astropy.table.Table` 

86 The table as an astropy table. 

87 """ 

88 return AstropyTable.read(textTable, 

89 format="ascii", 

90 fill_values=[("", 0, "")]) 

91 

92 

93class MetricTestRepo: 

94 """Creates and manage a test repository on disk with datasets that 

95 may be queried and modified for unit tests. 

96 

97 Parameters 

98 ---------- 

99 root : `str` 

100 The location of the repository, to pass to ``Butler.makeRepo``. 

101 configFile : `str` 

102 The path to the config file, to pass to ``Butler.makeRepo``. 

103 """ 

104 

105 @staticmethod 

106 def _makeExampleMetrics(): 

107 """Make an object to put into the repository. 

108 """ 

109 return MetricsExample({"AM1": 5.2, "AM2": 30.6}, 

110 {"a": [1, 2, 3], 

111 "b": {"blue": 5, "red": "green"}}, 

112 [563, 234, 456.7, 752, 8, 9, 27]) 

113 

114 @staticmethod 

115 def _makeDimensionData(id, name, datetimeBegin=None, datetimeEnd=None): 

116 """Make a dict of dimensional data with default values to insert into 

117 the registry. 

118 """ 

119 data = dict(instrument="DummyCamComp", 

120 id=id, 

121 name=name, 

122 physical_filter="d-r", 

123 visit_system=1) 

124 if datetimeBegin: 

125 data["datetime_begin"] = datetimeBegin 

126 data["datetime_end"] = datetimeEnd 

127 return data 

128 

129 def __init__(self, root, configFile): 

130 self.root = root 

131 Butler.makeRepo(self.root, config=Config(configFile)) 

132 butlerConfigFile = os.path.join(self.root, "butler.yaml") 

133 self.storageClassFactory = StorageClassFactory() 

134 self.storageClassFactory.addFromConfig(butlerConfigFile) 

135 

136 # New datasets will be added to run and tag, but we will only look in 

137 # tag when looking up datasets. 

138 run = "ingest/run" 

139 tag = "ingest" 

140 self.butler = Butler(butlerConfigFile, run=run, collections=[tag], tags=[tag]) 

141 

142 # Create and register a DatasetType 

143 self.datasetType = addDatasetType(self.butler, "test_metric_comp", ("instrument", "visit"), 

144 "StructuredCompositeReadComp") 

145 

146 # Add needed Dimensions 

147 self.butler.registry.insertDimensionData("instrument", {"name": "DummyCamComp"}) 

148 self.butler.registry.insertDimensionData("physical_filter", {"instrument": "DummyCamComp", 

149 "name": "d-r", 

150 "band": "R"}) 

151 self.butler.registry.insertDimensionData("visit_system", {"instrument": "DummyCamComp", 

152 "id": 1, 

153 "name": "default"}) 

154 visitStart = astropy.time.Time("2020-01-01 08:00:00.123456789", scale="tai") 

155 visitEnd = astropy.time.Time("2020-01-01 08:00:36.66", scale="tai") 

156 self.butler.registry.insertDimensionData("visit", dict(instrument="DummyCamComp", 

157 id=423, 

158 name="fourtwentythree", 

159 physical_filter="d-r", 

160 visit_system=1, 

161 datetimeBegin=visitStart, 

162 datetimeEnd=visitEnd)) 

163 self.butler.registry.insertDimensionData("visit", dict(instrument="DummyCamComp", 

164 id=424, 

165 name="fourtwentyfour", 

166 physical_filter="d-r", 

167 visit_system=1)) 

168 

169 self.addDataset({"instrument": "DummyCamComp", "visit": 423}) 

170 self.addDataset({"instrument": "DummyCamComp", "visit": 424}) 

171 

172 def addDataset(self, dataId, run=None, datasetType=None): 

173 """Create a new example metric and add it to the named run with the 

174 given dataId. 

175 

176 Overwrites tags, so this does not try to associate the new dataset with 

177 existing tags. (If/when tags are needed this can be added to the 

178 arguments of this function.) 

179 

180 Parameters 

181 ---------- 

182 dataId : `dict` 

183 The dataId for the new metric. 

184 run : `str`, optional 

185 The name of the run to create and add a dataset to. If `None`, the 

186 dataset will be added to the root butler. 

187 datasetType : ``DatasetType``, optional 

188 The dataset type of the added dataset. If `None`, will use the 

189 default dataset type. 

190 """ 

191 if run: 

192 self.butler.registry.registerCollection(run, type=CollectionType.RUN) 

193 metric = self._makeExampleMetrics() 

194 self.butler.put(metric, 

195 self.datasetType if datasetType is None else datasetType, 

196 dataId, 

197 run=run, 

198 tags=())