Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22 

23import astropy 

24from astropy.table import Table as AstropyTable 

25from astropy.utils.diff import report_diff_values 

26import io 

27import os 

28 

29 

30from .. import ( 

31 Butler, 

32 Config, 

33 StorageClassFactory, 

34) 

35from ..tests import addDatasetType, MetricsExample 

36from ..registry import CollectionType 

37 

38 

39class ButlerTestHelper: 

40 """Mixin with helpers for unit tests.""" 

41 

42 def assertAstropyTablesEqual(self, tables, expectedTables): 

43 """Verify that a list of astropy tables matches a list of expected 

44 astropy tables. 

45 

46 Parameters 

47 ---------- 

48 tables : `astropy.table.Table` or iterable [`astropy.table.Table`] 

49 The table or tables that should match the expected tables. 

50 expectedTables : `astropy.table.Table` 

51 or iterable [`astropy.table.Table`] 

52 The tables with expected values to which the tables under test will 

53 be compared. 

54 """ 

55 # If a single table is passed in for tables or expectedTables, put it 

56 # in a list. 

57 if isinstance(tables, AstropyTable): 

58 tables = [tables] 

59 if isinstance(expectedTables, AstropyTable): 

60 expectedTables = [expectedTables] 

61 diff = io.StringIO() 

62 self.assertEqual(len(tables), len(expectedTables)) 

63 for table, expected in zip(tables, expectedTables): 

64 # Assert that we are testing what we think we are testing: 

65 self.assertIsInstance(table, AstropyTable) 

66 self.assertIsInstance(expected, AstropyTable) 

67 # Assert that they match: 

68 self.assertTrue(report_diff_values(table, expected, fileobj=diff), msg="\n" + diff.getvalue()) 

69 

70 

71def readTable(textTable): 

72 """Read an astropy table from formatted text. 

73 

74 Contains formatting that causes the astropy table to print an empty string 

75 instead of "--" for missing/unpopulated values in the text table. 

76 

77 

78 Parameters 

79 ---------- 

80 textTable : `str` 

81 The text version of the table to read. 

82 

83 Returns 

84 ------- 

85 table : `astropy.table.Table` 

86 The table as an astropy table. 

87 """ 

88 return AstropyTable.read(textTable, 

89 format="ascii", 

90 data_start=2, # skip the header row and the header row underlines. 

91 fill_values=[("", 0, "")]) 

92 

93 

94class MetricTestRepo: 

95 """Creates and manage a test repository on disk with datasets that 

96 may be queried and modified for unit tests. 

97 

98 Parameters 

99 ---------- 

100 root : `str` 

101 The location of the repository, to pass to ``Butler.makeRepo``. 

102 configFile : `str` 

103 The path to the config file, to pass to ``Butler.makeRepo``. 

104 """ 

105 

106 @staticmethod 

107 def _makeExampleMetrics(): 

108 """Make an object to put into the repository. 

109 """ 

110 return MetricsExample({"AM1": 5.2, "AM2": 30.6}, 

111 {"a": [1, 2, 3], 

112 "b": {"blue": 5, "red": "green"}}, 

113 [563, 234, 456.7, 752, 8, 9, 27]) 

114 

115 @staticmethod 

116 def _makeDimensionData(id, name, datetimeBegin=None, datetimeEnd=None): 

117 """Make a dict of dimensional data with default values to insert into 

118 the registry. 

119 """ 

120 data = dict(instrument="DummyCamComp", 

121 id=id, 

122 name=name, 

123 physical_filter="d-r", 

124 visit_system=1) 

125 if datetimeBegin: 

126 data["datetime_begin"] = datetimeBegin 

127 data["datetime_end"] = datetimeEnd 

128 return data 

129 

130 def __init__(self, root, configFile): 

131 self.root = root 

132 Butler.makeRepo(self.root, config=Config(configFile)) 

133 butlerConfigFile = os.path.join(self.root, "butler.yaml") 

134 self.storageClassFactory = StorageClassFactory() 

135 self.storageClassFactory.addFromConfig(butlerConfigFile) 

136 

137 # New datasets will be added to run and tag, but we will only look in 

138 # tag when looking up datasets. 

139 run = "ingest/run" 

140 tag = "ingest" 

141 self.butler = Butler(butlerConfigFile, run=run, collections=[tag], tags=[tag]) 

142 

143 # Create and register a DatasetType 

144 self.datasetType = addDatasetType(self.butler, "test_metric_comp", ("instrument", "visit"), 

145 "StructuredCompositeReadComp") 

146 

147 # Add needed Dimensions 

148 self.butler.registry.insertDimensionData("instrument", {"name": "DummyCamComp"}) 

149 self.butler.registry.insertDimensionData("physical_filter", {"instrument": "DummyCamComp", 

150 "name": "d-r", 

151 "band": "R"}) 

152 self.butler.registry.insertDimensionData("visit_system", {"instrument": "DummyCamComp", 

153 "id": 1, 

154 "name": "default"}) 

155 visitStart = astropy.time.Time("2020-01-01 08:00:00.123456789", scale="tai") 

156 visitEnd = astropy.time.Time("2020-01-01 08:00:36.66", scale="tai") 

157 self.butler.registry.insertDimensionData("visit", dict(instrument="DummyCamComp", 

158 id=423, 

159 name="fourtwentythree", 

160 physical_filter="d-r", 

161 visit_system=1, 

162 datetimeBegin=visitStart, 

163 datetimeEnd=visitEnd)) 

164 self.butler.registry.insertDimensionData("visit", dict(instrument="DummyCamComp", 

165 id=424, 

166 name="fourtwentyfour", 

167 physical_filter="d-r", 

168 visit_system=1)) 

169 

170 self.addDataset({"instrument": "DummyCamComp", "visit": 423}) 

171 self.addDataset({"instrument": "DummyCamComp", "visit": 424}) 

172 

173 def addDataset(self, dataId, run=None, datasetType=None): 

174 """Create a new example metric and add it to the named run with the 

175 given dataId. 

176 

177 Overwrites tags, so this does not try to associate the new dataset with 

178 existing tags. (If/when tags are needed this can be added to the 

179 arguments of this function.) 

180 

181 Parameters 

182 ---------- 

183 dataId : `dict` 

184 The dataId for the new metric. 

185 run : `str`, optional 

186 The name of the run to create and add a dataset to. If `None`, the 

187 dataset will be added to the root butler. 

188 datasetType : ``DatasetType``, optional 

189 The dataset type of the added dataset. If `None`, will use the 

190 default dataset type. 

191 """ 

192 if run: 

193 self.butler.registry.registerCollection(run, type=CollectionType.RUN) 

194 metric = self._makeExampleMetrics() 

195 self.butler.put(metric, 

196 self.datasetType if datasetType is None else datasetType, 

197 dataId, 

198 run=run, 

199 tags=())