Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for daf_butler CLI query-collections command. 

23""" 

24 

25from astropy.table import Table as AstropyTable 

26from numpy import array 

27import os 

28import unittest 

29 

30from lsst.daf.butler import StorageClassFactory 

31from lsst.daf.butler import script 

32from lsst.daf.butler.tests import addDatasetType 

33from lsst.daf.butler.tests.utils import ButlerTestHelper, makeTestTempDir, MetricTestRepo, removeTestTempDir 

34 

35 

36TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

37 

38 

39class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper): 

40 

41 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

42 storageClassFactory = StorageClassFactory() 

43 

44 @staticmethod 

45 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False): 

46 return script.QueryDatasets(repo, glob, collections, where, find_first, show_uri).getTables() 

47 

48 def setUp(self): 

49 self.root = makeTestTempDir(TESTDIR) 

50 self.testRepo = MetricTestRepo(self.root, 

51 configFile=os.path.join(TESTDIR, "config/basic/butler.yaml")) 

52 

53 def tearDown(self): 

54 removeTestTempDir(self.root) 

55 

56 def testShowURI(self): 

57 """Test for expected output with show_uri=True.""" 

58 tables = self._queryDatasets(repo=self.root, show_uri=True) 

59 

60 expectedTables = ( 

61 AstropyTable(array(( 

62 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

63 self.testRepo.butler.datastore.root.join( 

64 "ingest/run/test_metric_comp.data/" 

65 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")), 

66 ("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

67 self.testRepo.butler.datastore.root.join( 

68 "ingest/run/test_metric_comp.data/" 

69 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")))), 

70 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

71 "visit", "URI")), 

72 AstropyTable(array(( 

73 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

74 self.testRepo.butler.datastore.root.join( 

75 "ingest/run/test_metric_comp.output/" 

76 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")), 

77 ("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

78 self.testRepo.butler.datastore.root.join( 

79 "ingest/run/test_metric_comp.output/" 

80 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")))), 

81 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

82 "visit", "URI")), 

83 AstropyTable(array(( 

84 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

85 self.testRepo.butler.datastore.root.join( 

86 "ingest/run/test_metric_comp.summary/" 

87 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")), 

88 ("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

89 self.testRepo.butler.datastore.root.join( 

90 "ingest/run/test_metric_comp.summary/" 

91 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")))), 

92 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

93 "visit", "URI")), 

94 ) 

95 

96 self.assertAstropyTablesEqual(tables, expectedTables) 

97 

98 def testNoShowURI(self): 

99 """Test for expected output without show_uri (default is False).""" 

100 tables = self._queryDatasets(repo=self.root) 

101 

102 expectedTables = ( 

103 AstropyTable(array(( 

104 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"), 

105 ("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))), 

106 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit") 

107 ), 

108 ) 

109 

110 self.assertAstropyTablesEqual(tables, expectedTables) 

111 

112 def testWhere(self): 

113 """Test using the where clause to reduce the number of rows returned. 

114 """ 

115 tables = self._queryDatasets(repo=self.root, where="instrument='DummyCamComp' AND visit=423") 

116 

117 expectedTables = ( 

118 AstropyTable(array( 

119 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423")), 

120 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit"), 

121 ), 

122 ) 

123 

124 self.assertAstropyTablesEqual(tables, expectedTables) 

125 

126 def testGlobDatasetType(self): 

127 """Test specifying dataset type.""" 

128 # Create and register an additional DatasetType 

129 

130 self.testRepo.butler.registry.insertDimensionData("visit", 

131 {"instrument": "DummyCamComp", "id": 425, 

132 "name": "fourtwentyfive", "physical_filter": "d-r", 

133 "visit_system": 1}) 

134 

135 datasetType = addDatasetType(self.testRepo.butler, 

136 "alt_test_metric_comp", 

137 ("instrument", "visit"), 

138 "StructuredCompositeReadComp") 

139 

140 self.testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType) 

141 

142 # verify the new dataset type increases the number of tables found: 

143 tables = self._queryDatasets(repo=self.root) 

144 

145 expectedTables = ( 

146 AstropyTable(array(( 

147 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"), 

148 ("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))), 

149 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit") 

150 ), 

151 AstropyTable(array(( 

152 ("alt_test_metric_comp", "ingest/run", "3", "R", "DummyCamComp", "d-r", "1", "425"))), 

153 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit") 

154 ) 

155 ) 

156 

157 self.assertAstropyTablesEqual(tables, expectedTables) 

158 

159 def testFindFirstAndCollections(self): 

160 """Test the find-first option, and the collections option, since it 

161 is required for find-first.""" 

162 

163 # Add a new run, and add a dataset to shadow an existing dataset. 

164 self.testRepo.addDataset(run="foo", 

165 dataId={"instrument": "DummyCamComp", "visit": 424}) 

166 

167 # Verify that without find-first, duplicate datasets are returned 

168 tables = self._queryDatasets(repo=self.root, 

169 collections=["foo", "ingest/run"], 

170 show_uri=True) 

171 

172 expectedTables = ( 

173 AstropyTable(array( 

174 ( 

175 ("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

176 self.testRepo.butler.datastore.root.join( 

177 "foo/test_metric_comp.data/" 

178 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")), 

179 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

180 self.testRepo.butler.datastore.root.join( 

181 "ingest/run/test_metric_comp.data/" 

182 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")), 

183 ("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

184 self.testRepo.butler.datastore.root.join( 

185 "ingest/run/test_metric_comp.data/" 

186 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")), 

187 )), 

188 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

189 "visit", "URI")), 

190 AstropyTable(array( 

191 ( 

192 ("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

193 self.testRepo.butler.datastore.root.join( 

194 "foo/test_metric_comp.output/" 

195 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")), 

196 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

197 self.testRepo.butler.datastore.root.join( 

198 "ingest/run/test_metric_comp.output/" 

199 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")), 

200 ("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

201 self.testRepo.butler.datastore.root.join( 

202 "ingest/run/test_metric_comp.output/" 

203 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")), 

204 )), 

205 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

206 "visit", "URI")), 

207 AstropyTable(array( 

208 ( 

209 ("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

210 self.testRepo.butler.datastore.root.join( 

211 "foo/test_metric_comp.summary/" 

212 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")), 

213 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

214 self.testRepo.butler.datastore.root.join( 

215 "ingest/run/test_metric_comp.summary/" 

216 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")), 

217 ("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

218 self.testRepo.butler.datastore.root.join( 

219 "ingest/run/test_metric_comp.summary/" 

220 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")), 

221 )), 

222 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

223 "visit", "URI")), 

224 ) 

225 

226 self.assertAstropyTablesEqual(tables, expectedTables) 

227 

228 # Verify that with find first the duplicate dataset is eliminated and 

229 # the more recent dataset is returned. 

230 tables = self._queryDatasets(repo=self.root, 

231 collections=["foo", "ingest/run"], 

232 show_uri=True, 

233 find_first=True) 

234 

235 expectedTables = ( 

236 AstropyTable(array( 

237 ( 

238 ("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

239 self.testRepo.butler.datastore.root.join( 

240 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml")), 

241 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

242 self.testRepo.butler.datastore.root.join( 

243 "ingest/run/test_metric_comp.data/" 

244 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")), 

245 )), 

246 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

247 "visit", "URI")), 

248 AstropyTable(array( 

249 ( 

250 ("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

251 self.testRepo.butler.datastore.root.join( 

252 "foo/test_metric_comp.output/" 

253 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")), 

254 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

255 self.testRepo.butler.datastore.root.join( 

256 "ingest/run/test_metric_comp.output/" 

257 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")), 

258 )), 

259 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

260 "visit", "URI")), 

261 AstropyTable(array( 

262 ( 

263 ("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

264 self.testRepo.butler.datastore.root.join( 

265 "foo/test_metric_comp.summary/" 

266 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")), 

267 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

268 self.testRepo.butler.datastore.root.join( 

269 "ingest/run/test_metric_comp.summary/" 

270 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")), 

271 )), 

272 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

273 "visit", "URI")), 

274 ) 

275 

276 self.assertAstropyTablesEqual(tables, expectedTables) 

277 

278 

279if __name__ == "__main__": 279 ↛ 280line 279 didn't jump to line 280, because the condition on line 279 was never true

280 unittest.main()