Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for daf_butler CLI query-collections command. 

23""" 

24 

25from astropy.table import Table as AstropyTable 

26from numpy import array 

27import os 

28import unittest 

29 

30from lsst.daf.butler import StorageClassFactory 

31from lsst.daf.butler import script 

32from lsst.daf.butler.tests import addDatasetType 

33from lsst.daf.butler.tests.utils import ButlerTestHelper, makeTestTempDir, MetricTestRepo, removeTestTempDir 

34 

35 

36TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

37 

38 

39class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper): 

40 

41 mockFuncName = "lsst.daf.butler.cli.cmd.commands.script.queryDatasets" 

42 

43 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

44 storageClassFactory = StorageClassFactory() 

45 

46 @staticmethod 

47 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False): 

48 return script.queryDatasets(repo, glob, collections, where, find_first, show_uri) 

49 

50 def setUp(self): 

51 self.root = makeTestTempDir(TESTDIR) 

52 self.testRepo = MetricTestRepo(self.root, 

53 configFile=os.path.join(TESTDIR, "config/basic/butler.yaml")) 

54 

55 def tearDown(self): 

56 removeTestTempDir(self.root) 

57 

58 def testShowURI(self): 

59 """Test for expected output with show_uri=True.""" 

60 tables = self._queryDatasets(repo=self.root, show_uri=True) 

61 

62 expectedTables = ( 

63 AstropyTable(array(( 

64 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

65 self.testRepo.butler.datastore.root.join( 

66 "ingest/run/test_metric_comp.data/" 

67 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")), 

68 ("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

69 self.testRepo.butler.datastore.root.join( 

70 "ingest/run/test_metric_comp.data/" 

71 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")))), 

72 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

73 "visit", "URI")), 

74 AstropyTable(array(( 

75 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

76 self.testRepo.butler.datastore.root.join( 

77 "ingest/run/test_metric_comp.output/" 

78 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")), 

79 ("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

80 self.testRepo.butler.datastore.root.join( 

81 "ingest/run/test_metric_comp.output/" 

82 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")))), 

83 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

84 "visit", "URI")), 

85 AstropyTable(array(( 

86 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

87 self.testRepo.butler.datastore.root.join( 

88 "ingest/run/test_metric_comp.summary/" 

89 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")), 

90 ("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

91 self.testRepo.butler.datastore.root.join( 

92 "ingest/run/test_metric_comp.summary/" 

93 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")))), 

94 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

95 "visit", "URI")), 

96 ) 

97 

98 self.assertAstropyTablesEqual(tables, expectedTables) 

99 

100 def testNoShowURI(self): 

101 """Test for expected output without show_uri (default is False).""" 

102 tables = self._queryDatasets(repo=self.root) 

103 

104 expectedTables = ( 

105 AstropyTable(array(( 

106 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"), 

107 ("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))), 

108 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit") 

109 ), 

110 ) 

111 

112 self.assertAstropyTablesEqual(tables, expectedTables) 

113 

114 def testWhere(self): 

115 """Test using the where clause to reduce the number of rows returned. 

116 """ 

117 tables = self._queryDatasets(repo=self.root, where="instrument='DummyCamComp' AND visit=423") 

118 

119 expectedTables = ( 

120 AstropyTable(array( 

121 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423")), 

122 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit"), 

123 ), 

124 ) 

125 

126 self.assertAstropyTablesEqual(tables, expectedTables) 

127 

128 def testGlobDatasetType(self): 

129 """Test specifying dataset type.""" 

130 # Create and register an additional DatasetType 

131 

132 self.testRepo.butler.registry.insertDimensionData("visit", 

133 {"instrument": "DummyCamComp", "id": 425, 

134 "name": "fourtwentyfive", "physical_filter": "d-r", 

135 "visit_system": 1}) 

136 

137 datasetType = addDatasetType(self.testRepo.butler, 

138 "alt_test_metric_comp", 

139 ("instrument", "visit"), 

140 "StructuredCompositeReadComp") 

141 

142 self.testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType) 

143 

144 # verify the new dataset type increases the number of tables found: 

145 tables = self._queryDatasets(repo=self.root) 

146 

147 expectedTables = ( 

148 AstropyTable(array(( 

149 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"), 

150 ("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))), 

151 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit") 

152 ), 

153 AstropyTable(array(( 

154 ("alt_test_metric_comp", "ingest/run", "3", "R", "DummyCamComp", "d-r", "1", "425"))), 

155 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit") 

156 ) 

157 ) 

158 

159 self.assertAstropyTablesEqual(tables, expectedTables) 

160 

161 def testFindFirstAndCollections(self): 

162 """Test the find-first option, and the collections option, since it 

163 is required for find-first.""" 

164 

165 # Add a new run, and add a dataset to shadow an existing dataset. 

166 self.testRepo.addDataset(run="foo", 

167 dataId={"instrument": "DummyCamComp", "visit": 424}) 

168 

169 # Verify that without find-first, duplicate datasets are returned 

170 tables = self._queryDatasets(repo=self.root, 

171 collections=["foo", "ingest/run"], 

172 show_uri=True) 

173 

174 expectedTables = ( 

175 AstropyTable(array( 

176 ( 

177 ("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

178 self.testRepo.butler.datastore.root.join( 

179 "foo/test_metric_comp.data/" 

180 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")), 

181 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

182 self.testRepo.butler.datastore.root.join( 

183 "ingest/run/test_metric_comp.data/" 

184 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")), 

185 ("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

186 self.testRepo.butler.datastore.root.join( 

187 "ingest/run/test_metric_comp.data/" 

188 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")), 

189 )), 

190 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

191 "visit", "URI")), 

192 AstropyTable(array( 

193 ( 

194 ("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

195 self.testRepo.butler.datastore.root.join( 

196 "foo/test_metric_comp.output/" 

197 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")), 

198 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

199 self.testRepo.butler.datastore.root.join( 

200 "ingest/run/test_metric_comp.output/" 

201 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")), 

202 ("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

203 self.testRepo.butler.datastore.root.join( 

204 "ingest/run/test_metric_comp.output/" 

205 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")), 

206 )), 

207 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

208 "visit", "URI")), 

209 AstropyTable(array( 

210 ( 

211 ("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

212 self.testRepo.butler.datastore.root.join( 

213 "foo/test_metric_comp.summary/" 

214 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")), 

215 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

216 self.testRepo.butler.datastore.root.join( 

217 "ingest/run/test_metric_comp.summary/" 

218 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")), 

219 ("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

220 self.testRepo.butler.datastore.root.join( 

221 "ingest/run/test_metric_comp.summary/" 

222 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")), 

223 )), 

224 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

225 "visit", "URI")), 

226 ) 

227 

228 self.assertAstropyTablesEqual(tables, expectedTables) 

229 

230 # Verify that with find first the duplicate dataset is eliminated and 

231 # the more recent dataset is returned. 

232 tables = self._queryDatasets(repo=self.root, 

233 collections=["foo", "ingest/run"], 

234 show_uri=True, 

235 find_first=True) 

236 

237 expectedTables = ( 

238 AstropyTable(array( 

239 ( 

240 ("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

241 self.testRepo.butler.datastore.root.join( 

242 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml")), 

243 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

244 self.testRepo.butler.datastore.root.join( 

245 "ingest/run/test_metric_comp.data/" 

246 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")), 

247 )), 

248 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

249 "visit", "URI")), 

250 AstropyTable(array( 

251 ( 

252 ("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

253 self.testRepo.butler.datastore.root.join( 

254 "foo/test_metric_comp.output/" 

255 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")), 

256 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

257 self.testRepo.butler.datastore.root.join( 

258 "ingest/run/test_metric_comp.output/" 

259 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")), 

260 )), 

261 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

262 "visit", "URI")), 

263 AstropyTable(array( 

264 ( 

265 ("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

266 self.testRepo.butler.datastore.root.join( 

267 "foo/test_metric_comp.summary/" 

268 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")), 

269 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

270 self.testRepo.butler.datastore.root.join( 

271 "ingest/run/test_metric_comp.summary/" 

272 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")), 

273 )), 

274 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

275 "visit", "URI")), 

276 ) 

277 

278 self.assertAstropyTablesEqual(tables, expectedTables) 

279 

280 

281if __name__ == "__main__": 281 ↛ 282line 281 didn't jump to line 282, because the condition on line 281 was never true

282 unittest.main()