Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for daf_butler CLI query-collections command. 

23""" 

24 

25from astropy.table import Table as AstropyTable 

26from numpy import array 

27import os 

28import shutil 

29import tempfile 

30import unittest 

31 

32from lsst.daf.butler import StorageClassFactory 

33from lsst.daf.butler import script 

34from lsst.daf.butler.tests import addDatasetType 

35from lsst.daf.butler.tests.utils import ButlerTestHelper, MetricTestRepo 

36 

37 

38TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

39 

40 

41class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper): 

42 

43 mockFuncName = "lsst.daf.butler.cli.cmd.commands.script.queryDatasets" 

44 

45 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

46 storageClassFactory = StorageClassFactory() 

47 

48 @staticmethod 

49 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False): 

50 return script.queryDatasets(repo, glob, collections, where, find_first, show_uri) 

51 

52 def setUp(self): 

53 self.root = tempfile.mkdtemp(dir=TESTDIR) 

54 self.testRepo = MetricTestRepo(self.root, 

55 configFile=os.path.join(TESTDIR, "config/basic/butler.yaml")) 

56 

57 def tearDown(self): 

58 if os.path.exists(self.root): 

59 shutil.rmtree(self.root, ignore_errors=True) 

60 

61 def testShowURI(self): 

62 """Test for expected output with show_uri=True.""" 

63 tables = self._queryDatasets(repo=self.root, show_uri=True) 

64 

65 expectedTables = ( 

66 AstropyTable(array(( 

67 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

68 self.testRepo.butler.datastore.root.join( 

69 "ingest/run/test_metric_comp.data/" 

70 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")), 

71 ("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

72 self.testRepo.butler.datastore.root.join( 

73 "ingest/run/test_metric_comp.data/" 

74 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")))), 

75 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

76 "visit", "URI")), 

77 AstropyTable(array(( 

78 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

79 self.testRepo.butler.datastore.root.join( 

80 "ingest/run/test_metric_comp.output/" 

81 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")), 

82 ("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

83 self.testRepo.butler.datastore.root.join( 

84 "ingest/run/test_metric_comp.output/" 

85 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")))), 

86 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

87 "visit", "URI")), 

88 AstropyTable(array(( 

89 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

90 self.testRepo.butler.datastore.root.join( 

91 "ingest/run/test_metric_comp.summary/" 

92 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")), 

93 ("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

94 self.testRepo.butler.datastore.root.join( 

95 "ingest/run/test_metric_comp.summary/" 

96 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")))), 

97 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

98 "visit", "URI")), 

99 ) 

100 

101 self.assertAstropyTablesEqual(tables, expectedTables) 

102 

103 def testNoShowURI(self): 

104 """Test for expected output without show_uri (default is False).""" 

105 tables = self._queryDatasets(repo=self.root) 

106 

107 expectedTables = ( 

108 AstropyTable(array(( 

109 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"), 

110 ("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))), 

111 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit") 

112 ), 

113 ) 

114 

115 self.assertAstropyTablesEqual(tables, expectedTables) 

116 

117 def testWhere(self): 

118 """Test using the where clause to reduce the number of rows returned. 

119 """ 

120 tables = self._queryDatasets(repo=self.root, where="instrument='DummyCamComp' AND visit=423") 

121 

122 expectedTables = ( 

123 AstropyTable(array( 

124 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423")), 

125 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit"), 

126 ), 

127 ) 

128 

129 self.assertAstropyTablesEqual(tables, expectedTables) 

130 

131 def testGlobDatasetType(self): 

132 """Test specifying dataset type.""" 

133 # Create and register an additional DatasetType 

134 

135 self.testRepo.butler.registry.insertDimensionData("visit", 

136 {"instrument": "DummyCamComp", "id": 425, 

137 "name": "fourtwentyfive", "physical_filter": "d-r", 

138 "visit_system": 1}) 

139 

140 datasetType = addDatasetType(self.testRepo.butler, 

141 "alt_test_metric_comp", 

142 ("instrument", "visit"), 

143 "StructuredCompositeReadComp") 

144 

145 self.testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType) 

146 

147 # verify the new dataset type increases the number of tables found: 

148 tables = self._queryDatasets(repo=self.root) 

149 

150 expectedTables = ( 

151 AstropyTable(array(( 

152 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"), 

153 ("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))), 

154 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit") 

155 ), 

156 AstropyTable(array(( 

157 ("alt_test_metric_comp", "ingest/run", "3", "R", "DummyCamComp", "d-r", "1", "425"))), 

158 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit") 

159 ) 

160 ) 

161 

162 self.assertAstropyTablesEqual(tables, expectedTables) 

163 

164 def testFindFirstAndCollections(self): 

165 """Test the find-first option, and the collections option, since it 

166 is required for find-first.""" 

167 

168 # Add a new run, and add a dataset to shadow an existing dataset. 

169 self.testRepo.addDataset(run="foo", 

170 dataId={"instrument": "DummyCamComp", "visit": 424}) 

171 

172 # Verify that without find-first, duplicate datasets are returned 

173 tables = self._queryDatasets(repo=self.root, 

174 collections=["foo", "ingest/run"], 

175 show_uri=True) 

176 

177 expectedTables = ( 

178 AstropyTable(array( 

179 ( 

180 ("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

181 self.testRepo.butler.datastore.root.join( 

182 "foo/test_metric_comp.data/" 

183 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")), 

184 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

185 self.testRepo.butler.datastore.root.join( 

186 "ingest/run/test_metric_comp.data/" 

187 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")), 

188 ("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

189 self.testRepo.butler.datastore.root.join( 

190 "ingest/run/test_metric_comp.data/" 

191 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")), 

192 )), 

193 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

194 "visit", "URI")), 

195 AstropyTable(array( 

196 ( 

197 ("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

198 self.testRepo.butler.datastore.root.join( 

199 "foo/test_metric_comp.output/" 

200 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")), 

201 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

202 self.testRepo.butler.datastore.root.join( 

203 "ingest/run/test_metric_comp.output/" 

204 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")), 

205 ("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

206 self.testRepo.butler.datastore.root.join( 

207 "ingest/run/test_metric_comp.output/" 

208 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")), 

209 )), 

210 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

211 "visit", "URI")), 

212 AstropyTable(array( 

213 ( 

214 ("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

215 self.testRepo.butler.datastore.root.join( 

216 "foo/test_metric_comp.summary/" 

217 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")), 

218 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

219 self.testRepo.butler.datastore.root.join( 

220 "ingest/run/test_metric_comp.summary/" 

221 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")), 

222 ("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

223 self.testRepo.butler.datastore.root.join( 

224 "ingest/run/test_metric_comp.summary/" 

225 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")), 

226 )), 

227 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

228 "visit", "URI")), 

229 ) 

230 

231 self.assertAstropyTablesEqual(tables, expectedTables) 

232 

233 # Verify that with find first the duplicate dataset is eliminated and 

234 # the more recent dataset is returned. 

235 tables = self._queryDatasets(repo=self.root, 

236 collections=["foo", "ingest/run"], 

237 show_uri=True, 

238 find_first=True) 

239 

240 expectedTables = ( 

241 AstropyTable(array( 

242 ( 

243 ("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

244 self.testRepo.butler.datastore.root.join( 

245 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml")), 

246 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

247 self.testRepo.butler.datastore.root.join( 

248 "ingest/run/test_metric_comp.data/" 

249 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")), 

250 )), 

251 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

252 "visit", "URI")), 

253 AstropyTable(array( 

254 ( 

255 ("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

256 self.testRepo.butler.datastore.root.join( 

257 "foo/test_metric_comp.output/" 

258 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")), 

259 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

260 self.testRepo.butler.datastore.root.join( 

261 "ingest/run/test_metric_comp.output/" 

262 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")), 

263 )), 

264 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

265 "visit", "URI")), 

266 AstropyTable(array( 

267 ( 

268 ("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

269 self.testRepo.butler.datastore.root.join( 

270 "foo/test_metric_comp.summary/" 

271 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")), 

272 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

273 self.testRepo.butler.datastore.root.join( 

274 "ingest/run/test_metric_comp.summary/" 

275 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")), 

276 )), 

277 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

278 "visit", "URI")), 

279 ) 

280 

281 self.assertAstropyTablesEqual(tables, expectedTables) 

282 

283 

284if __name__ == "__main__": 284 ↛ 285line 284 didn't jump to line 285, because the condition on line 284 was never true

285 unittest.main()