Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for daf_butler CLI query-collections command. 

23""" 

24 

25import astropy 

26from astropy.table import Table as AstropyTable 

27from astropy.utils.diff import report_diff_values 

28from numpy import array 

29import io 

30import os 

31import shutil 

32import tempfile 

33import unittest 

34 

35from lsst.daf.butler import ( 

36 Butler, 

37 Config, 

38 DatasetRef, 

39 DatasetType, 

40 StorageClassFactory 

41) 

42from lsst.daf.butler import script 

43from lsst.daf.butler.tests import MetricsExample 

44 

45 

46TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

47 

48 

49class QueryDatasetsTest(unittest.TestCase): 

50 

51 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

52 storageClassFactory = StorageClassFactory() 

53 

54 def _assertTablesEqual(self, tables, expectedTables): 

55 """Verify that a list of astropy tables matches a list of expected 

56 astropy tables.""" 

57 diff = io.StringIO() 

58 self.assertEqual(len(tables), len(expectedTables)) 

59 for table, expected in zip(tables, expectedTables): 

60 # Assert that we are testing what we think we are testing 

61 self.assertIsInstance(table, AstropyTable) 

62 self.assertIsInstance(expected, AstropyTable) 

63 self.assertTrue(report_diff_values(table, expected, fileobj=diff), msg=diff.getvalue()) 

64 

65 @staticmethod 

66 def _makeExampleMetrics(): 

67 return MetricsExample({"AM1": 5.2, "AM2": 30.6}, 

68 {"a": [1, 2, 3], 

69 "b": {"blue": 5, "red": "green"}}, 

70 [563, 234, 456.7, 752, 8, 9, 27]) 

71 

72 @staticmethod 

73 def _addDatasetType(datasetTypeName, dimensions, storageClass, registry): 

74 """Create a DatasetType and register it 

75 """ 

76 datasetType = DatasetType(datasetTypeName, dimensions, storageClass) 

77 registry.registerDatasetType(datasetType) 

78 return datasetType 

79 

80 @staticmethod 

81 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False): 

82 return script.queryDatasets(repo, glob, collections, where, find_first, show_uri) 

83 

84 def setUp(self): 

85 self.root = tempfile.mkdtemp(dir=TESTDIR) 

86 Butler.makeRepo(self.root, config=Config(self.configFile)) 

87 self.butlerConfigFile = os.path.join(self.root, "butler.yaml") 

88 self.storageClassFactory.addFromConfig(self.configFile) 

89 

90 # New datasets will be added to run and tag, but we will only look in 

91 # tag when looking up datasets. 

92 run = "ingest/run" 

93 tag = "ingest" 

94 self.butler = Butler(self.butlerConfigFile, run=run, collections=[tag], tags=[tag]) 

95 

96 # There will not be a collection yet 

97 collections = set(self.butler.registry.queryCollections()) 

98 self.assertEqual(collections, set([run, tag])) 

99 

100 storageClass = self.storageClassFactory.getStorageClass("StructuredCompositeReadComp") 

101 

102 # Create and register a DatasetType 

103 dimensions = self.butler.registry.dimensions.extract(["instrument", "visit"]) 

104 datasetTypeName = "test_metric_comp" 

105 self.datasetType = self._addDatasetType(datasetTypeName, dimensions, storageClass, 

106 self.butler.registry) 

107 

108 # Add needed Dimensions 

109 self.butler.registry.insertDimensionData("instrument", {"name": "DummyCamComp"}) 

110 self.butler.registry.insertDimensionData("physical_filter", {"instrument": "DummyCamComp", 

111 "name": "d-r", 

112 "band": "R"}) 

113 self.butler.registry.insertDimensionData("visit_system", {"instrument": "DummyCamComp", 

114 "id": 1, 

115 "name": "default"}) 

116 visit_start = astropy.time.Time("2020-01-01 08:00:00.123456789", scale="tai") 

117 visit_end = astropy.time.Time("2020-01-01 08:00:36.66", scale="tai") 

118 self.butler.registry.insertDimensionData("visit", 

119 {"instrument": "DummyCamComp", "id": 423, 

120 "name": "fourtwentythree", "physical_filter": "d-r", 

121 "visit_system": 1, "datetime_begin": visit_start, 

122 "datetime_end": visit_end}) 

123 self.butler.registry.insertDimensionData("visit", {"instrument": "DummyCamComp", "id": 424, 

124 "name": "fourtwentyfour", "physical_filter": "d-r", 

125 "visit_system": 1}) 

126 metric = self._makeExampleMetrics() 

127 dataId = {"instrument": "DummyCamComp", "visit": 423} 

128 ref = DatasetRef(self.datasetType, dataId, id=None) 

129 self.butler.put(metric, ref) 

130 

131 metric = self._makeExampleMetrics() 

132 dataId = {"instrument": "DummyCamComp", "visit": 424} 

133 ref = DatasetRef(self.datasetType, dataId, id=None) 

134 self.butler.put(metric, ref) 

135 

136 def tearDown(self): 

137 if os.path.exists(self.root): 

138 shutil.rmtree(self.root, ignore_errors=True) 

139 

140 def testShowURI(self): 

141 """Test for expected output with show_uri=True.""" 

142 tables = self._queryDatasets(repo=self.butlerConfigFile, show_uri=True) 

143 

144 expectedTables = ( 

145 AstropyTable(array(( 

146 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

147 self.butler.datastore.root.join( 

148 "ingest/run/test_metric_comp.data/" 

149 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")), 

150 ("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

151 self.butler.datastore.root.join( 

152 "ingest/run/test_metric_comp.data/" 

153 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")))), 

154 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

155 "visit", "URI")), 

156 AstropyTable(array(( 

157 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

158 self.butler.datastore.root.join( 

159 "ingest/run/test_metric_comp.output/" 

160 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")), 

161 ("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

162 self.butler.datastore.root.join( 

163 "ingest/run/test_metric_comp.output/" 

164 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")))), 

165 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

166 "visit", "URI")), 

167 AstropyTable(array(( 

168 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

169 self.butler.datastore.root.join( 

170 "ingest/run/test_metric_comp.summary/" 

171 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")), 

172 ("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

173 self.butler.datastore.root.join( 

174 "ingest/run/test_metric_comp.summary/" 

175 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")))), 

176 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

177 "visit", "URI")), 

178 ) 

179 

180 self._assertTablesEqual(tables, expectedTables) 

181 

182 def testNoShowURI(self): 

183 """Test for expected output without show_uri (default is False).""" 

184 tables = self._queryDatasets(repo=self.butlerConfigFile) 

185 

186 expectedTables = ( 

187 AstropyTable(array(( 

188 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"), 

189 ("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))), 

190 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit") 

191 ), 

192 ) 

193 

194 self._assertTablesEqual(tables, expectedTables) 

195 

196 def testWhere(self): 

197 """Test using the where clause to reduce the number of rows returned. 

198 """ 

199 tables = self._queryDatasets(repo=self.butlerConfigFile, where="visit=423") 

200 

201 expectedTables = ( 

202 AstropyTable(array( 

203 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423")), 

204 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit"), 

205 ), 

206 ) 

207 

208 self._assertTablesEqual(tables, expectedTables) 

209 

210 def testGlobDatasetType(self): 

211 """Test specifying dataset type.""" 

212 # Create and register an additional DatasetType 

213 dimensions = self.butler.registry.dimensions.extract(["instrument", "visit"]) 

214 datasetTypeName = "alt_test_metric_comp" 

215 storageClass = self.storageClassFactory.getStorageClass("StructuredCompositeReadComp") 

216 datasetType = self._addDatasetType(datasetTypeName, dimensions, storageClass, self.butler.registry) 

217 self.butler.registry.insertDimensionData( 

218 "visit", {"instrument": "DummyCamComp", "id": 425, 

219 "name": "fourtwentyfive", "physical_filter": "d-r", 

220 "visit_system": 1}) 

221 metric = self._makeExampleMetrics() 

222 dataId = {"instrument": "DummyCamComp", "visit": 425} 

223 ref = DatasetRef(datasetType, dataId, id=None) 

224 self.butler.put(metric, ref) 

225 

226 # verify the new dataset type increases the number of tables found: 

227 tables = self._queryDatasets(repo=self.butlerConfigFile) 

228 

229 expectedTables = ( 

230 AstropyTable(array(( 

231 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"), 

232 ("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))), 

233 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit") 

234 ), 

235 AstropyTable(array(( 

236 ("alt_test_metric_comp", "ingest/run", "3", "R", "DummyCamComp", "d-r", "1", "425"))), 

237 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit") 

238 ) 

239 ) 

240 

241 self._assertTablesEqual(tables, expectedTables) 

242 

243 def testFindFirstAndCollections(self): 

244 """Test the find-first option, and the collections option, since it 

245 is required for find-first.""" 

246 

247 # Create a new butler with a new run, and add a dataset to shadow an 

248 # existing dataset. 

249 self.butler = Butler(self.root, run="foo") 

250 metric = self._makeExampleMetrics() 

251 dataId = {"instrument": "DummyCamComp", "visit": 424} 

252 ref = DatasetRef(self.datasetType, dataId, id=None) 

253 self.butler.put(metric, ref) 

254 

255 # Verify that without find-first, duplicate datasets are returned 

256 tables = self._queryDatasets(repo=self.butlerConfigFile, 

257 collections=["foo", "ingest/run"], 

258 show_uri=True) 

259 

260 expectedTables = ( 

261 AstropyTable(array( 

262 ( 

263 ("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

264 self.butler.datastore.root.join( 

265 "foo/test_metric_comp.data/" 

266 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")), 

267 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

268 self.butler.datastore.root.join( 

269 "ingest/run/test_metric_comp.data/" 

270 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")), 

271 ("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

272 self.butler.datastore.root.join( 

273 "ingest/run/test_metric_comp.data/" 

274 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")), 

275 )), 

276 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

277 "visit", "URI")), 

278 AstropyTable(array( 

279 ( 

280 ("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

281 self.butler.datastore.root.join( 

282 "foo/test_metric_comp.output/" 

283 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")), 

284 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

285 self.butler.datastore.root.join( 

286 "ingest/run/test_metric_comp.output/" 

287 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")), 

288 ("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

289 self.butler.datastore.root.join( 

290 "ingest/run/test_metric_comp.output/" 

291 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")), 

292 )), 

293 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

294 "visit", "URI")), 

295 AstropyTable(array( 

296 ( 

297 ("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

298 self.butler.datastore.root.join( 

299 "foo/test_metric_comp.summary/" 

300 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")), 

301 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

302 self.butler.datastore.root.join( 

303 "ingest/run/test_metric_comp.summary/" 

304 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")), 

305 ("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

306 self.butler.datastore.root.join( 

307 "ingest/run/test_metric_comp.summary/" 

308 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")), 

309 )), 

310 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

311 "visit", "URI")), 

312 ) 

313 

314 self._assertTablesEqual(tables, expectedTables) 

315 

316 # Verify that with find first the duplicate dataset is eliminated and 

317 # the more recent dataset is returned. 

318 tables = self._queryDatasets(repo=self.butlerConfigFile, 

319 collections=["foo", "ingest/run"], 

320 show_uri=True, 

321 find_first=True) 

322 

323 expectedTables = ( 

324 AstropyTable(array( 

325 ( 

326 ("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

327 self.butler.datastore.root.join( 

328 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml")), 

329 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

330 self.butler.datastore.root.join( 

331 "ingest/run/test_metric_comp.data/" 

332 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")), 

333 )), 

334 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

335 "visit", "URI")), 

336 AstropyTable(array( 

337 ( 

338 ("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

339 self.butler.datastore.root.join( 

340 "foo/test_metric_comp.output/" 

341 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")), 

342 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

343 self.butler.datastore.root.join( 

344 "ingest/run/test_metric_comp.output/" 

345 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")), 

346 )), 

347 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

348 "visit", "URI")), 

349 AstropyTable(array( 

350 ( 

351 ("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

352 self.butler.datastore.root.join( 

353 "foo/test_metric_comp.summary/" 

354 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")), 

355 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

356 self.butler.datastore.root.join( 

357 "ingest/run/test_metric_comp.summary/" 

358 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")), 

359 )), 

360 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

361 "visit", "URI")), 

362 ) 

363 

364 self._assertTablesEqual(tables, expectedTables) 

365 

366 

367if __name__ == "__main__": 367 ↛ 368line 367 didn't jump to line 368, because the condition on line 367 was never true

368 unittest.main()