Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for daf_butler CLI query-collections command. 

23""" 

24 

25import astropy 

26from astropy.table import Table as AstropyTable 

27from numpy import array 

28import os 

29import shutil 

30import tempfile 

31import unittest 

32 

33from lsst.daf.butler import ( 

34 Butler, 

35 Config, 

36 DatasetRef, 

37 DatasetType, 

38 StorageClassFactory 

39) 

40from lsst.daf.butler import script 

41from lsst.daf.butler.tests import MetricsExample 

42from lsst.daf.butler.tests.utils import ButlerTestHelper 

43 

44 

45TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

46 

47 

48class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper): 

49 

50 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

51 storageClassFactory = StorageClassFactory() 

52 

53 @staticmethod 

54 def _makeExampleMetrics(): 

55 return MetricsExample({"AM1": 5.2, "AM2": 30.6}, 

56 {"a": [1, 2, 3], 

57 "b": {"blue": 5, "red": "green"}}, 

58 [563, 234, 456.7, 752, 8, 9, 27]) 

59 

60 @staticmethod 

61 def _addDatasetType(datasetTypeName, dimensions, storageClass, registry): 

62 """Create a DatasetType and register it 

63 """ 

64 datasetType = DatasetType(datasetTypeName, dimensions, storageClass) 

65 registry.registerDatasetType(datasetType) 

66 return datasetType 

67 

68 @staticmethod 

69 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False): 

70 return script.queryDatasets(repo, glob, collections, where, find_first, show_uri) 

71 

72 def setUp(self): 

73 self.root = tempfile.mkdtemp(dir=TESTDIR) 

74 Butler.makeRepo(self.root, config=Config(self.configFile)) 

75 self.butlerConfigFile = os.path.join(self.root, "butler.yaml") 

76 self.storageClassFactory.addFromConfig(self.configFile) 

77 

78 # New datasets will be added to run and tag, but we will only look in 

79 # tag when looking up datasets. 

80 run = "ingest/run" 

81 tag = "ingest" 

82 self.butler = Butler(self.butlerConfigFile, run=run, collections=[tag], tags=[tag]) 

83 

84 # There will not be a collection yet 

85 collections = set(self.butler.registry.queryCollections()) 

86 self.assertEqual(collections, set([run, tag])) 

87 

88 storageClass = self.storageClassFactory.getStorageClass("StructuredCompositeReadComp") 

89 

90 # Create and register a DatasetType 

91 dimensions = self.butler.registry.dimensions.extract(["instrument", "visit"]) 

92 datasetTypeName = "test_metric_comp" 

93 self.datasetType = self._addDatasetType(datasetTypeName, dimensions, storageClass, 

94 self.butler.registry) 

95 

96 # Add needed Dimensions 

97 self.butler.registry.insertDimensionData("instrument", {"name": "DummyCamComp"}) 

98 self.butler.registry.insertDimensionData("physical_filter", {"instrument": "DummyCamComp", 

99 "name": "d-r", 

100 "band": "R"}) 

101 self.butler.registry.insertDimensionData("visit_system", {"instrument": "DummyCamComp", 

102 "id": 1, 

103 "name": "default"}) 

104 visit_start = astropy.time.Time("2020-01-01 08:00:00.123456789", scale="tai") 

105 visit_end = astropy.time.Time("2020-01-01 08:00:36.66", scale="tai") 

106 self.butler.registry.insertDimensionData("visit", 

107 {"instrument": "DummyCamComp", "id": 423, 

108 "name": "fourtwentythree", "physical_filter": "d-r", 

109 "visit_system": 1, "datetime_begin": visit_start, 

110 "datetime_end": visit_end}) 

111 self.butler.registry.insertDimensionData("visit", {"instrument": "DummyCamComp", "id": 424, 

112 "name": "fourtwentyfour", "physical_filter": "d-r", 

113 "visit_system": 1}) 

114 metric = self._makeExampleMetrics() 

115 dataId = {"instrument": "DummyCamComp", "visit": 423} 

116 ref = DatasetRef(self.datasetType, dataId, id=None) 

117 self.butler.put(metric, ref) 

118 

119 metric = self._makeExampleMetrics() 

120 dataId = {"instrument": "DummyCamComp", "visit": 424} 

121 ref = DatasetRef(self.datasetType, dataId, id=None) 

122 self.butler.put(metric, ref) 

123 

124 def tearDown(self): 

125 if os.path.exists(self.root): 

126 shutil.rmtree(self.root, ignore_errors=True) 

127 

128 def testShowURI(self): 

129 """Test for expected output with show_uri=True.""" 

130 tables = self._queryDatasets(repo=self.butlerConfigFile, show_uri=True) 

131 

132 expectedTables = ( 

133 AstropyTable(array(( 

134 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

135 self.butler.datastore.root.join( 

136 "ingest/run/test_metric_comp.data/" 

137 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")), 

138 ("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

139 self.butler.datastore.root.join( 

140 "ingest/run/test_metric_comp.data/" 

141 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")))), 

142 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

143 "visit", "URI")), 

144 AstropyTable(array(( 

145 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

146 self.butler.datastore.root.join( 

147 "ingest/run/test_metric_comp.output/" 

148 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")), 

149 ("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

150 self.butler.datastore.root.join( 

151 "ingest/run/test_metric_comp.output/" 

152 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")))), 

153 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

154 "visit", "URI")), 

155 AstropyTable(array(( 

156 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

157 self.butler.datastore.root.join( 

158 "ingest/run/test_metric_comp.summary/" 

159 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")), 

160 ("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

161 self.butler.datastore.root.join( 

162 "ingest/run/test_metric_comp.summary/" 

163 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")))), 

164 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

165 "visit", "URI")), 

166 ) 

167 

168 self.assertAstropyTablesEqual(tables, expectedTables) 

169 

170 def testNoShowURI(self): 

171 """Test for expected output without show_uri (default is False).""" 

172 tables = self._queryDatasets(repo=self.butlerConfigFile) 

173 

174 expectedTables = ( 

175 AstropyTable(array(( 

176 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"), 

177 ("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))), 

178 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit") 

179 ), 

180 ) 

181 

182 self.assertAstropyTablesEqual(tables, expectedTables) 

183 

184 def testWhere(self): 

185 """Test using the where clause to reduce the number of rows returned. 

186 """ 

187 tables = self._queryDatasets(repo=self.butlerConfigFile, where="visit=423") 

188 

189 expectedTables = ( 

190 AstropyTable(array( 

191 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423")), 

192 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit"), 

193 ), 

194 ) 

195 

196 self.assertAstropyTablesEqual(tables, expectedTables) 

197 

198 def testGlobDatasetType(self): 

199 """Test specifying dataset type.""" 

200 # Create and register an additional DatasetType 

201 dimensions = self.butler.registry.dimensions.extract(["instrument", "visit"]) 

202 datasetTypeName = "alt_test_metric_comp" 

203 storageClass = self.storageClassFactory.getStorageClass("StructuredCompositeReadComp") 

204 datasetType = self._addDatasetType(datasetTypeName, dimensions, storageClass, self.butler.registry) 

205 self.butler.registry.insertDimensionData( 

206 "visit", {"instrument": "DummyCamComp", "id": 425, 

207 "name": "fourtwentyfive", "physical_filter": "d-r", 

208 "visit_system": 1}) 

209 metric = self._makeExampleMetrics() 

210 dataId = {"instrument": "DummyCamComp", "visit": 425} 

211 ref = DatasetRef(datasetType, dataId, id=None) 

212 self.butler.put(metric, ref) 

213 

214 # verify the new dataset type increases the number of tables found: 

215 tables = self._queryDatasets(repo=self.butlerConfigFile) 

216 

217 expectedTables = ( 

218 AstropyTable(array(( 

219 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"), 

220 ("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))), 

221 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit") 

222 ), 

223 AstropyTable(array(( 

224 ("alt_test_metric_comp", "ingest/run", "3", "R", "DummyCamComp", "d-r", "1", "425"))), 

225 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit") 

226 ) 

227 ) 

228 

229 self.assertAstropyTablesEqual(tables, expectedTables) 

230 

231 def testFindFirstAndCollections(self): 

232 """Test the find-first option, and the collections option, since it 

233 is required for find-first.""" 

234 

235 # Create a new butler with a new run, and add a dataset to shadow an 

236 # existing dataset. 

237 self.butler = Butler(self.root, run="foo") 

238 metric = self._makeExampleMetrics() 

239 dataId = {"instrument": "DummyCamComp", "visit": 424} 

240 ref = DatasetRef(self.datasetType, dataId, id=None) 

241 self.butler.put(metric, ref) 

242 

243 # Verify that without find-first, duplicate datasets are returned 

244 tables = self._queryDatasets(repo=self.butlerConfigFile, 

245 collections=["foo", "ingest/run"], 

246 show_uri=True) 

247 

248 expectedTables = ( 

249 AstropyTable(array( 

250 ( 

251 ("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

252 self.butler.datastore.root.join( 

253 "foo/test_metric_comp.data/" 

254 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")), 

255 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

256 self.butler.datastore.root.join( 

257 "ingest/run/test_metric_comp.data/" 

258 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")), 

259 ("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

260 self.butler.datastore.root.join( 

261 "ingest/run/test_metric_comp.data/" 

262 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")), 

263 )), 

264 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

265 "visit", "URI")), 

266 AstropyTable(array( 

267 ( 

268 ("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

269 self.butler.datastore.root.join( 

270 "foo/test_metric_comp.output/" 

271 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")), 

272 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

273 self.butler.datastore.root.join( 

274 "ingest/run/test_metric_comp.output/" 

275 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")), 

276 ("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

277 self.butler.datastore.root.join( 

278 "ingest/run/test_metric_comp.output/" 

279 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")), 

280 )), 

281 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

282 "visit", "URI")), 

283 AstropyTable(array( 

284 ( 

285 ("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

286 self.butler.datastore.root.join( 

287 "foo/test_metric_comp.summary/" 

288 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")), 

289 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

290 self.butler.datastore.root.join( 

291 "ingest/run/test_metric_comp.summary/" 

292 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")), 

293 ("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

294 self.butler.datastore.root.join( 

295 "ingest/run/test_metric_comp.summary/" 

296 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")), 

297 )), 

298 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

299 "visit", "URI")), 

300 ) 

301 

302 self.assertAstropyTablesEqual(tables, expectedTables) 

303 

304 # Verify that with find first the duplicate dataset is eliminated and 

305 # the more recent dataset is returned. 

306 tables = self._queryDatasets(repo=self.butlerConfigFile, 

307 collections=["foo", "ingest/run"], 

308 show_uri=True, 

309 find_first=True) 

310 

311 expectedTables = ( 

312 AstropyTable(array( 

313 ( 

314 ("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

315 self.butler.datastore.root.join( 

316 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml")), 

317 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

318 self.butler.datastore.root.join( 

319 "ingest/run/test_metric_comp.data/" 

320 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")), 

321 )), 

322 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

323 "visit", "URI")), 

324 AstropyTable(array( 

325 ( 

326 ("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

327 self.butler.datastore.root.join( 

328 "foo/test_metric_comp.output/" 

329 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")), 

330 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

331 self.butler.datastore.root.join( 

332 "ingest/run/test_metric_comp.output/" 

333 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")), 

334 )), 

335 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

336 "visit", "URI")), 

337 AstropyTable(array( 

338 ( 

339 ("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

340 self.butler.datastore.root.join( 

341 "foo/test_metric_comp.summary/" 

342 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")), 

343 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

344 self.butler.datastore.root.join( 

345 "ingest/run/test_metric_comp.summary/" 

346 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")), 

347 )), 

348 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

349 "visit", "URI")), 

350 ) 

351 

352 self.assertAstropyTablesEqual(tables, expectedTables) 

353 

354 

355if __name__ == "__main__": 355 ↛ 356line 355 didn't jump to line 356, because the condition on line 355 was never true

356 unittest.main()