Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Unit tests for daf_butler CLI query-collections command. 

23""" 

24 

25from astropy.table import Table as AstropyTable 

26from numpy import array 

27import os 

28import shutil 

29import tempfile 

30import unittest 

31 

32from lsst.daf.butler import StorageClassFactory 

33from lsst.daf.butler import script 

34from lsst.daf.butler.tests import addDatasetType 

35from lsst.daf.butler.tests.utils import ButlerTestHelper, MetricTestRepo 

36 

37 

38TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

39 

40 

41class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper): 

42 

43 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

44 storageClassFactory = StorageClassFactory() 

45 

46 @staticmethod 

47 def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False): 

48 return script.queryDatasets(repo, glob, collections, where, find_first, show_uri) 

49 

50 def setUp(self): 

51 self.root = tempfile.mkdtemp(dir=TESTDIR) 

52 self.testRepo = MetricTestRepo(self.root, 

53 configFile=os.path.join(TESTDIR, "config/basic/butler.yaml")) 

54 

55 def tearDown(self): 

56 if os.path.exists(self.root): 

57 shutil.rmtree(self.root, ignore_errors=True) 

58 

59 def testShowURI(self): 

60 """Test for expected output with show_uri=True.""" 

61 tables = self._queryDatasets(repo=self.root, show_uri=True) 

62 

63 expectedTables = ( 

64 AstropyTable(array(( 

65 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

66 self.testRepo.butler.datastore.root.join( 

67 "ingest/run/test_metric_comp.data/" 

68 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")), 

69 ("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

70 self.testRepo.butler.datastore.root.join( 

71 "ingest/run/test_metric_comp.data/" 

72 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")))), 

73 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

74 "visit", "URI")), 

75 AstropyTable(array(( 

76 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

77 self.testRepo.butler.datastore.root.join( 

78 "ingest/run/test_metric_comp.output/" 

79 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")), 

80 ("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

81 self.testRepo.butler.datastore.root.join( 

82 "ingest/run/test_metric_comp.output/" 

83 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")))), 

84 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

85 "visit", "URI")), 

86 AstropyTable(array(( 

87 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

88 self.testRepo.butler.datastore.root.join( 

89 "ingest/run/test_metric_comp.summary/" 

90 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")), 

91 ("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

92 self.testRepo.butler.datastore.root.join( 

93 "ingest/run/test_metric_comp.summary/" 

94 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")))), 

95 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

96 "visit", "URI")), 

97 ) 

98 

99 self.assertAstropyTablesEqual(tables, expectedTables) 

100 

101 def testNoShowURI(self): 

102 """Test for expected output without show_uri (default is False).""" 

103 tables = self._queryDatasets(repo=self.root) 

104 

105 expectedTables = ( 

106 AstropyTable(array(( 

107 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"), 

108 ("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))), 

109 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit") 

110 ), 

111 ) 

112 

113 self.assertAstropyTablesEqual(tables, expectedTables) 

114 

115 def testWhere(self): 

116 """Test using the where clause to reduce the number of rows returned. 

117 """ 

118 tables = self._queryDatasets(repo=self.root, where="instrument='DummyCamComp' AND visit=423") 

119 

120 expectedTables = ( 

121 AstropyTable(array( 

122 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423")), 

123 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit"), 

124 ), 

125 ) 

126 

127 self.assertAstropyTablesEqual(tables, expectedTables) 

128 

129 def testGlobDatasetType(self): 

130 """Test specifying dataset type.""" 

131 # Create and register an additional DatasetType 

132 

133 self.testRepo.butler.registry.insertDimensionData("visit", 

134 {"instrument": "DummyCamComp", "id": 425, 

135 "name": "fourtwentyfive", "physical_filter": "d-r", 

136 "visit_system": 1}) 

137 

138 datasetType = addDatasetType(self.testRepo.butler, 

139 "alt_test_metric_comp", 

140 ("instrument", "visit"), 

141 "StructuredCompositeReadComp") 

142 

143 self.testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType) 

144 

145 # verify the new dataset type increases the number of tables found: 

146 tables = self._queryDatasets(repo=self.root) 

147 

148 expectedTables = ( 

149 AstropyTable(array(( 

150 ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"), 

151 ("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))), 

152 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit") 

153 ), 

154 AstropyTable(array(( 

155 ("alt_test_metric_comp", "ingest/run", "3", "R", "DummyCamComp", "d-r", "1", "425"))), 

156 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit") 

157 ) 

158 ) 

159 

160 self.assertAstropyTablesEqual(tables, expectedTables) 

161 

162 def testFindFirstAndCollections(self): 

163 """Test the find-first option, and the collections option, since it 

164 is required for find-first.""" 

165 

166 # Add a new run, and add a dataset to shadow an existing dataset. 

167 self.testRepo.addDataset(run="foo", 

168 dataId={"instrument": "DummyCamComp", "visit": 424}) 

169 

170 # Verify that without find-first, duplicate datasets are returned 

171 tables = self._queryDatasets(repo=self.root, 

172 collections=["foo", "ingest/run"], 

173 show_uri=True) 

174 

175 expectedTables = ( 

176 AstropyTable(array( 

177 ( 

178 ("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

179 self.testRepo.butler.datastore.root.join( 

180 "foo/test_metric_comp.data/" 

181 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")), 

182 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

183 self.testRepo.butler.datastore.root.join( 

184 "ingest/run/test_metric_comp.data/" 

185 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")), 

186 ("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

187 self.testRepo.butler.datastore.root.join( 

188 "ingest/run/test_metric_comp.data/" 

189 "test_metric_comp_v00000424_fDummyCamComp_data.yaml")), 

190 )), 

191 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

192 "visit", "URI")), 

193 AstropyTable(array( 

194 ( 

195 ("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

196 self.testRepo.butler.datastore.root.join( 

197 "foo/test_metric_comp.output/" 

198 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")), 

199 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

200 self.testRepo.butler.datastore.root.join( 

201 "ingest/run/test_metric_comp.output/" 

202 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")), 

203 ("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

204 self.testRepo.butler.datastore.root.join( 

205 "ingest/run/test_metric_comp.output/" 

206 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")), 

207 )), 

208 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

209 "visit", "URI")), 

210 AstropyTable(array( 

211 ( 

212 ("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

213 self.testRepo.butler.datastore.root.join( 

214 "foo/test_metric_comp.summary/" 

215 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")), 

216 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

217 self.testRepo.butler.datastore.root.join( 

218 "ingest/run/test_metric_comp.summary/" 

219 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")), 

220 ("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", 

221 self.testRepo.butler.datastore.root.join( 

222 "ingest/run/test_metric_comp.summary/" 

223 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")), 

224 )), 

225 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

226 "visit", "URI")), 

227 ) 

228 

229 self.assertAstropyTablesEqual(tables, expectedTables) 

230 

231 # Verify that with find first the duplicate dataset is eliminated and 

232 # the more recent dataset is returned. 

233 tables = self._queryDatasets(repo=self.root, 

234 collections=["foo", "ingest/run"], 

235 show_uri=True, 

236 find_first=True) 

237 

238 expectedTables = ( 

239 AstropyTable(array( 

240 ( 

241 ("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

242 self.testRepo.butler.datastore.root.join( 

243 "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml")), 

244 ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

245 self.testRepo.butler.datastore.root.join( 

246 "ingest/run/test_metric_comp.data/" 

247 "test_metric_comp_v00000423_fDummyCamComp_data.yaml")), 

248 )), 

249 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

250 "visit", "URI")), 

251 AstropyTable(array( 

252 ( 

253 ("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

254 self.testRepo.butler.datastore.root.join( 

255 "foo/test_metric_comp.output/" 

256 "test_metric_comp_v00000424_fDummyCamComp_output.yaml")), 

257 ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

258 self.testRepo.butler.datastore.root.join( 

259 "ingest/run/test_metric_comp.output/" 

260 "test_metric_comp_v00000423_fDummyCamComp_output.yaml")), 

261 )), 

262 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

263 "visit", "URI")), 

264 AstropyTable(array( 

265 ( 

266 ("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", 

267 self.testRepo.butler.datastore.root.join( 

268 "foo/test_metric_comp.summary/" 

269 "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")), 

270 ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", 

271 self.testRepo.butler.datastore.root.join( 

272 "ingest/run/test_metric_comp.summary/" 

273 "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")), 

274 )), 

275 names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", 

276 "visit", "URI")), 

277 ) 

278 

279 self.assertAstropyTablesEqual(tables, expectedTables) 

280 

281 

282if __name__ == "__main__": 282 ↛ 283line 282 didn't jump to line 283, because the condition on line 282 was never true

283 unittest.main()