Coverage for tests/test_templates.py: 9%

171 statements  

« prev     ^ index     » next       coverage.py v7.2.3, created at 2023-04-22 02:18 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Test file name templating.""" 

23 

24import os.path 

25import unittest 

26 

27from lsst.daf.butler import ( 

28 DatasetRef, 

29 DatasetType, 

30 DimensionGraph, 

31 DimensionUniverse, 

32 FileTemplate, 

33 FileTemplates, 

34 FileTemplatesConfig, 

35 FileTemplateValidationError, 

36 StorageClass, 

37) 

38 

39TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

40 

41PlaceHolder = StorageClass("PlaceHolder") 

42 

43 

44class TestFileTemplates(unittest.TestCase): 

45 """Test creation of paths from templates.""" 

46 

47 def makeDatasetRef( 

48 self, datasetTypeName, dataId=None, storageClassName="DefaultStorageClass", run="run2", conform=True 

49 ): 

50 """Make a simple DatasetRef""" 

51 if dataId is None: 

52 dataId = self.dataId 

53 

54 # Pretend we have a parent if this looks like a composite 

55 compositeName, componentName = DatasetType.splitDatasetTypeName(datasetTypeName) 

56 parentStorageClass = PlaceHolder if componentName else None 

57 

58 datasetType = DatasetType( 

59 datasetTypeName, 

60 DimensionGraph(self.universe, names=dataId.keys()), 

61 StorageClass(storageClassName), 

62 parentStorageClass=parentStorageClass, 

63 ) 

64 return DatasetRef(datasetType, dataId, id=1, run=run, conform=conform) 

65 

66 def setUp(self): 

67 self.universe = DimensionUniverse() 

68 self.dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "Most Amazing U Filter Ever"} 

69 

70 def assertTemplate(self, template, answer, ref): 

71 fileTmpl = FileTemplate(template) 

72 path = fileTmpl.format(ref) 

73 self.assertEqual(path, answer) 

74 

75 def testBasic(self): 

76 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}" 

77 self.assertTemplate( 

78 tmplstr, 

79 "run2/calexp/00052/Most_Amazing_U_Filter_Ever", 

80 self.makeDatasetRef("calexp", conform=False), 

81 ) 

82 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail" 

83 self.assertTemplate( 

84 tmplstr, 

85 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail", 

86 self.makeDatasetRef("calexp", conform=False), 

87 ) 

88 

89 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}" 

90 self.assertTemplate( 

91 tmplstr, 

92 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run2", 

93 self.makeDatasetRef("calexp", conform=False), 

94 ) 

95 self.assertTemplate( 

96 tmplstr, 

97 "run_2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2", 

98 self.makeDatasetRef("calexp", run="run/2", conform=False), 

99 ) 

100 

101 # Check that the id is sufficient without any other information. 

102 self.assertTemplate("{id}", "1", self.makeDatasetRef("calexp", run="run2", conform=False)) 

103 

104 self.assertTemplate("{run}/{id}", "run2/1", self.makeDatasetRef("calexp", run="run2", conform=False)) 

105 

106 self.assertTemplate( 

107 "fixed/{id}", 

108 "fixed/1", 

109 self.makeDatasetRef("calexp", run="run2", conform=False), 

110 ) 

111 

112 self.assertTemplate( 

113 "fixed/{id}_{physical_filter}", 

114 "fixed/1_Most_Amazing_U_Filter_Ever", 

115 self.makeDatasetRef("calexp", run="run2", conform=False), 

116 ) 

117 

118 # Retain any "/" in run 

119 tmplstr = "{run:/}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}" 

120 self.assertTemplate( 

121 tmplstr, 

122 "run/2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2", 

123 self.makeDatasetRef("calexp", run="run/2", conform=False), 

124 ) 

125 

126 # Check that "." are replaced in the file basename, but not directory. 

127 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "g.10"} 

128 self.assertTemplate( 

129 tmplstr, 

130 "run.2/calexp/00052/g_10-trail-run_2", 

131 self.makeDatasetRef("calexp", run="run.2", dataId=dataId, conform=False), 

132 ) 

133 

134 with self.assertRaises(FileTemplateValidationError): 

135 FileTemplate("no fields at all") 

136 

137 with self.assertRaises(FileTemplateValidationError): 

138 FileTemplate("{visit}") 

139 

140 with self.assertRaises(FileTemplateValidationError): 

141 FileTemplate("{run}_{datasetType}") 

142 

143 with self.assertRaises(FileTemplateValidationError): 

144 FileTemplate("{id}/fixed") 

145 

146 def testRunOrCollectionNeeded(self): 

147 tmplstr = "{datasetType}/{visit:05d}/{physical_filter}" 

148 with self.assertRaises(FileTemplateValidationError): 

149 self.assertTemplate(tmplstr, "run2/calexp/00052/U", self.makeDatasetRef("calexp")) 

150 

151 def testNoRecord(self): 

152 # Attaching records is not possible in this test code but we can check 

153 # that a missing record when a metadata entry has been requested 

154 # does fail. 

155 tmplstr = "{run}/{datasetType}/{visit.name}/{physical_filter}" 

156 with self.assertRaises(RuntimeError) as cm: 

157 self.assertTemplate(tmplstr, "", self.makeDatasetRef("calexp")) 

158 self.assertIn("No metadata", str(cm.exception)) 

159 

160 def testOptional(self): 

161 """Optional units in templates.""" 

162 ref = self.makeDatasetRef("calexp", conform=False) 

163 tmplstr = "{run}/{datasetType}/v{visit:05d}_f{physical_filter:?}" 

164 self.assertTemplate( 

165 tmplstr, 

166 "run2/calexp/v00052_fMost_Amazing_U_Filter_Ever", 

167 self.makeDatasetRef("calexp", conform=False), 

168 ) 

169 

170 du = {"visit": 48, "tract": 265, "skymap": "big", "instrument": "dummy"} 

171 self.assertTemplate(tmplstr, "run2/calexpT/v00048", self.makeDatasetRef("calexpT", du, conform=False)) 

172 

173 # Ensure that this returns a relative path even if the first field 

174 # is optional 

175 tmplstr = "{run}/{tract:?}/{visit:?}/f{physical_filter}" 

176 self.assertTemplate(tmplstr, "run2/52/fMost_Amazing_U_Filter_Ever", ref) 

177 

178 # Ensure that // from optionals are converted to singles 

179 tmplstr = "{run}/{datasetType}/{patch:?}/{tract:?}/f{physical_filter}" 

180 self.assertTemplate(tmplstr, "run2/calexp/fMost_Amazing_U_Filter_Ever", ref) 

181 

182 # Optionals with some text between fields 

183 tmplstr = "{run}/{datasetType}/p{patch:?}_t{tract:?}/f{physical_filter}" 

184 self.assertTemplate(tmplstr, "run2/calexp/p/fMost_Amazing_U_Filter_Ever", ref) 

185 tmplstr = "{run}/{datasetType}/p{patch:?}_t{visit:04d?}/f{physical_filter}" 

186 self.assertTemplate(tmplstr, "run2/calexp/p_t0052/fMost_Amazing_U_Filter_Ever", ref) 

187 

188 def testComponent(self): 

189 """Test handling of components in templates.""" 

190 refMetricOutput = self.makeDatasetRef("metric.output") 

191 refMetric = self.makeDatasetRef("metric") 

192 refMaskedImage = self.makeDatasetRef("calexp.maskedimage.variance") 

193 refWcs = self.makeDatasetRef("calexp.wcs") 

194 

195 tmplstr = "{run}_c_{component}_v{visit}" 

196 self.assertTemplate(tmplstr, "run2_c_output_v52", refMetricOutput) 

197 

198 # We want this template to have both a directory and basename, to 

199 # test that the right parts of the output are replaced. 

200 tmplstr = "{component:?}/{run}_{component:?}_{visit}" 

201 self.assertTemplate(tmplstr, "run2_52", refMetric) 

202 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput) 

203 self.assertTemplate(tmplstr, "maskedimage.variance/run2_maskedimage_variance_52", refMaskedImage) 

204 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput) 

205 

206 # Providing a component but not using it 

207 tmplstr = "{run}/{datasetType}/v{visit:05d}" 

208 with self.assertRaises(KeyError): 

209 self.assertTemplate(tmplstr, "", refWcs) 

210 

211 def testFields(self): 

212 # Template, mandatory fields, optional non-special fields, 

213 # special fields, optional special fields 

214 testData = ( 

215 ( 

216 "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail", 

217 set(["visit", "physical_filter"]), 

218 set(), 

219 set(["run", "datasetType"]), 

220 set(), 

221 ), 

222 ( 

223 "{run}/{component:?}_{visit}", 

224 set(["visit"]), 

225 set(), 

226 set(["run"]), 

227 set(["component"]), 

228 ), 

229 ( 

230 "{run}/{component:?}_{visit:?}_{physical_filter}_{instrument}_{datasetType}", 

231 set(["physical_filter", "instrument"]), 

232 set(["visit"]), 

233 set(["run", "datasetType"]), 

234 set(["component"]), 

235 ), 

236 ) 

237 for tmplstr, mandatory, optional, special, optionalSpecial in testData: 

238 with self.subTest(template=tmplstr): 

239 tmpl = FileTemplate(tmplstr) 

240 fields = tmpl.fields() 

241 self.assertEqual(fields, mandatory) 

242 fields = tmpl.fields(optionals=True) 

243 self.assertEqual(fields, mandatory | optional) 

244 fields = tmpl.fields(specials=True) 

245 self.assertEqual(fields, mandatory | special) 

246 fields = tmpl.fields(specials=True, optionals=True) 

247 self.assertEqual(fields, mandatory | special | optional | optionalSpecial) 

248 

249 def testSimpleConfig(self): 

250 """Test reading from config file""" 

251 configRoot = os.path.join(TESTDIR, "config", "templates") 

252 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml")) 

253 templates = FileTemplates(config1, universe=self.universe) 

254 ref = self.makeDatasetRef("calexp") 

255 tmpl = templates.getTemplate(ref) 

256 self.assertIsInstance(tmpl, FileTemplate) 

257 

258 # This config file should not allow defaulting 

259 ref2 = self.makeDatasetRef("unknown") 

260 with self.assertRaises(KeyError): 

261 templates.getTemplate(ref2) 

262 

263 # This should fall through the datasetTypeName check and use 

264 # StorageClass instead 

265 ref3 = self.makeDatasetRef("unknown2", storageClassName="StorageClassX") 

266 tmplSc = templates.getTemplate(ref3) 

267 self.assertIsInstance(tmplSc, FileTemplate) 

268 

269 # Try with a component: one with defined formatter and one without 

270 refWcs = self.makeDatasetRef("calexp.wcs") 

271 refImage = self.makeDatasetRef("calexp.image") 

272 tmplCalexp = templates.getTemplate(ref) 

273 tmplWcs = templates.getTemplate(refWcs) # Should be special 

274 tmpl_image = templates.getTemplate(refImage) 

275 self.assertIsInstance(tmplCalexp, FileTemplate) 

276 self.assertIsInstance(tmpl_image, FileTemplate) 

277 self.assertIsInstance(tmplWcs, FileTemplate) 

278 self.assertEqual(tmplCalexp, tmpl_image) 

279 self.assertNotEqual(tmplCalexp, tmplWcs) 

280 

281 # Check dimensions lookup order. 

282 # The order should be: dataset type name, dimension, storage class 

283 # This one will not match name but might match storage class. 

284 # It should match dimensions 

285 refDims = self.makeDatasetRef( 

286 "nomatch", dataId={"instrument": "LSST", "physical_filter": "z"}, storageClassName="StorageClassX" 

287 ) 

288 tmplDims = templates.getTemplate(refDims) 

289 self.assertIsInstance(tmplDims, FileTemplate) 

290 self.assertNotEqual(tmplDims, tmplSc) 

291 

292 # Test that instrument overrides retrieve specialist templates 

293 refPvi = self.makeDatasetRef("pvi") 

294 refPviHsc = self.makeDatasetRef("pvi", dataId={"instrument": "HSC", "physical_filter": "z"}) 

295 refPviLsst = self.makeDatasetRef("pvi", dataId={"instrument": "LSST", "physical_filter": "z"}) 

296 

297 tmplPvi = templates.getTemplate(refPvi) 

298 tmplPviHsc = templates.getTemplate(refPviHsc) 

299 tmplPviLsst = templates.getTemplate(refPviLsst) 

300 self.assertEqual(tmplPvi, tmplPviLsst) 

301 self.assertNotEqual(tmplPvi, tmplPviHsc) 

302 

303 # Have instrument match and dimensions look up with no name match 

304 refNoPviHsc = self.makeDatasetRef( 

305 "pvix", dataId={"instrument": "HSC", "physical_filter": "z"}, storageClassName="StorageClassX" 

306 ) 

307 tmplNoPviHsc = templates.getTemplate(refNoPviHsc) 

308 self.assertNotEqual(tmplNoPviHsc, tmplDims) 

309 self.assertNotEqual(tmplNoPviHsc, tmplPviHsc) 

310 

311 # Format config file with defaulting 

312 config2 = FileTemplatesConfig(os.path.join(configRoot, "templates-withdefault.yaml")) 

313 templates = FileTemplates(config2, universe=self.universe) 

314 tmpl = templates.getTemplate(ref2) 

315 self.assertIsInstance(tmpl, FileTemplate) 

316 

317 # Format config file with bad format string 

318 with self.assertRaises(FileTemplateValidationError): 

319 FileTemplates(os.path.join(configRoot, "templates-bad.yaml"), universe=self.universe) 

320 

321 # Config file with no defaulting mentioned 

322 config3 = os.path.join(configRoot, "templates-nodefault2.yaml") 

323 templates = FileTemplates(config3, universe=self.universe) 

324 with self.assertRaises(KeyError): 

325 templates.getTemplate(ref2) 

326 

327 # Try again but specify a default in the constructor 

328 default = "{run}/{datasetType}/{physical_filter}" 

329 templates = FileTemplates(config3, default=default, universe=self.universe) 

330 tmpl = templates.getTemplate(ref2) 

331 self.assertEqual(tmpl.template, default) 

332 

333 def testValidation(self): 

334 configRoot = os.path.join(TESTDIR, "config", "templates") 

335 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml")) 

336 templates = FileTemplates(config1, universe=self.universe) 

337 

338 entities = {} 

339 entities["calexp"] = self.makeDatasetRef( 

340 "calexp", 

341 storageClassName="StorageClassX", 

342 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 52}, 

343 ) 

344 

345 with self.assertLogs(level="WARNING") as cm: 

346 templates.validateTemplates(entities.values(), logFailures=True) 

347 self.assertIn("Unchecked keys", cm.output[0]) 

348 self.assertIn("StorageClassX", cm.output[0]) 

349 

350 entities["pvi"] = self.makeDatasetRef( 

351 "pvi", storageClassName="StorageClassX", dataId={"instrument": "dummy", "physical_filter": "i"} 

352 ) 

353 entities["StorageClassX"] = self.makeDatasetRef( 

354 "storageClass", storageClassName="StorageClassX", dataId={"instrument": "dummy", "visit": 2} 

355 ) 

356 entities["calexp.wcs"] = self.makeDatasetRef( 

357 "calexp.wcs", 

358 storageClassName="StorageClassX", 

359 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 23}, 

360 conform=False, 

361 ) 

362 

363 entities["instrument+physical_filter"] = self.makeDatasetRef( 

364 "filter_inst", 

365 storageClassName="StorageClassX", 

366 dataId={"physical_filter": "i", "instrument": "SCUBA"}, 

367 ) 

368 entities["hsc+pvi"] = self.makeDatasetRef( 

369 "pvi", storageClassName="StorageClassX", dataId={"physical_filter": "i", "instrument": "HSC"} 

370 ) 

371 

372 entities["hsc+instrument+physical_filter"] = self.makeDatasetRef( 

373 "filter_inst", 

374 storageClassName="StorageClassX", 

375 dataId={"physical_filter": "i", "instrument": "HSC"}, 

376 ) 

377 

378 entities["metric6"] = self.makeDatasetRef( 

379 "filter_inst", 

380 storageClassName="Integer", 

381 dataId={"physical_filter": "i", "instrument": "HSC"}, 

382 ) 

383 

384 templates.validateTemplates(entities.values(), logFailures=True) 

385 

386 # Rerun but with a failure 

387 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX", dataId={"band": "i"}) 

388 with self.assertRaises(FileTemplateValidationError): 

389 with self.assertLogs(level="FATAL"): 

390 templates.validateTemplates(entities.values(), logFailures=True) 

391 

392 

393if __name__ == "__main__": 

394 unittest.main()