Coverage for tests/test_templates.py: 9%

175 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-23 09:30 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Test file name templating.""" 

23 

24import os.path 

25import unittest 

26 

27from lsst.daf.butler import ( 

28 DataCoordinate, 

29 DatasetRef, 

30 DatasetType, 

31 DimensionGraph, 

32 DimensionUniverse, 

33 FileTemplate, 

34 FileTemplates, 

35 FileTemplatesConfig, 

36 FileTemplateValidationError, 

37 StorageClass, 

38) 

39 

40TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

41 

42PlaceHolder = StorageClass("PlaceHolder") 

43 

44 

45class TestFileTemplates(unittest.TestCase): 

46 """Test creation of paths from templates.""" 

47 

48 def makeDatasetRef( 

49 self, datasetTypeName, dataId=None, storageClassName="DefaultStorageClass", run="run2", conform=True 

50 ): 

51 """Make a simple DatasetRef""" 

52 

53 if dataId is None: 

54 dataId = self.dataId 

55 if "physical_filter" in dataId and "band" not in dataId: 

56 dataId["band"] = "b" # Add fake band. 

57 dimensions = DimensionGraph(self.universe, names=dataId.keys()) 

58 dataId = DataCoordinate.standardize(dataId, graph=dimensions) 

59 

60 # Pretend we have a parent if this looks like a composite 

61 compositeName, componentName = DatasetType.splitDatasetTypeName(datasetTypeName) 

62 parentStorageClass = PlaceHolder if componentName else None 

63 

64 datasetType = DatasetType( 

65 datasetTypeName, 

66 dimensions, 

67 StorageClass(storageClassName), 

68 parentStorageClass=parentStorageClass, 

69 ) 

70 return DatasetRef(datasetType, dataId, id=1, run=run, conform=conform) 

71 

72 def setUp(self): 

73 self.universe = DimensionUniverse() 

74 self.dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "Most Amazing U Filter Ever"} 

75 

76 def assertTemplate(self, template, answer, ref): 

77 fileTmpl = FileTemplate(template) 

78 path = fileTmpl.format(ref) 

79 self.assertEqual(path, answer) 

80 

81 def testBasic(self): 

82 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}" 

83 self.assertTemplate( 

84 tmplstr, 

85 "run2/calexp/00052/Most_Amazing_U_Filter_Ever", 

86 self.makeDatasetRef("calexp"), 

87 ) 

88 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail" 

89 self.assertTemplate( 

90 tmplstr, 

91 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail", 

92 self.makeDatasetRef("calexp"), 

93 ) 

94 

95 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}" 

96 self.assertTemplate( 

97 tmplstr, 

98 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run2", 

99 self.makeDatasetRef("calexp"), 

100 ) 

101 self.assertTemplate( 

102 tmplstr, 

103 "run_2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2", 

104 self.makeDatasetRef("calexp", run="run/2"), 

105 ) 

106 

107 # Check that the id is sufficient without any other information. 

108 self.assertTemplate("{id}", "1", self.makeDatasetRef("calexp", run="run2")) 

109 

110 self.assertTemplate("{run}/{id}", "run2/1", self.makeDatasetRef("calexp", run="run2")) 

111 

112 self.assertTemplate( 

113 "fixed/{id}", 

114 "fixed/1", 

115 self.makeDatasetRef("calexp", run="run2"), 

116 ) 

117 

118 self.assertTemplate( 

119 "fixed/{id}_{physical_filter}", 

120 "fixed/1_Most_Amazing_U_Filter_Ever", 

121 self.makeDatasetRef("calexp", run="run2"), 

122 ) 

123 

124 # Retain any "/" in run 

125 tmplstr = "{run:/}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}" 

126 self.assertTemplate( 

127 tmplstr, 

128 "run/2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2", 

129 self.makeDatasetRef("calexp", run="run/2"), 

130 ) 

131 

132 # Check that "." are replaced in the file basename, but not directory. 

133 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "g.10"} 

134 self.assertTemplate( 

135 tmplstr, 

136 "run.2/calexp/00052/g_10-trail-run_2", 

137 self.makeDatasetRef("calexp", run="run.2", dataId=dataId), 

138 ) 

139 

140 with self.assertRaises(FileTemplateValidationError): 

141 FileTemplate("no fields at all") 

142 

143 with self.assertRaises(FileTemplateValidationError): 

144 FileTemplate("{visit}") 

145 

146 with self.assertRaises(FileTemplateValidationError): 

147 FileTemplate("{run}_{datasetType}") 

148 

149 with self.assertRaises(FileTemplateValidationError): 

150 FileTemplate("{id}/fixed") 

151 

152 def testRunOrCollectionNeeded(self): 

153 tmplstr = "{datasetType}/{visit:05d}/{physical_filter}" 

154 with self.assertRaises(FileTemplateValidationError): 

155 self.assertTemplate(tmplstr, "run2/calexp/00052/U", self.makeDatasetRef("calexp")) 

156 

157 def testNoRecord(self): 

158 # Attaching records is not possible in this test code but we can check 

159 # that a missing record when a metadata entry has been requested 

160 # does fail. 

161 tmplstr = "{run}/{datasetType}/{visit.name}/{physical_filter}" 

162 with self.assertRaises(RuntimeError) as cm: 

163 self.assertTemplate(tmplstr, "", self.makeDatasetRef("calexp")) 

164 self.assertIn("No metadata", str(cm.exception)) 

165 

166 def testOptional(self): 

167 """Optional units in templates.""" 

168 ref = self.makeDatasetRef("calexp") 

169 tmplstr = "{run}/{datasetType}/v{visit:05d}_f{physical_filter:?}" 

170 self.assertTemplate( 

171 tmplstr, 

172 "run2/calexp/v00052_fMost_Amazing_U_Filter_Ever", 

173 self.makeDatasetRef("calexp"), 

174 ) 

175 

176 du = {"visit": 48, "tract": 265, "skymap": "big", "instrument": "dummy"} 

177 self.assertTemplate(tmplstr, "run2/calexpT/v00048", self.makeDatasetRef("calexpT", du)) 

178 

179 # Ensure that this returns a relative path even if the first field 

180 # is optional 

181 tmplstr = "{run}/{tract:?}/{visit:?}/f{physical_filter}" 

182 self.assertTemplate(tmplstr, "run2/52/fMost_Amazing_U_Filter_Ever", ref) 

183 

184 # Ensure that // from optionals are converted to singles 

185 tmplstr = "{run}/{datasetType}/{patch:?}/{tract:?}/f{physical_filter}" 

186 self.assertTemplate(tmplstr, "run2/calexp/fMost_Amazing_U_Filter_Ever", ref) 

187 

188 # Optionals with some text between fields 

189 tmplstr = "{run}/{datasetType}/p{patch:?}_t{tract:?}/f{physical_filter}" 

190 self.assertTemplate(tmplstr, "run2/calexp/p/fMost_Amazing_U_Filter_Ever", ref) 

191 tmplstr = "{run}/{datasetType}/p{patch:?}_t{visit:04d?}/f{physical_filter}" 

192 self.assertTemplate(tmplstr, "run2/calexp/p_t0052/fMost_Amazing_U_Filter_Ever", ref) 

193 

194 def testComponent(self): 

195 """Test handling of components in templates.""" 

196 refMetricOutput = self.makeDatasetRef("metric.output") 

197 refMetric = self.makeDatasetRef("metric") 

198 refMaskedImage = self.makeDatasetRef("calexp.maskedimage.variance") 

199 refWcs = self.makeDatasetRef("calexp.wcs") 

200 

201 tmplstr = "{run}_c_{component}_v{visit}" 

202 self.assertTemplate(tmplstr, "run2_c_output_v52", refMetricOutput) 

203 

204 # We want this template to have both a directory and basename, to 

205 # test that the right parts of the output are replaced. 

206 tmplstr = "{component:?}/{run}_{component:?}_{visit}" 

207 self.assertTemplate(tmplstr, "run2_52", refMetric) 

208 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput) 

209 self.assertTemplate(tmplstr, "maskedimage.variance/run2_maskedimage_variance_52", refMaskedImage) 

210 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput) 

211 

212 # Providing a component but not using it 

213 tmplstr = "{run}/{datasetType}/v{visit:05d}" 

214 with self.assertRaises(KeyError): 

215 self.assertTemplate(tmplstr, "", refWcs) 

216 

217 def testFields(self): 

218 # Template, mandatory fields, optional non-special fields, 

219 # special fields, optional special fields 

220 testData = ( 

221 ( 

222 "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail", 

223 {"visit", "physical_filter"}, 

224 set(), 

225 {"run", "datasetType"}, 

226 set(), 

227 ), 

228 ( 

229 "{run}/{component:?}_{visit}", 

230 {"visit"}, 

231 set(), 

232 {"run"}, 

233 {"component"}, 

234 ), 

235 ( 

236 "{run}/{component:?}_{visit:?}_{physical_filter}_{instrument}_{datasetType}", 

237 {"physical_filter", "instrument"}, 

238 {"visit"}, 

239 {"run", "datasetType"}, 

240 {"component"}, 

241 ), 

242 ) 

243 for tmplstr, mandatory, optional, special, optionalSpecial in testData: 

244 with self.subTest(template=tmplstr): 

245 tmpl = FileTemplate(tmplstr) 

246 fields = tmpl.fields() 

247 self.assertEqual(fields, mandatory) 

248 fields = tmpl.fields(optionals=True) 

249 self.assertEqual(fields, mandatory | optional) 

250 fields = tmpl.fields(specials=True) 

251 self.assertEqual(fields, mandatory | special) 

252 fields = tmpl.fields(specials=True, optionals=True) 

253 self.assertEqual(fields, mandatory | special | optional | optionalSpecial) 

254 

255 def testSimpleConfig(self): 

256 """Test reading from config file""" 

257 configRoot = os.path.join(TESTDIR, "config", "templates") 

258 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml")) 

259 templates = FileTemplates(config1, universe=self.universe) 

260 ref = self.makeDatasetRef("calexp") 

261 tmpl = templates.getTemplate(ref) 

262 self.assertIsInstance(tmpl, FileTemplate) 

263 

264 # This config file should not allow defaulting 

265 ref2 = self.makeDatasetRef("unknown") 

266 with self.assertRaises(KeyError): 

267 templates.getTemplate(ref2) 

268 

269 # This should fall through the datasetTypeName check and use 

270 # StorageClass instead 

271 ref3 = self.makeDatasetRef("unknown2", storageClassName="StorageClassX") 

272 tmplSc = templates.getTemplate(ref3) 

273 self.assertIsInstance(tmplSc, FileTemplate) 

274 

275 # Try with a component: one with defined formatter and one without 

276 refWcs = self.makeDatasetRef("calexp.wcs") 

277 refImage = self.makeDatasetRef("calexp.image") 

278 tmplCalexp = templates.getTemplate(ref) 

279 tmplWcs = templates.getTemplate(refWcs) # Should be special 

280 tmpl_image = templates.getTemplate(refImage) 

281 self.assertIsInstance(tmplCalexp, FileTemplate) 

282 self.assertIsInstance(tmpl_image, FileTemplate) 

283 self.assertIsInstance(tmplWcs, FileTemplate) 

284 self.assertEqual(tmplCalexp, tmpl_image) 

285 self.assertNotEqual(tmplCalexp, tmplWcs) 

286 

287 # Check dimensions lookup order. 

288 # The order should be: dataset type name, dimension, storage class 

289 # This one will not match name but might match storage class. 

290 # It should match dimensions 

291 refDims = self.makeDatasetRef( 

292 "nomatch", dataId={"instrument": "LSST", "physical_filter": "z"}, storageClassName="StorageClassX" 

293 ) 

294 tmplDims = templates.getTemplate(refDims) 

295 self.assertIsInstance(tmplDims, FileTemplate) 

296 self.assertNotEqual(tmplDims, tmplSc) 

297 

298 # Test that instrument overrides retrieve specialist templates 

299 refPvi = self.makeDatasetRef("pvi") 

300 refPviHsc = self.makeDatasetRef("pvi", dataId={"instrument": "HSC", "physical_filter": "z"}) 

301 refPviLsst = self.makeDatasetRef("pvi", dataId={"instrument": "LSST", "physical_filter": "z"}) 

302 

303 tmplPvi = templates.getTemplate(refPvi) 

304 tmplPviHsc = templates.getTemplate(refPviHsc) 

305 tmplPviLsst = templates.getTemplate(refPviLsst) 

306 self.assertEqual(tmplPvi, tmplPviLsst) 

307 self.assertNotEqual(tmplPvi, tmplPviHsc) 

308 

309 # Have instrument match and dimensions look up with no name match 

310 refNoPviHsc = self.makeDatasetRef( 

311 "pvix", dataId={"instrument": "HSC", "physical_filter": "z"}, storageClassName="StorageClassX" 

312 ) 

313 tmplNoPviHsc = templates.getTemplate(refNoPviHsc) 

314 self.assertNotEqual(tmplNoPviHsc, tmplDims) 

315 self.assertNotEqual(tmplNoPviHsc, tmplPviHsc) 

316 

317 # Format config file with defaulting 

318 config2 = FileTemplatesConfig(os.path.join(configRoot, "templates-withdefault.yaml")) 

319 templates = FileTemplates(config2, universe=self.universe) 

320 tmpl = templates.getTemplate(ref2) 

321 self.assertIsInstance(tmpl, FileTemplate) 

322 

323 # Format config file with bad format string 

324 with self.assertRaises(FileTemplateValidationError): 

325 FileTemplates(os.path.join(configRoot, "templates-bad.yaml"), universe=self.universe) 

326 

327 # Config file with no defaulting mentioned 

328 config3 = os.path.join(configRoot, "templates-nodefault2.yaml") 

329 templates = FileTemplates(config3, universe=self.universe) 

330 with self.assertRaises(KeyError): 

331 templates.getTemplate(ref2) 

332 

333 # Try again but specify a default in the constructor 

334 default = "{run}/{datasetType}/{physical_filter}" 

335 templates = FileTemplates(config3, default=default, universe=self.universe) 

336 tmpl = templates.getTemplate(ref2) 

337 self.assertEqual(tmpl.template, default) 

338 

339 def testValidation(self): 

340 configRoot = os.path.join(TESTDIR, "config", "templates") 

341 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml")) 

342 templates = FileTemplates(config1, universe=self.universe) 

343 

344 entities = {} 

345 entities["calexp"] = self.makeDatasetRef( 

346 "calexp", 

347 storageClassName="StorageClassX", 

348 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 52}, 

349 ) 

350 

351 with self.assertLogs(level="WARNING") as cm: 

352 templates.validateTemplates(entities.values(), logFailures=True) 

353 self.assertIn("Unchecked keys", cm.output[0]) 

354 self.assertIn("StorageClassX", cm.output[0]) 

355 

356 entities["pvi"] = self.makeDatasetRef( 

357 "pvi", storageClassName="StorageClassX", dataId={"instrument": "dummy", "physical_filter": "i"} 

358 ) 

359 entities["StorageClassX"] = self.makeDatasetRef( 

360 "storageClass", storageClassName="StorageClassX", dataId={"instrument": "dummy", "visit": 2} 

361 ) 

362 entities["calexp.wcs"] = self.makeDatasetRef( 

363 "calexp.wcs", 

364 storageClassName="StorageClassX", 

365 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 23}, 

366 conform=False, 

367 ) 

368 

369 entities["instrument+physical_filter"] = self.makeDatasetRef( 

370 "filter_inst", 

371 storageClassName="StorageClassX", 

372 dataId={"physical_filter": "i", "instrument": "SCUBA"}, 

373 ) 

374 entities["hsc+pvi"] = self.makeDatasetRef( 

375 "pvi", storageClassName="StorageClassX", dataId={"physical_filter": "i", "instrument": "HSC"} 

376 ) 

377 

378 entities["hsc+instrument+physical_filter"] = self.makeDatasetRef( 

379 "filter_inst", 

380 storageClassName="StorageClassX", 

381 dataId={"physical_filter": "i", "instrument": "HSC"}, 

382 ) 

383 

384 entities["metric6"] = self.makeDatasetRef( 

385 "filter_inst", 

386 storageClassName="Integer", 

387 dataId={"physical_filter": "i", "instrument": "HSC"}, 

388 ) 

389 

390 templates.validateTemplates(entities.values(), logFailures=True) 

391 

392 # Rerun but with a failure 

393 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX", dataId={"band": "i"}) 

394 with self.assertRaises(FileTemplateValidationError): 

395 with self.assertLogs(level="FATAL"): 

396 templates.validateTemplates(entities.values(), logFailures=True) 

397 

398 

399if __name__ == "__main__": 

400 unittest.main()