Coverage for tests/test_templates.py: 9%

175 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-28 10:10 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Test file name templating.""" 

23 

24import os.path 

25import unittest 

26 

27from lsst.daf.butler import ( 

28 DataCoordinate, 

29 DatasetRef, 

30 DatasetType, 

31 DimensionGraph, 

32 DimensionUniverse, 

33 FileTemplate, 

34 FileTemplates, 

35 FileTemplatesConfig, 

36 FileTemplateValidationError, 

37 StorageClass, 

38) 

39 

40TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

41 

42PlaceHolder = StorageClass("PlaceHolder") 

43 

44 

45class TestFileTemplates(unittest.TestCase): 

46 """Test creation of paths from templates.""" 

47 

48 def makeDatasetRef( 

49 self, datasetTypeName, dataId=None, storageClassName="DefaultStorageClass", run="run2", conform=True 

50 ): 

51 """Make a simple DatasetRef""" 

52 if dataId is None: 

53 dataId = self.dataId 

54 if "physical_filter" in dataId and "band" not in dataId: 

55 dataId["band"] = "b" # Add fake band. 

56 dimensions = DimensionGraph(self.universe, names=dataId.keys()) 

57 dataId = DataCoordinate.standardize(dataId, graph=dimensions) 

58 

59 # Pretend we have a parent if this looks like a composite 

60 compositeName, componentName = DatasetType.splitDatasetTypeName(datasetTypeName) 

61 parentStorageClass = PlaceHolder if componentName else None 

62 

63 datasetType = DatasetType( 

64 datasetTypeName, 

65 dimensions, 

66 StorageClass(storageClassName), 

67 parentStorageClass=parentStorageClass, 

68 ) 

69 return DatasetRef(datasetType, dataId, id=1, run=run, conform=conform) 

70 

71 def setUp(self): 

72 self.universe = DimensionUniverse() 

73 self.dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "Most Amazing U Filter Ever"} 

74 

75 def assertTemplate(self, template, answer, ref): 

76 fileTmpl = FileTemplate(template) 

77 path = fileTmpl.format(ref) 

78 self.assertEqual(path, answer) 

79 

80 def testBasic(self): 

81 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}" 

82 self.assertTemplate( 

83 tmplstr, 

84 "run2/calexp/00052/Most_Amazing_U_Filter_Ever", 

85 self.makeDatasetRef("calexp"), 

86 ) 

87 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail" 

88 self.assertTemplate( 

89 tmplstr, 

90 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail", 

91 self.makeDatasetRef("calexp"), 

92 ) 

93 

94 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}" 

95 self.assertTemplate( 

96 tmplstr, 

97 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run2", 

98 self.makeDatasetRef("calexp"), 

99 ) 

100 self.assertTemplate( 

101 tmplstr, 

102 "run_2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2", 

103 self.makeDatasetRef("calexp", run="run/2"), 

104 ) 

105 

106 # Check that the id is sufficient without any other information. 

107 self.assertTemplate("{id}", "1", self.makeDatasetRef("calexp", run="run2")) 

108 

109 self.assertTemplate("{run}/{id}", "run2/1", self.makeDatasetRef("calexp", run="run2")) 

110 

111 self.assertTemplate( 

112 "fixed/{id}", 

113 "fixed/1", 

114 self.makeDatasetRef("calexp", run="run2"), 

115 ) 

116 

117 self.assertTemplate( 

118 "fixed/{id}_{physical_filter}", 

119 "fixed/1_Most_Amazing_U_Filter_Ever", 

120 self.makeDatasetRef("calexp", run="run2"), 

121 ) 

122 

123 # Retain any "/" in run 

124 tmplstr = "{run:/}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}" 

125 self.assertTemplate( 

126 tmplstr, 

127 "run/2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2", 

128 self.makeDatasetRef("calexp", run="run/2"), 

129 ) 

130 

131 # Check that "." are replaced in the file basename, but not directory. 

132 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "g.10"} 

133 self.assertTemplate( 

134 tmplstr, 

135 "run.2/calexp/00052/g_10-trail-run_2", 

136 self.makeDatasetRef("calexp", run="run.2", dataId=dataId), 

137 ) 

138 

139 with self.assertRaises(FileTemplateValidationError): 

140 FileTemplate("no fields at all") 

141 

142 with self.assertRaises(FileTemplateValidationError): 

143 FileTemplate("{visit}") 

144 

145 with self.assertRaises(FileTemplateValidationError): 

146 FileTemplate("{run}_{datasetType}") 

147 

148 with self.assertRaises(FileTemplateValidationError): 

149 FileTemplate("{id}/fixed") 

150 

151 def testRunOrCollectionNeeded(self): 

152 tmplstr = "{datasetType}/{visit:05d}/{physical_filter}" 

153 with self.assertRaises(FileTemplateValidationError): 

154 self.assertTemplate(tmplstr, "run2/calexp/00052/U", self.makeDatasetRef("calexp")) 

155 

156 def testNoRecord(self): 

157 # Attaching records is not possible in this test code but we can check 

158 # that a missing record when a metadata entry has been requested 

159 # does fail. 

160 tmplstr = "{run}/{datasetType}/{visit.name}/{physical_filter}" 

161 with self.assertRaises(RuntimeError) as cm: 

162 self.assertTemplate(tmplstr, "", self.makeDatasetRef("calexp")) 

163 self.assertIn("No metadata", str(cm.exception)) 

164 

165 def testOptional(self): 

166 """Optional units in templates.""" 

167 ref = self.makeDatasetRef("calexp") 

168 tmplstr = "{run}/{datasetType}/v{visit:05d}_f{physical_filter:?}" 

169 self.assertTemplate( 

170 tmplstr, 

171 "run2/calexp/v00052_fMost_Amazing_U_Filter_Ever", 

172 self.makeDatasetRef("calexp"), 

173 ) 

174 

175 du = {"visit": 48, "tract": 265, "skymap": "big", "instrument": "dummy"} 

176 self.assertTemplate(tmplstr, "run2/calexpT/v00048", self.makeDatasetRef("calexpT", du)) 

177 

178 # Ensure that this returns a relative path even if the first field 

179 # is optional 

180 tmplstr = "{run}/{tract:?}/{visit:?}/f{physical_filter}" 

181 self.assertTemplate(tmplstr, "run2/52/fMost_Amazing_U_Filter_Ever", ref) 

182 

183 # Ensure that // from optionals are converted to singles 

184 tmplstr = "{run}/{datasetType}/{patch:?}/{tract:?}/f{physical_filter}" 

185 self.assertTemplate(tmplstr, "run2/calexp/fMost_Amazing_U_Filter_Ever", ref) 

186 

187 # Optionals with some text between fields 

188 tmplstr = "{run}/{datasetType}/p{patch:?}_t{tract:?}/f{physical_filter}" 

189 self.assertTemplate(tmplstr, "run2/calexp/p/fMost_Amazing_U_Filter_Ever", ref) 

190 tmplstr = "{run}/{datasetType}/p{patch:?}_t{visit:04d?}/f{physical_filter}" 

191 self.assertTemplate(tmplstr, "run2/calexp/p_t0052/fMost_Amazing_U_Filter_Ever", ref) 

192 

193 def testComponent(self): 

194 """Test handling of components in templates.""" 

195 refMetricOutput = self.makeDatasetRef("metric.output") 

196 refMetric = self.makeDatasetRef("metric") 

197 refMaskedImage = self.makeDatasetRef("calexp.maskedimage.variance") 

198 refWcs = self.makeDatasetRef("calexp.wcs") 

199 

200 tmplstr = "{run}_c_{component}_v{visit}" 

201 self.assertTemplate(tmplstr, "run2_c_output_v52", refMetricOutput) 

202 

203 # We want this template to have both a directory and basename, to 

204 # test that the right parts of the output are replaced. 

205 tmplstr = "{component:?}/{run}_{component:?}_{visit}" 

206 self.assertTemplate(tmplstr, "run2_52", refMetric) 

207 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput) 

208 self.assertTemplate(tmplstr, "maskedimage.variance/run2_maskedimage_variance_52", refMaskedImage) 

209 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput) 

210 

211 # Providing a component but not using it 

212 tmplstr = "{run}/{datasetType}/v{visit:05d}" 

213 with self.assertRaises(KeyError): 

214 self.assertTemplate(tmplstr, "", refWcs) 

215 

216 def testFields(self): 

217 # Template, mandatory fields, optional non-special fields, 

218 # special fields, optional special fields 

219 testData = ( 

220 ( 

221 "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail", 

222 {"visit", "physical_filter"}, 

223 set(), 

224 {"run", "datasetType"}, 

225 set(), 

226 ), 

227 ( 

228 "{run}/{component:?}_{visit}", 

229 {"visit"}, 

230 set(), 

231 {"run"}, 

232 {"component"}, 

233 ), 

234 ( 

235 "{run}/{component:?}_{visit:?}_{physical_filter}_{instrument}_{datasetType}", 

236 {"physical_filter", "instrument"}, 

237 {"visit"}, 

238 {"run", "datasetType"}, 

239 {"component"}, 

240 ), 

241 ) 

242 for tmplstr, mandatory, optional, special, optionalSpecial in testData: 

243 with self.subTest(template=tmplstr): 

244 tmpl = FileTemplate(tmplstr) 

245 fields = tmpl.fields() 

246 self.assertEqual(fields, mandatory) 

247 fields = tmpl.fields(optionals=True) 

248 self.assertEqual(fields, mandatory | optional) 

249 fields = tmpl.fields(specials=True) 

250 self.assertEqual(fields, mandatory | special) 

251 fields = tmpl.fields(specials=True, optionals=True) 

252 self.assertEqual(fields, mandatory | special | optional | optionalSpecial) 

253 

254 def testSimpleConfig(self): 

255 """Test reading from config file""" 

256 configRoot = os.path.join(TESTDIR, "config", "templates") 

257 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml")) 

258 templates = FileTemplates(config1, universe=self.universe) 

259 ref = self.makeDatasetRef("calexp") 

260 tmpl = templates.getTemplate(ref) 

261 self.assertIsInstance(tmpl, FileTemplate) 

262 

263 # This config file should not allow defaulting 

264 ref2 = self.makeDatasetRef("unknown") 

265 with self.assertRaises(KeyError): 

266 templates.getTemplate(ref2) 

267 

268 # This should fall through the datasetTypeName check and use 

269 # StorageClass instead 

270 ref3 = self.makeDatasetRef("unknown2", storageClassName="StorageClassX") 

271 tmplSc = templates.getTemplate(ref3) 

272 self.assertIsInstance(tmplSc, FileTemplate) 

273 

274 # Try with a component: one with defined formatter and one without 

275 refWcs = self.makeDatasetRef("calexp.wcs") 

276 refImage = self.makeDatasetRef("calexp.image") 

277 tmplCalexp = templates.getTemplate(ref) 

278 tmplWcs = templates.getTemplate(refWcs) # Should be special 

279 tmpl_image = templates.getTemplate(refImage) 

280 self.assertIsInstance(tmplCalexp, FileTemplate) 

281 self.assertIsInstance(tmpl_image, FileTemplate) 

282 self.assertIsInstance(tmplWcs, FileTemplate) 

283 self.assertEqual(tmplCalexp, tmpl_image) 

284 self.assertNotEqual(tmplCalexp, tmplWcs) 

285 

286 # Check dimensions lookup order. 

287 # The order should be: dataset type name, dimension, storage class 

288 # This one will not match name but might match storage class. 

289 # It should match dimensions 

290 refDims = self.makeDatasetRef( 

291 "nomatch", dataId={"instrument": "LSST", "physical_filter": "z"}, storageClassName="StorageClassX" 

292 ) 

293 tmplDims = templates.getTemplate(refDims) 

294 self.assertIsInstance(tmplDims, FileTemplate) 

295 self.assertNotEqual(tmplDims, tmplSc) 

296 

297 # Test that instrument overrides retrieve specialist templates 

298 refPvi = self.makeDatasetRef("pvi") 

299 refPviHsc = self.makeDatasetRef("pvi", dataId={"instrument": "HSC", "physical_filter": "z"}) 

300 refPviLsst = self.makeDatasetRef("pvi", dataId={"instrument": "LSST", "physical_filter": "z"}) 

301 

302 tmplPvi = templates.getTemplate(refPvi) 

303 tmplPviHsc = templates.getTemplate(refPviHsc) 

304 tmplPviLsst = templates.getTemplate(refPviLsst) 

305 self.assertEqual(tmplPvi, tmplPviLsst) 

306 self.assertNotEqual(tmplPvi, tmplPviHsc) 

307 

308 # Have instrument match and dimensions look up with no name match 

309 refNoPviHsc = self.makeDatasetRef( 

310 "pvix", dataId={"instrument": "HSC", "physical_filter": "z"}, storageClassName="StorageClassX" 

311 ) 

312 tmplNoPviHsc = templates.getTemplate(refNoPviHsc) 

313 self.assertNotEqual(tmplNoPviHsc, tmplDims) 

314 self.assertNotEqual(tmplNoPviHsc, tmplPviHsc) 

315 

316 # Format config file with defaulting 

317 config2 = FileTemplatesConfig(os.path.join(configRoot, "templates-withdefault.yaml")) 

318 templates = FileTemplates(config2, universe=self.universe) 

319 tmpl = templates.getTemplate(ref2) 

320 self.assertIsInstance(tmpl, FileTemplate) 

321 

322 # Format config file with bad format string 

323 with self.assertRaises(FileTemplateValidationError): 

324 FileTemplates(os.path.join(configRoot, "templates-bad.yaml"), universe=self.universe) 

325 

326 # Config file with no defaulting mentioned 

327 config3 = os.path.join(configRoot, "templates-nodefault2.yaml") 

328 templates = FileTemplates(config3, universe=self.universe) 

329 with self.assertRaises(KeyError): 

330 templates.getTemplate(ref2) 

331 

332 # Try again but specify a default in the constructor 

333 default = "{run}/{datasetType}/{physical_filter}" 

334 templates = FileTemplates(config3, default=default, universe=self.universe) 

335 tmpl = templates.getTemplate(ref2) 

336 self.assertEqual(tmpl.template, default) 

337 

338 def testValidation(self): 

339 configRoot = os.path.join(TESTDIR, "config", "templates") 

340 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml")) 

341 templates = FileTemplates(config1, universe=self.universe) 

342 

343 entities = {} 

344 entities["calexp"] = self.makeDatasetRef( 

345 "calexp", 

346 storageClassName="StorageClassX", 

347 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 52}, 

348 ) 

349 

350 with self.assertLogs(level="WARNING") as cm: 

351 templates.validateTemplates(entities.values(), logFailures=True) 

352 self.assertIn("Unchecked keys", cm.output[0]) 

353 self.assertIn("StorageClassX", cm.output[0]) 

354 

355 entities["pvi"] = self.makeDatasetRef( 

356 "pvi", storageClassName="StorageClassX", dataId={"instrument": "dummy", "physical_filter": "i"} 

357 ) 

358 entities["StorageClassX"] = self.makeDatasetRef( 

359 "storageClass", storageClassName="StorageClassX", dataId={"instrument": "dummy", "visit": 2} 

360 ) 

361 entities["calexp.wcs"] = self.makeDatasetRef( 

362 "calexp.wcs", 

363 storageClassName="StorageClassX", 

364 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 23}, 

365 conform=False, 

366 ) 

367 

368 entities["instrument+physical_filter"] = self.makeDatasetRef( 

369 "filter_inst", 

370 storageClassName="StorageClassX", 

371 dataId={"physical_filter": "i", "instrument": "SCUBA"}, 

372 ) 

373 entities["hsc+pvi"] = self.makeDatasetRef( 

374 "pvi", storageClassName="StorageClassX", dataId={"physical_filter": "i", "instrument": "HSC"} 

375 ) 

376 

377 entities["hsc+instrument+physical_filter"] = self.makeDatasetRef( 

378 "filter_inst", 

379 storageClassName="StorageClassX", 

380 dataId={"physical_filter": "i", "instrument": "HSC"}, 

381 ) 

382 

383 entities["metric6"] = self.makeDatasetRef( 

384 "filter_inst", 

385 storageClassName="Integer", 

386 dataId={"physical_filter": "i", "instrument": "HSC"}, 

387 ) 

388 

389 templates.validateTemplates(entities.values(), logFailures=True) 

390 

391 # Rerun but with a failure 

392 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX", dataId={"band": "i"}) 

393 with self.assertRaises(FileTemplateValidationError): 

394 with self.assertLogs(level="FATAL"): 

395 templates.validateTemplates(entities.values(), logFailures=True) 

396 

397 

398if __name__ == "__main__": 

399 unittest.main()