Coverage for tests/test_templates.py: 9%

177 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-10-25 15:14 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Test file name templating.""" 

23 

24import os.path 

25import unittest 

26import uuid 

27 

28from lsst.daf.butler import ( 

29 DataCoordinate, 

30 DatasetId, 

31 DatasetRef, 

32 DatasetType, 

33 DimensionGraph, 

34 DimensionUniverse, 

35 FileTemplate, 

36 FileTemplates, 

37 FileTemplatesConfig, 

38 FileTemplateValidationError, 

39 StorageClass, 

40) 

41 

42TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

43 

44PlaceHolder = StorageClass("PlaceHolder") 

45 

46REFUUID = DatasetId(int=uuid.uuid4().int) 

47 

48 

49class TestFileTemplates(unittest.TestCase): 

50 """Test creation of paths from templates.""" 

51 

52 def makeDatasetRef( 

53 self, datasetTypeName, dataId=None, storageClassName="DefaultStorageClass", run="run2", conform=True 

54 ): 

55 """Make a simple DatasetRef""" 

56 if dataId is None: 

57 dataId = self.dataId 

58 if "physical_filter" in dataId and "band" not in dataId: 

59 dataId["band"] = "b" # Add fake band. 

60 dimensions = DimensionGraph(self.universe, names=dataId.keys()) 

61 dataId = DataCoordinate.standardize(dataId, graph=dimensions) 

62 

63 # Pretend we have a parent if this looks like a composite 

64 compositeName, componentName = DatasetType.splitDatasetTypeName(datasetTypeName) 

65 parentStorageClass = PlaceHolder if componentName else None 

66 

67 datasetType = DatasetType( 

68 datasetTypeName, 

69 dimensions, 

70 StorageClass(storageClassName), 

71 parentStorageClass=parentStorageClass, 

72 ) 

73 return DatasetRef(datasetType, dataId, id=REFUUID, run=run, conform=conform) 

74 

75 def setUp(self): 

76 self.universe = DimensionUniverse() 

77 self.dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "Most Amazing U Filter Ever"} 

78 

79 def assertTemplate(self, template, answer, ref): 

80 fileTmpl = FileTemplate(template) 

81 path = fileTmpl.format(ref) 

82 self.assertEqual(path, answer) 

83 

84 def testBasic(self): 

85 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}" 

86 self.assertTemplate( 

87 tmplstr, 

88 "run2/calexp/00052/Most_Amazing_U_Filter_Ever", 

89 self.makeDatasetRef("calexp"), 

90 ) 

91 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail" 

92 self.assertTemplate( 

93 tmplstr, 

94 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail", 

95 self.makeDatasetRef("calexp"), 

96 ) 

97 

98 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}" 

99 self.assertTemplate( 

100 tmplstr, 

101 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run2", 

102 self.makeDatasetRef("calexp"), 

103 ) 

104 self.assertTemplate( 

105 tmplstr, 

106 "run_2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2", 

107 self.makeDatasetRef("calexp", run="run/2"), 

108 ) 

109 

110 # Check that the id is sufficient without any other information. 

111 self.assertTemplate("{id}", str(REFUUID), self.makeDatasetRef("calexp", run="run2")) 

112 

113 self.assertTemplate("{run}/{id}", f"run2/{str(REFUUID)}", self.makeDatasetRef("calexp", run="run2")) 

114 

115 self.assertTemplate( 

116 "fixed/{id}", 

117 f"fixed/{str(REFUUID)}", 

118 self.makeDatasetRef("calexp", run="run2"), 

119 ) 

120 

121 self.assertTemplate( 

122 "fixed/{id}_{physical_filter}", 

123 f"fixed/{str(REFUUID)}_Most_Amazing_U_Filter_Ever", 

124 self.makeDatasetRef("calexp", run="run2"), 

125 ) 

126 

127 # Retain any "/" in run 

128 tmplstr = "{run:/}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}" 

129 self.assertTemplate( 

130 tmplstr, 

131 "run/2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2", 

132 self.makeDatasetRef("calexp", run="run/2"), 

133 ) 

134 

135 # Check that "." are replaced in the file basename, but not directory. 

136 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "g.10"} 

137 self.assertTemplate( 

138 tmplstr, 

139 "run.2/calexp/00052/g_10-trail-run_2", 

140 self.makeDatasetRef("calexp", run="run.2", dataId=dataId), 

141 ) 

142 

143 with self.assertRaises(FileTemplateValidationError): 

144 FileTemplate("no fields at all") 

145 

146 with self.assertRaises(FileTemplateValidationError): 

147 FileTemplate("{visit}") 

148 

149 with self.assertRaises(FileTemplateValidationError): 

150 FileTemplate("{run}_{datasetType}") 

151 

152 with self.assertRaises(FileTemplateValidationError): 

153 FileTemplate("{id}/fixed") 

154 

155 def testRunOrCollectionNeeded(self): 

156 tmplstr = "{datasetType}/{visit:05d}/{physical_filter}" 

157 with self.assertRaises(FileTemplateValidationError): 

158 self.assertTemplate(tmplstr, "run2/calexp/00052/U", self.makeDatasetRef("calexp")) 

159 

160 def testNoRecord(self): 

161 # Attaching records is not possible in this test code but we can check 

162 # that a missing record when a metadata entry has been requested 

163 # does fail. 

164 tmplstr = "{run}/{datasetType}/{visit.name}/{physical_filter}" 

165 with self.assertRaises(RuntimeError) as cm: 

166 self.assertTemplate(tmplstr, "", self.makeDatasetRef("calexp")) 

167 self.assertIn("No metadata", str(cm.exception)) 

168 

169 def testOptional(self): 

170 """Optional units in templates.""" 

171 ref = self.makeDatasetRef("calexp") 

172 tmplstr = "{run}/{datasetType}/v{visit:05d}_f{physical_filter:?}" 

173 self.assertTemplate( 

174 tmplstr, 

175 "run2/calexp/v00052_fMost_Amazing_U_Filter_Ever", 

176 self.makeDatasetRef("calexp"), 

177 ) 

178 

179 du = {"visit": 48, "tract": 265, "skymap": "big", "instrument": "dummy"} 

180 self.assertTemplate(tmplstr, "run2/calexpT/v00048", self.makeDatasetRef("calexpT", du)) 

181 

182 # Ensure that this returns a relative path even if the first field 

183 # is optional 

184 tmplstr = "{run}/{tract:?}/{visit:?}/f{physical_filter}" 

185 self.assertTemplate(tmplstr, "run2/52/fMost_Amazing_U_Filter_Ever", ref) 

186 

187 # Ensure that // from optionals are converted to singles 

188 tmplstr = "{run}/{datasetType}/{patch:?}/{tract:?}/f{physical_filter}" 

189 self.assertTemplate(tmplstr, "run2/calexp/fMost_Amazing_U_Filter_Ever", ref) 

190 

191 # Optionals with some text between fields 

192 tmplstr = "{run}/{datasetType}/p{patch:?}_t{tract:?}/f{physical_filter}" 

193 self.assertTemplate(tmplstr, "run2/calexp/p/fMost_Amazing_U_Filter_Ever", ref) 

194 tmplstr = "{run}/{datasetType}/p{patch:?}_t{visit:04d?}/f{physical_filter}" 

195 self.assertTemplate(tmplstr, "run2/calexp/p_t0052/fMost_Amazing_U_Filter_Ever", ref) 

196 

197 def testComponent(self): 

198 """Test handling of components in templates.""" 

199 refMetricOutput = self.makeDatasetRef("metric.output") 

200 refMetric = self.makeDatasetRef("metric") 

201 refMaskedImage = self.makeDatasetRef("calexp.maskedimage.variance") 

202 refWcs = self.makeDatasetRef("calexp.wcs") 

203 

204 tmplstr = "{run}_c_{component}_v{visit}" 

205 self.assertTemplate(tmplstr, "run2_c_output_v52", refMetricOutput) 

206 

207 # We want this template to have both a directory and basename, to 

208 # test that the right parts of the output are replaced. 

209 tmplstr = "{component:?}/{run}_{component:?}_{visit}" 

210 self.assertTemplate(tmplstr, "run2_52", refMetric) 

211 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput) 

212 self.assertTemplate(tmplstr, "maskedimage.variance/run2_maskedimage_variance_52", refMaskedImage) 

213 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput) 

214 

215 # Providing a component but not using it 

216 tmplstr = "{run}/{datasetType}/v{visit:05d}" 

217 with self.assertRaises(KeyError): 

218 self.assertTemplate(tmplstr, "", refWcs) 

219 

220 def testFields(self): 

221 # Template, mandatory fields, optional non-special fields, 

222 # special fields, optional special fields 

223 testData = ( 

224 ( 

225 "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail", 

226 {"visit", "physical_filter"}, 

227 set(), 

228 {"run", "datasetType"}, 

229 set(), 

230 ), 

231 ( 

232 "{run}/{component:?}_{visit}", 

233 {"visit"}, 

234 set(), 

235 {"run"}, 

236 {"component"}, 

237 ), 

238 ( 

239 "{run}/{component:?}_{visit:?}_{physical_filter}_{instrument}_{datasetType}", 

240 {"physical_filter", "instrument"}, 

241 {"visit"}, 

242 {"run", "datasetType"}, 

243 {"component"}, 

244 ), 

245 ) 

246 for tmplstr, mandatory, optional, special, optionalSpecial in testData: 

247 with self.subTest(template=tmplstr): 

248 tmpl = FileTemplate(tmplstr) 

249 fields = tmpl.fields() 

250 self.assertEqual(fields, mandatory) 

251 fields = tmpl.fields(optionals=True) 

252 self.assertEqual(fields, mandatory | optional) 

253 fields = tmpl.fields(specials=True) 

254 self.assertEqual(fields, mandatory | special) 

255 fields = tmpl.fields(specials=True, optionals=True) 

256 self.assertEqual(fields, mandatory | special | optional | optionalSpecial) 

257 

258 def testSimpleConfig(self): 

259 """Test reading from config file""" 

260 configRoot = os.path.join(TESTDIR, "config", "templates") 

261 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml")) 

262 templates = FileTemplates(config1, universe=self.universe) 

263 ref = self.makeDatasetRef("calexp") 

264 tmpl = templates.getTemplate(ref) 

265 self.assertIsInstance(tmpl, FileTemplate) 

266 

267 # This config file should not allow defaulting 

268 ref2 = self.makeDatasetRef("unknown") 

269 with self.assertRaises(KeyError): 

270 templates.getTemplate(ref2) 

271 

272 # This should fall through the datasetTypeName check and use 

273 # StorageClass instead 

274 ref3 = self.makeDatasetRef("unknown2", storageClassName="StorageClassX") 

275 tmplSc = templates.getTemplate(ref3) 

276 self.assertIsInstance(tmplSc, FileTemplate) 

277 

278 # Try with a component: one with defined formatter and one without 

279 refWcs = self.makeDatasetRef("calexp.wcs") 

280 refImage = self.makeDatasetRef("calexp.image") 

281 tmplCalexp = templates.getTemplate(ref) 

282 tmplWcs = templates.getTemplate(refWcs) # Should be special 

283 tmpl_image = templates.getTemplate(refImage) 

284 self.assertIsInstance(tmplCalexp, FileTemplate) 

285 self.assertIsInstance(tmpl_image, FileTemplate) 

286 self.assertIsInstance(tmplWcs, FileTemplate) 

287 self.assertEqual(tmplCalexp, tmpl_image) 

288 self.assertNotEqual(tmplCalexp, tmplWcs) 

289 

290 # Check dimensions lookup order. 

291 # The order should be: dataset type name, dimension, storage class 

292 # This one will not match name but might match storage class. 

293 # It should match dimensions 

294 refDims = self.makeDatasetRef( 

295 "nomatch", dataId={"instrument": "LSST", "physical_filter": "z"}, storageClassName="StorageClassX" 

296 ) 

297 tmplDims = templates.getTemplate(refDims) 

298 self.assertIsInstance(tmplDims, FileTemplate) 

299 self.assertNotEqual(tmplDims, tmplSc) 

300 

301 # Test that instrument overrides retrieve specialist templates 

302 refPvi = self.makeDatasetRef("pvi") 

303 refPviHsc = self.makeDatasetRef("pvi", dataId={"instrument": "HSC", "physical_filter": "z"}) 

304 refPviLsst = self.makeDatasetRef("pvi", dataId={"instrument": "LSST", "physical_filter": "z"}) 

305 

306 tmplPvi = templates.getTemplate(refPvi) 

307 tmplPviHsc = templates.getTemplate(refPviHsc) 

308 tmplPviLsst = templates.getTemplate(refPviLsst) 

309 self.assertEqual(tmplPvi, tmplPviLsst) 

310 self.assertNotEqual(tmplPvi, tmplPviHsc) 

311 

312 # Have instrument match and dimensions look up with no name match 

313 refNoPviHsc = self.makeDatasetRef( 

314 "pvix", dataId={"instrument": "HSC", "physical_filter": "z"}, storageClassName="StorageClassX" 

315 ) 

316 tmplNoPviHsc = templates.getTemplate(refNoPviHsc) 

317 self.assertNotEqual(tmplNoPviHsc, tmplDims) 

318 self.assertNotEqual(tmplNoPviHsc, tmplPviHsc) 

319 

320 # Format config file with defaulting 

321 config2 = FileTemplatesConfig(os.path.join(configRoot, "templates-withdefault.yaml")) 

322 templates = FileTemplates(config2, universe=self.universe) 

323 tmpl = templates.getTemplate(ref2) 

324 self.assertIsInstance(tmpl, FileTemplate) 

325 

326 # Format config file with bad format string 

327 with self.assertRaises(FileTemplateValidationError): 

328 FileTemplates(os.path.join(configRoot, "templates-bad.yaml"), universe=self.universe) 

329 

330 # Config file with no defaulting mentioned 

331 config3 = os.path.join(configRoot, "templates-nodefault2.yaml") 

332 templates = FileTemplates(config3, universe=self.universe) 

333 with self.assertRaises(KeyError): 

334 templates.getTemplate(ref2) 

335 

336 # Try again but specify a default in the constructor 

337 default = "{run}/{datasetType}/{physical_filter}" 

338 templates = FileTemplates(config3, default=default, universe=self.universe) 

339 tmpl = templates.getTemplate(ref2) 

340 self.assertEqual(tmpl.template, default) 

341 

342 def testValidation(self): 

343 configRoot = os.path.join(TESTDIR, "config", "templates") 

344 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml")) 

345 templates = FileTemplates(config1, universe=self.universe) 

346 

347 entities = {} 

348 entities["calexp"] = self.makeDatasetRef( 

349 "calexp", 

350 storageClassName="StorageClassX", 

351 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 52}, 

352 ) 

353 

354 with self.assertLogs(level="WARNING") as cm: 

355 templates.validateTemplates(entities.values(), logFailures=True) 

356 self.assertIn("Unchecked keys", cm.output[0]) 

357 self.assertIn("StorageClassX", cm.output[0]) 

358 

359 entities["pvi"] = self.makeDatasetRef( 

360 "pvi", storageClassName="StorageClassX", dataId={"instrument": "dummy", "physical_filter": "i"} 

361 ) 

362 entities["StorageClassX"] = self.makeDatasetRef( 

363 "storageClass", storageClassName="StorageClassX", dataId={"instrument": "dummy", "visit": 2} 

364 ) 

365 entities["calexp.wcs"] = self.makeDatasetRef( 

366 "calexp.wcs", 

367 storageClassName="StorageClassX", 

368 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 23}, 

369 conform=False, 

370 ) 

371 

372 entities["instrument+physical_filter"] = self.makeDatasetRef( 

373 "filter_inst", 

374 storageClassName="StorageClassX", 

375 dataId={"physical_filter": "i", "instrument": "SCUBA"}, 

376 ) 

377 entities["hsc+pvi"] = self.makeDatasetRef( 

378 "pvi", storageClassName="StorageClassX", dataId={"physical_filter": "i", "instrument": "HSC"} 

379 ) 

380 

381 entities["hsc+instrument+physical_filter"] = self.makeDatasetRef( 

382 "filter_inst", 

383 storageClassName="StorageClassX", 

384 dataId={"physical_filter": "i", "instrument": "HSC"}, 

385 ) 

386 

387 entities["metric6"] = self.makeDatasetRef( 

388 "filter_inst", 

389 storageClassName="Integer", 

390 dataId={"physical_filter": "i", "instrument": "HSC"}, 

391 ) 

392 

393 templates.validateTemplates(entities.values(), logFailures=True) 

394 

395 # Rerun but with a failure 

396 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX", dataId={"band": "i"}) 

397 with self.assertRaises(FileTemplateValidationError): 

398 with self.assertLogs(level="FATAL"): 

399 templates.validateTemplates(entities.values(), logFailures=True) 

400 

401 

402if __name__ == "__main__": 

403 unittest.main()