Coverage for tests/test_templates.py: 10%

178 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-10-27 09:44 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Test file name templating.""" 

29 

30import os.path 

31import unittest 

32import uuid 

33 

34from lsst.daf.butler import ( 

35 DataCoordinate, 

36 DatasetId, 

37 DatasetRef, 

38 DatasetType, 

39 DimensionGraph, 

40 DimensionUniverse, 

41 StorageClass, 

42) 

43from lsst.daf.butler.datastore.file_templates import ( 

44 FileTemplate, 

45 FileTemplates, 

46 FileTemplatesConfig, 

47 FileTemplateValidationError, 

48) 

49 

50TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

51 

52PlaceHolder = StorageClass("PlaceHolder") 

53 

54REFUUID = DatasetId(int=uuid.uuid4().int) 

55 

56 

57class TestFileTemplates(unittest.TestCase): 

58 """Test creation of paths from templates.""" 

59 

60 def makeDatasetRef( 

61 self, datasetTypeName, dataId=None, storageClassName="DefaultStorageClass", run="run2", conform=True 

62 ): 

63 """Make a simple DatasetRef""" 

64 if dataId is None: 

65 dataId = self.dataId 

66 if "physical_filter" in dataId and "band" not in dataId: 

67 dataId["band"] = "b" # Add fake band. 

68 dimensions = DimensionGraph(self.universe, names=dataId.keys()) 

69 dataId = DataCoordinate.standardize(dataId, graph=dimensions) 

70 

71 # Pretend we have a parent if this looks like a composite 

72 compositeName, componentName = DatasetType.splitDatasetTypeName(datasetTypeName) 

73 parentStorageClass = PlaceHolder if componentName else None 

74 

75 datasetType = DatasetType( 

76 datasetTypeName, 

77 dimensions, 

78 StorageClass(storageClassName), 

79 parentStorageClass=parentStorageClass, 

80 ) 

81 return DatasetRef(datasetType, dataId, id=REFUUID, run=run, conform=conform) 

82 

83 def setUp(self): 

84 self.universe = DimensionUniverse() 

85 self.dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "Most Amazing U Filter Ever"} 

86 

87 def assertTemplate(self, template, answer, ref): 

88 fileTmpl = FileTemplate(template) 

89 path = fileTmpl.format(ref) 

90 self.assertEqual(path, answer) 

91 

92 def testBasic(self): 

93 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}" 

94 self.assertTemplate( 

95 tmplstr, 

96 "run2/calexp/00052/Most_Amazing_U_Filter_Ever", 

97 self.makeDatasetRef("calexp"), 

98 ) 

99 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail" 

100 self.assertTemplate( 

101 tmplstr, 

102 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail", 

103 self.makeDatasetRef("calexp"), 

104 ) 

105 

106 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}" 

107 self.assertTemplate( 

108 tmplstr, 

109 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run2", 

110 self.makeDatasetRef("calexp"), 

111 ) 

112 self.assertTemplate( 

113 tmplstr, 

114 "run_2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2", 

115 self.makeDatasetRef("calexp", run="run/2"), 

116 ) 

117 

118 # Check that the id is sufficient without any other information. 

119 self.assertTemplate("{id}", str(REFUUID), self.makeDatasetRef("calexp", run="run2")) 

120 

121 self.assertTemplate("{run}/{id}", f"run2/{str(REFUUID)}", self.makeDatasetRef("calexp", run="run2")) 

122 

123 self.assertTemplate( 

124 "fixed/{id}", 

125 f"fixed/{str(REFUUID)}", 

126 self.makeDatasetRef("calexp", run="run2"), 

127 ) 

128 

129 self.assertTemplate( 

130 "fixed/{id}_{physical_filter}", 

131 f"fixed/{str(REFUUID)}_Most_Amazing_U_Filter_Ever", 

132 self.makeDatasetRef("calexp", run="run2"), 

133 ) 

134 

135 # Retain any "/" in run 

136 tmplstr = "{run:/}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}" 

137 self.assertTemplate( 

138 tmplstr, 

139 "run/2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2", 

140 self.makeDatasetRef("calexp", run="run/2"), 

141 ) 

142 

143 # Check that "." are replaced in the file basename, but not directory. 

144 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "g.10"} 

145 self.assertTemplate( 

146 tmplstr, 

147 "run.2/calexp/00052/g_10-trail-run_2", 

148 self.makeDatasetRef("calexp", run="run.2", dataId=dataId), 

149 ) 

150 

151 with self.assertRaises(FileTemplateValidationError): 

152 FileTemplate("no fields at all") 

153 

154 with self.assertRaises(FileTemplateValidationError): 

155 FileTemplate("{visit}") 

156 

157 with self.assertRaises(FileTemplateValidationError): 

158 FileTemplate("{run}_{datasetType}") 

159 

160 with self.assertRaises(FileTemplateValidationError): 

161 FileTemplate("{id}/fixed") 

162 

163 def testRunOrCollectionNeeded(self): 

164 tmplstr = "{datasetType}/{visit:05d}/{physical_filter}" 

165 with self.assertRaises(FileTemplateValidationError): 

166 self.assertTemplate(tmplstr, "run2/calexp/00052/U", self.makeDatasetRef("calexp")) 

167 

168 def testNoRecord(self): 

169 # Attaching records is not possible in this test code but we can check 

170 # that a missing record when a metadata entry has been requested 

171 # does fail. 

172 tmplstr = "{run}/{datasetType}/{visit.name}/{physical_filter}" 

173 with self.assertRaises(RuntimeError) as cm: 

174 self.assertTemplate(tmplstr, "", self.makeDatasetRef("calexp")) 

175 self.assertIn("No metadata", str(cm.exception)) 

176 

177 def testOptional(self): 

178 """Optional units in templates.""" 

179 ref = self.makeDatasetRef("calexp") 

180 tmplstr = "{run}/{datasetType}/v{visit:05d}_f{physical_filter:?}" 

181 self.assertTemplate( 

182 tmplstr, 

183 "run2/calexp/v00052_fMost_Amazing_U_Filter_Ever", 

184 self.makeDatasetRef("calexp"), 

185 ) 

186 

187 du = {"visit": 48, "tract": 265, "skymap": "big", "instrument": "dummy"} 

188 self.assertTemplate(tmplstr, "run2/calexpT/v00048", self.makeDatasetRef("calexpT", du)) 

189 

190 # Ensure that this returns a relative path even if the first field 

191 # is optional 

192 tmplstr = "{run}/{tract:?}/{visit:?}/f{physical_filter}" 

193 self.assertTemplate(tmplstr, "run2/52/fMost_Amazing_U_Filter_Ever", ref) 

194 

195 # Ensure that // from optionals are converted to singles 

196 tmplstr = "{run}/{datasetType}/{patch:?}/{tract:?}/f{physical_filter}" 

197 self.assertTemplate(tmplstr, "run2/calexp/fMost_Amazing_U_Filter_Ever", ref) 

198 

199 # Optionals with some text between fields 

200 tmplstr = "{run}/{datasetType}/p{patch:?}_t{tract:?}/f{physical_filter}" 

201 self.assertTemplate(tmplstr, "run2/calexp/p/fMost_Amazing_U_Filter_Ever", ref) 

202 tmplstr = "{run}/{datasetType}/p{patch:?}_t{visit:04d?}/f{physical_filter}" 

203 self.assertTemplate(tmplstr, "run2/calexp/p_t0052/fMost_Amazing_U_Filter_Ever", ref) 

204 

205 def testComponent(self): 

206 """Test handling of components in templates.""" 

207 refMetricOutput = self.makeDatasetRef("metric.output") 

208 refMetric = self.makeDatasetRef("metric") 

209 refMaskedImage = self.makeDatasetRef("calexp.maskedimage.variance") 

210 refWcs = self.makeDatasetRef("calexp.wcs") 

211 

212 tmplstr = "{run}_c_{component}_v{visit}" 

213 self.assertTemplate(tmplstr, "run2_c_output_v52", refMetricOutput) 

214 

215 # We want this template to have both a directory and basename, to 

216 # test that the right parts of the output are replaced. 

217 tmplstr = "{component:?}/{run}_{component:?}_{visit}" 

218 self.assertTemplate(tmplstr, "run2_52", refMetric) 

219 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput) 

220 self.assertTemplate(tmplstr, "maskedimage.variance/run2_maskedimage_variance_52", refMaskedImage) 

221 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput) 

222 

223 # Providing a component but not using it 

224 tmplstr = "{run}/{datasetType}/v{visit:05d}" 

225 with self.assertRaises(KeyError): 

226 self.assertTemplate(tmplstr, "", refWcs) 

227 

228 def testFields(self): 

229 # Template, mandatory fields, optional non-special fields, 

230 # special fields, optional special fields 

231 testData = ( 

232 ( 

233 "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail", 

234 {"visit", "physical_filter"}, 

235 set(), 

236 {"run", "datasetType"}, 

237 set(), 

238 ), 

239 ( 

240 "{run}/{component:?}_{visit}", 

241 {"visit"}, 

242 set(), 

243 {"run"}, 

244 {"component"}, 

245 ), 

246 ( 

247 "{run}/{component:?}_{visit:?}_{physical_filter}_{instrument}_{datasetType}", 

248 {"physical_filter", "instrument"}, 

249 {"visit"}, 

250 {"run", "datasetType"}, 

251 {"component"}, 

252 ), 

253 ) 

254 for tmplstr, mandatory, optional, special, optionalSpecial in testData: 

255 with self.subTest(template=tmplstr): 

256 tmpl = FileTemplate(tmplstr) 

257 fields = tmpl.fields() 

258 self.assertEqual(fields, mandatory) 

259 fields = tmpl.fields(optionals=True) 

260 self.assertEqual(fields, mandatory | optional) 

261 fields = tmpl.fields(specials=True) 

262 self.assertEqual(fields, mandatory | special) 

263 fields = tmpl.fields(specials=True, optionals=True) 

264 self.assertEqual(fields, mandatory | special | optional | optionalSpecial) 

265 

266 def testSimpleConfig(self): 

267 """Test reading from config file""" 

268 configRoot = os.path.join(TESTDIR, "config", "templates") 

269 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml")) 

270 templates = FileTemplates(config1, universe=self.universe) 

271 ref = self.makeDatasetRef("calexp") 

272 tmpl = templates.getTemplate(ref) 

273 self.assertIsInstance(tmpl, FileTemplate) 

274 

275 # This config file should not allow defaulting 

276 ref2 = self.makeDatasetRef("unknown") 

277 with self.assertRaises(KeyError): 

278 templates.getTemplate(ref2) 

279 

280 # This should fall through the datasetTypeName check and use 

281 # StorageClass instead 

282 ref3 = self.makeDatasetRef("unknown2", storageClassName="StorageClassX") 

283 tmplSc = templates.getTemplate(ref3) 

284 self.assertIsInstance(tmplSc, FileTemplate) 

285 

286 # Try with a component: one with defined formatter and one without 

287 refWcs = self.makeDatasetRef("calexp.wcs") 

288 refImage = self.makeDatasetRef("calexp.image") 

289 tmplCalexp = templates.getTemplate(ref) 

290 tmplWcs = templates.getTemplate(refWcs) # Should be special 

291 tmpl_image = templates.getTemplate(refImage) 

292 self.assertIsInstance(tmplCalexp, FileTemplate) 

293 self.assertIsInstance(tmpl_image, FileTemplate) 

294 self.assertIsInstance(tmplWcs, FileTemplate) 

295 self.assertEqual(tmplCalexp, tmpl_image) 

296 self.assertNotEqual(tmplCalexp, tmplWcs) 

297 

298 # Check dimensions lookup order. 

299 # The order should be: dataset type name, dimension, storage class 

300 # This one will not match name but might match storage class. 

301 # It should match dimensions 

302 refDims = self.makeDatasetRef( 

303 "nomatch", dataId={"instrument": "LSST", "physical_filter": "z"}, storageClassName="StorageClassX" 

304 ) 

305 tmplDims = templates.getTemplate(refDims) 

306 self.assertIsInstance(tmplDims, FileTemplate) 

307 self.assertNotEqual(tmplDims, tmplSc) 

308 

309 # Test that instrument overrides retrieve specialist templates 

310 refPvi = self.makeDatasetRef("pvi") 

311 refPviHsc = self.makeDatasetRef("pvi", dataId={"instrument": "HSC", "physical_filter": "z"}) 

312 refPviLsst = self.makeDatasetRef("pvi", dataId={"instrument": "LSST", "physical_filter": "z"}) 

313 

314 tmplPvi = templates.getTemplate(refPvi) 

315 tmplPviHsc = templates.getTemplate(refPviHsc) 

316 tmplPviLsst = templates.getTemplate(refPviLsst) 

317 self.assertEqual(tmplPvi, tmplPviLsst) 

318 self.assertNotEqual(tmplPvi, tmplPviHsc) 

319 

320 # Have instrument match and dimensions look up with no name match 

321 refNoPviHsc = self.makeDatasetRef( 

322 "pvix", dataId={"instrument": "HSC", "physical_filter": "z"}, storageClassName="StorageClassX" 

323 ) 

324 tmplNoPviHsc = templates.getTemplate(refNoPviHsc) 

325 self.assertNotEqual(tmplNoPviHsc, tmplDims) 

326 self.assertNotEqual(tmplNoPviHsc, tmplPviHsc) 

327 

328 # Format config file with defaulting 

329 config2 = FileTemplatesConfig(os.path.join(configRoot, "templates-withdefault.yaml")) 

330 templates = FileTemplates(config2, universe=self.universe) 

331 tmpl = templates.getTemplate(ref2) 

332 self.assertIsInstance(tmpl, FileTemplate) 

333 

334 # Format config file with bad format string 

335 with self.assertRaises(FileTemplateValidationError): 

336 FileTemplates(os.path.join(configRoot, "templates-bad.yaml"), universe=self.universe) 

337 

338 # Config file with no defaulting mentioned 

339 config3 = os.path.join(configRoot, "templates-nodefault2.yaml") 

340 templates = FileTemplates(config3, universe=self.universe) 

341 with self.assertRaises(KeyError): 

342 templates.getTemplate(ref2) 

343 

344 # Try again but specify a default in the constructor 

345 default = "{run}/{datasetType}/{physical_filter}" 

346 templates = FileTemplates(config3, default=default, universe=self.universe) 

347 tmpl = templates.getTemplate(ref2) 

348 self.assertEqual(tmpl.template, default) 

349 

350 def testValidation(self): 

351 configRoot = os.path.join(TESTDIR, "config", "templates") 

352 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml")) 

353 templates = FileTemplates(config1, universe=self.universe) 

354 

355 entities = {} 

356 entities["calexp"] = self.makeDatasetRef( 

357 "calexp", 

358 storageClassName="StorageClassX", 

359 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 52}, 

360 ) 

361 

362 with self.assertLogs(level="WARNING") as cm: 

363 templates.validateTemplates(entities.values(), logFailures=True) 

364 self.assertIn("Unchecked keys", cm.output[0]) 

365 self.assertIn("StorageClassX", cm.output[0]) 

366 

367 entities["pvi"] = self.makeDatasetRef( 

368 "pvi", storageClassName="StorageClassX", dataId={"instrument": "dummy", "physical_filter": "i"} 

369 ) 

370 entities["StorageClassX"] = self.makeDatasetRef( 

371 "storageClass", storageClassName="StorageClassX", dataId={"instrument": "dummy", "visit": 2} 

372 ) 

373 entities["calexp.wcs"] = self.makeDatasetRef( 

374 "calexp.wcs", 

375 storageClassName="StorageClassX", 

376 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 23}, 

377 conform=False, 

378 ) 

379 

380 entities["instrument+physical_filter"] = self.makeDatasetRef( 

381 "filter_inst", 

382 storageClassName="StorageClassX", 

383 dataId={"physical_filter": "i", "instrument": "SCUBA"}, 

384 ) 

385 entities["hsc+pvi"] = self.makeDatasetRef( 

386 "pvi", storageClassName="StorageClassX", dataId={"physical_filter": "i", "instrument": "HSC"} 

387 ) 

388 

389 entities["hsc+instrument+physical_filter"] = self.makeDatasetRef( 

390 "filter_inst", 

391 storageClassName="StorageClassX", 

392 dataId={"physical_filter": "i", "instrument": "HSC"}, 

393 ) 

394 

395 entities["metric6"] = self.makeDatasetRef( 

396 "filter_inst", 

397 storageClassName="Integer", 

398 dataId={"physical_filter": "i", "instrument": "HSC"}, 

399 ) 

400 

401 templates.validateTemplates(entities.values(), logFailures=True) 

402 

403 # Rerun but with a failure 

404 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX", dataId={"band": "i"}) 

405 with self.assertRaises(FileTemplateValidationError): 

406 with self.assertLogs(level="FATAL"): 

407 templates.validateTemplates(entities.values(), logFailures=True) 

408 

409 

410if __name__ == "__main__": 

411 unittest.main()