Coverage for tests/test_templates.py: 10%

178 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-12-01 11:00 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Test file name templating.""" 

29 

30import os.path 

31import unittest 

32import uuid 

33 

34from lsst.daf.butler import ( 

35 DataCoordinate, 

36 DatasetId, 

37 DatasetRef, 

38 DatasetType, 

39 DimensionUniverse, 

40 StorageClass, 

41) 

42from lsst.daf.butler.datastore.file_templates import ( 

43 FileTemplate, 

44 FileTemplates, 

45 FileTemplatesConfig, 

46 FileTemplateValidationError, 

47) 

48 

49TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

50 

51PlaceHolder = StorageClass("PlaceHolder") 

52 

53REFUUID = DatasetId(int=uuid.uuid4().int) 

54 

55 

56class TestFileTemplates(unittest.TestCase): 

57 """Test creation of paths from templates.""" 

58 

59 def makeDatasetRef( 

60 self, datasetTypeName, dataId=None, storageClassName="DefaultStorageClass", run="run2", conform=True 

61 ): 

62 """Make a simple DatasetRef""" 

63 if dataId is None: 

64 dataId = self.dataId 

65 if "physical_filter" in dataId and "band" not in dataId: 

66 dataId["band"] = "b" # Add fake band. 

67 dimensions = self.universe.conform(dataId.keys()) 

68 dataId = DataCoordinate.standardize(dataId, dimensions=dimensions) 

69 

70 # Pretend we have a parent if this looks like a composite 

71 compositeName, componentName = DatasetType.splitDatasetTypeName(datasetTypeName) 

72 parentStorageClass = PlaceHolder if componentName else None 

73 

74 datasetType = DatasetType( 

75 datasetTypeName, 

76 dimensions, 

77 StorageClass(storageClassName), 

78 parentStorageClass=parentStorageClass, 

79 ) 

80 return DatasetRef(datasetType, dataId, id=REFUUID, run=run, conform=conform) 

81 

82 def setUp(self): 

83 self.universe = DimensionUniverse() 

84 self.dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "Most Amazing U Filter Ever"} 

85 

86 def assertTemplate(self, template, answer, ref): 

87 fileTmpl = FileTemplate(template) 

88 path = fileTmpl.format(ref) 

89 self.assertEqual(path, answer) 

90 

91 def testBasic(self): 

92 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}" 

93 self.assertTemplate( 

94 tmplstr, 

95 "run2/calexp/00052/Most_Amazing_U_Filter_Ever", 

96 self.makeDatasetRef("calexp"), 

97 ) 

98 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail" 

99 self.assertTemplate( 

100 tmplstr, 

101 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail", 

102 self.makeDatasetRef("calexp"), 

103 ) 

104 

105 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}" 

106 self.assertTemplate( 

107 tmplstr, 

108 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run2", 

109 self.makeDatasetRef("calexp"), 

110 ) 

111 self.assertTemplate( 

112 tmplstr, 

113 "run_2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2", 

114 self.makeDatasetRef("calexp", run="run/2"), 

115 ) 

116 

117 # Check that the id is sufficient without any other information. 

118 self.assertTemplate("{id}", str(REFUUID), self.makeDatasetRef("calexp", run="run2")) 

119 

120 self.assertTemplate("{run}/{id}", f"run2/{str(REFUUID)}", self.makeDatasetRef("calexp", run="run2")) 

121 

122 self.assertTemplate( 

123 "fixed/{id}", 

124 f"fixed/{str(REFUUID)}", 

125 self.makeDatasetRef("calexp", run="run2"), 

126 ) 

127 

128 self.assertTemplate( 

129 "fixed/{id}_{physical_filter}", 

130 f"fixed/{str(REFUUID)}_Most_Amazing_U_Filter_Ever", 

131 self.makeDatasetRef("calexp", run="run2"), 

132 ) 

133 

134 # Retain any "/" in run 

135 tmplstr = "{run:/}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}" 

136 self.assertTemplate( 

137 tmplstr, 

138 "run/2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2", 

139 self.makeDatasetRef("calexp", run="run/2"), 

140 ) 

141 

142 # Check that "." are replaced in the file basename, but not directory. 

143 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "g.10"} 

144 self.assertTemplate( 

145 tmplstr, 

146 "run.2/calexp/00052/g_10-trail-run_2", 

147 self.makeDatasetRef("calexp", run="run.2", dataId=dataId), 

148 ) 

149 

150 with self.assertRaises(FileTemplateValidationError): 

151 FileTemplate("no fields at all") 

152 

153 with self.assertRaises(FileTemplateValidationError): 

154 FileTemplate("{visit}") 

155 

156 with self.assertRaises(FileTemplateValidationError): 

157 FileTemplate("{run}_{datasetType}") 

158 

159 with self.assertRaises(FileTemplateValidationError): 

160 FileTemplate("{id}/fixed") 

161 

162 def testRunOrCollectionNeeded(self): 

163 tmplstr = "{datasetType}/{visit:05d}/{physical_filter}" 

164 with self.assertRaises(FileTemplateValidationError): 

165 self.assertTemplate(tmplstr, "run2/calexp/00052/U", self.makeDatasetRef("calexp")) 

166 

167 def testNoRecord(self): 

168 # Attaching records is not possible in this test code but we can check 

169 # that a missing record when a metadata entry has been requested 

170 # does fail. 

171 tmplstr = "{run}/{datasetType}/{visit.name}/{physical_filter}" 

172 with self.assertRaises(RuntimeError) as cm: 

173 self.assertTemplate(tmplstr, "", self.makeDatasetRef("calexp")) 

174 self.assertIn("No metadata", str(cm.exception)) 

175 

176 def testOptional(self): 

177 """Optional units in templates.""" 

178 ref = self.makeDatasetRef("calexp") 

179 tmplstr = "{run}/{datasetType}/v{visit:05d}_f{physical_filter:?}" 

180 self.assertTemplate( 

181 tmplstr, 

182 "run2/calexp/v00052_fMost_Amazing_U_Filter_Ever", 

183 self.makeDatasetRef("calexp"), 

184 ) 

185 

186 du = {"visit": 48, "tract": 265, "skymap": "big", "instrument": "dummy"} 

187 self.assertTemplate(tmplstr, "run2/calexpT/v00048", self.makeDatasetRef("calexpT", du)) 

188 

189 # Ensure that this returns a relative path even if the first field 

190 # is optional 

191 tmplstr = "{run}/{tract:?}/{visit:?}/f{physical_filter}" 

192 self.assertTemplate(tmplstr, "run2/52/fMost_Amazing_U_Filter_Ever", ref) 

193 

194 # Ensure that // from optionals are converted to singles 

195 tmplstr = "{run}/{datasetType}/{patch:?}/{tract:?}/f{physical_filter}" 

196 self.assertTemplate(tmplstr, "run2/calexp/fMost_Amazing_U_Filter_Ever", ref) 

197 

198 # Optionals with some text between fields 

199 tmplstr = "{run}/{datasetType}/p{patch:?}_t{tract:?}/f{physical_filter}" 

200 self.assertTemplate(tmplstr, "run2/calexp/p/fMost_Amazing_U_Filter_Ever", ref) 

201 tmplstr = "{run}/{datasetType}/p{patch:?}_t{visit:04d?}/f{physical_filter}" 

202 self.assertTemplate(tmplstr, "run2/calexp/p_t0052/fMost_Amazing_U_Filter_Ever", ref) 

203 

204 def testComponent(self): 

205 """Test handling of components in templates.""" 

206 refMetricOutput = self.makeDatasetRef("metric.output") 

207 refMetric = self.makeDatasetRef("metric") 

208 refMaskedImage = self.makeDatasetRef("calexp.maskedimage.variance") 

209 refWcs = self.makeDatasetRef("calexp.wcs") 

210 

211 tmplstr = "{run}_c_{component}_v{visit}" 

212 self.assertTemplate(tmplstr, "run2_c_output_v52", refMetricOutput) 

213 

214 # We want this template to have both a directory and basename, to 

215 # test that the right parts of the output are replaced. 

216 tmplstr = "{component:?}/{run}_{component:?}_{visit}" 

217 self.assertTemplate(tmplstr, "run2_52", refMetric) 

218 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput) 

219 self.assertTemplate(tmplstr, "maskedimage.variance/run2_maskedimage_variance_52", refMaskedImage) 

220 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput) 

221 

222 # Providing a component but not using it 

223 tmplstr = "{run}/{datasetType}/v{visit:05d}" 

224 with self.assertRaises(KeyError): 

225 self.assertTemplate(tmplstr, "", refWcs) 

226 

227 def testFields(self): 

228 # Template, mandatory fields, optional non-special fields, 

229 # special fields, optional special fields 

230 testData = ( 

231 ( 

232 "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail", 

233 {"visit", "physical_filter"}, 

234 set(), 

235 {"run", "datasetType"}, 

236 set(), 

237 ), 

238 ( 

239 "{run}/{component:?}_{visit}", 

240 {"visit"}, 

241 set(), 

242 {"run"}, 

243 {"component"}, 

244 ), 

245 ( 

246 "{run}/{component:?}_{visit:?}_{physical_filter}_{instrument}_{datasetType}", 

247 {"physical_filter", "instrument"}, 

248 {"visit"}, 

249 {"run", "datasetType"}, 

250 {"component"}, 

251 ), 

252 ) 

253 for tmplstr, mandatory, optional, special, optionalSpecial in testData: 

254 with self.subTest(template=tmplstr): 

255 tmpl = FileTemplate(tmplstr) 

256 fields = tmpl.fields() 

257 self.assertEqual(fields, mandatory) 

258 fields = tmpl.fields(optionals=True) 

259 self.assertEqual(fields, mandatory | optional) 

260 fields = tmpl.fields(specials=True) 

261 self.assertEqual(fields, mandatory | special) 

262 fields = tmpl.fields(specials=True, optionals=True) 

263 self.assertEqual(fields, mandatory | special | optional | optionalSpecial) 

264 

265 def testSimpleConfig(self): 

266 """Test reading from config file""" 

267 configRoot = os.path.join(TESTDIR, "config", "templates") 

268 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml")) 

269 templates = FileTemplates(config1, universe=self.universe) 

270 ref = self.makeDatasetRef("calexp") 

271 tmpl = templates.getTemplate(ref) 

272 self.assertIsInstance(tmpl, FileTemplate) 

273 

274 # This config file should not allow defaulting 

275 ref2 = self.makeDatasetRef("unknown") 

276 with self.assertRaises(KeyError): 

277 templates.getTemplate(ref2) 

278 

279 # This should fall through the datasetTypeName check and use 

280 # StorageClass instead 

281 ref3 = self.makeDatasetRef("unknown2", storageClassName="StorageClassX") 

282 tmplSc = templates.getTemplate(ref3) 

283 self.assertIsInstance(tmplSc, FileTemplate) 

284 

285 # Try with a component: one with defined formatter and one without 

286 refWcs = self.makeDatasetRef("calexp.wcs") 

287 refImage = self.makeDatasetRef("calexp.image") 

288 tmplCalexp = templates.getTemplate(ref) 

289 tmplWcs = templates.getTemplate(refWcs) # Should be special 

290 tmpl_image = templates.getTemplate(refImage) 

291 self.assertIsInstance(tmplCalexp, FileTemplate) 

292 self.assertIsInstance(tmpl_image, FileTemplate) 

293 self.assertIsInstance(tmplWcs, FileTemplate) 

294 self.assertEqual(tmplCalexp, tmpl_image) 

295 self.assertNotEqual(tmplCalexp, tmplWcs) 

296 

297 # Check dimensions lookup order. 

298 # The order should be: dataset type name, dimension, storage class 

299 # This one will not match name but might match storage class. 

300 # It should match dimensions 

301 refDims = self.makeDatasetRef( 

302 "nomatch", dataId={"instrument": "LSST", "physical_filter": "z"}, storageClassName="StorageClassX" 

303 ) 

304 tmplDims = templates.getTemplate(refDims) 

305 self.assertIsInstance(tmplDims, FileTemplate) 

306 self.assertNotEqual(tmplDims, tmplSc) 

307 

308 # Test that instrument overrides retrieve specialist templates 

309 refPvi = self.makeDatasetRef("pvi") 

310 refPviHsc = self.makeDatasetRef("pvi", dataId={"instrument": "HSC", "physical_filter": "z"}) 

311 refPviLsst = self.makeDatasetRef("pvi", dataId={"instrument": "LSST", "physical_filter": "z"}) 

312 

313 tmplPvi = templates.getTemplate(refPvi) 

314 tmplPviHsc = templates.getTemplate(refPviHsc) 

315 tmplPviLsst = templates.getTemplate(refPviLsst) 

316 self.assertEqual(tmplPvi, tmplPviLsst) 

317 self.assertNotEqual(tmplPvi, tmplPviHsc) 

318 

319 # Have instrument match and dimensions look up with no name match 

320 refNoPviHsc = self.makeDatasetRef( 

321 "pvix", dataId={"instrument": "HSC", "physical_filter": "z"}, storageClassName="StorageClassX" 

322 ) 

323 tmplNoPviHsc = templates.getTemplate(refNoPviHsc) 

324 self.assertNotEqual(tmplNoPviHsc, tmplDims) 

325 self.assertNotEqual(tmplNoPviHsc, tmplPviHsc) 

326 

327 # Format config file with defaulting 

328 config2 = FileTemplatesConfig(os.path.join(configRoot, "templates-withdefault.yaml")) 

329 templates = FileTemplates(config2, universe=self.universe) 

330 tmpl = templates.getTemplate(ref2) 

331 self.assertIsInstance(tmpl, FileTemplate) 

332 

333 # Format config file with bad format string 

334 with self.assertRaises(FileTemplateValidationError): 

335 FileTemplates(os.path.join(configRoot, "templates-bad.yaml"), universe=self.universe) 

336 

337 # Config file with no defaulting mentioned 

338 config3 = os.path.join(configRoot, "templates-nodefault2.yaml") 

339 templates = FileTemplates(config3, universe=self.universe) 

340 with self.assertRaises(KeyError): 

341 templates.getTemplate(ref2) 

342 

343 # Try again but specify a default in the constructor 

344 default = "{run}/{datasetType}/{physical_filter}" 

345 templates = FileTemplates(config3, default=default, universe=self.universe) 

346 tmpl = templates.getTemplate(ref2) 

347 self.assertEqual(tmpl.template, default) 

348 

349 def testValidation(self): 

350 configRoot = os.path.join(TESTDIR, "config", "templates") 

351 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml")) 

352 templates = FileTemplates(config1, universe=self.universe) 

353 

354 entities = {} 

355 entities["calexp"] = self.makeDatasetRef( 

356 "calexp", 

357 storageClassName="StorageClassX", 

358 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 52}, 

359 ) 

360 

361 with self.assertLogs(level="WARNING") as cm: 

362 templates.validateTemplates(entities.values(), logFailures=True) 

363 self.assertIn("Unchecked keys", cm.output[0]) 

364 self.assertIn("StorageClassX", cm.output[0]) 

365 

366 entities["pvi"] = self.makeDatasetRef( 

367 "pvi", storageClassName="StorageClassX", dataId={"instrument": "dummy", "physical_filter": "i"} 

368 ) 

369 entities["StorageClassX"] = self.makeDatasetRef( 

370 "storageClass", storageClassName="StorageClassX", dataId={"instrument": "dummy", "visit": 2} 

371 ) 

372 entities["calexp.wcs"] = self.makeDatasetRef( 

373 "calexp.wcs", 

374 storageClassName="StorageClassX", 

375 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 23}, 

376 conform=False, 

377 ) 

378 

379 entities["instrument+physical_filter"] = self.makeDatasetRef( 

380 "filter_inst", 

381 storageClassName="StorageClassX", 

382 dataId={"physical_filter": "i", "instrument": "SCUBA"}, 

383 ) 

384 entities["hsc+pvi"] = self.makeDatasetRef( 

385 "pvi", storageClassName="StorageClassX", dataId={"physical_filter": "i", "instrument": "HSC"} 

386 ) 

387 

388 entities["hsc+instrument+physical_filter"] = self.makeDatasetRef( 

389 "filter_inst", 

390 storageClassName="StorageClassX", 

391 dataId={"physical_filter": "i", "instrument": "HSC"}, 

392 ) 

393 

394 entities["metric6"] = self.makeDatasetRef( 

395 "filter_inst", 

396 storageClassName="Integer", 

397 dataId={"physical_filter": "i", "instrument": "HSC"}, 

398 ) 

399 

400 templates.validateTemplates(entities.values(), logFailures=True) 

401 

402 # Rerun but with a failure 

403 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX", dataId={"band": "i"}) 

404 with self.assertRaises(FileTemplateValidationError): 

405 with self.assertLogs(level="FATAL"): 

406 templates.validateTemplates(entities.values(), logFailures=True) 

407 

408 

409if __name__ == "__main__": 

410 unittest.main()