Coverage for tests/test_templates.py: 10%

185 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-05-02 10:24 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Test file name templating.""" 

29 

30import os.path 

31import unittest 

32import uuid 

33 

34from lsst.daf.butler import ( 

35 DataCoordinate, 

36 DatasetId, 

37 DatasetRef, 

38 DatasetType, 

39 DimensionUniverse, 

40 StorageClass, 

41) 

42from lsst.daf.butler.datastore.file_templates import ( 

43 FileTemplate, 

44 FileTemplates, 

45 FileTemplatesConfig, 

46 FileTemplateValidationError, 

47) 

48 

49TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

50 

51PlaceHolder = StorageClass("PlaceHolder") 

52 

53REFUUID = DatasetId(int=uuid.uuid4().int) 

54 

55 

56class TestFileTemplates(unittest.TestCase): 

57 """Test creation of paths from templates.""" 

58 

59 def makeDatasetRef( 

60 self, datasetTypeName, dataId=None, storageClassName="DefaultStorageClass", run="run2", conform=True 

61 ): 

62 """Make a simple DatasetRef""" 

63 if dataId is None: 

64 dataId = self.dataId 

65 if "physical_filter" in dataId and "band" not in dataId: 

66 dataId["band"] = "b" # Add fake band. 

67 dimensions = self.universe.conform(dataId.keys()) 

68 dataId = DataCoordinate.standardize(dataId, dimensions=dimensions) 

69 

70 # Pretend we have a parent if this looks like a composite 

71 compositeName, componentName = DatasetType.splitDatasetTypeName(datasetTypeName) 

72 parentStorageClass = PlaceHolder if componentName else None 

73 

74 datasetType = DatasetType( 

75 datasetTypeName, 

76 dimensions, 

77 StorageClass(storageClassName), 

78 parentStorageClass=parentStorageClass, 

79 ) 

80 return DatasetRef(datasetType, dataId, id=REFUUID, run=run, conform=conform) 

81 

82 def setUp(self): 

83 self.universe = DimensionUniverse() 

84 self.dataId = { 

85 "instrument": "dummy", 

86 "visit": 52, 

87 "physical_filter": "Most Amazing U Filter Ever", 

88 "day_obs": 20200101, 

89 } 

90 

91 def assertTemplate(self, template, answer, ref): 

92 fileTmpl = FileTemplate(template) 

93 path = fileTmpl.format(ref) 

94 self.assertEqual(path, answer) 

95 

96 def testBasic(self): 

97 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}" 

98 self.assertTemplate( 

99 tmplstr, 

100 "run2/calexp/00052/Most_Amazing_U_Filter_Ever", 

101 self.makeDatasetRef("calexp"), 

102 ) 

103 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail" 

104 self.assertTemplate( 

105 tmplstr, 

106 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail", 

107 self.makeDatasetRef("calexp"), 

108 ) 

109 

110 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}" 

111 self.assertTemplate( 

112 tmplstr, 

113 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run2", 

114 self.makeDatasetRef("calexp"), 

115 ) 

116 self.assertTemplate( 

117 tmplstr, 

118 "run_2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2", 

119 self.makeDatasetRef("calexp", run="run/2"), 

120 ) 

121 

122 # Check that the id is sufficient without any other information. 

123 self.assertTemplate("{id}", str(REFUUID), self.makeDatasetRef("calexp", run="run2")) 

124 

125 self.assertTemplate("{run}/{id}", f"run2/{str(REFUUID)}", self.makeDatasetRef("calexp", run="run2")) 

126 

127 self.assertTemplate( 

128 "fixed/{id}", 

129 f"fixed/{str(REFUUID)}", 

130 self.makeDatasetRef("calexp", run="run2"), 

131 ) 

132 

133 self.assertTemplate( 

134 "fixed/{id}_{physical_filter}", 

135 f"fixed/{str(REFUUID)}_Most_Amazing_U_Filter_Ever", 

136 self.makeDatasetRef("calexp", run="run2"), 

137 ) 

138 

139 # Retain any "/" in run 

140 tmplstr = "{run:/}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}" 

141 self.assertTemplate( 

142 tmplstr, 

143 "run/2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2", 

144 self.makeDatasetRef("calexp", run="run/2"), 

145 ) 

146 

147 # Check that "." are replaced in the file basename, but not directory. 

148 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "g.10", "day_obs": 20250101} 

149 self.assertTemplate( 

150 tmplstr, 

151 "run.2/calexp/00052/g_10-trail-run_2", 

152 self.makeDatasetRef("calexp", run="run.2", dataId=dataId), 

153 ) 

154 

155 with self.assertRaises(FileTemplateValidationError): 

156 FileTemplate("no fields at all") 

157 

158 with self.assertRaises(FileTemplateValidationError): 

159 FileTemplate("{visit}") 

160 

161 with self.assertRaises(FileTemplateValidationError): 

162 FileTemplate("{run}_{datasetType}") 

163 

164 with self.assertRaises(FileTemplateValidationError): 

165 FileTemplate("{id}/fixed") 

166 

167 with self.assertRaises(FileTemplateValidationError): 

168 FileTemplate("{run}/../{datasetType}_{visit}") 

169 

170 def testAlternates(self): 

171 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter|day_obs}_{day_obs|physical_filter}" 

172 self.assertTemplate( 

173 tmplstr, 

174 "run2/calexp/00052/Most_Amazing_U_Filter_Ever_20200101", 

175 self.makeDatasetRef("calexp"), 

176 ) 

177 tmplstr = "{run}/{datasetType}/{exposure|visit:05d}/{physical_filter|day_obs}_{group|exposure:?}" 

178 self.assertTemplate( 

179 tmplstr, 

180 "run2/calexp/00052/Most_Amazing_U_Filter_Ever", 

181 self.makeDatasetRef("calexp"), 

182 ) 

183 

184 def testRunOrCollectionNeeded(self): 

185 tmplstr = "{datasetType}/{visit:05d}/{physical_filter}" 

186 with self.assertRaises(FileTemplateValidationError): 

187 self.assertTemplate(tmplstr, "run2/calexp/00052/U", self.makeDatasetRef("calexp")) 

188 

189 def testNoRecord(self): 

190 # Attaching records is not possible in this test code but we can check 

191 # that a missing record when a metadata entry has been requested 

192 # does fail. 

193 tmplstr = "{run}/{datasetType}/{visit.name}/{physical_filter}" 

194 with self.assertRaises(RuntimeError) as cm: 

195 self.assertTemplate(tmplstr, "", self.makeDatasetRef("calexp")) 

196 self.assertIn("No metadata", str(cm.exception)) 

197 

198 def testOptional(self): 

199 """Optional units in templates.""" 

200 ref = self.makeDatasetRef("calexp") 

201 tmplstr = "{run}/{datasetType}/v{visit:05d}_f{physical_filter:?}_{skypix:?}" 

202 self.assertTemplate( 

203 tmplstr, 

204 "run2/calexp/v00052_fMost_Amazing_U_Filter_Ever", 

205 self.makeDatasetRef("calexp"), 

206 ) 

207 

208 du = {"visit": 48, "tract": 265, "skymap": "big", "instrument": "dummy", "htm7": 12345} 

209 self.assertTemplate(tmplstr, "run2/calexpT/v00048_12345", self.makeDatasetRef("calexpT", du)) 

210 

211 # Ensure that this returns a relative path even if the first field 

212 # is optional 

213 tmplstr = "{run}/{tract:?}/{visit:?}/f{physical_filter}" 

214 self.assertTemplate(tmplstr, "run2/52/fMost_Amazing_U_Filter_Ever", ref) 

215 

216 # Ensure that // from optionals are converted to singles 

217 tmplstr = "{run}/{datasetType}/{patch:?}/{tract:?}/f{physical_filter}" 

218 self.assertTemplate(tmplstr, "run2/calexp/fMost_Amazing_U_Filter_Ever", ref) 

219 

220 # Optionals with some text between fields 

221 tmplstr = "{run}/{datasetType}/p{patch:?}_t{tract:?}/f{physical_filter}" 

222 self.assertTemplate(tmplstr, "run2/calexp/p/fMost_Amazing_U_Filter_Ever", ref) 

223 tmplstr = "{run}/{datasetType}/p{patch:?}_t{visit:04d?}/f{physical_filter}" 

224 self.assertTemplate(tmplstr, "run2/calexp/p_t0052/fMost_Amazing_U_Filter_Ever", ref) 

225 

226 def testComponent(self): 

227 """Test handling of components in templates.""" 

228 refMetricOutput = self.makeDatasetRef("metric.output") 

229 refMetric = self.makeDatasetRef("metric") 

230 refMaskedImage = self.makeDatasetRef("calexp.maskedimage.variance") 

231 refWcs = self.makeDatasetRef("calexp.wcs") 

232 

233 tmplstr = "{run}_c_{component}_v{visit}" 

234 self.assertTemplate(tmplstr, "run2_c_output_v52", refMetricOutput) 

235 

236 # We want this template to have both a directory and basename, to 

237 # test that the right parts of the output are replaced. 

238 tmplstr = "{component:?}/{run}_{component:?}_{visit}" 

239 self.assertTemplate(tmplstr, "run2_52", refMetric) 

240 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput) 

241 self.assertTemplate(tmplstr, "maskedimage.variance/run2_maskedimage_variance_52", refMaskedImage) 

242 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput) 

243 

244 # Providing a component but not using it 

245 tmplstr = "{run}/{datasetType}/v{visit:05d}" 

246 with self.assertRaises(KeyError): 

247 self.assertTemplate(tmplstr, "", refWcs) 

248 

249 def testFields(self): 

250 # Template, mandatory fields, optional non-special fields, 

251 # special fields, optional special fields 

252 testData = ( 

253 ( 

254 "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail", 

255 {"visit", "physical_filter"}, 

256 set(), 

257 {"run", "datasetType"}, 

258 set(), 

259 ), 

260 ( 

261 "{run}/{component:?}_{visit}", 

262 {"visit"}, 

263 set(), 

264 {"run"}, 

265 {"component"}, 

266 ), 

267 ( 

268 "{run}/{component:?}_{visit:?}_{physical_filter}_{instrument}_{datasetType}", 

269 {"physical_filter", "instrument"}, 

270 {"visit"}, 

271 {"run", "datasetType"}, 

272 {"component"}, 

273 ), 

274 ) 

275 for tmplstr, mandatory, optional, special, optionalSpecial in testData: 

276 with self.subTest(template=tmplstr): 

277 tmpl = FileTemplate(tmplstr) 

278 fields = tmpl.fields() 

279 self.assertEqual(fields, mandatory) 

280 fields = tmpl.fields(optionals=True) 

281 self.assertEqual(fields, mandatory | optional) 

282 fields = tmpl.fields(specials=True) 

283 self.assertEqual(fields, mandatory | special) 

284 fields = tmpl.fields(specials=True, optionals=True) 

285 self.assertEqual(fields, mandatory | special | optional | optionalSpecial) 

286 

287 def testSimpleConfig(self): 

288 """Test reading from config file""" 

289 configRoot = os.path.join(TESTDIR, "config", "templates") 

290 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml")) 

291 templates = FileTemplates(config1, universe=self.universe) 

292 ref = self.makeDatasetRef("calexp") 

293 tmpl = templates.getTemplate(ref) 

294 self.assertIsInstance(tmpl, FileTemplate) 

295 

296 # This config file should not allow defaulting 

297 ref2 = self.makeDatasetRef("unknown") 

298 with self.assertRaises(KeyError): 

299 templates.getTemplate(ref2) 

300 

301 # This should fall through the datasetTypeName check and use 

302 # StorageClass instead 

303 ref3 = self.makeDatasetRef("unknown2", storageClassName="StorageClassX") 

304 tmplSc = templates.getTemplate(ref3) 

305 self.assertIsInstance(tmplSc, FileTemplate) 

306 

307 # Try with a component: one with defined formatter and one without 

308 refWcs = self.makeDatasetRef("calexp.wcs") 

309 refImage = self.makeDatasetRef("calexp.image") 

310 tmplCalexp = templates.getTemplate(ref) 

311 tmplWcs = templates.getTemplate(refWcs) # Should be special 

312 tmpl_image = templates.getTemplate(refImage) 

313 self.assertIsInstance(tmplCalexp, FileTemplate) 

314 self.assertIsInstance(tmpl_image, FileTemplate) 

315 self.assertIsInstance(tmplWcs, FileTemplate) 

316 self.assertEqual(tmplCalexp, tmpl_image) 

317 self.assertNotEqual(tmplCalexp, tmplWcs) 

318 

319 # Check dimensions lookup order. 

320 # The order should be: dataset type name, dimension, storage class 

321 # This one will not match name but might match storage class. 

322 # It should match dimensions 

323 refDims = self.makeDatasetRef( 

324 "nomatch", dataId={"instrument": "LSST", "physical_filter": "z"}, storageClassName="StorageClassX" 

325 ) 

326 tmplDims = templates.getTemplate(refDims) 

327 self.assertIsInstance(tmplDims, FileTemplate) 

328 self.assertNotEqual(tmplDims, tmplSc) 

329 

330 # Test that instrument overrides retrieve specialist templates 

331 refPvi = self.makeDatasetRef("pvi") 

332 refPviHsc = self.makeDatasetRef("pvi", dataId={"instrument": "HSC", "physical_filter": "z"}) 

333 refPviLsst = self.makeDatasetRef("pvi", dataId={"instrument": "LSST", "physical_filter": "z"}) 

334 

335 tmplPvi = templates.getTemplate(refPvi) 

336 tmplPviHsc = templates.getTemplate(refPviHsc) 

337 tmplPviLsst = templates.getTemplate(refPviLsst) 

338 self.assertEqual(tmplPvi, tmplPviLsst) 

339 self.assertNotEqual(tmplPvi, tmplPviHsc) 

340 

341 # Have instrument match and dimensions look up with no name match 

342 refNoPviHsc = self.makeDatasetRef( 

343 "pvix", dataId={"instrument": "HSC", "physical_filter": "z"}, storageClassName="StorageClassX" 

344 ) 

345 tmplNoPviHsc = templates.getTemplate(refNoPviHsc) 

346 self.assertNotEqual(tmplNoPviHsc, tmplDims) 

347 self.assertNotEqual(tmplNoPviHsc, tmplPviHsc) 

348 

349 # Format config file with defaulting 

350 config2 = FileTemplatesConfig(os.path.join(configRoot, "templates-withdefault.yaml")) 

351 templates = FileTemplates(config2, universe=self.universe) 

352 tmpl = templates.getTemplate(ref2) 

353 self.assertIsInstance(tmpl, FileTemplate) 

354 

355 # Format config file with bad format string 

356 with self.assertRaises(FileTemplateValidationError): 

357 FileTemplates(os.path.join(configRoot, "templates-bad.yaml"), universe=self.universe) 

358 

359 # Config file with no defaulting mentioned 

360 config3 = os.path.join(configRoot, "templates-nodefault2.yaml") 

361 templates = FileTemplates(config3, universe=self.universe) 

362 with self.assertRaises(KeyError): 

363 templates.getTemplate(ref2) 

364 

365 # Try again but specify a default in the constructor 

366 default = "{run}/{datasetType}/{physical_filter}" 

367 templates = FileTemplates(config3, default=default, universe=self.universe) 

368 tmpl = templates.getTemplate(ref2) 

369 self.assertEqual(tmpl.template, default) 

370 

371 def testValidation(self): 

372 configRoot = os.path.join(TESTDIR, "config", "templates") 

373 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml")) 

374 templates = FileTemplates(config1, universe=self.universe) 

375 

376 entities = {} 

377 entities["calexp"] = self.makeDatasetRef( 

378 "calexp", 

379 storageClassName="StorageClassX", 

380 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 52}, 

381 ) 

382 

383 with self.assertLogs(level="WARNING") as cm: 

384 templates.validateTemplates(entities.values(), logFailures=True) 

385 self.assertIn("Unchecked keys", cm.output[0]) 

386 self.assertIn("StorageClassX", cm.output[0]) 

387 

388 entities["pvi"] = self.makeDatasetRef( 

389 "pvi", storageClassName="StorageClassX", dataId={"instrument": "dummy", "physical_filter": "i"} 

390 ) 

391 entities["StorageClassX"] = self.makeDatasetRef( 

392 "storageClass", storageClassName="StorageClassX", dataId={"instrument": "dummy", "visit": 2} 

393 ) 

394 entities["calexp.wcs"] = self.makeDatasetRef( 

395 "calexp.wcs", 

396 storageClassName="StorageClassX", 

397 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 23}, 

398 conform=False, 

399 ) 

400 

401 entities["instrument+physical_filter"] = self.makeDatasetRef( 

402 "filter_inst", 

403 storageClassName="StorageClassX", 

404 dataId={"physical_filter": "i", "instrument": "SCUBA"}, 

405 ) 

406 entities["hsc+pvi"] = self.makeDatasetRef( 

407 "pvi", storageClassName="StorageClassX", dataId={"physical_filter": "i", "instrument": "HSC"} 

408 ) 

409 

410 entities["hsc+instrument+physical_filter"] = self.makeDatasetRef( 

411 "filter_inst", 

412 storageClassName="StorageClassX", 

413 dataId={"physical_filter": "i", "instrument": "HSC"}, 

414 ) 

415 

416 entities["metric6"] = self.makeDatasetRef( 

417 "filter_inst", 

418 storageClassName="Integer", 

419 dataId={"physical_filter": "i", "instrument": "HSC"}, 

420 ) 

421 

422 templates.validateTemplates(entities.values(), logFailures=True) 

423 

424 # Rerun but with a failure 

425 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX", dataId={"band": "i"}) 

426 with self.assertRaises(FileTemplateValidationError): 

427 with self.assertLogs(level="FATAL"): 

428 templates.validateTemplates(entities.values(), logFailures=True) 

429 

430 

431if __name__ == "__main__": 

432 unittest.main()