Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Test file name templating.""" 

23 

24import os.path 

25import unittest 

26 

27from lsst.daf.butler import DatasetType, DatasetRef, FileTemplates, DimensionUniverse, \ 

28 FileTemplate, FileTemplatesConfig, StorageClass, FileTemplateValidationError, \ 

29 DimensionGraph 

30 

31TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

32 

33PlaceHolder = StorageClass("PlaceHolder") 

34 

35 

36class TestFileTemplates(unittest.TestCase): 

37 """Test creation of paths from templates.""" 

38 

39 def makeDatasetRef(self, datasetTypeName, dataId=None, storageClassName="DefaultStorageClass", 

40 run="run2", conform=True): 

41 """Make a simple DatasetRef""" 

42 if dataId is None: 

43 dataId = self.dataId 

44 

45 # Pretend we have a parent if this looks like a composite 

46 compositeName, componentName = DatasetType.splitDatasetTypeName(datasetTypeName) 

47 parentStorageClass = PlaceHolder if componentName else None 

48 

49 datasetType = DatasetType(datasetTypeName, DimensionGraph(self.universe, names=dataId.keys()), 

50 StorageClass(storageClassName), 

51 parentStorageClass=parentStorageClass) 

52 return DatasetRef(datasetType, dataId, id=1, run=run, conform=conform) 

53 

54 def setUp(self): 

55 self.universe = DimensionUniverse() 

56 self.dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "Most Amazing U Filter Ever"} 

57 

58 def assertTemplate(self, template, answer, ref): 

59 fileTmpl = FileTemplate(template) 

60 path = fileTmpl.format(ref) 

61 self.assertEqual(path, answer) 

62 

63 def testBasic(self): 

64 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}" 

65 self.assertTemplate(tmplstr, 

66 "run2/calexp/00052/Most_Amazing_U_Filter_Ever", 

67 self.makeDatasetRef("calexp", conform=False)) 

68 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail" 

69 self.assertTemplate(tmplstr, 

70 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail", 

71 self.makeDatasetRef("calexp", conform=False)) 

72 

73 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}" 

74 self.assertTemplate(tmplstr, 

75 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run2", 

76 self.makeDatasetRef("calexp", conform=False)) 

77 self.assertTemplate(tmplstr, 

78 "run_2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2", 

79 self.makeDatasetRef("calexp", run="run/2", conform=False)) 

80 

81 # Retain any "/" in run 

82 tmplstr = "{run:/}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}" 

83 self.assertTemplate(tmplstr, 

84 "run/2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2", 

85 self.makeDatasetRef("calexp", run="run/2", conform=False)) 

86 

87 # Check that "." are replaced in the file basename, but not directory. 

88 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "g.10"} 

89 self.assertTemplate(tmplstr, 

90 "run.2/calexp/00052/g_10-trail-run_2", 

91 self.makeDatasetRef("calexp", run="run.2", dataId=dataId, conform=False)) 

92 

93 with self.assertRaises(FileTemplateValidationError): 

94 FileTemplate("no fields at all") 

95 

96 with self.assertRaises(FileTemplateValidationError): 

97 FileTemplate("{visit}") 

98 

99 with self.assertRaises(FileTemplateValidationError): 

100 FileTemplate("{run}_{datasetType}") 

101 

102 def testRunOrCollectionNeeded(self): 

103 tmplstr = "{datasetType}/{visit:05d}/{physical_filter}" 

104 with self.assertRaises(FileTemplateValidationError): 

105 self.assertTemplate(tmplstr, 

106 "run2/calexp/00052/U", 

107 self.makeDatasetRef("calexp")) 

108 

109 def testOptional(self): 

110 """Optional units in templates.""" 

111 ref = self.makeDatasetRef("calexp", conform=False) 

112 tmplstr = "{run}/{datasetType}/v{visit:05d}_f{physical_filter:?}" 

113 self.assertTemplate(tmplstr, "run2/calexp/v00052_fMost_Amazing_U_Filter_Ever", 

114 self.makeDatasetRef("calexp", conform=False)) 

115 

116 du = {"visit": 48, "tract": 265, "skymap": "big", "instrument": "dummy"} 

117 self.assertTemplate(tmplstr, "run2/calexpT/v00048", 

118 self.makeDatasetRef("calexpT", du, conform=False)) 

119 

120 # Ensure that this returns a relative path even if the first field 

121 # is optional 

122 tmplstr = "{run}/{tract:?}/{visit:?}/f{physical_filter}" 

123 self.assertTemplate(tmplstr, "run2/52/fMost_Amazing_U_Filter_Ever", ref) 

124 

125 # Ensure that // from optionals are converted to singles 

126 tmplstr = "{run}/{datasetType}/{patch:?}/{tract:?}/f{physical_filter}" 

127 self.assertTemplate(tmplstr, "run2/calexp/fMost_Amazing_U_Filter_Ever", ref) 

128 

129 # Optionals with some text between fields 

130 tmplstr = "{run}/{datasetType}/p{patch:?}_t{tract:?}/f{physical_filter}" 

131 self.assertTemplate(tmplstr, "run2/calexp/p/fMost_Amazing_U_Filter_Ever", ref) 

132 tmplstr = "{run}/{datasetType}/p{patch:?}_t{visit:04d?}/f{physical_filter}" 

133 self.assertTemplate(tmplstr, "run2/calexp/p_t0052/fMost_Amazing_U_Filter_Ever", ref) 

134 

135 def testComponent(self): 

136 """Test handling of components in templates.""" 

137 refMetricOutput = self.makeDatasetRef("metric.output") 

138 refMetric = self.makeDatasetRef("metric") 

139 refMaskedImage = self.makeDatasetRef("calexp.maskedimage.variance") 

140 refWcs = self.makeDatasetRef("calexp.wcs") 

141 

142 tmplstr = "{run}_c_{component}_v{visit}" 

143 self.assertTemplate(tmplstr, "run2_c_output_v52", refMetricOutput) 

144 

145 # We want this template to have both a directory and basename, to 

146 # test that the right parts of the output are replaced. 

147 tmplstr = "{component:?}/{run}_{component:?}_{visit}" 

148 self.assertTemplate(tmplstr, "run2_52", refMetric) 

149 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput) 

150 self.assertTemplate(tmplstr, "maskedimage.variance/run2_maskedimage_variance_52", refMaskedImage) 

151 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput) 

152 

153 # Providing a component but not using it 

154 tmplstr = "{run}/{datasetType}/v{visit:05d}" 

155 with self.assertRaises(KeyError): 

156 self.assertTemplate(tmplstr, "", refWcs) 

157 

158 def testFields(self): 

159 # Template, mandatory fields, optional non-special fields, 

160 # special fields, optional special fields 

161 testData = (("{run}/{datasetType}/{visit:05d}/{physical_filter}-trail", 

162 set(["visit", "physical_filter"]), 

163 set(), 

164 set(["run", "datasetType"]), 

165 set()), 

166 ("{run}/{component:?}_{visit}", 

167 set(["visit"]), 

168 set(), 

169 set(["run"]), 

170 set(["component"]),), 

171 ("{run}/{component:?}_{visit:?}_{physical_filter}_{instrument}_{datasetType}", 

172 set(["physical_filter", "instrument"]), 

173 set(["visit"]), 

174 set(["run", "datasetType"]), 

175 set(["component"]),), 

176 ) 

177 for tmplstr, mandatory, optional, special, optionalSpecial in testData: 

178 with self.subTest(template=tmplstr): 

179 tmpl = FileTemplate(tmplstr) 

180 fields = tmpl.fields() 

181 self.assertEqual(fields, mandatory) 

182 fields = tmpl.fields(optionals=True) 

183 self.assertEqual(fields, mandatory | optional) 

184 fields = tmpl.fields(specials=True) 

185 self.assertEqual(fields, mandatory | special) 

186 fields = tmpl.fields(specials=True, optionals=True) 

187 self.assertEqual(fields, mandatory | special | optional | optionalSpecial) 

188 

189 def testSimpleConfig(self): 

190 """Test reading from config file""" 

191 configRoot = os.path.join(TESTDIR, "config", "templates") 

192 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml")) 

193 templates = FileTemplates(config1, universe=self.universe) 

194 ref = self.makeDatasetRef("calexp") 

195 tmpl = templates.getTemplate(ref) 

196 self.assertIsInstance(tmpl, FileTemplate) 

197 

198 # This config file should not allow defaulting 

199 ref2 = self.makeDatasetRef("unknown") 

200 with self.assertRaises(KeyError): 

201 templates.getTemplate(ref2) 

202 

203 # This should fall through the datasetTypeName check and use 

204 # StorageClass instead 

205 ref3 = self.makeDatasetRef("unknown2", storageClassName="StorageClassX") 

206 tmplSc = templates.getTemplate(ref3) 

207 self.assertIsInstance(tmplSc, FileTemplate) 

208 

209 # Try with a component: one with defined formatter and one without 

210 refWcs = self.makeDatasetRef("calexp.wcs") 

211 refImage = self.makeDatasetRef("calexp.image") 

212 tmplCalexp = templates.getTemplate(ref) 

213 tmplWcs = templates.getTemplate(refWcs) # Should be special 

214 tmpl_image = templates.getTemplate(refImage) 

215 self.assertIsInstance(tmplCalexp, FileTemplate) 

216 self.assertIsInstance(tmpl_image, FileTemplate) 

217 self.assertIsInstance(tmplWcs, FileTemplate) 

218 self.assertEqual(tmplCalexp, tmpl_image) 

219 self.assertNotEqual(tmplCalexp, tmplWcs) 

220 

221 # Check dimensions lookup order. 

222 # The order should be: dataset type name, dimension, storage class 

223 # This one will not match name but might match storage class. 

224 # It should match dimensions 

225 refDims = self.makeDatasetRef("nomatch", dataId={"instrument": "LSST", "physical_filter": "z"}, 

226 storageClassName="StorageClassX") 

227 tmplDims = templates.getTemplate(refDims) 

228 self.assertIsInstance(tmplDims, FileTemplate) 

229 self.assertNotEqual(tmplDims, tmplSc) 

230 

231 # Test that instrument overrides retrieve specialist templates 

232 refPvi = self.makeDatasetRef("pvi") 

233 refPviHsc = self.makeDatasetRef("pvi", dataId={"instrument": "HSC", "physical_filter": "z"}) 

234 refPviLsst = self.makeDatasetRef("pvi", dataId={"instrument": "LSST", "physical_filter": "z"}) 

235 

236 tmplPvi = templates.getTemplate(refPvi) 

237 tmplPviHsc = templates.getTemplate(refPviHsc) 

238 tmplPviLsst = templates.getTemplate(refPviLsst) 

239 self.assertEqual(tmplPvi, tmplPviLsst) 

240 self.assertNotEqual(tmplPvi, tmplPviHsc) 

241 

242 # Have instrument match and dimensions look up with no name match 

243 refNoPviHsc = self.makeDatasetRef("pvix", dataId={"instrument": "HSC", "physical_filter": "z"}, 

244 storageClassName="StorageClassX") 

245 tmplNoPviHsc = templates.getTemplate(refNoPviHsc) 

246 self.assertNotEqual(tmplNoPviHsc, tmplDims) 

247 self.assertNotEqual(tmplNoPviHsc, tmplPviHsc) 

248 

249 # Format config file with defaulting 

250 config2 = FileTemplatesConfig(os.path.join(configRoot, "templates-withdefault.yaml")) 

251 templates = FileTemplates(config2, universe=self.universe) 

252 tmpl = templates.getTemplate(ref2) 

253 self.assertIsInstance(tmpl, FileTemplate) 

254 

255 # Format config file with bad format string 

256 with self.assertRaises(FileTemplateValidationError): 

257 FileTemplates(os.path.join(configRoot, "templates-bad.yaml"), universe=self.universe) 

258 

259 # Config file with no defaulting mentioned 

260 config3 = os.path.join(configRoot, "templates-nodefault2.yaml") 

261 templates = FileTemplates(config3, universe=self.universe) 

262 with self.assertRaises(KeyError): 

263 templates.getTemplate(ref2) 

264 

265 # Try again but specify a default in the constructor 

266 default = "{run}/{datasetType}/{physical_filter}" 

267 templates = FileTemplates(config3, default=default, universe=self.universe) 

268 tmpl = templates.getTemplate(ref2) 

269 self.assertEqual(tmpl.template, default) 

270 

271 def testValidation(self): 

272 configRoot = os.path.join(TESTDIR, "config", "templates") 

273 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml")) 

274 templates = FileTemplates(config1, universe=self.universe) 

275 

276 entities = {} 

277 entities["calexp"] = self.makeDatasetRef("calexp", storageClassName="StorageClassX", 

278 dataId={"instrument": "dummy", "physical_filter": "i", 

279 "visit": 52}) 

280 

281 with self.assertLogs(level="WARNING") as cm: 

282 templates.validateTemplates(entities.values(), logFailures=True) 

283 self.assertIn("Unchecked keys", cm.output[0]) 

284 self.assertIn("StorageClassX", cm.output[0]) 

285 

286 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX", 

287 dataId={"instrument": "dummy", "physical_filter": "i"}) 

288 entities["StorageClassX"] = self.makeDatasetRef("storageClass", 

289 storageClassName="StorageClassX", 

290 dataId={"instrument": "dummy", "visit": 2}) 

291 entities["calexp.wcs"] = self.makeDatasetRef("calexp.wcs", 

292 storageClassName="StorageClassX", 

293 dataId={"instrument": "dummy", 

294 "physical_filter": "i", "visit": 23}, 

295 conform=False) 

296 

297 entities["instrument+physical_filter"] = self.makeDatasetRef("filter_inst", 

298 storageClassName="StorageClassX", 

299 dataId={"physical_filter": "i", 

300 "instrument": "SCUBA"}) 

301 entities["hsc+pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX", 

302 dataId={"physical_filter": "i", "instrument": "HSC"}) 

303 

304 entities["hsc+instrument+physical_filter"] = self.makeDatasetRef("filter_inst", 

305 storageClassName="StorageClassX", 

306 dataId={"physical_filter": "i", 

307 "instrument": "HSC"}) 

308 

309 templates.validateTemplates(entities.values(), logFailures=True) 

310 

311 # Rerun but with a failure 

312 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX", 

313 dataId={"band": "i"}) 

314 with self.assertRaises(FileTemplateValidationError): 

315 with self.assertLogs(level="FATAL"): 

316 templates.validateTemplates(entities.values(), logFailures=True) 

317 

318 

319if __name__ == "__main__": 319 ↛ 320line 319 didn't jump to line 320, because the condition on line 319 was never true

320 unittest.main()