Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Test file name templating.""" 

23 

24import os.path 

25import unittest 

26 

27from lsst.daf.butler import DatasetType, DatasetRef, FileTemplates, DimensionUniverse, \ 

28 FileTemplate, FileTemplatesConfig, StorageClass, FileTemplateValidationError, \ 

29 DimensionGraph 

30 

31TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

32 

33 

34class TestFileTemplates(unittest.TestCase): 

35 """Test creation of paths from templates.""" 

36 

37 def makeDatasetRef(self, datasetTypeName, dataId=None, storageClassName="DefaultStorageClass", 

38 run="run2", conform=True): 

39 """Make a simple DatasetRef""" 

40 if dataId is None: 

41 dataId = self.dataId 

42 datasetType = DatasetType(datasetTypeName, DimensionGraph(self.universe, names=dataId.keys()), 

43 StorageClass(storageClassName)) 

44 return DatasetRef(datasetType, dataId, id=1, run=run, conform=conform) 

45 

46 def setUp(self): 

47 self.universe = DimensionUniverse() 

48 self.dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "U"} 

49 

50 def assertTemplate(self, template, answer, ref): 

51 fileTmpl = FileTemplate(template) 

52 path = fileTmpl.format(ref) 

53 self.assertEqual(path, answer) 

54 

55 def testBasic(self): 

56 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}" 

57 self.assertTemplate(tmplstr, 

58 "run2/calexp/00052/U", 

59 self.makeDatasetRef("calexp", conform=False)) 

60 tmplstr = "{collection}/{datasetType}/{visit:05d}/{physical_filter}-trail" 

61 self.assertTemplate(tmplstr, 

62 "run2/calexp/00052/U-trail", 

63 self.makeDatasetRef("calexp", conform=False)) 

64 

65 tmplstr = "{collection}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}" 

66 self.assertTemplate(tmplstr, 

67 "run2/calexp/00052/U-trail-run2", 

68 self.makeDatasetRef("calexp", conform=False)) 

69 self.assertTemplate(tmplstr, 

70 "run_2/calexp/00052/U-trail-run_2", 

71 self.makeDatasetRef("calexp", run="run/2", conform=False)) 

72 

73 # Retain any "/" in collection 

74 tmplstr = "{collection:/}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}" 

75 self.assertTemplate(tmplstr, 

76 "run/2/calexp/00052/U-trail-run_2", 

77 self.makeDatasetRef("calexp", run="run/2", conform=False)) 

78 

79 with self.assertRaises(FileTemplateValidationError): 

80 FileTemplate("no fields at all") 

81 

82 with self.assertRaises(FileTemplateValidationError): 

83 FileTemplate("{visit}") 

84 

85 with self.assertRaises(FileTemplateValidationError): 

86 FileTemplate("{run}_{datasetType}") 

87 

88 def testRunOrCollectionNeeded(self): 

89 tmplstr = "{datasetType}/{visit:05d}/{physical_filter}" 

90 with self.assertRaises(FileTemplateValidationError): 

91 self.assertTemplate(tmplstr, 

92 "run2/calexp/00052/U", 

93 self.makeDatasetRef("calexp")) 

94 

95 def testOptional(self): 

96 """Optional units in templates.""" 

97 ref = self.makeDatasetRef("calexp", conform=False) 

98 tmplstr = "{run}/{datasetType}/v{visit:05d}_f{physical_filter:?}" 

99 self.assertTemplate(tmplstr, "run2/calexp/v00052_fU", 

100 self.makeDatasetRef("calexp", conform=False)) 

101 

102 du = {"visit": 48, "tract": 265, "skymap": "big", "instrument": "dummy"} 

103 self.assertTemplate(tmplstr, "run2/calexpT/v00048", 

104 self.makeDatasetRef("calexpT", du, conform=False)) 

105 

106 # Ensure that this returns a relative path even if the first field 

107 # is optional 

108 tmplstr = "{run}/{tract:?}/{visit:?}/f{physical_filter}" 

109 self.assertTemplate(tmplstr, "run2/52/fU", ref) 

110 

111 # Ensure that // from optionals are converted to singles 

112 tmplstr = "{run}/{datasetType}/{patch:?}/{tract:?}/f{physical_filter}" 

113 self.assertTemplate(tmplstr, "run2/calexp/fU", ref) 

114 

115 # Optionals with some text between fields 

116 tmplstr = "{run}/{datasetType}/p{patch:?}_t{tract:?}/f{physical_filter}" 

117 self.assertTemplate(tmplstr, "run2/calexp/p/fU", ref) 

118 tmplstr = "{run}/{datasetType}/p{patch:?}_t{visit:04d?}/f{physical_filter}" 

119 self.assertTemplate(tmplstr, "run2/calexp/p_t0052/fU", ref) 

120 

121 def testComponent(self): 

122 """Test handling of components in templates.""" 

123 refMetricOutput = self.makeDatasetRef("metric.output") 

124 refMetric = self.makeDatasetRef("metric") 

125 refMaskedImage = self.makeDatasetRef("calexp.maskedimage.variance") 

126 refWcs = self.makeDatasetRef("calexp.wcs") 

127 

128 tmplstr = "{run}_c_{component}_v{visit}" 

129 self.assertTemplate(tmplstr, "run2_c_output_v52", refMetricOutput) 

130 

131 tmplstr = "{run}_{component:?}_{visit}" 

132 self.assertTemplate(tmplstr, "run2_52", refMetric) 

133 self.assertTemplate(tmplstr, "run2_output_52", refMetricOutput) 

134 self.assertTemplate(tmplstr, "run2_maskedimage.variance_52", refMaskedImage) 

135 self.assertTemplate(tmplstr, "run2_output_52", refMetricOutput) 

136 

137 # Providing a component but not using it 

138 tmplstr = "{collection}/{datasetType}/v{visit:05d}" 

139 with self.assertRaises(KeyError): 

140 self.assertTemplate(tmplstr, "", refWcs) 

141 

142 def testFields(self): 

143 # Template, mandatory fields, optional non-special fields, 

144 # special fields, optional special fields 

145 testData = (("{collection}/{datasetType}/{visit:05d}/{physical_filter}-trail", 

146 set(["visit", "physical_filter"]), 

147 set(), 

148 set(["collection", "datasetType"]), 

149 set()), 

150 ("{collection}/{component:?}_{visit}", 

151 set(["visit"]), 

152 set(), 

153 set(["collection"]), 

154 set(["component"]),), 

155 ("{run}/{component:?}_{visit:?}_{physical_filter}_{instrument}_{datasetType}", 

156 set(["physical_filter", "instrument"]), 

157 set(["visit"]), 

158 set(["run", "datasetType"]), 

159 set(["component"]),), 

160 ) 

161 for tmplstr, mandatory, optional, special, optionalSpecial in testData: 

162 with self.subTest(template=tmplstr): 

163 tmpl = FileTemplate(tmplstr) 

164 fields = tmpl.fields() 

165 self.assertEqual(fields, mandatory) 

166 fields = tmpl.fields(optionals=True) 

167 self.assertEqual(fields, mandatory | optional) 

168 fields = tmpl.fields(specials=True) 

169 self.assertEqual(fields, mandatory | special) 

170 fields = tmpl.fields(specials=True, optionals=True) 

171 self.assertEqual(fields, mandatory | special | optional | optionalSpecial) 

172 

173 def testSimpleConfig(self): 

174 """Test reading from config file""" 

175 configRoot = os.path.join(TESTDIR, "config", "templates") 

176 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml")) 

177 templates = FileTemplates(config1, universe=self.universe) 

178 ref = self.makeDatasetRef("calexp") 

179 tmpl = templates.getTemplate(ref) 

180 self.assertIsInstance(tmpl, FileTemplate) 

181 

182 # This config file should not allow defaulting 

183 ref2 = self.makeDatasetRef("unknown") 

184 with self.assertRaises(KeyError): 

185 templates.getTemplate(ref2) 

186 

187 # This should fall through the datasetTypeName check and use 

188 # StorageClass instead 

189 ref3 = self.makeDatasetRef("unknown2", storageClassName="StorageClassX") 

190 tmplSc = templates.getTemplate(ref3) 

191 self.assertIsInstance(tmplSc, FileTemplate) 

192 

193 # Try with a component: one with defined formatter and one without 

194 refWcs = self.makeDatasetRef("calexp.wcs") 

195 refImage = self.makeDatasetRef("calexp.image") 

196 tmplCalexp = templates.getTemplate(ref) 

197 tmplWcs = templates.getTemplate(refWcs) # Should be special 

198 tmpl_image = templates.getTemplate(refImage) 

199 self.assertIsInstance(tmplCalexp, FileTemplate) 

200 self.assertIsInstance(tmpl_image, FileTemplate) 

201 self.assertIsInstance(tmplWcs, FileTemplate) 

202 self.assertEqual(tmplCalexp, tmpl_image) 

203 self.assertNotEqual(tmplCalexp, tmplWcs) 

204 

205 # Check dimensions lookup order. 

206 # The order should be: dataset type name, dimension, storage class 

207 # This one will not match name but might match storage class. 

208 # It should match dimensions 

209 refDims = self.makeDatasetRef("nomatch", dataId={"instrument": "LSST", "physical_filter": "z"}, 

210 storageClassName="StorageClassX") 

211 tmplDims = templates.getTemplate(refDims) 

212 self.assertIsInstance(tmplDims, FileTemplate) 

213 self.assertNotEqual(tmplDims, tmplSc) 

214 

215 # Test that instrument overrides retrieve specialist templates 

216 refPvi = self.makeDatasetRef("pvi") 

217 refPviHsc = self.makeDatasetRef("pvi", dataId={"instrument": "HSC", "physical_filter": "z"}) 

218 refPviLsst = self.makeDatasetRef("pvi", dataId={"instrument": "LSST", "physical_filter": "z"}) 

219 

220 tmplPvi = templates.getTemplate(refPvi) 

221 tmplPviHsc = templates.getTemplate(refPviHsc) 

222 tmplPviLsst = templates.getTemplate(refPviLsst) 

223 self.assertEqual(tmplPvi, tmplPviLsst) 

224 self.assertNotEqual(tmplPvi, tmplPviHsc) 

225 

226 # Have instrument match and dimensions look up with no name match 

227 refNoPviHsc = self.makeDatasetRef("pvix", dataId={"instrument": "HSC", "physical_filter": "z"}, 

228 storageClassName="StorageClassX") 

229 tmplNoPviHsc = templates.getTemplate(refNoPviHsc) 

230 self.assertNotEqual(tmplNoPviHsc, tmplDims) 

231 self.assertNotEqual(tmplNoPviHsc, tmplPviHsc) 

232 

233 # Format config file with defaulting 

234 config2 = FileTemplatesConfig(os.path.join(configRoot, "templates-withdefault.yaml")) 

235 templates = FileTemplates(config2, universe=self.universe) 

236 tmpl = templates.getTemplate(ref2) 

237 self.assertIsInstance(tmpl, FileTemplate) 

238 

239 # Format config file with bad format string 

240 with self.assertRaises(FileTemplateValidationError): 

241 FileTemplates(os.path.join(configRoot, "templates-bad.yaml"), universe=self.universe) 

242 

243 # Config file with no defaulting mentioned 

244 config3 = os.path.join(configRoot, "templates-nodefault2.yaml") 

245 templates = FileTemplates(config3, universe=self.universe) 

246 with self.assertRaises(KeyError): 

247 templates.getTemplate(ref2) 

248 

249 # Try again but specify a default in the constructor 

250 default = "{run}/{datasetType}/{physical_filter}" 

251 templates = FileTemplates(config3, default=default, universe=self.universe) 

252 tmpl = templates.getTemplate(ref2) 

253 self.assertEqual(tmpl.template, default) 

254 

255 def testValidation(self): 

256 configRoot = os.path.join(TESTDIR, "config", "templates") 

257 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml")) 

258 templates = FileTemplates(config1, universe=self.universe) 

259 

260 entities = {} 

261 entities["calexp"] = self.makeDatasetRef("calexp", storageClassName="StorageClassX", 

262 dataId={"instrument": "dummy", "physical_filter": "i", 

263 "visit": 52}) 

264 

265 with self.assertLogs(level="WARNING") as cm: 

266 templates.validateTemplates(entities.values(), logFailures=True) 

267 self.assertIn("Unchecked keys", cm.output[0]) 

268 self.assertIn("StorageClassX", cm.output[0]) 

269 

270 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX", 

271 dataId={"instrument": "dummy", "physical_filter": "i"}) 

272 entities["StorageClassX"] = self.makeDatasetRef("storageClass", 

273 storageClassName="StorageClassX", 

274 dataId={"instrument": "dummy", "visit": 2}) 

275 entities["calexp.wcs"] = self.makeDatasetRef("calexp.wcs", 

276 storageClassName="StorageClassX", 

277 dataId={"instrument": "dummy", 

278 "physical_filter": "i", "visit": 23}, 

279 conform=False) 

280 

281 entities["instrument+physical_filter"] = self.makeDatasetRef("filter_inst", 

282 storageClassName="StorageClassX", 

283 dataId={"physical_filter": "i", 

284 "instrument": "SCUBA"}) 

285 entities["hsc+pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX", 

286 dataId={"physical_filter": "i", "instrument": "HSC"}) 

287 

288 entities["hsc+instrument+physical_filter"] = self.makeDatasetRef("filter_inst", 

289 storageClassName="StorageClassX", 

290 dataId={"physical_filter": "i", 

291 "instrument": "HSC"}) 

292 

293 templates.validateTemplates(entities.values(), logFailures=True) 

294 

295 # Rerun but with a failure 

296 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX", 

297 dataId={"abstract_filter": "i"}) 

298 with self.assertRaises(FileTemplateValidationError): 

299 with self.assertLogs(level="FATAL"): 

300 templates.validateTemplates(entities.values(), logFailures=True) 

301 

302 

303if __name__ == "__main__": 303 ↛ 304line 303 didn't jump to line 304, because the condition on line 303 was never true

304 unittest.main()