Coverage for tests/test_templates.py: 12%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Test file name templating."""
24import os.path
25import unittest
27from lsst.daf.butler import (
28 DatasetRef,
29 DatasetType,
30 DimensionGraph,
31 DimensionUniverse,
32 FileTemplate,
33 FileTemplates,
34 FileTemplatesConfig,
35 FileTemplateValidationError,
36 StorageClass,
37)
39TESTDIR = os.path.abspath(os.path.dirname(__file__))
41PlaceHolder = StorageClass("PlaceHolder")
44class TestFileTemplates(unittest.TestCase):
45 """Test creation of paths from templates."""
47 def makeDatasetRef(
48 self, datasetTypeName, dataId=None, storageClassName="DefaultStorageClass", run="run2", conform=True
49 ):
50 """Make a simple DatasetRef"""
51 if dataId is None:
52 dataId = self.dataId
54 # Pretend we have a parent if this looks like a composite
55 compositeName, componentName = DatasetType.splitDatasetTypeName(datasetTypeName)
56 parentStorageClass = PlaceHolder if componentName else None
58 datasetType = DatasetType(
59 datasetTypeName,
60 DimensionGraph(self.universe, names=dataId.keys()),
61 StorageClass(storageClassName),
62 parentStorageClass=parentStorageClass,
63 )
64 return DatasetRef(datasetType, dataId, id=1, run=run, conform=conform)
66 def setUp(self):
67 self.universe = DimensionUniverse()
68 self.dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "Most Amazing U Filter Ever"}
70 def assertTemplate(self, template, answer, ref):
71 fileTmpl = FileTemplate(template)
72 path = fileTmpl.format(ref)
73 self.assertEqual(path, answer)
75 def testBasic(self):
76 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}"
77 self.assertTemplate(
78 tmplstr,
79 "run2/calexp/00052/Most_Amazing_U_Filter_Ever",
80 self.makeDatasetRef("calexp", conform=False),
81 )
82 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail"
83 self.assertTemplate(
84 tmplstr,
85 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail",
86 self.makeDatasetRef("calexp", conform=False),
87 )
89 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
90 self.assertTemplate(
91 tmplstr,
92 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run2",
93 self.makeDatasetRef("calexp", conform=False),
94 )
95 self.assertTemplate(
96 tmplstr,
97 "run_2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2",
98 self.makeDatasetRef("calexp", run="run/2", conform=False),
99 )
101 # Check that the id is sufficient without any other information.
102 self.assertTemplate("{id}", "1", self.makeDatasetRef("calexp", run="run2", conform=False))
104 self.assertTemplate("{run}/{id}", "run2/1", self.makeDatasetRef("calexp", run="run2", conform=False))
106 self.assertTemplate(
107 "fixed/{id}",
108 "fixed/1",
109 self.makeDatasetRef("calexp", run="run2", conform=False),
110 )
112 self.assertTemplate(
113 "fixed/{id}_{physical_filter}",
114 "fixed/1_Most_Amazing_U_Filter_Ever",
115 self.makeDatasetRef("calexp", run="run2", conform=False),
116 )
118 # Retain any "/" in run
119 tmplstr = "{run:/}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
120 self.assertTemplate(
121 tmplstr,
122 "run/2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2",
123 self.makeDatasetRef("calexp", run="run/2", conform=False),
124 )
126 # Check that "." are replaced in the file basename, but not directory.
127 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "g.10"}
128 self.assertTemplate(
129 tmplstr,
130 "run.2/calexp/00052/g_10-trail-run_2",
131 self.makeDatasetRef("calexp", run="run.2", dataId=dataId, conform=False),
132 )
134 with self.assertRaises(FileTemplateValidationError):
135 FileTemplate("no fields at all")
137 with self.assertRaises(FileTemplateValidationError):
138 FileTemplate("{visit}")
140 with self.assertRaises(FileTemplateValidationError):
141 FileTemplate("{run}_{datasetType}")
143 with self.assertRaises(FileTemplateValidationError):
144 FileTemplate("{id}/fixed")
146 def testRunOrCollectionNeeded(self):
147 tmplstr = "{datasetType}/{visit:05d}/{physical_filter}"
148 with self.assertRaises(FileTemplateValidationError):
149 self.assertTemplate(tmplstr, "run2/calexp/00052/U", self.makeDatasetRef("calexp"))
151 def testOptional(self):
152 """Optional units in templates."""
153 ref = self.makeDatasetRef("calexp", conform=False)
154 tmplstr = "{run}/{datasetType}/v{visit:05d}_f{physical_filter:?}"
155 self.assertTemplate(
156 tmplstr,
157 "run2/calexp/v00052_fMost_Amazing_U_Filter_Ever",
158 self.makeDatasetRef("calexp", conform=False),
159 )
161 du = {"visit": 48, "tract": 265, "skymap": "big", "instrument": "dummy"}
162 self.assertTemplate(tmplstr, "run2/calexpT/v00048", self.makeDatasetRef("calexpT", du, conform=False))
164 # Ensure that this returns a relative path even if the first field
165 # is optional
166 tmplstr = "{run}/{tract:?}/{visit:?}/f{physical_filter}"
167 self.assertTemplate(tmplstr, "run2/52/fMost_Amazing_U_Filter_Ever", ref)
169 # Ensure that // from optionals are converted to singles
170 tmplstr = "{run}/{datasetType}/{patch:?}/{tract:?}/f{physical_filter}"
171 self.assertTemplate(tmplstr, "run2/calexp/fMost_Amazing_U_Filter_Ever", ref)
173 # Optionals with some text between fields
174 tmplstr = "{run}/{datasetType}/p{patch:?}_t{tract:?}/f{physical_filter}"
175 self.assertTemplate(tmplstr, "run2/calexp/p/fMost_Amazing_U_Filter_Ever", ref)
176 tmplstr = "{run}/{datasetType}/p{patch:?}_t{visit:04d?}/f{physical_filter}"
177 self.assertTemplate(tmplstr, "run2/calexp/p_t0052/fMost_Amazing_U_Filter_Ever", ref)
179 def testComponent(self):
180 """Test handling of components in templates."""
181 refMetricOutput = self.makeDatasetRef("metric.output")
182 refMetric = self.makeDatasetRef("metric")
183 refMaskedImage = self.makeDatasetRef("calexp.maskedimage.variance")
184 refWcs = self.makeDatasetRef("calexp.wcs")
186 tmplstr = "{run}_c_{component}_v{visit}"
187 self.assertTemplate(tmplstr, "run2_c_output_v52", refMetricOutput)
189 # We want this template to have both a directory and basename, to
190 # test that the right parts of the output are replaced.
191 tmplstr = "{component:?}/{run}_{component:?}_{visit}"
192 self.assertTemplate(tmplstr, "run2_52", refMetric)
193 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput)
194 self.assertTemplate(tmplstr, "maskedimage.variance/run2_maskedimage_variance_52", refMaskedImage)
195 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput)
197 # Providing a component but not using it
198 tmplstr = "{run}/{datasetType}/v{visit:05d}"
199 with self.assertRaises(KeyError):
200 self.assertTemplate(tmplstr, "", refWcs)
202 def testFields(self):
203 # Template, mandatory fields, optional non-special fields,
204 # special fields, optional special fields
205 testData = (
206 (
207 "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail",
208 set(["visit", "physical_filter"]),
209 set(),
210 set(["run", "datasetType"]),
211 set(),
212 ),
213 (
214 "{run}/{component:?}_{visit}",
215 set(["visit"]),
216 set(),
217 set(["run"]),
218 set(["component"]),
219 ),
220 (
221 "{run}/{component:?}_{visit:?}_{physical_filter}_{instrument}_{datasetType}",
222 set(["physical_filter", "instrument"]),
223 set(["visit"]),
224 set(["run", "datasetType"]),
225 set(["component"]),
226 ),
227 )
228 for tmplstr, mandatory, optional, special, optionalSpecial in testData:
229 with self.subTest(template=tmplstr):
230 tmpl = FileTemplate(tmplstr)
231 fields = tmpl.fields()
232 self.assertEqual(fields, mandatory)
233 fields = tmpl.fields(optionals=True)
234 self.assertEqual(fields, mandatory | optional)
235 fields = tmpl.fields(specials=True)
236 self.assertEqual(fields, mandatory | special)
237 fields = tmpl.fields(specials=True, optionals=True)
238 self.assertEqual(fields, mandatory | special | optional | optionalSpecial)
240 def testSimpleConfig(self):
241 """Test reading from config file"""
242 configRoot = os.path.join(TESTDIR, "config", "templates")
243 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
244 templates = FileTemplates(config1, universe=self.universe)
245 ref = self.makeDatasetRef("calexp")
246 tmpl = templates.getTemplate(ref)
247 self.assertIsInstance(tmpl, FileTemplate)
249 # This config file should not allow defaulting
250 ref2 = self.makeDatasetRef("unknown")
251 with self.assertRaises(KeyError):
252 templates.getTemplate(ref2)
254 # This should fall through the datasetTypeName check and use
255 # StorageClass instead
256 ref3 = self.makeDatasetRef("unknown2", storageClassName="StorageClassX")
257 tmplSc = templates.getTemplate(ref3)
258 self.assertIsInstance(tmplSc, FileTemplate)
260 # Try with a component: one with defined formatter and one without
261 refWcs = self.makeDatasetRef("calexp.wcs")
262 refImage = self.makeDatasetRef("calexp.image")
263 tmplCalexp = templates.getTemplate(ref)
264 tmplWcs = templates.getTemplate(refWcs) # Should be special
265 tmpl_image = templates.getTemplate(refImage)
266 self.assertIsInstance(tmplCalexp, FileTemplate)
267 self.assertIsInstance(tmpl_image, FileTemplate)
268 self.assertIsInstance(tmplWcs, FileTemplate)
269 self.assertEqual(tmplCalexp, tmpl_image)
270 self.assertNotEqual(tmplCalexp, tmplWcs)
272 # Check dimensions lookup order.
273 # The order should be: dataset type name, dimension, storage class
274 # This one will not match name but might match storage class.
275 # It should match dimensions
276 refDims = self.makeDatasetRef(
277 "nomatch", dataId={"instrument": "LSST", "physical_filter": "z"}, storageClassName="StorageClassX"
278 )
279 tmplDims = templates.getTemplate(refDims)
280 self.assertIsInstance(tmplDims, FileTemplate)
281 self.assertNotEqual(tmplDims, tmplSc)
283 # Test that instrument overrides retrieve specialist templates
284 refPvi = self.makeDatasetRef("pvi")
285 refPviHsc = self.makeDatasetRef("pvi", dataId={"instrument": "HSC", "physical_filter": "z"})
286 refPviLsst = self.makeDatasetRef("pvi", dataId={"instrument": "LSST", "physical_filter": "z"})
288 tmplPvi = templates.getTemplate(refPvi)
289 tmplPviHsc = templates.getTemplate(refPviHsc)
290 tmplPviLsst = templates.getTemplate(refPviLsst)
291 self.assertEqual(tmplPvi, tmplPviLsst)
292 self.assertNotEqual(tmplPvi, tmplPviHsc)
294 # Have instrument match and dimensions look up with no name match
295 refNoPviHsc = self.makeDatasetRef(
296 "pvix", dataId={"instrument": "HSC", "physical_filter": "z"}, storageClassName="StorageClassX"
297 )
298 tmplNoPviHsc = templates.getTemplate(refNoPviHsc)
299 self.assertNotEqual(tmplNoPviHsc, tmplDims)
300 self.assertNotEqual(tmplNoPviHsc, tmplPviHsc)
302 # Format config file with defaulting
303 config2 = FileTemplatesConfig(os.path.join(configRoot, "templates-withdefault.yaml"))
304 templates = FileTemplates(config2, universe=self.universe)
305 tmpl = templates.getTemplate(ref2)
306 self.assertIsInstance(tmpl, FileTemplate)
308 # Format config file with bad format string
309 with self.assertRaises(FileTemplateValidationError):
310 FileTemplates(os.path.join(configRoot, "templates-bad.yaml"), universe=self.universe)
312 # Config file with no defaulting mentioned
313 config3 = os.path.join(configRoot, "templates-nodefault2.yaml")
314 templates = FileTemplates(config3, universe=self.universe)
315 with self.assertRaises(KeyError):
316 templates.getTemplate(ref2)
318 # Try again but specify a default in the constructor
319 default = "{run}/{datasetType}/{physical_filter}"
320 templates = FileTemplates(config3, default=default, universe=self.universe)
321 tmpl = templates.getTemplate(ref2)
322 self.assertEqual(tmpl.template, default)
324 def testValidation(self):
325 configRoot = os.path.join(TESTDIR, "config", "templates")
326 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
327 templates = FileTemplates(config1, universe=self.universe)
329 entities = {}
330 entities["calexp"] = self.makeDatasetRef(
331 "calexp",
332 storageClassName="StorageClassX",
333 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 52},
334 )
336 with self.assertLogs(level="WARNING") as cm:
337 templates.validateTemplates(entities.values(), logFailures=True)
338 self.assertIn("Unchecked keys", cm.output[0])
339 self.assertIn("StorageClassX", cm.output[0])
341 entities["pvi"] = self.makeDatasetRef(
342 "pvi", storageClassName="StorageClassX", dataId={"instrument": "dummy", "physical_filter": "i"}
343 )
344 entities["StorageClassX"] = self.makeDatasetRef(
345 "storageClass", storageClassName="StorageClassX", dataId={"instrument": "dummy", "visit": 2}
346 )
347 entities["calexp.wcs"] = self.makeDatasetRef(
348 "calexp.wcs",
349 storageClassName="StorageClassX",
350 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 23},
351 conform=False,
352 )
354 entities["instrument+physical_filter"] = self.makeDatasetRef(
355 "filter_inst",
356 storageClassName="StorageClassX",
357 dataId={"physical_filter": "i", "instrument": "SCUBA"},
358 )
359 entities["hsc+pvi"] = self.makeDatasetRef(
360 "pvi", storageClassName="StorageClassX", dataId={"physical_filter": "i", "instrument": "HSC"}
361 )
363 entities["hsc+instrument+physical_filter"] = self.makeDatasetRef(
364 "filter_inst",
365 storageClassName="StorageClassX",
366 dataId={"physical_filter": "i", "instrument": "HSC"},
367 )
369 entities["metric6"] = self.makeDatasetRef(
370 "filter_inst",
371 storageClassName="Integer",
372 dataId={"physical_filter": "i", "instrument": "HSC"},
373 )
375 templates.validateTemplates(entities.values(), logFailures=True)
377 # Rerun but with a failure
378 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX", dataId={"band": "i"})
379 with self.assertRaises(FileTemplateValidationError):
380 with self.assertLogs(level="FATAL"):
381 templates.validateTemplates(entities.values(), logFailures=True)
384if __name__ == "__main__": 384 ↛ 385line 384 didn't jump to line 385, because the condition on line 384 was never true
385 unittest.main()