Coverage for tests/test_templates.py: 9%
171 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-08 05:05 -0700
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-08 05:05 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Test file name templating."""
24import os.path
25import unittest
27from lsst.daf.butler import (
28 DatasetRef,
29 DatasetType,
30 DimensionGraph,
31 DimensionUniverse,
32 FileTemplate,
33 FileTemplates,
34 FileTemplatesConfig,
35 FileTemplateValidationError,
36 StorageClass,
37)
39TESTDIR = os.path.abspath(os.path.dirname(__file__))
41PlaceHolder = StorageClass("PlaceHolder")
44class TestFileTemplates(unittest.TestCase):
45 """Test creation of paths from templates."""
47 def makeDatasetRef(
48 self, datasetTypeName, dataId=None, storageClassName="DefaultStorageClass", run="run2", conform=True
49 ):
50 """Make a simple DatasetRef"""
51 if dataId is None:
52 dataId = self.dataId
54 # Pretend we have a parent if this looks like a composite
55 compositeName, componentName = DatasetType.splitDatasetTypeName(datasetTypeName)
56 parentStorageClass = PlaceHolder if componentName else None
58 datasetType = DatasetType(
59 datasetTypeName,
60 DimensionGraph(self.universe, names=dataId.keys()),
61 StorageClass(storageClassName),
62 parentStorageClass=parentStorageClass,
63 )
64 return DatasetRef(datasetType, dataId, id=1, run=run, conform=conform)
66 def setUp(self):
67 self.universe = DimensionUniverse()
68 self.dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "Most Amazing U Filter Ever"}
70 def assertTemplate(self, template, answer, ref):
71 fileTmpl = FileTemplate(template)
72 path = fileTmpl.format(ref)
73 self.assertEqual(path, answer)
75 def testBasic(self):
76 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}"
77 self.assertTemplate(
78 tmplstr,
79 "run2/calexp/00052/Most_Amazing_U_Filter_Ever",
80 self.makeDatasetRef("calexp", conform=False),
81 )
82 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail"
83 self.assertTemplate(
84 tmplstr,
85 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail",
86 self.makeDatasetRef("calexp", conform=False),
87 )
89 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
90 self.assertTemplate(
91 tmplstr,
92 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run2",
93 self.makeDatasetRef("calexp", conform=False),
94 )
95 self.assertTemplate(
96 tmplstr,
97 "run_2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2",
98 self.makeDatasetRef("calexp", run="run/2", conform=False),
99 )
101 # Check that the id is sufficient without any other information.
102 self.assertTemplate("{id}", "1", self.makeDatasetRef("calexp", run="run2", conform=False))
104 self.assertTemplate("{run}/{id}", "run2/1", self.makeDatasetRef("calexp", run="run2", conform=False))
106 self.assertTemplate(
107 "fixed/{id}",
108 "fixed/1",
109 self.makeDatasetRef("calexp", run="run2", conform=False),
110 )
112 self.assertTemplate(
113 "fixed/{id}_{physical_filter}",
114 "fixed/1_Most_Amazing_U_Filter_Ever",
115 self.makeDatasetRef("calexp", run="run2", conform=False),
116 )
118 # Retain any "/" in run
119 tmplstr = "{run:/}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
120 self.assertTemplate(
121 tmplstr,
122 "run/2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2",
123 self.makeDatasetRef("calexp", run="run/2", conform=False),
124 )
126 # Check that "." are replaced in the file basename, but not directory.
127 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "g.10"}
128 self.assertTemplate(
129 tmplstr,
130 "run.2/calexp/00052/g_10-trail-run_2",
131 self.makeDatasetRef("calexp", run="run.2", dataId=dataId, conform=False),
132 )
134 with self.assertRaises(FileTemplateValidationError):
135 FileTemplate("no fields at all")
137 with self.assertRaises(FileTemplateValidationError):
138 FileTemplate("{visit}")
140 with self.assertRaises(FileTemplateValidationError):
141 FileTemplate("{run}_{datasetType}")
143 with self.assertRaises(FileTemplateValidationError):
144 FileTemplate("{id}/fixed")
146 def testRunOrCollectionNeeded(self):
147 tmplstr = "{datasetType}/{visit:05d}/{physical_filter}"
148 with self.assertRaises(FileTemplateValidationError):
149 self.assertTemplate(tmplstr, "run2/calexp/00052/U", self.makeDatasetRef("calexp"))
151 def testNoRecord(self):
152 # Attaching records is not possible in this test code but we can check
153 # that a missing record when a metadata entry has been requested
154 # does fail.
155 tmplstr = "{run}/{datasetType}/{visit.name}/{physical_filter}"
156 with self.assertRaises(RuntimeError) as cm:
157 self.assertTemplate(tmplstr, "", self.makeDatasetRef("calexp"))
158 self.assertIn("No metadata", str(cm.exception))
160 def testOptional(self):
161 """Optional units in templates."""
162 ref = self.makeDatasetRef("calexp", conform=False)
163 tmplstr = "{run}/{datasetType}/v{visit:05d}_f{physical_filter:?}"
164 self.assertTemplate(
165 tmplstr,
166 "run2/calexp/v00052_fMost_Amazing_U_Filter_Ever",
167 self.makeDatasetRef("calexp", conform=False),
168 )
170 du = {"visit": 48, "tract": 265, "skymap": "big", "instrument": "dummy"}
171 self.assertTemplate(tmplstr, "run2/calexpT/v00048", self.makeDatasetRef("calexpT", du, conform=False))
173 # Ensure that this returns a relative path even if the first field
174 # is optional
175 tmplstr = "{run}/{tract:?}/{visit:?}/f{physical_filter}"
176 self.assertTemplate(tmplstr, "run2/52/fMost_Amazing_U_Filter_Ever", ref)
178 # Ensure that // from optionals are converted to singles
179 tmplstr = "{run}/{datasetType}/{patch:?}/{tract:?}/f{physical_filter}"
180 self.assertTemplate(tmplstr, "run2/calexp/fMost_Amazing_U_Filter_Ever", ref)
182 # Optionals with some text between fields
183 tmplstr = "{run}/{datasetType}/p{patch:?}_t{tract:?}/f{physical_filter}"
184 self.assertTemplate(tmplstr, "run2/calexp/p/fMost_Amazing_U_Filter_Ever", ref)
185 tmplstr = "{run}/{datasetType}/p{patch:?}_t{visit:04d?}/f{physical_filter}"
186 self.assertTemplate(tmplstr, "run2/calexp/p_t0052/fMost_Amazing_U_Filter_Ever", ref)
188 def testComponent(self):
189 """Test handling of components in templates."""
190 refMetricOutput = self.makeDatasetRef("metric.output")
191 refMetric = self.makeDatasetRef("metric")
192 refMaskedImage = self.makeDatasetRef("calexp.maskedimage.variance")
193 refWcs = self.makeDatasetRef("calexp.wcs")
195 tmplstr = "{run}_c_{component}_v{visit}"
196 self.assertTemplate(tmplstr, "run2_c_output_v52", refMetricOutput)
198 # We want this template to have both a directory and basename, to
199 # test that the right parts of the output are replaced.
200 tmplstr = "{component:?}/{run}_{component:?}_{visit}"
201 self.assertTemplate(tmplstr, "run2_52", refMetric)
202 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput)
203 self.assertTemplate(tmplstr, "maskedimage.variance/run2_maskedimage_variance_52", refMaskedImage)
204 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput)
206 # Providing a component but not using it
207 tmplstr = "{run}/{datasetType}/v{visit:05d}"
208 with self.assertRaises(KeyError):
209 self.assertTemplate(tmplstr, "", refWcs)
211 def testFields(self):
212 # Template, mandatory fields, optional non-special fields,
213 # special fields, optional special fields
214 testData = (
215 (
216 "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail",
217 set(["visit", "physical_filter"]),
218 set(),
219 set(["run", "datasetType"]),
220 set(),
221 ),
222 (
223 "{run}/{component:?}_{visit}",
224 set(["visit"]),
225 set(),
226 set(["run"]),
227 set(["component"]),
228 ),
229 (
230 "{run}/{component:?}_{visit:?}_{physical_filter}_{instrument}_{datasetType}",
231 set(["physical_filter", "instrument"]),
232 set(["visit"]),
233 set(["run", "datasetType"]),
234 set(["component"]),
235 ),
236 )
237 for tmplstr, mandatory, optional, special, optionalSpecial in testData:
238 with self.subTest(template=tmplstr):
239 tmpl = FileTemplate(tmplstr)
240 fields = tmpl.fields()
241 self.assertEqual(fields, mandatory)
242 fields = tmpl.fields(optionals=True)
243 self.assertEqual(fields, mandatory | optional)
244 fields = tmpl.fields(specials=True)
245 self.assertEqual(fields, mandatory | special)
246 fields = tmpl.fields(specials=True, optionals=True)
247 self.assertEqual(fields, mandatory | special | optional | optionalSpecial)
249 def testSimpleConfig(self):
250 """Test reading from config file"""
251 configRoot = os.path.join(TESTDIR, "config", "templates")
252 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
253 templates = FileTemplates(config1, universe=self.universe)
254 ref = self.makeDatasetRef("calexp")
255 tmpl = templates.getTemplate(ref)
256 self.assertIsInstance(tmpl, FileTemplate)
258 # This config file should not allow defaulting
259 ref2 = self.makeDatasetRef("unknown")
260 with self.assertRaises(KeyError):
261 templates.getTemplate(ref2)
263 # This should fall through the datasetTypeName check and use
264 # StorageClass instead
265 ref3 = self.makeDatasetRef("unknown2", storageClassName="StorageClassX")
266 tmplSc = templates.getTemplate(ref3)
267 self.assertIsInstance(tmplSc, FileTemplate)
269 # Try with a component: one with defined formatter and one without
270 refWcs = self.makeDatasetRef("calexp.wcs")
271 refImage = self.makeDatasetRef("calexp.image")
272 tmplCalexp = templates.getTemplate(ref)
273 tmplWcs = templates.getTemplate(refWcs) # Should be special
274 tmpl_image = templates.getTemplate(refImage)
275 self.assertIsInstance(tmplCalexp, FileTemplate)
276 self.assertIsInstance(tmpl_image, FileTemplate)
277 self.assertIsInstance(tmplWcs, FileTemplate)
278 self.assertEqual(tmplCalexp, tmpl_image)
279 self.assertNotEqual(tmplCalexp, tmplWcs)
281 # Check dimensions lookup order.
282 # The order should be: dataset type name, dimension, storage class
283 # This one will not match name but might match storage class.
284 # It should match dimensions
285 refDims = self.makeDatasetRef(
286 "nomatch", dataId={"instrument": "LSST", "physical_filter": "z"}, storageClassName="StorageClassX"
287 )
288 tmplDims = templates.getTemplate(refDims)
289 self.assertIsInstance(tmplDims, FileTemplate)
290 self.assertNotEqual(tmplDims, tmplSc)
292 # Test that instrument overrides retrieve specialist templates
293 refPvi = self.makeDatasetRef("pvi")
294 refPviHsc = self.makeDatasetRef("pvi", dataId={"instrument": "HSC", "physical_filter": "z"})
295 refPviLsst = self.makeDatasetRef("pvi", dataId={"instrument": "LSST", "physical_filter": "z"})
297 tmplPvi = templates.getTemplate(refPvi)
298 tmplPviHsc = templates.getTemplate(refPviHsc)
299 tmplPviLsst = templates.getTemplate(refPviLsst)
300 self.assertEqual(tmplPvi, tmplPviLsst)
301 self.assertNotEqual(tmplPvi, tmplPviHsc)
303 # Have instrument match and dimensions look up with no name match
304 refNoPviHsc = self.makeDatasetRef(
305 "pvix", dataId={"instrument": "HSC", "physical_filter": "z"}, storageClassName="StorageClassX"
306 )
307 tmplNoPviHsc = templates.getTemplate(refNoPviHsc)
308 self.assertNotEqual(tmplNoPviHsc, tmplDims)
309 self.assertNotEqual(tmplNoPviHsc, tmplPviHsc)
311 # Format config file with defaulting
312 config2 = FileTemplatesConfig(os.path.join(configRoot, "templates-withdefault.yaml"))
313 templates = FileTemplates(config2, universe=self.universe)
314 tmpl = templates.getTemplate(ref2)
315 self.assertIsInstance(tmpl, FileTemplate)
317 # Format config file with bad format string
318 with self.assertRaises(FileTemplateValidationError):
319 FileTemplates(os.path.join(configRoot, "templates-bad.yaml"), universe=self.universe)
321 # Config file with no defaulting mentioned
322 config3 = os.path.join(configRoot, "templates-nodefault2.yaml")
323 templates = FileTemplates(config3, universe=self.universe)
324 with self.assertRaises(KeyError):
325 templates.getTemplate(ref2)
327 # Try again but specify a default in the constructor
328 default = "{run}/{datasetType}/{physical_filter}"
329 templates = FileTemplates(config3, default=default, universe=self.universe)
330 tmpl = templates.getTemplate(ref2)
331 self.assertEqual(tmpl.template, default)
333 def testValidation(self):
334 configRoot = os.path.join(TESTDIR, "config", "templates")
335 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
336 templates = FileTemplates(config1, universe=self.universe)
338 entities = {}
339 entities["calexp"] = self.makeDatasetRef(
340 "calexp",
341 storageClassName="StorageClassX",
342 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 52},
343 )
345 with self.assertLogs(level="WARNING") as cm:
346 templates.validateTemplates(entities.values(), logFailures=True)
347 self.assertIn("Unchecked keys", cm.output[0])
348 self.assertIn("StorageClassX", cm.output[0])
350 entities["pvi"] = self.makeDatasetRef(
351 "pvi", storageClassName="StorageClassX", dataId={"instrument": "dummy", "physical_filter": "i"}
352 )
353 entities["StorageClassX"] = self.makeDatasetRef(
354 "storageClass", storageClassName="StorageClassX", dataId={"instrument": "dummy", "visit": 2}
355 )
356 entities["calexp.wcs"] = self.makeDatasetRef(
357 "calexp.wcs",
358 storageClassName="StorageClassX",
359 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 23},
360 conform=False,
361 )
363 entities["instrument+physical_filter"] = self.makeDatasetRef(
364 "filter_inst",
365 storageClassName="StorageClassX",
366 dataId={"physical_filter": "i", "instrument": "SCUBA"},
367 )
368 entities["hsc+pvi"] = self.makeDatasetRef(
369 "pvi", storageClassName="StorageClassX", dataId={"physical_filter": "i", "instrument": "HSC"}
370 )
372 entities["hsc+instrument+physical_filter"] = self.makeDatasetRef(
373 "filter_inst",
374 storageClassName="StorageClassX",
375 dataId={"physical_filter": "i", "instrument": "HSC"},
376 )
378 entities["metric6"] = self.makeDatasetRef(
379 "filter_inst",
380 storageClassName="Integer",
381 dataId={"physical_filter": "i", "instrument": "HSC"},
382 )
384 templates.validateTemplates(entities.values(), logFailures=True)
386 # Rerun but with a failure
387 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX", dataId={"band": "i"})
388 with self.assertRaises(FileTemplateValidationError):
389 with self.assertLogs(level="FATAL"):
390 templates.validateTemplates(entities.values(), logFailures=True)
393if __name__ == "__main__":
394 unittest.main()