Coverage for tests/test_templates.py: 9%
175 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-28 10:10 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-28 10:10 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Test file name templating."""
24import os.path
25import unittest
27from lsst.daf.butler import (
28 DataCoordinate,
29 DatasetRef,
30 DatasetType,
31 DimensionGraph,
32 DimensionUniverse,
33 FileTemplate,
34 FileTemplates,
35 FileTemplatesConfig,
36 FileTemplateValidationError,
37 StorageClass,
38)
40TESTDIR = os.path.abspath(os.path.dirname(__file__))
42PlaceHolder = StorageClass("PlaceHolder")
45class TestFileTemplates(unittest.TestCase):
46 """Test creation of paths from templates."""
48 def makeDatasetRef(
49 self, datasetTypeName, dataId=None, storageClassName="DefaultStorageClass", run="run2", conform=True
50 ):
51 """Make a simple DatasetRef"""
52 if dataId is None:
53 dataId = self.dataId
54 if "physical_filter" in dataId and "band" not in dataId:
55 dataId["band"] = "b" # Add fake band.
56 dimensions = DimensionGraph(self.universe, names=dataId.keys())
57 dataId = DataCoordinate.standardize(dataId, graph=dimensions)
59 # Pretend we have a parent if this looks like a composite
60 compositeName, componentName = DatasetType.splitDatasetTypeName(datasetTypeName)
61 parentStorageClass = PlaceHolder if componentName else None
63 datasetType = DatasetType(
64 datasetTypeName,
65 dimensions,
66 StorageClass(storageClassName),
67 parentStorageClass=parentStorageClass,
68 )
69 return DatasetRef(datasetType, dataId, id=1, run=run, conform=conform)
71 def setUp(self):
72 self.universe = DimensionUniverse()
73 self.dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "Most Amazing U Filter Ever"}
75 def assertTemplate(self, template, answer, ref):
76 fileTmpl = FileTemplate(template)
77 path = fileTmpl.format(ref)
78 self.assertEqual(path, answer)
80 def testBasic(self):
81 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}"
82 self.assertTemplate(
83 tmplstr,
84 "run2/calexp/00052/Most_Amazing_U_Filter_Ever",
85 self.makeDatasetRef("calexp"),
86 )
87 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail"
88 self.assertTemplate(
89 tmplstr,
90 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail",
91 self.makeDatasetRef("calexp"),
92 )
94 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
95 self.assertTemplate(
96 tmplstr,
97 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run2",
98 self.makeDatasetRef("calexp"),
99 )
100 self.assertTemplate(
101 tmplstr,
102 "run_2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2",
103 self.makeDatasetRef("calexp", run="run/2"),
104 )
106 # Check that the id is sufficient without any other information.
107 self.assertTemplate("{id}", "1", self.makeDatasetRef("calexp", run="run2"))
109 self.assertTemplate("{run}/{id}", "run2/1", self.makeDatasetRef("calexp", run="run2"))
111 self.assertTemplate(
112 "fixed/{id}",
113 "fixed/1",
114 self.makeDatasetRef("calexp", run="run2"),
115 )
117 self.assertTemplate(
118 "fixed/{id}_{physical_filter}",
119 "fixed/1_Most_Amazing_U_Filter_Ever",
120 self.makeDatasetRef("calexp", run="run2"),
121 )
123 # Retain any "/" in run
124 tmplstr = "{run:/}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
125 self.assertTemplate(
126 tmplstr,
127 "run/2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2",
128 self.makeDatasetRef("calexp", run="run/2"),
129 )
131 # Check that "." are replaced in the file basename, but not directory.
132 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "g.10"}
133 self.assertTemplate(
134 tmplstr,
135 "run.2/calexp/00052/g_10-trail-run_2",
136 self.makeDatasetRef("calexp", run="run.2", dataId=dataId),
137 )
139 with self.assertRaises(FileTemplateValidationError):
140 FileTemplate("no fields at all")
142 with self.assertRaises(FileTemplateValidationError):
143 FileTemplate("{visit}")
145 with self.assertRaises(FileTemplateValidationError):
146 FileTemplate("{run}_{datasetType}")
148 with self.assertRaises(FileTemplateValidationError):
149 FileTemplate("{id}/fixed")
151 def testRunOrCollectionNeeded(self):
152 tmplstr = "{datasetType}/{visit:05d}/{physical_filter}"
153 with self.assertRaises(FileTemplateValidationError):
154 self.assertTemplate(tmplstr, "run2/calexp/00052/U", self.makeDatasetRef("calexp"))
156 def testNoRecord(self):
157 # Attaching records is not possible in this test code but we can check
158 # that a missing record when a metadata entry has been requested
159 # does fail.
160 tmplstr = "{run}/{datasetType}/{visit.name}/{physical_filter}"
161 with self.assertRaises(RuntimeError) as cm:
162 self.assertTemplate(tmplstr, "", self.makeDatasetRef("calexp"))
163 self.assertIn("No metadata", str(cm.exception))
165 def testOptional(self):
166 """Optional units in templates."""
167 ref = self.makeDatasetRef("calexp")
168 tmplstr = "{run}/{datasetType}/v{visit:05d}_f{physical_filter:?}"
169 self.assertTemplate(
170 tmplstr,
171 "run2/calexp/v00052_fMost_Amazing_U_Filter_Ever",
172 self.makeDatasetRef("calexp"),
173 )
175 du = {"visit": 48, "tract": 265, "skymap": "big", "instrument": "dummy"}
176 self.assertTemplate(tmplstr, "run2/calexpT/v00048", self.makeDatasetRef("calexpT", du))
178 # Ensure that this returns a relative path even if the first field
179 # is optional
180 tmplstr = "{run}/{tract:?}/{visit:?}/f{physical_filter}"
181 self.assertTemplate(tmplstr, "run2/52/fMost_Amazing_U_Filter_Ever", ref)
183 # Ensure that // from optionals are converted to singles
184 tmplstr = "{run}/{datasetType}/{patch:?}/{tract:?}/f{physical_filter}"
185 self.assertTemplate(tmplstr, "run2/calexp/fMost_Amazing_U_Filter_Ever", ref)
187 # Optionals with some text between fields
188 tmplstr = "{run}/{datasetType}/p{patch:?}_t{tract:?}/f{physical_filter}"
189 self.assertTemplate(tmplstr, "run2/calexp/p/fMost_Amazing_U_Filter_Ever", ref)
190 tmplstr = "{run}/{datasetType}/p{patch:?}_t{visit:04d?}/f{physical_filter}"
191 self.assertTemplate(tmplstr, "run2/calexp/p_t0052/fMost_Amazing_U_Filter_Ever", ref)
193 def testComponent(self):
194 """Test handling of components in templates."""
195 refMetricOutput = self.makeDatasetRef("metric.output")
196 refMetric = self.makeDatasetRef("metric")
197 refMaskedImage = self.makeDatasetRef("calexp.maskedimage.variance")
198 refWcs = self.makeDatasetRef("calexp.wcs")
200 tmplstr = "{run}_c_{component}_v{visit}"
201 self.assertTemplate(tmplstr, "run2_c_output_v52", refMetricOutput)
203 # We want this template to have both a directory and basename, to
204 # test that the right parts of the output are replaced.
205 tmplstr = "{component:?}/{run}_{component:?}_{visit}"
206 self.assertTemplate(tmplstr, "run2_52", refMetric)
207 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput)
208 self.assertTemplate(tmplstr, "maskedimage.variance/run2_maskedimage_variance_52", refMaskedImage)
209 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput)
211 # Providing a component but not using it
212 tmplstr = "{run}/{datasetType}/v{visit:05d}"
213 with self.assertRaises(KeyError):
214 self.assertTemplate(tmplstr, "", refWcs)
216 def testFields(self):
217 # Template, mandatory fields, optional non-special fields,
218 # special fields, optional special fields
219 testData = (
220 (
221 "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail",
222 {"visit", "physical_filter"},
223 set(),
224 {"run", "datasetType"},
225 set(),
226 ),
227 (
228 "{run}/{component:?}_{visit}",
229 {"visit"},
230 set(),
231 {"run"},
232 {"component"},
233 ),
234 (
235 "{run}/{component:?}_{visit:?}_{physical_filter}_{instrument}_{datasetType}",
236 {"physical_filter", "instrument"},
237 {"visit"},
238 {"run", "datasetType"},
239 {"component"},
240 ),
241 )
242 for tmplstr, mandatory, optional, special, optionalSpecial in testData:
243 with self.subTest(template=tmplstr):
244 tmpl = FileTemplate(tmplstr)
245 fields = tmpl.fields()
246 self.assertEqual(fields, mandatory)
247 fields = tmpl.fields(optionals=True)
248 self.assertEqual(fields, mandatory | optional)
249 fields = tmpl.fields(specials=True)
250 self.assertEqual(fields, mandatory | special)
251 fields = tmpl.fields(specials=True, optionals=True)
252 self.assertEqual(fields, mandatory | special | optional | optionalSpecial)
254 def testSimpleConfig(self):
255 """Test reading from config file"""
256 configRoot = os.path.join(TESTDIR, "config", "templates")
257 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
258 templates = FileTemplates(config1, universe=self.universe)
259 ref = self.makeDatasetRef("calexp")
260 tmpl = templates.getTemplate(ref)
261 self.assertIsInstance(tmpl, FileTemplate)
263 # This config file should not allow defaulting
264 ref2 = self.makeDatasetRef("unknown")
265 with self.assertRaises(KeyError):
266 templates.getTemplate(ref2)
268 # This should fall through the datasetTypeName check and use
269 # StorageClass instead
270 ref3 = self.makeDatasetRef("unknown2", storageClassName="StorageClassX")
271 tmplSc = templates.getTemplate(ref3)
272 self.assertIsInstance(tmplSc, FileTemplate)
274 # Try with a component: one with defined formatter and one without
275 refWcs = self.makeDatasetRef("calexp.wcs")
276 refImage = self.makeDatasetRef("calexp.image")
277 tmplCalexp = templates.getTemplate(ref)
278 tmplWcs = templates.getTemplate(refWcs) # Should be special
279 tmpl_image = templates.getTemplate(refImage)
280 self.assertIsInstance(tmplCalexp, FileTemplate)
281 self.assertIsInstance(tmpl_image, FileTemplate)
282 self.assertIsInstance(tmplWcs, FileTemplate)
283 self.assertEqual(tmplCalexp, tmpl_image)
284 self.assertNotEqual(tmplCalexp, tmplWcs)
286 # Check dimensions lookup order.
287 # The order should be: dataset type name, dimension, storage class
288 # This one will not match name but might match storage class.
289 # It should match dimensions
290 refDims = self.makeDatasetRef(
291 "nomatch", dataId={"instrument": "LSST", "physical_filter": "z"}, storageClassName="StorageClassX"
292 )
293 tmplDims = templates.getTemplate(refDims)
294 self.assertIsInstance(tmplDims, FileTemplate)
295 self.assertNotEqual(tmplDims, tmplSc)
297 # Test that instrument overrides retrieve specialist templates
298 refPvi = self.makeDatasetRef("pvi")
299 refPviHsc = self.makeDatasetRef("pvi", dataId={"instrument": "HSC", "physical_filter": "z"})
300 refPviLsst = self.makeDatasetRef("pvi", dataId={"instrument": "LSST", "physical_filter": "z"})
302 tmplPvi = templates.getTemplate(refPvi)
303 tmplPviHsc = templates.getTemplate(refPviHsc)
304 tmplPviLsst = templates.getTemplate(refPviLsst)
305 self.assertEqual(tmplPvi, tmplPviLsst)
306 self.assertNotEqual(tmplPvi, tmplPviHsc)
308 # Have instrument match and dimensions look up with no name match
309 refNoPviHsc = self.makeDatasetRef(
310 "pvix", dataId={"instrument": "HSC", "physical_filter": "z"}, storageClassName="StorageClassX"
311 )
312 tmplNoPviHsc = templates.getTemplate(refNoPviHsc)
313 self.assertNotEqual(tmplNoPviHsc, tmplDims)
314 self.assertNotEqual(tmplNoPviHsc, tmplPviHsc)
316 # Format config file with defaulting
317 config2 = FileTemplatesConfig(os.path.join(configRoot, "templates-withdefault.yaml"))
318 templates = FileTemplates(config2, universe=self.universe)
319 tmpl = templates.getTemplate(ref2)
320 self.assertIsInstance(tmpl, FileTemplate)
322 # Format config file with bad format string
323 with self.assertRaises(FileTemplateValidationError):
324 FileTemplates(os.path.join(configRoot, "templates-bad.yaml"), universe=self.universe)
326 # Config file with no defaulting mentioned
327 config3 = os.path.join(configRoot, "templates-nodefault2.yaml")
328 templates = FileTemplates(config3, universe=self.universe)
329 with self.assertRaises(KeyError):
330 templates.getTemplate(ref2)
332 # Try again but specify a default in the constructor
333 default = "{run}/{datasetType}/{physical_filter}"
334 templates = FileTemplates(config3, default=default, universe=self.universe)
335 tmpl = templates.getTemplate(ref2)
336 self.assertEqual(tmpl.template, default)
338 def testValidation(self):
339 configRoot = os.path.join(TESTDIR, "config", "templates")
340 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
341 templates = FileTemplates(config1, universe=self.universe)
343 entities = {}
344 entities["calexp"] = self.makeDatasetRef(
345 "calexp",
346 storageClassName="StorageClassX",
347 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 52},
348 )
350 with self.assertLogs(level="WARNING") as cm:
351 templates.validateTemplates(entities.values(), logFailures=True)
352 self.assertIn("Unchecked keys", cm.output[0])
353 self.assertIn("StorageClassX", cm.output[0])
355 entities["pvi"] = self.makeDatasetRef(
356 "pvi", storageClassName="StorageClassX", dataId={"instrument": "dummy", "physical_filter": "i"}
357 )
358 entities["StorageClassX"] = self.makeDatasetRef(
359 "storageClass", storageClassName="StorageClassX", dataId={"instrument": "dummy", "visit": 2}
360 )
361 entities["calexp.wcs"] = self.makeDatasetRef(
362 "calexp.wcs",
363 storageClassName="StorageClassX",
364 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 23},
365 conform=False,
366 )
368 entities["instrument+physical_filter"] = self.makeDatasetRef(
369 "filter_inst",
370 storageClassName="StorageClassX",
371 dataId={"physical_filter": "i", "instrument": "SCUBA"},
372 )
373 entities["hsc+pvi"] = self.makeDatasetRef(
374 "pvi", storageClassName="StorageClassX", dataId={"physical_filter": "i", "instrument": "HSC"}
375 )
377 entities["hsc+instrument+physical_filter"] = self.makeDatasetRef(
378 "filter_inst",
379 storageClassName="StorageClassX",
380 dataId={"physical_filter": "i", "instrument": "HSC"},
381 )
383 entities["metric6"] = self.makeDatasetRef(
384 "filter_inst",
385 storageClassName="Integer",
386 dataId={"physical_filter": "i", "instrument": "HSC"},
387 )
389 templates.validateTemplates(entities.values(), logFailures=True)
391 # Rerun but with a failure
392 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX", dataId={"band": "i"})
393 with self.assertRaises(FileTemplateValidationError):
394 with self.assertLogs(level="FATAL"):
395 templates.validateTemplates(entities.values(), logFailures=True)
398if __name__ == "__main__":
399 unittest.main()