Coverage for tests/test_templates.py: 9%
175 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-23 09:30 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-23 09:30 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Test file name templating."""
24import os.path
25import unittest
27from lsst.daf.butler import (
28 DataCoordinate,
29 DatasetRef,
30 DatasetType,
31 DimensionGraph,
32 DimensionUniverse,
33 FileTemplate,
34 FileTemplates,
35 FileTemplatesConfig,
36 FileTemplateValidationError,
37 StorageClass,
38)
40TESTDIR = os.path.abspath(os.path.dirname(__file__))
42PlaceHolder = StorageClass("PlaceHolder")
45class TestFileTemplates(unittest.TestCase):
46 """Test creation of paths from templates."""
48 def makeDatasetRef(
49 self, datasetTypeName, dataId=None, storageClassName="DefaultStorageClass", run="run2", conform=True
50 ):
51 """Make a simple DatasetRef"""
53 if dataId is None:
54 dataId = self.dataId
55 if "physical_filter" in dataId and "band" not in dataId:
56 dataId["band"] = "b" # Add fake band.
57 dimensions = DimensionGraph(self.universe, names=dataId.keys())
58 dataId = DataCoordinate.standardize(dataId, graph=dimensions)
60 # Pretend we have a parent if this looks like a composite
61 compositeName, componentName = DatasetType.splitDatasetTypeName(datasetTypeName)
62 parentStorageClass = PlaceHolder if componentName else None
64 datasetType = DatasetType(
65 datasetTypeName,
66 dimensions,
67 StorageClass(storageClassName),
68 parentStorageClass=parentStorageClass,
69 )
70 return DatasetRef(datasetType, dataId, id=1, run=run, conform=conform)
72 def setUp(self):
73 self.universe = DimensionUniverse()
74 self.dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "Most Amazing U Filter Ever"}
76 def assertTemplate(self, template, answer, ref):
77 fileTmpl = FileTemplate(template)
78 path = fileTmpl.format(ref)
79 self.assertEqual(path, answer)
81 def testBasic(self):
82 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}"
83 self.assertTemplate(
84 tmplstr,
85 "run2/calexp/00052/Most_Amazing_U_Filter_Ever",
86 self.makeDatasetRef("calexp"),
87 )
88 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail"
89 self.assertTemplate(
90 tmplstr,
91 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail",
92 self.makeDatasetRef("calexp"),
93 )
95 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
96 self.assertTemplate(
97 tmplstr,
98 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run2",
99 self.makeDatasetRef("calexp"),
100 )
101 self.assertTemplate(
102 tmplstr,
103 "run_2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2",
104 self.makeDatasetRef("calexp", run="run/2"),
105 )
107 # Check that the id is sufficient without any other information.
108 self.assertTemplate("{id}", "1", self.makeDatasetRef("calexp", run="run2"))
110 self.assertTemplate("{run}/{id}", "run2/1", self.makeDatasetRef("calexp", run="run2"))
112 self.assertTemplate(
113 "fixed/{id}",
114 "fixed/1",
115 self.makeDatasetRef("calexp", run="run2"),
116 )
118 self.assertTemplate(
119 "fixed/{id}_{physical_filter}",
120 "fixed/1_Most_Amazing_U_Filter_Ever",
121 self.makeDatasetRef("calexp", run="run2"),
122 )
124 # Retain any "/" in run
125 tmplstr = "{run:/}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
126 self.assertTemplate(
127 tmplstr,
128 "run/2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2",
129 self.makeDatasetRef("calexp", run="run/2"),
130 )
132 # Check that "." are replaced in the file basename, but not directory.
133 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "g.10"}
134 self.assertTemplate(
135 tmplstr,
136 "run.2/calexp/00052/g_10-trail-run_2",
137 self.makeDatasetRef("calexp", run="run.2", dataId=dataId),
138 )
140 with self.assertRaises(FileTemplateValidationError):
141 FileTemplate("no fields at all")
143 with self.assertRaises(FileTemplateValidationError):
144 FileTemplate("{visit}")
146 with self.assertRaises(FileTemplateValidationError):
147 FileTemplate("{run}_{datasetType}")
149 with self.assertRaises(FileTemplateValidationError):
150 FileTemplate("{id}/fixed")
152 def testRunOrCollectionNeeded(self):
153 tmplstr = "{datasetType}/{visit:05d}/{physical_filter}"
154 with self.assertRaises(FileTemplateValidationError):
155 self.assertTemplate(tmplstr, "run2/calexp/00052/U", self.makeDatasetRef("calexp"))
157 def testNoRecord(self):
158 # Attaching records is not possible in this test code but we can check
159 # that a missing record when a metadata entry has been requested
160 # does fail.
161 tmplstr = "{run}/{datasetType}/{visit.name}/{physical_filter}"
162 with self.assertRaises(RuntimeError) as cm:
163 self.assertTemplate(tmplstr, "", self.makeDatasetRef("calexp"))
164 self.assertIn("No metadata", str(cm.exception))
166 def testOptional(self):
167 """Optional units in templates."""
168 ref = self.makeDatasetRef("calexp")
169 tmplstr = "{run}/{datasetType}/v{visit:05d}_f{physical_filter:?}"
170 self.assertTemplate(
171 tmplstr,
172 "run2/calexp/v00052_fMost_Amazing_U_Filter_Ever",
173 self.makeDatasetRef("calexp"),
174 )
176 du = {"visit": 48, "tract": 265, "skymap": "big", "instrument": "dummy"}
177 self.assertTemplate(tmplstr, "run2/calexpT/v00048", self.makeDatasetRef("calexpT", du))
179 # Ensure that this returns a relative path even if the first field
180 # is optional
181 tmplstr = "{run}/{tract:?}/{visit:?}/f{physical_filter}"
182 self.assertTemplate(tmplstr, "run2/52/fMost_Amazing_U_Filter_Ever", ref)
184 # Ensure that // from optionals are converted to singles
185 tmplstr = "{run}/{datasetType}/{patch:?}/{tract:?}/f{physical_filter}"
186 self.assertTemplate(tmplstr, "run2/calexp/fMost_Amazing_U_Filter_Ever", ref)
188 # Optionals with some text between fields
189 tmplstr = "{run}/{datasetType}/p{patch:?}_t{tract:?}/f{physical_filter}"
190 self.assertTemplate(tmplstr, "run2/calexp/p/fMost_Amazing_U_Filter_Ever", ref)
191 tmplstr = "{run}/{datasetType}/p{patch:?}_t{visit:04d?}/f{physical_filter}"
192 self.assertTemplate(tmplstr, "run2/calexp/p_t0052/fMost_Amazing_U_Filter_Ever", ref)
194 def testComponent(self):
195 """Test handling of components in templates."""
196 refMetricOutput = self.makeDatasetRef("metric.output")
197 refMetric = self.makeDatasetRef("metric")
198 refMaskedImage = self.makeDatasetRef("calexp.maskedimage.variance")
199 refWcs = self.makeDatasetRef("calexp.wcs")
201 tmplstr = "{run}_c_{component}_v{visit}"
202 self.assertTemplate(tmplstr, "run2_c_output_v52", refMetricOutput)
204 # We want this template to have both a directory and basename, to
205 # test that the right parts of the output are replaced.
206 tmplstr = "{component:?}/{run}_{component:?}_{visit}"
207 self.assertTemplate(tmplstr, "run2_52", refMetric)
208 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput)
209 self.assertTemplate(tmplstr, "maskedimage.variance/run2_maskedimage_variance_52", refMaskedImage)
210 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput)
212 # Providing a component but not using it
213 tmplstr = "{run}/{datasetType}/v{visit:05d}"
214 with self.assertRaises(KeyError):
215 self.assertTemplate(tmplstr, "", refWcs)
217 def testFields(self):
218 # Template, mandatory fields, optional non-special fields,
219 # special fields, optional special fields
220 testData = (
221 (
222 "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail",
223 {"visit", "physical_filter"},
224 set(),
225 {"run", "datasetType"},
226 set(),
227 ),
228 (
229 "{run}/{component:?}_{visit}",
230 {"visit"},
231 set(),
232 {"run"},
233 {"component"},
234 ),
235 (
236 "{run}/{component:?}_{visit:?}_{physical_filter}_{instrument}_{datasetType}",
237 {"physical_filter", "instrument"},
238 {"visit"},
239 {"run", "datasetType"},
240 {"component"},
241 ),
242 )
243 for tmplstr, mandatory, optional, special, optionalSpecial in testData:
244 with self.subTest(template=tmplstr):
245 tmpl = FileTemplate(tmplstr)
246 fields = tmpl.fields()
247 self.assertEqual(fields, mandatory)
248 fields = tmpl.fields(optionals=True)
249 self.assertEqual(fields, mandatory | optional)
250 fields = tmpl.fields(specials=True)
251 self.assertEqual(fields, mandatory | special)
252 fields = tmpl.fields(specials=True, optionals=True)
253 self.assertEqual(fields, mandatory | special | optional | optionalSpecial)
255 def testSimpleConfig(self):
256 """Test reading from config file"""
257 configRoot = os.path.join(TESTDIR, "config", "templates")
258 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
259 templates = FileTemplates(config1, universe=self.universe)
260 ref = self.makeDatasetRef("calexp")
261 tmpl = templates.getTemplate(ref)
262 self.assertIsInstance(tmpl, FileTemplate)
264 # This config file should not allow defaulting
265 ref2 = self.makeDatasetRef("unknown")
266 with self.assertRaises(KeyError):
267 templates.getTemplate(ref2)
269 # This should fall through the datasetTypeName check and use
270 # StorageClass instead
271 ref3 = self.makeDatasetRef("unknown2", storageClassName="StorageClassX")
272 tmplSc = templates.getTemplate(ref3)
273 self.assertIsInstance(tmplSc, FileTemplate)
275 # Try with a component: one with defined formatter and one without
276 refWcs = self.makeDatasetRef("calexp.wcs")
277 refImage = self.makeDatasetRef("calexp.image")
278 tmplCalexp = templates.getTemplate(ref)
279 tmplWcs = templates.getTemplate(refWcs) # Should be special
280 tmpl_image = templates.getTemplate(refImage)
281 self.assertIsInstance(tmplCalexp, FileTemplate)
282 self.assertIsInstance(tmpl_image, FileTemplate)
283 self.assertIsInstance(tmplWcs, FileTemplate)
284 self.assertEqual(tmplCalexp, tmpl_image)
285 self.assertNotEqual(tmplCalexp, tmplWcs)
287 # Check dimensions lookup order.
288 # The order should be: dataset type name, dimension, storage class
289 # This one will not match name but might match storage class.
290 # It should match dimensions
291 refDims = self.makeDatasetRef(
292 "nomatch", dataId={"instrument": "LSST", "physical_filter": "z"}, storageClassName="StorageClassX"
293 )
294 tmplDims = templates.getTemplate(refDims)
295 self.assertIsInstance(tmplDims, FileTemplate)
296 self.assertNotEqual(tmplDims, tmplSc)
298 # Test that instrument overrides retrieve specialist templates
299 refPvi = self.makeDatasetRef("pvi")
300 refPviHsc = self.makeDatasetRef("pvi", dataId={"instrument": "HSC", "physical_filter": "z"})
301 refPviLsst = self.makeDatasetRef("pvi", dataId={"instrument": "LSST", "physical_filter": "z"})
303 tmplPvi = templates.getTemplate(refPvi)
304 tmplPviHsc = templates.getTemplate(refPviHsc)
305 tmplPviLsst = templates.getTemplate(refPviLsst)
306 self.assertEqual(tmplPvi, tmplPviLsst)
307 self.assertNotEqual(tmplPvi, tmplPviHsc)
309 # Have instrument match and dimensions look up with no name match
310 refNoPviHsc = self.makeDatasetRef(
311 "pvix", dataId={"instrument": "HSC", "physical_filter": "z"}, storageClassName="StorageClassX"
312 )
313 tmplNoPviHsc = templates.getTemplate(refNoPviHsc)
314 self.assertNotEqual(tmplNoPviHsc, tmplDims)
315 self.assertNotEqual(tmplNoPviHsc, tmplPviHsc)
317 # Format config file with defaulting
318 config2 = FileTemplatesConfig(os.path.join(configRoot, "templates-withdefault.yaml"))
319 templates = FileTemplates(config2, universe=self.universe)
320 tmpl = templates.getTemplate(ref2)
321 self.assertIsInstance(tmpl, FileTemplate)
323 # Format config file with bad format string
324 with self.assertRaises(FileTemplateValidationError):
325 FileTemplates(os.path.join(configRoot, "templates-bad.yaml"), universe=self.universe)
327 # Config file with no defaulting mentioned
328 config3 = os.path.join(configRoot, "templates-nodefault2.yaml")
329 templates = FileTemplates(config3, universe=self.universe)
330 with self.assertRaises(KeyError):
331 templates.getTemplate(ref2)
333 # Try again but specify a default in the constructor
334 default = "{run}/{datasetType}/{physical_filter}"
335 templates = FileTemplates(config3, default=default, universe=self.universe)
336 tmpl = templates.getTemplate(ref2)
337 self.assertEqual(tmpl.template, default)
339 def testValidation(self):
340 configRoot = os.path.join(TESTDIR, "config", "templates")
341 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
342 templates = FileTemplates(config1, universe=self.universe)
344 entities = {}
345 entities["calexp"] = self.makeDatasetRef(
346 "calexp",
347 storageClassName="StorageClassX",
348 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 52},
349 )
351 with self.assertLogs(level="WARNING") as cm:
352 templates.validateTemplates(entities.values(), logFailures=True)
353 self.assertIn("Unchecked keys", cm.output[0])
354 self.assertIn("StorageClassX", cm.output[0])
356 entities["pvi"] = self.makeDatasetRef(
357 "pvi", storageClassName="StorageClassX", dataId={"instrument": "dummy", "physical_filter": "i"}
358 )
359 entities["StorageClassX"] = self.makeDatasetRef(
360 "storageClass", storageClassName="StorageClassX", dataId={"instrument": "dummy", "visit": 2}
361 )
362 entities["calexp.wcs"] = self.makeDatasetRef(
363 "calexp.wcs",
364 storageClassName="StorageClassX",
365 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 23},
366 conform=False,
367 )
369 entities["instrument+physical_filter"] = self.makeDatasetRef(
370 "filter_inst",
371 storageClassName="StorageClassX",
372 dataId={"physical_filter": "i", "instrument": "SCUBA"},
373 )
374 entities["hsc+pvi"] = self.makeDatasetRef(
375 "pvi", storageClassName="StorageClassX", dataId={"physical_filter": "i", "instrument": "HSC"}
376 )
378 entities["hsc+instrument+physical_filter"] = self.makeDatasetRef(
379 "filter_inst",
380 storageClassName="StorageClassX",
381 dataId={"physical_filter": "i", "instrument": "HSC"},
382 )
384 entities["metric6"] = self.makeDatasetRef(
385 "filter_inst",
386 storageClassName="Integer",
387 dataId={"physical_filter": "i", "instrument": "HSC"},
388 )
390 templates.validateTemplates(entities.values(), logFailures=True)
392 # Rerun but with a failure
393 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX", dataId={"band": "i"})
394 with self.assertRaises(FileTemplateValidationError):
395 with self.assertLogs(level="FATAL"):
396 templates.validateTemplates(entities.values(), logFailures=True)
399if __name__ == "__main__":
400 unittest.main()