Coverage for tests/test_templates.py: 11%
160 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-01 19:55 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-01 19:55 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Test file name templating."""
24import os.path
25import unittest
27from lsst.daf.butler import DatasetType, DatasetRef, FileTemplates, DimensionUniverse, \
28 FileTemplate, FileTemplatesConfig, StorageClass, FileTemplateValidationError, \
29 DimensionGraph
31TESTDIR = os.path.abspath(os.path.dirname(__file__))
34class TestFileTemplates(unittest.TestCase):
35 """Test creation of paths from templates."""
37 def makeDatasetRef(self, datasetTypeName, dataId=None, storageClassName="DefaultStorageClass",
38 run="run2", conform=True):
39 """Make a simple DatasetRef"""
40 if dataId is None:
41 dataId = self.dataId
43 # Pretend we have a parent if this looks like a composite
44 compositeName, componentName = DatasetType.splitDatasetTypeName(datasetTypeName)
45 parentStorageClass = DatasetType.PlaceholderParentStorageClass if componentName else None
47 datasetType = DatasetType(datasetTypeName, DimensionGraph(self.universe, names=dataId.keys()),
48 StorageClass(storageClassName),
49 parentStorageClass=parentStorageClass)
50 return DatasetRef(datasetType, dataId, id=1, run=run, conform=conform)
52 def setUp(self):
53 self.universe = DimensionUniverse()
54 self.dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "Most Amazing U Filter Ever"}
56 def assertTemplate(self, template, answer, ref):
57 fileTmpl = FileTemplate(template)
58 path = fileTmpl.format(ref)
59 self.assertEqual(path, answer)
61 def testBasic(self):
62 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}"
63 self.assertTemplate(tmplstr,
64 "run2/calexp/00052/Most_Amazing_U_Filter_Ever",
65 self.makeDatasetRef("calexp", conform=False))
66 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail"
67 self.assertTemplate(tmplstr,
68 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail",
69 self.makeDatasetRef("calexp", conform=False))
71 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
72 self.assertTemplate(tmplstr,
73 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run2",
74 self.makeDatasetRef("calexp", conform=False))
75 self.assertTemplate(tmplstr,
76 "run_2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2",
77 self.makeDatasetRef("calexp", run="run/2", conform=False))
79 # Retain any "/" in run
80 tmplstr = "{run:/}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
81 self.assertTemplate(tmplstr,
82 "run/2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2",
83 self.makeDatasetRef("calexp", run="run/2", conform=False))
85 # Check that "." are replaced in the file basename, but not directory.
86 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "g.10"}
87 self.assertTemplate(tmplstr,
88 "run.2/calexp/00052/g_10-trail-run_2",
89 self.makeDatasetRef("calexp", run="run.2", dataId=dataId, conform=False))
91 with self.assertRaises(FileTemplateValidationError):
92 FileTemplate("no fields at all")
94 with self.assertRaises(FileTemplateValidationError):
95 FileTemplate("{visit}")
97 with self.assertRaises(FileTemplateValidationError):
98 FileTemplate("{run}_{datasetType}")
100 def testRunOrCollectionNeeded(self):
101 tmplstr = "{datasetType}/{visit:05d}/{physical_filter}"
102 with self.assertRaises(FileTemplateValidationError):
103 self.assertTemplate(tmplstr,
104 "run2/calexp/00052/U",
105 self.makeDatasetRef("calexp"))
107 def testOptional(self):
108 """Optional units in templates."""
109 ref = self.makeDatasetRef("calexp", conform=False)
110 tmplstr = "{run}/{datasetType}/v{visit:05d}_f{physical_filter:?}"
111 self.assertTemplate(tmplstr, "run2/calexp/v00052_fMost_Amazing_U_Filter_Ever",
112 self.makeDatasetRef("calexp", conform=False))
114 du = {"visit": 48, "tract": 265, "skymap": "big", "instrument": "dummy"}
115 self.assertTemplate(tmplstr, "run2/calexpT/v00048",
116 self.makeDatasetRef("calexpT", du, conform=False))
118 # Ensure that this returns a relative path even if the first field
119 # is optional
120 tmplstr = "{run}/{tract:?}/{visit:?}/f{physical_filter}"
121 self.assertTemplate(tmplstr, "run2/52/fMost_Amazing_U_Filter_Ever", ref)
123 # Ensure that // from optionals are converted to singles
124 tmplstr = "{run}/{datasetType}/{patch:?}/{tract:?}/f{physical_filter}"
125 self.assertTemplate(tmplstr, "run2/calexp/fMost_Amazing_U_Filter_Ever", ref)
127 # Optionals with some text between fields
128 tmplstr = "{run}/{datasetType}/p{patch:?}_t{tract:?}/f{physical_filter}"
129 self.assertTemplate(tmplstr, "run2/calexp/p/fMost_Amazing_U_Filter_Ever", ref)
130 tmplstr = "{run}/{datasetType}/p{patch:?}_t{visit:04d?}/f{physical_filter}"
131 self.assertTemplate(tmplstr, "run2/calexp/p_t0052/fMost_Amazing_U_Filter_Ever", ref)
133 def testComponent(self):
134 """Test handling of components in templates."""
135 refMetricOutput = self.makeDatasetRef("metric.output")
136 refMetric = self.makeDatasetRef("metric")
137 refMaskedImage = self.makeDatasetRef("calexp.maskedimage.variance")
138 refWcs = self.makeDatasetRef("calexp.wcs")
140 tmplstr = "{run}_c_{component}_v{visit}"
141 self.assertTemplate(tmplstr, "run2_c_output_v52", refMetricOutput)
143 # We want this template to have both a directory and basename, to
144 # test that the right parts of the output are replaced.
145 tmplstr = "{component:?}/{run}_{component:?}_{visit}"
146 self.assertTemplate(tmplstr, "run2_52", refMetric)
147 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput)
148 self.assertTemplate(tmplstr, "maskedimage.variance/run2_maskedimage_variance_52", refMaskedImage)
149 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput)
151 # Providing a component but not using it
152 tmplstr = "{run}/{datasetType}/v{visit:05d}"
153 with self.assertRaises(KeyError):
154 self.assertTemplate(tmplstr, "", refWcs)
156 def testFields(self):
157 # Template, mandatory fields, optional non-special fields,
158 # special fields, optional special fields
159 testData = (("{run}/{datasetType}/{visit:05d}/{physical_filter}-trail",
160 set(["visit", "physical_filter"]),
161 set(),
162 set(["run", "datasetType"]),
163 set()),
164 ("{run}/{component:?}_{visit}",
165 set(["visit"]),
166 set(),
167 set(["run"]),
168 set(["component"]),),
169 ("{run}/{component:?}_{visit:?}_{physical_filter}_{instrument}_{datasetType}",
170 set(["physical_filter", "instrument"]),
171 set(["visit"]),
172 set(["run", "datasetType"]),
173 set(["component"]),),
174 )
175 for tmplstr, mandatory, optional, special, optionalSpecial in testData:
176 with self.subTest(template=tmplstr):
177 tmpl = FileTemplate(tmplstr)
178 fields = tmpl.fields()
179 self.assertEqual(fields, mandatory)
180 fields = tmpl.fields(optionals=True)
181 self.assertEqual(fields, mandatory | optional)
182 fields = tmpl.fields(specials=True)
183 self.assertEqual(fields, mandatory | special)
184 fields = tmpl.fields(specials=True, optionals=True)
185 self.assertEqual(fields, mandatory | special | optional | optionalSpecial)
187 def testSimpleConfig(self):
188 """Test reading from config file"""
189 configRoot = os.path.join(TESTDIR, "config", "templates")
190 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
191 templates = FileTemplates(config1, universe=self.universe)
192 ref = self.makeDatasetRef("calexp")
193 tmpl = templates.getTemplate(ref)
194 self.assertIsInstance(tmpl, FileTemplate)
196 # This config file should not allow defaulting
197 ref2 = self.makeDatasetRef("unknown")
198 with self.assertRaises(KeyError):
199 templates.getTemplate(ref2)
201 # This should fall through the datasetTypeName check and use
202 # StorageClass instead
203 ref3 = self.makeDatasetRef("unknown2", storageClassName="StorageClassX")
204 tmplSc = templates.getTemplate(ref3)
205 self.assertIsInstance(tmplSc, FileTemplate)
207 # Try with a component: one with defined formatter and one without
208 refWcs = self.makeDatasetRef("calexp.wcs")
209 refImage = self.makeDatasetRef("calexp.image")
210 tmplCalexp = templates.getTemplate(ref)
211 tmplWcs = templates.getTemplate(refWcs) # Should be special
212 tmpl_image = templates.getTemplate(refImage)
213 self.assertIsInstance(tmplCalexp, FileTemplate)
214 self.assertIsInstance(tmpl_image, FileTemplate)
215 self.assertIsInstance(tmplWcs, FileTemplate)
216 self.assertEqual(tmplCalexp, tmpl_image)
217 self.assertNotEqual(tmplCalexp, tmplWcs)
219 # Check dimensions lookup order.
220 # The order should be: dataset type name, dimension, storage class
221 # This one will not match name but might match storage class.
222 # It should match dimensions
223 refDims = self.makeDatasetRef("nomatch", dataId={"instrument": "LSST", "physical_filter": "z"},
224 storageClassName="StorageClassX")
225 tmplDims = templates.getTemplate(refDims)
226 self.assertIsInstance(tmplDims, FileTemplate)
227 self.assertNotEqual(tmplDims, tmplSc)
229 # Test that instrument overrides retrieve specialist templates
230 refPvi = self.makeDatasetRef("pvi")
231 refPviHsc = self.makeDatasetRef("pvi", dataId={"instrument": "HSC", "physical_filter": "z"})
232 refPviLsst = self.makeDatasetRef("pvi", dataId={"instrument": "LSST", "physical_filter": "z"})
234 tmplPvi = templates.getTemplate(refPvi)
235 tmplPviHsc = templates.getTemplate(refPviHsc)
236 tmplPviLsst = templates.getTemplate(refPviLsst)
237 self.assertEqual(tmplPvi, tmplPviLsst)
238 self.assertNotEqual(tmplPvi, tmplPviHsc)
240 # Have instrument match and dimensions look up with no name match
241 refNoPviHsc = self.makeDatasetRef("pvix", dataId={"instrument": "HSC", "physical_filter": "z"},
242 storageClassName="StorageClassX")
243 tmplNoPviHsc = templates.getTemplate(refNoPviHsc)
244 self.assertNotEqual(tmplNoPviHsc, tmplDims)
245 self.assertNotEqual(tmplNoPviHsc, tmplPviHsc)
247 # Format config file with defaulting
248 config2 = FileTemplatesConfig(os.path.join(configRoot, "templates-withdefault.yaml"))
249 templates = FileTemplates(config2, universe=self.universe)
250 tmpl = templates.getTemplate(ref2)
251 self.assertIsInstance(tmpl, FileTemplate)
253 # Format config file with bad format string
254 with self.assertRaises(FileTemplateValidationError):
255 FileTemplates(os.path.join(configRoot, "templates-bad.yaml"), universe=self.universe)
257 # Config file with no defaulting mentioned
258 config3 = os.path.join(configRoot, "templates-nodefault2.yaml")
259 templates = FileTemplates(config3, universe=self.universe)
260 with self.assertRaises(KeyError):
261 templates.getTemplate(ref2)
263 # Try again but specify a default in the constructor
264 default = "{run}/{datasetType}/{physical_filter}"
265 templates = FileTemplates(config3, default=default, universe=self.universe)
266 tmpl = templates.getTemplate(ref2)
267 self.assertEqual(tmpl.template, default)
269 def testValidation(self):
270 configRoot = os.path.join(TESTDIR, "config", "templates")
271 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
272 templates = FileTemplates(config1, universe=self.universe)
274 entities = {}
275 entities["calexp"] = self.makeDatasetRef("calexp", storageClassName="StorageClassX",
276 dataId={"instrument": "dummy", "physical_filter": "i",
277 "visit": 52})
279 with self.assertLogs(level="WARNING") as cm:
280 templates.validateTemplates(entities.values(), logFailures=True)
281 self.assertIn("Unchecked keys", cm.output[0])
282 self.assertIn("StorageClassX", cm.output[0])
284 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX",
285 dataId={"instrument": "dummy", "physical_filter": "i"})
286 entities["StorageClassX"] = self.makeDatasetRef("storageClass",
287 storageClassName="StorageClassX",
288 dataId={"instrument": "dummy", "visit": 2})
289 entities["calexp.wcs"] = self.makeDatasetRef("calexp.wcs",
290 storageClassName="StorageClassX",
291 dataId={"instrument": "dummy",
292 "physical_filter": "i", "visit": 23},
293 conform=False)
295 entities["instrument+physical_filter"] = self.makeDatasetRef("filter_inst",
296 storageClassName="StorageClassX",
297 dataId={"physical_filter": "i",
298 "instrument": "SCUBA"})
299 entities["hsc+pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX",
300 dataId={"physical_filter": "i", "instrument": "HSC"})
302 entities["hsc+instrument+physical_filter"] = self.makeDatasetRef("filter_inst",
303 storageClassName="StorageClassX",
304 dataId={"physical_filter": "i",
305 "instrument": "HSC"})
307 templates.validateTemplates(entities.values(), logFailures=True)
309 # Rerun but with a failure
310 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX",
311 dataId={"band": "i"})
312 with self.assertRaises(FileTemplateValidationError):
313 with self.assertLogs(level="FATAL"):
314 templates.validateTemplates(entities.values(), logFailures=True)
317if __name__ == "__main__": 317 ↛ 318line 317 didn't jump to line 318, because the condition on line 317 was never true
318 unittest.main()