Coverage for tests/test_templates.py : 11%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Test file name templating."""
24import os.path
25import unittest
27from lsst.daf.butler import DatasetType, DatasetRef, FileTemplates, DimensionUniverse, \
28 FileTemplate, FileTemplatesConfig, StorageClass, FileTemplateValidationError, \
29 DimensionGraph
31TESTDIR = os.path.abspath(os.path.dirname(__file__))
34class TestFileTemplates(unittest.TestCase):
35 """Test creation of paths from templates."""
37 def makeDatasetRef(self, datasetTypeName, dataId=None, storageClassName="DefaultStorageClass",
38 run="run2", conform=True):
39 """Make a simple DatasetRef"""
40 if dataId is None:
41 dataId = self.dataId
42 datasetType = DatasetType(datasetTypeName, DimensionGraph(self.universe, names=dataId.keys()),
43 StorageClass(storageClassName))
44 return DatasetRef(datasetType, dataId, id=1, run=run, conform=conform)
46 def setUp(self):
47 self.universe = DimensionUniverse()
48 self.dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "U"}
50 def assertTemplate(self, template, answer, ref):
51 fileTmpl = FileTemplate(template)
52 path = fileTmpl.format(ref)
53 self.assertEqual(path, answer)
55 def testBasic(self):
56 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}"
57 self.assertTemplate(tmplstr,
58 "run2/calexp/00052/U",
59 self.makeDatasetRef("calexp", conform=False))
60 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail"
61 self.assertTemplate(tmplstr,
62 "run2/calexp/00052/U-trail",
63 self.makeDatasetRef("calexp", conform=False))
65 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
66 self.assertTemplate(tmplstr,
67 "run2/calexp/00052/U-trail-run2",
68 self.makeDatasetRef("calexp", conform=False))
69 self.assertTemplate(tmplstr,
70 "run_2/calexp/00052/U-trail-run_2",
71 self.makeDatasetRef("calexp", run="run/2", conform=False))
73 # Retain any "/" in run
74 tmplstr = "{run:/}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
75 self.assertTemplate(tmplstr,
76 "run/2/calexp/00052/U-trail-run_2",
77 self.makeDatasetRef("calexp", run="run/2", conform=False))
79 # Check that "." are replaced in the file basename, but not directory.
80 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "g.10"}
81 self.assertTemplate(tmplstr,
82 "run.2/calexp/00052/g_10-trail-run_2",
83 self.makeDatasetRef("calexp", run="run.2", dataId=dataId, conform=False))
85 with self.assertRaises(FileTemplateValidationError):
86 FileTemplate("no fields at all")
88 with self.assertRaises(FileTemplateValidationError):
89 FileTemplate("{visit}")
91 with self.assertRaises(FileTemplateValidationError):
92 FileTemplate("{run}_{datasetType}")
94 def testRunOrCollectionNeeded(self):
95 tmplstr = "{datasetType}/{visit:05d}/{physical_filter}"
96 with self.assertRaises(FileTemplateValidationError):
97 self.assertTemplate(tmplstr,
98 "run2/calexp/00052/U",
99 self.makeDatasetRef("calexp"))
101 def testOptional(self):
102 """Optional units in templates."""
103 ref = self.makeDatasetRef("calexp", conform=False)
104 tmplstr = "{run}/{datasetType}/v{visit:05d}_f{physical_filter:?}"
105 self.assertTemplate(tmplstr, "run2/calexp/v00052_fU",
106 self.makeDatasetRef("calexp", conform=False))
108 du = {"visit": 48, "tract": 265, "skymap": "big", "instrument": "dummy"}
109 self.assertTemplate(tmplstr, "run2/calexpT/v00048",
110 self.makeDatasetRef("calexpT", du, conform=False))
112 # Ensure that this returns a relative path even if the first field
113 # is optional
114 tmplstr = "{run}/{tract:?}/{visit:?}/f{physical_filter}"
115 self.assertTemplate(tmplstr, "run2/52/fU", ref)
117 # Ensure that // from optionals are converted to singles
118 tmplstr = "{run}/{datasetType}/{patch:?}/{tract:?}/f{physical_filter}"
119 self.assertTemplate(tmplstr, "run2/calexp/fU", ref)
121 # Optionals with some text between fields
122 tmplstr = "{run}/{datasetType}/p{patch:?}_t{tract:?}/f{physical_filter}"
123 self.assertTemplate(tmplstr, "run2/calexp/p/fU", ref)
124 tmplstr = "{run}/{datasetType}/p{patch:?}_t{visit:04d?}/f{physical_filter}"
125 self.assertTemplate(tmplstr, "run2/calexp/p_t0052/fU", ref)
127 def testComponent(self):
128 """Test handling of components in templates."""
129 refMetricOutput = self.makeDatasetRef("metric.output")
130 refMetric = self.makeDatasetRef("metric")
131 refMaskedImage = self.makeDatasetRef("calexp.maskedimage.variance")
132 refWcs = self.makeDatasetRef("calexp.wcs")
134 tmplstr = "{run}_c_{component}_v{visit}"
135 self.assertTemplate(tmplstr, "run2_c_output_v52", refMetricOutput)
137 # We want this template to have both a directory and basename, to
138 # test that the right parts of the output are replaced.
139 tmplstr = "{component:?}/{run}_{component:?}_{visit}"
140 self.assertTemplate(tmplstr, "run2_52", refMetric)
141 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput)
142 self.assertTemplate(tmplstr, "maskedimage.variance/run2_maskedimage_variance_52", refMaskedImage)
143 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput)
145 # Providing a component but not using it
146 tmplstr = "{run}/{datasetType}/v{visit:05d}"
147 with self.assertRaises(KeyError):
148 self.assertTemplate(tmplstr, "", refWcs)
150 def testFields(self):
151 # Template, mandatory fields, optional non-special fields,
152 # special fields, optional special fields
153 testData = (("{run}/{datasetType}/{visit:05d}/{physical_filter}-trail",
154 set(["visit", "physical_filter"]),
155 set(),
156 set(["run", "datasetType"]),
157 set()),
158 ("{run}/{component:?}_{visit}",
159 set(["visit"]),
160 set(),
161 set(["run"]),
162 set(["component"]),),
163 ("{run}/{component:?}_{visit:?}_{physical_filter}_{instrument}_{datasetType}",
164 set(["physical_filter", "instrument"]),
165 set(["visit"]),
166 set(["run", "datasetType"]),
167 set(["component"]),),
168 )
169 for tmplstr, mandatory, optional, special, optionalSpecial in testData:
170 with self.subTest(template=tmplstr):
171 tmpl = FileTemplate(tmplstr)
172 fields = tmpl.fields()
173 self.assertEqual(fields, mandatory)
174 fields = tmpl.fields(optionals=True)
175 self.assertEqual(fields, mandatory | optional)
176 fields = tmpl.fields(specials=True)
177 self.assertEqual(fields, mandatory | special)
178 fields = tmpl.fields(specials=True, optionals=True)
179 self.assertEqual(fields, mandatory | special | optional | optionalSpecial)
181 def testSimpleConfig(self):
182 """Test reading from config file"""
183 configRoot = os.path.join(TESTDIR, "config", "templates")
184 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
185 templates = FileTemplates(config1, universe=self.universe)
186 ref = self.makeDatasetRef("calexp")
187 tmpl = templates.getTemplate(ref)
188 self.assertIsInstance(tmpl, FileTemplate)
190 # This config file should not allow defaulting
191 ref2 = self.makeDatasetRef("unknown")
192 with self.assertRaises(KeyError):
193 templates.getTemplate(ref2)
195 # This should fall through the datasetTypeName check and use
196 # StorageClass instead
197 ref3 = self.makeDatasetRef("unknown2", storageClassName="StorageClassX")
198 tmplSc = templates.getTemplate(ref3)
199 self.assertIsInstance(tmplSc, FileTemplate)
201 # Try with a component: one with defined formatter and one without
202 refWcs = self.makeDatasetRef("calexp.wcs")
203 refImage = self.makeDatasetRef("calexp.image")
204 tmplCalexp = templates.getTemplate(ref)
205 tmplWcs = templates.getTemplate(refWcs) # Should be special
206 tmpl_image = templates.getTemplate(refImage)
207 self.assertIsInstance(tmplCalexp, FileTemplate)
208 self.assertIsInstance(tmpl_image, FileTemplate)
209 self.assertIsInstance(tmplWcs, FileTemplate)
210 self.assertEqual(tmplCalexp, tmpl_image)
211 self.assertNotEqual(tmplCalexp, tmplWcs)
213 # Check dimensions lookup order.
214 # The order should be: dataset type name, dimension, storage class
215 # This one will not match name but might match storage class.
216 # It should match dimensions
217 refDims = self.makeDatasetRef("nomatch", dataId={"instrument": "LSST", "physical_filter": "z"},
218 storageClassName="StorageClassX")
219 tmplDims = templates.getTemplate(refDims)
220 self.assertIsInstance(tmplDims, FileTemplate)
221 self.assertNotEqual(tmplDims, tmplSc)
223 # Test that instrument overrides retrieve specialist templates
224 refPvi = self.makeDatasetRef("pvi")
225 refPviHsc = self.makeDatasetRef("pvi", dataId={"instrument": "HSC", "physical_filter": "z"})
226 refPviLsst = self.makeDatasetRef("pvi", dataId={"instrument": "LSST", "physical_filter": "z"})
228 tmplPvi = templates.getTemplate(refPvi)
229 tmplPviHsc = templates.getTemplate(refPviHsc)
230 tmplPviLsst = templates.getTemplate(refPviLsst)
231 self.assertEqual(tmplPvi, tmplPviLsst)
232 self.assertNotEqual(tmplPvi, tmplPviHsc)
234 # Have instrument match and dimensions look up with no name match
235 refNoPviHsc = self.makeDatasetRef("pvix", dataId={"instrument": "HSC", "physical_filter": "z"},
236 storageClassName="StorageClassX")
237 tmplNoPviHsc = templates.getTemplate(refNoPviHsc)
238 self.assertNotEqual(tmplNoPviHsc, tmplDims)
239 self.assertNotEqual(tmplNoPviHsc, tmplPviHsc)
241 # Format config file with defaulting
242 config2 = FileTemplatesConfig(os.path.join(configRoot, "templates-withdefault.yaml"))
243 templates = FileTemplates(config2, universe=self.universe)
244 tmpl = templates.getTemplate(ref2)
245 self.assertIsInstance(tmpl, FileTemplate)
247 # Format config file with bad format string
248 with self.assertRaises(FileTemplateValidationError):
249 FileTemplates(os.path.join(configRoot, "templates-bad.yaml"), universe=self.universe)
251 # Config file with no defaulting mentioned
252 config3 = os.path.join(configRoot, "templates-nodefault2.yaml")
253 templates = FileTemplates(config3, universe=self.universe)
254 with self.assertRaises(KeyError):
255 templates.getTemplate(ref2)
257 # Try again but specify a default in the constructor
258 default = "{run}/{datasetType}/{physical_filter}"
259 templates = FileTemplates(config3, default=default, universe=self.universe)
260 tmpl = templates.getTemplate(ref2)
261 self.assertEqual(tmpl.template, default)
263 def testValidation(self):
264 configRoot = os.path.join(TESTDIR, "config", "templates")
265 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
266 templates = FileTemplates(config1, universe=self.universe)
268 entities = {}
269 entities["calexp"] = self.makeDatasetRef("calexp", storageClassName="StorageClassX",
270 dataId={"instrument": "dummy", "physical_filter": "i",
271 "visit": 52})
273 with self.assertLogs(level="WARNING") as cm:
274 templates.validateTemplates(entities.values(), logFailures=True)
275 self.assertIn("Unchecked keys", cm.output[0])
276 self.assertIn("StorageClassX", cm.output[0])
278 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX",
279 dataId={"instrument": "dummy", "physical_filter": "i"})
280 entities["StorageClassX"] = self.makeDatasetRef("storageClass",
281 storageClassName="StorageClassX",
282 dataId={"instrument": "dummy", "visit": 2})
283 entities["calexp.wcs"] = self.makeDatasetRef("calexp.wcs",
284 storageClassName="StorageClassX",
285 dataId={"instrument": "dummy",
286 "physical_filter": "i", "visit": 23},
287 conform=False)
289 entities["instrument+physical_filter"] = self.makeDatasetRef("filter_inst",
290 storageClassName="StorageClassX",
291 dataId={"physical_filter": "i",
292 "instrument": "SCUBA"})
293 entities["hsc+pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX",
294 dataId={"physical_filter": "i", "instrument": "HSC"})
296 entities["hsc+instrument+physical_filter"] = self.makeDatasetRef("filter_inst",
297 storageClassName="StorageClassX",
298 dataId={"physical_filter": "i",
299 "instrument": "HSC"})
301 templates.validateTemplates(entities.values(), logFailures=True)
303 # Rerun but with a failure
304 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX",
305 dataId={"abstract_filter": "i"})
306 with self.assertRaises(FileTemplateValidationError):
307 with self.assertLogs(level="FATAL"):
308 templates.validateTemplates(entities.values(), logFailures=True)
311if __name__ == "__main__": 311 ↛ 312line 311 didn't jump to line 312, because the condition on line 311 was never true
312 unittest.main()