Coverage for tests/test_templates.py: 11%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Test file name templating."""
24import os.path
25import unittest
27from lsst.daf.butler import DatasetType, DatasetRef, FileTemplates, DimensionUniverse, \
28 FileTemplate, FileTemplatesConfig, StorageClass, FileTemplateValidationError, \
29 DimensionGraph
31TESTDIR = os.path.abspath(os.path.dirname(__file__))
33PlaceHolder = StorageClass("PlaceHolder")
36class TestFileTemplates(unittest.TestCase):
37 """Test creation of paths from templates."""
39 def makeDatasetRef(self, datasetTypeName, dataId=None, storageClassName="DefaultStorageClass",
40 run="run2", conform=True):
41 """Make a simple DatasetRef"""
42 if dataId is None:
43 dataId = self.dataId
45 # Pretend we have a parent if this looks like a composite
46 compositeName, componentName = DatasetType.splitDatasetTypeName(datasetTypeName)
47 parentStorageClass = PlaceHolder if componentName else None
49 datasetType = DatasetType(datasetTypeName, DimensionGraph(self.universe, names=dataId.keys()),
50 StorageClass(storageClassName),
51 parentStorageClass=parentStorageClass)
52 return DatasetRef(datasetType, dataId, id=1, run=run, conform=conform)
54 def setUp(self):
55 self.universe = DimensionUniverse()
56 self.dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "Most Amazing U Filter Ever"}
58 def assertTemplate(self, template, answer, ref):
59 fileTmpl = FileTemplate(template)
60 path = fileTmpl.format(ref)
61 self.assertEqual(path, answer)
63 def testBasic(self):
64 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}"
65 self.assertTemplate(tmplstr,
66 "run2/calexp/00052/Most_Amazing_U_Filter_Ever",
67 self.makeDatasetRef("calexp", conform=False))
68 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail"
69 self.assertTemplate(tmplstr,
70 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail",
71 self.makeDatasetRef("calexp", conform=False))
73 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
74 self.assertTemplate(tmplstr,
75 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run2",
76 self.makeDatasetRef("calexp", conform=False))
77 self.assertTemplate(tmplstr,
78 "run_2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2",
79 self.makeDatasetRef("calexp", run="run/2", conform=False))
81 # Retain any "/" in run
82 tmplstr = "{run:/}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
83 self.assertTemplate(tmplstr,
84 "run/2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2",
85 self.makeDatasetRef("calexp", run="run/2", conform=False))
87 # Check that "." are replaced in the file basename, but not directory.
88 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "g.10"}
89 self.assertTemplate(tmplstr,
90 "run.2/calexp/00052/g_10-trail-run_2",
91 self.makeDatasetRef("calexp", run="run.2", dataId=dataId, conform=False))
93 with self.assertRaises(FileTemplateValidationError):
94 FileTemplate("no fields at all")
96 with self.assertRaises(FileTemplateValidationError):
97 FileTemplate("{visit}")
99 with self.assertRaises(FileTemplateValidationError):
100 FileTemplate("{run}_{datasetType}")
102 def testRunOrCollectionNeeded(self):
103 tmplstr = "{datasetType}/{visit:05d}/{physical_filter}"
104 with self.assertRaises(FileTemplateValidationError):
105 self.assertTemplate(tmplstr,
106 "run2/calexp/00052/U",
107 self.makeDatasetRef("calexp"))
109 def testOptional(self):
110 """Optional units in templates."""
111 ref = self.makeDatasetRef("calexp", conform=False)
112 tmplstr = "{run}/{datasetType}/v{visit:05d}_f{physical_filter:?}"
113 self.assertTemplate(tmplstr, "run2/calexp/v00052_fMost_Amazing_U_Filter_Ever",
114 self.makeDatasetRef("calexp", conform=False))
116 du = {"visit": 48, "tract": 265, "skymap": "big", "instrument": "dummy"}
117 self.assertTemplate(tmplstr, "run2/calexpT/v00048",
118 self.makeDatasetRef("calexpT", du, conform=False))
120 # Ensure that this returns a relative path even if the first field
121 # is optional
122 tmplstr = "{run}/{tract:?}/{visit:?}/f{physical_filter}"
123 self.assertTemplate(tmplstr, "run2/52/fMost_Amazing_U_Filter_Ever", ref)
125 # Ensure that // from optionals are converted to singles
126 tmplstr = "{run}/{datasetType}/{patch:?}/{tract:?}/f{physical_filter}"
127 self.assertTemplate(tmplstr, "run2/calexp/fMost_Amazing_U_Filter_Ever", ref)
129 # Optionals with some text between fields
130 tmplstr = "{run}/{datasetType}/p{patch:?}_t{tract:?}/f{physical_filter}"
131 self.assertTemplate(tmplstr, "run2/calexp/p/fMost_Amazing_U_Filter_Ever", ref)
132 tmplstr = "{run}/{datasetType}/p{patch:?}_t{visit:04d?}/f{physical_filter}"
133 self.assertTemplate(tmplstr, "run2/calexp/p_t0052/fMost_Amazing_U_Filter_Ever", ref)
135 def testComponent(self):
136 """Test handling of components in templates."""
137 refMetricOutput = self.makeDatasetRef("metric.output")
138 refMetric = self.makeDatasetRef("metric")
139 refMaskedImage = self.makeDatasetRef("calexp.maskedimage.variance")
140 refWcs = self.makeDatasetRef("calexp.wcs")
142 tmplstr = "{run}_c_{component}_v{visit}"
143 self.assertTemplate(tmplstr, "run2_c_output_v52", refMetricOutput)
145 # We want this template to have both a directory and basename, to
146 # test that the right parts of the output are replaced.
147 tmplstr = "{component:?}/{run}_{component:?}_{visit}"
148 self.assertTemplate(tmplstr, "run2_52", refMetric)
149 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput)
150 self.assertTemplate(tmplstr, "maskedimage.variance/run2_maskedimage_variance_52", refMaskedImage)
151 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput)
153 # Providing a component but not using it
154 tmplstr = "{run}/{datasetType}/v{visit:05d}"
155 with self.assertRaises(KeyError):
156 self.assertTemplate(tmplstr, "", refWcs)
158 def testFields(self):
159 # Template, mandatory fields, optional non-special fields,
160 # special fields, optional special fields
161 testData = (("{run}/{datasetType}/{visit:05d}/{physical_filter}-trail",
162 set(["visit", "physical_filter"]),
163 set(),
164 set(["run", "datasetType"]),
165 set()),
166 ("{run}/{component:?}_{visit}",
167 set(["visit"]),
168 set(),
169 set(["run"]),
170 set(["component"]),),
171 ("{run}/{component:?}_{visit:?}_{physical_filter}_{instrument}_{datasetType}",
172 set(["physical_filter", "instrument"]),
173 set(["visit"]),
174 set(["run", "datasetType"]),
175 set(["component"]),),
176 )
177 for tmplstr, mandatory, optional, special, optionalSpecial in testData:
178 with self.subTest(template=tmplstr):
179 tmpl = FileTemplate(tmplstr)
180 fields = tmpl.fields()
181 self.assertEqual(fields, mandatory)
182 fields = tmpl.fields(optionals=True)
183 self.assertEqual(fields, mandatory | optional)
184 fields = tmpl.fields(specials=True)
185 self.assertEqual(fields, mandatory | special)
186 fields = tmpl.fields(specials=True, optionals=True)
187 self.assertEqual(fields, mandatory | special | optional | optionalSpecial)
189 def testSimpleConfig(self):
190 """Test reading from config file"""
191 configRoot = os.path.join(TESTDIR, "config", "templates")
192 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
193 templates = FileTemplates(config1, universe=self.universe)
194 ref = self.makeDatasetRef("calexp")
195 tmpl = templates.getTemplate(ref)
196 self.assertIsInstance(tmpl, FileTemplate)
198 # This config file should not allow defaulting
199 ref2 = self.makeDatasetRef("unknown")
200 with self.assertRaises(KeyError):
201 templates.getTemplate(ref2)
203 # This should fall through the datasetTypeName check and use
204 # StorageClass instead
205 ref3 = self.makeDatasetRef("unknown2", storageClassName="StorageClassX")
206 tmplSc = templates.getTemplate(ref3)
207 self.assertIsInstance(tmplSc, FileTemplate)
209 # Try with a component: one with defined formatter and one without
210 refWcs = self.makeDatasetRef("calexp.wcs")
211 refImage = self.makeDatasetRef("calexp.image")
212 tmplCalexp = templates.getTemplate(ref)
213 tmplWcs = templates.getTemplate(refWcs) # Should be special
214 tmpl_image = templates.getTemplate(refImage)
215 self.assertIsInstance(tmplCalexp, FileTemplate)
216 self.assertIsInstance(tmpl_image, FileTemplate)
217 self.assertIsInstance(tmplWcs, FileTemplate)
218 self.assertEqual(tmplCalexp, tmpl_image)
219 self.assertNotEqual(tmplCalexp, tmplWcs)
221 # Check dimensions lookup order.
222 # The order should be: dataset type name, dimension, storage class
223 # This one will not match name but might match storage class.
224 # It should match dimensions
225 refDims = self.makeDatasetRef("nomatch", dataId={"instrument": "LSST", "physical_filter": "z"},
226 storageClassName="StorageClassX")
227 tmplDims = templates.getTemplate(refDims)
228 self.assertIsInstance(tmplDims, FileTemplate)
229 self.assertNotEqual(tmplDims, tmplSc)
231 # Test that instrument overrides retrieve specialist templates
232 refPvi = self.makeDatasetRef("pvi")
233 refPviHsc = self.makeDatasetRef("pvi", dataId={"instrument": "HSC", "physical_filter": "z"})
234 refPviLsst = self.makeDatasetRef("pvi", dataId={"instrument": "LSST", "physical_filter": "z"})
236 tmplPvi = templates.getTemplate(refPvi)
237 tmplPviHsc = templates.getTemplate(refPviHsc)
238 tmplPviLsst = templates.getTemplate(refPviLsst)
239 self.assertEqual(tmplPvi, tmplPviLsst)
240 self.assertNotEqual(tmplPvi, tmplPviHsc)
242 # Have instrument match and dimensions look up with no name match
243 refNoPviHsc = self.makeDatasetRef("pvix", dataId={"instrument": "HSC", "physical_filter": "z"},
244 storageClassName="StorageClassX")
245 tmplNoPviHsc = templates.getTemplate(refNoPviHsc)
246 self.assertNotEqual(tmplNoPviHsc, tmplDims)
247 self.assertNotEqual(tmplNoPviHsc, tmplPviHsc)
249 # Format config file with defaulting
250 config2 = FileTemplatesConfig(os.path.join(configRoot, "templates-withdefault.yaml"))
251 templates = FileTemplates(config2, universe=self.universe)
252 tmpl = templates.getTemplate(ref2)
253 self.assertIsInstance(tmpl, FileTemplate)
255 # Format config file with bad format string
256 with self.assertRaises(FileTemplateValidationError):
257 FileTemplates(os.path.join(configRoot, "templates-bad.yaml"), universe=self.universe)
259 # Config file with no defaulting mentioned
260 config3 = os.path.join(configRoot, "templates-nodefault2.yaml")
261 templates = FileTemplates(config3, universe=self.universe)
262 with self.assertRaises(KeyError):
263 templates.getTemplate(ref2)
265 # Try again but specify a default in the constructor
266 default = "{run}/{datasetType}/{physical_filter}"
267 templates = FileTemplates(config3, default=default, universe=self.universe)
268 tmpl = templates.getTemplate(ref2)
269 self.assertEqual(tmpl.template, default)
271 def testValidation(self):
272 configRoot = os.path.join(TESTDIR, "config", "templates")
273 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
274 templates = FileTemplates(config1, universe=self.universe)
276 entities = {}
277 entities["calexp"] = self.makeDatasetRef("calexp", storageClassName="StorageClassX",
278 dataId={"instrument": "dummy", "physical_filter": "i",
279 "visit": 52})
281 with self.assertLogs(level="WARNING") as cm:
282 templates.validateTemplates(entities.values(), logFailures=True)
283 self.assertIn("Unchecked keys", cm.output[0])
284 self.assertIn("StorageClassX", cm.output[0])
286 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX",
287 dataId={"instrument": "dummy", "physical_filter": "i"})
288 entities["StorageClassX"] = self.makeDatasetRef("storageClass",
289 storageClassName="StorageClassX",
290 dataId={"instrument": "dummy", "visit": 2})
291 entities["calexp.wcs"] = self.makeDatasetRef("calexp.wcs",
292 storageClassName="StorageClassX",
293 dataId={"instrument": "dummy",
294 "physical_filter": "i", "visit": 23},
295 conform=False)
297 entities["instrument+physical_filter"] = self.makeDatasetRef("filter_inst",
298 storageClassName="StorageClassX",
299 dataId={"physical_filter": "i",
300 "instrument": "SCUBA"})
301 entities["hsc+pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX",
302 dataId={"physical_filter": "i", "instrument": "HSC"})
304 entities["hsc+instrument+physical_filter"] = self.makeDatasetRef("filter_inst",
305 storageClassName="StorageClassX",
306 dataId={"physical_filter": "i",
307 "instrument": "HSC"})
309 templates.validateTemplates(entities.values(), logFailures=True)
311 # Rerun but with a failure
312 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX",
313 dataId={"band": "i"})
314 with self.assertRaises(FileTemplateValidationError):
315 with self.assertLogs(level="FATAL"):
316 templates.validateTemplates(entities.values(), logFailures=True)
319if __name__ == "__main__": 319 ↛ 320line 319 didn't jump to line 320, because the condition on line 319 was never true
320 unittest.main()