Coverage for tests/test_templates.py : 11%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Test file name templating."""
24import os.path
25import unittest
27from lsst.daf.butler import DatasetType, DatasetRef, FileTemplates, DimensionUniverse, \
28 FileTemplate, FileTemplatesConfig, StorageClass, FileTemplateValidationError, \
29 DimensionGraph
31TESTDIR = os.path.abspath(os.path.dirname(__file__))
34class TestFileTemplates(unittest.TestCase):
35 """Test creation of paths from templates."""
37 def makeDatasetRef(self, datasetTypeName, dataId=None, storageClassName="DefaultStorageClass",
38 run="run2", conform=True):
39 """Make a simple DatasetRef"""
40 if dataId is None:
41 dataId = self.dataId
42 datasetType = DatasetType(datasetTypeName, DimensionGraph(self.universe, names=dataId.keys()),
43 StorageClass(storageClassName))
44 return DatasetRef(datasetType, dataId, id=1, run=run, conform=conform)
46 def setUp(self):
47 self.universe = DimensionUniverse()
48 self.dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "U"}
50 def assertTemplate(self, template, answer, ref):
51 fileTmpl = FileTemplate(template)
52 path = fileTmpl.format(ref)
53 self.assertEqual(path, answer)
55 def testBasic(self):
56 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}"
57 self.assertTemplate(tmplstr,
58 "run2/calexp/00052/U",
59 self.makeDatasetRef("calexp", conform=False))
60 tmplstr = "{collection}/{datasetType}/{visit:05d}/{physical_filter}-trail"
61 self.assertTemplate(tmplstr,
62 "run2/calexp/00052/U-trail",
63 self.makeDatasetRef("calexp", conform=False))
65 tmplstr = "{collection}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
66 self.assertTemplate(tmplstr,
67 "run2/calexp/00052/U-trail-run2",
68 self.makeDatasetRef("calexp", conform=False))
69 self.assertTemplate(tmplstr,
70 "run_2/calexp/00052/U-trail-run_2",
71 self.makeDatasetRef("calexp", run="run/2", conform=False))
73 # Retain any "/" in collection
74 tmplstr = "{collection:/}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
75 self.assertTemplate(tmplstr,
76 "run/2/calexp/00052/U-trail-run_2",
77 self.makeDatasetRef("calexp", run="run/2", conform=False))
79 with self.assertRaises(FileTemplateValidationError):
80 FileTemplate("no fields at all")
82 with self.assertRaises(FileTemplateValidationError):
83 FileTemplate("{visit}")
85 with self.assertRaises(FileTemplateValidationError):
86 FileTemplate("{run}_{datasetType}")
88 def testRunOrCollectionNeeded(self):
89 tmplstr = "{datasetType}/{visit:05d}/{physical_filter}"
90 with self.assertRaises(FileTemplateValidationError):
91 self.assertTemplate(tmplstr,
92 "run2/calexp/00052/U",
93 self.makeDatasetRef("calexp"))
95 def testOptional(self):
96 """Optional units in templates."""
97 ref = self.makeDatasetRef("calexp", conform=False)
98 tmplstr = "{run}/{datasetType}/v{visit:05d}_f{physical_filter:?}"
99 self.assertTemplate(tmplstr, "run2/calexp/v00052_fU",
100 self.makeDatasetRef("calexp", conform=False))
102 du = {"visit": 48, "tract": 265, "skymap": "big", "instrument": "dummy"}
103 self.assertTemplate(tmplstr, "run2/calexpT/v00048",
104 self.makeDatasetRef("calexpT", du, conform=False))
106 # Ensure that this returns a relative path even if the first field
107 # is optional
108 tmplstr = "{run}/{tract:?}/{visit:?}/f{physical_filter}"
109 self.assertTemplate(tmplstr, "run2/52/fU", ref)
111 # Ensure that // from optionals are converted to singles
112 tmplstr = "{run}/{datasetType}/{patch:?}/{tract:?}/f{physical_filter}"
113 self.assertTemplate(tmplstr, "run2/calexp/fU", ref)
115 # Optionals with some text between fields
116 tmplstr = "{run}/{datasetType}/p{patch:?}_t{tract:?}/f{physical_filter}"
117 self.assertTemplate(tmplstr, "run2/calexp/p/fU", ref)
118 tmplstr = "{run}/{datasetType}/p{patch:?}_t{visit:04d?}/f{physical_filter}"
119 self.assertTemplate(tmplstr, "run2/calexp/p_t0052/fU", ref)
121 def testComponent(self):
122 """Test handling of components in templates."""
123 refMetricOutput = self.makeDatasetRef("metric.output")
124 refMetric = self.makeDatasetRef("metric")
125 refMaskedImage = self.makeDatasetRef("calexp.maskedimage.variance")
126 refWcs = self.makeDatasetRef("calexp.wcs")
128 tmplstr = "{run}_c_{component}_v{visit}"
129 self.assertTemplate(tmplstr, "run2_c_output_v52", refMetricOutput)
131 tmplstr = "{run}_{component:?}_{visit}"
132 self.assertTemplate(tmplstr, "run2_52", refMetric)
133 self.assertTemplate(tmplstr, "run2_output_52", refMetricOutput)
134 self.assertTemplate(tmplstr, "run2_maskedimage.variance_52", refMaskedImage)
135 self.assertTemplate(tmplstr, "run2_output_52", refMetricOutput)
137 # Providing a component but not using it
138 tmplstr = "{collection}/{datasetType}/v{visit:05d}"
139 with self.assertRaises(KeyError):
140 self.assertTemplate(tmplstr, "", refWcs)
142 def testFields(self):
143 # Template, mandatory fields, optional non-special fields,
144 # special fields, optional special fields
145 testData = (("{collection}/{datasetType}/{visit:05d}/{physical_filter}-trail",
146 set(["visit", "physical_filter"]),
147 set(),
148 set(["collection", "datasetType"]),
149 set()),
150 ("{collection}/{component:?}_{visit}",
151 set(["visit"]),
152 set(),
153 set(["collection"]),
154 set(["component"]),),
155 ("{run}/{component:?}_{visit:?}_{physical_filter}_{instrument}_{datasetType}",
156 set(["physical_filter", "instrument"]),
157 set(["visit"]),
158 set(["run", "datasetType"]),
159 set(["component"]),),
160 )
161 for tmplstr, mandatory, optional, special, optionalSpecial in testData:
162 with self.subTest(template=tmplstr):
163 tmpl = FileTemplate(tmplstr)
164 fields = tmpl.fields()
165 self.assertEqual(fields, mandatory)
166 fields = tmpl.fields(optionals=True)
167 self.assertEqual(fields, mandatory | optional)
168 fields = tmpl.fields(specials=True)
169 self.assertEqual(fields, mandatory | special)
170 fields = tmpl.fields(specials=True, optionals=True)
171 self.assertEqual(fields, mandatory | special | optional | optionalSpecial)
173 def testSimpleConfig(self):
174 """Test reading from config file"""
175 configRoot = os.path.join(TESTDIR, "config", "templates")
176 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
177 templates = FileTemplates(config1, universe=self.universe)
178 ref = self.makeDatasetRef("calexp")
179 tmpl = templates.getTemplate(ref)
180 self.assertIsInstance(tmpl, FileTemplate)
182 # This config file should not allow defaulting
183 ref2 = self.makeDatasetRef("unknown")
184 with self.assertRaises(KeyError):
185 templates.getTemplate(ref2)
187 # This should fall through the datasetTypeName check and use
188 # StorageClass instead
189 ref3 = self.makeDatasetRef("unknown2", storageClassName="StorageClassX")
190 tmplSc = templates.getTemplate(ref3)
191 self.assertIsInstance(tmplSc, FileTemplate)
193 # Try with a component: one with defined formatter and one without
194 refWcs = self.makeDatasetRef("calexp.wcs")
195 refImage = self.makeDatasetRef("calexp.image")
196 tmplCalexp = templates.getTemplate(ref)
197 tmplWcs = templates.getTemplate(refWcs) # Should be special
198 tmpl_image = templates.getTemplate(refImage)
199 self.assertIsInstance(tmplCalexp, FileTemplate)
200 self.assertIsInstance(tmpl_image, FileTemplate)
201 self.assertIsInstance(tmplWcs, FileTemplate)
202 self.assertEqual(tmplCalexp, tmpl_image)
203 self.assertNotEqual(tmplCalexp, tmplWcs)
205 # Check dimensions lookup order.
206 # The order should be: dataset type name, dimension, storage class
207 # This one will not match name but might match storage class.
208 # It should match dimensions
209 refDims = self.makeDatasetRef("nomatch", dataId={"instrument": "LSST", "physical_filter": "z"},
210 storageClassName="StorageClassX")
211 tmplDims = templates.getTemplate(refDims)
212 self.assertIsInstance(tmplDims, FileTemplate)
213 self.assertNotEqual(tmplDims, tmplSc)
215 # Test that instrument overrides retrieve specialist templates
216 refPvi = self.makeDatasetRef("pvi")
217 refPviHsc = self.makeDatasetRef("pvi", dataId={"instrument": "HSC", "physical_filter": "z"})
218 refPviLsst = self.makeDatasetRef("pvi", dataId={"instrument": "LSST", "physical_filter": "z"})
220 tmplPvi = templates.getTemplate(refPvi)
221 tmplPviHsc = templates.getTemplate(refPviHsc)
222 tmplPviLsst = templates.getTemplate(refPviLsst)
223 self.assertEqual(tmplPvi, tmplPviLsst)
224 self.assertNotEqual(tmplPvi, tmplPviHsc)
226 # Have instrument match and dimensions look up with no name match
227 refNoPviHsc = self.makeDatasetRef("pvix", dataId={"instrument": "HSC", "physical_filter": "z"},
228 storageClassName="StorageClassX")
229 tmplNoPviHsc = templates.getTemplate(refNoPviHsc)
230 self.assertNotEqual(tmplNoPviHsc, tmplDims)
231 self.assertNotEqual(tmplNoPviHsc, tmplPviHsc)
233 # Format config file with defaulting
234 config2 = FileTemplatesConfig(os.path.join(configRoot, "templates-withdefault.yaml"))
235 templates = FileTemplates(config2, universe=self.universe)
236 tmpl = templates.getTemplate(ref2)
237 self.assertIsInstance(tmpl, FileTemplate)
239 # Format config file with bad format string
240 with self.assertRaises(FileTemplateValidationError):
241 FileTemplates(os.path.join(configRoot, "templates-bad.yaml"), universe=self.universe)
243 # Config file with no defaulting mentioned
244 config3 = os.path.join(configRoot, "templates-nodefault2.yaml")
245 templates = FileTemplates(config3, universe=self.universe)
246 with self.assertRaises(KeyError):
247 templates.getTemplate(ref2)
249 # Try again but specify a default in the constructor
250 default = "{run}/{datasetType}/{physical_filter}"
251 templates = FileTemplates(config3, default=default, universe=self.universe)
252 tmpl = templates.getTemplate(ref2)
253 self.assertEqual(tmpl.template, default)
255 def testValidation(self):
256 configRoot = os.path.join(TESTDIR, "config", "templates")
257 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
258 templates = FileTemplates(config1, universe=self.universe)
260 entities = {}
261 entities["calexp"] = self.makeDatasetRef("calexp", storageClassName="StorageClassX",
262 dataId={"instrument": "dummy", "physical_filter": "i",
263 "visit": 52})
265 with self.assertLogs(level="WARNING") as cm:
266 templates.validateTemplates(entities.values(), logFailures=True)
267 self.assertIn("Unchecked keys", cm.output[0])
268 self.assertIn("StorageClassX", cm.output[0])
270 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX",
271 dataId={"instrument": "dummy", "physical_filter": "i"})
272 entities["StorageClassX"] = self.makeDatasetRef("storageClass",
273 storageClassName="StorageClassX",
274 dataId={"instrument": "dummy", "visit": 2})
275 entities["calexp.wcs"] = self.makeDatasetRef("calexp.wcs",
276 storageClassName="StorageClassX",
277 dataId={"instrument": "dummy",
278 "physical_filter": "i", "visit": 23},
279 conform=False)
281 entities["instrument+physical_filter"] = self.makeDatasetRef("filter_inst",
282 storageClassName="StorageClassX",
283 dataId={"physical_filter": "i",
284 "instrument": "SCUBA"})
285 entities["hsc+pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX",
286 dataId={"physical_filter": "i", "instrument": "HSC"})
288 entities["hsc+instrument+physical_filter"] = self.makeDatasetRef("filter_inst",
289 storageClassName="StorageClassX",
290 dataId={"physical_filter": "i",
291 "instrument": "HSC"})
293 templates.validateTemplates(entities.values(), logFailures=True)
295 # Rerun but with a failure
296 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX",
297 dataId={"abstract_filter": "i"})
298 with self.assertRaises(FileTemplateValidationError):
299 with self.assertLogs(level="FATAL"):
300 templates.validateTemplates(entities.values(), logFailures=True)
303if __name__ == "__main__": 303 ↛ 304line 303 didn't jump to line 304, because the condition on line 303 was never true
304 unittest.main()