Coverage for tests/test_templates.py: 9%
177 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-14 19:21 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-14 19:21 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Test file name templating."""
24import os.path
25import unittest
26import uuid
28from lsst.daf.butler import (
29 DataCoordinate,
30 DatasetId,
31 DatasetRef,
32 DatasetType,
33 DimensionGraph,
34 DimensionUniverse,
35 FileTemplate,
36 FileTemplates,
37 FileTemplatesConfig,
38 FileTemplateValidationError,
39 StorageClass,
40)
42TESTDIR = os.path.abspath(os.path.dirname(__file__))
44PlaceHolder = StorageClass("PlaceHolder")
46REFUUID = DatasetId(int=uuid.uuid4().int)
49class TestFileTemplates(unittest.TestCase):
50 """Test creation of paths from templates."""
52 def makeDatasetRef(
53 self, datasetTypeName, dataId=None, storageClassName="DefaultStorageClass", run="run2", conform=True
54 ):
55 """Make a simple DatasetRef"""
56 if dataId is None:
57 dataId = self.dataId
58 if "physical_filter" in dataId and "band" not in dataId:
59 dataId["band"] = "b" # Add fake band.
60 dimensions = DimensionGraph(self.universe, names=dataId.keys())
61 dataId = DataCoordinate.standardize(dataId, graph=dimensions)
63 # Pretend we have a parent if this looks like a composite
64 compositeName, componentName = DatasetType.splitDatasetTypeName(datasetTypeName)
65 parentStorageClass = PlaceHolder if componentName else None
67 datasetType = DatasetType(
68 datasetTypeName,
69 dimensions,
70 StorageClass(storageClassName),
71 parentStorageClass=parentStorageClass,
72 )
73 return DatasetRef(datasetType, dataId, id=REFUUID, run=run, conform=conform)
75 def setUp(self):
76 self.universe = DimensionUniverse()
77 self.dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "Most Amazing U Filter Ever"}
79 def assertTemplate(self, template, answer, ref):
80 fileTmpl = FileTemplate(template)
81 path = fileTmpl.format(ref)
82 self.assertEqual(path, answer)
84 def testBasic(self):
85 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}"
86 self.assertTemplate(
87 tmplstr,
88 "run2/calexp/00052/Most_Amazing_U_Filter_Ever",
89 self.makeDatasetRef("calexp"),
90 )
91 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail"
92 self.assertTemplate(
93 tmplstr,
94 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail",
95 self.makeDatasetRef("calexp"),
96 )
98 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
99 self.assertTemplate(
100 tmplstr,
101 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run2",
102 self.makeDatasetRef("calexp"),
103 )
104 self.assertTemplate(
105 tmplstr,
106 "run_2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2",
107 self.makeDatasetRef("calexp", run="run/2"),
108 )
110 # Check that the id is sufficient without any other information.
111 self.assertTemplate("{id}", str(REFUUID), self.makeDatasetRef("calexp", run="run2"))
113 self.assertTemplate("{run}/{id}", f"run2/{str(REFUUID)}", self.makeDatasetRef("calexp", run="run2"))
115 self.assertTemplate(
116 "fixed/{id}",
117 f"fixed/{str(REFUUID)}",
118 self.makeDatasetRef("calexp", run="run2"),
119 )
121 self.assertTemplate(
122 "fixed/{id}_{physical_filter}",
123 f"fixed/{str(REFUUID)}_Most_Amazing_U_Filter_Ever",
124 self.makeDatasetRef("calexp", run="run2"),
125 )
127 # Retain any "/" in run
128 tmplstr = "{run:/}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
129 self.assertTemplate(
130 tmplstr,
131 "run/2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2",
132 self.makeDatasetRef("calexp", run="run/2"),
133 )
135 # Check that "." are replaced in the file basename, but not directory.
136 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "g.10"}
137 self.assertTemplate(
138 tmplstr,
139 "run.2/calexp/00052/g_10-trail-run_2",
140 self.makeDatasetRef("calexp", run="run.2", dataId=dataId),
141 )
143 with self.assertRaises(FileTemplateValidationError):
144 FileTemplate("no fields at all")
146 with self.assertRaises(FileTemplateValidationError):
147 FileTemplate("{visit}")
149 with self.assertRaises(FileTemplateValidationError):
150 FileTemplate("{run}_{datasetType}")
152 with self.assertRaises(FileTemplateValidationError):
153 FileTemplate("{id}/fixed")
155 def testRunOrCollectionNeeded(self):
156 tmplstr = "{datasetType}/{visit:05d}/{physical_filter}"
157 with self.assertRaises(FileTemplateValidationError):
158 self.assertTemplate(tmplstr, "run2/calexp/00052/U", self.makeDatasetRef("calexp"))
160 def testNoRecord(self):
161 # Attaching records is not possible in this test code but we can check
162 # that a missing record when a metadata entry has been requested
163 # does fail.
164 tmplstr = "{run}/{datasetType}/{visit.name}/{physical_filter}"
165 with self.assertRaises(RuntimeError) as cm:
166 self.assertTemplate(tmplstr, "", self.makeDatasetRef("calexp"))
167 self.assertIn("No metadata", str(cm.exception))
169 def testOptional(self):
170 """Optional units in templates."""
171 ref = self.makeDatasetRef("calexp")
172 tmplstr = "{run}/{datasetType}/v{visit:05d}_f{physical_filter:?}"
173 self.assertTemplate(
174 tmplstr,
175 "run2/calexp/v00052_fMost_Amazing_U_Filter_Ever",
176 self.makeDatasetRef("calexp"),
177 )
179 du = {"visit": 48, "tract": 265, "skymap": "big", "instrument": "dummy"}
180 self.assertTemplate(tmplstr, "run2/calexpT/v00048", self.makeDatasetRef("calexpT", du))
182 # Ensure that this returns a relative path even if the first field
183 # is optional
184 tmplstr = "{run}/{tract:?}/{visit:?}/f{physical_filter}"
185 self.assertTemplate(tmplstr, "run2/52/fMost_Amazing_U_Filter_Ever", ref)
187 # Ensure that // from optionals are converted to singles
188 tmplstr = "{run}/{datasetType}/{patch:?}/{tract:?}/f{physical_filter}"
189 self.assertTemplate(tmplstr, "run2/calexp/fMost_Amazing_U_Filter_Ever", ref)
191 # Optionals with some text between fields
192 tmplstr = "{run}/{datasetType}/p{patch:?}_t{tract:?}/f{physical_filter}"
193 self.assertTemplate(tmplstr, "run2/calexp/p/fMost_Amazing_U_Filter_Ever", ref)
194 tmplstr = "{run}/{datasetType}/p{patch:?}_t{visit:04d?}/f{physical_filter}"
195 self.assertTemplate(tmplstr, "run2/calexp/p_t0052/fMost_Amazing_U_Filter_Ever", ref)
197 def testComponent(self):
198 """Test handling of components in templates."""
199 refMetricOutput = self.makeDatasetRef("metric.output")
200 refMetric = self.makeDatasetRef("metric")
201 refMaskedImage = self.makeDatasetRef("calexp.maskedimage.variance")
202 refWcs = self.makeDatasetRef("calexp.wcs")
204 tmplstr = "{run}_c_{component}_v{visit}"
205 self.assertTemplate(tmplstr, "run2_c_output_v52", refMetricOutput)
207 # We want this template to have both a directory and basename, to
208 # test that the right parts of the output are replaced.
209 tmplstr = "{component:?}/{run}_{component:?}_{visit}"
210 self.assertTemplate(tmplstr, "run2_52", refMetric)
211 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput)
212 self.assertTemplate(tmplstr, "maskedimage.variance/run2_maskedimage_variance_52", refMaskedImage)
213 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput)
215 # Providing a component but not using it
216 tmplstr = "{run}/{datasetType}/v{visit:05d}"
217 with self.assertRaises(KeyError):
218 self.assertTemplate(tmplstr, "", refWcs)
220 def testFields(self):
221 # Template, mandatory fields, optional non-special fields,
222 # special fields, optional special fields
223 testData = (
224 (
225 "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail",
226 {"visit", "physical_filter"},
227 set(),
228 {"run", "datasetType"},
229 set(),
230 ),
231 (
232 "{run}/{component:?}_{visit}",
233 {"visit"},
234 set(),
235 {"run"},
236 {"component"},
237 ),
238 (
239 "{run}/{component:?}_{visit:?}_{physical_filter}_{instrument}_{datasetType}",
240 {"physical_filter", "instrument"},
241 {"visit"},
242 {"run", "datasetType"},
243 {"component"},
244 ),
245 )
246 for tmplstr, mandatory, optional, special, optionalSpecial in testData:
247 with self.subTest(template=tmplstr):
248 tmpl = FileTemplate(tmplstr)
249 fields = tmpl.fields()
250 self.assertEqual(fields, mandatory)
251 fields = tmpl.fields(optionals=True)
252 self.assertEqual(fields, mandatory | optional)
253 fields = tmpl.fields(specials=True)
254 self.assertEqual(fields, mandatory | special)
255 fields = tmpl.fields(specials=True, optionals=True)
256 self.assertEqual(fields, mandatory | special | optional | optionalSpecial)
258 def testSimpleConfig(self):
259 """Test reading from config file"""
260 configRoot = os.path.join(TESTDIR, "config", "templates")
261 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
262 templates = FileTemplates(config1, universe=self.universe)
263 ref = self.makeDatasetRef("calexp")
264 tmpl = templates.getTemplate(ref)
265 self.assertIsInstance(tmpl, FileTemplate)
267 # This config file should not allow defaulting
268 ref2 = self.makeDatasetRef("unknown")
269 with self.assertRaises(KeyError):
270 templates.getTemplate(ref2)
272 # This should fall through the datasetTypeName check and use
273 # StorageClass instead
274 ref3 = self.makeDatasetRef("unknown2", storageClassName="StorageClassX")
275 tmplSc = templates.getTemplate(ref3)
276 self.assertIsInstance(tmplSc, FileTemplate)
278 # Try with a component: one with defined formatter and one without
279 refWcs = self.makeDatasetRef("calexp.wcs")
280 refImage = self.makeDatasetRef("calexp.image")
281 tmplCalexp = templates.getTemplate(ref)
282 tmplWcs = templates.getTemplate(refWcs) # Should be special
283 tmpl_image = templates.getTemplate(refImage)
284 self.assertIsInstance(tmplCalexp, FileTemplate)
285 self.assertIsInstance(tmpl_image, FileTemplate)
286 self.assertIsInstance(tmplWcs, FileTemplate)
287 self.assertEqual(tmplCalexp, tmpl_image)
288 self.assertNotEqual(tmplCalexp, tmplWcs)
290 # Check dimensions lookup order.
291 # The order should be: dataset type name, dimension, storage class
292 # This one will not match name but might match storage class.
293 # It should match dimensions
294 refDims = self.makeDatasetRef(
295 "nomatch", dataId={"instrument": "LSST", "physical_filter": "z"}, storageClassName="StorageClassX"
296 )
297 tmplDims = templates.getTemplate(refDims)
298 self.assertIsInstance(tmplDims, FileTemplate)
299 self.assertNotEqual(tmplDims, tmplSc)
301 # Test that instrument overrides retrieve specialist templates
302 refPvi = self.makeDatasetRef("pvi")
303 refPviHsc = self.makeDatasetRef("pvi", dataId={"instrument": "HSC", "physical_filter": "z"})
304 refPviLsst = self.makeDatasetRef("pvi", dataId={"instrument": "LSST", "physical_filter": "z"})
306 tmplPvi = templates.getTemplate(refPvi)
307 tmplPviHsc = templates.getTemplate(refPviHsc)
308 tmplPviLsst = templates.getTemplate(refPviLsst)
309 self.assertEqual(tmplPvi, tmplPviLsst)
310 self.assertNotEqual(tmplPvi, tmplPviHsc)
312 # Have instrument match and dimensions look up with no name match
313 refNoPviHsc = self.makeDatasetRef(
314 "pvix", dataId={"instrument": "HSC", "physical_filter": "z"}, storageClassName="StorageClassX"
315 )
316 tmplNoPviHsc = templates.getTemplate(refNoPviHsc)
317 self.assertNotEqual(tmplNoPviHsc, tmplDims)
318 self.assertNotEqual(tmplNoPviHsc, tmplPviHsc)
320 # Format config file with defaulting
321 config2 = FileTemplatesConfig(os.path.join(configRoot, "templates-withdefault.yaml"))
322 templates = FileTemplates(config2, universe=self.universe)
323 tmpl = templates.getTemplate(ref2)
324 self.assertIsInstance(tmpl, FileTemplate)
326 # Format config file with bad format string
327 with self.assertRaises(FileTemplateValidationError):
328 FileTemplates(os.path.join(configRoot, "templates-bad.yaml"), universe=self.universe)
330 # Config file with no defaulting mentioned
331 config3 = os.path.join(configRoot, "templates-nodefault2.yaml")
332 templates = FileTemplates(config3, universe=self.universe)
333 with self.assertRaises(KeyError):
334 templates.getTemplate(ref2)
336 # Try again but specify a default in the constructor
337 default = "{run}/{datasetType}/{physical_filter}"
338 templates = FileTemplates(config3, default=default, universe=self.universe)
339 tmpl = templates.getTemplate(ref2)
340 self.assertEqual(tmpl.template, default)
342 def testValidation(self):
343 configRoot = os.path.join(TESTDIR, "config", "templates")
344 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
345 templates = FileTemplates(config1, universe=self.universe)
347 entities = {}
348 entities["calexp"] = self.makeDatasetRef(
349 "calexp",
350 storageClassName="StorageClassX",
351 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 52},
352 )
354 with self.assertLogs(level="WARNING") as cm:
355 templates.validateTemplates(entities.values(), logFailures=True)
356 self.assertIn("Unchecked keys", cm.output[0])
357 self.assertIn("StorageClassX", cm.output[0])
359 entities["pvi"] = self.makeDatasetRef(
360 "pvi", storageClassName="StorageClassX", dataId={"instrument": "dummy", "physical_filter": "i"}
361 )
362 entities["StorageClassX"] = self.makeDatasetRef(
363 "storageClass", storageClassName="StorageClassX", dataId={"instrument": "dummy", "visit": 2}
364 )
365 entities["calexp.wcs"] = self.makeDatasetRef(
366 "calexp.wcs",
367 storageClassName="StorageClassX",
368 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 23},
369 conform=False,
370 )
372 entities["instrument+physical_filter"] = self.makeDatasetRef(
373 "filter_inst",
374 storageClassName="StorageClassX",
375 dataId={"physical_filter": "i", "instrument": "SCUBA"},
376 )
377 entities["hsc+pvi"] = self.makeDatasetRef(
378 "pvi", storageClassName="StorageClassX", dataId={"physical_filter": "i", "instrument": "HSC"}
379 )
381 entities["hsc+instrument+physical_filter"] = self.makeDatasetRef(
382 "filter_inst",
383 storageClassName="StorageClassX",
384 dataId={"physical_filter": "i", "instrument": "HSC"},
385 )
387 entities["metric6"] = self.makeDatasetRef(
388 "filter_inst",
389 storageClassName="Integer",
390 dataId={"physical_filter": "i", "instrument": "HSC"},
391 )
393 templates.validateTemplates(entities.values(), logFailures=True)
395 # Rerun but with a failure
396 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX", dataId={"band": "i"})
397 with self.assertRaises(FileTemplateValidationError):
398 with self.assertLogs(level="FATAL"):
399 templates.validateTemplates(entities.values(), logFailures=True)
402if __name__ == "__main__":
403 unittest.main()