Coverage for tests/test_templates.py: 10%
178 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-27 09:44 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-27 09:44 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28"""Test file name templating."""
30import os.path
31import unittest
32import uuid
34from lsst.daf.butler import (
35 DataCoordinate,
36 DatasetId,
37 DatasetRef,
38 DatasetType,
39 DimensionGraph,
40 DimensionUniverse,
41 StorageClass,
42)
43from lsst.daf.butler.datastore.file_templates import (
44 FileTemplate,
45 FileTemplates,
46 FileTemplatesConfig,
47 FileTemplateValidationError,
48)
50TESTDIR = os.path.abspath(os.path.dirname(__file__))
52PlaceHolder = StorageClass("PlaceHolder")
54REFUUID = DatasetId(int=uuid.uuid4().int)
57class TestFileTemplates(unittest.TestCase):
58 """Test creation of paths from templates."""
60 def makeDatasetRef(
61 self, datasetTypeName, dataId=None, storageClassName="DefaultStorageClass", run="run2", conform=True
62 ):
63 """Make a simple DatasetRef"""
64 if dataId is None:
65 dataId = self.dataId
66 if "physical_filter" in dataId and "band" not in dataId:
67 dataId["band"] = "b" # Add fake band.
68 dimensions = DimensionGraph(self.universe, names=dataId.keys())
69 dataId = DataCoordinate.standardize(dataId, graph=dimensions)
71 # Pretend we have a parent if this looks like a composite
72 compositeName, componentName = DatasetType.splitDatasetTypeName(datasetTypeName)
73 parentStorageClass = PlaceHolder if componentName else None
75 datasetType = DatasetType(
76 datasetTypeName,
77 dimensions,
78 StorageClass(storageClassName),
79 parentStorageClass=parentStorageClass,
80 )
81 return DatasetRef(datasetType, dataId, id=REFUUID, run=run, conform=conform)
83 def setUp(self):
84 self.universe = DimensionUniverse()
85 self.dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "Most Amazing U Filter Ever"}
87 def assertTemplate(self, template, answer, ref):
88 fileTmpl = FileTemplate(template)
89 path = fileTmpl.format(ref)
90 self.assertEqual(path, answer)
92 def testBasic(self):
93 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}"
94 self.assertTemplate(
95 tmplstr,
96 "run2/calexp/00052/Most_Amazing_U_Filter_Ever",
97 self.makeDatasetRef("calexp"),
98 )
99 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail"
100 self.assertTemplate(
101 tmplstr,
102 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail",
103 self.makeDatasetRef("calexp"),
104 )
106 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
107 self.assertTemplate(
108 tmplstr,
109 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run2",
110 self.makeDatasetRef("calexp"),
111 )
112 self.assertTemplate(
113 tmplstr,
114 "run_2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2",
115 self.makeDatasetRef("calexp", run="run/2"),
116 )
118 # Check that the id is sufficient without any other information.
119 self.assertTemplate("{id}", str(REFUUID), self.makeDatasetRef("calexp", run="run2"))
121 self.assertTemplate("{run}/{id}", f"run2/{str(REFUUID)}", self.makeDatasetRef("calexp", run="run2"))
123 self.assertTemplate(
124 "fixed/{id}",
125 f"fixed/{str(REFUUID)}",
126 self.makeDatasetRef("calexp", run="run2"),
127 )
129 self.assertTemplate(
130 "fixed/{id}_{physical_filter}",
131 f"fixed/{str(REFUUID)}_Most_Amazing_U_Filter_Ever",
132 self.makeDatasetRef("calexp", run="run2"),
133 )
135 # Retain any "/" in run
136 tmplstr = "{run:/}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
137 self.assertTemplate(
138 tmplstr,
139 "run/2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2",
140 self.makeDatasetRef("calexp", run="run/2"),
141 )
143 # Check that "." are replaced in the file basename, but not directory.
144 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "g.10"}
145 self.assertTemplate(
146 tmplstr,
147 "run.2/calexp/00052/g_10-trail-run_2",
148 self.makeDatasetRef("calexp", run="run.2", dataId=dataId),
149 )
151 with self.assertRaises(FileTemplateValidationError):
152 FileTemplate("no fields at all")
154 with self.assertRaises(FileTemplateValidationError):
155 FileTemplate("{visit}")
157 with self.assertRaises(FileTemplateValidationError):
158 FileTemplate("{run}_{datasetType}")
160 with self.assertRaises(FileTemplateValidationError):
161 FileTemplate("{id}/fixed")
163 def testRunOrCollectionNeeded(self):
164 tmplstr = "{datasetType}/{visit:05d}/{physical_filter}"
165 with self.assertRaises(FileTemplateValidationError):
166 self.assertTemplate(tmplstr, "run2/calexp/00052/U", self.makeDatasetRef("calexp"))
168 def testNoRecord(self):
169 # Attaching records is not possible in this test code but we can check
170 # that a missing record when a metadata entry has been requested
171 # does fail.
172 tmplstr = "{run}/{datasetType}/{visit.name}/{physical_filter}"
173 with self.assertRaises(RuntimeError) as cm:
174 self.assertTemplate(tmplstr, "", self.makeDatasetRef("calexp"))
175 self.assertIn("No metadata", str(cm.exception))
177 def testOptional(self):
178 """Optional units in templates."""
179 ref = self.makeDatasetRef("calexp")
180 tmplstr = "{run}/{datasetType}/v{visit:05d}_f{physical_filter:?}"
181 self.assertTemplate(
182 tmplstr,
183 "run2/calexp/v00052_fMost_Amazing_U_Filter_Ever",
184 self.makeDatasetRef("calexp"),
185 )
187 du = {"visit": 48, "tract": 265, "skymap": "big", "instrument": "dummy"}
188 self.assertTemplate(tmplstr, "run2/calexpT/v00048", self.makeDatasetRef("calexpT", du))
190 # Ensure that this returns a relative path even if the first field
191 # is optional
192 tmplstr = "{run}/{tract:?}/{visit:?}/f{physical_filter}"
193 self.assertTemplate(tmplstr, "run2/52/fMost_Amazing_U_Filter_Ever", ref)
195 # Ensure that // from optionals are converted to singles
196 tmplstr = "{run}/{datasetType}/{patch:?}/{tract:?}/f{physical_filter}"
197 self.assertTemplate(tmplstr, "run2/calexp/fMost_Amazing_U_Filter_Ever", ref)
199 # Optionals with some text between fields
200 tmplstr = "{run}/{datasetType}/p{patch:?}_t{tract:?}/f{physical_filter}"
201 self.assertTemplate(tmplstr, "run2/calexp/p/fMost_Amazing_U_Filter_Ever", ref)
202 tmplstr = "{run}/{datasetType}/p{patch:?}_t{visit:04d?}/f{physical_filter}"
203 self.assertTemplate(tmplstr, "run2/calexp/p_t0052/fMost_Amazing_U_Filter_Ever", ref)
205 def testComponent(self):
206 """Test handling of components in templates."""
207 refMetricOutput = self.makeDatasetRef("metric.output")
208 refMetric = self.makeDatasetRef("metric")
209 refMaskedImage = self.makeDatasetRef("calexp.maskedimage.variance")
210 refWcs = self.makeDatasetRef("calexp.wcs")
212 tmplstr = "{run}_c_{component}_v{visit}"
213 self.assertTemplate(tmplstr, "run2_c_output_v52", refMetricOutput)
215 # We want this template to have both a directory and basename, to
216 # test that the right parts of the output are replaced.
217 tmplstr = "{component:?}/{run}_{component:?}_{visit}"
218 self.assertTemplate(tmplstr, "run2_52", refMetric)
219 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput)
220 self.assertTemplate(tmplstr, "maskedimage.variance/run2_maskedimage_variance_52", refMaskedImage)
221 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput)
223 # Providing a component but not using it
224 tmplstr = "{run}/{datasetType}/v{visit:05d}"
225 with self.assertRaises(KeyError):
226 self.assertTemplate(tmplstr, "", refWcs)
228 def testFields(self):
229 # Template, mandatory fields, optional non-special fields,
230 # special fields, optional special fields
231 testData = (
232 (
233 "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail",
234 {"visit", "physical_filter"},
235 set(),
236 {"run", "datasetType"},
237 set(),
238 ),
239 (
240 "{run}/{component:?}_{visit}",
241 {"visit"},
242 set(),
243 {"run"},
244 {"component"},
245 ),
246 (
247 "{run}/{component:?}_{visit:?}_{physical_filter}_{instrument}_{datasetType}",
248 {"physical_filter", "instrument"},
249 {"visit"},
250 {"run", "datasetType"},
251 {"component"},
252 ),
253 )
254 for tmplstr, mandatory, optional, special, optionalSpecial in testData:
255 with self.subTest(template=tmplstr):
256 tmpl = FileTemplate(tmplstr)
257 fields = tmpl.fields()
258 self.assertEqual(fields, mandatory)
259 fields = tmpl.fields(optionals=True)
260 self.assertEqual(fields, mandatory | optional)
261 fields = tmpl.fields(specials=True)
262 self.assertEqual(fields, mandatory | special)
263 fields = tmpl.fields(specials=True, optionals=True)
264 self.assertEqual(fields, mandatory | special | optional | optionalSpecial)
266 def testSimpleConfig(self):
267 """Test reading from config file"""
268 configRoot = os.path.join(TESTDIR, "config", "templates")
269 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
270 templates = FileTemplates(config1, universe=self.universe)
271 ref = self.makeDatasetRef("calexp")
272 tmpl = templates.getTemplate(ref)
273 self.assertIsInstance(tmpl, FileTemplate)
275 # This config file should not allow defaulting
276 ref2 = self.makeDatasetRef("unknown")
277 with self.assertRaises(KeyError):
278 templates.getTemplate(ref2)
280 # This should fall through the datasetTypeName check and use
281 # StorageClass instead
282 ref3 = self.makeDatasetRef("unknown2", storageClassName="StorageClassX")
283 tmplSc = templates.getTemplate(ref3)
284 self.assertIsInstance(tmplSc, FileTemplate)
286 # Try with a component: one with defined formatter and one without
287 refWcs = self.makeDatasetRef("calexp.wcs")
288 refImage = self.makeDatasetRef("calexp.image")
289 tmplCalexp = templates.getTemplate(ref)
290 tmplWcs = templates.getTemplate(refWcs) # Should be special
291 tmpl_image = templates.getTemplate(refImage)
292 self.assertIsInstance(tmplCalexp, FileTemplate)
293 self.assertIsInstance(tmpl_image, FileTemplate)
294 self.assertIsInstance(tmplWcs, FileTemplate)
295 self.assertEqual(tmplCalexp, tmpl_image)
296 self.assertNotEqual(tmplCalexp, tmplWcs)
298 # Check dimensions lookup order.
299 # The order should be: dataset type name, dimension, storage class
300 # This one will not match name but might match storage class.
301 # It should match dimensions
302 refDims = self.makeDatasetRef(
303 "nomatch", dataId={"instrument": "LSST", "physical_filter": "z"}, storageClassName="StorageClassX"
304 )
305 tmplDims = templates.getTemplate(refDims)
306 self.assertIsInstance(tmplDims, FileTemplate)
307 self.assertNotEqual(tmplDims, tmplSc)
309 # Test that instrument overrides retrieve specialist templates
310 refPvi = self.makeDatasetRef("pvi")
311 refPviHsc = self.makeDatasetRef("pvi", dataId={"instrument": "HSC", "physical_filter": "z"})
312 refPviLsst = self.makeDatasetRef("pvi", dataId={"instrument": "LSST", "physical_filter": "z"})
314 tmplPvi = templates.getTemplate(refPvi)
315 tmplPviHsc = templates.getTemplate(refPviHsc)
316 tmplPviLsst = templates.getTemplate(refPviLsst)
317 self.assertEqual(tmplPvi, tmplPviLsst)
318 self.assertNotEqual(tmplPvi, tmplPviHsc)
320 # Have instrument match and dimensions look up with no name match
321 refNoPviHsc = self.makeDatasetRef(
322 "pvix", dataId={"instrument": "HSC", "physical_filter": "z"}, storageClassName="StorageClassX"
323 )
324 tmplNoPviHsc = templates.getTemplate(refNoPviHsc)
325 self.assertNotEqual(tmplNoPviHsc, tmplDims)
326 self.assertNotEqual(tmplNoPviHsc, tmplPviHsc)
328 # Format config file with defaulting
329 config2 = FileTemplatesConfig(os.path.join(configRoot, "templates-withdefault.yaml"))
330 templates = FileTemplates(config2, universe=self.universe)
331 tmpl = templates.getTemplate(ref2)
332 self.assertIsInstance(tmpl, FileTemplate)
334 # Format config file with bad format string
335 with self.assertRaises(FileTemplateValidationError):
336 FileTemplates(os.path.join(configRoot, "templates-bad.yaml"), universe=self.universe)
338 # Config file with no defaulting mentioned
339 config3 = os.path.join(configRoot, "templates-nodefault2.yaml")
340 templates = FileTemplates(config3, universe=self.universe)
341 with self.assertRaises(KeyError):
342 templates.getTemplate(ref2)
344 # Try again but specify a default in the constructor
345 default = "{run}/{datasetType}/{physical_filter}"
346 templates = FileTemplates(config3, default=default, universe=self.universe)
347 tmpl = templates.getTemplate(ref2)
348 self.assertEqual(tmpl.template, default)
350 def testValidation(self):
351 configRoot = os.path.join(TESTDIR, "config", "templates")
352 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
353 templates = FileTemplates(config1, universe=self.universe)
355 entities = {}
356 entities["calexp"] = self.makeDatasetRef(
357 "calexp",
358 storageClassName="StorageClassX",
359 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 52},
360 )
362 with self.assertLogs(level="WARNING") as cm:
363 templates.validateTemplates(entities.values(), logFailures=True)
364 self.assertIn("Unchecked keys", cm.output[0])
365 self.assertIn("StorageClassX", cm.output[0])
367 entities["pvi"] = self.makeDatasetRef(
368 "pvi", storageClassName="StorageClassX", dataId={"instrument": "dummy", "physical_filter": "i"}
369 )
370 entities["StorageClassX"] = self.makeDatasetRef(
371 "storageClass", storageClassName="StorageClassX", dataId={"instrument": "dummy", "visit": 2}
372 )
373 entities["calexp.wcs"] = self.makeDatasetRef(
374 "calexp.wcs",
375 storageClassName="StorageClassX",
376 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 23},
377 conform=False,
378 )
380 entities["instrument+physical_filter"] = self.makeDatasetRef(
381 "filter_inst",
382 storageClassName="StorageClassX",
383 dataId={"physical_filter": "i", "instrument": "SCUBA"},
384 )
385 entities["hsc+pvi"] = self.makeDatasetRef(
386 "pvi", storageClassName="StorageClassX", dataId={"physical_filter": "i", "instrument": "HSC"}
387 )
389 entities["hsc+instrument+physical_filter"] = self.makeDatasetRef(
390 "filter_inst",
391 storageClassName="StorageClassX",
392 dataId={"physical_filter": "i", "instrument": "HSC"},
393 )
395 entities["metric6"] = self.makeDatasetRef(
396 "filter_inst",
397 storageClassName="Integer",
398 dataId={"physical_filter": "i", "instrument": "HSC"},
399 )
401 templates.validateTemplates(entities.values(), logFailures=True)
403 # Rerun but with a failure
404 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX", dataId={"band": "i"})
405 with self.assertRaises(FileTemplateValidationError):
406 with self.assertLogs(level="FATAL"):
407 templates.validateTemplates(entities.values(), logFailures=True)
410if __name__ == "__main__":
411 unittest.main()