Coverage for tests/test_templates.py: 9%
177 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-10-02 08:00 +0000
« prev ^ index » next coverage.py v7.3.1, created at 2023-10-02 08:00 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28"""Test file name templating."""
30import os.path
31import unittest
32import uuid
34from lsst.daf.butler import (
35 DataCoordinate,
36 DatasetId,
37 DatasetRef,
38 DatasetType,
39 DimensionGraph,
40 DimensionUniverse,
41 FileTemplate,
42 FileTemplates,
43 FileTemplatesConfig,
44 FileTemplateValidationError,
45 StorageClass,
46)
48TESTDIR = os.path.abspath(os.path.dirname(__file__))
50PlaceHolder = StorageClass("PlaceHolder")
52REFUUID = DatasetId(int=uuid.uuid4().int)
55class TestFileTemplates(unittest.TestCase):
56 """Test creation of paths from templates."""
58 def makeDatasetRef(
59 self, datasetTypeName, dataId=None, storageClassName="DefaultStorageClass", run="run2", conform=True
60 ):
61 """Make a simple DatasetRef"""
62 if dataId is None:
63 dataId = self.dataId
64 if "physical_filter" in dataId and "band" not in dataId:
65 dataId["band"] = "b" # Add fake band.
66 dimensions = DimensionGraph(self.universe, names=dataId.keys())
67 dataId = DataCoordinate.standardize(dataId, graph=dimensions)
69 # Pretend we have a parent if this looks like a composite
70 compositeName, componentName = DatasetType.splitDatasetTypeName(datasetTypeName)
71 parentStorageClass = PlaceHolder if componentName else None
73 datasetType = DatasetType(
74 datasetTypeName,
75 dimensions,
76 StorageClass(storageClassName),
77 parentStorageClass=parentStorageClass,
78 )
79 return DatasetRef(datasetType, dataId, id=REFUUID, run=run, conform=conform)
81 def setUp(self):
82 self.universe = DimensionUniverse()
83 self.dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "Most Amazing U Filter Ever"}
85 def assertTemplate(self, template, answer, ref):
86 fileTmpl = FileTemplate(template)
87 path = fileTmpl.format(ref)
88 self.assertEqual(path, answer)
90 def testBasic(self):
91 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}"
92 self.assertTemplate(
93 tmplstr,
94 "run2/calexp/00052/Most_Amazing_U_Filter_Ever",
95 self.makeDatasetRef("calexp"),
96 )
97 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail"
98 self.assertTemplate(
99 tmplstr,
100 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail",
101 self.makeDatasetRef("calexp"),
102 )
104 tmplstr = "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
105 self.assertTemplate(
106 tmplstr,
107 "run2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run2",
108 self.makeDatasetRef("calexp"),
109 )
110 self.assertTemplate(
111 tmplstr,
112 "run_2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2",
113 self.makeDatasetRef("calexp", run="run/2"),
114 )
116 # Check that the id is sufficient without any other information.
117 self.assertTemplate("{id}", str(REFUUID), self.makeDatasetRef("calexp", run="run2"))
119 self.assertTemplate("{run}/{id}", f"run2/{str(REFUUID)}", self.makeDatasetRef("calexp", run="run2"))
121 self.assertTemplate(
122 "fixed/{id}",
123 f"fixed/{str(REFUUID)}",
124 self.makeDatasetRef("calexp", run="run2"),
125 )
127 self.assertTemplate(
128 "fixed/{id}_{physical_filter}",
129 f"fixed/{str(REFUUID)}_Most_Amazing_U_Filter_Ever",
130 self.makeDatasetRef("calexp", run="run2"),
131 )
133 # Retain any "/" in run
134 tmplstr = "{run:/}/{datasetType}/{visit:05d}/{physical_filter}-trail-{run}"
135 self.assertTemplate(
136 tmplstr,
137 "run/2/calexp/00052/Most_Amazing_U_Filter_Ever-trail-run_2",
138 self.makeDatasetRef("calexp", run="run/2"),
139 )
141 # Check that "." are replaced in the file basename, but not directory.
142 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "g.10"}
143 self.assertTemplate(
144 tmplstr,
145 "run.2/calexp/00052/g_10-trail-run_2",
146 self.makeDatasetRef("calexp", run="run.2", dataId=dataId),
147 )
149 with self.assertRaises(FileTemplateValidationError):
150 FileTemplate("no fields at all")
152 with self.assertRaises(FileTemplateValidationError):
153 FileTemplate("{visit}")
155 with self.assertRaises(FileTemplateValidationError):
156 FileTemplate("{run}_{datasetType}")
158 with self.assertRaises(FileTemplateValidationError):
159 FileTemplate("{id}/fixed")
161 def testRunOrCollectionNeeded(self):
162 tmplstr = "{datasetType}/{visit:05d}/{physical_filter}"
163 with self.assertRaises(FileTemplateValidationError):
164 self.assertTemplate(tmplstr, "run2/calexp/00052/U", self.makeDatasetRef("calexp"))
166 def testNoRecord(self):
167 # Attaching records is not possible in this test code but we can check
168 # that a missing record when a metadata entry has been requested
169 # does fail.
170 tmplstr = "{run}/{datasetType}/{visit.name}/{physical_filter}"
171 with self.assertRaises(RuntimeError) as cm:
172 self.assertTemplate(tmplstr, "", self.makeDatasetRef("calexp"))
173 self.assertIn("No metadata", str(cm.exception))
175 def testOptional(self):
176 """Optional units in templates."""
177 ref = self.makeDatasetRef("calexp")
178 tmplstr = "{run}/{datasetType}/v{visit:05d}_f{physical_filter:?}"
179 self.assertTemplate(
180 tmplstr,
181 "run2/calexp/v00052_fMost_Amazing_U_Filter_Ever",
182 self.makeDatasetRef("calexp"),
183 )
185 du = {"visit": 48, "tract": 265, "skymap": "big", "instrument": "dummy"}
186 self.assertTemplate(tmplstr, "run2/calexpT/v00048", self.makeDatasetRef("calexpT", du))
188 # Ensure that this returns a relative path even if the first field
189 # is optional
190 tmplstr = "{run}/{tract:?}/{visit:?}/f{physical_filter}"
191 self.assertTemplate(tmplstr, "run2/52/fMost_Amazing_U_Filter_Ever", ref)
193 # Ensure that // from optionals are converted to singles
194 tmplstr = "{run}/{datasetType}/{patch:?}/{tract:?}/f{physical_filter}"
195 self.assertTemplate(tmplstr, "run2/calexp/fMost_Amazing_U_Filter_Ever", ref)
197 # Optionals with some text between fields
198 tmplstr = "{run}/{datasetType}/p{patch:?}_t{tract:?}/f{physical_filter}"
199 self.assertTemplate(tmplstr, "run2/calexp/p/fMost_Amazing_U_Filter_Ever", ref)
200 tmplstr = "{run}/{datasetType}/p{patch:?}_t{visit:04d?}/f{physical_filter}"
201 self.assertTemplate(tmplstr, "run2/calexp/p_t0052/fMost_Amazing_U_Filter_Ever", ref)
203 def testComponent(self):
204 """Test handling of components in templates."""
205 refMetricOutput = self.makeDatasetRef("metric.output")
206 refMetric = self.makeDatasetRef("metric")
207 refMaskedImage = self.makeDatasetRef("calexp.maskedimage.variance")
208 refWcs = self.makeDatasetRef("calexp.wcs")
210 tmplstr = "{run}_c_{component}_v{visit}"
211 self.assertTemplate(tmplstr, "run2_c_output_v52", refMetricOutput)
213 # We want this template to have both a directory and basename, to
214 # test that the right parts of the output are replaced.
215 tmplstr = "{component:?}/{run}_{component:?}_{visit}"
216 self.assertTemplate(tmplstr, "run2_52", refMetric)
217 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput)
218 self.assertTemplate(tmplstr, "maskedimage.variance/run2_maskedimage_variance_52", refMaskedImage)
219 self.assertTemplate(tmplstr, "output/run2_output_52", refMetricOutput)
221 # Providing a component but not using it
222 tmplstr = "{run}/{datasetType}/v{visit:05d}"
223 with self.assertRaises(KeyError):
224 self.assertTemplate(tmplstr, "", refWcs)
226 def testFields(self):
227 # Template, mandatory fields, optional non-special fields,
228 # special fields, optional special fields
229 testData = (
230 (
231 "{run}/{datasetType}/{visit:05d}/{physical_filter}-trail",
232 {"visit", "physical_filter"},
233 set(),
234 {"run", "datasetType"},
235 set(),
236 ),
237 (
238 "{run}/{component:?}_{visit}",
239 {"visit"},
240 set(),
241 {"run"},
242 {"component"},
243 ),
244 (
245 "{run}/{component:?}_{visit:?}_{physical_filter}_{instrument}_{datasetType}",
246 {"physical_filter", "instrument"},
247 {"visit"},
248 {"run", "datasetType"},
249 {"component"},
250 ),
251 )
252 for tmplstr, mandatory, optional, special, optionalSpecial in testData:
253 with self.subTest(template=tmplstr):
254 tmpl = FileTemplate(tmplstr)
255 fields = tmpl.fields()
256 self.assertEqual(fields, mandatory)
257 fields = tmpl.fields(optionals=True)
258 self.assertEqual(fields, mandatory | optional)
259 fields = tmpl.fields(specials=True)
260 self.assertEqual(fields, mandatory | special)
261 fields = tmpl.fields(specials=True, optionals=True)
262 self.assertEqual(fields, mandatory | special | optional | optionalSpecial)
264 def testSimpleConfig(self):
265 """Test reading from config file"""
266 configRoot = os.path.join(TESTDIR, "config", "templates")
267 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
268 templates = FileTemplates(config1, universe=self.universe)
269 ref = self.makeDatasetRef("calexp")
270 tmpl = templates.getTemplate(ref)
271 self.assertIsInstance(tmpl, FileTemplate)
273 # This config file should not allow defaulting
274 ref2 = self.makeDatasetRef("unknown")
275 with self.assertRaises(KeyError):
276 templates.getTemplate(ref2)
278 # This should fall through the datasetTypeName check and use
279 # StorageClass instead
280 ref3 = self.makeDatasetRef("unknown2", storageClassName="StorageClassX")
281 tmplSc = templates.getTemplate(ref3)
282 self.assertIsInstance(tmplSc, FileTemplate)
284 # Try with a component: one with defined formatter and one without
285 refWcs = self.makeDatasetRef("calexp.wcs")
286 refImage = self.makeDatasetRef("calexp.image")
287 tmplCalexp = templates.getTemplate(ref)
288 tmplWcs = templates.getTemplate(refWcs) # Should be special
289 tmpl_image = templates.getTemplate(refImage)
290 self.assertIsInstance(tmplCalexp, FileTemplate)
291 self.assertIsInstance(tmpl_image, FileTemplate)
292 self.assertIsInstance(tmplWcs, FileTemplate)
293 self.assertEqual(tmplCalexp, tmpl_image)
294 self.assertNotEqual(tmplCalexp, tmplWcs)
296 # Check dimensions lookup order.
297 # The order should be: dataset type name, dimension, storage class
298 # This one will not match name but might match storage class.
299 # It should match dimensions
300 refDims = self.makeDatasetRef(
301 "nomatch", dataId={"instrument": "LSST", "physical_filter": "z"}, storageClassName="StorageClassX"
302 )
303 tmplDims = templates.getTemplate(refDims)
304 self.assertIsInstance(tmplDims, FileTemplate)
305 self.assertNotEqual(tmplDims, tmplSc)
307 # Test that instrument overrides retrieve specialist templates
308 refPvi = self.makeDatasetRef("pvi")
309 refPviHsc = self.makeDatasetRef("pvi", dataId={"instrument": "HSC", "physical_filter": "z"})
310 refPviLsst = self.makeDatasetRef("pvi", dataId={"instrument": "LSST", "physical_filter": "z"})
312 tmplPvi = templates.getTemplate(refPvi)
313 tmplPviHsc = templates.getTemplate(refPviHsc)
314 tmplPviLsst = templates.getTemplate(refPviLsst)
315 self.assertEqual(tmplPvi, tmplPviLsst)
316 self.assertNotEqual(tmplPvi, tmplPviHsc)
318 # Have instrument match and dimensions look up with no name match
319 refNoPviHsc = self.makeDatasetRef(
320 "pvix", dataId={"instrument": "HSC", "physical_filter": "z"}, storageClassName="StorageClassX"
321 )
322 tmplNoPviHsc = templates.getTemplate(refNoPviHsc)
323 self.assertNotEqual(tmplNoPviHsc, tmplDims)
324 self.assertNotEqual(tmplNoPviHsc, tmplPviHsc)
326 # Format config file with defaulting
327 config2 = FileTemplatesConfig(os.path.join(configRoot, "templates-withdefault.yaml"))
328 templates = FileTemplates(config2, universe=self.universe)
329 tmpl = templates.getTemplate(ref2)
330 self.assertIsInstance(tmpl, FileTemplate)
332 # Format config file with bad format string
333 with self.assertRaises(FileTemplateValidationError):
334 FileTemplates(os.path.join(configRoot, "templates-bad.yaml"), universe=self.universe)
336 # Config file with no defaulting mentioned
337 config3 = os.path.join(configRoot, "templates-nodefault2.yaml")
338 templates = FileTemplates(config3, universe=self.universe)
339 with self.assertRaises(KeyError):
340 templates.getTemplate(ref2)
342 # Try again but specify a default in the constructor
343 default = "{run}/{datasetType}/{physical_filter}"
344 templates = FileTemplates(config3, default=default, universe=self.universe)
345 tmpl = templates.getTemplate(ref2)
346 self.assertEqual(tmpl.template, default)
348 def testValidation(self):
349 configRoot = os.path.join(TESTDIR, "config", "templates")
350 config1 = FileTemplatesConfig(os.path.join(configRoot, "templates-nodefault.yaml"))
351 templates = FileTemplates(config1, universe=self.universe)
353 entities = {}
354 entities["calexp"] = self.makeDatasetRef(
355 "calexp",
356 storageClassName="StorageClassX",
357 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 52},
358 )
360 with self.assertLogs(level="WARNING") as cm:
361 templates.validateTemplates(entities.values(), logFailures=True)
362 self.assertIn("Unchecked keys", cm.output[0])
363 self.assertIn("StorageClassX", cm.output[0])
365 entities["pvi"] = self.makeDatasetRef(
366 "pvi", storageClassName="StorageClassX", dataId={"instrument": "dummy", "physical_filter": "i"}
367 )
368 entities["StorageClassX"] = self.makeDatasetRef(
369 "storageClass", storageClassName="StorageClassX", dataId={"instrument": "dummy", "visit": 2}
370 )
371 entities["calexp.wcs"] = self.makeDatasetRef(
372 "calexp.wcs",
373 storageClassName="StorageClassX",
374 dataId={"instrument": "dummy", "physical_filter": "i", "visit": 23},
375 conform=False,
376 )
378 entities["instrument+physical_filter"] = self.makeDatasetRef(
379 "filter_inst",
380 storageClassName="StorageClassX",
381 dataId={"physical_filter": "i", "instrument": "SCUBA"},
382 )
383 entities["hsc+pvi"] = self.makeDatasetRef(
384 "pvi", storageClassName="StorageClassX", dataId={"physical_filter": "i", "instrument": "HSC"}
385 )
387 entities["hsc+instrument+physical_filter"] = self.makeDatasetRef(
388 "filter_inst",
389 storageClassName="StorageClassX",
390 dataId={"physical_filter": "i", "instrument": "HSC"},
391 )
393 entities["metric6"] = self.makeDatasetRef(
394 "filter_inst",
395 storageClassName="Integer",
396 dataId={"physical_filter": "i", "instrument": "HSC"},
397 )
399 templates.validateTemplates(entities.values(), logFailures=True)
401 # Rerun but with a failure
402 entities["pvi"] = self.makeDatasetRef("pvi", storageClassName="StorageClassX", dataId={"band": "i"})
403 with self.assertRaises(FileTemplateValidationError):
404 with self.assertLogs(level="FATAL"):
405 templates.validateTemplates(entities.values(), logFailures=True)
408if __name__ == "__main__":
409 unittest.main()