Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

# This file is part of daf_butler. 

# 

# Developed for the LSST Data Management System. 

# This product includes software developed by the LSST Project 

# (http://www.lsst.org). 

# See the COPYRIGHT file at the top-level directory of this distribution 

# for details of code ownership. 

# 

# This program is free software: you can redistribute it and/or modify 

# it under the terms of the GNU General Public License as published by 

# the Free Software Foundation, either version 3 of the License, or 

# (at your option) any later version. 

# 

# This program is distributed in the hope that it will be useful, 

# but WITHOUT ANY WARRANTY; without even the implied warranty of 

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

# GNU General Public License for more details. 

# 

# You should have received a copy of the GNU General Public License 

# along with this program. If not, see <http://www.gnu.org/licenses/>. 

 

"""Support for file template string expansion.""" 

 

__all__ = ("FileTemplates", "FileTemplate", "FileTemplatesConfig") 

 

import os.path 

import string 

 

from .config import Config 

 

 

class FileTemplatesConfig(Config): 

"""Configuration information for `FileTemplates`""" 

pass 

 

 

class FileTemplates: 

"""Collection of `FileTemplate` templates. 

 

Parameters 

---------- 

config : `FileTemplatesConfig` or `str` 

Load configuration. 

default : `str`, optional 

If not `None`, a default template to use if no template has 

been specified explicitly in the configuration. 

""" 

 

def __init__(self, config, default=None): 

self.config = FileTemplatesConfig(config) 

self.templates = {} 

self.default = FileTemplate(default) if default is not None else None 

for name, templateStr in self.config.items(): 

# We can disable defaulting with an empty string in a config 

# or by using a boolean 

if name == "default": 

if not templateStr: 

self.default = None 

else: 

self.default = FileTemplate(templateStr) 

else: 

self.templates[name] = FileTemplate(templateStr) 

 

def getTemplate(self, entity): 

"""Retrieve the `FileTemplate` associated with the dataset type. 

 

If the lookup name corresponds to a component the base name for 

the component will be examined if the full component name does 

not match. 

 

Parameters 

---------- 

entity : `DatasetType`, `DatasetRef`, or `StorageClass` 

Instance to use to look for a corresponding template. 

A `DatasetType` name or a `StorageClass` name will be used 

depending on the supplied entity. Priority is given to a 

`DatasetType` name. 

 

Returns 

------- 

template : `FileTemplate` 

Template instance to use with that dataset type. 

 

Raises 

------ 

KeyError 

No template could be located for this Dataset type. 

""" 

 

# Get the names to use for lookup 

names = entity._lookupNames() 

 

# Get a location from the templates 

template = None 

for name in names: 

if name in self.templates: 

template = self.templates[name] 

elif "." in name: 

baseType, component = name.split(".", maxsplit=1) 

if baseType in self.templates: 

template = self.templates[baseType] 

if template is not None: 

break 

 

if template is None and self.default is not None: 

template = self.default 

 

# if still not template give up for now. 

if template is None: 

raise KeyError(f"Unable to determine file template from supplied argument [{entity}]") 

 

return template 

 

 

class FileTemplate: 

"""Format a path template into a fully expanded path. 

 

Parameters 

---------- 

template : `str` 

Template string. 

 

Notes 

----- 

The templates use the standard Format Specification Mini-Language 

with the caveat that only named fields can be used. The field names 

are taken from the DataUnits along with several additional fields: 

 

- datasetType: `str`, `DatasetType.name` 

- component: `str`, name of the StorageClass component 

- collection: `str`, `Run.collection` 

- run: `int`, `Run.id` 

 

At least one or both of `run` or `collection` must be provided to ensure 

unique filenames. 

 

The mini-language is extended to understand a "?" in the format 

specification. This indicates that a field is optional. If that 

DataUnit is missing the field, along with the text before the field, 

unless it is a path separator, will be removed from the output path. 

""" 

 

def __init__(self, template): 

if not isinstance(template, str) or "{" not in template: 

raise ValueError(f"Template ({template}) does not contain any format specifiers") 

self.template = template 

 

def format(self, ref): 

"""Format a template string into a full path. 

 

Parameters 

---------- 

ref : `DatasetRef` 

The dataset to be formatted. 

 

Returns 

------- 

path : `str` 

Expanded path. 

 

Raises 

------ 

KeyError 

Requested field is not defined and the field is not optional. 

Or, `component` is specified but "component" was not part of 

the template. 

""" 

# Extract defined non-None units from the dataId 

fields = {k: v for k, v in ref.dataId.items() if v is not None} 

 

datasetType = ref.datasetType 

fields["datasetType"] = datasetType.name 

component = datasetType.component() 

 

usedComponent = False 

if component is not None: 

fields["component"] = component 

 

usedRunOrCollection = False 

fields["collection"] = ref.run.collection 

fields["run"] = ref.run.id 

 

fmt = string.Formatter() 

parts = fmt.parse(self.template) 

output = "" 

 

for literal, field_name, format_spec, conversion in parts: 

 

if field_name == "component": 

usedComponent = True 

 

if format_spec is None: 

output = output + literal 

continue 

 

if "?" in format_spec: 

optional = True 

# Remove the non-standard character from the spec 

format_spec = format_spec.replace("?", "") 

else: 

optional = False 

 

if field_name in ("run", "collection"): 

usedRunOrCollection = True 

 

if field_name in fields: 

value = fields[field_name] 

elif optional: 

# If this is optional ignore the format spec 

# and do not include the literal text prior to the optional 

# field unless it contains a "/" path separator 

format_spec = "" 

value = "" 

if "/" not in literal: 

literal = "" 

else: 

raise KeyError("{} requested in template but not defined and not optional".format(field_name)) 

 

# Now use standard formatting 

output = output + literal + format(value, format_spec) 

 

# Complain if we were meant to use a component 

if component is not None and not usedComponent: 

raise KeyError("Component '{}' specified but template {} did not use it".format(component, 

self.template)) 

 

# Complain if there's no run or collection 

if not usedRunOrCollection: 

raise KeyError("Template does not include 'run' or 'collection'.") 

 

# Since this is known to be a path, normalize it in case some double 

# slashes have crept in 

path = os.path.normpath(output) 

 

# It should not be an absolute path (may happen with optionals) 

if os.path.isabs(path): 

path = os.path.relpath(path, start="/") 

 

return path