Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

#!/usr/bin/env python 

 

# 

# LSST Data Management System 

# Copyright 2008, 2009, 2010 LSST Corporation. 

# 

# This product includes software developed by the 

# LSST Project (http://www.lsst.org/). 

# 

# This program is free software: you can redistribute it and/or modify 

# it under the terms of the GNU General Public License as published by 

# the Free Software Foundation, either version 3 of the License, or 

# (at your option) any later version. 

# 

# This program is distributed in the hope that it will be useful, 

# but WITHOUT ANY WARRANTY; without even the implied warranty of 

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

# GNU General Public License for more details. 

# 

# You should have received a copy of the LSST License Statement and 

# the GNU General Public License along with this program. If not, 

# see <http://www.lsstcorp.org/LegalNotices/>. 

# 

 

 

"""This module defines the ButlerLocation class.""" 

 

import lsst.daf.base as dafBase 

 

import os 

from past.builtins import basestring 

import yaml 

 

from . import iterify, doImport 

 

 

class ButlerComposite(object): 

"""Initializer 

 

Parameters 

---------- 

assembler : function object 

Function object or importable string to a function object that can be called with the assembler 

signature: (dataId, componentDict, cls). 

disassembler : function object 

Function object or importable string to a function object that can be called with the disassembler 

signature: (object, dataId, componentDict). 

python : class object 

A python class object or importable string to a class object that can be used by the assembler to 

instantiate an object to be returned. 

dataId : dict or DataId 

The dataId that is used to look up components. 

mapper : Mapper instance 

A reference to the mapper that created this ButlerComposite object. 

""" 

 

class ComponentInfo(): 

"""Information about a butler composite object. Some details come from the policy and some are filled 

in by the butler. Component info is used while assembling and disassembling a composite object in 

butler. It is used as an input to assemblers and disassemblers (which are part of the butler public 

API). 

 

Parameters 

---------- 

datasetType : string 

The datasetType of the component. 

obj : object instance 

The python object instance that is this component. 

setter : string 

The name of the function in the parent object to set this component. 

Optional - may be None 

getter : string 

The name of the function in the parent object to get this component. 

Optional - may be None 

subset : bool 

If true, indicates that the obj should be a list of objects found via butlerSubset. 

inputOnly : bool 

If true, indicates that the obj should not be serialized when performing a butler.put. 

""" 

def __init__(self, datasetType, obj, setter, getter, subset, inputOnly): 

self.datasetType = datasetType 

self.obj = obj 

self.setter = setter 

self.getter = getter 

self.subset = subset 

self.inputOnly = inputOnly 

 

def __repr__(self): 

return 'ComponentInfo(datasetType:%s, obj:%s, setter:%s, getter:%s, subset:%s)' % \ 

(self.datasetType, self.obj, self.setter, self.getter, self.subset) 

 

def __repr__(self): 

return 'ButlerComposite(assembler:%s, disassembler:%s, python:%s, dataId:%s, mapper:%s, ' \ 

'componentInfo:%s, repository:%s)' % \ 

(self.assembler, 

self.disassembler, 

self.python, 

self.dataId, 

self.mapper, 

self.componentInfo, 

self.repository) 

 

def __init__(self, assembler, disassembler, python, dataId, mapper): 

self.assembler = doImport(assembler) if isinstance(assembler, basestring) else assembler 

self.disassembler = doImport(disassembler) if isinstance(disassembler, basestring) else disassembler 

self.python = doImport(python) if isinstance(python, basestring) else python 

self.dataId = dataId 

self.mapper = mapper 

self.componentInfo = {} 

self.repository = None 

 

def add(self, id, datasetType, setter, getter, subset, inputOnly): 

"""Add a description of a component needed to fetch the composite dataset. 

 

Parameters 

---------- 

id : string 

The name of the component in the policy definition. 

datasetType : string 

The name of the datasetType of the component. 

setter : string or None 

The name of the function used to set this component into the python type that contains it. 

Specifying a setter is optional, use None if the setter won't be specified or used. 

getter : string or None 

The name of the function used to get this component from the python type that contains it. 

Specifying a setter is optional, use None if the setter won't be specified or used. 

subset : bool 

If true, indicates that the obj should be a list of objects found via butlerSubset. 

inputOnly : bool 

If true, indicates that the obj should not be serialized when performing a butler.put. 

""" 

self.componentInfo[id] = ButlerComposite.ComponentInfo(datasetType=datasetType, 

obj=None, 

setter=setter, 

getter=getter, 

subset=subset, 

inputOnly=inputOnly) 

 

def setRepository(self, repository): 

self.repository = repository 

 

def getRepository(self): 

return self.repository 

 

def getPythonType(self): 

return self.python 

 

 

class ButlerLocation(yaml.YAMLObject): 

"""ButlerLocation is a struct-like class that holds information needed to 

persist and retrieve an object using the LSST Persistence Framework. 

 

Mappers should create and return ButlerLocations from their 

map_{datasetType} methods. 

 

Parameters 

---------- 

pythonType - string or class instance 

This is the type of python object that should be created when reading the location. 

 

cppType - string or None 

The type of cpp object represented by the location (optional, may be None) 

 

storageName - string 

The type of storage the object is in or should be place into. 

 

locationList - list of string 

A list of URI to place the object or where the object might be found. (Typically when reading the 

length is expected to be exactly 1). 

 

dataId - dict 

The dataId that was passed in when mapping the location. This may include keys that were not used for 

mapping this location. 

 

mapper - mapper class instance 

The mapper object that mapped this location. 

 

storage - storage class instance 

The storage interface that can be used to read or write this location. 

 

usedDataId - dict 

The dataId components that were used to map this location. If the mapper had to look up keys those 

will be in this dict (even though they may not appear in the dataId parameter). If the dataId 

parameter contained keys that were not required to map this item then those keys will NOT be in this 

parameter. 

 

datasetType - string 

The datasetType that this location represents. 

 

additionalData : `lsst.daf.base.PropertySet`, optional 

Additional metadata to be passed to the persistence framework, 

or `None`. 

""" 

 

yaml_tag = u"!ButlerLocation" 

try: 

# PyYAML >=5.1 prefers a different loader 

yaml_loader = yaml.FullLoader 

except AttributeError: 

yaml_loader = yaml.Loader 

yaml_dumper = yaml.Dumper 

 

def __repr__(self): 

return \ 

'ButlerLocation(pythonType=%r, cppType=%r, storageName=%r, storage=%r, locationList=%r,' \ 

' additionalData=%r, mapper=%r, dataId=%r)' % \ 

(self.pythonType, self.cppType, self.storageName, self.storage, self.locationList, 

self.additionalData, self.mapper, self.dataId) 

 

def __init__(self, pythonType, cppType, storageName, locationList, dataId, mapper, storage, 

usedDataId=None, datasetType=None, additionalData=None): 

# pythonType is sometimes unicode with Python 2 and pybind11; this breaks the interpreter 

self.pythonType = str(pythonType) if isinstance(pythonType, basestring) else pythonType 

self.cppType = cppType 

self.storageName = storageName 

self.mapper = mapper 

self.storage = storage 

self.locationList = iterify(locationList) 

self.additionalData = additionalData if additionalData else dafBase.PropertySet() 

for k, v in dataId.items(): 

self.additionalData.set(k, v) 

self.dataId = dataId 

self.usedDataId = usedDataId 

self.datasetType = datasetType 

 

def __str__(self): 

s = "%s at %s(%s)" % (self.pythonType, self.storageName, 

", ".join(self.locationList)) 

return s 

 

@staticmethod 

def to_yaml(dumper, obj): 

"""Representer for dumping to YAML 

:param dumper: 

:param obj: 

:return: 

""" 

return dumper.represent_mapping(ButlerLocation.yaml_tag, 

{'pythonType': obj.pythonType, 'cppType': obj.cppType, 

'storageName': obj.storageName, 

'locationList': obj.locationList, 'mapper': obj.mapper, 

'storage': obj.storage, 'dataId': obj.dataId}) 

 

@staticmethod 

def from_yaml(loader, node): 

obj = loader.construct_mapping(node) 

return ButlerLocation(**obj) 

 

def setRepository(self, repository): 

self.repository = repository 

 

def getRepository(self): 

return self.repository 

 

def getPythonType(self): 

return self.pythonType 

 

def getCppType(self): 

return self.cppType 

 

def getStorageName(self): 

return self.storageName 

 

def getLocations(self): 

return self.locationList 

 

def getLocationsWithRoot(self): 

return [os.path.join(self.storage.root, l) for l in self.getLocations()] 

 

def getAdditionalData(self): 

return self.additionalData 

 

def getStorage(self): 

return self.storage