Coverage for python/lsst/daf/persistence/butlerLocation.py: 41%

84 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-12-14 03:44 -0800

1#!/usr/bin/env python 

2 

3# 

4# LSST Data Management System 

5# Copyright 2008, 2009, 2010 LSST Corporation. 

6# 

7# This product includes software developed by the 

8# LSST Project (http://www.lsst.org/). 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the LSST License Statement and 

21# the GNU General Public License along with this program. If not, 

22# see <http://www.lsstcorp.org/LegalNotices/>. 

23# 

24 

25 

26"""This module defines the ButlerLocation class.""" 

27 

28import lsst.daf.base as dafBase 

29 

30import os 

31import yaml 

32 

33from . import iterify, doImport 

34 

35 

36class ButlerComposite: 

37 """Initializer 

38 

39 Parameters 

40 ---------- 

41 assembler : function object 

42 Function object or importable string to a function object that can be called with the assembler 

43 signature: (dataId, componentDict, cls). 

44 disassembler : function object 

45 Function object or importable string to a function object that can be called with the disassembler 

46 signature: (object, dataId, componentDict). 

47 python : class object 

48 A python class object or importable string to a class object that can be used by the assembler to 

49 instantiate an object to be returned. 

50 dataId : dict or DataId 

51 The dataId that is used to look up components. 

52 mapper : Mapper instance 

53 A reference to the mapper that created this ButlerComposite object. 

54 """ 

55 

56 class ComponentInfo(): 

57 """Information about a butler composite object. Some details come from the policy and some are filled 

58 in by the butler. Component info is used while assembling and disassembling a composite object in 

59 butler. It is used as an input to assemblers and disassemblers (which are part of the butler public 

60 API). 

61 

62 Parameters 

63 ---------- 

64 datasetType : string 

65 The datasetType of the component. 

66 obj : object instance 

67 The python object instance that is this component. 

68 setter : string 

69 The name of the function in the parent object to set this component. 

70 Optional - may be None 

71 getter : string 

72 The name of the function in the parent object to get this component. 

73 Optional - may be None 

74 subset : bool 

75 If true, indicates that the obj should be a list of objects found via butlerSubset. 

76 inputOnly : bool 

77 If true, indicates that the obj should not be serialized when performing a butler.put. 

78 """ 

79 def __init__(self, datasetType, obj, setter, getter, subset, inputOnly): 

80 self.datasetType = datasetType 

81 self.obj = obj 

82 self.setter = setter 

83 self.getter = getter 

84 self.subset = subset 

85 self.inputOnly = inputOnly 

86 

87 def __repr__(self): 

88 return 'ComponentInfo(datasetType:%s, obj:%s, setter:%s, getter:%s, subset:%s)' % \ 

89 (self.datasetType, self.obj, self.setter, self.getter, self.subset) 

90 

91 def __repr__(self): 

92 return 'ButlerComposite(assembler:%s, disassembler:%s, python:%s, dataId:%s, mapper:%s, ' \ 

93 'componentInfo:%s, repository:%s)' % \ 

94 (self.assembler, 

95 self.disassembler, 

96 self.python, 

97 self.dataId, 

98 self.mapper, 

99 self.componentInfo, 

100 self.repository) 

101 

102 def __init__(self, assembler, disassembler, python, dataId, mapper): 

103 self.assembler = doImport(assembler) if isinstance(assembler, str) else assembler 

104 self.disassembler = doImport(disassembler) if isinstance(disassembler, str) else disassembler 

105 self.python = doImport(python) if isinstance(python, str) else python 

106 self.dataId = dataId 

107 self.mapper = mapper 

108 self.componentInfo = {} 

109 self.repository = None 

110 

111 def add(self, id, datasetType, setter, getter, subset, inputOnly): 

112 """Add a description of a component needed to fetch the composite dataset. 

113 

114 Parameters 

115 ---------- 

116 id : string 

117 The name of the component in the policy definition. 

118 datasetType : string 

119 The name of the datasetType of the component. 

120 setter : string or None 

121 The name of the function used to set this component into the python type that contains it. 

122 Specifying a setter is optional, use None if the setter won't be specified or used. 

123 getter : string or None 

124 The name of the function used to get this component from the python type that contains it. 

125 Specifying a setter is optional, use None if the setter won't be specified or used. 

126 subset : bool 

127 If true, indicates that the obj should be a list of objects found via butlerSubset. 

128 inputOnly : bool 

129 If true, indicates that the obj should not be serialized when performing a butler.put. 

130 """ 

131 self.componentInfo[id] = ButlerComposite.ComponentInfo(datasetType=datasetType, 

132 obj=None, 

133 setter=setter, 

134 getter=getter, 

135 subset=subset, 

136 inputOnly=inputOnly) 

137 

138 def setRepository(self, repository): 

139 self.repository = repository 

140 

141 def getRepository(self): 

142 return self.repository 

143 

144 def getPythonType(self): 

145 return self.python 

146 

147 

148class ButlerLocation(yaml.YAMLObject): 

149 """ButlerLocation is a struct-like class that holds information needed to 

150 persist and retrieve an object using the LSST Persistence Framework. 

151 

152 Mappers should create and return ButlerLocations from their 

153 map_{datasetType} methods. 

154 

155 Parameters 

156 ---------- 

157 pythonType - string or class instance 

158 This is the type of python object that should be created when reading the location. 

159 

160 cppType - string or None 

161 The type of cpp object represented by the location (optional, may be None) 

162 

163 storageName - string 

164 The type of storage the object is in or should be place into. 

165 

166 locationList - list of string 

167 A list of URI to place the object or where the object might be found. (Typically when reading the 

168 length is expected to be exactly 1). 

169 

170 dataId - dict 

171 The dataId that was passed in when mapping the location. This may include keys that were not used for 

172 mapping this location. 

173 

174 mapper - mapper class instance 

175 The mapper object that mapped this location. 

176 

177 storage - storage class instance 

178 The storage interface that can be used to read or write this location. 

179 

180 usedDataId - dict 

181 The dataId components that were used to map this location. If the mapper had to look up keys those 

182 will be in this dict (even though they may not appear in the dataId parameter). If the dataId 

183 parameter contained keys that were not required to map this item then those keys will NOT be in this 

184 parameter. 

185 

186 datasetType - string 

187 The datasetType that this location represents. 

188 

189 additionalData : `lsst.daf.base.PropertySet`, optional 

190 Additional metadata to be passed to the persistence framework, 

191 or `None`. 

192 """ 

193 

194 yaml_tag = u"!ButlerLocation" 

195 try: 

196 # PyYAML >=5.1 prefers a different loader 

197 yaml_loader = yaml.UnsafeLoader 

198 except AttributeError: 

199 yaml_loader = yaml.Loader 

200 yaml_dumper = yaml.Dumper 

201 

202 def __repr__(self): 

203 return \ 

204 'ButlerLocation(pythonType=%r, cppType=%r, storageName=%r, storage=%r, locationList=%r,' \ 

205 ' additionalData=%r, mapper=%r, dataId=%r)' % \ 

206 (self.pythonType, self.cppType, self.storageName, self.storage, self.locationList, 

207 self.additionalData, self.mapper, self.dataId) 

208 

209 def __init__(self, pythonType, cppType, storageName, locationList, dataId, mapper, storage, 

210 usedDataId=None, datasetType=None, additionalData=None): 

211 self.pythonType = pythonType 

212 self.cppType = cppType 

213 self.storageName = storageName 

214 self.mapper = mapper 

215 self.storage = storage 

216 self.locationList = iterify(locationList) 

217 self.additionalData = additionalData if additionalData else dafBase.PropertySet() 

218 for k, v in dataId.items(): 

219 self.additionalData.set(k, v) 

220 self.dataId = dataId 

221 self.usedDataId = usedDataId 

222 self.datasetType = datasetType 

223 

224 def __str__(self): 

225 s = "%s at %s(%s)" % (self.pythonType, self.storageName, 

226 ", ".join(self.locationList)) 

227 return s 

228 

229 @staticmethod 

230 def to_yaml(dumper, obj): 

231 """Representer for dumping to YAML 

232 :param dumper: 

233 :param obj: 

234 :return: 

235 """ 

236 return dumper.represent_mapping(ButlerLocation.yaml_tag, 

237 {'pythonType': obj.pythonType, 'cppType': obj.cppType, 

238 'storageName': obj.storageName, 

239 'locationList': obj.locationList, 'mapper': obj.mapper, 

240 'storage': obj.storage, 'dataId': obj.dataId}) 

241 

242 @staticmethod 

243 def from_yaml(loader, node): 

244 obj = loader.construct_mapping(node) 

245 return ButlerLocation(**obj) 

246 

247 def setRepository(self, repository): 

248 self.repository = repository 

249 

250 def getRepository(self): 

251 return self.repository 

252 

253 def getPythonType(self): 

254 return self.pythonType 

255 

256 def getCppType(self): 

257 return self.cppType 

258 

259 def getStorageName(self): 

260 return self.storageName 

261 

262 def getLocations(self): 

263 return self.locationList 

264 

265 def getLocationsWithRoot(self): 

266 return [os.path.join(self.storage.root, loc) for loc in self.getLocations()] 

267 

268 def getAdditionalData(self): 

269 return self.additionalData 

270 

271 def getStorage(self): 

272 return self.storage