Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

#!/usr/bin/env python 

 

# 

# LSST Data Management System 

# Copyright 2008, 2009, 2010 LSST Corporation. 

# 

# This product includes software developed by the 

# LSST Project (http://www.lsst.org/). 

# 

# This program is free software: you can redistribute it and/or modify 

# it under the terms of the GNU General Public License as published by 

# the Free Software Foundation, either version 3 of the License, or 

# (at your option) any later version. 

# 

# This program is distributed in the hope that it will be useful, 

# but WITHOUT ANY WARRANTY; without even the implied warranty of 

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

# GNU General Public License for more details. 

# 

# You should have received a copy of the LSST License Statement and 

# the GNU General Public License along with this program. If not, 

# see <http://www.lsstcorp.org/LegalNotices/>. 

# 

 

# -*- python -*- 

 

"""This module defines the ButlerSubset class and the ButlerDataRefs contained 

within it as well as an iterator over the subset.""" 

from builtins import next 

from builtins import range 

from builtins import object 

 

from . import DataId 

 

 

class ButlerSubset(object): 

 

"""ButlerSubset is a container for ButlerDataRefs. It represents a 

collection of data ids that can be used to obtain datasets of the type 

used when creating the collection or a compatible dataset type. It can be 

thought of as the result of a query for datasets matching a partial data 

id. 

 

The ButlerDataRefs are generated at a specified level of the data id 

hierarchy. If that is not the level at which datasets are specified, the 

ButlerDataRef.subItems() method may be used to dive further into the 

ButlerDataRefs. 

 

ButlerSubsets should generally be created using Butler.subset(). 

 

This mechanism replaces the creation of butlers using partial dataIds. 

 

Public methods: 

 

__init__(self, butler, datasetType, level, dataId) 

 

__len__(self) 

 

__iter__(self) 

 

""" 

 

def __init__(self, butler, datasetType, level, dataId): 

""" 

Create a ButlerSubset by querying a butler for data ids matching a 

given partial data id for a given dataset type at a given hierarchy 

level. 

 

@param butler (Butler) butler that is being queried. 

@param datasetType (str) the type of dataset to query. 

@param level (str) the hierarchy level to descend to. if empty string will look up the default 

level. 

@param dataId (dict) the (partial or complete) data id. 

""" 

self.butler = butler 

self.datasetType = datasetType 

self.dataId = DataId(dataId) 

self.cache = [] 

self.level = level 

 

keys = self.butler.getKeys(datasetType, level, tag=dataId.tag) 

82 ↛ 83line 82 didn't jump to line 83, because the condition on line 82 was never true if keys is None: 

return 

fmt = list(keys.keys()) 

 

# Don't query if we already have a complete dataId 

completeId = True 

for key in fmt: 

if key not in dataId: 

completeId = False 

break 

if completeId: 

self.cache.append(dataId) 

return 

 

idTuples = butler.queryMetadata(self.datasetType, fmt, self.dataId) 

for idTuple in idTuples: 

tempId = dict(self.dataId) 

99 ↛ 100line 99 didn't jump to line 100, because the condition on line 99 was never true if len(fmt) == 1: 

tempId[fmt[0]] = idTuple 

else: 

for i in range(len(fmt)): 

tempId[fmt[i]] = idTuple[i] 

self.cache.append(tempId) 

 

def __repr__(self): 

return "ButlerSubset(butler=%s, datasetType=%s, dataId=%s, cache=%s, level=%s)" % ( 

self.butler, self.datasetType, self.dataId, self.cache, self.level) 

 

def __len__(self): 

""" 

Number of ButlerDataRefs in the ButlerSubset. 

 

@returns (int) 

""" 

 

return len(self.cache) 

 

def __iter__(self): 

""" 

Iterator over the ButlerDataRefs in the ButlerSubset. 

 

@returns (ButlerIterator) 

""" 

 

return ButlerSubsetIterator(self) 

 

 

class ButlerSubsetIterator(object): 

""" 

An iterator over the ButlerDataRefs in a ButlerSubset. 

""" 

 

def __init__(self, butlerSubset): 

self.butlerSubset = butlerSubset 

self.iter = iter(butlerSubset.cache) 

 

def __iter__(self): 

return self 

 

def __next__(self): 

return ButlerDataRef(self.butlerSubset, next(self.iter)) 

 

 

class ButlerDataRef(object): 

""" 

A ButlerDataRef is a reference to a potential dataset or group of datasets 

that is portable between compatible dataset types. As such, it can be 

used to create or retrieve datasets. 

 

ButlerDataRefs are (conceptually) created as elements of a ButlerSubset by 

Butler.subset(). They are initially specific to the dataset type passed 

to that call, but they may be used with any other compatible dataset type. 

Dataset type compatibility must be determined externally (or by trial and 

error). 

 

ButlerDataRefs may be created at any level of a data identifier hierarchy. 

If the level is not one at which datasets exist, a ButlerSubset 

with lower-level ButlerDataRefs can be created using 

ButlerDataRef.subItems(). 

 

Public methods: 

 

get(self, datasetType=None, **rest) 

 

put(self, obj, datasetType=None, **rest) 

 

subItems(self, level=None) 

 

datasetExists(self, datasetType=None, **rest) 

 

getButler(self) 

""" 

 

def __init__(self, butlerSubset, dataId): 

""" 

For internal use only. ButlerDataRefs should only be created by 

ButlerSubset and ButlerSubsetIterator. 

""" 

 

self.butlerSubset = butlerSubset 

self.dataId = dataId 

 

def __repr__(self): 

return 'ButlerDataRef(butlerSubset=%s, dataId=%s)' % (self.butlerSubset, self.dataId) 

 

def get(self, datasetType=None, **rest): 

""" 

Retrieve a dataset of the given type (or the type used when creating 

the ButlerSubset, if None) as specified by the ButlerDataRef. 

 

@param datasetType (str) dataset type to retrieve. 

@param **rest keyword arguments with data identifiers 

@returns object corresponding to the given dataset type. 

""" 

196 ↛ 197line 196 didn't jump to line 197, because the condition on line 196 was never true if datasetType is None: 

datasetType = self.butlerSubset.datasetType 

return self.butlerSubset.butler.get(datasetType, self.dataId, **rest) 

 

def put(self, obj, datasetType=None, doBackup=False, **rest): 

""" 

Persist a dataset of the given type (or the type used when creating 

the ButlerSubset, if None) as specified by the ButlerDataRef. 

 

@param obj object to persist. 

@param datasetType (str) dataset type to persist. 

@param doBackup if True, rename existing instead of overwriting 

@param **rest keyword arguments with data identifiers 

 

WARNING: Setting doBackup=True is not safe for parallel processing, as it 

may be subject to race conditions. 

""" 

 

214 ↛ 215line 214 didn't jump to line 215, because the condition on line 214 was never true if datasetType is None: 

datasetType = self.butlerSubset.datasetType 

self.butlerSubset.butler.put(obj, datasetType, self.dataId, doBackup=doBackup, **rest) 

 

def getUri(self, datasetType=None, write=False, **rest): 

"""Return the URL for a dataset 

 

.. warning:: This is intended only for debugging. The URI should 

never be used for anything other than printing. 

 

.. note:: In the event there are multiple URIs, we return only 

the first. 

 

.. note:: getUri() does not currently support composite datasets. 

 

Parameters 

---------- 

datasetType : `str`, optional 

The dataset type of interest. 

write : `bool`, optional 

Return the URI for writing? 

rest : `dict`, optional 

Keyword arguments for the data id. 

 

Returns 

------- 

uri : `str` 

URI for dataset 

""" 

 

244 ↛ 245line 244 didn't jump to line 245, because the condition on line 244 was never true if datasetType is None: 

datasetType = self.butlerSubset.datasetType 

return self.butlerSubset.butler.getUri(datasetType, self.dataId, write=write, **rest) 

 

def subLevels(self): 

""" 

Return a list of the lower levels of the hierarchy than this 

ButlerDataRef. 

 

@returns (iterable) list of strings with level keys.""" 

 

return set( 

self.butlerSubset.butler.getKeys( 

self.butlerSubset.datasetType, 

tag=self.butlerSubset.dataId.tag).keys() 

) - set( 

self.butlerSubset.butler.getKeys( 

self.butlerSubset.datasetType, 

self.butlerSubset.level, 

tag=self.butlerSubset.dataId.tag).keys() 

) 

 

def subItems(self, level=None): 

""" 

Generate a ButlerSubset at a lower level of the hierarchy than this 

ButlerDataRef, using it as a partial data id. If level is None, a 

default lower level for the original ButlerSubset level and dataset 

type is used. 

 

As currently implemented, the default sublevels for all the 

repositories used by this Butler instance must match for the Butler to 

be able to select a default sublevel to get the subset. 

 

@param level (str) the hierarchy level to descend to. 

@returns (ButlerSubset) resulting from the lower-level query or () if 

there is no lower level. 

""" 

 

282 ↛ 293line 282 didn't jump to line 293, because the condition on line 282 was never false if level is None: 

levelSet = set() 

for repoData in self.butlerSubset.butler._repos.all(): 

levelSet.add(repoData.repo._mapper.getDefaultSubLevel( 

self.butlerSubset.level)) 

287 ↛ 288line 287 didn't jump to line 288, because the condition on line 287 was never true if len(levelSet) > 1: 

raise RuntimeError( 

"Support for multiple levels not implemented.") 

level = levelSet.pop() 

291 ↛ 292line 291 didn't jump to line 292, because the condition on line 291 was never true if level is None: 

return () 

return self.butlerSubset.butler.subset(self.butlerSubset.datasetType, 

level, self.dataId) 

 

def datasetExists(self, datasetType=None, write=False, **rest): 

""" 

Determine if a dataset exists of the given type (or the type used when 

creating the ButlerSubset, if None) as specified by the ButlerDataRef. 

 

@param datasetType (str) dataset type to check. 

@param write (bool) if True, search only in output repositories 

@param **rest keywords arguments with data identifiers 

@returns bool 

""" 

306 ↛ 307line 306 didn't jump to line 307, because the condition on line 306 was never true if datasetType is None: 

datasetType = self.butlerSubset.datasetType 

return self.butlerSubset.butler.datasetExists( 

datasetType, self.dataId, write=write, **rest) 

 

def getButler(self): 

""" 

Return the butler associated with this data reference. 

""" 

return self.butlerSubset.butler