lsst.daf.persistence  13.0-32-g7b14ddd
 All Classes Namespaces Files Functions Variables Typedefs Friends Macros
butlerLocation.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 #
4 # LSST Data Management System
5 # Copyright 2008, 2009, 2010 LSST Corporation.
6 #
7 # This product includes software developed by the
8 # LSST Project (http://www.lsst.org/).
9 #
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the LSST License Statement and
21 # the GNU General Public License along with this program. If not,
22 # see <http://www.lsstcorp.org/LegalNotices/>.
23 #
24 
25 
26 """This module defines the ButlerLocation class."""
27 
28 import lsst.daf.base as dafBase
29 
30 import os
31 from past.builtins import basestring
32 import yaml
33 
34 from . import iterify, doImport
35 
36 
37 class ButlerComposite(object):
38  """Initializer
39 
40  Parameters
41  ----------
42  assembler : function object
43  Function object or importable string to a function object that can be called with the assembler
44  signature: (dataId, componentDict, cls).
45  disassembler : function object
46  Function object or importable string to a function object that can be called with the disassembler
47  signature: (object, dataId, componentDict).
48  python : class object
49  A python class object or importable string to a class object that can be used by the assembler to
50  instantiate an object to be returned.
51  dataId : dict or DataId
52  The dataId that is used to look up components.
53  mapper : Mapper instance
54  A reference to the mapper that created this ButlerComposite object.
55  """
56 
57  class ComponentInfo():
58  """Information about a butler composite object. Some details come from the policy and some are filled
59  in by the butler. Component info is used while assembling and disassembling a composite object in
60  butler. It is used as an input to assemblers and disassemblers (which are part of the butler public
61  API).
62 
63  Parameters
64  ----------
65  datasetType : string
66  The datasetType of the component.
67  obj : object instance
68  The python object instance that is this component.
69  setter : string
70  The name of the function in the parent object to set this component.
71  Optional - may be None
72  getter : string
73  The name of the function in the parent object to get this component.
74  Optional - may be None
75  subset : bool
76  If true, indicates that the obj should be a list of objects found via butlerSubset.
77  inputOnly : bool
78  If true, indicates that the obj should not be serialized when performing a butler.put.
79  """
80  def __init__(self, datasetType, obj, setter, getter, subset, inputOnly):
81  self.datasetType = datasetType
82  self.obj = obj
83  self.setter = setter
84  self.getter = getter
85  self.subset = subset
86  self.inputOnly = inputOnly
87 
88  def __repr__(self):
89  return 'ComponentInfo(datasetType:%s, obj:%s, setter:%s, getter:%s, subset:%s)' % \
90  (self.datasetType, self.obj, self.setter, self.getter, self.subset)
91 
92  def __repr__(self):
93  return 'ButlerComposite(assembler:%s, disassembler:%s, python:%s, dataId:%s, mapper:%s, ' \
94  'componentInfo:%s, repository:%s)' % \
95  (self.assembler,
96  self.disassembler,
97  self.python,
98  self.dataId,
99  self.mapper,
100  self.componentInfo,
101  self.repository)
102 
103  def __init__(self, assembler, disassembler, python, dataId, mapper):
104  self.assembler = doImport(assembler) if isinstance(assembler, basestring) else assembler
105  self.disassembler = doImport(disassembler) if isinstance(disassembler, basestring) else disassembler
106  self.python = doImport(python) if isinstance(python, basestring) else python
107  self.dataId = dataId
108  self.mapper = mapper
109  self.componentInfo = {}
110  self.repository = None
111 
112  def add(self, id, datasetType, setter, getter, subset, inputOnly):
113  """Add a description of a component needed to fetch the composite dataset.
114 
115  Parameters
116  ----------
117  id : string
118  The name of the component in the policy definition.
119  datasetType : string
120  The name of the datasetType of the component.
121  setter : string or None
122  The name of the function used to set this component into the python type that contains it.
123  Specifying a setter is optional, use None if the setter won't be specified or used.
124  getter : string or None
125  The name of the function used to get this component from the python type that contains it.
126  Specifying a setter is optional, use None if the setter won't be specified or used.
127  subset : bool
128  If true, indicates that the obj should be a list of objects found via butlerSubset.
129  inputOnly : bool
130  If true, indicates that the obj should not be serialized when performing a butler.put.
131  """
132  self.componentInfo[id] = ButlerComposite.ComponentInfo(datasetType=datasetType,
133  obj=None,
134  setter=setter,
135  getter=getter,
136  subset=subset,
137  inputOnly=inputOnly)
138 
139  def setRepository(self, repository):
140  self.repository = repository
141 
142  def getRepository(self):
143  return self.repository
144 
145  def getPythonType(self):
146  return self.python
147 
148 
149 class ButlerLocation(yaml.YAMLObject):
150  """ButlerLocation is a struct-like class that holds information needed to
151  persist and retrieve an object using the LSST Persistence Framework.
152 
153  Mappers should create and return ButlerLocations from their
154  map_{datasetType} methods.
155 
156  Parameters
157  ----------
158  pythonType - string or class instance
159  This is the type of python object that should be created when reading the location.
160 
161  cppType - string or None
162  The type of cpp object represented by the location (optional, may be None)
163 
164  storageName - string
165  The type of storage the object is in or should be place into.
166 
167  locationList - list of string
168  A list of URI to place the object or where the object might be found. (Typically when reading the
169  length is expected to be exactly 1).
170 
171  dataId - dict
172  The dataId that was passed in when mapping the location. This may include keys that were not used for
173  mapping this location.
174 
175  mapper - mapper class instance
176  The mapper object that mapped this location.
177 
178  storage - storage class instance
179  The storage interface that can be used to read or write this location.
180 
181  usedDataId - dict
182  The dataId components that were used to map this location. If the mapper had to look up keys those
183  will be in this dict (even though they may not appear in the dataId parameter). If the dataId
184  parameter contained keys that were not required to map this item then those keys will NOT be in this
185  parameter.
186 
187  datasetType - string
188  The datasetType that this location represents.
189  """
190 
191  yaml_tag = u"!ButlerLocation"
192  yaml_loader = yaml.Loader
193  yaml_dumper = yaml.Dumper
194 
195  def __repr__(self):
196  return \
197  'ButlerLocation(pythonType=%r, cppType=%r, storageName=%r, storage=%r, locationList=%r,' \
198  ' additionalData=%r, mapper=%r, dataId=%r)' % \
199  (self.pythonType, self.cppType, self.storageName, self.storage, self.locationList,
200  self.additionalData, self.mapper, self.dataId)
201 
202  def __init__(self, pythonType, cppType, storageName, locationList, dataId, mapper, storage,
203  usedDataId=None, datasetType=None):
204  # pythonType is sometimes unicode with Python 2 and pybind11; this breaks the interpreter
205  self.pythonType = str(pythonType) if isinstance(pythonType, basestring) else pythonType
206  self.cppType = cppType
207  self.storageName = storageName
208  self.mapper = mapper
209  self.storage = storage
210  self.locationList = iterify(locationList)
211  self.additionalData = dafBase.PropertySet()
212  for k, v in dataId.items():
213  self.additionalData.set(k, v)
214  self.dataId = dataId
215  self.usedDataId = usedDataId
216  self.datasetType = datasetType
217 
218  def __str__(self):
219  s = "%s at %s(%s)" % (self.pythonType, self.storageName,
220  ", ".join(self.locationList))
221  return s
222 
223  @staticmethod
224  def to_yaml(dumper, obj):
225  """Representer for dumping to YAML
226  :param dumper:
227  :param obj:
228  :return:
229  """
230  return dumper.represent_mapping(ButlerLocation.yaml_tag,
231  {'pythonType': obj.pythonType, 'cppType': obj.cppType,
232  'storageName': obj.storageName,
233  'locationList': obj.locationList, 'mapper': obj.mapper,
234  'storage': obj.storage, 'dataId': obj.dataId})
235 
236  @staticmethod
237  def from_yaml(loader, node):
238  obj = loader.construct_mapping(node)
239  return ButlerLocation(**obj)
240 
241  def setRepository(self, repository):
242  self.repository = repository
243 
244  def getRepository(self):
245  return self.repository
246 
247  def getPythonType(self):
248  return self.pythonType
249 
250  def getCppType(self):
251  return self.cppType
252 
253  def getStorageName(self):
254  return self.storageName
255 
256  def getLocations(self):
257  return self.locationList
258 
260  return [os.path.join(self.storage.root, l) for l in self.getLocations()]
261 
262  def getAdditionalData(self):
263  return self.additionalData
264 
265  def getStorage(self):
266  return self.storage