lsst.daf.persistence  13.0-11-gfc17871
 All Classes Namespaces Files Functions Variables Typedefs Friends Macros
butlerLocation.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 #
4 # LSST Data Management System
5 # Copyright 2008, 2009, 2010 LSST Corporation.
6 #
7 # This product includes software developed by the
8 # LSST Project (http://www.lsst.org/).
9 #
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the LSST License Statement and
21 # the GNU General Public License along with this program. If not,
22 # see <http://www.lsstcorp.org/LegalNotices/>.
23 #
24 
25 
26 """This module defines the ButlerLocation class."""
27 
28 import lsst.daf.base as dafBase
29 
30 from collections import namedtuple
31 import os
32 from past.builtins import basestring
33 import yaml
34 
35 from . import iterify, doImport
36 
37 
38 class ButlerComposite(object):
39  """Initializer
40 
41  Parameters
42  ----------
43  assembler : function object
44  Function object or importable string to a function object that can be called with the assembler
45  signature: (dataId, componentDict, cls).
46  disassembler : function object
47  Function object or importable string to a function object that can be called with the disassembler
48  signature: (object, dataId, componentDict).
49  python : class object
50  A python class object or importable string to a class object that can be used by the assembler to
51  instantiate an object to be returned.
52  dataId : dict or DataId
53  The dataId that is used to look up components.
54  mapper : Mapper instance
55  A reference to the mapper that created this ButlerComposite object.
56  """
57 
58 
59  class ComponentInfo():
60  """Information about a butler composite object. Some details come from the policy and some are filled
61  in by the butler. Component info is used while assembling and disassembling a composite object in
62  butler. It is used as an input to assemblers and disassemblers (which are part of the butler public
63  API).
64 
65  Parameters
66  ----------
67  datasetType : string
68  The datasetType of the component.
69  obj : object instance
70  The python object instance that is this component.
71  setter : string
72  The name of the function in the parent object to set this component.
73  Optional - may be None
74  getter : string
75  The name of the function in the parent object to get this component.
76  Optional - may be None
77  subset : bool
78  If true, indicates that the obj should be a list of objects found via butlerSubset.
79  inputOnly : bool
80  If true, indicates that the obj should not be serialized when performing a butler.put.
81  """
82  def __init__(self, datasetType, obj, setter, getter, subset, inputOnly):
83  self.datasetType = datasetType
84  self.obj = obj
85  self.setter = setter
86  self.getter = getter
87  self.subset = subset
88  self.inputOnly = inputOnly
89 
90  def __repr__(self):
91  return 'ComponentInfo(datasetType:%s, obj:%s, setter:%s, getter:%s, subset:%s)' % \
92  (self.datasetType, self.obj, self.setter, self.getter, self.subset)
93 
94 
95  def __repr__(self):
96  return 'ButlerComposite(assembler:%s, disassembler:%s, python:%s, dataId:%s, mapper:%s, componentInfo:%s, repository:%s)' % \
97  (self.assembler,
98  self.disassembler,
99  self.python,
100  self.dataId,
101  self.mapper,
102  self.componentInfo,
103  self.repository)
104 
105  def __repr__(self):
106  return "ComponentInfo(datasetType=%s, obj=%s, setter=%s, getter=%s)" % (
107  self.datasetType, self.obj, self.setter, self.getter)
108 
109 
110  def __init__(self, assembler, disassembler, python, dataId, mapper):
111  self.assembler = doImport(assembler) if isinstance(assembler, basestring) else assembler
112  self.disassembler = doImport(disassembler) if isinstance(disassembler, basestring) else disassembler
113  self.python = doImport(python) if isinstance(python, basestring) else python
114  self.dataId = dataId
115  self.mapper = mapper
116  self.componentInfo = {}
117  self.repository = None
118 
119  def add(self, id, datasetType, setter, getter, subset, inputOnly):
120  """Add a description of a component needed to fetch the composite dataset.
121 
122  Parameters
123  ----------
124  id : string
125  The name of the component in the policy definition.
126  datasetType : string
127  The name of the datasetType of the component.
128  setter : string or None
129  The name of the function used to set this component into the python type that contains it.
130  Specifying a setter is optional, use None if the setter won't be specified or used.
131  getter : string or None
132  The name of the function used to get this component from the python type that contains it.
133  Specifying a setter is optional, use None if the setter won't be specified or used.
134  subset : bool
135  If true, indicates that the obj should be a list of objects found via butlerSubset.
136  inputOnly : bool
137  If true, indicates that the obj should not be serialized when performing a butler.put.
138  """
139  self.componentInfo[id] = ButlerComposite.ComponentInfo(datasetType=datasetType,
140  obj = None,
141  setter=setter,
142  getter=getter,
143  subset=subset,
144  inputOnly=inputOnly)
145 
146  def __repr__(self):
147  return "ButlerComposite(assembler=%s, disassembler=%s, python=%s, dataId=%s, components=%s)" % (
148  self.assembler, self.disassembler, self.python, self.dataId, self.componentInfo)
149 
150  def setRepository(self, repository):
151  self.repository = repository
152 
153  def getRepository(self):
154  return self.repository
155 
156 
157 class ButlerLocation(yaml.YAMLObject):
158  """ButlerLocation is a struct-like class that holds information needed to
159  persist and retrieve an object using the LSST Persistence Framework.
160 
161  Mappers should create and return ButlerLocations from their
162  map_{datasetType} methods.
163 
164  Parameters
165  ----------
166  pythonType - string or class instance
167  This is the type of python object that should be created when reading the location.
168 
169  cppType - string or None
170  The type of cpp object represented by the location (optional, may be None)
171 
172  storageName - string
173  The type of storage the object is in or should be place into.
174 
175  locationList - list of string
176  A list of URI to place the object or where the object might be found. (Typically when reading the
177  length is expected to be exactly 1).
178 
179  dataId - dict
180  The dataId that was passed in when mapping the location. This may include keys that were not used for
181  mapping this location.
182 
183  mapper - mapper class instance
184  The mapper object that mapped this location.
185 
186  storage - storage class instance
187  The storage interface that can be used to read or write this location.
188 
189  usedDataId - dict
190  The dataId components that were used to map this location. If the mapper had to look up keys those
191  will be in this dict (even though they may not appear in the dataId parameter). If the dataId
192  parameter contained keys that were not required to map this item then those keys will NOT be in this
193  parameter.
194 
195  datasetType - string
196  The datasetType that this location represents.
197  """
198 
199  yaml_tag = u"!ButlerLocation"
200  yaml_loader = yaml.Loader
201  yaml_dumper = yaml.Dumper
202 
203  def __repr__(self):
204  return \
205  'ButlerLocation(pythonType=%r, cppType=%r, storageName=%r, storage=%r, locationList=%r,' \
206  ' additionalData=%r, mapper=%r, dataId=%r)' % \
207  (self.pythonType, self.cppType, self.storageName, self.storage, self.locationList,
208  self.additionalData, self.mapper, self.dataId)
209 
210  def __init__(self, pythonType, cppType, storageName, locationList, dataId, mapper, storage,
211  usedDataId=None, datasetType=None):
212  # pythonType is sometimes unicode with Python 2 and pybind11; this breaks the interpreter
213  self.pythonType = str(pythonType) if isinstance(pythonType, basestring) else pythonType
214  self.cppType = cppType
215  self.storageName = storageName
216  self.mapper = mapper
217  self.storage = storage
218  self.locationList = iterify(locationList)
219  self.additionalData = dafBase.PropertySet()
220  for k, v in dataId.items():
221  self.additionalData.set(k, v)
222  self.dataId = dataId
223  self.usedDataId = usedDataId
224  self.datasetType = datasetType
225 
226  def __str__(self):
227  s = "%s at %s(%s)" % (self.pythonType, self.storageName,
228  ", ".join(self.locationList))
229  return s
230 
231  @staticmethod
232  def to_yaml(dumper, obj):
233  """Representer for dumping to YAML
234  :param dumper:
235  :param obj:
236  :return:
237  """
238  return dumper.represent_mapping(ButlerLocation.yaml_tag,
239  {'pythonType': obj.pythonType, 'cppType': obj.cppType,
240  'storageName': obj.storageName,
241  'locationList': obj.locationList, 'mapper': obj.mapper,
242  'storage': obj.storage, 'dataId': obj.dataId})
243 
244  @staticmethod
245  def from_yaml(loader, node):
246  obj = loader.construct_mapping(node)
247  return ButlerLocation(**obj)
248 
249  def setRepository(self, repository):
250  self.repository = repository
251 
252  def getRepository(self):
253  return self.repository
254 
255  def getPythonType(self):
256  return self.pythonType
257 
258  def getCppType(self):
259  return self.cppType
260 
261  def getStorageName(self):
262  return self.storageName
263 
264  def getLocations(self):
265  return self.locationList
266 
268  return [os.path.join(self.storage.root, l) for l in self.getLocations()]
269 
270  def getAdditionalData(self):
271  return self.additionalData
272 
273  def getStorage(self):
274  return self.storage