lsst.daf.persistence  13.0-31-g48013df+4
butlerLocation.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 #
4 # LSST Data Management System
5 # Copyright 2008, 2009, 2010 LSST Corporation.
6 #
7 # This product includes software developed by the
8 # LSST Project (http://www.lsst.org/).
9 #
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the LSST License Statement and
21 # the GNU General Public License along with this program. If not,
22 # see <http://www.lsstcorp.org/LegalNotices/>.
23 #
24 
25 
26 """This module defines the ButlerLocation class."""
27 
28 import lsst.daf.base as dafBase
29 
30 from collections import namedtuple
31 import os
32 from past.builtins import basestring
33 import yaml
34 
35 from . import iterify, doImport
36 
37 
38 class ButlerComposite(object):
39  """Initializer
40 
41  Parameters
42  ----------
43  assembler : function object
44  Function object or importable string to a function object that can be called with the assembler
45  signature: (dataId, componentDict, cls).
46  disassembler : function object
47  Function object or importable string to a function object that can be called with the disassembler
48  signature: (object, dataId, componentDict).
49  python : class object
50  A python class object or importable string to a class object that can be used by the assembler to
51  instantiate an object to be returned.
52  dataId : dict or DataId
53  The dataId that is used to look up components.
54  mapper : Mapper instance
55  A reference to the mapper that created this ButlerComposite object.
56  """
57 
58 
59  class ComponentInfo():
60  """Information about a butler composite object. Some details come from the policy and some are filled
61  in by the butler. Component info is used while assembling and disassembling a composite object in
62  butler. It is used as an input to assemblers and disassemblers (which are part of the butler public
63  API).
64 
65  Parameters
66  ----------
67  datasetType : string
68  The datasetType of the component.
69  obj : object instance
70  The python object instance that is this component.
71  setter : string
72  The name of the function in the parent object to set this component.
73  Optional - may be None
74  getter : string
75  The name of the function in the parent object to get this component.
76  Optional - may be None
77  subset : bool
78  If true, indicates that the obj should be a list of objects found via butlerSubset.
79  inputOnly : bool
80  If true, indicates that the obj should not be serialized when performing a butler.put.
81  """
82  def __init__(self, datasetType, obj, setter, getter, subset, inputOnly):
83  self.datasetType = datasetType
84  self.obj = obj
85  self.setter = setter
86  self.getter = getter
87  self.subset = subset
88  self.inputOnly = inputOnly
89 
90  def __repr__(self):
91  return 'ComponentInfo(datasetType:%s, obj:%s, setter:%s, getter:%s, subset:%s)' % \
92  (self.datasetType, self.obj, self.setter, self.getter, self.subset)
93 
94 
95  def __repr__(self):
96  return 'ButlerComposite(assembler:%s, disassembler:%s, python:%s, dataId:%s, mapper:%s, componentInfo:%s, repository:%s)' % \
97  (self.assembler,
98  self.disassembler,
99  self.python,
100  self.dataId,
101  self.mapper,
102  self.componentInfo,
103  self.repository)
104 
105  def __repr__(self):
106  return "ComponentInfo(datasetType=%s, obj=%s, setter=%s, getter=%s)" % (
107  self.datasetType, self.obj, self.setter, self.getter)
108 
109 
110  def __init__(self, assembler, disassembler, python, dataId, mapper):
111  self.assembler = doImport(assembler) if isinstance(assembler, basestring) else assembler
112  self.disassembler = doImport(disassembler) if isinstance(disassembler, basestring) else disassembler
113  self.python = doImport(python) if isinstance(python, basestring) else python
114  self.dataId = dataId
115  self.mapper = mapper
116  self.componentInfo = {}
117  self.repository = None
118 
119  def add(self, id, datasetType, setter, getter, subset, inputOnly):
120  """Add a description of a component needed to fetch the composite dataset.
121 
122  Parameters
123  ----------
124  id : string
125  The name of the component in the policy definition.
126  datasetType : string
127  The name of the datasetType of the component.
128  setter : string or None
129  The name of the function used to set this component into the python type that contains it.
130  Specifying a setter is optional, use None if the setter won't be specified or used.
131  getter : string or None
132  The name of the function used to get this component from the python type that contains it.
133  Specifying a setter is optional, use None if the setter won't be specified or used.
134  subset : bool
135  If true, indicates that the obj should be a list of objects found via butlerSubset.
136  inputOnly : bool
137  If true, indicates that the obj should not be serialized when performing a butler.put.
138  """
139  self.componentInfo[id] = ButlerComposite.ComponentInfo(datasetType=datasetType,
140  obj = None,
141  setter=setter,
142  getter=getter,
143  subset=subset,
144  inputOnly=inputOnly)
145 
146  def __repr__(self):
147  return "ButlerComposite(assembler=%s, disassembler=%s, python=%s, dataId=%s, components=%s)" % (
148  self.assembler, self.disassembler, self.python, self.dataId, self.componentInfo)
149 
150  def setRepository(self, repository):
151  self.repository = repository
152 
153  def getRepository(self):
154  return self.repository
155 
156  def getPythonType(self):
157  return self.python
158 
159 
160 class ButlerLocation(yaml.YAMLObject):
161  """ButlerLocation is a struct-like class that holds information needed to
162  persist and retrieve an object using the LSST Persistence Framework.
163 
164  Mappers should create and return ButlerLocations from their
165  map_{datasetType} methods.
166 
167  Parameters
168  ----------
169  pythonType - string or class instance
170  This is the type of python object that should be created when reading the location.
171 
172  cppType - string or None
173  The type of cpp object represented by the location (optional, may be None)
174 
175  storageName - string
176  The type of storage the object is in or should be place into.
177 
178  locationList - list of string
179  A list of URI to place the object or where the object might be found. (Typically when reading the
180  length is expected to be exactly 1).
181 
182  dataId - dict
183  The dataId that was passed in when mapping the location. This may include keys that were not used for
184  mapping this location.
185 
186  mapper - mapper class instance
187  The mapper object that mapped this location.
188 
189  storage - storage class instance
190  The storage interface that can be used to read or write this location.
191 
192  usedDataId - dict
193  The dataId components that were used to map this location. If the mapper had to look up keys those
194  will be in this dict (even though they may not appear in the dataId parameter). If the dataId
195  parameter contained keys that were not required to map this item then those keys will NOT be in this
196  parameter.
197 
198  datasetType - string
199  The datasetType that this location represents.
200  """
201 
202  yaml_tag = u"!ButlerLocation"
203  yaml_loader = yaml.Loader
204  yaml_dumper = yaml.Dumper
205 
206  def __repr__(self):
207  return \
208  'ButlerLocation(pythonType=%r, cppType=%r, storageName=%r, storage=%r, locationList=%r,' \
209  ' additionalData=%r, mapper=%r, dataId=%r)' % \
210  (self.pythonType, self.cppType, self.storageName, self.storage, self.locationList,
211  self.additionalData, self.mapper, self.dataId)
212 
213  def __init__(self, pythonType, cppType, storageName, locationList, dataId, mapper, storage,
214  usedDataId=None, datasetType=None):
215  # pythonType is sometimes unicode with Python 2 and pybind11; this breaks the interpreter
216  self.pythonType = str(pythonType) if isinstance(pythonType, basestring) else pythonType
217  self.cppType = cppType
218  self.storageName = storageName
219  self.mapper = mapper
220  self.storage = storage
221  self.locationList = iterify(locationList)
222  self.additionalData = dafBase.PropertySet()
223  for k, v in dataId.items():
224  self.additionalData.set(k, v)
225  self.dataId = dataId
226  self.usedDataId = usedDataId
227  self.datasetType = datasetType
228 
229  def __str__(self):
230  s = "%s at %s(%s)" % (self.pythonType, self.storageName,
231  ", ".join(self.locationList))
232  return s
233 
234  @staticmethod
235  def to_yaml(dumper, obj):
236  """Representer for dumping to YAML
237  :param dumper:
238  :param obj:
239  :return:
240  """
241  return dumper.represent_mapping(ButlerLocation.yaml_tag,
242  {'pythonType': obj.pythonType, 'cppType': obj.cppType,
243  'storageName': obj.storageName,
244  'locationList': obj.locationList, 'mapper': obj.mapper,
245  'storage': obj.storage, 'dataId': obj.dataId})
246 
247  @staticmethod
248  def from_yaml(loader, node):
249  obj = loader.construct_mapping(node)
250  return ButlerLocation(**obj)
251 
252  def setRepository(self, repository):
253  self.repository = repository
254 
255  def getRepository(self):
256  return self.repository
257 
258  def getPythonType(self):
259  return self.pythonType
260 
261  def getCppType(self):
262  return self.cppType
263 
264  def getStorageName(self):
265  return self.storageName
266 
267  def getLocations(self):
268  return self.locationList
269 
271  return [os.path.join(self.storage.root, l) for l in self.getLocations()]
272 
273  def getAdditionalData(self):
274  return self.additionalData
275 
276  def getStorage(self):
277  return self.storage
def __init__(self, datasetType, obj, setter, getter, subset, inputOnly)
def __init__(self, assembler, disassembler, python, dataId, mapper)
def __init__(self, pythonType, cppType, storageName, locationList, dataId, mapper, storage, usedDataId=None, datasetType=None)
def doImport(pythonType)
Definition: utils.py:109
def add(self, id, datasetType, setter, getter, subset, inputOnly)