Coverage for python/lsst/daf/persistence/butlerLocation.py: 45%
84 statements
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-11 02:48 -0700
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-11 02:48 -0700
1#!/usr/bin/env python
3#
4# LSST Data Management System
5# Copyright 2008, 2009, 2010 LSST Corporation.
6#
7# This product includes software developed by the
8# LSST Project (http://www.lsst.org/).
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the LSST License Statement and
21# the GNU General Public License along with this program. If not,
22# see <http://www.lsstcorp.org/LegalNotices/>.
23#
26"""This module defines the ButlerLocation class."""
28import lsst.daf.base as dafBase
30import os
31import yaml
33from . import iterify, doImport
36class ButlerComposite:
37 """Initializer
39 Parameters
40 ----------
41 assembler : function object
42 Function object or importable string to a function object that can be called with the assembler
43 signature: (dataId, componentDict, cls).
44 disassembler : function object
45 Function object or importable string to a function object that can be called with the disassembler
46 signature: (object, dataId, componentDict).
47 python : class object
48 A python class object or importable string to a class object that can be used by the assembler to
49 instantiate an object to be returned.
50 dataId : dict or DataId
51 The dataId that is used to look up components.
52 mapper : Mapper instance
53 A reference to the mapper that created this ButlerComposite object.
54 """
56 class ComponentInfo():
57 """Information about a butler composite object. Some details come from the policy and some are filled
58 in by the butler. Component info is used while assembling and disassembling a composite object in
59 butler. It is used as an input to assemblers and disassemblers (which are part of the butler public
60 API).
62 Parameters
63 ----------
64 datasetType : string
65 The datasetType of the component.
66 obj : object instance
67 The python object instance that is this component.
68 setter : string
69 The name of the function in the parent object to set this component.
70 Optional - may be None
71 getter : string
72 The name of the function in the parent object to get this component.
73 Optional - may be None
74 subset : bool
75 If true, indicates that the obj should be a list of objects found via butlerSubset.
76 inputOnly : bool
77 If true, indicates that the obj should not be serialized when performing a butler.put.
78 """
79 def __init__(self, datasetType, obj, setter, getter, subset, inputOnly):
80 self.datasetType = datasetType
81 self.obj = obj
82 self.setter = setter
83 self.getter = getter
84 self.subset = subset
85 self.inputOnly = inputOnly
87 def __repr__(self):
88 return 'ComponentInfo(datasetType:%s, obj:%s, setter:%s, getter:%s, subset:%s)' % \
89 (self.datasetType, self.obj, self.setter, self.getter, self.subset)
91 def __repr__(self):
92 return 'ButlerComposite(assembler:%s, disassembler:%s, python:%s, dataId:%s, mapper:%s, ' \
93 'componentInfo:%s, repository:%s)' % \
94 (self.assembler,
95 self.disassembler,
96 self.python,
97 self.dataId,
98 self.mapper,
99 self.componentInfo,
100 self.repository)
102 def __init__(self, assembler, disassembler, python, dataId, mapper):
103 self.assembler = doImport(assembler) if isinstance(assembler, str) else assembler
104 self.disassembler = doImport(disassembler) if isinstance(disassembler, str) else disassembler
105 self.python = doImport(python) if isinstance(python, str) else python
106 self.dataId = dataId
107 self.mapper = mapper
108 self.componentInfo = {}
109 self.repository = None
111 def add(self, id, datasetType, setter, getter, subset, inputOnly):
112 """Add a description of a component needed to fetch the composite dataset.
114 Parameters
115 ----------
116 id : string
117 The name of the component in the policy definition.
118 datasetType : string
119 The name of the datasetType of the component.
120 setter : string or None
121 The name of the function used to set this component into the python type that contains it.
122 Specifying a setter is optional, use None if the setter won't be specified or used.
123 getter : string or None
124 The name of the function used to get this component from the python type that contains it.
125 Specifying a setter is optional, use None if the setter won't be specified or used.
126 subset : bool
127 If true, indicates that the obj should be a list of objects found via butlerSubset.
128 inputOnly : bool
129 If true, indicates that the obj should not be serialized when performing a butler.put.
130 """
131 self.componentInfo[id] = ButlerComposite.ComponentInfo(datasetType=datasetType,
132 obj=None,
133 setter=setter,
134 getter=getter,
135 subset=subset,
136 inputOnly=inputOnly)
138 def setRepository(self, repository):
139 self.repository = repository
141 def getRepository(self):
142 return self.repository
144 def getPythonType(self):
145 return self.python
148class ButlerLocation(yaml.YAMLObject):
149 """ButlerLocation is a struct-like class that holds information needed to
150 persist and retrieve an object using the LSST Persistence Framework.
152 Mappers should create and return ButlerLocations from their
153 map_{datasetType} methods.
155 Parameters
156 ----------
157 pythonType - string or class instance
158 This is the type of python object that should be created when reading the location.
160 cppType - string or None
161 The type of cpp object represented by the location (optional, may be None)
163 storageName - string
164 The type of storage the object is in or should be place into.
166 locationList - list of string
167 A list of URI to place the object or where the object might be found. (Typically when reading the
168 length is expected to be exactly 1).
170 dataId - dict
171 The dataId that was passed in when mapping the location. This may include keys that were not used for
172 mapping this location.
174 mapper - mapper class instance
175 The mapper object that mapped this location.
177 storage - storage class instance
178 The storage interface that can be used to read or write this location.
180 usedDataId - dict
181 The dataId components that were used to map this location. If the mapper had to look up keys those
182 will be in this dict (even though they may not appear in the dataId parameter). If the dataId
183 parameter contained keys that were not required to map this item then those keys will NOT be in this
184 parameter.
186 datasetType - string
187 The datasetType that this location represents.
189 additionalData : `lsst.daf.base.PropertySet`, optional
190 Additional metadata to be passed to the persistence framework,
191 or `None`.
192 """
194 yaml_tag = u"!ButlerLocation"
195 try:
196 # PyYAML >=5.1 prefers a different loader
197 yaml_loader = yaml.UnsafeLoader
198 except AttributeError:
199 yaml_loader = yaml.Loader
200 yaml_dumper = yaml.Dumper
202 def __repr__(self):
203 return \
204 'ButlerLocation(pythonType=%r, cppType=%r, storageName=%r, storage=%r, locationList=%r,' \
205 ' additionalData=%r, mapper=%r, dataId=%r)' % \
206 (self.pythonType, self.cppType, self.storageName, self.storage, self.locationList,
207 self.additionalData, self.mapper, self.dataId)
209 def __init__(self, pythonType, cppType, storageName, locationList, dataId, mapper, storage,
210 usedDataId=None, datasetType=None, additionalData=None):
211 self.pythonType = pythonType
212 self.cppType = cppType
213 self.storageName = storageName
214 self.mapper = mapper
215 self.storage = storage
216 self.locationList = iterify(locationList)
217 self.additionalData = additionalData if additionalData else dafBase.PropertySet()
218 for k, v in dataId.items():
219 self.additionalData.set(k, v)
220 self.dataId = dataId
221 self.usedDataId = usedDataId
222 self.datasetType = datasetType
224 def __str__(self):
225 s = "%s at %s(%s)" % (self.pythonType, self.storageName,
226 ", ".join(self.locationList))
227 return s
229 @staticmethod
230 def to_yaml(dumper, obj):
231 """Representer for dumping to YAML
232 :param dumper:
233 :param obj:
234 :return:
235 """
236 return dumper.represent_mapping(ButlerLocation.yaml_tag,
237 {'pythonType': obj.pythonType, 'cppType': obj.cppType,
238 'storageName': obj.storageName,
239 'locationList': obj.locationList, 'mapper': obj.mapper,
240 'storage': obj.storage, 'dataId': obj.dataId})
242 @staticmethod
243 def from_yaml(loader, node):
244 obj = loader.construct_mapping(node)
245 return ButlerLocation(**obj)
247 def setRepository(self, repository):
248 self.repository = repository
250 def getRepository(self):
251 return self.repository
253 def getPythonType(self):
254 return self.pythonType
256 def getCppType(self):
257 return self.cppType
259 def getStorageName(self):
260 return self.storageName
262 def getLocations(self):
263 return self.locationList
265 def getLocationsWithRoot(self):
266 return [os.path.join(self.storage.root, loc) for loc in self.getLocations()]
268 def getAdditionalData(self):
269 return self.additionalData
271 def getStorage(self):
272 return self.storage