23 from builtins
import zip
24 from builtins
import object
25 from collections
import OrderedDict
28 from lsst.daf.persistence
import ButlerLocation, NoResults
29 from lsst.daf.persistence.policy
import Policy
30 import lsst.pex.policy
as pexPolicy
32 """This module defines the Mapping base class.""" 37 """Mapping is a base class for all mappings. Mappings are used by 38 the Mapper to map (determine a path to some data given some 39 identifiers) and standardize (convert data into some standard 40 format or type) data, and to query the associated registry to see 41 what data is available. 43 Subclasses must specify self.storage or else override self.map(). 45 Public methods: lookup, have, need, getKeys, map 47 Mappings are specified mainly by policy. A Mapping policy should 50 template (string): a Python string providing the filename for that 51 particular dataset type based on some data identifiers. In the 52 case of redundancy in the path (e.g., file uniquely specified by 53 the exposure number, but filter in the path), the 54 redundant/dependent identifiers can be looked up in the registry. 56 python (string): the Python type for the retrieved data (e.g. 57 lsst.afw.image.ExposureF) 59 persistable (string): the Persistable registration for the on-disk data 62 storage (string, optional): Storage type for this dataset type (e.g. 65 level (string, optional): the level in the camera hierarchy at which the 66 data is stored (Amp, Ccd or skyTile), if relevant 68 tables (string, optional): a whitespace-delimited list of tables in the 69 registry that can be NATURAL JOIN-ed to look up additional 72 def __init__(self, datasetType, policy, registry, rootStorage, provided=None):
73 """Constructor for Mapping class. 74 @param datasetType (string) 75 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility)) 77 @param registry (lsst.obs.base.Registry) Registry for metadata lookups 78 @param rootStorage (Storage subclass instance) Interface to persisted repository data 79 @param provided (list of strings) Keys provided by the mapper 83 raise RuntimeError(
"No policy provided for mapping")
85 if isinstance(policy, pexPolicy.Policy):
86 policy = Policy(policy)
94 (k, _formatMap(v, k, datasetType))
96 re.findall(
r'\%\((\w+)\).*?([diouxXeEfFgGcrs])', self.
template)
98 if provided
is not None:
105 if 'level' in policy:
107 if 'tables' in policy:
113 self.
obsTimeName = policy[
'obsTimeName']
if 'obsTimeName' in policy
else None 116 """Return the dict of keys and value types required for this mapping.""" 119 def map(self, mapper, dataId, write=False):
120 """Standard implementation of map function. 121 @param mapper (lsst.daf.persistence.Mapper) 122 @param dataId (dict) Dataset identifier 123 @return (lsst.daf.persistence.ButlerLocation)""" 125 usedDataId = {key: actualId[key]
for key
in self.
keyDict.
keys()}
126 path = mapper._mapActualToPath(self.
template, actualId)
127 if os.path.isabs(path):
128 raise RuntimeError(
"Mapped path should not be absolute.")
135 for ext
in (
None,
'.gz',
'.fz'):
136 if ext
and path.endswith(ext):
138 extPath = path + ext
if ext
else path
143 assert path,
"Fully-qualified filename is empty." 146 if hasattr(mapper, addFunc):
147 addFunc = getattr(mapper, addFunc)
148 additionalData = addFunc(actualId)
149 assert isinstance(additionalData, dict),
"Bad type for returned data" 151 additionalData = actualId.copy()
154 locationList=path, dataId=additionalData, mapper=mapper,
158 """Look up properties for in a metadata registry given a partial 160 @param properties (list of strings) 161 @param dataId (dict) Dataset identifier 162 @return (list of tuples) values of properties""" 165 raise RuntimeError(
"No registry for lookup")
167 skyMapKeys = (
"tract",
"patch")
179 substitutions = OrderedDict()
181 properties = list(properties)
185 substitutions[p] = dataId[p]
189 "Cannot look up skymap key '%s'; it must be explicitly included in the data ID" % p
192 substitutions[p] = index
200 if p
not in (
'filter',
'expTime',
'taiObs'):
203 if fastPath
and 'visit' in dataId
and "raw" in self.
tables:
204 lookupDataId = {
'visit': dataId[
'visit']}
207 if dataId
is not None:
208 for k, v
in dataId.items():
215 where.append((k,
'?'))
217 lookupDataId = {k[0]: v
for k, v
in zip(where, values)}
226 result = [tuple(v
if k
in removed
else item[v]
for k, v
in substitutions.items())
230 def have(self, properties, dataId):
231 """Returns whether the provided data identifier has all 232 the properties in the provided list. 233 @param properties (list of strings) Properties required 234 @parm dataId (dict) Dataset identifier 235 @return (bool) True if all properties are present""" 236 for prop
in properties:
237 if prop
not in dataId:
241 def need(self, properties, dataId):
242 """Ensures all properties in the provided list are present in 243 the data identifier, looking them up as needed. This is only 244 possible for the case where the data identifies a single 246 @param properties (list of strings) Properties required 247 @param dataId (dict) Partial dataset identifier 248 @return (dict) copy of dataset identifier with enhanced values 250 newId = dataId.copy()
252 for prop
in properties:
253 if prop
not in newId:
254 newProps.append(prop)
255 if len(newProps) == 0:
258 lookups = self.
lookup(newProps, newId)
259 if len(lookups) != 1:
260 raise NoResults(
"No unique lookup for %s from %s: %d matches" %
261 (newProps, newId, len(lookups)),
263 for i, prop
in enumerate(newProps):
264 newId[prop] = lookups[0][i]
268 def _formatMap(ch, k, datasetType):
269 """Convert a format character into a Python type.""" 277 raise RuntimeError(
"Unexpected format specifier %s" 278 " for field %s in template for dataset %s" %
279 (ch, k, datasetType))
283 """ImageMapping is a Mapping subclass for non-camera images.""" 285 def __init__(self, datasetType, policy, registry, root, **kwargs):
286 """Constructor for Mapping class. 287 @param datasetType (string) 288 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility)) 290 @param registry (lsst.obs.base.Registry) Registry for metadata lookups 291 @param root (string) Path of root directory""" 292 if isinstance(policy, pexPolicy.Policy):
293 policy = Policy(policy)
294 Mapping.__init__(self, datasetType, policy, registry, root, **kwargs)
295 self.
columns = policy.asArray(
'columns')
if 'columns' in policy
else None 299 """ExposureMapping is a Mapping subclass for normal exposures.""" 301 def __init__(self, datasetType, policy, registry, root, **kwargs):
302 """Constructor for Mapping class. 303 @param datasetType (string) 304 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility)) 306 @param registry (lsst.obs.base.Registry) Registry for metadata lookups 307 @param root (string) Path of root directory""" 308 if isinstance(policy, pexPolicy.Policy):
309 policy = Policy(policy)
310 Mapping.__init__(self, datasetType, policy, registry, root, **kwargs)
311 self.
columns = policy.asArray(
'columns')
if 'columns' in policy
else None 314 return mapper._standardizeExposure(self, item, dataId)
318 """CalibrationMapping is a Mapping subclass for calibration-type products. 320 The difference is that data properties in the query or template 321 can be looked up using a reference Mapping in addition to this one. 323 CalibrationMapping Policies can contain the following: 325 reference (string, optional): a list of tables for finding missing dataset 326 identifier components (including the observation time, if a validity range 327 is required) in the exposure registry; note that the "tables" entry refers 328 to the calibration registry 330 refCols (string, optional): a list of dataset properties required from the 331 reference tables for lookups in the calibration registry 333 validRange (bool): true if the calibration dataset has a validity range 334 specified by a column in the tables of the reference dataset in the 335 exposure registry) and two columns in the tables of this calibration 336 dataset in the calibration registry) 338 obsTimeName (string, optional): the name of the column in the reference 339 dataset tables containing the observation time (default "taiObs") 341 validStartName (string, optional): the name of the column in the 342 calibration dataset tables containing the start of the validity range 343 (default "validStart") 345 validEndName (string, optional): the name of the column in the 346 calibration dataset tables containing the end of the validity range 347 (default "validEnd") """ 349 def __init__(self, datasetType, policy, registry, calibRegistry, calibRoot, dataRoot=None, **kwargs):
350 """Constructor for Mapping class. 351 @param datasetType (string) 352 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility)) 354 @param registry (lsst.obs.base.Registry) Registry for metadata lookups 355 @param calibRegistry (lsst.obs.base.Registry) Registry for calibration metadata lookups 356 @param calibRoot (string) Path of calibration root directory 357 @param dataRoot. (string) Path of data root directory; used for outputs only 359 if isinstance(policy, pexPolicy.Policy):
360 policy = Policy(policy)
361 Mapping.__init__(self, datasetType, policy, calibRegistry, calibRoot, **kwargs)
362 self.
reference = policy.asArray(
"reference")
if "reference" in policy
else None 363 self.
refCols = policy.asArray(
"refCols")
if "refCols" in policy
else None 366 if "validRange" in policy
and policy[
"validRange"]:
367 self.
range = (
"?", policy[
"validStartName"], policy[
"validEndName"])
368 if "columns" in policy:
370 if "filter" in policy:
373 if "metadataKey" in policy:
376 def map(self, mapper, dataId, write=False):
377 location = Mapping.map(self, mapper, dataId, write=write)
384 """Look up properties for in a metadata registry given a partial 386 @param properties (list of strings) 387 @param dataId (dict) Dataset identifier 388 @return (list of tuples) values of properties""" 393 newId = dataId.copy()
397 for k, v
in dataId.items():
406 for k
in dataId.keys():
409 columns = set(properties)
413 return Mapping.lookup(self, properties, newId)
415 lookupDataId = dict(zip(where, values))
417 if len(lookups) != 1:
418 raise RuntimeError(
"No unique lookup for %s from %s: %d matches" %
419 (columns, dataId, len(lookups)))
420 if columns == set(properties):
423 for i, prop
in enumerate(columns):
424 newId[prop] = lookups[0][i]
425 return Mapping.lookup(self, properties, newId)
428 return mapper._standardizeExposure(self, item, dataId, filter=self.
setFilter)
432 """DatasetMapping is a Mapping subclass for non-Exposure datasets that can 433 be retrieved by the standard daf_persistence mechanism. 435 The differences are that the Storage type must be specified and no 436 Exposure standardization is performed. 438 The "storage" entry in the Policy is mandatory; the "tables" entry is 439 optional; no "level" entry is allowed. """ 441 def __init__(self, datasetType, policy, registry, root, **kwargs):
442 """Constructor for DatasetMapping class. 443 @param[in,out] mapper (lsst.daf.persistence.Mapper) Mapper object 444 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility)) 446 @param datasetType (string) 447 @param registry (lsst.obs.base.Registry) Registry for metadata lookups 448 @param root (string) Path of root directory""" 449 if isinstance(policy, pexPolicy.Policy):
450 policy = Policy(policy)
451 Mapping.__init__(self, datasetType, policy, registry, root, **kwargs)
def __init__(self, datasetType, policy, registry, root, kwargs)
def __init__(self, datasetType, policy, registry, calibRegistry, calibRoot, dataRoot=None, kwargs)
def __init__(self, datasetType, policy, registry, root, kwargs)
def standardize(self, mapper, item, dataId)
def map(self, mapper, dataId, write=False)
def have(self, properties, dataId)
def standardize(self, mapper, item, dataId)
def need(self, properties, dataId)
def lookup(self, properties, dataId)
def __init__(self, datasetType, policy, registry, root, kwargs)
def lookup(self, properties, dataId)
def __init__(self, datasetType, policy, registry, rootStorage, provided=None)
def map(self, mapper, dataId, write=False)