23 from builtins
import zip
24 from builtins
import object
25 from collections
import OrderedDict
28 from lsst.daf.base
import PropertySet
29 from lsst.daf.persistence
import ButlerLocation, NoResults
30 from lsst.daf.persistence.policy
import Policy
31 import lsst.pex.policy
as pexPolicy
33 """This module defines the Mapping base class.""" 38 """Mapping is a base class for all mappings. Mappings are used by 39 the Mapper to map (determine a path to some data given some 40 identifiers) and standardize (convert data into some standard 41 format or type) data, and to query the associated registry to see 42 what data is available. 44 Subclasses must specify self.storage or else override self.map(). 46 Public methods: lookup, have, need, getKeys, map 48 Mappings are specified mainly by policy. A Mapping policy should 51 template (string): a Python string providing the filename for that 52 particular dataset type based on some data identifiers. In the 53 case of redundancy in the path (e.g., file uniquely specified by 54 the exposure number, but filter in the path), the 55 redundant/dependent identifiers can be looked up in the registry. 57 python (string): the Python type for the retrieved data (e.g. 58 lsst.afw.image.ExposureF) 60 persistable (string): the Persistable registration for the on-disk data 63 storage (string, optional): Storage type for this dataset type (e.g. 66 level (string, optional): the level in the camera hierarchy at which the 67 data is stored (Amp, Ccd or skyTile), if relevant 69 tables (string, optional): a whitespace-delimited list of tables in the 70 registry that can be NATURAL JOIN-ed to look up additional 73 def __init__(self, datasetType, policy, registry, rootStorage, provided=None):
74 """Constructor for Mapping class. 75 @param datasetType (string) 76 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility)) 78 @param registry (lsst.obs.base.Registry) Registry for metadata lookups 79 @param rootStorage (Storage subclass instance) Interface to persisted repository data 80 @param provided (list of strings) Keys provided by the mapper 84 raise RuntimeError(
"No policy provided for mapping")
86 if isinstance(policy, pexPolicy.Policy):
87 policy = Policy(policy)
101 (k, _formatMap(v, k, datasetType))
103 re.findall(
r'\%\((\w+)\).*?([diouxXeEfFgGcrs])', self.
template)
107 if provided
is not None:
114 if 'level' in policy:
116 if 'tables' in policy:
122 self.
obsTimeName = policy[
'obsTimeName']
if 'obsTimeName' in policy
else None 123 self.
recipe = policy[
'recipe']
if 'recipe' in policy
else 'default' 130 raise RuntimeError(
"Template is not defined for the {} dataset type, ".format(self.
datasetType) +
131 "it must be set before it can be used.")
134 """Return the dict of keys and value types required for this mapping.""" 137 def map(self, mapper, dataId, write=False):
138 """Standard implementation of map function. 139 @param mapper (lsst.daf.persistence.Mapper) 140 @param dataId (dict) Dataset identifier 141 @return (lsst.daf.persistence.ButlerLocation)""" 143 usedDataId = {key: actualId[key]
for key
in self.
keyDict.
keys()}
144 path = mapper._mapActualToPath(self.
template, actualId)
145 if os.path.isabs(path):
146 raise RuntimeError(
"Mapped path should not be absolute.")
153 for ext
in (
None,
'.gz',
'.fz'):
154 if ext
and path.endswith(ext):
156 extPath = path + ext
if ext
else path
161 assert path,
"Fully-qualified filename is empty." 164 if hasattr(mapper, addFunc):
165 addFunc = getattr(mapper, addFunc)
166 additionalData = addFunc(self.
datasetType, actualId)
167 assert isinstance(additionalData, PropertySet), \
168 "Bad type for returned data: %s" (type(additionalData),)
170 additionalData =
None 173 locationList=path, dataId=actualId.copy(), mapper=mapper,
175 additionalData=additionalData)
178 """Look up properties for in a metadata registry given a partial 180 @param properties (list of strings) 181 @param dataId (dict) Dataset identifier 182 @return (list of tuples) values of properties""" 185 raise RuntimeError(
"No registry for lookup")
187 skyMapKeys = (
"tract",
"patch")
199 substitutions = OrderedDict()
201 properties = list(properties)
205 substitutions[p] = dataId[p]
209 "Cannot look up skymap key '%s'; it must be explicitly included in the data ID" % p
212 substitutions[p] = index
220 if p
not in (
'filter',
'expTime',
'taiObs'):
223 if fastPath
and 'visit' in dataId
and "raw" in self.
tables:
224 lookupDataId = {
'visit': dataId[
'visit']}
227 if dataId
is not None:
228 for k, v
in dataId.items():
235 where.append((k,
'?'))
237 lookupDataId = {k[0]: v
for k, v
in zip(where, values)}
246 result = [tuple(v
if k
in removed
else item[v]
for k, v
in substitutions.items())
250 def have(self, properties, dataId):
251 """Returns whether the provided data identifier has all 252 the properties in the provided list. 253 @param properties (list of strings) Properties required 254 @parm dataId (dict) Dataset identifier 255 @return (bool) True if all properties are present""" 256 for prop
in properties:
257 if prop
not in dataId:
261 def need(self, properties, dataId):
262 """Ensures all properties in the provided list are present in 263 the data identifier, looking them up as needed. This is only 264 possible for the case where the data identifies a single 266 @param properties (list of strings) Properties required 267 @param dataId (dict) Partial dataset identifier 268 @return (dict) copy of dataset identifier with enhanced values 270 newId = dataId.copy()
272 for prop
in properties:
273 if prop
not in newId:
274 newProps.append(prop)
275 if len(newProps) == 0:
278 lookups = self.
lookup(newProps, newId)
279 if len(lookups) != 1:
280 raise NoResults(
"No unique lookup for %s from %s: %d matches" %
281 (newProps, newId, len(lookups)),
283 for i, prop
in enumerate(newProps):
284 newId[prop] = lookups[0][i]
288 def _formatMap(ch, k, datasetType):
289 """Convert a format character into a Python type.""" 297 raise RuntimeError(
"Unexpected format specifier %s" 298 " for field %s in template for dataset %s" %
299 (ch, k, datasetType))
303 """ImageMapping is a Mapping subclass for non-camera images.""" 305 def __init__(self, datasetType, policy, registry, root, **kwargs):
306 """Constructor for Mapping class. 307 @param datasetType (string) 308 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility)) 310 @param registry (lsst.obs.base.Registry) Registry for metadata lookups 311 @param root (string) Path of root directory""" 312 if isinstance(policy, pexPolicy.Policy):
313 policy = Policy(policy)
314 Mapping.__init__(self, datasetType, policy, registry, root, **kwargs)
315 self.
columns = policy.asArray(
'columns')
if 'columns' in policy
else None 319 """ExposureMapping is a Mapping subclass for normal exposures.""" 321 def __init__(self, datasetType, policy, registry, root, **kwargs):
322 """Constructor for Mapping class. 323 @param datasetType (string) 324 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility)) 326 @param registry (lsst.obs.base.Registry) Registry for metadata lookups 327 @param root (string) Path of root directory""" 328 if isinstance(policy, pexPolicy.Policy):
329 policy = Policy(policy)
330 Mapping.__init__(self, datasetType, policy, registry, root, **kwargs)
331 self.
columns = policy.asArray(
'columns')
if 'columns' in policy
else None 334 return mapper._standardizeExposure(self, item, dataId)
338 """CalibrationMapping is a Mapping subclass for calibration-type products. 340 The difference is that data properties in the query or template 341 can be looked up using a reference Mapping in addition to this one. 343 CalibrationMapping Policies can contain the following: 345 reference (string, optional): a list of tables for finding missing dataset 346 identifier components (including the observation time, if a validity range 347 is required) in the exposure registry; note that the "tables" entry refers 348 to the calibration registry 350 refCols (string, optional): a list of dataset properties required from the 351 reference tables for lookups in the calibration registry 353 validRange (bool): true if the calibration dataset has a validity range 354 specified by a column in the tables of the reference dataset in the 355 exposure registry) and two columns in the tables of this calibration 356 dataset in the calibration registry) 358 obsTimeName (string, optional): the name of the column in the reference 359 dataset tables containing the observation time (default "taiObs") 361 validStartName (string, optional): the name of the column in the 362 calibration dataset tables containing the start of the validity range 363 (default "validStart") 365 validEndName (string, optional): the name of the column in the 366 calibration dataset tables containing the end of the validity range 367 (default "validEnd") """ 369 def __init__(self, datasetType, policy, registry, calibRegistry, calibRoot, dataRoot=None, **kwargs):
370 """Constructor for Mapping class. 371 @param datasetType (string) 372 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility)) 374 @param registry (lsst.obs.base.Registry) Registry for metadata lookups 375 @param calibRegistry (lsst.obs.base.Registry) Registry for calibration metadata lookups 376 @param calibRoot (string) Path of calibration root directory 377 @param dataRoot. (string) Path of data root directory; used for outputs only 379 if isinstance(policy, pexPolicy.Policy):
380 policy = Policy(policy)
381 Mapping.__init__(self, datasetType, policy, calibRegistry, calibRoot, **kwargs)
382 self.
reference = policy.asArray(
"reference")
if "reference" in policy
else None 383 self.
refCols = policy.asArray(
"refCols")
if "refCols" in policy
else None 386 if "validRange" in policy
and policy[
"validRange"]:
387 self.
range = (
"?", policy[
"validStartName"], policy[
"validEndName"])
388 if "columns" in policy:
390 if "filter" in policy:
393 if "metadataKey" in policy:
396 def map(self, mapper, dataId, write=False):
397 location = Mapping.map(self, mapper, dataId, write=write)
404 """Look up properties for in a metadata registry given a partial 406 @param properties (list of strings) 407 @param dataId (dict) Dataset identifier 408 @return (list of tuples) values of properties""" 413 newId = dataId.copy()
417 for k, v
in dataId.items():
426 for k
in dataId.keys():
429 columns = set(properties)
433 return Mapping.lookup(self, properties, newId)
435 lookupDataId = dict(zip(where, values))
437 if len(lookups) != 1:
438 raise RuntimeError(
"No unique lookup for %s from %s: %d matches" %
439 (columns, dataId, len(lookups)))
440 if columns == set(properties):
443 for i, prop
in enumerate(columns):
444 newId[prop] = lookups[0][i]
445 return Mapping.lookup(self, properties, newId)
448 return mapper._standardizeExposure(self, item, dataId, filter=self.
setFilter)
452 """DatasetMapping is a Mapping subclass for non-Exposure datasets that can 453 be retrieved by the standard daf_persistence mechanism. 455 The differences are that the Storage type must be specified and no 456 Exposure standardization is performed. 458 The "storage" entry in the Policy is mandatory; the "tables" entry is 459 optional; no "level" entry is allowed. """ 461 def __init__(self, datasetType, policy, registry, root, **kwargs):
462 """Constructor for DatasetMapping class. 463 @param[in,out] mapper (lsst.daf.persistence.Mapper) Mapper object 464 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility)) 466 @param datasetType (string) 467 @param registry (lsst.obs.base.Registry) Registry for metadata lookups 468 @param root (string) Path of root directory""" 469 if isinstance(policy, pexPolicy.Policy):
470 policy = Policy(policy)
471 Mapping.__init__(self, datasetType, policy, registry, root, **kwargs)
def __init__(self, datasetType, policy, registry, root, kwargs)
def __init__(self, datasetType, policy, registry, calibRegistry, calibRoot, dataRoot=None, kwargs)
def __init__(self, datasetType, policy, registry, root, kwargs)
def standardize(self, mapper, item, dataId)
def map(self, mapper, dataId, write=False)
def have(self, properties, dataId)
def standardize(self, mapper, item, dataId)
def need(self, properties, dataId)
def lookup(self, properties, dataId)
def __init__(self, datasetType, policy, registry, root, kwargs)
def lookup(self, properties, dataId)
def __init__(self, datasetType, policy, registry, rootStorage, provided=None)
def map(self, mapper, dataId, write=False)