23 from builtins
import zip
24 from builtins
import object
25 from collections
import OrderedDict
28 from lsst.daf.persistence
import ButlerLocation, NoResults
29 from lsst.daf.persistence.policy
import Policy
30 import lsst.pex.policy
as pexPolicy
32 """This module defines the Mapping base class."""
37 """Mapping is a base class for all mappings. Mappings are used by
38 the Mapper to map (determine a path to some data given some
39 identifiers) and standardize (convert data into some standard
40 format or type) data, and to query the associated registry to see
41 what data is available.
43 Subclasses must specify self.storage or else override self.map().
45 Public methods: lookup, have, need, getKeys, map
47 Mappings are specified mainly by policy. A Mapping policy should
50 template (string): a Python string providing the filename for that
51 particular dataset type based on some data identifiers. In the
52 case of redundancy in the path (e.g., file uniquely specified by
53 the exposure number, but filter in the path), the
54 redundant/dependent identifiers can be looked up in the registry.
56 python (string): the Python type for the retrieved data (e.g.
57 lsst.afw.image.ExposureF)
59 persistable (string): the Persistable registration for the on-disk data
62 storage (string, optional): Storage type for this dataset type (e.g.
65 level (string, optional): the level in the camera hierarchy at which the
66 data is stored (Amp, Ccd or skyTile), if relevant
68 tables (string, optional): a whitespace-delimited list of tables in the
69 registry that can be NATURAL JOIN-ed to look up additional
72 def __init__(self, datasetType, policy, registry, rootStorage, provided=None):
73 """Constructor for Mapping class.
74 @param datasetType (string)
75 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility))
77 @param registry (lsst.obs.base.Registry) Registry for metadata lookups
78 @param rootStorage (Storage subclass instance) Interface to persisted repository data
79 @param provided (list of strings) Keys provided by the mapper
83 raise RuntimeError(
"No policy provided for mapping")
85 if isinstance(policy, pexPolicy.Policy):
86 policy = Policy(policy)
100 (k, _formatMap(v, k, datasetType))
102 re.findall(
r'\%\((\w+)\).*?([diouxXeEfFgGcrs])', self.
template)
106 if provided
is not None:
113 if 'level' in policy:
115 if 'tables' in policy:
121 self.
obsTimeName = policy[
'obsTimeName']
if 'obsTimeName' in policy
else None
128 raise RuntimeError(
"Template is not defined for the {} dataset type, ".format(self.
datasetType) +
129 "it must be set before it can be used.")
132 """Return the dict of keys and value types required for this mapping."""
135 def map(self, mapper, dataId, write=False):
136 """Standard implementation of map function.
137 @param mapper (lsst.daf.persistence.Mapper)
138 @param dataId (dict) Dataset identifier
139 @return (lsst.daf.persistence.ButlerLocation)"""
140 actualId = self.
need(iter(self.keyDict.keys()), dataId)
141 usedDataId = {key: actualId[key]
for key
in self.keyDict.keys()}
142 path = mapper._mapActualToPath(self.
template, actualId)
143 if os.path.isabs(path):
144 raise RuntimeError(
"Mapped path should not be absolute.")
151 for ext
in (
None,
'.gz',
'.fz'):
152 if ext
and path.endswith(ext):
154 extPath = path + ext
if ext
else path
155 newPath = self.rootStorage.instanceSearch(extPath)
159 assert path,
"Fully-qualified filename is empty."
162 if hasattr(mapper, addFunc):
163 addFunc = getattr(mapper, addFunc)
164 additionalData = addFunc(actualId)
165 assert isinstance(additionalData, dict),
"Bad type for returned data"
167 additionalData = actualId.copy()
170 locationList=path, dataId=additionalData, mapper=mapper,
174 """Look up properties for in a metadata registry given a partial
176 @param properties (list of strings)
177 @param dataId (dict) Dataset identifier
178 @return (list of tuples) values of properties"""
181 raise RuntimeError(
"No registry for lookup")
183 skyMapKeys = (
"tract",
"patch")
195 substitutions = OrderedDict()
197 properties = list(properties)
201 substitutions[p] = dataId[p]
205 "Cannot look up skymap key '%s'; it must be explicitly included in the data ID" % p
208 substitutions[p] = index
216 if p
not in (
'filter',
'expTime',
'taiObs'):
219 if fastPath
and 'visit' in dataId
and "raw" in self.
tables:
220 lookupDataId = {
'visit': dataId[
'visit']}
221 result = self.registry.lookup(properties,
'raw_visit', lookupDataId, template=self.
template)
223 if dataId
is not None:
224 for k, v
in dataId.items():
231 where.append((k,
'?'))
233 lookupDataId = {k[0]: v
for k, v
in zip(where, values)}
238 result = self.registry.lookup(properties, self.
tables, lookupDataId, template=self.
template)
242 result = [tuple(v
if k
in removed
else item[v]
for k, v
in substitutions.items())
246 def have(self, properties, dataId):
247 """Returns whether the provided data identifier has all
248 the properties in the provided list.
249 @param properties (list of strings) Properties required
250 @parm dataId (dict) Dataset identifier
251 @return (bool) True if all properties are present"""
252 for prop
in properties:
253 if prop
not in dataId:
257 def need(self, properties, dataId):
258 """Ensures all properties in the provided list are present in
259 the data identifier, looking them up as needed. This is only
260 possible for the case where the data identifies a single
262 @param properties (list of strings) Properties required
263 @param dataId (dict) Partial dataset identifier
264 @return (dict) copy of dataset identifier with enhanced values
266 newId = dataId.copy()
268 for prop
in properties:
269 if prop
not in newId:
270 newProps.append(prop)
271 if len(newProps) == 0:
274 lookups = self.
lookup(newProps, newId)
275 if len(lookups) != 1:
276 raise NoResults(
"No unique lookup for %s from %s: %d matches" %
277 (newProps, newId, len(lookups)),
279 for i, prop
in enumerate(newProps):
280 newId[prop] = lookups[0][i]
284 def _formatMap(ch, k, datasetType):
285 """Convert a format character into a Python type."""
293 raise RuntimeError(
"Unexpected format specifier %s"
294 " for field %s in template for dataset %s" %
295 (ch, k, datasetType))
299 """ImageMapping is a Mapping subclass for non-camera images."""
301 def __init__(self, datasetType, policy, registry, root, **kwargs):
302 """Constructor for Mapping class.
303 @param datasetType (string)
304 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility))
306 @param registry (lsst.obs.base.Registry) Registry for metadata lookups
307 @param root (string) Path of root directory"""
308 if isinstance(policy, pexPolicy.Policy):
309 policy = Policy(policy)
310 Mapping.__init__(self, datasetType, policy, registry, root, **kwargs)
311 self.
columns = policy.asArray(
'columns')
if 'columns' in policy
else None
315 """ExposureMapping is a Mapping subclass for normal exposures."""
317 def __init__(self, datasetType, policy, registry, root, **kwargs):
318 """Constructor for Mapping class.
319 @param datasetType (string)
320 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility))
322 @param registry (lsst.obs.base.Registry) Registry for metadata lookups
323 @param root (string) Path of root directory"""
324 if isinstance(policy, pexPolicy.Policy):
325 policy = Policy(policy)
326 Mapping.__init__(self, datasetType, policy, registry, root, **kwargs)
327 self.
columns = policy.asArray(
'columns')
if 'columns' in policy
else None
330 return mapper._standardizeExposure(self, item, dataId)
334 """CalibrationMapping is a Mapping subclass for calibration-type products.
336 The difference is that data properties in the query or template
337 can be looked up using a reference Mapping in addition to this one.
339 CalibrationMapping Policies can contain the following:
341 reference (string, optional): a list of tables for finding missing dataset
342 identifier components (including the observation time, if a validity range
343 is required) in the exposure registry; note that the "tables" entry refers
344 to the calibration registry
346 refCols (string, optional): a list of dataset properties required from the
347 reference tables for lookups in the calibration registry
349 validRange (bool): true if the calibration dataset has a validity range
350 specified by a column in the tables of the reference dataset in the
351 exposure registry) and two columns in the tables of this calibration
352 dataset in the calibration registry)
354 obsTimeName (string, optional): the name of the column in the reference
355 dataset tables containing the observation time (default "taiObs")
357 validStartName (string, optional): the name of the column in the
358 calibration dataset tables containing the start of the validity range
359 (default "validStart")
361 validEndName (string, optional): the name of the column in the
362 calibration dataset tables containing the end of the validity range
363 (default "validEnd") """
365 def __init__(self, datasetType, policy, registry, calibRegistry, calibRoot, dataRoot=None, **kwargs):
366 """Constructor for Mapping class.
367 @param datasetType (string)
368 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility))
370 @param registry (lsst.obs.base.Registry) Registry for metadata lookups
371 @param calibRegistry (lsst.obs.base.Registry) Registry for calibration metadata lookups
372 @param calibRoot (string) Path of calibration root directory
373 @param dataRoot. (string) Path of data root directory; used for outputs only
375 if isinstance(policy, pexPolicy.Policy):
376 policy = Policy(policy)
377 Mapping.__init__(self, datasetType, policy, calibRegistry, calibRoot, **kwargs)
378 self.
reference = policy.asArray(
"reference")
if "reference" in policy
else None
379 self.
refCols = policy.asArray(
"refCols")
if "refCols" in policy
else None
382 if "validRange" in policy
and policy[
"validRange"]:
383 self.
range = (
"?", policy[
"validStartName"], policy[
"validEndName"])
384 if "columns" in policy:
386 if "filter" in policy:
389 if "metadataKey" in policy:
392 def map(self, mapper, dataId, write=False):
393 location = Mapping.map(self, mapper, dataId, write=write)
400 """Look up properties for in a metadata registry given a partial
402 @param properties (list of strings)
403 @param dataId (dict) Dataset identifier
404 @return (list of tuples) values of properties"""
409 newId = dataId.copy()
413 for k, v
in dataId.items():
422 for k
in dataId.keys():
425 columns = set(properties)
429 return Mapping.lookup(self, properties, newId)
431 lookupDataId = dict(zip(where, values))
432 lookups = self.refRegistry.lookup(columns, self.
reference, lookupDataId)
433 if len(lookups) != 1:
434 raise RuntimeError(
"No unique lookup for %s from %s: %d matches" %
435 (columns, dataId, len(lookups)))
436 if columns == set(properties):
439 for i, prop
in enumerate(columns):
440 newId[prop] = lookups[0][i]
441 return Mapping.lookup(self, properties, newId)
444 return mapper._standardizeExposure(self, item, dataId, filter=self.
setFilter)
448 """DatasetMapping is a Mapping subclass for non-Exposure datasets that can
449 be retrieved by the standard daf_persistence mechanism.
451 The differences are that the Storage type must be specified and no
452 Exposure standardization is performed.
454 The "storage" entry in the Policy is mandatory; the "tables" entry is
455 optional; no "level" entry is allowed. """
457 def __init__(self, datasetType, policy, registry, root, **kwargs):
458 """Constructor for DatasetMapping class.
459 @param[in,out] mapper (lsst.daf.persistence.Mapper) Mapper object
460 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility))
462 @param datasetType (string)
463 @param registry (lsst.obs.base.Registry) Registry for metadata lookups
464 @param root (string) Path of root directory"""
465 if isinstance(policy, pexPolicy.Policy):
466 policy = Policy(policy)
467 Mapping.__init__(self, datasetType, policy, registry, root, **kwargs)