23 from builtins
import zip
24 from builtins
import object
25 from collections
import OrderedDict
28 from lsst.daf.base
import PropertySet
29 from lsst.daf.persistence
import ButlerLocation, NoResults
30 from lsst.daf.persistence.policy
import Policy
31 import lsst.pex.policy
as pexPolicy
33 """This module defines the Mapping base class."""
38 """Mapping is a base class for all mappings. Mappings are used by
39 the Mapper to map (determine a path to some data given some
40 identifiers) and standardize (convert data into some standard
41 format or type) data, and to query the associated registry to see
42 what data is available.
44 Subclasses must specify self.storage or else override self.map().
46 Public methods: lookup, have, need, getKeys, map
48 Mappings are specified mainly by policy. A Mapping policy should
51 template (string): a Python string providing the filename for that
52 particular dataset type based on some data identifiers. In the
53 case of redundancy in the path (e.g., file uniquely specified by
54 the exposure number, but filter in the path), the
55 redundant/dependent identifiers can be looked up in the registry.
57 python (string): the Python type for the retrieved data (e.g.
58 lsst.afw.image.ExposureF)
60 persistable (string): the Persistable registration for the on-disk data
63 storage (string, optional): Storage type for this dataset type (e.g.
66 level (string, optional): the level in the camera hierarchy at which the
67 data is stored (Amp, Ccd or skyTile), if relevant
69 tables (string, optional): a whitespace-delimited list of tables in the
70 registry that can be NATURAL JOIN-ed to look up additional
73 def __init__(self, datasetType, policy, registry, rootStorage, provided=None):
74 """Constructor for Mapping class.
75 @param datasetType (string)
76 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility))
78 @param registry (lsst.obs.base.Registry) Registry for metadata lookups
79 @param rootStorage (Storage subclass instance) Interface to persisted repository data
80 @param provided (list of strings) Keys provided by the mapper
84 raise RuntimeError(
"No policy provided for mapping")
86 if isinstance(policy, pexPolicy.Policy):
87 policy = Policy(policy)
101 (k, _formatMap(v, k, datasetType))
103 re.findall(
r'\%\((\w+)\).*?([diouxXeEfFgGcrs])', self.
template)
107 if provided
is not None:
114 if 'level' in policy:
116 if 'tables' in policy:
122 self.
obsTimeName = policy[
'obsTimeName']
if 'obsTimeName' in policy
else None
123 self.
recipe = policy[
'recipe']
if 'recipe' in policy
else 'default'
130 raise RuntimeError(
"Template is not defined for the {} dataset type, ".format(self.
datasetType) +
131 "it must be set before it can be used.")
134 """Return the dict of keys and value types required for this mapping."""
137 def map(self, mapper, dataId, write=False):
138 """Standard implementation of map function.
139 @param mapper (lsst.daf.persistence.Mapper)
140 @param dataId (dict) Dataset identifier
141 @return (lsst.daf.persistence.ButlerLocation)"""
142 actualId = self.
need(iter(self.keyDict.keys()), dataId)
143 usedDataId = {key: actualId[key]
for key
in self.keyDict.keys()}
144 path = mapper._mapActualToPath(self.
template, actualId)
145 if os.path.isabs(path):
146 raise RuntimeError(
"Mapped path should not be absolute.")
153 for ext
in (
None,
'.gz',
'.fz'):
154 if ext
and path.endswith(ext):
156 extPath = path + ext
if ext
else path
157 newPath = self.rootStorage.instanceSearch(extPath)
161 assert path,
"Fully-qualified filename is empty."
164 if hasattr(mapper, addFunc):
165 addFunc = getattr(mapper, addFunc)
166 additionalData = addFunc(self.
datasetType, actualId)
167 assert isinstance(additionalData, PropertySet), \
168 "Bad type for returned data: %s" (type(additionalData),)
170 additionalData =
None
173 locationList=path, dataId=actualId.copy(), mapper=mapper,
175 additionalData=additionalData)
178 """Look up properties for in a metadata registry given a partial
180 @param properties (list of strings)
181 @param dataId (dict) Dataset identifier
182 @return (list of tuples) values of properties"""
185 raise RuntimeError(
"No registry for lookup")
187 skyMapKeys = (
"tract",
"patch")
199 substitutions = OrderedDict()
201 properties = list(properties)
205 substitutions[p] = dataId[p]
209 "Cannot look up skymap key '%s'; it must be explicitly included in the data ID" % p
212 substitutions[p] = index
220 if p
not in (
'filter',
'expTime',
'taiObs'):
223 if fastPath
and 'visit' in dataId
and "raw" in self.
tables:
224 lookupDataId = {
'visit': dataId[
'visit']}
225 result = self.registry.lookup(properties,
'raw_visit', lookupDataId, template=self.
template)
227 if dataId
is not None:
228 for k, v
in dataId.items():
235 where.append((k,
'?'))
237 lookupDataId = {k[0]: v
for k, v
in zip(where, values)}
242 result = self.registry.lookup(properties, self.
tables, lookupDataId, template=self.
template)
246 result = [tuple(v
if k
in removed
else item[v]
for k, v
in substitutions.items())
250 def have(self, properties, dataId):
251 """Returns whether the provided data identifier has all
252 the properties in the provided list.
253 @param properties (list of strings) Properties required
254 @parm dataId (dict) Dataset identifier
255 @return (bool) True if all properties are present"""
256 for prop
in properties:
257 if prop
not in dataId:
261 def need(self, properties, dataId):
262 """Ensures all properties in the provided list are present in
263 the data identifier, looking them up as needed. This is only
264 possible for the case where the data identifies a single
266 @param properties (list of strings) Properties required
267 @param dataId (dict) Partial dataset identifier
268 @return (dict) copy of dataset identifier with enhanced values
270 newId = dataId.copy()
272 for prop
in properties:
273 if prop
not in newId:
274 newProps.append(prop)
275 if len(newProps) == 0:
278 lookups = self.
lookup(newProps, newId)
279 if len(lookups) != 1:
280 raise NoResults(
"No unique lookup for %s from %s: %d matches" %
281 (newProps, newId, len(lookups)),
283 for i, prop
in enumerate(newProps):
284 newId[prop] = lookups[0][i]
288 def _formatMap(ch, k, datasetType):
289 """Convert a format character into a Python type."""
297 raise RuntimeError(
"Unexpected format specifier %s"
298 " for field %s in template for dataset %s" %
299 (ch, k, datasetType))
303 """ImageMapping is a Mapping subclass for non-camera images."""
305 def __init__(self, datasetType, policy, registry, root, **kwargs):
306 """Constructor for Mapping class.
307 @param datasetType (string)
308 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility))
310 @param registry (lsst.obs.base.Registry) Registry for metadata lookups
311 @param root (string) Path of root directory"""
312 if isinstance(policy, pexPolicy.Policy):
313 policy = Policy(policy)
314 Mapping.__init__(self, datasetType, policy, registry, root, **kwargs)
315 self.
columns = policy.asArray(
'columns')
if 'columns' in policy
else None
319 """ExposureMapping is a Mapping subclass for normal exposures."""
321 def __init__(self, datasetType, policy, registry, root, **kwargs):
322 """Constructor for Mapping class.
323 @param datasetType (string)
324 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility))
326 @param registry (lsst.obs.base.Registry) Registry for metadata lookups
327 @param root (string) Path of root directory"""
328 if isinstance(policy, pexPolicy.Policy):
329 policy = Policy(policy)
330 Mapping.__init__(self, datasetType, policy, registry, root, **kwargs)
331 self.
columns = policy.asArray(
'columns')
if 'columns' in policy
else None
334 return mapper._standardizeExposure(self, item, dataId)
338 """CalibrationMapping is a Mapping subclass for calibration-type products.
340 The difference is that data properties in the query or template
341 can be looked up using a reference Mapping in addition to this one.
343 CalibrationMapping Policies can contain the following:
345 reference (string, optional): a list of tables for finding missing dataset
346 identifier components (including the observation time, if a validity range
347 is required) in the exposure registry; note that the "tables" entry refers
348 to the calibration registry
350 refCols (string, optional): a list of dataset properties required from the
351 reference tables for lookups in the calibration registry
353 validRange (bool): true if the calibration dataset has a validity range
354 specified by a column in the tables of the reference dataset in the
355 exposure registry) and two columns in the tables of this calibration
356 dataset in the calibration registry)
358 obsTimeName (string, optional): the name of the column in the reference
359 dataset tables containing the observation time (default "taiObs")
361 validStartName (string, optional): the name of the column in the
362 calibration dataset tables containing the start of the validity range
363 (default "validStart")
365 validEndName (string, optional): the name of the column in the
366 calibration dataset tables containing the end of the validity range
367 (default "validEnd") """
369 def __init__(self, datasetType, policy, registry, calibRegistry, calibRoot, dataRoot=None, **kwargs):
370 """Constructor for Mapping class.
371 @param datasetType (string)
372 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility))
374 @param registry (lsst.obs.base.Registry) Registry for metadata lookups
375 @param calibRegistry (lsst.obs.base.Registry) Registry for calibration metadata lookups
376 @param calibRoot (string) Path of calibration root directory
377 @param dataRoot. (string) Path of data root directory; used for outputs only
379 if isinstance(policy, pexPolicy.Policy):
380 policy = Policy(policy)
381 Mapping.__init__(self, datasetType, policy, calibRegistry, calibRoot, **kwargs)
382 self.
reference = policy.asArray(
"reference")
if "reference" in policy
else None
383 self.
refCols = policy.asArray(
"refCols")
if "refCols" in policy
else None
386 if "validRange" in policy
and policy[
"validRange"]:
387 self.
range = (
"?", policy[
"validStartName"], policy[
"validEndName"])
388 if "columns" in policy:
390 if "filter" in policy:
393 if "metadataKey" in policy:
396 def map(self, mapper, dataId, write=False):
397 location = Mapping.map(self, mapper, dataId, write=write)
404 """Look up properties for in a metadata registry given a partial
406 @param properties (list of strings)
407 @param dataId (dict) Dataset identifier
408 @return (list of tuples) values of properties"""
413 newId = dataId.copy()
417 for k, v
in dataId.items():
426 for k
in dataId.keys():
429 columns = set(properties)
433 return Mapping.lookup(self, properties, newId)
435 lookupDataId = dict(zip(where, values))
436 lookups = self.refRegistry.lookup(columns, self.
reference, lookupDataId)
437 if len(lookups) != 1:
438 raise RuntimeError(
"No unique lookup for %s from %s: %d matches" %
439 (columns, dataId, len(lookups)))
440 if columns == set(properties):
443 for i, prop
in enumerate(columns):
444 newId[prop] = lookups[0][i]
445 return Mapping.lookup(self, properties, newId)
448 return mapper._standardizeExposure(self, item, dataId, filter=self.
setFilter)
452 """DatasetMapping is a Mapping subclass for non-Exposure datasets that can
453 be retrieved by the standard daf_persistence mechanism.
455 The differences are that the Storage type must be specified and no
456 Exposure standardization is performed.
458 The "storage" entry in the Policy is mandatory; the "tables" entry is
459 optional; no "level" entry is allowed. """
461 def __init__(self, datasetType, policy, registry, root, **kwargs):
462 """Constructor for DatasetMapping class.
463 @param[in,out] mapper (lsst.daf.persistence.Mapper) Mapper object
464 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility))
466 @param datasetType (string)
467 @param registry (lsst.obs.base.Registry) Registry for metadata lookups
468 @param root (string) Path of root directory"""
469 if isinstance(policy, pexPolicy.Policy):
470 policy = Policy(policy)
471 Mapping.__init__(self, datasetType, policy, registry, root, **kwargs)