23 from builtins
import zip
24 from builtins
import object
25 from collections
import OrderedDict
28 from lsst.daf.persistence
import ButlerLocation, NoResults
29 from lsst.daf.persistence.policy
import Policy
30 import lsst.pex.policy
as pexPolicy
32 """This module defines the Mapping base class."""
37 """Mapping is a base class for all mappings. Mappings are used by
38 the Mapper to map (determine a path to some data given some
39 identifiers) and standardize (convert data into some standard
40 format or type) data, and to query the associated registry to see
41 what data is available.
43 Subclasses must specify self.storage or else override self.map().
45 Public methods: lookup, have, need, getKeys, map
47 Mappings are specified mainly by policy. A Mapping policy should
50 template (string): a Python string providing the filename for that
51 particular dataset type based on some data identifiers. In the
52 case of redundancy in the path (e.g., file uniquely specified by
53 the exposure number, but filter in the path), the
54 redundant/dependent identifiers can be looked up in the registry.
56 python (string): the Python type for the retrieved data (e.g.
57 lsst.afw.image.ExposureF)
59 persistable (string): the Persistable registration for the on-disk data
62 storage (string, optional): Storage type for this dataset type (e.g.
65 level (string, optional): the level in the camera hierarchy at which the
66 data is stored (Amp, Ccd or skyTile), if relevant
68 tables (string, optional): a whitespace-delimited list of tables in the
69 registry that can be NATURAL JOIN-ed to look up additional
72 def __init__(self, datasetType, policy, registry, rootStorage, provided=None):
73 """Constructor for Mapping class.
74 @param datasetType (string)
75 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility))
77 @param registry (lsst.obs.base.Registry) Registry for metadata lookups
78 @param rootStorage (Storage subclass instance) Interface to persisted repository data
79 @param provided (list of strings) Keys provided by the mapper
83 raise RuntimeError(
"No policy provided for mapping")
85 if isinstance(policy, pexPolicy.Policy):
86 policy = Policy(policy)
94 (k, _formatMap(v, k, datasetType))
96 re.findall(
r'\%\((\w+)\).*?([diouxXeEfFgGcrs])', self.
template)
98 if provided
is not None:
105 if 'level' in policy:
107 if 'tables' in policy:
113 self.
obsTimeName = policy[
'obsTimeName']
if 'obsTimeName' in policy
else None
116 """Return the dict of keys and value types required for this mapping."""
119 def map(self, mapper, dataId, write=False):
120 """Standard implementation of map function.
121 @param mapper (lsst.daf.persistence.Mapper)
122 @param dataId (dict) Dataset identifier
123 @return (lsst.daf.persistence.ButlerLocation)"""
124 actualId = self.
need(iter(self.keyDict.keys()), dataId)
125 usedDataId = {key: actualId[key]
for key
in self.keyDict.keys()}
126 path = mapper._mapActualToPath(self.
template, actualId)
127 if os.path.isabs(path):
128 raise RuntimeError(
"Mapped path should not be absolute.")
135 for ext
in (
None,
'.gz',
'.fz'):
136 if ext
and path.endswith(ext):
138 extPath = path + ext
if ext
else path
139 newPath = self.rootStorage.instanceSearch(extPath)
143 assert path,
"Fully-qualified filename is empty."
146 if hasattr(mapper, addFunc):
147 addFunc = getattr(mapper, addFunc)
148 additionalData = addFunc(actualId)
149 assert isinstance(additionalData, dict),
"Bad type for returned data"
151 additionalData = actualId.copy()
154 locationList=path, dataId=additionalData, mapper=mapper,
158 """Look up properties for in a metadata registry given a partial
160 @param properties (list of strings)
161 @param dataId (dict) Dataset identifier
162 @return (list of tuples) values of properties"""
165 raise RuntimeError(
"No registry for lookup")
167 skyMapKeys = (
"tract",
"patch")
179 substitutions = OrderedDict()
181 properties = list(properties)
185 substitutions[p] = dataId[p]
189 "Cannot look up skymap key '%s'; it must be explicitly included in the data ID" % p
192 substitutions[p] = index
200 if p
not in (
'filter',
'expTime',
'taiObs'):
203 if fastPath
and 'visit' in dataId
and "raw" in self.
tables:
204 lookupDataId = {
'visit': dataId[
'visit']}
205 result = self.registry.lookup(properties,
'raw_visit', lookupDataId, template=self.
template)
207 if dataId
is not None:
208 for k, v
in dataId.items():
215 where.append((k,
'?'))
217 lookupDataId = {k[0]: v
for k, v
in zip(where, values)}
222 result = self.registry.lookup(properties, self.
tables, lookupDataId, template=self.
template)
226 result = [tuple(v
if k
in removed
else item[v]
for k, v
in substitutions.items())
230 def have(self, properties, dataId):
231 """Returns whether the provided data identifier has all
232 the properties in the provided list.
233 @param properties (list of strings) Properties required
234 @parm dataId (dict) Dataset identifier
235 @return (bool) True if all properties are present"""
236 for prop
in properties:
237 if prop
not in dataId:
241 def need(self, properties, dataId):
242 """Ensures all properties in the provided list are present in
243 the data identifier, looking them up as needed. This is only
244 possible for the case where the data identifies a single
246 @param properties (list of strings) Properties required
247 @param dataId (dict) Partial dataset identifier
248 @return (dict) copy of dataset identifier with enhanced values
250 newId = dataId.copy()
252 for prop
in properties:
253 if prop
not in newId:
254 newProps.append(prop)
255 if len(newProps) == 0:
258 lookups = self.
lookup(newProps, newId)
259 if len(lookups) != 1:
260 raise NoResults(
"No unique lookup for %s from %s: %d matches" %
261 (newProps, newId, len(lookups)),
263 for i, prop
in enumerate(newProps):
264 newId[prop] = lookups[0][i]
268 def _formatMap(ch, k, datasetType):
269 """Convert a format character into a Python type."""
277 raise RuntimeError(
"Unexpected format specifier %s"
278 " for field %s in template for dataset %s" %
279 (ch, k, datasetType))
283 """ImageMapping is a Mapping subclass for non-camera images."""
285 def __init__(self, datasetType, policy, registry, root, **kwargs):
286 """Constructor for Mapping class.
287 @param datasetType (string)
288 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility))
290 @param registry (lsst.obs.base.Registry) Registry for metadata lookups
291 @param root (string) Path of root directory"""
292 if isinstance(policy, pexPolicy.Policy):
293 policy = Policy(policy)
294 Mapping.__init__(self, datasetType, policy, registry, root, **kwargs)
295 self.
columns = policy.asArray(
'columns')
if 'columns' in policy
else None
299 """ExposureMapping is a Mapping subclass for normal exposures."""
301 def __init__(self, datasetType, policy, registry, root, **kwargs):
302 """Constructor for Mapping class.
303 @param datasetType (string)
304 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility))
306 @param registry (lsst.obs.base.Registry) Registry for metadata lookups
307 @param root (string) Path of root directory"""
308 if isinstance(policy, pexPolicy.Policy):
309 policy = Policy(policy)
310 Mapping.__init__(self, datasetType, policy, registry, root, **kwargs)
311 self.
columns = policy.asArray(
'columns')
if 'columns' in policy
else None
314 return mapper._standardizeExposure(self, item, dataId)
318 """CalibrationMapping is a Mapping subclass for calibration-type products.
320 The difference is that data properties in the query or template
321 can be looked up using a reference Mapping in addition to this one.
323 CalibrationMapping Policies can contain the following:
325 reference (string, optional): a list of tables for finding missing dataset
326 identifier components (including the observation time, if a validity range
327 is required) in the exposure registry; note that the "tables" entry refers
328 to the calibration registry
330 refCols (string, optional): a list of dataset properties required from the
331 reference tables for lookups in the calibration registry
333 validRange (bool): true if the calibration dataset has a validity range
334 specified by a column in the tables of the reference dataset in the
335 exposure registry) and two columns in the tables of this calibration
336 dataset in the calibration registry)
338 obsTimeName (string, optional): the name of the column in the reference
339 dataset tables containing the observation time (default "taiObs")
341 validStartName (string, optional): the name of the column in the
342 calibration dataset tables containing the start of the validity range
343 (default "validStart")
345 validEndName (string, optional): the name of the column in the
346 calibration dataset tables containing the end of the validity range
347 (default "validEnd") """
349 def __init__(self, datasetType, policy, registry, calibRegistry, calibRoot, dataRoot=None, **kwargs):
350 """Constructor for Mapping class.
351 @param datasetType (string)
352 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility))
354 @param registry (lsst.obs.base.Registry) Registry for metadata lookups
355 @param calibRegistry (lsst.obs.base.Registry) Registry for calibration metadata lookups
356 @param calibRoot (string) Path of calibration root directory
357 @param dataRoot. (string) Path of data root directory; used for outputs only
359 if isinstance(policy, pexPolicy.Policy):
360 policy = Policy(policy)
361 Mapping.__init__(self, datasetType, policy, calibRegistry, calibRoot, **kwargs)
362 self.
reference = policy.asArray(
"reference")
if "reference" in policy
else None
363 self.
refCols = policy.asArray(
"refCols")
if "refCols" in policy
else None
366 if "validRange" in policy
and policy[
"validRange"]:
367 self.
range = (
"?", policy[
"validStartName"], policy[
"validEndName"])
368 if "columns" in policy:
370 if "filter" in policy:
373 if "metadataKey" in policy:
376 def map(self, mapper, dataId, write=False):
377 location = Mapping.map(self, mapper, dataId, write=write)
384 """Look up properties for in a metadata registry given a partial
386 @param properties (list of strings)
387 @param dataId (dict) Dataset identifier
388 @return (list of tuples) values of properties"""
393 newId = dataId.copy()
397 for k, v
in dataId.items():
406 for k
in dataId.keys():
409 columns = set(properties)
413 return Mapping.lookup(self, properties, newId)
415 lookupDataId = dict(zip(where, values))
416 lookups = self.refRegistry.lookup(columns, self.
reference, lookupDataId)
417 if len(lookups) != 1:
418 raise RuntimeError(
"No unique lookup for %s from %s: %d matches" %
419 (columns, dataId, len(lookups)))
420 if columns == set(properties):
423 for i, prop
in enumerate(columns):
424 newId[prop] = lookups[0][i]
425 return Mapping.lookup(self, properties, newId)
428 return mapper._standardizeExposure(self, item, dataId, filter=self.
setFilter)
432 """DatasetMapping is a Mapping subclass for non-Exposure datasets that can
433 be retrieved by the standard daf_persistence mechanism.
435 The differences are that the Storage type must be specified and no
436 Exposure standardization is performed.
438 The "storage" entry in the Policy is mandatory; the "tables" entry is
439 optional; no "level" entry is allowed. """
441 def __init__(self, datasetType, policy, registry, root, **kwargs):
442 """Constructor for DatasetMapping class.
443 @param[in,out] mapper (lsst.daf.persistence.Mapper) Mapper object
444 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility))
446 @param datasetType (string)
447 @param registry (lsst.obs.base.Registry) Registry for metadata lookups
448 @param root (string) Path of root directory"""
449 if isinstance(policy, pexPolicy.Policy):
450 policy = Policy(policy)
451 Mapping.__init__(self, datasetType, policy, registry, root, **kwargs)