lsst.obs.base  13.0-46-g1d866da+4
 All Classes Namespaces Files Functions Variables
cameraMapper.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from builtins import str
24 import copy
25 import os
26 import pyfits # required by _makeDefectsDict until defects are written as AFW tables
27 import re
28 import weakref
29 import lsst.daf.persistence as dafPersist
30 from . import ImageMapping, ExposureMapping, CalibrationMapping, DatasetMapping
31 import lsst.afw.geom as afwGeom
32 import lsst.afw.image as afwImage
33 import lsst.afw.table as afwTable
34 import lsst.afw.cameraGeom as afwCameraGeom
35 import lsst.log as lsstLog
36 import lsst.pex.policy as pexPolicy
37 import lsst.pex.exceptions as pexExcept
38 from .exposureIdInfo import ExposureIdInfo
39 from .makeRawVisitInfo import MakeRawVisitInfo
40 from lsst.utils import getPackageDir
41 
42 """This module defines the CameraMapper base class."""
43 
44 
45 class CameraMapper(dafPersist.Mapper):
46 
47  """CameraMapper is a base class for mappers that handle images from a
48  camera and products derived from them. This provides an abstraction layer
49  between the data on disk and the code.
50 
51  Public methods: keys, queryMetadata, getDatasetTypes, map,
52  canStandardize, standardize
53 
54  Mappers for specific data sources (e.g., CFHT Megacam, LSST
55  simulations, etc.) should inherit this class.
56 
57  The CameraMapper manages datasets within a "root" directory. Note that
58  writing to a dataset present in the input root will hide the existing
59  dataset but not overwrite it. See #2160 for design discussion.
60 
61  A camera is assumed to consist of one or more rafts, each composed of
62  multiple CCDs. Each CCD is in turn composed of one or more amplifiers
63  (amps). A camera is also assumed to have a camera geometry description
64  (CameraGeom object) as a policy file, a filter description (Filter class
65  static configuration) as another policy file, and an optional defects
66  description directory.
67 
68  Information from the camera geometry and defects are inserted into all
69  Exposure objects returned.
70 
71  The mapper uses one or two registries to retrieve metadata about the
72  images. The first is a registry of all raw exposures. This must contain
73  the time of the observation. One or more tables (or the equivalent)
74  within the registry are used to look up data identifier components that
75  are not specified by the user (e.g. filter) and to return results for
76  metadata queries. The second is an optional registry of all calibration
77  data. This should contain validity start and end entries for each
78  calibration dataset in the same timescale as the observation time.
79 
80  Subclasses will typically set MakeRawVisitInfoClass:
81 
82  MakeRawVisitInfoClass: a class variable that points to a subclass of
83  MakeRawVisitInfo, a functor that creates an
84  lsst.afw.image.VisitInfo from the FITS metadata of a raw image.
85 
86  Subclasses must provide the following methods:
87 
88  _extractDetectorName(self, dataId): returns the detector name for a CCD
89  (e.g., "CFHT 21", "R:1,2 S:3,4") as used in the AFW CameraGeom class given
90  a dataset identifier referring to that CCD or a subcomponent of it.
91 
92  _computeCcdExposureId(self, dataId): see below
93 
94  _computeCoaddExposureId(self, dataId, singleFilter): see below
95 
96  Subclasses may also need to override the following methods:
97 
98  _transformId(self, dataId): transformation of a data identifier
99  from colloquial usage (e.g., "ccdname") to proper/actual usage (e.g., "ccd"),
100  including making suitable for path expansion (e.g. removing commas).
101  The default implementation does nothing. Note that this
102  method should not modify its input parameter.
103 
104  getShortCcdName(self, ccdName): a static method that returns a shortened name
105  suitable for use as a filename. The default version converts spaces to underscores.
106 
107  _getCcdKeyVal(self, dataId): return a CCD key and value
108  by which to look up defects in the defects registry.
109  The default value returns ("ccd", detector name)
110 
111  _mapActualToPath(self, template, actualId): convert a template path to an
112  actual path, using the actual dataset identifier.
113 
114  The mapper's behaviors are largely specified by the policy file.
115  See the MapperDictionary.paf for descriptions of the available items.
116 
117  The 'exposures', 'calibrations', and 'datasets' subpolicies configure
118  mappings (see Mappings class).
119 
120  Common default mappings for all subclasses can be specified in the
121  "policy/{images,exposures,calibrations,datasets}.yaml" files. This provides
122  a simple way to add a product to all camera mappers.
123 
124  Functions to map (provide a path to the data given a dataset
125  identifier dictionary) and standardize (convert data into some standard
126  format or type) may be provided in the subclass as "map_{dataset type}"
127  and "std_{dataset type}", respectively.
128 
129  If non-Exposure datasets cannot be retrieved using standard
130  daf_persistence methods alone, a "bypass_{dataset type}" function may be
131  provided in the subclass to return the dataset instead of using the
132  "datasets" subpolicy.
133 
134  Implementations of map_camera and bypass_camera that should typically be
135  sufficient are provided in this base class.
136 
137  @todo
138  * Handle defects the same was as all other calibration products, using the calibration registry
139  * Instead of auto-loading the camera at construction time, load it from the calibration registry
140  * Rewrite defects as AFW tables so we don't need pyfits to unpersist them; then remove all mention
141  of pyfits from this package.
142  """
143  packageName = None
144 
145  # a class or subclass of MakeRawVisitInfo, a functor that makes an
146  # lsst.afw.image.VisitInfo from the FITS metadata of a raw image
147  MakeRawVisitInfoClass = MakeRawVisitInfo
148 
149  # a class or subclass of PupilFactory
150  PupilFactoryClass = afwCameraGeom.PupilFactory
151 
152  def __init__(self, policy, repositoryDir,
153  root=None, registry=None, calibRoot=None, calibRegistry=None,
154  provided=None, parentRegistry=None, repositoryCfg=None):
155  """Initialize the CameraMapper.
156 
157  Parameters
158  ----------
159  policy : daf_persistence.Policy,
160  Can also be pexPolicy.Policy, only for backward compatibility.
161  Policy with per-camera defaults already merged.
162  repositoryDir : string
163  Policy repository for the subclassing module (obtained with
164  getRepositoryPath() on the per-camera default dictionary).
165  root : string, optional
166  Path to the root directory for data.
167  registry : string, optional
168  Path to registry with data's metadata.
169  calibRoot : string, optional
170  Root directory for calibrations.
171  calibRegistry : string, optional
172  Path to registry with calibrations' metadata.
173  provided : list of string, optional
174  Keys provided by the mapper.
175  parentRegistry : Registry subclass, optional
176  Registry from a parent repository that may be used to look up
177  data's metadata.
178  repositoryCfg : daf_persistence.RepositoryCfg or None, optional
179  The configuration information for the repository this mapper is
180  being used with.
181  """
182 
183  dafPersist.Mapper.__init__(self)
184 
185  self.log = lsstLog.Log.getLogger("CameraMapper")
186 
187  if root:
188  self.root = root
189  elif repositoryCfg:
190  self.root = repositoryCfg.root
191  else:
192  self.root = None
193  if isinstance(policy, pexPolicy.Policy):
194  policy = dafPersist.Policy(policy)
195 
196  repoPolicy = repositoryCfg.policy if repositoryCfg else None
197  if repoPolicy is not None:
198  policy.update(repoPolicy)
199 
200  defaultPolicyFile = dafPersist.Policy.defaultPolicyFile("obs_base",
201  "MapperDictionary.paf",
202  "policy")
203  dictPolicy = dafPersist.Policy(defaultPolicyFile)
204  policy.merge(dictPolicy)
205 
206  # Levels
207  self.levels = dict()
208  if 'levels' in policy:
209  levelsPolicy = policy['levels']
210  for key in levelsPolicy.names(True):
211  self.levels[key] = set(levelsPolicy.asArray(key))
212  self.defaultLevel = policy['defaultLevel']
213  self.defaultSubLevels = dict()
214  if 'defaultSubLevels' in policy:
215  self.defaultSubLevels = policy['defaultSubLevels']
216 
217  # Root directories
218  if root is None:
219  root = "."
220  root = dafPersist.LogicalLocation(root).locString()
221 
222  self.rootStorage = dafPersist.Storage.makeFromURI(uri=root)
223 
224  # If the calibRoot is passed in, use that. If not and it's indicated in
225  # the policy, use that. And otherwise, the calibs are in the regular
226  # root.
227  # If the location indicated by the calib root does not exist, do not
228  # create it.
229  calibStorage = None
230  if calibRoot is not None:
231  calibRoot = dafPersist.Storage.absolutePath(root, calibRoot)
232  calibStorage = dafPersist.Storage.makeFromURI(uri=calibRoot,
233  create=False)
234  else:
235  calibRoot = policy.get('calibRoot', None)
236  if calibRoot:
237  calibStorage = dafPersist.Storage.makeFromURI(uri=calibRoot,
238  create=False)
239  if calibStorage is None:
240  calibStorage = self.rootStorage
241 
242  self.root = root
243 
244  # Registries
245  self.registry = self._setupRegistry("registry", "exposure", registry, policy, "registryPath",
246  self.rootStorage, searchParents=False,
247  posixIfNoSql=(not parentRegistry))
248  if not self.registry:
249  self.registry = parentRegistry
250  needCalibRegistry = policy.get('needCalibRegistry', None)
251  if needCalibRegistry:
252  if calibStorage:
253  self.calibRegistry = self._setupRegistry("calibRegistry", "calib", calibRegistry, policy,
254  "calibRegistryPath", calibStorage)
255  else:
256  raise RuntimeError(
257  "'needCalibRegistry' is true in Policy, but was unable to locate a repo at " +
258  "calibRoot ivar:%s or policy['calibRoot']:%s" %
259  (calibRoot, policy.get('calibRoot', None)))
260  else:
261  self.calibRegistry = None
262 
263  # Dict of valid keys and their value types
264  self.keyDict = dict()
265 
266  self._initMappings(policy, self.rootStorage, calibStorage, provided=None)
267 
268  # Camera geometry
269  self.cameraDataLocation = None # path to camera geometry config file
270  self.camera = self._makeCamera(policy=policy, repositoryDir=repositoryDir)
271 
272  # Defect registry and root. Defects are stored with the camera and the registry is loaded from the
273  # camera package, which is on the local filesystem.
274  self.defectRegistry = None
275  if 'defects' in policy:
276  self.defectPath = os.path.join(repositoryDir, policy['defects'])
277  defectRegistryLocation = os.path.join(self.defectPath, "defectRegistry.sqlite3")
278  self.defectRegistry = dafPersist.Registry.create(defectRegistryLocation)
279 
280  # Filter translation table
281  self.filters = None
282 
283  # Skytile policy
284  self.skypolicy = policy['skytiles']
285 
286  # verify that the class variable packageName is set before attempting
287  # to instantiate an instance
288  if self.packageName is None:
289  raise ValueError('class variable packageName must not be None')
290 
292 
293  def _initMappings(self, policy, rootStorage=None, calibStorage=None, provided=None):
294  """Initialize mappings
295 
296  For each of the dataset types that we want to be able to read, there are
297  methods that can be created to support them:
298  * map_<dataset> : determine the path for dataset
299  * std_<dataset> : standardize the retrieved dataset
300  * bypass_<dataset> : retrieve the dataset (bypassing the usual retrieval machinery)
301  * query_<dataset> : query the registry
302 
303  Besides the dataset types explicitly listed in the policy, we create
304  additional, derived datasets for additional conveniences, e.g., reading
305  the header of an image, retrieving only the size of a catalog.
306 
307  @param policy (Policy) Policy with per-camera defaults already merged
308  @param rootStorage (Storage subclass instance) Interface to persisted repository data
309  @param calibRoot (Storage subclass instance) Interface to persisted calib repository data
310  @param provided (list of strings) Keys provided by the mapper
311  """
312  # Sub-dictionaries (for exposure/calibration/dataset types)
313  imgMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
314  "obs_base", "ImageMappingDictionary.paf", "policy"))
315  expMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
316  "obs_base", "ExposureMappingDictionary.paf", "policy"))
317  calMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
318  "obs_base", "CalibrationMappingDictionary.paf", "policy"))
319  dsMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
320  "obs_base", "DatasetMappingDictionary.paf", "policy"))
321 
322  # Mappings
323  mappingList = (
324  ("images", imgMappingPolicy, ImageMapping),
325  ("exposures", expMappingPolicy, ExposureMapping),
326  ("calibrations", calMappingPolicy, CalibrationMapping),
327  ("datasets", dsMappingPolicy, DatasetMapping)
328  )
329  self.mappings = dict()
330  for name, defPolicy, cls in mappingList:
331  if name in policy:
332  datasets = policy[name]
333 
334  # Centrally-defined datasets
335  defaultsPath = os.path.join(getPackageDir("obs_base"), "policy", name + ".yaml")
336  if os.path.exists(defaultsPath):
337  datasets.merge(dafPersist.Policy(defaultsPath))
338 
339  mappings = dict()
340  setattr(self, name, mappings)
341  for datasetType in datasets.names(True):
342  subPolicy = datasets[datasetType]
343  subPolicy.merge(defPolicy)
344 
345  if not hasattr(self, "map_" + datasetType) and 'composite' in subPolicy:
346  def compositeClosure(dataId, write=False, mapper=None, mapping=None,
347  subPolicy=subPolicy):
348  components = subPolicy.get('composite')
349  assembler = subPolicy['assembler'] if 'assembler' in subPolicy else None
350  disassembler = subPolicy['disassembler'] if 'disassembler' in subPolicy else None
351  python = subPolicy['python']
352  butlerComposite = dafPersist.ButlerComposite(assembler=assembler,
353  disassembler=disassembler,
354  python=python,
355  dataId=dataId,
356  mapper=self)
357  for name, component in components.items():
358  butlerComposite.add(id=name,
359  datasetType=component.get('datasetType'),
360  setter=component.get('setter', None),
361  getter=component.get('getter', None),
362  subset=component.get('subset', False),
363  inputOnly=component.get('inputOnly', False))
364  return butlerComposite
365  setattr(self, "map_" + datasetType, compositeClosure)
366  # for now at least, don't set up any other handling for this dataset type.
367  continue
368 
369  if name == "calibrations":
370  mapping = cls(datasetType, subPolicy, self.registry, self.calibRegistry, calibStorage,
371  provided=provided, dataRoot=rootStorage)
372  else:
373  mapping = cls(datasetType, subPolicy, self.registry, rootStorage, provided=provided)
374  self.keyDict.update(mapping.keys())
375  mappings[datasetType] = mapping
376  self.mappings[datasetType] = mapping
377  if not hasattr(self, "map_" + datasetType):
378  def mapClosure(dataId, write=False, mapper=weakref.proxy(self), mapping=mapping):
379  return mapping.map(mapper, dataId, write)
380  setattr(self, "map_" + datasetType, mapClosure)
381  if not hasattr(self, "query_" + datasetType):
382  def queryClosure(format, dataId, mapping=mapping):
383  return mapping.lookup(format, dataId)
384  setattr(self, "query_" + datasetType, queryClosure)
385  if hasattr(mapping, "standardize") and not hasattr(self, "std_" + datasetType):
386  def stdClosure(item, dataId, mapper=weakref.proxy(self), mapping=mapping):
387  return mapping.standardize(mapper, item, dataId)
388  setattr(self, "std_" + datasetType, stdClosure)
389 
390  def setMethods(suffix, mapImpl=None, bypassImpl=None, queryImpl=None):
391  """Set convenience methods on CameraMapper"""
392  mapName = "map_" + datasetType + "_" + suffix
393  bypassName = "bypass_" + datasetType + "_" + suffix
394  queryName = "query_" + datasetType + "_" + suffix
395  if not hasattr(self, mapName):
396  setattr(self, mapName, mapImpl or getattr(self, "map_" + datasetType))
397  if not hasattr(self, bypassName):
398  if bypassImpl is None and hasattr(self, "bypass_" + datasetType):
399  bypassImpl = getattr(self, "bypass_" + datasetType)
400  if bypassImpl is not None:
401  setattr(self, bypassName, bypassImpl)
402  if not hasattr(self, queryName):
403  setattr(self, queryName, queryImpl or getattr(self, "query_" + datasetType))
404 
405  # Filename of dataset
406  setMethods("filename", bypassImpl=lambda datasetType, pythonType, location, dataId:
407  [os.path.join(location.getStorage().root, p) for p in location.getLocations()])
408  # Metadata from FITS file
409  if subPolicy["storage"] == "FitsStorage": # a FITS image
410  setMethods("md", bypassImpl=lambda datasetType, pythonType, location, dataId:
411  afwImage.readMetadata(location.getLocationsWithRoot()[0]))
412  if name == "exposures":
413  setMethods("wcs", bypassImpl=lambda datasetType, pythonType, location, dataId:
414  afwImage.makeWcs(
415  afwImage.readMetadata(location.getLocationsWithRoot()[0])))
416  setMethods("calib", bypassImpl=lambda datasetType, pythonType, location, dataId:
417  afwImage.Calib(
418  afwImage.readMetadata(location.getLocationsWithRoot()[0])))
419  setMethods("visitInfo",
420  bypassImpl=lambda datasetType, pythonType, location, dataId:
421  afwImage.VisitInfo(
422  afwImage.readMetadata(location.getLocationsWithRoot()[0])))
423  setMethods("filter",
424  bypassImpl=lambda datasetType, pythonType, location, dataId:
425  afwImage.Filter(
426  afwImage.readMetadata(location.getLocationsWithRoot()[0])))
427  setMethods("detector",
428  mapImpl=lambda dataId, write=False:
429  dafPersist.ButlerLocation(
430  pythonType="lsst.afw.cameraGeom.CameraConfig",
431  cppType="Config",
432  storageName="Internal",
433  locationList="ignored",
434  dataId=dataId,
435  mapper=self,
436  storage=None,
437  ),
438  bypassImpl=lambda datasetType, pythonType, location, dataId:
439  self.camera[self._extractDetectorName(dataId)]
440  )
441  setMethods("bbox", bypassImpl=lambda dsType, pyType, location, dataId:
442  afwImage.bboxFromMetadata(
443  afwImage.readMetadata(location.getLocationsWithRoot()[0], hdu=1)))
444 
445  elif name == "images":
446  setMethods("bbox", bypassImpl=lambda dsType, pyType, location, dataId:
447  afwImage.bboxFromMetadata(
448  afwImage.readMetadata(location.getLocationsWithRoot()[0])))
449 
450  if subPolicy["storage"] == "FitsCatalogStorage": # a FITS catalog
451  setMethods("md", bypassImpl=lambda datasetType, pythonType, location, dataId:
452  afwImage.readMetadata(os.path.join(location.getStorage().root,
453  location.getLocations()[0]), hdu=1))
454 
455  # Sub-images
456  if subPolicy["storage"] == "FitsStorage":
457  def mapSubClosure(dataId, write=False, mapper=weakref.proxy(self), mapping=mapping):
458  subId = dataId.copy()
459  del subId['bbox']
460  loc = mapping.map(mapper, subId, write)
461  bbox = dataId['bbox']
462  llcX = bbox.getMinX()
463  llcY = bbox.getMinY()
464  width = bbox.getWidth()
465  height = bbox.getHeight()
466  loc.additionalData.set('llcX', llcX)
467  loc.additionalData.set('llcY', llcY)
468  loc.additionalData.set('width', width)
469  loc.additionalData.set('height', height)
470  if 'imageOrigin' in dataId:
471  loc.additionalData.set('imageOrigin',
472  dataId['imageOrigin'])
473  return loc
474 
475  def querySubClosure(key, format, dataId, mapping=mapping):
476  subId = dataId.copy()
477  del subId['bbox']
478  return mapping.lookup(format, subId)
479  setMethods("sub", mapImpl=mapSubClosure, queryImpl=querySubClosure)
480 
481  if subPolicy["storage"] == "FitsCatalogStorage":
482  # Length of catalog
483  setMethods("len", bypassImpl=lambda datasetType, pythonType, location, dataId:
484  afwImage.readMetadata(os.path.join(location.getStorage().root,
485  location.getLocations()[0]),
486  hdu=1).get("NAXIS2"))
487 
488  # Schema of catalog
489  if not datasetType.endswith("_schema") and datasetType + "_schema" not in datasets:
490  setMethods("schema", bypassImpl=lambda datasetType, pythonType, location, dataId:
491  afwTable.Schema.readFits(os.path.join(location.getStorage().root,
492  location.getLocations()[0])))
493 
494  def _computeCcdExposureId(self, dataId):
495  """Compute the 64-bit (long) identifier for a CCD exposure.
496 
497  Subclasses must override
498 
499  @param dataId (dict) Data identifier with visit, ccd
500  """
501  raise NotImplementedError()
502 
503  def _computeCoaddExposureId(self, dataId, singleFilter):
504  """Compute the 64-bit (long) identifier for a coadd.
505 
506  Subclasses must override
507 
508  @param dataId (dict) Data identifier with tract and patch.
509  @param singleFilter (bool) True means the desired ID is for a single-
510  filter coadd, in which case dataId
511  must contain filter.
512  """
513  raise NotImplementedError()
514 
515  def _search(self, path):
516  """Search for path in the associated repository's storage.
517 
518  Parameters
519  ----------
520  path : string
521  Path that describes an object in the repository associated with
522  this mapper.
523  Path may contain an HDU indicator, e.g. 'foo.fits[1]'. The
524  indicator will be stripped when searching and so will match
525  filenames without the HDU indicator, e.g. 'foo.fits'. The path
526  returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
527 
528  Returns
529  -------
530  string
531  The path for this object in the repository. Will return None if the
532  object can't be found. If the input argument path contained an HDU
533  indicator, the returned path will also contain the HDU indicator.
534  """
535  return self.rootStorage.search(path)
536 
537  def backup(self, datasetType, dataId):
538  """Rename any existing object with the given type and dataId.
539 
540  The CameraMapper implementation saves objects in a sequence of e.g.:
541  foo.fits
542  foo.fits~1
543  foo.fits~2
544  All of the backups will be placed in the output repo, however, and will
545  not be removed if they are found elsewhere in the _parent chain. This
546  means that the same file will be stored twice if the previous version was
547  found in an input repo.
548  """
549 
550  # Calling PosixStorage directly is not the long term solution in this
551  # function, this is work-in-progress on epic DM-6225. The plan is for
552  # parentSearch to be changed to 'search', and search only the storage
553  # associated with this mapper. All searching of parents will be handled
554  # by traversing the container of repositories in Butler.
555 
556  def firstElement(list):
557  """Get the first element in the list, or None if that can't be done.
558  """
559  return list[0] if list is not None and len(list) else None
560 
561  n = 0
562  newLocation = self.map(datasetType, dataId, write=True)
563  newPath = newLocation.getLocations()[0]
564  path = dafPersist.PosixStorage.search(self.root, newPath, searchParents=True)
565  path = firstElement(path)
566  oldPaths = []
567  while path is not None:
568  n += 1
569  oldPaths.append((n, path))
570  path = dafPersist.PosixStorage.search(self.root, "%s~%d" % (newPath, n), searchParents=True)
571  path = firstElement(path)
572  for n, oldPath in reversed(oldPaths):
573  self.rootStorage.copyFile(oldPath, "%s~%d" % (newPath, n))
574 
575  def keys(self):
576  """Return supported keys.
577  @return (iterable) List of keys usable in a dataset identifier"""
578  return iter(self.keyDict.keys())
579 
580  def getKeys(self, datasetType, level):
581  """Return a dict of supported keys and their value types for a given dataset
582  type at a given level of the key hierarchy.
583 
584  @param datasetType (str) dataset type or None for all dataset types
585  @param level (str) level or None for all levels or '' for the default level for the camera
586  @return (dict) dict keys are strings usable in a dataset identifier; values are their value types"""
587 
588  # not sure if this is how we want to do this. what if None was intended?
589  if level == '':
590  level = self.getDefaultLevel()
591 
592  if datasetType is None:
593  keyDict = copy.copy(self.keyDict)
594  else:
595  keyDict = self.mappings[datasetType].keys()
596  if level is not None and level in self.levels:
597  keyDict = copy.copy(keyDict)
598  for l in self.levels[level]:
599  if l in keyDict:
600  del keyDict[l]
601  return keyDict
602 
603  def getDefaultLevel(self):
604  return self.defaultLevel
605 
606  def getDefaultSubLevel(self, level):
607  if level in self.defaultSubLevels:
608  return self.defaultSubLevels[level]
609  return None
610 
611  @classmethod
612  def getCameraName(cls):
613  """Return the name of the camera that this CameraMapper is for."""
614  className = str(cls)
615  className = className[className.find('.'):-1]
616  m = re.search(r'(\w+)Mapper', className)
617  if m is None:
618  m = re.search(r"class '[\w.]*?(\w+)'", className)
619  name = m.group(1)
620  return name[:1].lower() + name[1:] if name else ''
621 
622  @classmethod
623  def getPackageName(cls):
624  """Return the name of the package containing this CameraMapper."""
625  if cls.packageName is None:
626  raise ValueError('class variable packageName must not be None')
627  return cls.packageName
628 
629  def map_camera(self, dataId, write=False):
630  """Map a camera dataset."""
631  if self.camera is None:
632  raise RuntimeError("No camera dataset available.")
633  actualId = self._transformId(dataId)
634  return dafPersist.ButlerLocation(
635  pythonType="lsst.afw.cameraGeom.CameraConfig",
636  cppType="Config",
637  storageName="ConfigStorage",
638  locationList=self.cameraDataLocation or "ignored",
639  dataId=actualId,
640  mapper=self,
641  storage=self.rootStorage
642  )
643 
644  def bypass_camera(self, datasetType, pythonType, butlerLocation, dataId):
645  """Return the (preloaded) camera object.
646  """
647  if self.camera is None:
648  raise RuntimeError("No camera dataset available.")
649  return self.camera
650 
651  def map_defects(self, dataId, write=False):
652  """Map defects dataset.
653 
654  @return a very minimal ButlerLocation containing just the locationList field
655  (just enough information that bypass_defects can use it).
656  """
657  defectFitsPath = self._defectLookup(dataId=dataId)
658  if defectFitsPath is None:
659  raise RuntimeError("No defects available for dataId=%s" % (dataId,))
660 
661  return dafPersist.ButlerLocation(None, None, None, defectFitsPath,
662  dataId, self,
663  storage=self.rootStorage)
664 
665  def bypass_defects(self, datasetType, pythonType, butlerLocation, dataId):
666  """Return a defect based on the butler location returned by map_defects
667 
668  @param[in] butlerLocation: a ButlerLocation with locationList = path to defects FITS file
669  @param[in] dataId: the usual data ID; "ccd" must be set
670 
671  Note: the name "bypass_XXX" means the butler makes no attempt to convert the ButlerLocation
672  into an object, which is what we want for now, since that conversion is a bit tricky.
673  """
674  detectorName = self._extractDetectorName(dataId)
675  defectsFitsPath = butlerLocation.locationList[0]
676  with pyfits.open(defectsFitsPath) as hduList:
677  for hdu in hduList[1:]:
678  if hdu.header["name"] != detectorName:
679  continue
680 
681  defectList = []
682  for data in hdu.data:
683  bbox = afwGeom.Box2I(
684  afwGeom.Point2I(int(data['x0']), int(data['y0'])),
685  afwGeom.Extent2I(int(data['width']), int(data['height'])),
686  )
687  defectList.append(afwImage.DefectBase(bbox))
688  return defectList
689 
690  raise RuntimeError("No defects for ccd %s in %s" % (detectorName, defectsFitsPath))
691 
692  def map_expIdInfo(self, dataId, write=False):
693  return dafPersist.ButlerLocation(
694  pythonType="lsst.obs.base.ExposureIdInfo",
695  cppType=None,
696  storageName="Internal",
697  locationList="ignored",
698  dataId=dataId,
699  mapper=self,
700  storage=self.rootStorage
701  )
702 
703  def bypass_expIdInfo(self, datasetType, pythonType, location, dataId):
704  """Hook to retrieve an lsst.obs.base.ExposureIdInfo for an exposure"""
705  expId = self.bypass_ccdExposureId(datasetType, pythonType, location, dataId)
706  expBits = self.bypass_ccdExposureId_bits(datasetType, pythonType, location, dataId)
707  return ExposureIdInfo(expId=expId, expBits=expBits)
708 
709  def std_bfKernel(self, item, dataId):
710  """Disable standardization for bfKernel
711 
712  bfKernel is a calibration product that is numpy array,
713  unlike other calibration products that are all images;
714  all calibration images are sent through _standardizeExposure
715  due to CalibrationMapping, but we don't want that to happen to bfKernel
716  """
717  return item
718 
719  def std_raw(self, item, dataId):
720  """Standardize a raw dataset by converting it to an Exposure instead of an Image"""
721  return self._standardizeExposure(self.exposures['raw'], item, dataId,
722  trimmed=False, setVisitInfo=True)
723 
724  def map_skypolicy(self, dataId):
725  """Map a sky policy."""
726  return dafPersist.ButlerLocation("lsst.pex.policy.Policy", "Policy",
727  "Internal", None, None, self,
728  storage=self.rootStorage)
729 
730  def std_skypolicy(self, item, dataId):
731  """Standardize a sky policy by returning the one we use."""
732  return self.skypolicy
733 
734 ###############################################################################
735 #
736 # Utility functions
737 #
738 ###############################################################################
739 
740  def _getCcdKeyVal(self, dataId):
741  """Return CCD key and value used to look a defect in the defect registry
742 
743  The default implementation simply returns ("ccd", full detector name)
744  """
745  return ("ccd", self._extractDetectorName(dataId))
746 
747  def _setupRegistry(self, name, description, path, policy, policyKey, storage, searchParents=True,
748  posixIfNoSql=True):
749  """Set up a registry (usually SQLite3), trying a number of possible
750  paths.
751 
752  Parameters
753  ----------
754  name : string
755  Name of registry.
756  description: `str`
757  Description of registry (for log messages)
758  path : string
759  Path for registry.
760  policy : string
761  Policy that contains the registry name, used if path is None.
762  policyKey : string
763  Key in policy for registry path.
764  storage : Storage subclass
765  Repository Storage to look in.
766  searchParents : bool, optional
767  True if the search for a registry should follow any Butler v1
768  _parent symlinks.
769  posixIfNoSql : bool, optional
770  If an sqlite registry is not found, will create a posix registry if
771  this is True.
772 
773  Returns
774  -------
775  lsst.daf.persistence.Registry
776  Registry object
777  """
778  if path is None and policyKey in policy:
779  path = dafPersist.LogicalLocation(policy[policyKey]).locString()
780  if os.path.isabs(path):
781  raise RuntimeError("Policy should not indicate an absolute path for registry.")
782  if not storage.exists(path):
783  newPath = storage.instanceSearch(path)
784 
785  newPath = newPath[0] if newPath is not None and len(newPath) else None
786  if newPath is None:
787  self.log.warn("Unable to locate registry at policy path (also looked in root): %s",
788  path)
789  path = newPath
790  else:
791  self.log.warn("Unable to locate registry at policy path: %s", path)
792  path = None
793 
794  # Old Butler API was to indicate the registry WITH the repo folder, New Butler expects the registry to
795  # be in the repo folder. To support Old API, check to see if path starts with root, and if so, strip
796  # root from path. Currently only works with PosixStorage
797  try:
798  root = storage.root
799  if path and (path.startswith(root)):
800  path = path[len(root + '/'):]
801  except AttributeError:
802  pass
803 
804  # determine if there is an sqlite registry and if not, try the posix registry.
805  registry = None
806 
807  def search(filename, description):
808  """Search for file in storage
809 
810  Parameters
811  ----------
812  filename : `str`
813  Filename to search for
814  description : `str`
815  Description of file, for error message.
816 
817  Returns
818  -------
819  path : `str` or `None`
820  Path to file, or None
821  """
822  result = storage.instanceSearch(filename)
823  if result:
824  return result[0]
825  self.log.debug("Unable to locate %s: %s", description, filename)
826  return None
827 
828  # Search for a suitable registry database
829  if path is None:
830  path = search("%s.pgsql" % name, "%s in root" % description)
831  if path is None:
832  path = search("%s.sqlite3" % name, "%s in root" % description)
833  if path is None:
834  path = search(os.path.join(".", "%s.sqlite3" % name), "%s in current dir" % description)
835 
836  if path is not None:
837  if not storage.exists(path):
838  newPath = storage.instanceSearch(path)
839  newPath = newPath[0] if newPath is not None and len(newPath) else None
840  if newPath is not None:
841  path = newPath
842  localFileObj = storage.getLocalFile(path)
843  self.log.info("Loading %s registry from %s", description, localFileObj.name)
844  registry = dafPersist.Registry.create(localFileObj.name)
845  localFileObj.close()
846  elif not registry and posixIfNoSql:
847  try:
848  self.log.info("Loading Posix %s registry from %s", description, storage.root)
849  registry = dafPersist.PosixRegistry(storage.root)
850  except:
851  registry = None
852 
853  return registry
854 
855  def _transformId(self, dataId):
856  """Generate a standard ID dict from a camera-specific ID dict.
857 
858  Canonical keys include:
859  - amp: amplifier name
860  - ccd: CCD name (in LSST this is a combination of raft and sensor)
861  The default implementation returns a copy of its input.
862 
863  @param dataId[in] (dict) Dataset identifier; this must not be modified
864  @return (dict) Transformed dataset identifier"""
865 
866  return dataId.copy()
867 
868  def _mapActualToPath(self, template, actualId):
869  """Convert a template path to an actual path, using the actual data
870  identifier. This implementation is usually sufficient but can be
871  overridden by the subclass.
872  @param template (string) Template path
873  @param actualId (dict) Dataset identifier
874  @return (string) Pathname"""
875 
876  try:
877  transformedId = self._transformId(actualId)
878  return template % transformedId
879  except Exception as e:
880  raise RuntimeError("Failed to format %r with data %r: %s" % (template, transformedId, e))
881 
882  @staticmethod
883  def getShortCcdName(ccdName):
884  """Convert a CCD name to a form useful as a filename
885 
886  The default implementation converts spaces to underscores.
887  """
888  return ccdName.replace(" ", "_")
889 
890  def _extractDetectorName(self, dataId):
891  """Extract the detector (CCD) name from the dataset identifier.
892 
893  The name in question is the detector name used by lsst.afw.cameraGeom.
894 
895  @param dataId (dict) Dataset identifier
896  @return (string) Detector name
897  """
898  raise NotImplementedError("No _extractDetectorName() function specified")
899 
900  def _extractAmpId(self, dataId):
901  """Extract the amplifier identifer from a dataset identifier.
902 
903  @warning this is deprecated; DO NOT USE IT
904 
905  amplifier identifier has two parts: the detector name for the CCD
906  containing the amplifier and index of the amplifier in the detector.
907  @param dataId (dict) Dataset identifer
908  @return (tuple) Amplifier identifier"""
909 
910  trDataId = self._transformId(dataId)
911  return (trDataId["ccd"], int(trDataId['amp']))
912 
913  def _setAmpDetector(self, item, dataId, trimmed=True):
914  """Set the detector object in an Exposure for an amplifier.
915  Defects are also added to the Exposure based on the detector object.
916  @param[in,out] item (lsst.afw.image.Exposure)
917  @param dataId (dict) Dataset identifier
918  @param trimmed (bool) Should detector be marked as trimmed? (ignored)"""
919 
920  return self._setCcdDetector(item=item, dataId=dataId, trimmed=trimmed)
921 
922  def _setCcdDetector(self, item, dataId, trimmed=True):
923  """Set the detector object in an Exposure for a CCD.
924  @param[in,out] item (lsst.afw.image.Exposure)
925  @param dataId (dict) Dataset identifier
926  @param trimmed (bool) Should detector be marked as trimmed? (ignored)"""
927 
928  if item.getDetector() is not None:
929  return
930 
931  detectorName = self._extractDetectorName(dataId)
932  detector = self.camera[detectorName]
933  item.setDetector(detector)
934 
935  def _setFilter(self, mapping, item, dataId):
936  """Set the filter object in an Exposure. If the Exposure had a FILTER
937  keyword, this was already processed during load. But if it didn't,
938  use the filter from the registry.
939  @param mapping (lsst.obs.base.Mapping)
940  @param[in,out] item (lsst.afw.image.Exposure)
941  @param dataId (dict) Dataset identifier"""
942 
943  if not (isinstance(item, afwImage.ExposureU) or isinstance(item, afwImage.ExposureI) or
944  isinstance(item, afwImage.ExposureF) or isinstance(item, afwImage.ExposureD)):
945  return
946 
947  if item.getFilter().getId() != afwImage.Filter.UNKNOWN:
948  return
949 
950  actualId = mapping.need(['filter'], dataId)
951  filterName = actualId['filter']
952  if self.filters is not None and filterName in self.filters:
953  filterName = self.filters[filterName]
954  item.setFilter(afwImage.Filter(filterName))
955 
956  # Default standardization function for exposures
957  def _standardizeExposure(self, mapping, item, dataId, filter=True,
958  trimmed=True, setVisitInfo=True):
959  """Default standardization function for images.
960 
961  This sets the Detector from the camera geometry
962  and optionally set the Fiter. In both cases this saves
963  having to persist some data in each exposure (or image).
964 
965  @param mapping (lsst.obs.base.Mapping)
966  @param[in,out] item image-like object; any of lsst.afw.image.Exposure,
967  lsst.afw.image.DecoratedImage, lsst.afw.image.Image
968  or lsst.afw.image.MaskedImage
969  @param dataId (dict) Dataset identifier
970  @param filter (bool) Set filter? Ignored if item is already an exposure
971  @param trimmed (bool) Should detector be marked as trimmed?
972  @param setVisitInfo (bool) Should Exposure have its VisitInfo filled out from the metadata?
973  @return (lsst.afw.image.Exposure) the standardized Exposure"""
974  try:
975  item = exposureFromImage(item, dataId, mapper=self, logger=self.log, setVisitInfo=setVisitInfo)
976  except Exception as e:
977  self.log.error("Could not turn item=%r into an exposure: %s" % (repr(item), e))
978  raise
979 
980  if mapping.level.lower() == "amp":
981  self._setAmpDetector(item, dataId, trimmed)
982  elif mapping.level.lower() == "ccd":
983  self._setCcdDetector(item, dataId, trimmed)
984 
985  if filter:
986  self._setFilter(mapping, item, dataId)
987 
988  return item
989 
990  def _defectLookup(self, dataId):
991  """Find the defects for a given CCD.
992  @param dataId (dict) Dataset identifier
993  @return (string) path to the defects file or None if not available"""
994  if self.defectRegistry is None:
995  return None
996  if self.registry is None:
997  raise RuntimeError("No registry for defect lookup")
998 
999  ccdKey, ccdVal = self._getCcdKeyVal(dataId)
1000 
1001  dataIdForLookup = {'visit': dataId['visit']}
1002  # .lookup will fail in a posix registry because there is no template to provide.
1003  rows = self.registry.lookup(('taiObs'), ('raw_visit'), dataIdForLookup)
1004  if len(rows) == 0:
1005  return None
1006  assert len(rows) == 1
1007  taiObs = rows[0][0]
1008 
1009  # Lookup the defects for this CCD serial number that are valid at the exposure midpoint.
1010  rows = self.defectRegistry.executeQuery(("path",), ("defect",),
1011  [(ccdKey, "?")],
1012  ("DATETIME(?)", "DATETIME(validStart)", "DATETIME(validEnd)"),
1013  (ccdVal, taiObs))
1014  if not rows or len(rows) == 0:
1015  return None
1016  if len(rows) == 1:
1017  return os.path.join(self.defectPath, rows[0][0])
1018  else:
1019  raise RuntimeError("Querying for defects (%s, %s) returns %d files: %s" %
1020  (ccdVal, taiObs, len(rows), ", ".join([_[0] for _ in rows])))
1021 
1022  def _makeCamera(self, policy, repositoryDir):
1023  """Make a camera (instance of lsst.afw.cameraGeom.Camera) describing the camera geometry
1024 
1025  Also set self.cameraDataLocation, if relevant (else it can be left None).
1026 
1027  This implementation assumes that policy contains an entry "camera" that points to the
1028  subdirectory in this package of camera data; specifically, that subdirectory must contain:
1029  - a file named `camera.py` that contains persisted camera config
1030  - ampInfo table FITS files, as required by lsst.afw.cameraGeom.makeCameraFromPath
1031 
1032  @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility))
1033  Policy with per-camera defaults already merged
1034  @param repositoryDir (string) Policy repository for the subclassing
1035  module (obtained with getRepositoryPath() on the
1036  per-camera default dictionary)
1037  """
1038  if isinstance(policy, pexPolicy.Policy):
1039  policy = dafPersist.Policy(pexPolicy=policy)
1040  if 'camera' not in policy:
1041  raise RuntimeError("Cannot find 'camera' in policy; cannot construct a camera")
1042  cameraDataSubdir = policy['camera']
1043  self.cameraDataLocation = os.path.normpath(
1044  os.path.join(repositoryDir, cameraDataSubdir, "camera.py"))
1045  cameraConfig = afwCameraGeom.CameraConfig()
1046  cameraConfig.load(self.cameraDataLocation)
1047  ampInfoPath = os.path.dirname(self.cameraDataLocation)
1048  return afwCameraGeom.makeCameraFromPath(
1049  cameraConfig=cameraConfig,
1050  ampInfoPath=ampInfoPath,
1051  shortNameFunc=self.getShortCcdName,
1052  pupilFactoryClass=self.PupilFactoryClass
1053  )
1054 
1055  def getRegistry(self):
1056  """Get the registry used by this mapper.
1057 
1058  Returns
1059  -------
1060  Registry or None
1061  The registry used by this mapper for this mapper's repository.
1062  """
1063  return self.registry
1064 
1065 
1066 def exposureFromImage(image, dataId=None, mapper=None, logger=None, setVisitInfo=True):
1067  """Generate an Exposure from an image-like object
1068 
1069  If the image is a DecoratedImage then also set its WCS and metadata
1070  (Image and MaskedImage are missing the necessary metadata
1071  and Exposure already has those set)
1072 
1073  @param[in] image Image-like object (lsst.afw.image.DecoratedImage, Image, MaskedImage or Exposure)
1074  @return (lsst.afw.image.Exposure) Exposure containing input image
1075  """
1076  metadata = None
1077  if isinstance(image, afwImage.MaskedImage):
1078  exposure = afwImage.makeExposure(image)
1079  elif isinstance(image, afwImage.DecoratedImage):
1080  exposure = afwImage.makeExposure(afwImage.makeMaskedImage(image.getImage()))
1081  metadata = image.getMetadata()
1082  try:
1083  wcs = afwImage.makeWcs(metadata, True)
1084  exposure.setWcs(wcs)
1085  except pexExcept.InvalidParameterError as e:
1086  # raised on failure to create a wcs (and possibly others)
1087  if logger is None:
1088  logger = lsstLog.Log.getLogger("CameraMapper")
1089  logger.warn("wcs set to None; insufficient information found in metadata to create a valid wcs: "
1090  "%s", e.args[0])
1091 
1092  exposure.setMetadata(metadata)
1093  elif isinstance(image, afwImage.Exposure):
1094  # Exposure
1095  exposure = image
1096  metadata = exposure.getMetadata()
1097  else:
1098  # Image
1099  exposure = afwImage.makeExposure(afwImage.makeMaskedImage(image))
1100  #
1101  # set VisitInfo if we can
1102  #
1103  if setVisitInfo and exposure.getInfo().getVisitInfo() is None:
1104  if metadata is not None:
1105  if mapper is None:
1106  if not logger:
1107  logger = lsstLog.Log.getLogger("CameraMapper")
1108  logger.warn("I can only set the VisitInfo if you provide a mapper")
1109  else:
1110  exposureId = mapper._computeCcdExposureId(dataId)
1111  visitInfo = mapper.makeRawVisitInfo(md=metadata, exposureId=exposureId)
1112 
1113  exposure.getInfo().setVisitInfo(visitInfo)
1114 
1115  return exposure
def _getCcdKeyVal
Utility functions.