lsst.obs.base  13.0-40-g96cd6c9+3
 All Classes Namespaces Files Functions Variables
cameraMapper.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from builtins import str
24 import copy
25 import os
26 import pyfits # required by _makeDefectsDict until defects are written as AFW tables
27 import re
28 import weakref
29 import lsst.daf.persistence as dafPersist
30 from . import ImageMapping, ExposureMapping, CalibrationMapping, DatasetMapping
31 import lsst.afw.geom as afwGeom
32 import lsst.afw.image as afwImage
33 import lsst.afw.table as afwTable
34 import lsst.afw.cameraGeom as afwCameraGeom
35 import lsst.log as lsstLog
36 import lsst.pex.policy as pexPolicy
37 import lsst.pex.exceptions as pexExcept
38 from .exposureIdInfo import ExposureIdInfo
39 from .makeRawVisitInfo import MakeRawVisitInfo
40 from lsst.utils import getPackageDir
41 
42 """This module defines the CameraMapper base class."""
43 
44 
45 class CameraMapper(dafPersist.Mapper):
46 
47  """CameraMapper is a base class for mappers that handle images from a
48  camera and products derived from them. This provides an abstraction layer
49  between the data on disk and the code.
50 
51  Public methods: keys, queryMetadata, getDatasetTypes, map,
52  canStandardize, standardize
53 
54  Mappers for specific data sources (e.g., CFHT Megacam, LSST
55  simulations, etc.) should inherit this class.
56 
57  The CameraMapper manages datasets within a "root" directory. Note that
58  writing to a dataset present in the input root will hide the existing
59  dataset but not overwrite it. See #2160 for design discussion.
60 
61  A camera is assumed to consist of one or more rafts, each composed of
62  multiple CCDs. Each CCD is in turn composed of one or more amplifiers
63  (amps). A camera is also assumed to have a camera geometry description
64  (CameraGeom object) as a policy file, a filter description (Filter class
65  static configuration) as another policy file, and an optional defects
66  description directory.
67 
68  Information from the camera geometry and defects are inserted into all
69  Exposure objects returned.
70 
71  The mapper uses one or two registries to retrieve metadata about the
72  images. The first is a registry of all raw exposures. This must contain
73  the time of the observation. One or more tables (or the equivalent)
74  within the registry are used to look up data identifier components that
75  are not specified by the user (e.g. filter) and to return results for
76  metadata queries. The second is an optional registry of all calibration
77  data. This should contain validity start and end entries for each
78  calibration dataset in the same timescale as the observation time.
79 
80  Subclasses will typically set MakeRawVisitInfoClass:
81 
82  MakeRawVisitInfoClass: a class variable that points to a subclass of
83  MakeRawVisitInfo, a functor that creates an
84  lsst.afw.image.VisitInfo from the FITS metadata of a raw image.
85 
86  Subclasses must provide the following methods:
87 
88  _extractDetectorName(self, dataId): returns the detector name for a CCD
89  (e.g., "CFHT 21", "R:1,2 S:3,4") as used in the AFW CameraGeom class given
90  a dataset identifier referring to that CCD or a subcomponent of it.
91 
92  _computeCcdExposureId(self, dataId): see below
93 
94  _computeCoaddExposureId(self, dataId, singleFilter): see below
95 
96  Subclasses may also need to override the following methods:
97 
98  _transformId(self, dataId): transformation of a data identifier
99  from colloquial usage (e.g., "ccdname") to proper/actual usage (e.g., "ccd"),
100  including making suitable for path expansion (e.g. removing commas).
101  The default implementation does nothing. Note that this
102  method should not modify its input parameter.
103 
104  getShortCcdName(self, ccdName): a static method that returns a shortened name
105  suitable for use as a filename. The default version converts spaces to underscores.
106 
107  _getCcdKeyVal(self, dataId): return a CCD key and value
108  by which to look up defects in the defects registry.
109  The default value returns ("ccd", detector name)
110 
111  _mapActualToPath(self, template, actualId): convert a template path to an
112  actual path, using the actual dataset identifier.
113 
114  The mapper's behaviors are largely specified by the policy file.
115  See the MapperDictionary.paf for descriptions of the available items.
116 
117  The 'exposures', 'calibrations', and 'datasets' subpolicies configure
118  mappings (see Mappings class).
119 
120  Common default mappings for all subclasses can be specified in the
121  "policy/{images,exposures,calibrations,datasets}.yaml" files. This provides
122  a simple way to add a product to all camera mappers.
123 
124  Functions to map (provide a path to the data given a dataset
125  identifier dictionary) and standardize (convert data into some standard
126  format or type) may be provided in the subclass as "map_{dataset type}"
127  and "std_{dataset type}", respectively.
128 
129  If non-Exposure datasets cannot be retrieved using standard
130  daf_persistence methods alone, a "bypass_{dataset type}" function may be
131  provided in the subclass to return the dataset instead of using the
132  "datasets" subpolicy.
133 
134  Implementations of map_camera and bypass_camera that should typically be
135  sufficient are provided in this base class.
136 
137  @todo
138  * Handle defects the same was as all other calibration products, using the calibration registry
139  * Instead of auto-loading the camera at construction time, load it from the calibration registry
140  * Rewrite defects as AFW tables so we don't need pyfits to unpersist them; then remove all mention
141  of pyfits from this package.
142  """
143  packageName = None
144 
145  # a class or subclass of MakeRawVisitInfo, a functor that makes an
146  # lsst.afw.image.VisitInfo from the FITS metadata of a raw image
147  MakeRawVisitInfoClass = MakeRawVisitInfo
148 
149  # a class or subclass of PupilFactory
150  PupilFactoryClass = afwCameraGeom.PupilFactory
151 
152  def __init__(self, policy, repositoryDir,
153  root=None, registry=None, calibRoot=None, calibRegistry=None,
154  provided=None, parentRegistry=None, repositoryCfg=None):
155  """Initialize the CameraMapper.
156 
157  Parameters
158  ----------
159  policy : daf_persistence.Policy,
160  Can also be pexPolicy.Policy, only for backward compatibility.
161  Policy with per-camera defaults already merged.
162  repositoryDir : string
163  Policy repository for the subclassing module (obtained with
164  getRepositoryPath() on the per-camera default dictionary).
165  root : string, optional
166  Path to the root directory for data.
167  registry : string, optional
168  Path to registry with data's metadata.
169  calibRoot : string, optional
170  Root directory for calibrations.
171  calibRegistry : string, optional
172  Path to registry with calibrations' metadata.
173  provided : list of string, optional
174  Keys provided by the mapper.
175  parentRegistry : Registry subclass, optional
176  Registry from a parent repository that may be used to look up
177  data's metadata.
178  repositoryCfg : daf_persistence.RepositoryCfg or None, optional
179  The configuration information for the repository this mapper is
180  being used with.
181  """
182 
183  dafPersist.Mapper.__init__(self)
184 
185  self.log = lsstLog.Log.getLogger("CameraMapper")
186 
187  if root:
188  self.root = root
189  elif repositoryCfg:
190  self.root = repositoryCfg.root
191  else:
192  self.root = None
193  if isinstance(policy, pexPolicy.Policy):
194  policy = dafPersist.Policy(policy)
195 
196  repoPolicy = repositoryCfg.policy if repositoryCfg else None
197  if repoPolicy is not None:
198  policy.update(repoPolicy)
199 
200  defaultPolicyFile = dafPersist.Policy.defaultPolicyFile("obs_base",
201  "MapperDictionary.paf",
202  "policy")
203  dictPolicy = dafPersist.Policy(defaultPolicyFile)
204  policy.merge(dictPolicy)
205 
206  # Levels
207  self.levels = dict()
208  if 'levels' in policy:
209  levelsPolicy = policy['levels']
210  for key in levelsPolicy.names(True):
211  self.levels[key] = set(levelsPolicy.asArray(key))
212  self.defaultLevel = policy['defaultLevel']
213  self.defaultSubLevels = dict()
214  if 'defaultSubLevels' in policy:
215  self.defaultSubLevels = policy['defaultSubLevels']
216 
217  # Root directories
218  if root is None:
219  root = "."
220  root = dafPersist.LogicalLocation(root).locString()
221 
222  self.rootStorage = dafPersist.Storage.makeFromURI(uri=root)
223 
224  # If the calibRoot is passed in, use that. If not and it's indicated in
225  # the policy, use that. And otherwise, the calibs are in the regular
226  # root.
227  # If the location indicated by the calib root does not exist, do not
228  # create it.
229  calibStorage = None
230  if calibRoot is not None:
231  calibRoot = dafPersist.Storage.absolutePath(root, calibRoot)
232  calibStorage = dafPersist.Storage.makeFromURI(uri=calibRoot,
233  create=False)
234  else:
235  calibRoot = policy.get('calibRoot', None)
236  if calibRoot:
237  calibStorage = dafPersist.Storage.makeFromURI(uri=calibRoot,
238  create=False)
239  if calibStorage is None:
240  calibStorage = self.rootStorage
241 
242  self.root = root
243 
244  # Registries
245  self.registry = self._setupRegistry("registry", "exposure", registry, policy, "registryPath",
246  self.rootStorage, searchParents=False,
247  posixIfNoSql=(not parentRegistry))
248  if not self.registry:
249  self.registry = parentRegistry
250  needCalibRegistry = policy.get('needCalibRegistry', None)
251  if needCalibRegistry:
252  if calibStorage:
253  self.calibRegistry = self._setupRegistry("calibRegistry", "calib", calibRegistry, policy,
254  "calibRegistryPath", calibStorage)
255  else:
256  raise RuntimeError(
257  "'needCalibRegistry' is true in Policy, but was unable to locate a repo at " +
258  "calibRoot ivar:%s or policy['calibRoot']:%s" %
259  (calibRoot, policy.get('calibRoot', None)))
260  else:
261  self.calibRegistry = None
262 
263  # Dict of valid keys and their value types
264  self.keyDict = dict()
265 
266  self._initMappings(policy, self.rootStorage, calibStorage, provided=None)
267 
268  # Camera geometry
269  self.cameraDataLocation = None # path to camera geometry config file
270  self.camera = self._makeCamera(policy=policy, repositoryDir=repositoryDir)
271 
272  # Defect registry and root. Defects are stored with the camera and the registry is loaded from the
273  # camera package, which is on the local filesystem.
274  self.defectRegistry = None
275  if 'defects' in policy:
276  self.defectPath = os.path.join(repositoryDir, policy['defects'])
277  defectRegistryLocation = os.path.join(self.defectPath, "defectRegistry.sqlite3")
278  self.defectRegistry = dafPersist.Registry.create(defectRegistryLocation)
279 
280  # Filter translation table
281  self.filters = None
282 
283  # Skytile policy
284  self.skypolicy = policy['skytiles']
285 
286  # verify that the class variable packageName is set before attempting
287  # to instantiate an instance
288  if self.packageName is None:
289  raise ValueError('class variable packageName must not be None')
290 
292 
293  def _initMappings(self, policy, rootStorage=None, calibStorage=None, provided=None):
294  """Initialize mappings
295 
296  For each of the dataset types that we want to be able to read, there are
297  methods that can be created to support them:
298  * map_<dataset> : determine the path for dataset
299  * std_<dataset> : standardize the retrieved dataset
300  * bypass_<dataset> : retrieve the dataset (bypassing the usual retrieval machinery)
301  * query_<dataset> : query the registry
302 
303  Besides the dataset types explicitly listed in the policy, we create
304  additional, derived datasets for additional conveniences, e.g., reading
305  the header of an image, retrieving only the size of a catalog.
306 
307  @param policy (Policy) Policy with per-camera defaults already merged
308  @param rootStorage (Storage subclass instance) Interface to persisted repository data
309  @param calibRoot (Storage subclass instance) Interface to persisted calib repository data
310  @param provided (list of strings) Keys provided by the mapper
311  """
312  # Sub-dictionaries (for exposure/calibration/dataset types)
313  imgMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
314  "obs_base", "ImageMappingDictionary.paf", "policy"))
315  expMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
316  "obs_base", "ExposureMappingDictionary.paf", "policy"))
317  calMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
318  "obs_base", "CalibrationMappingDictionary.paf", "policy"))
319  dsMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
320  "obs_base", "DatasetMappingDictionary.paf", "policy"))
321 
322  # Mappings
323  mappingList = (
324  ("images", imgMappingPolicy, ImageMapping),
325  ("exposures", expMappingPolicy, ExposureMapping),
326  ("calibrations", calMappingPolicy, CalibrationMapping),
327  ("datasets", dsMappingPolicy, DatasetMapping)
328  )
329  self.mappings = dict()
330  for name, defPolicy, cls in mappingList:
331  if name in policy:
332  datasets = policy[name]
333 
334  # Centrally-defined datasets
335  defaultsPath = os.path.join(getPackageDir("obs_base"), "policy", name + ".yaml")
336  if os.path.exists(defaultsPath):
337  datasets.merge(dafPersist.Policy(defaultsPath))
338 
339  mappings = dict()
340  setattr(self, name, mappings)
341  for datasetType in datasets.names(True):
342  subPolicy = datasets[datasetType]
343  subPolicy.merge(defPolicy)
344 
345  if not hasattr(self, "map_" + datasetType) and 'composite' in subPolicy:
346  def compositeClosure(dataId, write=False, mapper=None, mapping=None, subPolicy=subPolicy):
347  components = subPolicy.get('composite')
348  assembler = subPolicy['assembler'] if 'assembler' in subPolicy else None
349  disassembler = subPolicy['disassembler'] if 'disassembler' in subPolicy else None
350  python = subPolicy['python']
351  butlerComposite = dafPersist.ButlerComposite(assembler=assembler,
352  disassembler=disassembler,
353  python=python,
354  dataId=dataId,
355  mapper=self)
356  for name, component in components.items():
357  butlerComposite.add(id=name,
358  datasetType=component.get('datasetType'),
359  setter=component.get('setter', None),
360  getter=component.get('getter', None),
361  subset=component.get('subset', False),
362  inputOnly=component.get('inputOnly', False))
363  return butlerComposite
364  setattr(self, "map_" + datasetType, compositeClosure)
365  # for now at least, don't set up any other handling for this dataset type.
366  continue
367 
368  if name == "calibrations":
369  mapping = cls(datasetType, subPolicy, self.registry, self.calibRegistry, calibStorage,
370  provided=provided, dataRoot=rootStorage)
371  else:
372  mapping = cls(datasetType, subPolicy, self.registry, rootStorage, provided=provided)
373  self.keyDict.update(mapping.keys())
374  mappings[datasetType] = mapping
375  self.mappings[datasetType] = mapping
376  if not hasattr(self, "map_" + datasetType):
377  def mapClosure(dataId, write=False, mapper=weakref.proxy(self), mapping=mapping):
378  return mapping.map(mapper, dataId, write)
379  setattr(self, "map_" + datasetType, mapClosure)
380  if not hasattr(self, "query_" + datasetType):
381  def queryClosure(format, dataId, mapping=mapping):
382  return mapping.lookup(format, dataId)
383  setattr(self, "query_" + datasetType, queryClosure)
384  if hasattr(mapping, "standardize") and not hasattr(self, "std_" + datasetType):
385  def stdClosure(item, dataId, mapper=weakref.proxy(self), mapping=mapping):
386  return mapping.standardize(mapper, item, dataId)
387  setattr(self, "std_" + datasetType, stdClosure)
388 
389  def setMethods(suffix, mapImpl=None, bypassImpl=None, queryImpl=None):
390  """Set convenience methods on CameraMapper"""
391  mapName = "map_" + datasetType + "_" + suffix
392  bypassName = "bypass_" + datasetType + "_" + suffix
393  queryName = "query_" + datasetType + "_" + suffix
394  if not hasattr(self, mapName):
395  setattr(self, mapName, mapImpl or getattr(self, "map_" + datasetType))
396  if not hasattr(self, bypassName):
397  if bypassImpl is None and hasattr(self, "bypass_" + datasetType):
398  bypassImpl = getattr(self, "bypass_" + datasetType)
399  if bypassImpl is not None:
400  setattr(self, bypassName, bypassImpl)
401  if not hasattr(self, queryName):
402  setattr(self, queryName, queryImpl or getattr(self, "query_" + datasetType))
403 
404  # Filename of dataset
405  setMethods("filename", bypassImpl=lambda datasetType, pythonType, location, dataId:
406  [os.path.join(location.getStorage().root, p) for p in location.getLocations()])
407  # Metadata from FITS file
408  if subPolicy["storage"] == "FitsStorage": # a FITS image
409  setMethods("md", bypassImpl=lambda datasetType, pythonType, location, dataId:
410  afwImage.readMetadata(location.getLocationsWithRoot()[0]))
411  if name == "exposures":
412  setMethods("wcs", bypassImpl=lambda datasetType, pythonType, location, dataId:
413  afwImage.makeWcs(
414  afwImage.readMetadata(location.getLocationsWithRoot()[0])))
415  setMethods("calib", bypassImpl=lambda datasetType, pythonType, location, dataId:
416  afwImage.Calib(
417  afwImage.readMetadata(location.getLocationsWithRoot()[0])))
418  setMethods("visitInfo",
419  bypassImpl=lambda datasetType, pythonType, location, dataId:
420  afwImage.VisitInfo(
421  afwImage.readMetadata(location.getLocationsWithRoot()[0])))
422  setMethods("filter",
423  bypassImpl=lambda datasetType, pythonType, location, dataId:
424  afwImage.Filter(
425  afwImage.readMetadata(location.getLocationsWithRoot()[0])))
426  setMethods("detector",
427  mapImpl=lambda dataId, write=False:
428  dafPersist.ButlerLocation(
429  pythonType="lsst.afw.cameraGeom.CameraConfig",
430  cppType="Config",
431  storageName="Internal",
432  locationList="ignored",
433  dataId=dataId,
434  mapper=self,
435  storage=None,
436  ),
437  bypassImpl=lambda datasetType, pythonType, location, dataId:
438  self.camera[self._extractDetectorName(dataId)]
439  )
440  setMethods("bbox", bypassImpl=lambda dsType, pyType, location, dataId:
441  afwImage.bboxFromMetadata(
442  afwImage.readMetadata(location.getLocationsWithRoot()[0], hdu=1)))
443 
444  elif name == "images":
445  setMethods("bbox", bypassImpl=lambda dsType, pyType, location, dataId:
446  afwImage.bboxFromMetadata(
447  afwImage.readMetadata(location.getLocationsWithRoot()[0])))
448 
449  if subPolicy["storage"] == "FitsCatalogStorage": # a FITS catalog
450  setMethods("md", bypassImpl=lambda datasetType, pythonType, location, dataId:
451  afwImage.readMetadata(os.path.join(location.getStorage().root,
452  location.getLocations()[0]), hdu=1))
453 
454  # Sub-images
455  if subPolicy["storage"] == "FitsStorage":
456  def mapSubClosure(dataId, write=False, mapper=weakref.proxy(self), mapping=mapping):
457  subId = dataId.copy()
458  del subId['bbox']
459  loc = mapping.map(mapper, subId, write)
460  bbox = dataId['bbox']
461  llcX = bbox.getMinX()
462  llcY = bbox.getMinY()
463  width = bbox.getWidth()
464  height = bbox.getHeight()
465  loc.additionalData.set('llcX', llcX)
466  loc.additionalData.set('llcY', llcY)
467  loc.additionalData.set('width', width)
468  loc.additionalData.set('height', height)
469  if 'imageOrigin' in dataId:
470  loc.additionalData.set('imageOrigin',
471  dataId['imageOrigin'])
472  return loc
473  def querySubClosure(key, format, dataId, mapping=mapping):
474  subId = dataId.copy()
475  del subId['bbox']
476  return mapping.lookup(format, subId)
477  setMethods("sub", mapImpl=mapSubClosure, queryImpl=querySubClosure)
478 
479  if subPolicy["storage"] == "FitsCatalogStorage":
480  # Length of catalog
481  setMethods("len", bypassImpl=lambda datasetType, pythonType, location, dataId:
482  afwImage.readMetadata(os.path.join(location.getStorage().root,
483  location.getLocations()[0]),
484  hdu=1).get("NAXIS2"))
485 
486  # Schema of catalog
487  if not datasetType.endswith("_schema") and datasetType + "_schema" not in datasets:
488  setMethods("schema", bypassImpl=lambda datasetType, pythonType, location, dataId:
489  afwTable.Schema.readFits(os.path.join(location.getStorage().root,
490  location.getLocations()[0])))
491 
492  def _computeCcdExposureId(self, dataId):
493  """Compute the 64-bit (long) identifier for a CCD exposure.
494 
495  Subclasses must override
496 
497  @param dataId (dict) Data identifier with visit, ccd
498  """
499  raise NotImplementedError()
500 
501  def _computeCoaddExposureId(self, dataId, singleFilter):
502  """Compute the 64-bit (long) identifier for a coadd.
503 
504  Subclasses must override
505 
506  @param dataId (dict) Data identifier with tract and patch.
507  @param singleFilter (bool) True means the desired ID is for a single-
508  filter coadd, in which case dataId
509  must contain filter.
510  """
511  raise NotImplementedError()
512 
513  def _search(self, path):
514  """Search for path in the associated repository's storage.
515 
516  Parameters
517  ----------
518  path : string
519  Path that describes an object in the repository associated with
520  this mapper.
521  Path may contain an HDU indicator, e.g. 'foo.fits[1]'. The
522  indicator will be stripped when searching and so will match
523  filenames without the HDU indicator, e.g. 'foo.fits'. The path
524  returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
525 
526  Returns
527  -------
528  string
529  The path for this object in the repository. Will return None if the
530  object can't be found. If the input argument path contained an HDU
531  indicator, the returned path will also contain the HDU indicator.
532  """
533  return self.rootStorage.search(path)
534 
535  def backup(self, datasetType, dataId):
536  """Rename any existing object with the given type and dataId.
537 
538  The CameraMapper implementation saves objects in a sequence of e.g.:
539  foo.fits
540  foo.fits~1
541  foo.fits~2
542  All of the backups will be placed in the output repo, however, and will
543  not be removed if they are found elsewhere in the _parent chain. This
544  means that the same file will be stored twice if the previous version was
545  found in an input repo.
546  """
547 
548  # Calling PosixStorage directly is not the long term solution in this
549  # function, this is work-in-progress on epic DM-6225. The plan is for
550  # parentSearch to be changed to 'search', and search only the storage
551  # associated with this mapper. All searching of parents will be handled
552  # by traversing the container of repositories in Butler.
553 
554  def firstElement(list):
555  """Get the first element in the list, or None if that can't be done.
556  """
557  return list[0] if list is not None and len(list) else None
558 
559  n = 0
560  newLocation = self.map(datasetType, dataId, write=True)
561  newPath = newLocation.getLocations()[0]
562  path = dafPersist.PosixStorage.search(self.root, newPath, searchParents=True)
563  path = firstElement(path)
564  oldPaths = []
565  while path is not None:
566  n += 1
567  oldPaths.append((n, path))
568  path = dafPersist.PosixStorage.search(self.root, "%s~%d" % (newPath, n), searchParents=True)
569  path = firstElement(path)
570  for n, oldPath in reversed(oldPaths):
571  self.rootStorage.copyFile(oldPath, "%s~%d" % (newPath, n))
572 
573  def keys(self):
574  """Return supported keys.
575  @return (iterable) List of keys usable in a dataset identifier"""
576  return iter(self.keyDict.keys())
577 
578  def getKeys(self, datasetType, level):
579  """Return a dict of supported keys and their value types for a given dataset
580  type at a given level of the key hierarchy.
581 
582  @param datasetType (str) dataset type or None for all dataset types
583  @param level (str) level or None for all levels or '' for the default level for the camera
584  @return (dict) dict keys are strings usable in a dataset identifier; values are their value types"""
585 
586  # not sure if this is how we want to do this. what if None was intended?
587  if level == '':
588  level = self.getDefaultLevel()
589 
590  if datasetType is None:
591  keyDict = copy.copy(self.keyDict)
592  else:
593  keyDict = self.mappings[datasetType].keys()
594  if level is not None and level in self.levels:
595  keyDict = copy.copy(keyDict)
596  for l in self.levels[level]:
597  if l in keyDict:
598  del keyDict[l]
599  return keyDict
600 
601  def getDefaultLevel(self):
602  return self.defaultLevel
603 
604  def getDefaultSubLevel(self, level):
605  if level in self.defaultSubLevels:
606  return self.defaultSubLevels[level]
607  return None
608 
609  @classmethod
610  def getCameraName(cls):
611  """Return the name of the camera that this CameraMapper is for."""
612  className = str(cls)
613  className = className[className.find('.'):-1]
614  m = re.search(r'(\w+)Mapper', className)
615  if m is None:
616  m = re.search(r"class '[\w.]*?(\w+)'", className)
617  name = m.group(1)
618  return name[:1].lower() + name[1:] if name else ''
619 
620  @classmethod
621  def getPackageName(cls):
622  """Return the name of the package containing this CameraMapper."""
623  if cls.packageName is None:
624  raise ValueError('class variable packageName must not be None')
625  return cls.packageName
626 
627  def map_camera(self, dataId, write=False):
628  """Map a camera dataset."""
629  if self.camera is None:
630  raise RuntimeError("No camera dataset available.")
631  actualId = self._transformId(dataId)
632  return dafPersist.ButlerLocation(
633  pythonType="lsst.afw.cameraGeom.CameraConfig",
634  cppType="Config",
635  storageName="ConfigStorage",
636  locationList=self.cameraDataLocation or "ignored",
637  dataId=actualId,
638  mapper=self,
639  storage=self.rootStorage
640  )
641 
642  def bypass_camera(self, datasetType, pythonType, butlerLocation, dataId):
643  """Return the (preloaded) camera object.
644  """
645  if self.camera is None:
646  raise RuntimeError("No camera dataset available.")
647  return self.camera
648 
649  def map_defects(self, dataId, write=False):
650  """Map defects dataset.
651 
652  @return a very minimal ButlerLocation containing just the locationList field
653  (just enough information that bypass_defects can use it).
654  """
655  defectFitsPath = self._defectLookup(dataId=dataId)
656  if defectFitsPath is None:
657  raise RuntimeError("No defects available for dataId=%s" % (dataId,))
658 
659  return dafPersist.ButlerLocation(None, None, None, defectFitsPath,
660  dataId, self,
661  storage=self.rootStorage)
662 
663  def bypass_defects(self, datasetType, pythonType, butlerLocation, dataId):
664  """Return a defect based on the butler location returned by map_defects
665 
666  @param[in] butlerLocation: a ButlerLocation with locationList = path to defects FITS file
667  @param[in] dataId: the usual data ID; "ccd" must be set
668 
669  Note: the name "bypass_XXX" means the butler makes no attempt to convert the ButlerLocation
670  into an object, which is what we want for now, since that conversion is a bit tricky.
671  """
672  detectorName = self._extractDetectorName(dataId)
673  defectsFitsPath = butlerLocation.locationList[0]
674  with pyfits.open(defectsFitsPath) as hduList:
675  for hdu in hduList[1:]:
676  if hdu.header["name"] != detectorName:
677  continue
678 
679  defectList = []
680  for data in hdu.data:
681  bbox = afwGeom.Box2I(
682  afwGeom.Point2I(int(data['x0']), int(data['y0'])),
683  afwGeom.Extent2I(int(data['width']), int(data['height'])),
684  )
685  defectList.append(afwImage.DefectBase(bbox))
686  return defectList
687 
688  raise RuntimeError("No defects for ccd %s in %s" % (detectorName, defectsFitsPath))
689 
690  def map_expIdInfo(self, dataId, write=False):
691  return dafPersist.ButlerLocation(
692  pythonType="lsst.obs.base.ExposureIdInfo",
693  cppType=None,
694  storageName="Internal",
695  locationList="ignored",
696  dataId=dataId,
697  mapper=self,
698  storage=self.rootStorage
699  )
700 
701  def bypass_expIdInfo(self, datasetType, pythonType, location, dataId):
702  """Hook to retrieve an lsst.obs.base.ExposureIdInfo for an exposure"""
703  expId = self.bypass_ccdExposureId(datasetType, pythonType, location, dataId)
704  expBits = self.bypass_ccdExposureId_bits(datasetType, pythonType, location, dataId)
705  return ExposureIdInfo(expId=expId, expBits=expBits)
706 
707  def std_bfKernel(self, item, dataId):
708  """Disable standardization for bfKernel
709 
710  bfKernel is a calibration product that is numpy array,
711  unlike other calibration products that are all images;
712  all calibration images are sent through _standardizeExposure
713  due to CalibrationMapping, but we don't want that to happen to bfKernel
714  """
715  return item
716 
717  def std_raw(self, item, dataId):
718  """Standardize a raw dataset by converting it to an Exposure instead of an Image"""
719  return self._standardizeExposure(self.exposures['raw'], item, dataId,
720  trimmed=False, setVisitInfo=True)
721 
722  def map_skypolicy(self, dataId):
723  """Map a sky policy."""
724  return dafPersist.ButlerLocation("lsst.pex.policy.Policy", "Policy",
725  "Internal", None, None, self,
726  storage=self.rootStorage)
727 
728  def std_skypolicy(self, item, dataId):
729  """Standardize a sky policy by returning the one we use."""
730  return self.skypolicy
731 
732 ###############################################################################
733 #
734 # Utility functions
735 #
736 ###############################################################################
737 
738  def _getCcdKeyVal(self, dataId):
739  """Return CCD key and value used to look a defect in the defect registry
740 
741  The default implementation simply returns ("ccd", full detector name)
742  """
743  return ("ccd", self._extractDetectorName(dataId))
744 
745  def _setupRegistry(self, name, description, path, policy, policyKey, storage, searchParents=True,
746  posixIfNoSql=True):
747  """Set up a registry (usually SQLite3), trying a number of possible
748  paths.
749 
750  Parameters
751  ----------
752  name : string
753  Name of registry.
754  description: `str`
755  Description of registry (for log messages)
756  path : string
757  Path for registry.
758  policy : string
759  Policy that contains the registry name, used if path is None.
760  policyKey : string
761  Key in policy for registry path.
762  storage : Storage subclass
763  Repository Storage to look in.
764  searchParents : bool, optional
765  True if the search for a registry should follow any Butler v1
766  _parent symlinks.
767  posixIfNoSql : bool, optional
768  If an sqlite registry is not found, will create a posix registry if
769  this is True.
770 
771  Returns
772  -------
773  lsst.daf.persistence.Registry
774  Registry object
775  """
776  if path is None and policyKey in policy:
777  path = dafPersist.LogicalLocation(policy[policyKey]).locString()
778  if os.path.isabs(path):
779  raise RuntimeError("Policy should not indicate an absolute path for registry.")
780  if not storage.exists(path):
781  newPath = storage.instanceSearch(path)
782 
783  newPath = newPath[0] if newPath is not None and len(newPath) else None
784  if newPath is None:
785  self.log.warn("Unable to locate registry at policy path (also looked in root): %s",
786  path)
787  path = newPath
788  else:
789  self.log.warn("Unable to locate registry at policy path: %s", path)
790  path = None
791 
792  # Old Butler API was to indicate the registry WITH the repo folder, New Butler expects the registry to
793  # be in the repo folder. To support Old API, check to see if path starts with root, and if so, strip
794  # root from path. Currently only works with PosixStorage
795  try:
796  root = storage.root
797  if path and (path.startswith(root)):
798  path = path[len(root + '/'):]
799  except AttributeError:
800  pass
801 
802  # determine if there is an sqlite registry and if not, try the posix registry.
803  registry = None
804 
805  def search(filename, description):
806  """Search for file in storage
807 
808  Parameters
809  ----------
810  filename : `str`
811  Filename to search for
812  description : `str`
813  Description of file, for error message.
814 
815  Returns
816  -------
817  path : `str` or `None`
818  Path to file, or None
819  """
820  result = storage.instanceSearch(filename)
821  if result:
822  return result[0]
823  self.log.debug("Unable to locate %s: %s", description, filename)
824  return None
825 
826  # Search for a suitable registry database
827  if path is None:
828  path = search("%s.pgsql" % name, "%s in root" % description)
829  if path is None:
830  path = search("%s.sqlite3" % name, "%s in root" % description)
831  if path is None:
832  path = search(os.path.join(".", "%s.sqlite3" % name), "%s in current dir" % description)
833 
834  if path is not None:
835  if not storage.exists(path):
836  newPath = storage.instanceSearch(path)
837  newPath = newPath[0] if newPath is not None and len(newPath) else None
838  if newPath is not None:
839  path = newPath
840  localFileObj = storage.getLocalFile(path)
841  self.log.info("Loading %s registry from %s", description, localFileObj.name)
842  registry = dafPersist.Registry.create(localFileObj.name)
843  elif not registry and posixIfNoSql:
844  try:
845  self.log.info("Loading Posix %s registry from %s", description, storage.root)
846  registry = dafPersist.PosixRegistry(storage.root)
847  except:
848  registry = None
849 
850  return registry
851 
852  def _transformId(self, dataId):
853  """Generate a standard ID dict from a camera-specific ID dict.
854 
855  Canonical keys include:
856  - amp: amplifier name
857  - ccd: CCD name (in LSST this is a combination of raft and sensor)
858  The default implementation returns a copy of its input.
859 
860  @param dataId[in] (dict) Dataset identifier; this must not be modified
861  @return (dict) Transformed dataset identifier"""
862 
863  return dataId.copy()
864 
865  def _mapActualToPath(self, template, actualId):
866  """Convert a template path to an actual path, using the actual data
867  identifier. This implementation is usually sufficient but can be
868  overridden by the subclass.
869  @param template (string) Template path
870  @param actualId (dict) Dataset identifier
871  @return (string) Pathname"""
872 
873  try:
874  transformedId = self._transformId(actualId)
875  return template % transformedId
876  except Exception as e:
877  raise RuntimeError("Failed to format %r with data %r: %s" % (template, transformedId, e))
878 
879  @staticmethod
880  def getShortCcdName(ccdName):
881  """Convert a CCD name to a form useful as a filename
882 
883  The default implementation converts spaces to underscores.
884  """
885  return ccdName.replace(" ", "_")
886 
887  def _extractDetectorName(self, dataId):
888  """Extract the detector (CCD) name from the dataset identifier.
889 
890  The name in question is the detector name used by lsst.afw.cameraGeom.
891 
892  @param dataId (dict) Dataset identifier
893  @return (string) Detector name
894  """
895  raise NotImplementedError("No _extractDetectorName() function specified")
896 
897  def _extractAmpId(self, dataId):
898  """Extract the amplifier identifer from a dataset identifier.
899 
900  @warning this is deprecated; DO NOT USE IT
901 
902  amplifier identifier has two parts: the detector name for the CCD
903  containing the amplifier and index of the amplifier in the detector.
904  @param dataId (dict) Dataset identifer
905  @return (tuple) Amplifier identifier"""
906 
907  trDataId = self._transformId(dataId)
908  return (trDataId["ccd"], int(trDataId['amp']))
909 
910  def _setAmpDetector(self, item, dataId, trimmed=True):
911  """Set the detector object in an Exposure for an amplifier.
912  Defects are also added to the Exposure based on the detector object.
913  @param[in,out] item (lsst.afw.image.Exposure)
914  @param dataId (dict) Dataset identifier
915  @param trimmed (bool) Should detector be marked as trimmed? (ignored)"""
916 
917  return self._setCcdDetector(item=item, dataId=dataId, trimmed=trimmed)
918 
919  def _setCcdDetector(self, item, dataId, trimmed=True):
920  """Set the detector object in an Exposure for a CCD.
921  @param[in,out] item (lsst.afw.image.Exposure)
922  @param dataId (dict) Dataset identifier
923  @param trimmed (bool) Should detector be marked as trimmed? (ignored)"""
924 
925  if item.getDetector() is not None:
926  return
927 
928  detectorName = self._extractDetectorName(dataId)
929  detector = self.camera[detectorName]
930  item.setDetector(detector)
931 
932  def _setFilter(self, mapping, item, dataId):
933  """Set the filter object in an Exposure. If the Exposure had a FILTER
934  keyword, this was already processed during load. But if it didn't,
935  use the filter from the registry.
936  @param mapping (lsst.obs.base.Mapping)
937  @param[in,out] item (lsst.afw.image.Exposure)
938  @param dataId (dict) Dataset identifier"""
939 
940  if not (isinstance(item, afwImage.ExposureU) or isinstance(item, afwImage.ExposureI) or
941  isinstance(item, afwImage.ExposureF) or isinstance(item, afwImage.ExposureD)):
942  return
943 
944  if item.getFilter().getId() != afwImage.Filter.UNKNOWN:
945  return
946 
947  actualId = mapping.need(['filter'], dataId)
948  filterName = actualId['filter']
949  if self.filters is not None and filterName in self.filters:
950  filterName = self.filters[filterName]
951  item.setFilter(afwImage.Filter(filterName))
952 
953  # Default standardization function for exposures
954  def _standardizeExposure(self, mapping, item, dataId, filter=True,
955  trimmed=True, setVisitInfo=True):
956  """Default standardization function for images.
957 
958  This sets the Detector from the camera geometry
959  and optionally set the Fiter. In both cases this saves
960  having to persist some data in each exposure (or image).
961 
962  @param mapping (lsst.obs.base.Mapping)
963  @param[in,out] item image-like object; any of lsst.afw.image.Exposure,
964  lsst.afw.image.DecoratedImage, lsst.afw.image.Image
965  or lsst.afw.image.MaskedImage
966  @param dataId (dict) Dataset identifier
967  @param filter (bool) Set filter? Ignored if item is already an exposure
968  @param trimmed (bool) Should detector be marked as trimmed?
969  @param setVisitInfo (bool) Should Exposure have its VisitInfo filled out from the metadata?
970  @return (lsst.afw.image.Exposure) the standardized Exposure"""
971  try:
972  item = exposureFromImage(item, dataId, mapper=self, logger=self.log, setVisitInfo=setVisitInfo)
973  except Exception as e:
974  self.log.error("Could not turn item=%r into an exposure: %s" % (repr(item), e))
975  raise
976 
977  if mapping.level.lower() == "amp":
978  self._setAmpDetector(item, dataId, trimmed)
979  elif mapping.level.lower() == "ccd":
980  self._setCcdDetector(item, dataId, trimmed)
981 
982  if filter:
983  self._setFilter(mapping, item, dataId)
984 
985  return item
986 
987  def _defectLookup(self, dataId):
988  """Find the defects for a given CCD.
989  @param dataId (dict) Dataset identifier
990  @return (string) path to the defects file or None if not available"""
991  if self.defectRegistry is None:
992  return None
993  if self.registry is None:
994  raise RuntimeError("No registry for defect lookup")
995 
996  ccdKey, ccdVal = self._getCcdKeyVal(dataId)
997 
998  dataIdForLookup = {'visit': dataId['visit']}
999  # .lookup will fail in a posix registry because there is no template to provide.
1000  rows = self.registry.lookup(('taiObs'), ('raw_visit'), dataIdForLookup)
1001  if len(rows) == 0:
1002  return None
1003  assert len(rows) == 1
1004  taiObs = rows[0][0]
1005 
1006  # Lookup the defects for this CCD serial number that are valid at the exposure midpoint.
1007  rows = self.defectRegistry.executeQuery(("path",), ("defect",),
1008  [(ccdKey, "?")],
1009  ("DATETIME(?)", "DATETIME(validStart)", "DATETIME(validEnd)"),
1010  (ccdVal, taiObs))
1011  if not rows or len(rows) == 0:
1012  return None
1013  if len(rows) == 1:
1014  return os.path.join(self.defectPath, rows[0][0])
1015  else:
1016  raise RuntimeError("Querying for defects (%s, %s) returns %d files: %s" %
1017  (ccdVal, taiObs, len(rows), ", ".join([_[0] for _ in rows])))
1018 
1019  def _makeCamera(self, policy, repositoryDir):
1020  """Make a camera (instance of lsst.afw.cameraGeom.Camera) describing the camera geometry
1021 
1022  Also set self.cameraDataLocation, if relevant (else it can be left None).
1023 
1024  This implementation assumes that policy contains an entry "camera" that points to the
1025  subdirectory in this package of camera data; specifically, that subdirectory must contain:
1026  - a file named `camera.py` that contains persisted camera config
1027  - ampInfo table FITS files, as required by lsst.afw.cameraGeom.makeCameraFromPath
1028 
1029  @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility))
1030  Policy with per-camera defaults already merged
1031  @param repositoryDir (string) Policy repository for the subclassing
1032  module (obtained with getRepositoryPath() on the
1033  per-camera default dictionary)
1034  """
1035  if isinstance(policy, pexPolicy.Policy):
1036  policy = dafPersist.Policy(pexPolicy=policy)
1037  if 'camera' not in policy:
1038  raise RuntimeError("Cannot find 'camera' in policy; cannot construct a camera")
1039  cameraDataSubdir = policy['camera']
1040  self.cameraDataLocation = os.path.normpath(
1041  os.path.join(repositoryDir, cameraDataSubdir, "camera.py"))
1042  cameraConfig = afwCameraGeom.CameraConfig()
1043  cameraConfig.load(self.cameraDataLocation)
1044  ampInfoPath = os.path.dirname(self.cameraDataLocation)
1045  return afwCameraGeom.makeCameraFromPath(
1046  cameraConfig=cameraConfig,
1047  ampInfoPath=ampInfoPath,
1048  shortNameFunc=self.getShortCcdName,
1049  pupilFactoryClass=self.PupilFactoryClass
1050  )
1051 
1052  def getRegistry(self):
1053  """Get the registry used by this mapper.
1054 
1055  Returns
1056  -------
1057  Registry or None
1058  The registry used by this mapper for this mapper's repository.
1059  """
1060  return self.registry
1061 
1062 def exposureFromImage(image, dataId=None, mapper=None, logger=None, setVisitInfo=True):
1063  """Generate an Exposure from an image-like object
1064 
1065  If the image is a DecoratedImage then also set its WCS and metadata
1066  (Image and MaskedImage are missing the necessary metadata
1067  and Exposure already has those set)
1068 
1069  @param[in] image Image-like object (lsst.afw.image.DecoratedImage, Image, MaskedImage or Exposure)
1070  @return (lsst.afw.image.Exposure) Exposure containing input image
1071  """
1072  metadata = None
1073  if isinstance(image, afwImage.MaskedImage):
1074  exposure = afwImage.makeExposure(image)
1075  elif isinstance(image, afwImage.DecoratedImage):
1076  exposure = afwImage.makeExposure(afwImage.makeMaskedImage(image.getImage()))
1077  metadata = image.getMetadata()
1078  try:
1079  wcs = afwImage.makeWcs(metadata, True)
1080  exposure.setWcs(wcs)
1081  except pexExcept.InvalidParameterError as e:
1082  # raised on failure to create a wcs (and possibly others)
1083  if logger is None:
1084  logger = lsstLog.Log.getLogger("CameraMapper")
1085  logger.warn("wcs set to None; insufficient information found in metadata to create a valid wcs: "
1086  "%s", e.args[0])
1087 
1088  exposure.setMetadata(metadata)
1089  elif isinstance(image, afwImage.Exposure):
1090  # Exposure
1091  exposure = image
1092  metadata = exposure.getMetadata()
1093  else:
1094  # Image
1095  exposure = afwImage.makeExposure(afwImage.makeMaskedImage(image))
1096  #
1097  # set VisitInfo if we can
1098  #
1099  if setVisitInfo and exposure.getInfo().getVisitInfo() is None:
1100  if metadata is not None:
1101  if mapper is None:
1102  if not logger:
1103  logger = lsstLog.Log.getLogger("CameraMapper")
1104  logger.warn("I can only set the VisitInfo if you provide a mapper")
1105  else:
1106  exposureId = mapper._computeCcdExposureId(dataId)
1107  visitInfo = mapper.makeRawVisitInfo(md=metadata, exposureId=exposureId)
1108 
1109  exposure.getInfo().setVisitInfo(visitInfo)
1110 
1111  return exposure
def _getCcdKeyVal
Utility functions.