lsst.obs.base  13.0-27-g9d555d8+2
 All Classes Namespaces Files Functions Variables
cameraMapper.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from builtins import str
24 import copy
25 import os
26 import pyfits # required by _makeDefectsDict until defects are written as AFW tables
27 import re
28 import weakref
29 import lsst.daf.persistence as dafPersist
30 from . import ImageMapping, ExposureMapping, CalibrationMapping, DatasetMapping
31 import lsst.afw.geom as afwGeom
32 import lsst.afw.image as afwImage
33 import lsst.afw.table as afwTable
34 import lsst.afw.cameraGeom as afwCameraGeom
35 import lsst.log as lsstLog
36 import lsst.pex.policy as pexPolicy
37 import lsst.pex.exceptions as pexExcept
38 from .exposureIdInfo import ExposureIdInfo
39 from .makeRawVisitInfo import MakeRawVisitInfo
40 from lsst.utils import getPackageDir
41 
42 """This module defines the CameraMapper base class."""
43 
44 
45 class CameraMapper(dafPersist.Mapper):
46 
47  """CameraMapper is a base class for mappers that handle images from a
48  camera and products derived from them. This provides an abstraction layer
49  between the data on disk and the code.
50 
51  Public methods: keys, queryMetadata, getDatasetTypes, map,
52  canStandardize, standardize
53 
54  Mappers for specific data sources (e.g., CFHT Megacam, LSST
55  simulations, etc.) should inherit this class.
56 
57  The CameraMapper manages datasets within a "root" directory. Note that
58  writing to a dataset present in the input root will hide the existing
59  dataset but not overwrite it. See #2160 for design discussion.
60 
61  A camera is assumed to consist of one or more rafts, each composed of
62  multiple CCDs. Each CCD is in turn composed of one or more amplifiers
63  (amps). A camera is also assumed to have a camera geometry description
64  (CameraGeom object) as a policy file, a filter description (Filter class
65  static configuration) as another policy file, and an optional defects
66  description directory.
67 
68  Information from the camera geometry and defects are inserted into all
69  Exposure objects returned.
70 
71  The mapper uses one or two registries to retrieve metadata about the
72  images. The first is a registry of all raw exposures. This must contain
73  the time of the observation. One or more tables (or the equivalent)
74  within the registry are used to look up data identifier components that
75  are not specified by the user (e.g. filter) and to return results for
76  metadata queries. The second is an optional registry of all calibration
77  data. This should contain validity start and end entries for each
78  calibration dataset in the same timescale as the observation time.
79 
80  Subclasses will typically set MakeRawVisitInfoClass:
81 
82  MakeRawVisitInfoClass: a class variable that points to a subclass of
83  MakeRawVisitInfo, a functor that creates an
84  lsst.afw.image.VisitInfo from the FITS metadata of a raw image.
85 
86  Subclasses must provide the following methods:
87 
88  _extractDetectorName(self, dataId): returns the detector name for a CCD
89  (e.g., "CFHT 21", "R:1,2 S:3,4") as used in the AFW CameraGeom class given
90  a dataset identifier referring to that CCD or a subcomponent of it.
91 
92  _computeCcdExposureId(self, dataId): see below
93 
94  _computeCoaddExposureId(self, dataId, singleFilter): see below
95 
96  Subclasses may also need to override the following methods:
97 
98  _transformId(self, dataId): transformation of a data identifier
99  from colloquial usage (e.g., "ccdname") to proper/actual usage (e.g., "ccd"),
100  including making suitable for path expansion (e.g. removing commas).
101  The default implementation does nothing. Note that this
102  method should not modify its input parameter.
103 
104  getShortCcdName(self, ccdName): a static method that returns a shortened name
105  suitable for use as a filename. The default version converts spaces to underscores.
106 
107  _getCcdKeyVal(self, dataId): return a CCD key and value
108  by which to look up defects in the defects registry.
109  The default value returns ("ccd", detector name)
110 
111  _mapActualToPath(self, template, actualId): convert a template path to an
112  actual path, using the actual dataset identifier.
113 
114  The mapper's behaviors are largely specified by the policy file.
115  See the MapperDictionary.paf for descriptions of the available items.
116 
117  The 'exposures', 'calibrations', and 'datasets' subpolicies configure
118  mappings (see Mappings class).
119 
120  Common default mappings for all subclasses can be specified in the
121  "policy/{images,exposures,calibrations,datasets}.yaml" files. This provides
122  a simple way to add a product to all camera mappers.
123 
124  Functions to map (provide a path to the data given a dataset
125  identifier dictionary) and standardize (convert data into some standard
126  format or type) may be provided in the subclass as "map_{dataset type}"
127  and "std_{dataset type}", respectively.
128 
129  If non-Exposure datasets cannot be retrieved using standard
130  daf_persistence methods alone, a "bypass_{dataset type}" function may be
131  provided in the subclass to return the dataset instead of using the
132  "datasets" subpolicy.
133 
134  Implementations of map_camera and bypass_camera that should typically be
135  sufficient are provided in this base class.
136 
137  @todo
138  * Handle defects the same was as all other calibration products, using the calibration registry
139  * Instead of auto-loading the camera at construction time, load it from the calibration registry
140  * Rewrite defects as AFW tables so we don't need pyfits to unpersist them; then remove all mention
141  of pyfits from this package.
142  """
143  packageName = None
144 
145  # a class or subclass of MakeRawVisitInfo, a functor that makes an
146  # lsst.afw.image.VisitInfo from the FITS metadata of a raw image
147  MakeRawVisitInfoClass = MakeRawVisitInfo
148 
149  # a class or subclass of PupilFactory
150  PupilFactoryClass = afwCameraGeom.PupilFactory
151 
152  def __init__(self, policy, repositoryDir,
153  root=None, registry=None, calibRoot=None, calibRegistry=None,
154  provided=None, parentRegistry=None, repositoryCfg=None):
155  """Initialize the CameraMapper.
156 
157  Parameters
158  ----------
159  policy : daf_persistence.Policy,
160  Can also be pexPolicy.Policy, only for backward compatibility.
161  Policy with per-camera defaults already merged.
162  repositoryDir : string
163  Policy repository for the subclassing module (obtained with
164  getRepositoryPath() on the per-camera default dictionary).
165  root : string, optional
166  Path to the root directory for data.
167  registry : string, optional
168  Path to registry with data's metadata.
169  calibRoot : string, optional
170  Root directory for calibrations.
171  calibRegistry : string, optional
172  Path to registry with calibrations' metadata.
173  provided : list of string, optional
174  Keys provided by the mapper.
175  parentRegistry : Registry subclass, optional
176  Registry from a parent repository that may be used to look up
177  data's metadata.
178  repositoryCfg : daf_persistence.RepositoryCfg or None, optional
179  The configuration information for the repository this mapper is
180  being used with.
181  """
182 
183  dafPersist.Mapper.__init__(self)
184 
185  self.log = lsstLog.Log.getLogger("CameraMapper")
186 
187  if root:
188  self.root = root
189  elif repositoryCfg:
190  self.root = repositoryCfg.root
191  else:
192  self.root = None
193  if isinstance(policy, pexPolicy.Policy):
194  policy = dafPersist.Policy(policy)
195 
196  repoPolicy = repositoryCfg.policy if repositoryCfg else None
197  if repoPolicy is not None:
198  policy.update(repoPolicy)
199 
200  defaultPolicyFile = dafPersist.Policy.defaultPolicyFile("obs_base",
201  "MapperDictionary.paf",
202  "policy")
203  dictPolicy = dafPersist.Policy(defaultPolicyFile)
204  policy.merge(dictPolicy)
205 
206  # Levels
207  self.levels = dict()
208  if 'levels' in policy:
209  levelsPolicy = policy['levels']
210  for key in levelsPolicy.names(True):
211  self.levels[key] = set(levelsPolicy.asArray(key))
212  self.defaultLevel = policy['defaultLevel']
213  self.defaultSubLevels = dict()
214  if 'defaultSubLevels' in policy:
215  self.defaultSubLevels = policy['defaultSubLevels']
216 
217  # Root directories
218  if root is None:
219  root = "."
220  root = dafPersist.LogicalLocation(root).locString()
221 
222  self.rootStorage = dafPersist.Storage.makeFromURI(uri=root)
223 
224  # If the calibRoot is passed in, use that. If not and it's indicated in
225  # the policy, use that. And otherwise, the calibs are in the regular
226  # root.
227  # If the location indicated by the calib root does not exist, do not
228  # create it.
229  calibStorage = None
230  if calibRoot is not None:
231  calibRoot = dafPersist.Storage.absolutePath(root, calibRoot)
232  calibStorage = dafPersist.Storage.makeFromURI(uri=calibRoot,
233  create=False)
234  else:
235  calibRoot = policy.get('calibRoot', None)
236  if calibRoot:
237  calibStorage = dafPersist.Storage.makeFromURI(uri=calibRoot,
238  create=False)
239  if calibStorage is None:
240  calibStorage = self.rootStorage
241 
242  self.root = root
243 
244  # Registries
245  self.registry = self._setupRegistry("registry", "exposure", registry, policy, "registryPath",
246  self.rootStorage, searchParents=False,
247  posixIfNoSql=(not parentRegistry))
248  if not self.registry:
249  self.registry = parentRegistry
250  needCalibRegistry = policy.get('needCalibRegistry', None)
251  if needCalibRegistry:
252  if calibStorage:
253  self.calibRegistry = self._setupRegistry("calibRegistry", "calib", calibRegistry, policy,
254  "calibRegistryPath", calibStorage)
255  else:
256  raise RuntimeError(
257  "'needCalibRegistry' is true in Policy, but was unable to locate a repo at " +
258  "calibRoot ivar:%s or policy['calibRoot']:%s" %
259  (calibRoot, policy.get('calibRoot', None)))
260  else:
261  self.calibRegistry = None
262 
263  # Dict of valid keys and their value types
264  self.keyDict = dict()
265 
266  self._initMappings(policy, self.rootStorage, calibStorage, provided=None)
267 
268  # Camera geometry
269  self.cameraDataLocation = None # path to camera geometry config file
270  self.camera = self._makeCamera(policy=policy, repositoryDir=repositoryDir)
271 
272  # Defect registry and root. Defects are stored with the camera and the registry is loaded from the
273  # camera package, which is on the local filesystem.
274  self.defectRegistry = None
275  if 'defects' in policy:
276  self.defectPath = os.path.join(repositoryDir, policy['defects'])
277  defectRegistryLocation = os.path.join(self.defectPath, "defectRegistry.sqlite3")
278  self.defectRegistry = dafPersist.Registry.create(defectRegistryLocation)
279 
280  # Filter translation table
281  self.filters = None
282 
283  # Skytile policy
284  self.skypolicy = policy['skytiles']
285 
286  # verify that the class variable packageName is set before attempting
287  # to instantiate an instance
288  if self.packageName is None:
289  raise ValueError('class variable packageName must not be None')
290 
292 
293  def _initMappings(self, policy, rootStorage=None, calibStorage=None, provided=None):
294  """Initialize mappings
295 
296  For each of the dataset types that we want to be able to read, there are
297  methods that can be created to support them:
298  * map_<dataset> : determine the path for dataset
299  * std_<dataset> : standardize the retrieved dataset
300  * bypass_<dataset> : retrieve the dataset (bypassing the usual retrieval machinery)
301  * query_<dataset> : query the registry
302 
303  Besides the dataset types explicitly listed in the policy, we create
304  additional, derived datasets for additional conveniences, e.g., reading
305  the header of an image, retrieving only the size of a catalog.
306 
307  @param policy (Policy) Policy with per-camera defaults already merged
308  @param rootStorage (Storage subclass instance) Interface to persisted repository data
309  @param calibRoot (Storage subclass instance) Interface to persisted calib repository data
310  @param provided (list of strings) Keys provided by the mapper
311  """
312  # Sub-dictionaries (for exposure/calibration/dataset types)
313  imgMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
314  "obs_base", "ImageMappingDictionary.paf", "policy"))
315  expMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
316  "obs_base", "ExposureMappingDictionary.paf", "policy"))
317  calMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
318  "obs_base", "CalibrationMappingDictionary.paf", "policy"))
319  dsMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
320  "obs_base", "DatasetMappingDictionary.paf", "policy"))
321 
322  # Mappings
323  mappingList = (
324  ("images", imgMappingPolicy, ImageMapping),
325  ("exposures", expMappingPolicy, ExposureMapping),
326  ("calibrations", calMappingPolicy, CalibrationMapping),
327  ("datasets", dsMappingPolicy, DatasetMapping)
328  )
329  self.mappings = dict()
330  for name, defPolicy, cls in mappingList:
331  if name in policy:
332  datasets = policy[name]
333 
334  # Centrally-defined datasets
335  defaultsPath = os.path.join(getPackageDir("obs_base"), "policy", name + ".yaml")
336  if os.path.exists(defaultsPath):
337  datasets.merge(dafPersist.Policy(defaultsPath))
338 
339  mappings = dict()
340  setattr(self, name, mappings)
341  for datasetType in datasets.names(True):
342  subPolicy = datasets[datasetType]
343  subPolicy.merge(defPolicy)
344 
345  if not hasattr(self, "map_" + datasetType) and 'composite' in subPolicy:
346  def compositeClosure(dataId, write=False, mapper=None, mapping=None, subPolicy=subPolicy):
347  components = subPolicy.get('composite')
348  assembler = subPolicy['assembler'] if 'assembler' in subPolicy else None
349  disassembler = subPolicy['disassembler'] if 'disassembler' in subPolicy else None
350  python = subPolicy['python']
351  butlerComposite = dafPersist.ButlerComposite(assembler=assembler,
352  disassembler=disassembler,
353  python=python,
354  dataId=dataId,
355  mapper=self)
356  for name, component in components.items():
357  butlerComposite.add(id=name,
358  datasetType=component.get('datasetType'),
359  setter=component.get('setter', None),
360  getter=component.get('getter', None),
361  subset=component.get('subset', False),
362  inputOnly=component.get('inputOnly', False))
363  return butlerComposite
364  setattr(self, "map_" + datasetType, compositeClosure)
365  # for now at least, don't set up any other handling for this dataset type.
366  continue
367 
368  if name == "calibrations":
369  mapping = cls(datasetType, subPolicy, self.registry, self.calibRegistry, calibStorage,
370  provided=provided)
371  else:
372  mapping = cls(datasetType, subPolicy, self.registry, rootStorage, provided=provided)
373  self.keyDict.update(mapping.keys())
374  mappings[datasetType] = mapping
375  self.mappings[datasetType] = mapping
376  if not hasattr(self, "map_" + datasetType):
377  def mapClosure(dataId, write=False, mapper=weakref.proxy(self), mapping=mapping):
378  return mapping.map(mapper, dataId, write)
379  setattr(self, "map_" + datasetType, mapClosure)
380  if not hasattr(self, "query_" + datasetType):
381  def queryClosure(format, dataId, mapping=mapping):
382  return mapping.lookup(format, dataId)
383  setattr(self, "query_" + datasetType, queryClosure)
384  if hasattr(mapping, "standardize") and not hasattr(self, "std_" + datasetType):
385  def stdClosure(item, dataId, mapper=weakref.proxy(self), mapping=mapping):
386  return mapping.standardize(mapper, item, dataId)
387  setattr(self, "std_" + datasetType, stdClosure)
388 
389  def setMethods(suffix, mapImpl=None, bypassImpl=None, queryImpl=None):
390  """Set convenience methods on CameraMapper"""
391  mapName = "map_" + datasetType + "_" + suffix
392  bypassName = "bypass_" + datasetType + "_" + suffix
393  queryName = "query_" + datasetType + "_" + suffix
394  if not hasattr(self, mapName):
395  setattr(self, mapName, mapImpl or getattr(self, "map_" + datasetType))
396  if not hasattr(self, bypassName):
397  if bypassImpl is None and hasattr(self, "bypass_" + datasetType):
398  bypassImpl = getattr(self, "bypass_" + datasetType)
399  if bypassImpl is not None:
400  setattr(self, bypassName, bypassImpl)
401  if not hasattr(self, queryName):
402  setattr(self, queryName, queryImpl or getattr(self, "query_" + datasetType))
403 
404  # Filename of dataset
405  setMethods("filename", bypassImpl=lambda datasetType, pythonType, location, dataId:
406  [os.path.join(location.getStorage().root, p) for p in location.getLocations()])
407  # Metadata from FITS file
408  if subPolicy["storage"] == "FitsStorage": # a FITS image
409  setMethods("md", bypassImpl=lambda datasetType, pythonType, location, dataId:
410  afwImage.readMetadata(location.getLocationsWithRoot()[0]))
411  if name == "exposures":
412  setMethods("wcs", bypassImpl=lambda datasetType, pythonType, location, dataId:
413  afwImage.makeWcs(
414  afwImage.readMetadata(location.getLocationsWithRoot()[0])))
415  setMethods("calib", bypassImpl=lambda datasetType, pythonType, location, dataId:
416  afwImage.Calib(
417  afwImage.readMetadata(location.getLocationsWithRoot()[0])))
418  setMethods("visitInfo",
419  bypassImpl=lambda datasetType, pythonType, location, dataId:
420  afwImage.VisitInfo(
421  afwImage.readMetadata(location.getLocationsWithRoot()[0])))
422  setMethods("filter",
423  bypassImpl=lambda datasetType, pythonType, location, dataId:
424  afwImage.Filter(
425  afwImage.readMetadata(location.getLocationsWithRoot()[0])))
426  if subPolicy["storage"] == "FitsCatalogStorage": # a FITS catalog
427  setMethods("md", bypassImpl=lambda datasetType, pythonType, location, dataId:
428  afwImage.readMetadata(os.path.join(location.getStorage().root,
429  location.getLocations()[0]), hdu=1))
430 
431  # Sub-images
432  if subPolicy["storage"] == "FitsStorage":
433  def mapSubClosure(dataId, write=False, mapper=weakref.proxy(self), mapping=mapping):
434  subId = dataId.copy()
435  del subId['bbox']
436  loc = mapping.map(mapper, subId, write)
437  bbox = dataId['bbox']
438  llcX = bbox.getMinX()
439  llcY = bbox.getMinY()
440  width = bbox.getWidth()
441  height = bbox.getHeight()
442  loc.additionalData.set('llcX', llcX)
443  loc.additionalData.set('llcY', llcY)
444  loc.additionalData.set('width', width)
445  loc.additionalData.set('height', height)
446  if 'imageOrigin' in dataId:
447  loc.additionalData.set('imageOrigin',
448  dataId['imageOrigin'])
449  return loc
450  def querySubClosure(key, format, dataId, mapping=mapping):
451  subId = dataId.copy()
452  del subId['bbox']
453  return mapping.lookup(format, subId)
454  setMethods("sub", mapImpl=mapSubClosure, queryImpl=querySubClosure)
455 
456  if subPolicy["storage"] == "FitsCatalogStorage":
457  # Length of catalog
458  setMethods("len", bypassImpl=lambda datasetType, pythonType, location, dataId:
459  afwImage.readMetadata(os.path.join(location.getStorage().root,
460  location.getLocations()[0]),
461  hdu=1).get("NAXIS2"))
462 
463  # Schema of catalog
464  if not datasetType.endswith("_schema") and datasetType + "_schema" not in datasets:
465  setMethods("schema", bypassImpl=lambda datasetType, pythonType, location, dataId:
466  afwTable.Schema.readFits(os.path.join(location.getStorage().root,
467  location.getLocations()[0])))
468 
469  def _computeCcdExposureId(self, dataId):
470  """Compute the 64-bit (long) identifier for a CCD exposure.
471 
472  Subclasses must override
473 
474  @param dataId (dict) Data identifier with visit, ccd
475  """
476  raise NotImplementedError()
477 
478  def _computeCoaddExposureId(self, dataId, singleFilter):
479  """Compute the 64-bit (long) identifier for a coadd.
480 
481  Subclasses must override
482 
483  @param dataId (dict) Data identifier with tract and patch.
484  @param singleFilter (bool) True means the desired ID is for a single-
485  filter coadd, in which case dataId
486  must contain filter.
487  """
488  raise NotImplementedError()
489 
490  def _search(self, path):
491  """Search for path in the associated repository's storage.
492 
493  Parameters
494  ----------
495  path : string
496  Path that describes an object in the repository associated with
497  this mapper.
498  Path may contain an HDU indicator, e.g. 'foo.fits[1]'. The
499  indicator will be stripped when searching and so will match
500  filenames without the HDU indicator, e.g. 'foo.fits'. The path
501  returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
502 
503  Returns
504  -------
505  string
506  The path for this object in the repository. Will return None if the
507  object can't be found. If the input argument path contained an HDU
508  indicator, the returned path will also contain the HDU indicator.
509  """
510  return self.rootStorage.search(path)
511 
512  def backup(self, datasetType, dataId):
513  """Rename any existing object with the given type and dataId.
514 
515  The CameraMapper implementation saves objects in a sequence of e.g.:
516  foo.fits
517  foo.fits~1
518  foo.fits~2
519  All of the backups will be placed in the output repo, however, and will
520  not be removed if they are found elsewhere in the _parent chain. This
521  means that the same file will be stored twice if the previous version was
522  found in an input repo.
523  """
524 
525  # Calling PosixStorage directly is not the long term solution in this
526  # function, this is work-in-progress on epic DM-6225. The plan is for
527  # parentSearch to be changed to 'search', and search only the storage
528  # associated with this mapper. All searching of parents will be handled
529  # by traversing the container of repositories in Butler.
530 
531  def firstElement(list):
532  """Get the first element in the list, or None if that can't be done.
533  """
534  return list[0] if list is not None and len(list) else None
535 
536  n = 0
537  newLocation = self.map(datasetType, dataId, write=True)
538  newPath = newLocation.getLocations()[0]
539  path = dafPersist.PosixStorage.search(self.root, newPath, searchParents=True)
540  path = firstElement(path)
541  oldPaths = []
542  while path is not None:
543  n += 1
544  oldPaths.append((n, path))
545  path = dafPersist.PosixStorage.search(self.root, "%s~%d" % (newPath, n), searchParents=True)
546  path = firstElement(path)
547  for n, oldPath in reversed(oldPaths):
548  self.rootStorage.copyFile(oldPath, "%s~%d" % (newPath, n))
549 
550  def keys(self):
551  """Return supported keys.
552  @return (iterable) List of keys usable in a dataset identifier"""
553  return iter(self.keyDict.keys())
554 
555  def getKeys(self, datasetType, level):
556  """Return a dict of supported keys and their value types for a given dataset
557  type at a given level of the key hierarchy.
558 
559  @param datasetType (str) dataset type or None for all dataset types
560  @param level (str) level or None for all levels or '' for the default level for the camera
561  @return (dict) dict keys are strings usable in a dataset identifier; values are their value types"""
562 
563  # not sure if this is how we want to do this. what if None was intended?
564  if level == '':
565  level = self.getDefaultLevel()
566 
567  if datasetType is None:
568  keyDict = copy.copy(self.keyDict)
569  else:
570  keyDict = self.mappings[datasetType].keys()
571  if level is not None and level in self.levels:
572  keyDict = copy.copy(keyDict)
573  for l in self.levels[level]:
574  if l in keyDict:
575  del keyDict[l]
576  return keyDict
577 
578  def getDefaultLevel(self):
579  return self.defaultLevel
580 
581  def getDefaultSubLevel(self, level):
582  if level in self.defaultSubLevels:
583  return self.defaultSubLevels[level]
584  return None
585 
586  @classmethod
587  def getCameraName(cls):
588  """Return the name of the camera that this CameraMapper is for."""
589  className = str(cls)
590  className = className[className.find('.'):-1]
591  m = re.search(r'(\w+)Mapper', className)
592  if m is None:
593  m = re.search(r"class '[\w.]*?(\w+)'", className)
594  name = m.group(1)
595  return name[:1].lower() + name[1:] if name else ''
596 
597  @classmethod
598  def getPackageName(cls):
599  """Return the name of the package containing this CameraMapper."""
600  if cls.packageName is None:
601  raise ValueError('class variable packageName must not be None')
602  return cls.packageName
603 
604  def map_camera(self, dataId, write=False):
605  """Map a camera dataset."""
606  if self.camera is None:
607  raise RuntimeError("No camera dataset available.")
608  actualId = self._transformId(dataId)
609  return dafPersist.ButlerLocation(
610  pythonType="lsst.afw.cameraGeom.CameraConfig",
611  cppType="Config",
612  storageName="ConfigStorage",
613  locationList=self.cameraDataLocation or "ignored",
614  dataId=actualId,
615  mapper=self,
616  storage=self.rootStorage
617  )
618 
619  def bypass_camera(self, datasetType, pythonType, butlerLocation, dataId):
620  """Return the (preloaded) camera object.
621  """
622  if self.camera is None:
623  raise RuntimeError("No camera dataset available.")
624  return self.camera
625 
626  def map_defects(self, dataId, write=False):
627  """Map defects dataset.
628 
629  @return a very minimal ButlerLocation containing just the locationList field
630  (just enough information that bypass_defects can use it).
631  """
632  defectFitsPath = self._defectLookup(dataId=dataId)
633  if defectFitsPath is None:
634  raise RuntimeError("No defects available for dataId=%s" % (dataId,))
635 
636  return dafPersist.ButlerLocation(None, None, None, defectFitsPath,
637  dataId, self,
638  storage=self.rootStorage)
639 
640  def bypass_defects(self, datasetType, pythonType, butlerLocation, dataId):
641  """Return a defect based on the butler location returned by map_defects
642 
643  @param[in] butlerLocation: a ButlerLocation with locationList = path to defects FITS file
644  @param[in] dataId: the usual data ID; "ccd" must be set
645 
646  Note: the name "bypass_XXX" means the butler makes no attempt to convert the ButlerLocation
647  into an object, which is what we want for now, since that conversion is a bit tricky.
648  """
649  detectorName = self._extractDetectorName(dataId)
650  defectsFitsPath = butlerLocation.locationList[0]
651  with pyfits.open(defectsFitsPath) as hduList:
652  for hdu in hduList[1:]:
653  if hdu.header["name"] != detectorName:
654  continue
655 
656  defectList = []
657  for data in hdu.data:
658  bbox = afwGeom.Box2I(
659  afwGeom.Point2I(int(data['x0']), int(data['y0'])),
660  afwGeom.Extent2I(int(data['width']), int(data['height'])),
661  )
662  defectList.append(afwImage.DefectBase(bbox))
663  return defectList
664 
665  raise RuntimeError("No defects for ccd %s in %s" % (detectorName, defectsFitsPath))
666 
667  def map_expIdInfo(self, dataId, write=False):
668  return dafPersist.ButlerLocation(
669  pythonType="lsst.obs.base.ExposureIdInfo",
670  cppType=None,
671  storageName="Internal",
672  locationList="ignored",
673  dataId=dataId,
674  mapper=self,
675  storage=self.rootStorage
676  )
677 
678  def bypass_expIdInfo(self, datasetType, pythonType, location, dataId):
679  """Hook to retrieve an lsst.obs.base.ExposureIdInfo for an exposure"""
680  expId = self.bypass_ccdExposureId(datasetType, pythonType, location, dataId)
681  expBits = self.bypass_ccdExposureId_bits(datasetType, pythonType, location, dataId)
682  return ExposureIdInfo(expId=expId, expBits=expBits)
683 
684  def std_bfKernel(self, item, dataId):
685  """Disable standardization for bfKernel
686 
687  bfKernel is a calibration product that is numpy array,
688  unlike other calibration products that are all images;
689  all calibration images are sent through _standardizeExposure
690  due to CalibrationMapping, but we don't want that to happen to bfKernel
691  """
692  return item
693 
694  def std_raw(self, item, dataId):
695  """Standardize a raw dataset by converting it to an Exposure instead of an Image"""
696  exposure = exposureFromImage(item, logger=self.log)
697  exposureId = self._computeCcdExposureId(dataId)
698  md = exposure.getMetadata()
699  visitInfo = self.makeRawVisitInfo(md=md, exposureId=exposureId)
700  exposure.getInfo().setVisitInfo(visitInfo)
701  return self._standardizeExposure(self.exposures['raw'], exposure, dataId,
702  trimmed=False)
703 
704  def map_skypolicy(self, dataId):
705  """Map a sky policy."""
706  return dafPersist.ButlerLocation("lsst.pex.policy.Policy", "Policy",
707  "Internal", None, None, self,
708  storage=self.rootStorage)
709 
710  def std_skypolicy(self, item, dataId):
711  """Standardize a sky policy by returning the one we use."""
712  return self.skypolicy
713 
714 ###############################################################################
715 #
716 # Utility functions
717 #
718 ###############################################################################
719 
720  def _getCcdKeyVal(self, dataId):
721  """Return CCD key and value used to look a defect in the defect registry
722 
723  The default implementation simply returns ("ccd", full detector name)
724  """
725  return ("ccd", self._extractDetectorName(dataId))
726 
727  def _setupRegistry(self, name, description, path, policy, policyKey, storage, searchParents=True,
728  posixIfNoSql=True):
729  """Set up a registry (usually SQLite3), trying a number of possible
730  paths.
731 
732  Parameters
733  ----------
734  name : string
735  Name of registry.
736  description: `str`
737  Description of registry (for log messages)
738  path : string
739  Path for registry.
740  policy : string
741  Policy that contains the registry name, used if path is None.
742  policyKey : string
743  Key in policy for registry path.
744  storage : Storage subclass
745  Repository Storage to look in.
746  searchParents : bool, optional
747  True if the search for a registry should follow any Butler v1
748  _parent symlinks.
749  posixIfNoSql : bool, optional
750  If an sqlite registry is not found, will create a posix registry if
751  this is True.
752 
753  Returns
754  -------
755  lsst.daf.persistence.Registry
756  Registry object
757  """
758  if path is None and policyKey in policy:
759  path = dafPersist.LogicalLocation(policy[policyKey]).locString()
760  if os.path.isabs(path):
761  raise RuntimeError("Policy should not indicate an absolute path for registry.")
762  if not storage.exists(path):
763  newPath = storage.instanceSearch(path)
764 
765  newPath = newPath[0] if newPath is not None and len(newPath) else None
766  if newPath is None:
767  self.log.warn("Unable to locate registry at policy path (also looked in root): %s",
768  path)
769  path = newPath
770  else:
771  self.log.warn("Unable to locate registry at policy path: %s", path)
772  path = None
773 
774  # Old Butler API was to indicate the registry WITH the repo folder, New Butler expects the registry to
775  # be in the repo folder. To support Old API, check to see if path starts with root, and if so, strip
776  # root from path. Currently only works with PosixStorage
777  try:
778  root = storage.root
779  if path and (path.startswith(root)):
780  path = path[len(root + '/'):]
781  except AttributeError:
782  pass
783 
784  # determine if there is an sqlite registry and if not, try the posix registry.
785  registry = None
786 
787  def search(filename, description):
788  """Search for file in storage
789 
790  Parameters
791  ----------
792  filename : `str`
793  Filename to search for
794  description : `str`
795  Description of file, for error message.
796 
797  Returns
798  -------
799  path : `str` or `None`
800  Path to file, or None
801  """
802  result = storage.instanceSearch(filename)
803  if result:
804  return result[0]
805  self.log.debug("Unable to locate %s: %s", description, filename)
806  return None
807 
808  # Search for a suitable registry database
809  if path is None:
810  path = search("%s.pgsql" % name, "%s in root" % description)
811  if path is None:
812  path = search("%s.sqlite3" % name, "%s in root" % description)
813  if path is None:
814  path = search(os.path.join(".", "%s.sqlite3" % name), "%s in current dir" % description)
815 
816  if path is not None:
817  if not storage.exists(path):
818  newPath = storage.instanceSearch(path)
819  newPath = newPath[0] if newPath is not None and len(newPath) else None
820  if newPath is not None:
821  path = newPath
822  self.log.debug("Loading %s registry from %s", description, path)
823  localFileObj = storage.getLocalFile(path)
824  registry = dafPersist.Registry.create(localFileObj.name)
825  elif not registry and posixIfNoSql:
826  try:
827  self.log.info("Loading Posix %s registry from %s", description, storage.root)
828  registry = dafPersist.PosixRegistry(storage.root)
829  except:
830  registry = None
831 
832  return registry
833 
834  def _transformId(self, dataId):
835  """Generate a standard ID dict from a camera-specific ID dict.
836 
837  Canonical keys include:
838  - amp: amplifier name
839  - ccd: CCD name (in LSST this is a combination of raft and sensor)
840  The default implementation returns a copy of its input.
841 
842  @param dataId[in] (dict) Dataset identifier; this must not be modified
843  @return (dict) Transformed dataset identifier"""
844 
845  return dataId.copy()
846 
847  def _mapActualToPath(self, template, actualId):
848  """Convert a template path to an actual path, using the actual data
849  identifier. This implementation is usually sufficient but can be
850  overridden by the subclass.
851  @param template (string) Template path
852  @param actualId (dict) Dataset identifier
853  @return (string) Pathname"""
854 
855  try:
856  transformedId = self._transformId(actualId)
857  return template % transformedId
858  except Exception as e:
859  raise RuntimeError("Failed to format %r with data %r: %s" % (template, transformedId, e))
860 
861  @staticmethod
862  def getShortCcdName(ccdName):
863  """Convert a CCD name to a form useful as a filename
864 
865  The default implementation converts spaces to underscores.
866  """
867  return ccdName.replace(" ", "_")
868 
869  def _extractDetectorName(self, dataId):
870  """Extract the detector (CCD) name from the dataset identifier.
871 
872  The name in question is the detector name used by lsst.afw.cameraGeom.
873 
874  @param dataId (dict) Dataset identifier
875  @return (string) Detector name
876  """
877  raise NotImplementedError("No _extractDetectorName() function specified")
878 
879  def _extractAmpId(self, dataId):
880  """Extract the amplifier identifer from a dataset identifier.
881 
882  @warning this is deprecated; DO NOT USE IT
883 
884  amplifier identifier has two parts: the detector name for the CCD
885  containing the amplifier and index of the amplifier in the detector.
886  @param dataId (dict) Dataset identifer
887  @return (tuple) Amplifier identifier"""
888 
889  trDataId = self._transformId(dataId)
890  return (trDataId["ccd"], int(trDataId['amp']))
891 
892  def _setAmpDetector(self, item, dataId, trimmed=True):
893  """Set the detector object in an Exposure for an amplifier.
894  Defects are also added to the Exposure based on the detector object.
895  @param[in,out] item (lsst.afw.image.Exposure)
896  @param dataId (dict) Dataset identifier
897  @param trimmed (bool) Should detector be marked as trimmed? (ignored)"""
898 
899  return self._setCcdDetector(item=item, dataId=dataId, trimmed=trimmed)
900 
901  def _setCcdDetector(self, item, dataId, trimmed=True):
902  """Set the detector object in an Exposure for a CCD.
903  @param[in,out] item (lsst.afw.image.Exposure)
904  @param dataId (dict) Dataset identifier
905  @param trimmed (bool) Should detector be marked as trimmed? (ignored)"""
906 
907  if item.getDetector() is not None:
908  return
909 
910  detectorName = self._extractDetectorName(dataId)
911  detector = self.camera[detectorName]
912  item.setDetector(detector)
913 
914  def _setFilter(self, mapping, item, dataId):
915  """Set the filter object in an Exposure. If the Exposure had a FILTER
916  keyword, this was already processed during load. But if it didn't,
917  use the filter from the registry.
918  @param mapping (lsst.obs.base.Mapping)
919  @param[in,out] item (lsst.afw.image.Exposure)
920  @param dataId (dict) Dataset identifier"""
921 
922  if not (isinstance(item, afwImage.ExposureU) or isinstance(item, afwImage.ExposureI) or
923  isinstance(item, afwImage.ExposureF) or isinstance(item, afwImage.ExposureD)):
924  return
925 
926  if item.getFilter().getId() != afwImage.Filter.UNKNOWN:
927  return
928 
929  actualId = mapping.need(['filter'], dataId)
930  filterName = actualId['filter']
931  if self.filters is not None and filterName in self.filters:
932  filterName = self.filters[filterName]
933  item.setFilter(afwImage.Filter(filterName))
934 
935  # Default standardization function for exposures
936  def _standardizeExposure(self, mapping, item, dataId, filter=True,
937  trimmed=True):
938  """Default standardization function for images.
939 
940  This sets the Detector from the camera geometry
941  and optionally set the Fiter. In both cases this saves
942  having to persist some data in each exposure (or image).
943 
944  @param mapping (lsst.obs.base.Mapping)
945  @param[in,out] item image-like object; any of lsst.afw.image.Exposure,
946  lsst.afw.image.DecoratedImage, lsst.afw.image.Image
947  or lsst.afw.image.MaskedImage
948  @param dataId (dict) Dataset identifier
949  @param filter (bool) Set filter? Ignored if item is already an exposure
950  @param trimmed (bool) Should detector be marked as trimmed?
951  @return (lsst.afw.image.Exposure) the standardized Exposure"""
952  if not hasattr(item, "getMaskedImage"):
953  try:
954  item = exposureFromImage(item, logger=self.log)
955  except Exception as e:
956  self.log.error("Could not turn item=%r into an exposure: %s" % (repr(item), e))
957  raise
958 
959  if mapping.level.lower() == "amp":
960  self._setAmpDetector(item, dataId, trimmed)
961  elif mapping.level.lower() == "ccd":
962  self._setCcdDetector(item, dataId, trimmed)
963 
964  if filter:
965  self._setFilter(mapping, item, dataId)
966 
967  return item
968 
969  def _defectLookup(self, dataId):
970  """Find the defects for a given CCD.
971  @param dataId (dict) Dataset identifier
972  @return (string) path to the defects file or None if not available"""
973  if self.defectRegistry is None:
974  return None
975  if self.registry is None:
976  raise RuntimeError("No registry for defect lookup")
977 
978  ccdKey, ccdVal = self._getCcdKeyVal(dataId)
979 
980  dataIdForLookup = {'visit': dataId['visit']}
981  # .lookup will fail in a posix registry because there is no template to provide.
982  rows = self.registry.lookup(('taiObs'), ('raw_visit'), dataIdForLookup)
983  if len(rows) == 0:
984  return None
985  assert len(rows) == 1
986  taiObs = rows[0][0]
987 
988  # Lookup the defects for this CCD serial number that are valid at the exposure midpoint.
989  rows = self.defectRegistry.executeQuery(("path",), ("defect",),
990  [(ccdKey, "?")],
991  ("DATETIME(?)", "DATETIME(validStart)", "DATETIME(validEnd)"),
992  (ccdVal, taiObs))
993  if not rows or len(rows) == 0:
994  return None
995  if len(rows) == 1:
996  return os.path.join(self.defectPath, rows[0][0])
997  else:
998  raise RuntimeError("Querying for defects (%s, %s) returns %d files: %s" %
999  (ccdVal, taiObs, len(rows), ", ".join([_[0] for _ in rows])))
1000 
1001  def _makeCamera(self, policy, repositoryDir):
1002  """Make a camera (instance of lsst.afw.cameraGeom.Camera) describing the camera geometry
1003 
1004  Also set self.cameraDataLocation, if relevant (else it can be left None).
1005 
1006  This implementation assumes that policy contains an entry "camera" that points to the
1007  subdirectory in this package of camera data; specifically, that subdirectory must contain:
1008  - a file named `camera.py` that contains persisted camera config
1009  - ampInfo table FITS files, as required by lsst.afw.cameraGeom.makeCameraFromPath
1010 
1011  @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility))
1012  Policy with per-camera defaults already merged
1013  @param repositoryDir (string) Policy repository for the subclassing
1014  module (obtained with getRepositoryPath() on the
1015  per-camera default dictionary)
1016  """
1017  if isinstance(policy, pexPolicy.Policy):
1018  policy = dafPersist.Policy(pexPolicy=policy)
1019  if 'camera' not in policy:
1020  raise RuntimeError("Cannot find 'camera' in policy; cannot construct a camera")
1021  cameraDataSubdir = policy['camera']
1022  self.cameraDataLocation = os.path.normpath(
1023  os.path.join(repositoryDir, cameraDataSubdir, "camera.py"))
1024  cameraConfig = afwCameraGeom.CameraConfig()
1025  cameraConfig.load(self.cameraDataLocation)
1026  ampInfoPath = os.path.dirname(self.cameraDataLocation)
1027  return afwCameraGeom.makeCameraFromPath(
1028  cameraConfig=cameraConfig,
1029  ampInfoPath=ampInfoPath,
1030  shortNameFunc=self.getShortCcdName,
1031  pupilFactoryClass=self.PupilFactoryClass
1032  )
1033 
1034  def getRegistry(self):
1035  """Get the registry used by this mapper.
1036 
1037  Returns
1038  -------
1039  Registry or None
1040  The registry used by this mapper for this mapper's repository.
1041  """
1042  return self.registry
1043 
1044 def exposureFromImage(image, logger=None):
1045  """Generate an Exposure from an image-like object
1046 
1047  If the image is a DecoratedImage then also set its WCS and metadata
1048  (Image and MaskedImage are missing the necessary metadata
1049  and Exposure already has those set)
1050 
1051  @param[in] image Image-like object (lsst.afw.image.DecoratedImage, Image, MaskedImage or Exposure)
1052  @return (lsst.afw.image.Exposure) Exposure containing input image
1053  """
1054  if isinstance(image, afwImage.MaskedImage):
1055  exposure = afwImage.makeExposure(image)
1056  elif isinstance(image, afwImage.DecoratedImage):
1057  exposure = afwImage.makeExposure(afwImage.makeMaskedImage(image.getImage()))
1058  metadata = image.getMetadata()
1059  try:
1060  wcs = afwImage.makeWcs(metadata, True)
1061  exposure.setWcs(wcs)
1062  except pexExcept.InvalidParameterError as e:
1063  # raised on failure to create a wcs (and possibly others)
1064  if logger is None:
1065  logger = lsstLog.Log.getLogger("obs.base.cameraMapper")
1066  logger.warn("wcs set to None; insufficient information found in metadata to create a valid wcs: "
1067  "%s", e.args[0])
1068 
1069  exposure.setMetadata(metadata)
1070  elif isinstance(image, afwImage.Exposure):
1071  # Exposure
1072  exposure = image
1073  else:
1074  # Image
1075  exposure = afwImage.makeExposure(afwImage.makeMaskedImage(image))
1076 
1077  return exposure
def _getCcdKeyVal
Utility functions.