lsst.pipe.tasks  13.0-29-g7046ce1+1
 All Classes Namespaces Files Functions Variables Groups Pages
simpleMapper.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 """Mapper and cameraGeom definition for extremely simple mock data.
24 
25 SimpleMapper inherits directly from Mapper, not CameraMapper. This means
26 we can avoid any problems with paf files at the expense of reimplementing
27 some parts of CameraMapper here. Jim is not sure this was the best
28 possible approach, but it gave him an opportunity to play around with
29 prototyping a future paf-free mapper class, and it does everything it
30 needs to do right now.
31 """
32 from __future__ import absolute_import, division, print_function
33 from builtins import map
34 from builtins import range
35 from builtins import object
36 
37 import os
38 import shutil
39 import re
40 
41 import lsst.daf.persistence
42 import lsst.afw.cameraGeom
43 from lsst.afw.cameraGeom.testUtils import DetectorWrapper
44 import lsst.afw.image.utils as afwImageUtils
45 from future.utils import with_metaclass
46 
47 __all__ = ("SimpleMapper", "makeSimpleCamera", "makeDataRepo")
48 
49 
50 class PersistenceType(object):
51  """Base class of a hierarchy used by SimpleMapper to defined different kinds of types of objects
52  to persist.
53 
54  PersistenceType objects are never instantiated; only the type objects are used (we needed a
55  simple singleton struct that could be inherited, which is exactly what a Python type is).
56  """
57  python = None
58  cpp = "ignored"
59  storage = None
60  ext = ""
61  suffixes = ()
62 
63  @classmethod
64  def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
65  """Method called by SimpleMapping to implement a map_ method."""
66  return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [path], dataId,
67  mapper=mapper,
68  storage=storage)
69 
70  def canStandardize(self, datasetType):
71  return False
72 
73 
75  """Persistence type for things that don't actually use daf_persistence.
76  """
77 
78  python = "lsst.daf.base.PropertySet" # something to import even when we don't need to
79 
80  @classmethod
81  def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
82  """Method called by SimpleMapping to implement a map_ method; overridden to not use the path."""
83  return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [], dataId,
84  mapper=mapper, storage=storage)
85 
86 
88  """Persistence type of Exposure images.
89  """
90 
91  python = "lsst.afw.image.ExposureF"
92  cpp = "ExposureF"
93  storage = "FitsStorage"
94  ext = ".fits"
95  suffixes = ("_sub",)
96 
97  @classmethod
98  def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
99  """Method called by SimpleMapping to implement a map_ method; overridden to support subimages."""
100  if suffix is None:
101  loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, dataId, mapper, suffix=None,
102  storage=storage)
103  elif suffix == "_sub":
104  subId = dataId.copy()
105  bbox = subId.pop('bbox')
106  loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, subId, mapper, suffix=None,
107  storage=storage)
108  loc.additionalData.set('llcX', bbox.getMinX())
109  loc.additionalData.set('llcY', bbox.getMinY())
110  loc.additionalData.set('width', bbox.getWidth())
111  loc.additionalData.set('height', bbox.getHeight())
112  if 'imageOrigin' in dataId:
113  loc.additionalData.set('imageOrigin',
114  dataId['imageOrigin'])
115  return loc
116 
117 
119  python = "lsst.skymap.BaseSkyMap"
120  storage = "PickleStorage"
121  ext = ".pickle"
122 
123 
125  python = "lsst.afw.table.BaseCatalog"
126  cpp = "BaseCatalog"
127  storage = "FitsCatalogStorage"
128  ext = ".fits"
129 
130 
132  python = "lsst.afw.table.SimpleCatalog"
133  cpp = "SimpleCatalog"
134 
135 
137  python = "lsst.afw.table.SourceCatalog"
138  cpp = "SourceCatalog"
139 
140 
142  python = "lsst.afw.table.ExposureCatalog"
143  cpp = "ExposureCatalog"
144 
145 
147  python = "lsst.afw.detection.PeakCatalog"
148  cpp = "PeakCatalog"
149 
150 
151 class SimpleMapping(object):
152  """Mapping object used to implement SimpleMapper, similar in intent to lsst.daf.peristence.Mapping.
153  """
154 
155  template = None
156  keys = {}
157 
158  def __init__(self, persistence, template=None, keys=None):
159  self.persistence = persistence
160  if template is not None:
161  self.template = template
162  if keys is not None:
163  self.keys = keys
164 
165  def map(self, dataset, root, dataId, mapper, suffix=None, storage=None):
166  if self.template is not None:
167  path = self.template.format(dataset=dataset, ext=self.persistence.ext, **dataId)
168  else:
169  path = None
170  return self.persistence.makeButlerLocation(path, dataId, suffix=suffix, mapper=mapper,
171  storage=storage)
172 
173 
175  """Mapping for dataset types that are organized the same way as raw data (i.e. by CCD)."""
176 
177  template = "{dataset}-{visit:04d}-{ccd:01d}{ext}"
178  keys = dict(visit=int, ccd=int)
179 
180  def query(self, dataset, index, level, format, dataId):
181  dictList = index[dataset][level]
182  results = [list(d.values()) for d in dictList[dataId.get(level, None)]]
183  return results
184 
185 
187  """Mapping for dataset types that are organized according to a SkyMap subdivision of the sky."""
188 
189  template = "{dataset}-{filter}-{tract:02d}-{patch}{ext}"
190  keys = dict(filter=str, tract=int, patch=str)
191 
192 
194  """Mapping for CoaddTempExp datasets."""
195 
196  template = "{dataset}-{tract:02d}-{patch}-{visit:04d}{ext}"
197  keys = dict(tract=int, patch=str, visit=int)
198 
199 
201  """Mapping for forced_src datasets."""
202 
203  template = "{dataset}-{tract:02d}-{visit:04d}-{ccd:01d}{ext}"
204  keys = dict(tract=int, ccd=int, visit=int)
205 
206 
207 class MapperMeta(type):
208  """Metaclass for SimpleMapper that creates map_ and query_ methods for everything found in the
209  'mappings' class variable.
210  """
211 
212  @staticmethod
213  def _makeMapClosure(dataset, mapping, suffix=None):
214  def mapClosure(self, dataId, write=False):
215  return mapping.map(dataset, self.root, dataId, self, suffix=suffix, storage=self.storage)
216  return mapClosure
217 
218  @staticmethod
219  def _makeQueryClosure(dataset, mapping):
220  def queryClosure(self, level, format, dataId):
221  return mapping.query(dataset, self.index, level, format, dataId)
222  return queryClosure
223 
224  def __init__(cls, name, bases, dict_):
225  type.__init__(cls, name, bases, dict_)
226  cls.keyDict = dict()
227  for dataset, mapping in cls.mappings.items():
228  setattr(cls, "map_" + dataset, MapperMeta._makeMapClosure(dataset, mapping, suffix=None))
229  for suffix in mapping.persistence.suffixes:
230  setattr(cls, "map_" + dataset + suffix,
231  MapperMeta._makeMapClosure(dataset, mapping, suffix=suffix))
232  if hasattr(mapping, "query"):
233  setattr(cls, "query_" + dataset, MapperMeta._makeQueryClosure(dataset, mapping))
234  cls.keyDict.update(mapping.keys)
235 
236 
237 class SimpleMapper(with_metaclass(MapperMeta, lsst.daf.persistence.Mapper)):
238  """
239  An extremely simple mapper for an imaginary camera for use in integration tests.
240 
241  As SimpleMapper does not inherit from obs.base.CameraMapper, it does not
242  use a policy file to set mappings or a registry; all the information is here
243  (in the map_* and query_* methods).
244 
245  The imaginary camera's raw data format has only 'visit' and 'ccd' keys, with
246  two CCDs per visit (by default).
247  """
248 
249  mappings = dict(
250  calexp=RawMapping(ExposurePersistenceType),
251  forced_src=ForcedSrcMapping(SourceCatalogPersistenceType),
252  forced_src_schema=SimpleMapping(SourceCatalogPersistenceType,
253  template="{dataset}{ext}", keys={}),
254  truth=SimpleMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
255  keys={"tract": int}),
256  simsrc=RawMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
257  keys={"tract": int}),
258  observations=SimpleMapping(ExposureCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
259  keys={"tract": int}),
260  ccdExposureId=RawMapping(BypassPersistenceType),
261  ccdExposureId_bits=SimpleMapping(BypassPersistenceType),
262  deepCoaddId=SkyMapping(BypassPersistenceType),
263  deepCoaddId_bits=SimpleMapping(BypassPersistenceType),
264  deepMergedCoaddId=SkyMapping(BypassPersistenceType),
265  deepMergedCoaddId_bits=SimpleMapping(BypassPersistenceType),
266  deepCoadd_skyMap=SimpleMapping(SkyMapPersistenceType, template="{dataset}{ext}", keys={}),
267  deepCoadd=SkyMapping(ExposurePersistenceType),
268  deepCoadd_calexp=SkyMapping(ExposurePersistenceType),
269  deepCoadd_calexp_background=SkyMapping(CatalogPersistenceType),
270  deepCoadd_icSrc=SkyMapping(SourceCatalogPersistenceType),
271  deepCoadd_icSrc_schema=SimpleMapping(SourceCatalogPersistenceType,
272  template="{dataset}{ext}", keys={}),
273  deepCoadd_src=SkyMapping(SourceCatalogPersistenceType),
274  deepCoadd_src_schema=SimpleMapping(SourceCatalogPersistenceType,
275  template="{dataset}{ext}", keys={}),
276  deepCoadd_peak_schema=SimpleMapping(PeakCatalogPersistenceType,
277  template="{dataset}{ext}", keys={}),
278  deepCoadd_ref=SkyMapping(SourceCatalogPersistenceType),
279  deepCoadd_ref_schema=SimpleMapping(SourceCatalogPersistenceType,
280  template="{dataset}{ext}", keys={}),
281  deepCoadd_det=SkyMapping(SourceCatalogPersistenceType),
282  deepCoadd_det_schema=SimpleMapping(SourceCatalogPersistenceType,
283  template="{dataset}{ext}", keys={}),
284  deepCoadd_mergeDet=SkyMapping(SourceCatalogPersistenceType),
285  deepCoadd_mergeDet_schema=SimpleMapping(SourceCatalogPersistenceType,
286  template="{dataset}{ext}", keys={}),
287  deepCoadd_meas=SkyMapping(SourceCatalogPersistenceType),
288  deepCoadd_meas_schema=SimpleMapping(SourceCatalogPersistenceType,
289  template="{dataset}{ext}", keys={}),
290  deepCoadd_forced_src=SkyMapping(SourceCatalogPersistenceType),
291  deepCoadd_forced_src_schema=SimpleMapping(SourceCatalogPersistenceType,
292  template="{dataset}{ext}", keys={}),
293  deepCoadd_mock=SkyMapping(ExposurePersistenceType),
294  deepCoadd_tempExp=TempExpMapping(ExposurePersistenceType),
295  deepCoadd_tempExp_mock=TempExpMapping(ExposurePersistenceType),
296  )
297 
298  levels = dict(
299  visit=['ccd'],
300  ccd=[],
301  )
302 
303  def __init__(self, root, **kwargs):
304  self.storage = lsst.daf.persistence.Storage.makeFromURI(root)
305  super(SimpleMapper, self).__init__(**kwargs)
306  self.root = root
307  self.camera = makeSimpleCamera(nX=1, nY=2, sizeX=400, sizeY=200, gapX=2, gapY=2)
308  afwImageUtils.defineFilter('r', 619.42)
309  self.update()
310 
311  def getDefaultLevel(self): return "ccd"
312 
313  def getKeys(self, datasetType, level):
314  if datasetType is None:
315  keyDict = self.keyDict
316  else:
317  keyDict = self.mappings[datasetType].keys
318  if level is not None and level in self.levels:
319  keyDict = dict(keyDict)
320  for l in self.levels[level]:
321  if l in keyDict:
322  del keyDict[l]
323  return keyDict
324 
325  def update(self):
326  filenames = os.listdir(self.root)
327  rawRegex = re.compile(r"(?P<dataset>\w+)-(?P<visit>\d+)-(?P<ccd>\d).*")
328  self.index = {}
329  for filename in filenames:
330  m = rawRegex.match(filename)
331  if not m:
332  continue
333  index = self.index.setdefault(m.group('dataset'), dict(ccd={None: []}, visit={None: []}))
334  visit = int(m.group('visit'))
335  ccd = int(m.group('ccd'))
336  d1 = dict(visit=visit, ccd=ccd)
337  d2 = dict(visit=visit)
338  index['ccd'].setdefault(visit, []).append(d1)
339  index['ccd'][None].append(d1)
340  index['visit'][visit] = [d2]
341  index['visit'][None].append(d1)
342 
343  def keys(self):
344  return self.keyDict
345 
346  def bypass_camera(self, datasetType, pythonType, location, dataId):
347  return self.camera
348 
349  def map_camera(self, dataId, write=False):
350  return lsst.daf.persistence.ButlerLocation(
351  "lsst.afw.cameraGeom.Camera", "Camera", None, [], dataId, mapper=self, storage=self.storage
352  )
353 
354  def std_calexp(self, item, dataId):
355  detectorId = dataId["ccd"]
356  detector = self.camera[detectorId]
357  item.setDetector(detector)
358  return item
359 
360  def _computeCcdExposureId(self, dataId):
361  return int(dataId["visit"]) * 10 + int(dataId["ccd"])
362 
363  def _computeCoaddId(self, dataId):
364  # Note: for real IDs, we'd want to include filter here, but it doesn't actually matter
365  # for any of the tests we've done so far, which all assume filter='r'
366  tract = int(dataId['tract'])
367  if tract < 0 or tract >= 128:
368  raise RuntimeError('tract not in range [0,128)')
369  patchX, patchY = (int(c) for c in dataId['patch'].split(','))
370  for p in (patchX, patchY):
371  if p < 0 or p >= 2**13:
372  raise RuntimeError('patch component not in range [0, 8192)')
373  return (tract * 2**13 + patchX) * 2**13 + patchY
374 
375  def splitCcdExposureId(ccdExposureId):
376  return dict(visit=(int(ccdExposureId) // 10), ccd=(int(ccdExposureId) % 10))
377 
378  def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
379  return self._computeCcdExposureId(dataId)
380 
381  def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
382  return 32
383 
384  def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId):
385  return self._computeCoaddId(dataId)
386 
387  def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId):
388  return 1 + 7 + 13*2 + 3
389 
390  def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId):
391  return self._computeCoaddId(dataId)
392 
393  def bypass_deepMergedCoaddId_bits(self, datasetType, pythonType, location, dataId):
394  return 1 + 7 + 13*2 + 3
395 
396 
397 def makeSimpleCamera(
398  nX, nY,
399  sizeX, sizeY,
400  gapX, gapY,
401  pixelSize=1.0,
402  plateScale=20.0,
403  radialDistortion=0.925,
404 ):
405  """Create a camera
406 
407  @param[in] nx: number of detectors in x
408  @param[in] ny: number of detectors in y
409  @param[in] sizeX: detector size in x (pixels)
410  @param[in] sizeY: detector size in y (pixels)
411  @param[in] gapX: gap between detectors in x (mm)
412  @param[in] gapY: gap between detectors in y (mm)
413  @param[in] pixelSize: pixel size (mm) (a float)
414  @param[in] plateScale: plate scale in arcsec/mm; 20.0 is for LSST
415  @param[in] radialDistortion: radial distortion, in mm/rad^2
416  (the r^3 coefficient of the radial distortion polynomial
417  that converts PUPIL in radians to FOCAL_PLANE in mm);
418  0.925 is the value Dave Monet measured for lsstSim data
419 
420  Each detector will have one amplifier (with no raw information).
421  """
422  pScaleRad = lsst.afw.geom.arcsecToRad(plateScale)
423  radialDistortCoeffs = [0.0, 1.0/pScaleRad, 0.0, radialDistortion/pScaleRad]
424  focalPlaneToPupil = lsst.afw.geom.RadialXYTransform(radialDistortCoeffs)
425  nativeSys = lsst.afw.cameraGeom.FOCAL_PLANE
426  transforms = {
427  lsst.afw.cameraGeom.PUPIL: focalPlaneToPupil,
428  }
429  transformMap = lsst.afw.cameraGeom.CameraTransformMap(nativeSys, transforms)
430 
431  detectorList = []
432  ccdBBox = lsst.afw.geom.Box2I(lsst.afw.geom.Point2I(), lsst.afw.geom.Extent2I(sizeX, sizeY))
433  for iY in range(nY):
434  cY = (iY - 0.5 * (nY - 1)) * (pixelSize * sizeY + gapY)
435  for iX in range(nX):
436  cX = (iX - 0.5 * (nX - 1)) * (pixelSize * sizeY + gapX)
437  fpPos = lsst.afw.geom.Point2D(cX, cY)
438  detectorName = "detector %d,%d" % (iX, iY)
439  detectorId = len(detectorList) + 1
440  detectorList.append(DetectorWrapper(
441  name=detectorName,
442  id=detectorId,
443  serial=detectorName + " serial",
444  bbox=ccdBBox,
445  ampExtent=ccdBBox.getDimensions(),
446  numAmps=1,
447  pixelSize=lsst.afw.geom.Extent2D(pixelSize, pixelSize),
448  orientation=lsst.afw.cameraGeom.Orientation(fpPos),
449  plateScale=plateScale,
450  radialDistortion=radialDistortion,
451  ).detector)
452 
453  return lsst.afw.cameraGeom.Camera(
454  name="Simple Camera",
455  detectorList=detectorList,
456  transformMap=transformMap,
457  )
458 
459 
460 def makeDataRepo(root):
461  """
462  Create a data repository for SimpleMapper and return a butler for it.
463 
464  Clobbers anything already in the given path.
465  """
466  if os.path.exists(root):
467  shutil.rmtree(root)
468  os.makedirs(root)
469  with open(os.path.join(root, "_mapper"), "w") as f:
470  f.write("lsst.pipe.tasks.mocks.SimpleMapper\n")
471  return lsst.daf.persistence.Butler(root=root)