lsst.pipe.tasks  13.0-66-gfbf2f2ce+5
simpleMapper.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 """Mapper and cameraGeom definition for extremely simple mock data.
24 
25 SimpleMapper inherits directly from Mapper, not CameraMapper. This means
26 we can avoid any problems with paf files at the expense of reimplementing
27 some parts of CameraMapper here. Jim is not sure this was the best
28 possible approach, but it gave him an opportunity to play around with
29 prototyping a future paf-free mapper class, and it does everything it
30 needs to do right now.
31 """
32 from __future__ import absolute_import, division, print_function
33 from builtins import map
34 from builtins import range
35 from builtins import object
36 
37 import os
38 import shutil
39 import re
40 
41 import lsst.daf.persistence
42 import lsst.afw.cameraGeom
43 from lsst.afw.cameraGeom.testUtils import DetectorWrapper
44 import lsst.afw.image.utils as afwImageUtils
45 from future.utils import with_metaclass
46 
47 __all__ = ("SimpleMapper", "makeSimpleCamera", "makeDataRepo")
48 
49 
50 class PersistenceType(object):
51  """Base class of a hierarchy used by SimpleMapper to defined different kinds of types of objects
52  to persist.
53 
54  PersistenceType objects are never instantiated; only the type objects are used (we needed a
55  simple singleton struct that could be inherited, which is exactly what a Python type is).
56  """
57  python = None
58  cpp = "ignored"
59  storage = None
60  ext = ""
61  suffixes = ()
62 
63  @classmethod
64  def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
65  """Method called by SimpleMapping to implement a map_ method."""
66  return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [path], dataId,
67  mapper=mapper,
68  storage=storage)
69 
70  def canStandardize(self, datasetType):
71  return False
72 
73 
75  """Persistence type for things that don't actually use daf_persistence.
76  """
77 
78  python = "lsst.daf.base.PropertySet" # something to import even when we don't need to
79 
80  @classmethod
81  def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
82  """Method called by SimpleMapping to implement a map_ method; overridden to not use the path."""
83  return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [], dataId,
84  mapper=mapper, storage=storage)
85 
86 
88  """Persistence type of Exposure images.
89  """
90 
91  python = "lsst.afw.image.ExposureF"
92  cpp = "ExposureF"
93  storage = "FitsStorage"
94  ext = ".fits"
95  suffixes = ("_sub",)
96 
97  @classmethod
98  def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
99  """Method called by SimpleMapping to implement a map_ method; overridden to support subimages."""
100  if suffix is None:
101  loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, dataId, mapper, suffix=None,
102  storage=storage)
103  elif suffix == "_sub":
104  subId = dataId.copy()
105  bbox = subId.pop('bbox')
106  loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, subId, mapper, suffix=None,
107  storage=storage)
108  loc.additionalData.set('llcX', bbox.getMinX())
109  loc.additionalData.set('llcY', bbox.getMinY())
110  loc.additionalData.set('width', bbox.getWidth())
111  loc.additionalData.set('height', bbox.getHeight())
112  if 'imageOrigin' in dataId:
113  loc.additionalData.set('imageOrigin',
114  dataId['imageOrigin'])
115  return loc
116 
117 
119  python = "lsst.skymap.BaseSkyMap"
120  storage = "PickleStorage"
121  ext = ".pickle"
122 
123 
125  python = "lsst.afw.table.BaseCatalog"
126  cpp = "BaseCatalog"
127  storage = "FitsCatalogStorage"
128  ext = ".fits"
129 
130 
132  python = "lsst.afw.table.SimpleCatalog"
133  cpp = "SimpleCatalog"
134 
135 
137  python = "lsst.afw.table.SourceCatalog"
138  cpp = "SourceCatalog"
139 
140 
142  python = "lsst.afw.table.ExposureCatalog"
143  cpp = "ExposureCatalog"
144 
145 
147  python = "lsst.afw.detection.PeakCatalog"
148  cpp = "PeakCatalog"
149 
150 
151 class SimpleMapping(object):
152  """Mapping object used to implement SimpleMapper, similar in intent to lsst.daf.peristence.Mapping.
153  """
154 
155  template = None
156  keys = {}
157 
158  def __init__(self, persistence, template=None, keys=None):
159  self.persistence = persistence
160  if template is not None:
161  self.template = template
162  if keys is not None:
163  self.keys = keys
164 
165  def map(self, dataset, root, dataId, mapper, suffix=None, storage=None):
166  if self.template is not None:
167  path = self.template.format(dataset=dataset, ext=self.persistence.ext, **dataId)
168  else:
169  path = None
170  return self.persistence.makeButlerLocation(path, dataId, suffix=suffix, mapper=mapper,
171  storage=storage)
172 
173 
175  """Mapping for dataset types that are organized the same way as raw data (i.e. by CCD)."""
176 
177  template = "{dataset}-{visit:04d}-{ccd:01d}{ext}"
178  keys = dict(visit=int, ccd=int)
179 
180  def query(self, dataset, index, level, format, dataId):
181  dictList = index[dataset][level]
182  results = [list(d.values()) for d in dictList[dataId.get(level, None)]]
183  return results
184 
185 
187  """Mapping for dataset types that are organized according to a SkyMap subdivision of the sky."""
188 
189  template = "{dataset}-{filter}-{tract:02d}-{patch}{ext}"
190  keys = dict(filter=str, tract=int, patch=str)
191 
192 
194  """Mapping for CoaddTempExp datasets."""
195 
196  template = "{dataset}-{tract:02d}-{patch}-{visit:04d}{ext}"
197  keys = dict(tract=int, patch=str, visit=int)
198 
199 
201  """Mapping for forced_src datasets."""
202 
203  template = "{dataset}-{tract:02d}-{visit:04d}-{ccd:01d}{ext}"
204  keys = dict(tract=int, ccd=int, visit=int)
205 
206 
207 class MapperMeta(type):
208  """Metaclass for SimpleMapper that creates map_ and query_ methods for everything found in the
209  'mappings' class variable.
210  """
211 
212  @staticmethod
213  def _makeMapClosure(dataset, mapping, suffix=None):
214  def mapClosure(self, dataId, write=False):
215  return mapping.map(dataset, self.root, dataId, self, suffix=suffix, storage=self.storage)
216  return mapClosure
217 
218  @staticmethod
219  def _makeQueryClosure(dataset, mapping):
220  def queryClosure(self, level, format, dataId):
221  return mapping.query(dataset, self.index, level, format, dataId)
222  return queryClosure
223 
224  def __init__(cls, name, bases, dict_):
225  type.__init__(cls, name, bases, dict_)
226  cls.keyDict = dict()
227  for dataset, mapping in cls.mappings.items():
228  setattr(cls, "map_" + dataset, MapperMeta._makeMapClosure(dataset, mapping, suffix=None))
229  for suffix in mapping.persistence.suffixes:
230  setattr(cls, "map_" + dataset + suffix,
231  MapperMeta._makeMapClosure(dataset, mapping, suffix=suffix))
232  if hasattr(mapping, "query"):
233  setattr(cls, "query_" + dataset, MapperMeta._makeQueryClosure(dataset, mapping))
234  cls.keyDict.update(mapping.keys)
235 
236 
237 class SimpleMapper(with_metaclass(MapperMeta, lsst.daf.persistence.Mapper)):
238  """
239  An extremely simple mapper for an imaginary camera for use in integration tests.
240 
241  As SimpleMapper does not inherit from obs.base.CameraMapper, it does not
242  use a policy file to set mappings or a registry; all the information is here
243  (in the map_* and query_* methods).
244 
245  The imaginary camera's raw data format has only 'visit' and 'ccd' keys, with
246  two CCDs per visit (by default).
247  """
248 
249  mappings = dict(
250  calexp=RawMapping(ExposurePersistenceType),
251  forced_src=ForcedSrcMapping(SourceCatalogPersistenceType),
252  forced_src_schema=SimpleMapping(SourceCatalogPersistenceType,
253  template="{dataset}{ext}", keys={}),
254  truth=SimpleMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
255  keys={"tract": int}),
256  simsrc=RawMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
257  keys={"tract": int}),
258  observations=SimpleMapping(ExposureCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
259  keys={"tract": int}),
260  ccdExposureId=RawMapping(BypassPersistenceType),
261  ccdExposureId_bits=SimpleMapping(BypassPersistenceType),
262  deepCoaddId=SkyMapping(BypassPersistenceType),
263  deepCoaddId_bits=SimpleMapping(BypassPersistenceType),
264  deepMergedCoaddId=SkyMapping(BypassPersistenceType),
265  deepMergedCoaddId_bits=SimpleMapping(BypassPersistenceType),
266  deepCoadd_skyMap=SimpleMapping(SkyMapPersistenceType, template="{dataset}{ext}", keys={}),
267  deepCoadd=SkyMapping(ExposurePersistenceType),
268  deepCoaddPsfMatched=SkyMapping(ExposurePersistenceType),
269  deepCoadd_calexp=SkyMapping(ExposurePersistenceType),
270  deepCoadd_calexp_background=SkyMapping(CatalogPersistenceType),
271  deepCoadd_icSrc=SkyMapping(SourceCatalogPersistenceType),
272  deepCoadd_icSrc_schema=SimpleMapping(SourceCatalogPersistenceType,
273  template="{dataset}{ext}", keys={}),
274  deepCoadd_src=SkyMapping(SourceCatalogPersistenceType),
275  deepCoadd_src_schema=SimpleMapping(SourceCatalogPersistenceType,
276  template="{dataset}{ext}", keys={}),
277  deepCoadd_peak_schema=SimpleMapping(PeakCatalogPersistenceType,
278  template="{dataset}{ext}", keys={}),
279  deepCoadd_ref=SkyMapping(SourceCatalogPersistenceType),
280  deepCoadd_ref_schema=SimpleMapping(SourceCatalogPersistenceType,
281  template="{dataset}{ext}", keys={}),
282  deepCoadd_det=SkyMapping(SourceCatalogPersistenceType),
283  deepCoadd_det_schema=SimpleMapping(SourceCatalogPersistenceType,
284  template="{dataset}{ext}", keys={}),
285  deepCoadd_mergeDet=SkyMapping(SourceCatalogPersistenceType),
286  deepCoadd_mergeDet_schema=SimpleMapping(SourceCatalogPersistenceType,
287  template="{dataset}{ext}", keys={}),
288  deepCoadd_meas=SkyMapping(SourceCatalogPersistenceType),
289  deepCoadd_meas_schema=SimpleMapping(SourceCatalogPersistenceType,
290  template="{dataset}{ext}", keys={}),
291  deepCoadd_forced_src=SkyMapping(SourceCatalogPersistenceType),
292  deepCoadd_forced_src_schema=SimpleMapping(SourceCatalogPersistenceType,
293  template="{dataset}{ext}", keys={}),
294  deepCoadd_mock=SkyMapping(ExposurePersistenceType),
295  deepCoaddPsfMatched_mock=SkyMapping(ExposurePersistenceType),
296  deepCoadd_directWarp=TempExpMapping(ExposurePersistenceType),
297  deepCoadd_directWarp_mock=TempExpMapping(ExposurePersistenceType),
298  deepCoadd_psfMatchedWarp=TempExpMapping(ExposurePersistenceType),
299  deepCoadd_psfMatchedWarp_mock=TempExpMapping(ExposurePersistenceType),
300  )
301 
302  levels = dict(
303  visit=['ccd'],
304  ccd=[],
305  )
306 
307  def __init__(self, root, **kwargs):
308  self.storage = lsst.daf.persistence.Storage.makeFromURI(root)
309  super(SimpleMapper, self).__init__(**kwargs)
310  self.root = root
311  self.camera = makeSimpleCamera(nX=1, nY=2, sizeX=400, sizeY=200, gapX=2, gapY=2)
312  afwImageUtils.defineFilter('r', 619.42)
313  self.update()
314 
315  def getDefaultLevel(self): return "ccd"
316 
317  def getKeys(self, datasetType, level):
318  if datasetType is None:
319  keyDict = self.keyDict
320  else:
321  keyDict = self.mappings[datasetType].keys
322  if level is not None and level in self.levels:
323  keyDict = dict(keyDict)
324  for l in self.levels[level]:
325  if l in keyDict:
326  del keyDict[l]
327  return keyDict
328 
329  def update(self):
330  filenames = os.listdir(self.root)
331  rawRegex = re.compile(r"(?P<dataset>\w+)-(?P<visit>\d+)-(?P<ccd>\d).*")
332  self.index = {}
333  for filename in filenames:
334  m = rawRegex.match(filename)
335  if not m:
336  continue
337  index = self.index.setdefault(m.group('dataset'), dict(ccd={None: []}, visit={None: []}))
338  visit = int(m.group('visit'))
339  ccd = int(m.group('ccd'))
340  d1 = dict(visit=visit, ccd=ccd)
341  d2 = dict(visit=visit)
342  index['ccd'].setdefault(visit, []).append(d1)
343  index['ccd'][None].append(d1)
344  index['visit'][visit] = [d2]
345  index['visit'][None].append(d1)
346 
347  def keys(self):
348  return self.keyDict
349 
350  def bypass_camera(self, datasetType, pythonType, location, dataId):
351  return self.camera
352 
353  def map_camera(self, dataId, write=False):
354  return lsst.daf.persistence.ButlerLocation(
355  "lsst.afw.cameraGeom.Camera", "Camera", None, [], dataId, mapper=self, storage=self.storage
356  )
357 
358  def std_calexp(self, item, dataId):
359  detectorId = dataId["ccd"]
360  detector = self.camera[detectorId]
361  item.setDetector(detector)
362  return item
363 
364  def _computeCcdExposureId(self, dataId):
365  return int(dataId["visit"]) * 10 + int(dataId["ccd"])
366 
367  def _computeCoaddId(self, dataId):
368  # Note: for real IDs, we'd want to include filter here, but it doesn't actually matter
369  # for any of the tests we've done so far, which all assume filter='r'
370  tract = int(dataId['tract'])
371  if tract < 0 or tract >= 128:
372  raise RuntimeError('tract not in range [0,128)')
373  patchX, patchY = (int(c) for c in dataId['patch'].split(','))
374  for p in (patchX, patchY):
375  if p < 0 or p >= 2**13:
376  raise RuntimeError('patch component not in range [0, 8192)')
377  return (tract * 2**13 + patchX) * 2**13 + patchY
378 
379  def splitCcdExposureId(ccdExposureId):
380  return dict(visit=(int(ccdExposureId) // 10), ccd=(int(ccdExposureId) % 10))
381 
382  def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
383  return self._computeCcdExposureId(dataId)
384 
385  def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
386  return 32
387 
388  def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId):
389  return self._computeCoaddId(dataId)
390 
391  def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId):
392  return 1 + 7 + 13*2 + 3
393 
394  def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId):
395  return self._computeCoaddId(dataId)
396 
397  def bypass_deepMergedCoaddId_bits(self, datasetType, pythonType, location, dataId):
398  return 1 + 7 + 13*2 + 3
399 
400 
401 def makeSimpleCamera(
402  nX, nY,
403  sizeX, sizeY,
404  gapX, gapY,
405  pixelSize=1.0,
406  plateScale=20.0,
407  radialDistortion=0.925,
408 ):
409  """Create a camera
410 
411  @param[in] nx: number of detectors in x
412  @param[in] ny: number of detectors in y
413  @param[in] sizeX: detector size in x (pixels)
414  @param[in] sizeY: detector size in y (pixels)
415  @param[in] gapX: gap between detectors in x (mm)
416  @param[in] gapY: gap between detectors in y (mm)
417  @param[in] pixelSize: pixel size (mm) (a float)
418  @param[in] plateScale: plate scale in arcsec/mm; 20.0 is for LSST
419  @param[in] radialDistortion: radial distortion, in mm/rad^2
420  (the r^3 coefficient of the radial distortion polynomial
421  that converts FIELD_ANGLE in radians to FOCAL_PLANE in mm);
422  0.925 is the value Dave Monet measured for lsstSim data
423 
424  Each detector will have one amplifier (with no raw information).
425  """
426  pScaleRad = lsst.afw.geom.arcsecToRad(plateScale)
427  radialDistortCoeffs = [0.0, 1.0/pScaleRad, 0.0, radialDistortion/pScaleRad]
428  focalPlaneToFieldAngle = lsst.afw.geom.makeRadialTransform(radialDistortCoeffs)
429  nativeSys = lsst.afw.cameraGeom.FOCAL_PLANE
430  transforms = {
431  lsst.afw.cameraGeom.FIELD_ANGLE: focalPlaneToFieldAngle,
432  }
433  transformMap = lsst.afw.cameraGeom.TransformMap(nativeSys, transforms)
434 
435  detectorList = []
436  ccdBBox = lsst.afw.geom.Box2I(lsst.afw.geom.Point2I(), lsst.afw.geom.Extent2I(sizeX, sizeY))
437  for iY in range(nY):
438  cY = (iY - 0.5 * (nY - 1)) * (pixelSize * sizeY + gapY)
439  for iX in range(nX):
440  cX = (iX - 0.5 * (nX - 1)) * (pixelSize * sizeY + gapX)
441  fpPos = lsst.afw.geom.Point2D(cX, cY)
442  detectorName = "detector %d,%d" % (iX, iY)
443  detectorId = len(detectorList) + 1
444  detectorList.append(DetectorWrapper(
445  name=detectorName,
446  id=detectorId,
447  serial=detectorName + " serial",
448  bbox=ccdBBox,
449  ampExtent=ccdBBox.getDimensions(),
450  numAmps=1,
451  pixelSize=lsst.afw.geom.Extent2D(pixelSize, pixelSize),
452  orientation=lsst.afw.cameraGeom.Orientation(fpPos),
453  plateScale=plateScale,
454  radialDistortion=radialDistortion,
455  ).detector)
456 
457  return lsst.afw.cameraGeom.Camera(
458  name="Simple Camera",
459  detectorList=detectorList,
460  transformMap=transformMap,
461  )
462 
463 
464 def makeDataRepo(root):
465  """
466  Create a data repository for SimpleMapper and return a butler for it.
467 
468  Clobbers anything already in the given path.
469  """
470  if os.path.exists(root):
471  shutil.rmtree(root)
472  os.makedirs(root)
473  with open(os.path.join(root, "_mapper"), "w") as f:
474  f.write("lsst.pipe.tasks.mocks.SimpleMapper\n")
475  return lsst.daf.persistence.Butler(root=root)
def map_camera(self, dataId, write=False)
def bypass_camera(self, datasetType, pythonType, location, dataId)
def __init__(self, persistence, template=None, keys=None)
def makeSimpleCamera(nX, nY, sizeX, sizeY, gapX, gapY, pixelSize=1.0, plateScale=20.0, radialDistortion=0.925)
def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId)
def map(self, dataset, root, dataId, mapper, suffix=None, storage=None)
def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId)
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:81
def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId)
def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId)
def bypass_deepMergedCoaddId_bits(self, datasetType, pythonType, location, dataId)
def query(self, dataset, index, level, format, dataId)
def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId)
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:98
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:64