lsst.pipe.tasks  16.0-17-gd75be2c9+2
simpleMapper.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 """Mapper and cameraGeom definition for extremely simple mock data.
24 
25 SimpleMapper inherits directly from Mapper, not CameraMapper. This means
26 we can avoid any problems with paf files at the expense of reimplementing
27 some parts of CameraMapper here. Jim is not sure this was the best
28 possible approach, but it gave him an opportunity to play around with
29 prototyping a future paf-free mapper class, and it does everything it
30 needs to do right now.
31 """
32 import os
33 import shutil
34 import re
35 
38 from lsst.afw.cameraGeom.testUtils import DetectorWrapper
39 import lsst.afw.image.utils as afwImageUtils
40 import lsst.afw.image as afwImage
41 
42 __all__ = ("SimpleMapper", "makeSimpleCamera", "makeDataRepo")
43 
44 
46  """Base class of a hierarchy used by SimpleMapper to defined different kinds of types of objects
47  to persist.
48 
49  PersistenceType objects are never instantiated; only the type objects are used (we needed a
50  simple singleton struct that could be inherited, which is exactly what a Python type is).
51  """
52  python = None
53  cpp = "ignored"
54  storage = None
55  ext = ""
56  suffixes = ()
57 
58  @classmethod
59  def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
60  """Method called by SimpleMapping to implement a map_ method."""
61  return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [path], dataId,
62  mapper=mapper,
63  storage=storage)
64 
65  def canStandardize(self, datasetType):
66  return False
67 
68 
70  """Persistence type for things that don't actually use daf_persistence.
71  """
72 
73  python = "lsst.daf.base.PropertySet" # something to import even when we don't need to
74 
75  @classmethod
76  def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
77  """Method called by SimpleMapping to implement a map_ method; overridden to not use the path."""
78  return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [], dataId,
79  mapper=mapper, storage=storage)
80 
81 
83  """Persistence type of Exposure images.
84  """
85 
86  python = "lsst.afw.image.ExposureF"
87  cpp = "ExposureF"
88  storage = "FitsStorage"
89  ext = ".fits"
90  suffixes = ("_sub",)
91 
92  @classmethod
93  def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
94  """Method called by SimpleMapping to implement a map_ method; overridden to support subimages."""
95  if suffix is None:
96  loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, dataId, mapper, suffix=None,
97  storage=storage)
98  elif suffix == "_sub":
99  subId = dataId.copy()
100  bbox = subId.pop('bbox')
101  loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, subId, mapper, suffix=None,
102  storage=storage)
103  loc.additionalData.set('llcX', bbox.getMinX())
104  loc.additionalData.set('llcY', bbox.getMinY())
105  loc.additionalData.set('width', bbox.getWidth())
106  loc.additionalData.set('height', bbox.getHeight())
107  if 'imageOrigin' in dataId:
108  loc.additionalData.set('imageOrigin',
109  dataId['imageOrigin'])
110  return loc
111 
112 
114  python = "lsst.skymap.BaseSkyMap"
115  storage = "PickleStorage"
116  ext = ".pickle"
117 
118 
120  python = "lsst.afw.table.BaseCatalog"
121  cpp = "BaseCatalog"
122  storage = "FitsCatalogStorage"
123  ext = ".fits"
124 
125 
127  python = "lsst.afw.table.SimpleCatalog"
128  cpp = "SimpleCatalog"
129 
130 
132  python = "lsst.afw.table.SourceCatalog"
133  cpp = "SourceCatalog"
134 
135 
137  python = "lsst.afw.table.ExposureCatalog"
138  cpp = "ExposureCatalog"
139 
140 
142  python = "lsst.afw.detection.PeakCatalog"
143  cpp = "PeakCatalog"
144 
145 
147  """Mapping object used to implement SimpleMapper, similar in intent to lsst.daf.peristence.Mapping.
148  """
149 
150  template = None
151  keys = {}
152 
153  def __init__(self, persistence, template=None, keys=None):
154  self.persistence = persistence
155  if template is not None:
156  self.template = template
157  if keys is not None:
158  self.keys = keys
159 
160  def map(self, dataset, root, dataId, mapper, suffix=None, storage=None):
161  if self.template is not None:
162  path = self.template.format(dataset=dataset, ext=self.persistence.ext, **dataId)
163  else:
164  path = None
165  return self.persistence.makeButlerLocation(path, dataId, suffix=suffix, mapper=mapper,
166  storage=storage)
167 
168 
170  """Mapping for dataset types that are organized the same way as raw data (i.e. by CCD)."""
171 
172  template = "{dataset}-{visit:04d}-{ccd:01d}{ext}"
173  keys = dict(visit=int, ccd=int)
174 
175  def query(self, dataset, index, level, format, dataId):
176  dictList = index[dataset][level]
177  results = [list(d.values()) for d in dictList[dataId.get(level, None)]]
178  return results
179 
180 
182  """Mapping for dataset types that are organized according to a SkyMap subdivision of the sky."""
183 
184  template = "{dataset}-{filter}-{tract:02d}-{patch}{ext}"
185  keys = dict(filter=str, tract=int, patch=str)
186 
187 
189  """Mapping for CoaddTempExp datasets."""
190 
191  template = "{dataset}-{tract:02d}-{patch}-{visit:04d}{ext}"
192  keys = dict(tract=int, patch=str, visit=int)
193 
194 
196  """Mapping for forced_src datasets."""
197 
198  template = "{dataset}-{tract:02d}-{visit:04d}-{ccd:01d}{ext}"
199  keys = dict(tract=int, ccd=int, visit=int)
200 
201 
202 class MapperMeta(type):
203  """Metaclass for SimpleMapper that creates map_ and query_ methods for everything found in the
204  'mappings' class variable.
205  """
206 
207  @staticmethod
208  def _makeMapClosure(dataset, mapping, suffix=None):
209  def mapClosure(self, dataId, write=False):
210  return mapping.map(dataset, self.root, dataId, self, suffix=suffix, storage=self.storage)
211  return mapClosure
212 
213  @staticmethod
214  def _makeQueryClosure(dataset, mapping):
215  def queryClosure(self, level, format, dataId):
216  return mapping.query(dataset, self.index, level, format, dataId)
217  return queryClosure
218 
219  def __init__(cls, name, bases, dict_): # noqa allow "cls" instead of "self"
220  type.__init__(cls, name, bases, dict_)
221  cls.keyDict = dict()
222  for dataset, mapping in cls.mappings.items():
223  setattr(cls, "map_" + dataset, MapperMeta._makeMapClosure(dataset, mapping, suffix=None))
224  for suffix in mapping.persistence.suffixes:
225  setattr(cls, "map_" + dataset + suffix,
226  MapperMeta._makeMapClosure(dataset, mapping, suffix=suffix))
227  if hasattr(mapping, "query"):
228  setattr(cls, "query_" + dataset, MapperMeta._makeQueryClosure(dataset, mapping))
229  cls.keyDict.update(mapping.keys)
230 
231 
232 class SimpleMapper(lsst.daf.persistence.Mapper, metaclass=MapperMeta):
233  """
234  An extremely simple mapper for an imaginary camera for use in integration tests.
235 
236  As SimpleMapper does not inherit from obs.base.CameraMapper, it does not
237  use a policy file to set mappings or a registry; all the information is here
238  (in the map_* and query_* methods).
239 
240  The imaginary camera's raw data format has only 'visit' and 'ccd' keys, with
241  two CCDs per visit (by default).
242  """
243 
244  mappings = dict(
245  calexp=RawMapping(ExposurePersistenceType),
246  forced_src=ForcedSrcMapping(SourceCatalogPersistenceType),
247  forced_src_schema=SimpleMapping(SourceCatalogPersistenceType,
248  template="{dataset}{ext}", keys={}),
249  truth=SimpleMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
250  keys={"tract": int}),
251  simsrc=RawMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
252  keys={"tract": int}),
253  observations=SimpleMapping(ExposureCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
254  keys={"tract": int}),
255  ccdExposureId=RawMapping(BypassPersistenceType),
256  ccdExposureId_bits=SimpleMapping(BypassPersistenceType),
257  deepCoaddId=SkyMapping(BypassPersistenceType),
258  deepCoaddId_bits=SimpleMapping(BypassPersistenceType),
259  deepMergedCoaddId=SkyMapping(BypassPersistenceType),
260  deepMergedCoaddId_bits=SimpleMapping(BypassPersistenceType),
261  deepCoadd_skyMap=SimpleMapping(SkyMapPersistenceType, template="{dataset}{ext}", keys={}),
262  deepCoadd=SkyMapping(ExposurePersistenceType),
263  deepCoaddPsfMatched=SkyMapping(ExposurePersistenceType),
264  deepCoadd_calexp=SkyMapping(ExposurePersistenceType),
265  deepCoadd_calexp_background=SkyMapping(CatalogPersistenceType),
266  deepCoadd_icSrc=SkyMapping(SourceCatalogPersistenceType),
267  deepCoadd_icSrc_schema=SimpleMapping(SourceCatalogPersistenceType,
268  template="{dataset}{ext}", keys={}),
269  deepCoadd_src=SkyMapping(SourceCatalogPersistenceType),
270  deepCoadd_src_schema=SimpleMapping(SourceCatalogPersistenceType,
271  template="{dataset}{ext}", keys={}),
272  deepCoadd_peak_schema=SimpleMapping(PeakCatalogPersistenceType,
273  template="{dataset}{ext}", keys={}),
274  deepCoadd_ref=SkyMapping(SourceCatalogPersistenceType),
275  deepCoadd_ref_schema=SimpleMapping(SourceCatalogPersistenceType,
276  template="{dataset}{ext}", keys={}),
277  deepCoadd_det=SkyMapping(SourceCatalogPersistenceType),
278  deepCoadd_det_schema=SimpleMapping(SourceCatalogPersistenceType,
279  template="{dataset}{ext}", keys={}),
280  deepCoadd_mergeDet=SkyMapping(SourceCatalogPersistenceType),
281  deepCoadd_mergeDet_schema=SimpleMapping(SourceCatalogPersistenceType,
282  template="{dataset}{ext}", keys={}),
283  deepCoadd_deblendedFlux=SkyMapping(SourceCatalogPersistenceType),
284  deepCoadd_deblendedFlux_schema=SimpleMapping(SourceCatalogPersistenceType,
285  template="{dataset}{ext}", keys={}),
286  deepCoadd_deblendedModel=SkyMapping(SourceCatalogPersistenceType),
287  deepCoadd_deblendedModel_schema=SimpleMapping(SourceCatalogPersistenceType,
288  template="{dataset}{ext}", keys={}),
289  deepCoadd_meas=SkyMapping(SourceCatalogPersistenceType),
290  deepCoadd_meas_schema=SimpleMapping(SourceCatalogPersistenceType,
291  template="{dataset}{ext}", keys={}),
292  deepCoadd_forced_src=SkyMapping(SourceCatalogPersistenceType),
293  deepCoadd_forced_src_schema=SimpleMapping(SourceCatalogPersistenceType,
294  template="{dataset}{ext}", keys={}),
295  deepCoadd_mock=SkyMapping(ExposurePersistenceType),
296  deepCoaddPsfMatched_mock=SkyMapping(ExposurePersistenceType),
297  deepCoadd_directWarp=TempExpMapping(ExposurePersistenceType),
298  deepCoadd_directWarp_mock=TempExpMapping(ExposurePersistenceType),
299  deepCoadd_psfMatchedWarp=TempExpMapping(ExposurePersistenceType),
300  deepCoadd_psfMatchedWarp_mock=TempExpMapping(ExposurePersistenceType),
301  )
302 
303  levels = dict(
304  visit=['ccd'],
305  ccd=[],
306  )
307 
308  def __init__(self, root, **kwargs):
309  self.storage = lsst.daf.persistence.Storage.makeFromURI(root)
310  super(SimpleMapper, self).__init__(**kwargs)
311  self.root = root
312  self.camera = makeSimpleCamera(nX=1, nY=2, sizeX=400, sizeY=200, gapX=2, gapY=2)
313  afwImageUtils.defineFilter('r', 619.42)
314  self.update()
315 
316  def getDefaultLevel(self):
317  return "ccd"
318 
319  def getKeys(self, datasetType, level):
320  if datasetType is None:
321  keyDict = self.keyDict
322  else:
323  keyDict = self.mappings[datasetType].keys
324  if level is not None and level in self.levels:
325  keyDict = dict(keyDict)
326  for l in self.levels[level]:
327  if l in keyDict:
328  del keyDict[l]
329  return keyDict
330 
331  def update(self):
332  filenames = os.listdir(self.root)
333  rawRegex = re.compile(r"(?P<dataset>\w+)-(?P<visit>\d+)-(?P<ccd>\d).*")
334  self.index = {}
335  for filename in filenames:
336  m = rawRegex.match(filename)
337  if not m:
338  continue
339  index = self.index.setdefault(m.group('dataset'), dict(ccd={None: []}, visit={None: []}))
340  visit = int(m.group('visit'))
341  ccd = int(m.group('ccd'))
342  d1 = dict(visit=visit, ccd=ccd)
343  d2 = dict(visit=visit)
344  index['ccd'].setdefault(visit, []).append(d1)
345  index['ccd'][None].append(d1)
346  index['visit'][visit] = [d2]
347  index['visit'][None].append(d1)
348 
349  def keys(self):
350  return self.keyDict
351 
352  def bypass_camera(self, datasetType, pythonType, location, dataId):
353  return self.camera
354 
355  def map_camera(self, dataId, write=False):
356  return lsst.daf.persistence.ButlerLocation(
357  "lsst.afw.cameraGeom.Camera", "Camera", None, [], dataId, mapper=self, storage=self.storage
358  )
359 
360  def std_calexp(self, item, dataId):
361  detectorId = dataId["ccd"]
362  detector = self.camera[detectorId]
363  item.setDetector(detector)
364  item.setFilter(afwImage.Filter("r"))
365  return item
366 
367  def _computeCcdExposureId(self, dataId):
368  return int(dataId["visit"]) * 10 + int(dataId["ccd"])
369 
370  def _computeCoaddId(self, dataId):
371  # Note: for real IDs, we'd want to include filter here, but it doesn't actually matter
372  # for any of the tests we've done so far, which all assume filter='r'
373  tract = int(dataId['tract'])
374  if tract < 0 or tract >= 128:
375  raise RuntimeError('tract not in range [0,128)')
376  patchX, patchY = (int(c) for c in dataId['patch'].split(','))
377  for p in (patchX, patchY):
378  if p < 0 or p >= 2**13:
379  raise RuntimeError('patch component not in range [0, 8192)')
380  return (tract * 2**13 + patchX) * 2**13 + patchY
381 
382  @staticmethod
383  def splitCcdExposureId(ccdExposureId):
384  return dict(visit=(int(ccdExposureId) // 10), ccd=(int(ccdExposureId) % 10))
385 
386  def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
387  return self._computeCcdExposureId(dataId)
388 
389  def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
390  return 32
391 
392  def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId):
393  return self._computeCoaddId(dataId)
394 
395  def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId):
396  return 1 + 7 + 13*2 + 3
397 
398  def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId):
399  return self._computeCoaddId(dataId)
400 
401  def bypass_deepMergedCoaddId_bits(self, datasetType, pythonType, location, dataId):
402  return 1 + 7 + 13*2 + 3
403 
404 
405 def makeSimpleCamera(
406  nX, nY,
407  sizeX, sizeY,
408  gapX, gapY,
409  pixelSize=1.0,
410  plateScale=20.0,
411  radialDistortion=0.925,
412 ):
413  """Create a camera
414 
415  @param[in] nx: number of detectors in x
416  @param[in] ny: number of detectors in y
417  @param[in] sizeX: detector size in x (pixels)
418  @param[in] sizeY: detector size in y (pixels)
419  @param[in] gapX: gap between detectors in x (mm)
420  @param[in] gapY: gap between detectors in y (mm)
421  @param[in] pixelSize: pixel size (mm) (a float)
422  @param[in] plateScale: plate scale in arcsec/mm; 20.0 is for LSST
423  @param[in] radialDistortion: radial distortion, in mm/rad^2
424  (the r^3 coefficient of the radial distortion polynomial
425  that converts FIELD_ANGLE in radians to FOCAL_PLANE in mm);
426  0.925 is the value Dave Monet measured for lsstSim data
427 
428  Each detector will have one amplifier (with no raw information).
429  """
430  pScaleRad = lsst.afw.geom.arcsecToRad(plateScale)
431  radialDistortCoeffs = [0.0, 1.0/pScaleRad, 0.0, radialDistortion/pScaleRad]
432  focalPlaneToFieldAngle = lsst.afw.geom.makeRadialTransform(radialDistortCoeffs)
433  nativeSys = lsst.afw.cameraGeom.FOCAL_PLANE
434  transforms = {
435  lsst.afw.cameraGeom.FIELD_ANGLE: focalPlaneToFieldAngle,
436  }
437  transformMap = lsst.afw.cameraGeom.TransformMap(nativeSys, transforms)
438 
439  detectorList = []
440  ccdBBox = lsst.afw.geom.Box2I(lsst.afw.geom.Point2I(), lsst.afw.geom.Extent2I(sizeX, sizeY))
441  for iY in range(nY):
442  cY = (iY - 0.5 * (nY - 1)) * (pixelSize * sizeY + gapY)
443  for iX in range(nX):
444  cX = (iX - 0.5 * (nX - 1)) * (pixelSize * sizeY + gapX)
445  fpPos = lsst.afw.geom.Point2D(cX, cY)
446  detectorName = "detector %d,%d" % (iX, iY)
447  detectorId = len(detectorList) + 1
448  detectorList.append(DetectorWrapper(
449  name=detectorName,
450  id=detectorId,
451  serial=detectorName + " serial",
452  bbox=ccdBBox,
453  ampExtent=ccdBBox.getDimensions(),
454  numAmps=1,
455  pixelSize=lsst.afw.geom.Extent2D(pixelSize, pixelSize),
456  orientation=lsst.afw.cameraGeom.Orientation(fpPos),
457  plateScale=plateScale,
458  radialDistortion=radialDistortion,
459  ).detector)
460 
461  return lsst.afw.cameraGeom.Camera(
462  name="Simple Camera",
463  detectorList=detectorList,
464  transformMap=transformMap,
465  )
466 
467 
468 def makeDataRepo(root):
469  """
470  Create a data repository for SimpleMapper and return a butler for it.
471 
472  Clobbers anything already in the given path.
473  """
474  if os.path.exists(root):
475  shutil.rmtree(root)
476  os.makedirs(root)
477  with open(os.path.join(root, "_mapper"), "w") as f:
478  f.write("lsst.pipe.tasks.mocks.SimpleMapper\n")
479  return lsst.daf.persistence.Butler(root=root)
def map_camera(self, dataId, write=False)
def bypass_camera(self, datasetType, pythonType, location, dataId)
def __init__(self, persistence, template=None, keys=None)
def makeSimpleCamera(nX, nY, sizeX, sizeY, gapX, gapY, pixelSize=1.0, plateScale=20.0, radialDistortion=0.925)
def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId)
def map(self, dataset, root, dataId, mapper, suffix=None, storage=None)
def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId)
std::shared_ptr< TransformPoint2ToPoint2 > makeRadialTransform(std::vector< double > const &forwardCoeffs, std::vector< double > const &inverseCoeffs)
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:76
def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId)
def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId)
def bypass_deepMergedCoaddId_bits(self, datasetType, pythonType, location, dataId)
def query(self, dataset, index, level, format, dataId)
def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId)
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:93
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:59