lsst.pipe.tasks  16.0-8-g7b3fce8c+1
simpleMapper.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 """Mapper and cameraGeom definition for extremely simple mock data.
24 
25 SimpleMapper inherits directly from Mapper, not CameraMapper. This means
26 we can avoid any problems with paf files at the expense of reimplementing
27 some parts of CameraMapper here. Jim is not sure this was the best
28 possible approach, but it gave him an opportunity to play around with
29 prototyping a future paf-free mapper class, and it does everything it
30 needs to do right now.
31 """
32 import os
33 import shutil
34 import re
35 
38 from lsst.afw.cameraGeom.testUtils import DetectorWrapper
39 import lsst.afw.image.utils as afwImageUtils
40 import lsst.afw.image as afwImage
41 
42 __all__ = ("SimpleMapper", "makeSimpleCamera", "makeDataRepo")
43 
44 
46  """Base class of a hierarchy used by SimpleMapper to defined different kinds of types of objects
47  to persist.
48 
49  PersistenceType objects are never instantiated; only the type objects are used (we needed a
50  simple singleton struct that could be inherited, which is exactly what a Python type is).
51  """
52  python = None
53  cpp = "ignored"
54  storage = None
55  ext = ""
56  suffixes = ()
57 
58  @classmethod
59  def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
60  """Method called by SimpleMapping to implement a map_ method."""
61  return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [path], dataId,
62  mapper=mapper,
63  storage=storage)
64 
65  def canStandardize(self, datasetType):
66  return False
67 
68 
70  """Persistence type for things that don't actually use daf_persistence.
71  """
72 
73  python = "lsst.daf.base.PropertySet" # something to import even when we don't need to
74 
75  @classmethod
76  def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
77  """Method called by SimpleMapping to implement a map_ method; overridden to not use the path."""
78  return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [], dataId,
79  mapper=mapper, storage=storage)
80 
81 
83  """Persistence type of Exposure images.
84  """
85 
86  python = "lsst.afw.image.ExposureF"
87  cpp = "ExposureF"
88  storage = "FitsStorage"
89  ext = ".fits"
90  suffixes = ("_sub",)
91 
92  @classmethod
93  def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
94  """Method called by SimpleMapping to implement a map_ method; overridden to support subimages."""
95  if suffix is None:
96  loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, dataId, mapper, suffix=None,
97  storage=storage)
98  elif suffix == "_sub":
99  subId = dataId.copy()
100  bbox = subId.pop('bbox')
101  loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, subId, mapper, suffix=None,
102  storage=storage)
103  loc.additionalData.set('llcX', bbox.getMinX())
104  loc.additionalData.set('llcY', bbox.getMinY())
105  loc.additionalData.set('width', bbox.getWidth())
106  loc.additionalData.set('height', bbox.getHeight())
107  if 'imageOrigin' in dataId:
108  loc.additionalData.set('imageOrigin',
109  dataId['imageOrigin'])
110  return loc
111 
112 
114  python = "lsst.skymap.BaseSkyMap"
115  storage = "PickleStorage"
116  ext = ".pickle"
117 
118 
120  python = "lsst.afw.table.BaseCatalog"
121  cpp = "BaseCatalog"
122  storage = "FitsCatalogStorage"
123  ext = ".fits"
124 
125 
127  python = "lsst.afw.table.SimpleCatalog"
128  cpp = "SimpleCatalog"
129 
130 
132  python = "lsst.afw.table.SourceCatalog"
133  cpp = "SourceCatalog"
134 
135 
137  python = "lsst.afw.table.ExposureCatalog"
138  cpp = "ExposureCatalog"
139 
140 
142  python = "lsst.afw.detection.PeakCatalog"
143  cpp = "PeakCatalog"
144 
145 
147  """Mapping object used to implement SimpleMapper, similar in intent to lsst.daf.peristence.Mapping.
148  """
149 
150  template = None
151  keys = {}
152 
153  def __init__(self, persistence, template=None, keys=None):
154  self.persistence = persistence
155  if template is not None:
156  self.template = template
157  if keys is not None:
158  self.keys = keys
159 
160  def map(self, dataset, root, dataId, mapper, suffix=None, storage=None):
161  if self.template is not None:
162  path = self.template.format(dataset=dataset, ext=self.persistence.ext, **dataId)
163  else:
164  path = None
165  return self.persistence.makeButlerLocation(path, dataId, suffix=suffix, mapper=mapper,
166  storage=storage)
167 
168 
170  """Mapping for dataset types that are organized the same way as raw data (i.e. by CCD)."""
171 
172  template = "{dataset}-{visit:04d}-{ccd:01d}{ext}"
173  keys = dict(visit=int, ccd=int)
174 
175  def query(self, dataset, index, level, format, dataId):
176  dictList = index[dataset][level]
177  results = [list(d.values()) for d in dictList[dataId.get(level, None)]]
178  return results
179 
180 
182  """Mapping for dataset types that are organized according to a SkyMap subdivision of the sky."""
183 
184  template = "{dataset}-{filter}-{tract:02d}-{patch}{ext}"
185  keys = dict(filter=str, tract=int, patch=str)
186 
187 
189  """Mapping for CoaddTempExp datasets."""
190 
191  template = "{dataset}-{tract:02d}-{patch}-{visit:04d}{ext}"
192  keys = dict(tract=int, patch=str, visit=int)
193 
194 
196  """Mapping for forced_src datasets."""
197 
198  template = "{dataset}-{tract:02d}-{visit:04d}-{ccd:01d}{ext}"
199  keys = dict(tract=int, ccd=int, visit=int)
200 
201 
202 class MapperMeta(type):
203  """Metaclass for SimpleMapper that creates map_ and query_ methods for everything found in the
204  'mappings' class variable.
205  """
206 
207  @staticmethod
208  def _makeMapClosure(dataset, mapping, suffix=None):
209  def mapClosure(self, dataId, write=False):
210  return mapping.map(dataset, self.root, dataId, self, suffix=suffix, storage=self.storage)
211  return mapClosure
212 
213  @staticmethod
214  def _makeQueryClosure(dataset, mapping):
215  def queryClosure(self, level, format, dataId):
216  return mapping.query(dataset, self.index, level, format, dataId)
217  return queryClosure
218 
219  def __init__(cls, name, bases, dict_): # noqa allow "cls" instead of "self"
220  type.__init__(cls, name, bases, dict_)
221  cls.keyDict = dict()
222  for dataset, mapping in cls.mappings.items():
223  setattr(cls, "map_" + dataset, MapperMeta._makeMapClosure(dataset, mapping, suffix=None))
224  for suffix in mapping.persistence.suffixes:
225  setattr(cls, "map_" + dataset + suffix,
226  MapperMeta._makeMapClosure(dataset, mapping, suffix=suffix))
227  if hasattr(mapping, "query"):
228  setattr(cls, "query_" + dataset, MapperMeta._makeQueryClosure(dataset, mapping))
229  cls.keyDict.update(mapping.keys)
230 
231 
232 class SimpleMapper(lsst.daf.persistence.Mapper, metaclass=MapperMeta):
233  """
234  An extremely simple mapper for an imaginary camera for use in integration tests.
235 
236  As SimpleMapper does not inherit from obs.base.CameraMapper, it does not
237  use a policy file to set mappings or a registry; all the information is here
238  (in the map_* and query_* methods).
239 
240  The imaginary camera's raw data format has only 'visit' and 'ccd' keys, with
241  two CCDs per visit (by default).
242  """
243 
244  mappings = dict(
245  calexp=RawMapping(ExposurePersistenceType),
246  forced_src=ForcedSrcMapping(SourceCatalogPersistenceType),
247  forced_src_schema=SimpleMapping(SourceCatalogPersistenceType,
248  template="{dataset}{ext}", keys={}),
249  truth=SimpleMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
250  keys={"tract": int}),
251  simsrc=RawMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
252  keys={"tract": int}),
253  observations=SimpleMapping(ExposureCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
254  keys={"tract": int}),
255  ccdExposureId=RawMapping(BypassPersistenceType),
256  ccdExposureId_bits=SimpleMapping(BypassPersistenceType),
257  deepCoaddId=SkyMapping(BypassPersistenceType),
258  deepCoaddId_bits=SimpleMapping(BypassPersistenceType),
259  deepMergedCoaddId=SkyMapping(BypassPersistenceType),
260  deepMergedCoaddId_bits=SimpleMapping(BypassPersistenceType),
261  deepCoadd_skyMap=SimpleMapping(SkyMapPersistenceType, template="{dataset}{ext}", keys={}),
262  deepCoadd=SkyMapping(ExposurePersistenceType),
263  deepCoaddPsfMatched=SkyMapping(ExposurePersistenceType),
264  deepCoadd_calexp=SkyMapping(ExposurePersistenceType),
265  deepCoadd_calexp_background=SkyMapping(CatalogPersistenceType),
266  deepCoadd_icSrc=SkyMapping(SourceCatalogPersistenceType),
267  deepCoadd_icSrc_schema=SimpleMapping(SourceCatalogPersistenceType,
268  template="{dataset}{ext}", keys={}),
269  deepCoadd_src=SkyMapping(SourceCatalogPersistenceType),
270  deepCoadd_src_schema=SimpleMapping(SourceCatalogPersistenceType,
271  template="{dataset}{ext}", keys={}),
272  deepCoadd_peak_schema=SimpleMapping(PeakCatalogPersistenceType,
273  template="{dataset}{ext}", keys={}),
274  deepCoadd_ref=SkyMapping(SourceCatalogPersistenceType),
275  deepCoadd_ref_schema=SimpleMapping(SourceCatalogPersistenceType,
276  template="{dataset}{ext}", keys={}),
277  deepCoadd_det=SkyMapping(SourceCatalogPersistenceType),
278  deepCoadd_det_schema=SimpleMapping(SourceCatalogPersistenceType,
279  template="{dataset}{ext}", keys={}),
280  deepCoadd_mergeDet=SkyMapping(SourceCatalogPersistenceType),
281  deepCoadd_mergeDet_schema=SimpleMapping(SourceCatalogPersistenceType,
282  template="{dataset}{ext}", keys={}),
283  deepCoadd_meas=SkyMapping(SourceCatalogPersistenceType),
284  deepCoadd_meas_schema=SimpleMapping(SourceCatalogPersistenceType,
285  template="{dataset}{ext}", keys={}),
286  deepCoadd_forced_src=SkyMapping(SourceCatalogPersistenceType),
287  deepCoadd_forced_src_schema=SimpleMapping(SourceCatalogPersistenceType,
288  template="{dataset}{ext}", keys={}),
289  deepCoadd_mock=SkyMapping(ExposurePersistenceType),
290  deepCoaddPsfMatched_mock=SkyMapping(ExposurePersistenceType),
291  deepCoadd_directWarp=TempExpMapping(ExposurePersistenceType),
292  deepCoadd_directWarp_mock=TempExpMapping(ExposurePersistenceType),
293  deepCoadd_psfMatchedWarp=TempExpMapping(ExposurePersistenceType),
294  deepCoadd_psfMatchedWarp_mock=TempExpMapping(ExposurePersistenceType),
295  )
296 
297  levels = dict(
298  visit=['ccd'],
299  ccd=[],
300  )
301 
302  def __init__(self, root, **kwargs):
303  self.storage = lsst.daf.persistence.Storage.makeFromURI(root)
304  super(SimpleMapper, self).__init__(**kwargs)
305  self.root = root
306  self.camera = makeSimpleCamera(nX=1, nY=2, sizeX=400, sizeY=200, gapX=2, gapY=2)
307  afwImageUtils.defineFilter('r', 619.42)
308  self.update()
309 
310  def getDefaultLevel(self):
311  return "ccd"
312 
313  def getKeys(self, datasetType, level):
314  if datasetType is None:
315  keyDict = self.keyDict
316  else:
317  keyDict = self.mappings[datasetType].keys
318  if level is not None and level in self.levels:
319  keyDict = dict(keyDict)
320  for l in self.levels[level]:
321  if l in keyDict:
322  del keyDict[l]
323  return keyDict
324 
325  def update(self):
326  filenames = os.listdir(self.root)
327  rawRegex = re.compile(r"(?P<dataset>\w+)-(?P<visit>\d+)-(?P<ccd>\d).*")
328  self.index = {}
329  for filename in filenames:
330  m = rawRegex.match(filename)
331  if not m:
332  continue
333  index = self.index.setdefault(m.group('dataset'), dict(ccd={None: []}, visit={None: []}))
334  visit = int(m.group('visit'))
335  ccd = int(m.group('ccd'))
336  d1 = dict(visit=visit, ccd=ccd)
337  d2 = dict(visit=visit)
338  index['ccd'].setdefault(visit, []).append(d1)
339  index['ccd'][None].append(d1)
340  index['visit'][visit] = [d2]
341  index['visit'][None].append(d1)
342 
343  def keys(self):
344  return self.keyDict
345 
346  def bypass_camera(self, datasetType, pythonType, location, dataId):
347  return self.camera
348 
349  def map_camera(self, dataId, write=False):
350  return lsst.daf.persistence.ButlerLocation(
351  "lsst.afw.cameraGeom.Camera", "Camera", None, [], dataId, mapper=self, storage=self.storage
352  )
353 
354  def std_calexp(self, item, dataId):
355  detectorId = dataId["ccd"]
356  detector = self.camera[detectorId]
357  item.setDetector(detector)
358  item.setFilter(afwImage.Filter("r"))
359  return item
360 
361  def _computeCcdExposureId(self, dataId):
362  return int(dataId["visit"]) * 10 + int(dataId["ccd"])
363 
364  def _computeCoaddId(self, dataId):
365  # Note: for real IDs, we'd want to include filter here, but it doesn't actually matter
366  # for any of the tests we've done so far, which all assume filter='r'
367  tract = int(dataId['tract'])
368  if tract < 0 or tract >= 128:
369  raise RuntimeError('tract not in range [0,128)')
370  patchX, patchY = (int(c) for c in dataId['patch'].split(','))
371  for p in (patchX, patchY):
372  if p < 0 or p >= 2**13:
373  raise RuntimeError('patch component not in range [0, 8192)')
374  return (tract * 2**13 + patchX) * 2**13 + patchY
375 
376  @staticmethod
377  def splitCcdExposureId(ccdExposureId):
378  return dict(visit=(int(ccdExposureId) // 10), ccd=(int(ccdExposureId) % 10))
379 
380  def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
381  return self._computeCcdExposureId(dataId)
382 
383  def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
384  return 32
385 
386  def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId):
387  return self._computeCoaddId(dataId)
388 
389  def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId):
390  return 1 + 7 + 13*2 + 3
391 
392  def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId):
393  return self._computeCoaddId(dataId)
394 
395  def bypass_deepMergedCoaddId_bits(self, datasetType, pythonType, location, dataId):
396  return 1 + 7 + 13*2 + 3
397 
398 
399 def makeSimpleCamera(
400  nX, nY,
401  sizeX, sizeY,
402  gapX, gapY,
403  pixelSize=1.0,
404  plateScale=20.0,
405  radialDistortion=0.925,
406 ):
407  """Create a camera
408 
409  @param[in] nx: number of detectors in x
410  @param[in] ny: number of detectors in y
411  @param[in] sizeX: detector size in x (pixels)
412  @param[in] sizeY: detector size in y (pixels)
413  @param[in] gapX: gap between detectors in x (mm)
414  @param[in] gapY: gap between detectors in y (mm)
415  @param[in] pixelSize: pixel size (mm) (a float)
416  @param[in] plateScale: plate scale in arcsec/mm; 20.0 is for LSST
417  @param[in] radialDistortion: radial distortion, in mm/rad^2
418  (the r^3 coefficient of the radial distortion polynomial
419  that converts FIELD_ANGLE in radians to FOCAL_PLANE in mm);
420  0.925 is the value Dave Monet measured for lsstSim data
421 
422  Each detector will have one amplifier (with no raw information).
423  """
424  pScaleRad = lsst.afw.geom.arcsecToRad(plateScale)
425  radialDistortCoeffs = [0.0, 1.0/pScaleRad, 0.0, radialDistortion/pScaleRad]
426  focalPlaneToFieldAngle = lsst.afw.geom.makeRadialTransform(radialDistortCoeffs)
427  nativeSys = lsst.afw.cameraGeom.FOCAL_PLANE
428  transforms = {
429  lsst.afw.cameraGeom.FIELD_ANGLE: focalPlaneToFieldAngle,
430  }
431  transformMap = lsst.afw.cameraGeom.TransformMap(nativeSys, transforms)
432 
433  detectorList = []
434  ccdBBox = lsst.afw.geom.Box2I(lsst.afw.geom.Point2I(), lsst.afw.geom.Extent2I(sizeX, sizeY))
435  for iY in range(nY):
436  cY = (iY - 0.5 * (nY - 1)) * (pixelSize * sizeY + gapY)
437  for iX in range(nX):
438  cX = (iX - 0.5 * (nX - 1)) * (pixelSize * sizeY + gapX)
439  fpPos = lsst.afw.geom.Point2D(cX, cY)
440  detectorName = "detector %d,%d" % (iX, iY)
441  detectorId = len(detectorList) + 1
442  detectorList.append(DetectorWrapper(
443  name=detectorName,
444  id=detectorId,
445  serial=detectorName + " serial",
446  bbox=ccdBBox,
447  ampExtent=ccdBBox.getDimensions(),
448  numAmps=1,
449  pixelSize=lsst.afw.geom.Extent2D(pixelSize, pixelSize),
450  orientation=lsst.afw.cameraGeom.Orientation(fpPos),
451  plateScale=plateScale,
452  radialDistortion=radialDistortion,
453  ).detector)
454 
455  return lsst.afw.cameraGeom.Camera(
456  name="Simple Camera",
457  detectorList=detectorList,
458  transformMap=transformMap,
459  )
460 
461 
462 def makeDataRepo(root):
463  """
464  Create a data repository for SimpleMapper and return a butler for it.
465 
466  Clobbers anything already in the given path.
467  """
468  if os.path.exists(root):
469  shutil.rmtree(root)
470  os.makedirs(root)
471  with open(os.path.join(root, "_mapper"), "w") as f:
472  f.write("lsst.pipe.tasks.mocks.SimpleMapper\n")
473  return lsst.daf.persistence.Butler(root=root)
def map_camera(self, dataId, write=False)
def bypass_camera(self, datasetType, pythonType, location, dataId)
def __init__(self, persistence, template=None, keys=None)
def makeSimpleCamera(nX, nY, sizeX, sizeY, gapX, gapY, pixelSize=1.0, plateScale=20.0, radialDistortion=0.925)
def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId)
def map(self, dataset, root, dataId, mapper, suffix=None, storage=None)
def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId)
std::shared_ptr< TransformPoint2ToPoint2 > makeRadialTransform(std::vector< double > const &forwardCoeffs, std::vector< double > const &inverseCoeffs)
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:76
def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId)
def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId)
def bypass_deepMergedCoaddId_bits(self, datasetType, pythonType, location, dataId)
def query(self, dataset, index, level, format, dataId)
def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId)
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:93
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:59