lsst.pipe.tasks  14.0-32-gd046a3e2+3
simpleMapper.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 """Mapper and cameraGeom definition for extremely simple mock data.
24 
25 SimpleMapper inherits directly from Mapper, not CameraMapper. This means
26 we can avoid any problems with paf files at the expense of reimplementing
27 some parts of CameraMapper here. Jim is not sure this was the best
28 possible approach, but it gave him an opportunity to play around with
29 prototyping a future paf-free mapper class, and it does everything it
30 needs to do right now.
31 """
32 from __future__ import absolute_import, division, print_function
33 from builtins import map
34 from builtins import range
35 from builtins import object
36 
37 import os
38 import shutil
39 import re
40 
43 from lsst.afw.cameraGeom.testUtils import DetectorWrapper
44 import lsst.afw.image.utils as afwImageUtils
45 import lsst.afw.image as afwImage
46 from future.utils import with_metaclass
47 
48 __all__ = ("SimpleMapper", "makeSimpleCamera", "makeDataRepo")
49 
50 
51 class PersistenceType(object):
52  """Base class of a hierarchy used by SimpleMapper to defined different kinds of types of objects
53  to persist.
54 
55  PersistenceType objects are never instantiated; only the type objects are used (we needed a
56  simple singleton struct that could be inherited, which is exactly what a Python type is).
57  """
58  python = None
59  cpp = "ignored"
60  storage = None
61  ext = ""
62  suffixes = ()
63 
64  @classmethod
65  def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
66  """Method called by SimpleMapping to implement a map_ method."""
67  return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [path], dataId,
68  mapper=mapper,
69  storage=storage)
70 
71  def canStandardize(self, datasetType):
72  return False
73 
74 
76  """Persistence type for things that don't actually use daf_persistence.
77  """
78 
79  python = "lsst.daf.base.PropertySet" # something to import even when we don't need to
80 
81  @classmethod
82  def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
83  """Method called by SimpleMapping to implement a map_ method; overridden to not use the path."""
84  return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [], dataId,
85  mapper=mapper, storage=storage)
86 
87 
89  """Persistence type of Exposure images.
90  """
91 
92  python = "lsst.afw.image.ExposureF"
93  cpp = "ExposureF"
94  storage = "FitsStorage"
95  ext = ".fits"
96  suffixes = ("_sub",)
97 
98  @classmethod
99  def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
100  """Method called by SimpleMapping to implement a map_ method; overridden to support subimages."""
101  if suffix is None:
102  loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, dataId, mapper, suffix=None,
103  storage=storage)
104  elif suffix == "_sub":
105  subId = dataId.copy()
106  bbox = subId.pop('bbox')
107  loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, subId, mapper, suffix=None,
108  storage=storage)
109  loc.additionalData.set('llcX', bbox.getMinX())
110  loc.additionalData.set('llcY', bbox.getMinY())
111  loc.additionalData.set('width', bbox.getWidth())
112  loc.additionalData.set('height', bbox.getHeight())
113  if 'imageOrigin' in dataId:
114  loc.additionalData.set('imageOrigin',
115  dataId['imageOrigin'])
116  return loc
117 
118 
120  python = "lsst.skymap.BaseSkyMap"
121  storage = "PickleStorage"
122  ext = ".pickle"
123 
124 
126  python = "lsst.afw.table.BaseCatalog"
127  cpp = "BaseCatalog"
128  storage = "FitsCatalogStorage"
129  ext = ".fits"
130 
131 
133  python = "lsst.afw.table.SimpleCatalog"
134  cpp = "SimpleCatalog"
135 
136 
138  python = "lsst.afw.table.SourceCatalog"
139  cpp = "SourceCatalog"
140 
141 
143  python = "lsst.afw.table.ExposureCatalog"
144  cpp = "ExposureCatalog"
145 
146 
148  python = "lsst.afw.detection.PeakCatalog"
149  cpp = "PeakCatalog"
150 
151 
152 class SimpleMapping(object):
153  """Mapping object used to implement SimpleMapper, similar in intent to lsst.daf.peristence.Mapping.
154  """
155 
156  template = None
157  keys = {}
158 
159  def __init__(self, persistence, template=None, keys=None):
160  self.persistence = persistence
161  if template is not None:
162  self.template = template
163  if keys is not None:
164  self.keys = keys
165 
166  def map(self, dataset, root, dataId, mapper, suffix=None, storage=None):
167  if self.template is not None:
168  path = self.template.format(dataset=dataset, ext=self.persistence.ext, **dataId)
169  else:
170  path = None
171  return self.persistence.makeButlerLocation(path, dataId, suffix=suffix, mapper=mapper,
172  storage=storage)
173 
174 
176  """Mapping for dataset types that are organized the same way as raw data (i.e. by CCD)."""
177 
178  template = "{dataset}-{visit:04d}-{ccd:01d}{ext}"
179  keys = dict(visit=int, ccd=int)
180 
181  def query(self, dataset, index, level, format, dataId):
182  dictList = index[dataset][level]
183  results = [list(d.values()) for d in dictList[dataId.get(level, None)]]
184  return results
185 
186 
188  """Mapping for dataset types that are organized according to a SkyMap subdivision of the sky."""
189 
190  template = "{dataset}-{filter}-{tract:02d}-{patch}{ext}"
191  keys = dict(filter=str, tract=int, patch=str)
192 
193 
195  """Mapping for CoaddTempExp datasets."""
196 
197  template = "{dataset}-{tract:02d}-{patch}-{visit:04d}{ext}"
198  keys = dict(tract=int, patch=str, visit=int)
199 
200 
202  """Mapping for forced_src datasets."""
203 
204  template = "{dataset}-{tract:02d}-{visit:04d}-{ccd:01d}{ext}"
205  keys = dict(tract=int, ccd=int, visit=int)
206 
207 
208 class MapperMeta(type):
209  """Metaclass for SimpleMapper that creates map_ and query_ methods for everything found in the
210  'mappings' class variable.
211  """
212 
213  @staticmethod
214  def _makeMapClosure(dataset, mapping, suffix=None):
215  def mapClosure(self, dataId, write=False):
216  return mapping.map(dataset, self.root, dataId, self, suffix=suffix, storage=self.storage)
217  return mapClosure
218 
219  @staticmethod
220  def _makeQueryClosure(dataset, mapping):
221  def queryClosure(self, level, format, dataId):
222  return mapping.query(dataset, self.index, level, format, dataId)
223  return queryClosure
224 
225  def __init__(cls, name, bases, dict_):
226  type.__init__(cls, name, bases, dict_)
227  cls.keyDict = dict()
228  for dataset, mapping in cls.mappings.items():
229  setattr(cls, "map_" + dataset, MapperMeta._makeMapClosure(dataset, mapping, suffix=None))
230  for suffix in mapping.persistence.suffixes:
231  setattr(cls, "map_" + dataset + suffix,
232  MapperMeta._makeMapClosure(dataset, mapping, suffix=suffix))
233  if hasattr(mapping, "query"):
234  setattr(cls, "query_" + dataset, MapperMeta._makeQueryClosure(dataset, mapping))
235  cls.keyDict.update(mapping.keys)
236 
237 
238 class SimpleMapper(with_metaclass(MapperMeta, lsst.daf.persistence.Mapper)):
239  """
240  An extremely simple mapper for an imaginary camera for use in integration tests.
241 
242  As SimpleMapper does not inherit from obs.base.CameraMapper, it does not
243  use a policy file to set mappings or a registry; all the information is here
244  (in the map_* and query_* methods).
245 
246  The imaginary camera's raw data format has only 'visit' and 'ccd' keys, with
247  two CCDs per visit (by default).
248  """
249 
250  mappings = dict(
251  calexp=RawMapping(ExposurePersistenceType),
252  forced_src=ForcedSrcMapping(SourceCatalogPersistenceType),
253  forced_src_schema=SimpleMapping(SourceCatalogPersistenceType,
254  template="{dataset}{ext}", keys={}),
255  truth=SimpleMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
256  keys={"tract": int}),
257  simsrc=RawMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
258  keys={"tract": int}),
259  observations=SimpleMapping(ExposureCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
260  keys={"tract": int}),
261  ccdExposureId=RawMapping(BypassPersistenceType),
262  ccdExposureId_bits=SimpleMapping(BypassPersistenceType),
263  deepCoaddId=SkyMapping(BypassPersistenceType),
264  deepCoaddId_bits=SimpleMapping(BypassPersistenceType),
265  deepMergedCoaddId=SkyMapping(BypassPersistenceType),
266  deepMergedCoaddId_bits=SimpleMapping(BypassPersistenceType),
267  deepCoadd_skyMap=SimpleMapping(SkyMapPersistenceType, template="{dataset}{ext}", keys={}),
268  deepCoadd=SkyMapping(ExposurePersistenceType),
269  deepCoaddPsfMatched=SkyMapping(ExposurePersistenceType),
270  deepCoadd_calexp=SkyMapping(ExposurePersistenceType),
271  deepCoadd_calexp_background=SkyMapping(CatalogPersistenceType),
272  deepCoadd_icSrc=SkyMapping(SourceCatalogPersistenceType),
273  deepCoadd_icSrc_schema=SimpleMapping(SourceCatalogPersistenceType,
274  template="{dataset}{ext}", keys={}),
275  deepCoadd_src=SkyMapping(SourceCatalogPersistenceType),
276  deepCoadd_src_schema=SimpleMapping(SourceCatalogPersistenceType,
277  template="{dataset}{ext}", keys={}),
278  deepCoadd_peak_schema=SimpleMapping(PeakCatalogPersistenceType,
279  template="{dataset}{ext}", keys={}),
280  deepCoadd_ref=SkyMapping(SourceCatalogPersistenceType),
281  deepCoadd_ref_schema=SimpleMapping(SourceCatalogPersistenceType,
282  template="{dataset}{ext}", keys={}),
283  deepCoadd_det=SkyMapping(SourceCatalogPersistenceType),
284  deepCoadd_det_schema=SimpleMapping(SourceCatalogPersistenceType,
285  template="{dataset}{ext}", keys={}),
286  deepCoadd_mergeDet=SkyMapping(SourceCatalogPersistenceType),
287  deepCoadd_mergeDet_schema=SimpleMapping(SourceCatalogPersistenceType,
288  template="{dataset}{ext}", keys={}),
289  deepCoadd_meas=SkyMapping(SourceCatalogPersistenceType),
290  deepCoadd_meas_schema=SimpleMapping(SourceCatalogPersistenceType,
291  template="{dataset}{ext}", keys={}),
292  deepCoadd_forced_src=SkyMapping(SourceCatalogPersistenceType),
293  deepCoadd_forced_src_schema=SimpleMapping(SourceCatalogPersistenceType,
294  template="{dataset}{ext}", keys={}),
295  deepCoadd_mock=SkyMapping(ExposurePersistenceType),
296  deepCoaddPsfMatched_mock=SkyMapping(ExposurePersistenceType),
297  deepCoadd_directWarp=TempExpMapping(ExposurePersistenceType),
298  deepCoadd_directWarp_mock=TempExpMapping(ExposurePersistenceType),
299  deepCoadd_psfMatchedWarp=TempExpMapping(ExposurePersistenceType),
300  deepCoadd_psfMatchedWarp_mock=TempExpMapping(ExposurePersistenceType),
301  )
302 
303  levels = dict(
304  visit=['ccd'],
305  ccd=[],
306  )
307 
308  def __init__(self, root, **kwargs):
309  self.storage = lsst.daf.persistence.Storage.makeFromURI(root)
310  super(SimpleMapper, self).__init__(**kwargs)
311  self.root = root
312  self.camera = makeSimpleCamera(nX=1, nY=2, sizeX=400, sizeY=200, gapX=2, gapY=2)
313  afwImageUtils.defineFilter('r', 619.42)
314  self.update()
315 
316  def getDefaultLevel(self): return "ccd"
317 
318  def getKeys(self, datasetType, level):
319  if datasetType is None:
320  keyDict = self.keyDict
321  else:
322  keyDict = self.mappings[datasetType].keys
323  if level is not None and level in self.levels:
324  keyDict = dict(keyDict)
325  for l in self.levels[level]:
326  if l in keyDict:
327  del keyDict[l]
328  return keyDict
329 
330  def update(self):
331  filenames = os.listdir(self.root)
332  rawRegex = re.compile(r"(?P<dataset>\w+)-(?P<visit>\d+)-(?P<ccd>\d).*")
333  self.index = {}
334  for filename in filenames:
335  m = rawRegex.match(filename)
336  if not m:
337  continue
338  index = self.index.setdefault(m.group('dataset'), dict(ccd={None: []}, visit={None: []}))
339  visit = int(m.group('visit'))
340  ccd = int(m.group('ccd'))
341  d1 = dict(visit=visit, ccd=ccd)
342  d2 = dict(visit=visit)
343  index['ccd'].setdefault(visit, []).append(d1)
344  index['ccd'][None].append(d1)
345  index['visit'][visit] = [d2]
346  index['visit'][None].append(d1)
347 
348  def keys(self):
349  return self.keyDict
350 
351  def bypass_camera(self, datasetType, pythonType, location, dataId):
352  return self.camera
353 
354  def map_camera(self, dataId, write=False):
355  return lsst.daf.persistence.ButlerLocation(
356  "lsst.afw.cameraGeom.Camera", "Camera", None, [], dataId, mapper=self, storage=self.storage
357  )
358 
359  def std_calexp(self, item, dataId):
360  detectorId = dataId["ccd"]
361  detector = self.camera[detectorId]
362  item.setDetector(detector)
363  item.setFilter(afwImage.Filter("r"))
364  return item
365 
366  def _computeCcdExposureId(self, dataId):
367  return int(dataId["visit"]) * 10 + int(dataId["ccd"])
368 
369  def _computeCoaddId(self, dataId):
370  # Note: for real IDs, we'd want to include filter here, but it doesn't actually matter
371  # for any of the tests we've done so far, which all assume filter='r'
372  tract = int(dataId['tract'])
373  if tract < 0 or tract >= 128:
374  raise RuntimeError('tract not in range [0,128)')
375  patchX, patchY = (int(c) for c in dataId['patch'].split(','))
376  for p in (patchX, patchY):
377  if p < 0 or p >= 2**13:
378  raise RuntimeError('patch component not in range [0, 8192)')
379  return (tract * 2**13 + patchX) * 2**13 + patchY
380 
381  def splitCcdExposureId(ccdExposureId):
382  return dict(visit=(int(ccdExposureId) // 10), ccd=(int(ccdExposureId) % 10))
383 
384  def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
385  return self._computeCcdExposureId(dataId)
386 
387  def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
388  return 32
389 
390  def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId):
391  return self._computeCoaddId(dataId)
392 
393  def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId):
394  return 1 + 7 + 13*2 + 3
395 
396  def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId):
397  return self._computeCoaddId(dataId)
398 
399  def bypass_deepMergedCoaddId_bits(self, datasetType, pythonType, location, dataId):
400  return 1 + 7 + 13*2 + 3
401 
402 
403 def makeSimpleCamera(
404  nX, nY,
405  sizeX, sizeY,
406  gapX, gapY,
407  pixelSize=1.0,
408  plateScale=20.0,
409  radialDistortion=0.925,
410 ):
411  """Create a camera
412 
413  @param[in] nx: number of detectors in x
414  @param[in] ny: number of detectors in y
415  @param[in] sizeX: detector size in x (pixels)
416  @param[in] sizeY: detector size in y (pixels)
417  @param[in] gapX: gap between detectors in x (mm)
418  @param[in] gapY: gap between detectors in y (mm)
419  @param[in] pixelSize: pixel size (mm) (a float)
420  @param[in] plateScale: plate scale in arcsec/mm; 20.0 is for LSST
421  @param[in] radialDistortion: radial distortion, in mm/rad^2
422  (the r^3 coefficient of the radial distortion polynomial
423  that converts FIELD_ANGLE in radians to FOCAL_PLANE in mm);
424  0.925 is the value Dave Monet measured for lsstSim data
425 
426  Each detector will have one amplifier (with no raw information).
427  """
428  pScaleRad = lsst.afw.geom.arcsecToRad(plateScale)
429  radialDistortCoeffs = [0.0, 1.0/pScaleRad, 0.0, radialDistortion/pScaleRad]
430  focalPlaneToFieldAngle = lsst.afw.geom.makeRadialTransform(radialDistortCoeffs)
431  nativeSys = lsst.afw.cameraGeom.FOCAL_PLANE
432  transforms = {
433  lsst.afw.cameraGeom.FIELD_ANGLE: focalPlaneToFieldAngle,
434  }
435  transformMap = lsst.afw.cameraGeom.TransformMap(nativeSys, transforms)
436 
437  detectorList = []
439  for iY in range(nY):
440  cY = (iY - 0.5 * (nY - 1)) * (pixelSize * sizeY + gapY)
441  for iX in range(nX):
442  cX = (iX - 0.5 * (nX - 1)) * (pixelSize * sizeY + gapX)
443  fpPos = lsst.afw.geom.Point2D(cX, cY)
444  detectorName = "detector %d,%d" % (iX, iY)
445  detectorId = len(detectorList) + 1
446  detectorList.append(DetectorWrapper(
447  name=detectorName,
448  id=detectorId,
449  serial=detectorName + " serial",
450  bbox=ccdBBox,
451  ampExtent=ccdBBox.getDimensions(),
452  numAmps=1,
453  pixelSize=lsst.afw.geom.Extent2D(pixelSize, pixelSize),
454  orientation=lsst.afw.cameraGeom.Orientation(fpPos),
455  plateScale=plateScale,
456  radialDistortion=radialDistortion,
457  ).detector)
458 
459  return lsst.afw.cameraGeom.Camera(
460  name="Simple Camera",
461  detectorList=detectorList,
462  transformMap=transformMap,
463  )
464 
465 
466 def makeDataRepo(root):
467  """
468  Create a data repository for SimpleMapper and return a butler for it.
469 
470  Clobbers anything already in the given path.
471  """
472  if os.path.exists(root):
473  shutil.rmtree(root)
474  os.makedirs(root)
475  with open(os.path.join(root, "_mapper"), "w") as f:
476  f.write("lsst.pipe.tasks.mocks.SimpleMapper\n")
477  return lsst.daf.persistence.Butler(root=root)
def map_camera(self, dataId, write=False)
def bypass_camera(self, datasetType, pythonType, location, dataId)
std::shared_ptr< TransformPoint2ToPoint2 > makeRadialTransform(std::vector< double > const &forwardCoeffs, std::vector< double > const &inverseCoeffs)
def __init__(self, persistence, template=None, keys=None)
def makeSimpleCamera(nX, nY, sizeX, sizeY, gapX, gapY, pixelSize=1.0, plateScale=20.0, radialDistortion=0.925)
def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId)
def map(self, dataset, root, dataId, mapper, suffix=None, storage=None)
def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId)
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:82
def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId)
def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId)
def bypass_deepMergedCoaddId_bits(self, datasetType, pythonType, location, dataId)
constexpr double arcsecToRad(double x) noexcept
def query(self, dataset, index, level, format, dataId)
def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId)
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:99
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:65