lsst.pipe.tasks  15.0-5-g389937dc+5
simpleMapper.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 """Mapper and cameraGeom definition for extremely simple mock data.
24 
25 SimpleMapper inherits directly from Mapper, not CameraMapper. This means
26 we can avoid any problems with paf files at the expense of reimplementing
27 some parts of CameraMapper here. Jim is not sure this was the best
28 possible approach, but it gave him an opportunity to play around with
29 prototyping a future paf-free mapper class, and it does everything it
30 needs to do right now.
31 """
32 from __future__ import absolute_import, division, print_function
33 from builtins import range
34 from builtins import object
35 
36 import os
37 import shutil
38 import re
39 
42 from lsst.afw.cameraGeom.testUtils import DetectorWrapper
43 import lsst.afw.image.utils as afwImageUtils
44 import lsst.afw.image as afwImage
45 from future.utils import with_metaclass
46 
47 __all__ = ("SimpleMapper", "makeSimpleCamera", "makeDataRepo")
48 
49 
50 class PersistenceType(object):
51  """Base class of a hierarchy used by SimpleMapper to defined different kinds of types of objects
52  to persist.
53 
54  PersistenceType objects are never instantiated; only the type objects are used (we needed a
55  simple singleton struct that could be inherited, which is exactly what a Python type is).
56  """
57  python = None
58  cpp = "ignored"
59  storage = None
60  ext = ""
61  suffixes = ()
62 
63  @classmethod
64  def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
65  """Method called by SimpleMapping to implement a map_ method."""
66  return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [path], dataId,
67  mapper=mapper,
68  storage=storage)
69 
70  def canStandardize(self, datasetType):
71  return False
72 
73 
75  """Persistence type for things that don't actually use daf_persistence.
76  """
77 
78  python = "lsst.daf.base.PropertySet" # something to import even when we don't need to
79 
80  @classmethod
81  def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
82  """Method called by SimpleMapping to implement a map_ method; overridden to not use the path."""
83  return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [], dataId,
84  mapper=mapper, storage=storage)
85 
86 
88  """Persistence type of Exposure images.
89  """
90 
91  python = "lsst.afw.image.ExposureF"
92  cpp = "ExposureF"
93  storage = "FitsStorage"
94  ext = ".fits"
95  suffixes = ("_sub",)
96 
97  @classmethod
98  def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
99  """Method called by SimpleMapping to implement a map_ method; overridden to support subimages."""
100  if suffix is None:
101  loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, dataId, mapper, suffix=None,
102  storage=storage)
103  elif suffix == "_sub":
104  subId = dataId.copy()
105  bbox = subId.pop('bbox')
106  loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, subId, mapper, suffix=None,
107  storage=storage)
108  loc.additionalData.set('llcX', bbox.getMinX())
109  loc.additionalData.set('llcY', bbox.getMinY())
110  loc.additionalData.set('width', bbox.getWidth())
111  loc.additionalData.set('height', bbox.getHeight())
112  if 'imageOrigin' in dataId:
113  loc.additionalData.set('imageOrigin',
114  dataId['imageOrigin'])
115  return loc
116 
117 
119  python = "lsst.skymap.BaseSkyMap"
120  storage = "PickleStorage"
121  ext = ".pickle"
122 
123 
125  python = "lsst.afw.table.BaseCatalog"
126  cpp = "BaseCatalog"
127  storage = "FitsCatalogStorage"
128  ext = ".fits"
129 
130 
132  python = "lsst.afw.table.SimpleCatalog"
133  cpp = "SimpleCatalog"
134 
135 
137  python = "lsst.afw.table.SourceCatalog"
138  cpp = "SourceCatalog"
139 
140 
142  python = "lsst.afw.table.ExposureCatalog"
143  cpp = "ExposureCatalog"
144 
145 
147  python = "lsst.afw.detection.PeakCatalog"
148  cpp = "PeakCatalog"
149 
150 
151 class SimpleMapping(object):
152  """Mapping object used to implement SimpleMapper, similar in intent to lsst.daf.peristence.Mapping.
153  """
154 
155  template = None
156  keys = {}
157 
158  def __init__(self, persistence, template=None, keys=None):
159  self.persistence = persistence
160  if template is not None:
161  self.template = template
162  if keys is not None:
163  self.keys = keys
164 
165  def map(self, dataset, root, dataId, mapper, suffix=None, storage=None):
166  if self.template is not None:
167  path = self.template.format(dataset=dataset, ext=self.persistence.ext, **dataId)
168  else:
169  path = None
170  return self.persistence.makeButlerLocation(path, dataId, suffix=suffix, mapper=mapper,
171  storage=storage)
172 
173 
175  """Mapping for dataset types that are organized the same way as raw data (i.e. by CCD)."""
176 
177  template = "{dataset}-{visit:04d}-{ccd:01d}{ext}"
178  keys = dict(visit=int, ccd=int)
179 
180  def query(self, dataset, index, level, format, dataId):
181  dictList = index[dataset][level]
182  results = [list(d.values()) for d in dictList[dataId.get(level, None)]]
183  return results
184 
185 
187  """Mapping for dataset types that are organized according to a SkyMap subdivision of the sky."""
188 
189  template = "{dataset}-{filter}-{tract:02d}-{patch}{ext}"
190  keys = dict(filter=str, tract=int, patch=str)
191 
192 
194  """Mapping for CoaddTempExp datasets."""
195 
196  template = "{dataset}-{tract:02d}-{patch}-{visit:04d}{ext}"
197  keys = dict(tract=int, patch=str, visit=int)
198 
199 
201  """Mapping for forced_src datasets."""
202 
203  template = "{dataset}-{tract:02d}-{visit:04d}-{ccd:01d}{ext}"
204  keys = dict(tract=int, ccd=int, visit=int)
205 
206 
207 class MapperMeta(type):
208  """Metaclass for SimpleMapper that creates map_ and query_ methods for everything found in the
209  'mappings' class variable.
210  """
211 
212  @staticmethod
213  def _makeMapClosure(dataset, mapping, suffix=None):
214  def mapClosure(self, dataId, write=False):
215  return mapping.map(dataset, self.root, dataId, self, suffix=suffix, storage=self.storage)
216  return mapClosure
217 
218  @staticmethod
219  def _makeQueryClosure(dataset, mapping):
220  def queryClosure(self, level, format, dataId):
221  return mapping.query(dataset, self.index, level, format, dataId)
222  return queryClosure
223 
224  def __init__(cls, name, bases, dict_): # noqa allow "cls" instead of "self"
225  type.__init__(cls, name, bases, dict_)
226  cls.keyDict = dict()
227  for dataset, mapping in cls.mappings.items():
228  setattr(cls, "map_" + dataset, MapperMeta._makeMapClosure(dataset, mapping, suffix=None))
229  for suffix in mapping.persistence.suffixes:
230  setattr(cls, "map_" + dataset + suffix,
231  MapperMeta._makeMapClosure(dataset, mapping, suffix=suffix))
232  if hasattr(mapping, "query"):
233  setattr(cls, "query_" + dataset, MapperMeta._makeQueryClosure(dataset, mapping))
234  cls.keyDict.update(mapping.keys)
235 
236 
237 class SimpleMapper(with_metaclass(MapperMeta, lsst.daf.persistence.Mapper)):
238  """
239  An extremely simple mapper for an imaginary camera for use in integration tests.
240 
241  As SimpleMapper does not inherit from obs.base.CameraMapper, it does not
242  use a policy file to set mappings or a registry; all the information is here
243  (in the map_* and query_* methods).
244 
245  The imaginary camera's raw data format has only 'visit' and 'ccd' keys, with
246  two CCDs per visit (by default).
247  """
248 
249  mappings = dict(
250  calexp=RawMapping(ExposurePersistenceType),
251  forced_src=ForcedSrcMapping(SourceCatalogPersistenceType),
252  forced_src_schema=SimpleMapping(SourceCatalogPersistenceType,
253  template="{dataset}{ext}", keys={}),
254  truth=SimpleMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
255  keys={"tract": int}),
256  simsrc=RawMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
257  keys={"tract": int}),
258  observations=SimpleMapping(ExposureCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
259  keys={"tract": int}),
260  ccdExposureId=RawMapping(BypassPersistenceType),
261  ccdExposureId_bits=SimpleMapping(BypassPersistenceType),
262  deepCoaddId=SkyMapping(BypassPersistenceType),
263  deepCoaddId_bits=SimpleMapping(BypassPersistenceType),
264  deepMergedCoaddId=SkyMapping(BypassPersistenceType),
265  deepMergedCoaddId_bits=SimpleMapping(BypassPersistenceType),
266  deepCoadd_skyMap=SimpleMapping(SkyMapPersistenceType, template="{dataset}{ext}", keys={}),
267  deepCoadd=SkyMapping(ExposurePersistenceType),
268  deepCoaddPsfMatched=SkyMapping(ExposurePersistenceType),
269  deepCoadd_calexp=SkyMapping(ExposurePersistenceType),
270  deepCoadd_calexp_background=SkyMapping(CatalogPersistenceType),
271  deepCoadd_icSrc=SkyMapping(SourceCatalogPersistenceType),
272  deepCoadd_icSrc_schema=SimpleMapping(SourceCatalogPersistenceType,
273  template="{dataset}{ext}", keys={}),
274  deepCoadd_src=SkyMapping(SourceCatalogPersistenceType),
275  deepCoadd_src_schema=SimpleMapping(SourceCatalogPersistenceType,
276  template="{dataset}{ext}", keys={}),
277  deepCoadd_peak_schema=SimpleMapping(PeakCatalogPersistenceType,
278  template="{dataset}{ext}", keys={}),
279  deepCoadd_ref=SkyMapping(SourceCatalogPersistenceType),
280  deepCoadd_ref_schema=SimpleMapping(SourceCatalogPersistenceType,
281  template="{dataset}{ext}", keys={}),
282  deepCoadd_det=SkyMapping(SourceCatalogPersistenceType),
283  deepCoadd_det_schema=SimpleMapping(SourceCatalogPersistenceType,
284  template="{dataset}{ext}", keys={}),
285  deepCoadd_mergeDet=SkyMapping(SourceCatalogPersistenceType),
286  deepCoadd_mergeDet_schema=SimpleMapping(SourceCatalogPersistenceType,
287  template="{dataset}{ext}", keys={}),
288  deepCoadd_meas=SkyMapping(SourceCatalogPersistenceType),
289  deepCoadd_meas_schema=SimpleMapping(SourceCatalogPersistenceType,
290  template="{dataset}{ext}", keys={}),
291  deepCoadd_forced_src=SkyMapping(SourceCatalogPersistenceType),
292  deepCoadd_forced_src_schema=SimpleMapping(SourceCatalogPersistenceType,
293  template="{dataset}{ext}", keys={}),
294  deepCoadd_mock=SkyMapping(ExposurePersistenceType),
295  deepCoaddPsfMatched_mock=SkyMapping(ExposurePersistenceType),
296  deepCoadd_directWarp=TempExpMapping(ExposurePersistenceType),
297  deepCoadd_directWarp_mock=TempExpMapping(ExposurePersistenceType),
298  deepCoadd_psfMatchedWarp=TempExpMapping(ExposurePersistenceType),
299  deepCoadd_psfMatchedWarp_mock=TempExpMapping(ExposurePersistenceType),
300  )
301 
302  levels = dict(
303  visit=['ccd'],
304  ccd=[],
305  )
306 
307  def __init__(self, root, **kwargs):
308  self.storage = lsst.daf.persistence.Storage.makeFromURI(root)
309  super(SimpleMapper, self).__init__(**kwargs)
310  self.root = root
311  self.camera = makeSimpleCamera(nX=1, nY=2, sizeX=400, sizeY=200, gapX=2, gapY=2)
312  afwImageUtils.defineFilter('r', 619.42)
313  self.update()
314 
315  def getDefaultLevel(self):
316  return "ccd"
317 
318  def getKeys(self, datasetType, level):
319  if datasetType is None:
320  keyDict = self.keyDict
321  else:
322  keyDict = self.mappings[datasetType].keys
323  if level is not None and level in self.levels:
324  keyDict = dict(keyDict)
325  for l in self.levels[level]:
326  if l in keyDict:
327  del keyDict[l]
328  return keyDict
329 
330  def update(self):
331  filenames = os.listdir(self.root)
332  rawRegex = re.compile(r"(?P<dataset>\w+)-(?P<visit>\d+)-(?P<ccd>\d).*")
333  self.index = {}
334  for filename in filenames:
335  m = rawRegex.match(filename)
336  if not m:
337  continue
338  index = self.index.setdefault(m.group('dataset'), dict(ccd={None: []}, visit={None: []}))
339  visit = int(m.group('visit'))
340  ccd = int(m.group('ccd'))
341  d1 = dict(visit=visit, ccd=ccd)
342  d2 = dict(visit=visit)
343  index['ccd'].setdefault(visit, []).append(d1)
344  index['ccd'][None].append(d1)
345  index['visit'][visit] = [d2]
346  index['visit'][None].append(d1)
347 
348  def keys(self):
349  return self.keyDict
350 
351  def bypass_camera(self, datasetType, pythonType, location, dataId):
352  return self.camera
353 
354  def map_camera(self, dataId, write=False):
355  return lsst.daf.persistence.ButlerLocation(
356  "lsst.afw.cameraGeom.Camera", "Camera", None, [], dataId, mapper=self, storage=self.storage
357  )
358 
359  def std_calexp(self, item, dataId):
360  detectorId = dataId["ccd"]
361  detector = self.camera[detectorId]
362  item.setDetector(detector)
363  item.setFilter(afwImage.Filter("r"))
364  return item
365 
366  def _computeCcdExposureId(self, dataId):
367  return int(dataId["visit"]) * 10 + int(dataId["ccd"])
368 
369  def _computeCoaddId(self, dataId):
370  # Note: for real IDs, we'd want to include filter here, but it doesn't actually matter
371  # for any of the tests we've done so far, which all assume filter='r'
372  tract = int(dataId['tract'])
373  if tract < 0 or tract >= 128:
374  raise RuntimeError('tract not in range [0,128)')
375  patchX, patchY = (int(c) for c in dataId['patch'].split(','))
376  for p in (patchX, patchY):
377  if p < 0 or p >= 2**13:
378  raise RuntimeError('patch component not in range [0, 8192)')
379  return (tract * 2**13 + patchX) * 2**13 + patchY
380 
381  @staticmethod
382  def splitCcdExposureId(ccdExposureId):
383  return dict(visit=(int(ccdExposureId) // 10), ccd=(int(ccdExposureId) % 10))
384 
385  def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
386  return self._computeCcdExposureId(dataId)
387 
388  def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
389  return 32
390 
391  def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId):
392  return self._computeCoaddId(dataId)
393 
394  def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId):
395  return 1 + 7 + 13*2 + 3
396 
397  def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId):
398  return self._computeCoaddId(dataId)
399 
400  def bypass_deepMergedCoaddId_bits(self, datasetType, pythonType, location, dataId):
401  return 1 + 7 + 13*2 + 3
402 
403 
404 def makeSimpleCamera(
405  nX, nY,
406  sizeX, sizeY,
407  gapX, gapY,
408  pixelSize=1.0,
409  plateScale=20.0,
410  radialDistortion=0.925,
411 ):
412  """Create a camera
413 
414  @param[in] nx: number of detectors in x
415  @param[in] ny: number of detectors in y
416  @param[in] sizeX: detector size in x (pixels)
417  @param[in] sizeY: detector size in y (pixels)
418  @param[in] gapX: gap between detectors in x (mm)
419  @param[in] gapY: gap between detectors in y (mm)
420  @param[in] pixelSize: pixel size (mm) (a float)
421  @param[in] plateScale: plate scale in arcsec/mm; 20.0 is for LSST
422  @param[in] radialDistortion: radial distortion, in mm/rad^2
423  (the r^3 coefficient of the radial distortion polynomial
424  that converts FIELD_ANGLE in radians to FOCAL_PLANE in mm);
425  0.925 is the value Dave Monet measured for lsstSim data
426 
427  Each detector will have one amplifier (with no raw information).
428  """
429  pScaleRad = lsst.afw.geom.arcsecToRad(plateScale)
430  radialDistortCoeffs = [0.0, 1.0/pScaleRad, 0.0, radialDistortion/pScaleRad]
431  focalPlaneToFieldAngle = lsst.afw.geom.makeRadialTransform(radialDistortCoeffs)
432  nativeSys = lsst.afw.cameraGeom.FOCAL_PLANE
433  transforms = {
434  lsst.afw.cameraGeom.FIELD_ANGLE: focalPlaneToFieldAngle,
435  }
436  transformMap = lsst.afw.cameraGeom.TransformMap(nativeSys, transforms)
437 
438  detectorList = []
440  for iY in range(nY):
441  cY = (iY - 0.5 * (nY - 1)) * (pixelSize * sizeY + gapY)
442  for iX in range(nX):
443  cX = (iX - 0.5 * (nX - 1)) * (pixelSize * sizeY + gapX)
444  fpPos = lsst.afw.geom.Point2D(cX, cY)
445  detectorName = "detector %d,%d" % (iX, iY)
446  detectorId = len(detectorList) + 1
447  detectorList.append(DetectorWrapper(
448  name=detectorName,
449  id=detectorId,
450  serial=detectorName + " serial",
451  bbox=ccdBBox,
452  ampExtent=ccdBBox.getDimensions(),
453  numAmps=1,
454  pixelSize=lsst.afw.geom.Extent2D(pixelSize, pixelSize),
455  orientation=lsst.afw.cameraGeom.Orientation(fpPos),
456  plateScale=plateScale,
457  radialDistortion=radialDistortion,
458  ).detector)
459 
460  return lsst.afw.cameraGeom.Camera(
461  name="Simple Camera",
462  detectorList=detectorList,
463  transformMap=transformMap,
464  )
465 
466 
467 def makeDataRepo(root):
468  """
469  Create a data repository for SimpleMapper and return a butler for it.
470 
471  Clobbers anything already in the given path.
472  """
473  if os.path.exists(root):
474  shutil.rmtree(root)
475  os.makedirs(root)
476  with open(os.path.join(root, "_mapper"), "w") as f:
477  f.write("lsst.pipe.tasks.mocks.SimpleMapper\n")
478  return lsst.daf.persistence.Butler(root=root)
def map_camera(self, dataId, write=False)
def bypass_camera(self, datasetType, pythonType, location, dataId)
def __init__(self, persistence, template=None, keys=None)
def makeSimpleCamera(nX, nY, sizeX, sizeY, gapX, gapY, pixelSize=1.0, plateScale=20.0, radialDistortion=0.925)
def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId)
def map(self, dataset, root, dataId, mapper, suffix=None, storage=None)
def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId)
std::shared_ptr< TransformPoint2ToPoint2 > makeRadialTransform(std::vector< double > const &forwardCoeffs, std::vector< double > const &inverseCoeffs)
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:81
def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId)
def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId)
def bypass_deepMergedCoaddId_bits(self, datasetType, pythonType, location, dataId)
constexpr double arcsecToRad(double x) noexcept
def query(self, dataset, index, level, format, dataId)
def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId)
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:98
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:64