lsst.pipe.tasks  18.1.0-21-gb3d55290
simpleMapper.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 """Mapper and cameraGeom definition for extremely simple mock data.
24 
25 SimpleMapper inherits directly from Mapper, not CameraMapper. This means
26 we can avoid any problems with paf files at the expense of reimplementing
27 some parts of CameraMapper here. Jim is not sure this was the best
28 possible approach, but it gave him an opportunity to play around with
29 prototyping a future paf-free mapper class, and it does everything it
30 needs to do right now.
31 """
32 import os
33 import shutil
34 import re
35 
36 import lsst.geom
39 import lsst.afw.geom
40 from lsst.afw.cameraGeom.testUtils import DetectorWrapper
41 import lsst.afw.image.utils as afwImageUtils
42 import lsst.afw.image as afwImage
43 
44 __all__ = ("SimpleMapper", "makeSimpleCamera", "makeDataRepo")
45 
46 
48  """Base class of a hierarchy used by SimpleMapper to defined different kinds of types of objects
49  to persist.
50 
51  PersistenceType objects are never instantiated; only the type objects are used (we needed a
52  simple singleton struct that could be inherited, which is exactly what a Python type is).
53  """
54  python = None
55  cpp = "ignored"
56  storage = None
57  ext = ""
58  suffixes = ()
59 
60  @classmethod
61  def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
62  """Method called by SimpleMapping to implement a map_ method."""
63  return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [path], dataId,
64  mapper=mapper,
65  storage=storage)
66 
67  def canStandardize(self, datasetType):
68  return False
69 
70 
72  """Persistence type for things that don't actually use daf_persistence.
73  """
74 
75  python = "lsst.daf.base.PropertySet" # something to import even when we don't need to
76 
77  @classmethod
78  def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
79  """Method called by SimpleMapping to implement a map_ method; overridden to not use the path."""
80  return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [], dataId,
81  mapper=mapper, storage=storage)
82 
83 
85  """Persistence type of Exposure images.
86  """
87 
88  python = "lsst.afw.image.ExposureF"
89  cpp = "ExposureF"
90  storage = "FitsStorage"
91  ext = ".fits"
92  suffixes = ("_sub",)
93 
94  @classmethod
95  def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
96  """Method called by SimpleMapping to implement a map_ method; overridden to support subimages."""
97  if suffix is None:
98  loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, dataId, mapper, suffix=None,
99  storage=storage)
100  # Write options are never applicable for _sub, since that's only
101  # for read. None of the values aside from the "NONE"s matter, but
102  # writing explicit meaningless values for all of them to appease
103  # afw is the price we pay for trying to write a non-CameraMapper
104  # Mapper. It'll all get better with Gen3 (TM).
105  options = {
106  "compression.algorithm": "NONE",
107  "compression.columns": 0,
108  "compression.rows": 0,
109  "compression.quantizeLevel": 0.0,
110  "scaling.algorithm": "NONE",
111  "scaling.bzero": 0.0,
112  "scaling.bscale": 0.0,
113  "scaling.bitpix": 0,
114  "scaling.quantizeLevel": 0.0,
115  "scaling.quantizePad": 0.0,
116  "scaling.fuzz": False,
117  "scaling.seed": 0,
118  }
119  for prefix in ("image", "mask", "variance"):
120  for k, v in options.items():
121  loc.additionalData.set("{}.{}".format(prefix, k), v)
122  elif suffix == "_sub":
123  subId = dataId.copy()
124  bbox = subId.pop('bbox')
125  loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, subId, mapper, suffix=None,
126  storage=storage)
127  loc.additionalData.set('llcX', bbox.getMinX())
128  loc.additionalData.set('llcY', bbox.getMinY())
129  loc.additionalData.set('width', bbox.getWidth())
130  loc.additionalData.set('height', bbox.getHeight())
131  if 'imageOrigin' in dataId:
132  loc.additionalData.set('imageOrigin',
133  dataId['imageOrigin'])
134  return loc
135 
136 
138  python = "lsst.skymap.BaseSkyMap"
139  storage = "PickleStorage"
140  ext = ".pickle"
141 
142 
144  python = "lsst.afw.table.BaseCatalog"
145  cpp = "BaseCatalog"
146  storage = "FitsCatalogStorage"
147  ext = ".fits"
148 
149 
151  python = "lsst.afw.table.SimpleCatalog"
152  cpp = "SimpleCatalog"
153 
154 
156  python = "lsst.afw.table.SourceCatalog"
157  cpp = "SourceCatalog"
158 
159 
161  python = "lsst.afw.table.ExposureCatalog"
162  cpp = "ExposureCatalog"
163 
164 
166  python = "lsst.afw.detection.PeakCatalog"
167  cpp = "PeakCatalog"
168 
169 
171  """Mapping object used to implement SimpleMapper, similar in intent to lsst.daf.peristence.Mapping.
172  """
173 
174  template = None
175  keys = {}
176 
177  def __init__(self, persistence, template=None, keys=None):
178  self.persistence = persistence
179  if template is not None:
180  self.template = template
181  if keys is not None:
182  self.keys = keys
183 
184  def map(self, dataset, root, dataId, mapper, suffix=None, storage=None):
185  if self.template is not None:
186  path = self.template.format(dataset=dataset, ext=self.persistence.ext, **dataId)
187  else:
188  path = None
189  return self.persistence.makeButlerLocation(path, dataId, suffix=suffix, mapper=mapper,
190  storage=storage)
191 
192 
194  """Mapping for dataset types that are organized the same way as raw data (i.e. by CCD)."""
195 
196  template = "{dataset}-{visit:04d}-{ccd:01d}{ext}"
197  keys = dict(visit=int, ccd=int)
198 
199  def query(self, dataset, index, level, format, dataId):
200  dictList = index[dataset][level]
201  results = [list(d.values()) for d in dictList[dataId.get(level, None)]]
202  return results
203 
204 
206  """Mapping for dataset types that are organized according to a SkyMap subdivision of the sky."""
207 
208  template = "{dataset}-{filter}-{tract:02d}-{patch}{ext}"
209  keys = dict(filter=str, tract=int, patch=str)
210 
211 
213  """Mapping for CoaddTempExp datasets."""
214 
215  template = "{dataset}-{tract:02d}-{patch}-{visit:04d}{ext}"
216  keys = dict(tract=int, patch=str, visit=int)
217 
218 
220  """Mapping for forced_src datasets."""
221 
222  template = "{dataset}-{tract:02d}-{visit:04d}-{ccd:01d}{ext}"
223  keys = dict(tract=int, ccd=int, visit=int)
224 
225 
226 class MapperMeta(type):
227  """Metaclass for SimpleMapper that creates map_ and query_ methods for everything found in the
228  'mappings' class variable.
229  """
230 
231  @staticmethod
232  def _makeMapClosure(dataset, mapping, suffix=None):
233  def mapClosure(self, dataId, write=False):
234  return mapping.map(dataset, self.root, dataId, self, suffix=suffix, storage=self.storage)
235  return mapClosure
236 
237  @staticmethod
238  def _makeQueryClosure(dataset, mapping):
239  def queryClosure(self, level, format, dataId):
240  return mapping.query(dataset, self.index, level, format, dataId)
241  return queryClosure
242 
243  def __init__(cls, name, bases, dict_): # noqa allow "cls" instead of "self"
244  type.__init__(cls, name, bases, dict_)
245  cls.keyDict = dict()
246  for dataset, mapping in cls.mappings.items():
247  setattr(cls, "map_" + dataset, MapperMeta._makeMapClosure(dataset, mapping, suffix=None))
248  for suffix in mapping.persistence.suffixes:
249  setattr(cls, "map_" + dataset + suffix,
250  MapperMeta._makeMapClosure(dataset, mapping, suffix=suffix))
251  if hasattr(mapping, "query"):
252  setattr(cls, "query_" + dataset, MapperMeta._makeQueryClosure(dataset, mapping))
253  cls.keyDict.update(mapping.keys)
254 
255 
256 class SimpleMapper(lsst.daf.persistence.Mapper, metaclass=MapperMeta):
257  """
258  An extremely simple mapper for an imaginary camera for use in integration tests.
259 
260  As SimpleMapper does not inherit from obs.base.CameraMapper, it does not
261  use a policy file to set mappings or a registry; all the information is here
262  (in the map_* and query_* methods).
263 
264  The imaginary camera's raw data format has only 'visit' and 'ccd' keys, with
265  two CCDs per visit (by default).
266  """
267 
268  mappings = dict(
269  calexp=RawMapping(ExposurePersistenceType),
270  forced_src=ForcedSrcMapping(SourceCatalogPersistenceType),
271  forced_src_schema=SimpleMapping(SourceCatalogPersistenceType,
272  template="{dataset}{ext}", keys={}),
273  truth=SimpleMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
274  keys={"tract": int}),
275  simsrc=RawMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
276  keys={"tract": int}),
277  observations=SimpleMapping(ExposureCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
278  keys={"tract": int}),
279  ccdExposureId=RawMapping(BypassPersistenceType),
280  ccdExposureId_bits=SimpleMapping(BypassPersistenceType),
281  deepCoaddId=SkyMapping(BypassPersistenceType),
282  deepCoaddId_bits=SimpleMapping(BypassPersistenceType),
283  deepMergedCoaddId=SkyMapping(BypassPersistenceType),
284  deepMergedCoaddId_bits=SimpleMapping(BypassPersistenceType),
285  deepCoadd_skyMap=SimpleMapping(SkyMapPersistenceType, template="{dataset}{ext}", keys={}),
286  deepCoadd=SkyMapping(ExposurePersistenceType),
287  deepCoaddPsfMatched=SkyMapping(ExposurePersistenceType),
288  deepCoadd_calexp=SkyMapping(ExposurePersistenceType),
289  deepCoadd_calexp_background=SkyMapping(CatalogPersistenceType),
290  deepCoadd_icSrc=SkyMapping(SourceCatalogPersistenceType),
291  deepCoadd_icSrc_schema=SimpleMapping(SourceCatalogPersistenceType,
292  template="{dataset}{ext}", keys={}),
293  deepCoadd_src=SkyMapping(SourceCatalogPersistenceType),
294  deepCoadd_src_schema=SimpleMapping(SourceCatalogPersistenceType,
295  template="{dataset}{ext}", keys={}),
296  deepCoadd_peak_schema=SimpleMapping(PeakCatalogPersistenceType,
297  template="{dataset}{ext}", keys={}),
298  deepCoadd_ref=SkyMapping(SourceCatalogPersistenceType),
299  deepCoadd_ref_schema=SimpleMapping(SourceCatalogPersistenceType,
300  template="{dataset}{ext}", keys={}),
301  deepCoadd_det=SkyMapping(SourceCatalogPersistenceType),
302  deepCoadd_det_schema=SimpleMapping(SourceCatalogPersistenceType,
303  template="{dataset}{ext}", keys={}),
304  deepCoadd_mergeDet=SkyMapping(SourceCatalogPersistenceType),
305  deepCoadd_mergeDet_schema=SimpleMapping(SourceCatalogPersistenceType,
306  template="{dataset}{ext}", keys={}),
307  deepCoadd_deblendedFlux=SkyMapping(SourceCatalogPersistenceType),
308  deepCoadd_deblendedFlux_schema=SimpleMapping(SourceCatalogPersistenceType,
309  template="{dataset}{ext}", keys={}),
310  deepCoadd_deblendedModel=SkyMapping(SourceCatalogPersistenceType),
311  deepCoadd_deblendedModel_schema=SimpleMapping(SourceCatalogPersistenceType,
312  template="{dataset}{ext}", keys={}),
313  deepCoadd_meas=SkyMapping(SourceCatalogPersistenceType),
314  deepCoadd_meas_schema=SimpleMapping(SourceCatalogPersistenceType,
315  template="{dataset}{ext}", keys={}),
316  deepCoadd_forced_src=SkyMapping(SourceCatalogPersistenceType),
317  deepCoadd_forced_src_schema=SimpleMapping(SourceCatalogPersistenceType,
318  template="{dataset}{ext}", keys={}),
319  deepCoadd_mock=SkyMapping(ExposurePersistenceType),
320  deepCoaddPsfMatched_mock=SkyMapping(ExposurePersistenceType),
321  deepCoadd_directWarp=TempExpMapping(ExposurePersistenceType),
322  deepCoadd_directWarp_mock=TempExpMapping(ExposurePersistenceType),
323  deepCoadd_psfMatchedWarp=TempExpMapping(ExposurePersistenceType),
324  deepCoadd_psfMatchedWarp_mock=TempExpMapping(ExposurePersistenceType),
325  )
326 
327  levels = dict(
328  visit=['ccd'],
329  ccd=[],
330  )
331 
332  def __init__(self, root, **kwargs):
333  self.storage = lsst.daf.persistence.Storage.makeFromURI(root)
334  super(SimpleMapper, self).__init__(**kwargs)
335  self.root = root
336  self.camera = makeSimpleCamera(nX=1, nY=2, sizeX=400, sizeY=200, gapX=2, gapY=2)
337  afwImageUtils.defineFilter('r', 619.42)
338  self.update()
339 
340  def getDefaultLevel(self):
341  return "ccd"
342 
343  def getKeys(self, datasetType, level):
344  if datasetType is None:
345  keyDict = self.keyDict
346  else:
347  keyDict = self.mappings[datasetType].keys
348  if level is not None and level in self.levels:
349  keyDict = dict(keyDict)
350  for l in self.levels[level]:
351  if l in keyDict:
352  del keyDict[l]
353  return keyDict
354 
355  def update(self):
356  filenames = os.listdir(self.root)
357  rawRegex = re.compile(r"(?P<dataset>\w+)-(?P<visit>\d+)-(?P<ccd>\d).*")
358  self.index = {}
359  for filename in filenames:
360  m = rawRegex.match(filename)
361  if not m:
362  continue
363  index = self.index.setdefault(m.group('dataset'), dict(ccd={None: []}, visit={None: []}))
364  visit = int(m.group('visit'))
365  ccd = int(m.group('ccd'))
366  d1 = dict(visit=visit, ccd=ccd)
367  d2 = dict(visit=visit)
368  index['ccd'].setdefault(visit, []).append(d1)
369  index['ccd'][None].append(d1)
370  index['visit'][visit] = [d2]
371  index['visit'][None].append(d1)
372 
373  def keys(self):
374  return self.keyDict
375 
376  def bypass_camera(self, datasetType, pythonType, location, dataId):
377  return self.camera
378 
379  def map_camera(self, dataId, write=False):
380  return lsst.daf.persistence.ButlerLocation(
381  "lsst.afw.cameraGeom.Camera", "Camera", None, [], dataId, mapper=self, storage=self.storage
382  )
383 
384  def std_calexp(self, item, dataId):
385  detectorId = dataId["ccd"]
386  detector = self.camera[detectorId]
387  item.setDetector(detector)
388  item.setFilter(afwImage.Filter("r"))
389  return item
390 
391  def _computeCcdExposureId(self, dataId):
392  return int(dataId["visit"]) * 10 + int(dataId["ccd"])
393 
394  def _computeCoaddId(self, dataId):
395  # Note: for real IDs, we'd want to include filter here, but it doesn't actually matter
396  # for any of the tests we've done so far, which all assume filter='r'
397  tract = int(dataId['tract'])
398  if tract < 0 or tract >= 128:
399  raise RuntimeError('tract not in range [0,128)')
400  patchX, patchY = (int(c) for c in dataId['patch'].split(','))
401  for p in (patchX, patchY):
402  if p < 0 or p >= 2**13:
403  raise RuntimeError('patch component not in range [0, 8192)')
404  return (tract * 2**13 + patchX) * 2**13 + patchY
405 
406  @staticmethod
407  def splitCcdExposureId(ccdExposureId):
408  return dict(visit=(int(ccdExposureId) // 10), ccd=(int(ccdExposureId) % 10))
409 
410  def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
411  return self._computeCcdExposureId(dataId)
412 
413  def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
414  return 32
415 
416  def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId):
417  return self._computeCoaddId(dataId)
418 
419  def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId):
420  return 1 + 7 + 13*2 + 3
421 
422  def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId):
423  return self._computeCoaddId(dataId)
424 
425  def bypass_deepMergedCoaddId_bits(self, datasetType, pythonType, location, dataId):
426  return 1 + 7 + 13*2 + 3
427 
428 
429 def makeSimpleCamera(
430  nX, nY,
431  sizeX, sizeY,
432  gapX, gapY,
433  pixelSize=1.0,
434  plateScale=20.0,
435  radialDistortion=0.925,
436 ):
437  """Create a camera
438 
439  @param[in] nx: number of detectors in x
440  @param[in] ny: number of detectors in y
441  @param[in] sizeX: detector size in x (pixels)
442  @param[in] sizeY: detector size in y (pixels)
443  @param[in] gapX: gap between detectors in x (mm)
444  @param[in] gapY: gap between detectors in y (mm)
445  @param[in] pixelSize: pixel size (mm) (a float)
446  @param[in] plateScale: plate scale in arcsec/mm; 20.0 is for LSST
447  @param[in] radialDistortion: radial distortion, in mm/rad^2
448  (the r^3 coefficient of the radial distortion polynomial
449  that converts FIELD_ANGLE in radians to FOCAL_PLANE in mm);
450  0.925 is the value Dave Monet measured for lsstSim data
451 
452  Each detector will have one amplifier (with no raw information).
453  """
454  pScaleRad = lsst.geom.arcsecToRad(plateScale)
455  radialDistortCoeffs = [0.0, 1.0/pScaleRad, 0.0, radialDistortion/pScaleRad]
456  focalPlaneToFieldAngle = lsst.afw.geom.makeRadialTransform(radialDistortCoeffs)
457  nativeSys = lsst.afw.cameraGeom.FOCAL_PLANE
458  transforms = {
459  lsst.afw.cameraGeom.FIELD_ANGLE: focalPlaneToFieldAngle,
460  }
461  transformMap = lsst.afw.cameraGeom.TransformMap(nativeSys, transforms)
462 
463  detectorList = []
464  ccdBBox = lsst.geom.Box2I(lsst.geom.Point2I(), lsst.geom.Extent2I(sizeX, sizeY))
465  for iY in range(nY):
466  cY = (iY - 0.5 * (nY - 1)) * (pixelSize * sizeY + gapY)
467  for iX in range(nX):
468  cX = (iX - 0.5 * (nX - 1)) * (pixelSize * sizeY + gapX)
469  fpPos = lsst.geom.Point2D(cX, cY)
470  detectorName = "detector %d,%d" % (iX, iY)
471  detectorId = len(detectorList) + 1
472  detectorList.append(DetectorWrapper(
473  name=detectorName,
474  id=detectorId,
475  serial=detectorName + " serial",
476  bbox=ccdBBox,
477  ampExtent=ccdBBox.getDimensions(),
478  numAmps=1,
479  pixelSize=lsst.geom.Extent2D(pixelSize, pixelSize),
480  orientation=lsst.afw.cameraGeom.Orientation(fpPos),
481  plateScale=plateScale,
482  radialDistortion=radialDistortion,
483  ).detector)
484 
486  name="Simple Camera",
487  detectorList=detectorList,
488  transformMap=transformMap,
489  )
490 
491 
492 def makeDataRepo(root):
493  """
494  Create a data repository for SimpleMapper and return a butler for it.
495 
496  Clobbers anything already in the given path.
497  """
498  if os.path.exists(root):
499  shutil.rmtree(root)
500  os.makedirs(root)
501  with open(os.path.join(root, "_mapper"), "w") as f:
502  f.write("lsst.pipe.tasks.mocks.SimpleMapper\n")
503  return lsst.daf.persistence.Butler(root=root)
def map_camera(self, dataId, write=False)
def bypass_camera(self, datasetType, pythonType, location, dataId)
def __init__(self, persistence, template=None, keys=None)
def makeSimpleCamera(nX, nY, sizeX, sizeY, gapX, gapY, pixelSize=1.0, plateScale=20.0, radialDistortion=0.925)
def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId)
def map(self, dataset, root, dataId, mapper, suffix=None, storage=None)
def __init__(cls, name, bases, dict_)
def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId)
std::shared_ptr< TransformPoint2ToPoint2 > makeRadialTransform(std::vector< double > const &forwardCoeffs, std::vector< double > const &inverseCoeffs)
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:78
def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId)
def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId)
def bypass_deepMergedCoaddId_bits(self, datasetType, pythonType, location, dataId)
def query(self, dataset, index, level, format, dataId)
def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId)
constexpr double arcsecToRad(double x) noexcept
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:95
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:61