Coverage for python/lsst/pipe/tasks/mocks/simpleMapper.py : 44%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
23"""Mapper and cameraGeom definition for extremely simple mock data.
25SimpleMapper inherits directly from Mapper, not CameraMapper. This means
26we can avoid any problems with paf files at the expense of reimplementing
27some parts of CameraMapper here. Jim is not sure this was the best
28possible approach, but it gave him an opportunity to play around with
29prototyping a future paf-free mapper class, and it does everything it
30needs to do right now.
31"""
32import os
33import shutil
34import re
36import lsst.geom
37import lsst.daf.persistence
38import lsst.afw.cameraGeom
39import lsst.afw.geom
40import lsst.afw.image.utils as afwImageUtils
41import lsst.afw.image as afwImage
43__all__ = ("SimpleMapper", "makeSimpleCamera", "makeDataRepo")
46class PersistenceType:
47 """Base class of a hierarchy used by SimpleMapper to defined different kinds of types of objects
48 to persist.
50 PersistenceType objects are never instantiated; only the type objects are used (we needed a
51 simple singleton struct that could be inherited, which is exactly what a Python type is).
52 """
53 python = None
54 cpp = "ignored"
55 storage = None
56 ext = ""
57 suffixes = ()
59 @classmethod
60 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
61 """Method called by SimpleMapping to implement a map_ method."""
62 return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [path], dataId,
63 mapper=mapper,
64 storage=storage)
66 def canStandardize(self, datasetType):
67 return False
70class BypassPersistenceType(PersistenceType):
71 """Persistence type for things that don't actually use daf_persistence.
72 """
74 python = "lsst.daf.base.PropertySet" # something to import even when we don't need to
76 @classmethod
77 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
78 """Method called by SimpleMapping to implement a map_ method; overridden to not use the path."""
79 return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [], dataId,
80 mapper=mapper, storage=storage)
83class ExposurePersistenceType(PersistenceType):
84 """Persistence type of Exposure images.
85 """
87 python = "lsst.afw.image.ExposureF"
88 cpp = "ExposureF"
89 storage = "FitsStorage"
90 ext = ".fits"
91 suffixes = ("_sub",)
93 @classmethod
94 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
95 """Method called by SimpleMapping to implement a map_ method; overridden to support subimages."""
96 if suffix is None:
97 loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, dataId, mapper, suffix=None,
98 storage=storage)
99 # Write options are never applicable for _sub, since that's only
100 # for read. None of the values aside from the "NONE"s matter, but
101 # writing explicit meaningless values for all of them to appease
102 # afw is the price we pay for trying to write a non-CameraMapper
103 # Mapper. It'll all get better with Gen3 (TM).
104 options = {
105 "compression.algorithm": "NONE",
106 "compression.columns": 0,
107 "compression.rows": 0,
108 "compression.quantizeLevel": 0.0,
109 "scaling.algorithm": "NONE",
110 "scaling.bzero": 0.0,
111 "scaling.bscale": 0.0,
112 "scaling.bitpix": 0,
113 "scaling.quantizeLevel": 0.0,
114 "scaling.quantizePad": 0.0,
115 "scaling.fuzz": False,
116 "scaling.seed": 0,
117 }
118 for prefix in ("image", "mask", "variance"):
119 for k, v in options.items():
120 loc.additionalData.set("{}.{}".format(prefix, k), v)
121 elif suffix == "_sub":
122 subId = dataId.copy()
123 bbox = subId.pop('bbox')
124 loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, subId, mapper, suffix=None,
125 storage=storage)
126 loc.additionalData.set('llcX', bbox.getMinX())
127 loc.additionalData.set('llcY', bbox.getMinY())
128 loc.additionalData.set('width', bbox.getWidth())
129 loc.additionalData.set('height', bbox.getHeight())
130 if 'imageOrigin' in dataId:
131 loc.additionalData.set('imageOrigin',
132 dataId['imageOrigin'])
133 return loc
136class SkyMapPersistenceType(PersistenceType):
137 python = "lsst.skymap.BaseSkyMap"
138 storage = "PickleStorage"
139 ext = ".pickle"
142class CatalogPersistenceType(PersistenceType):
143 python = "lsst.afw.table.BaseCatalog"
144 cpp = "BaseCatalog"
145 storage = "FitsCatalogStorage"
146 ext = ".fits"
149class SimpleCatalogPersistenceType(CatalogPersistenceType):
150 python = "lsst.afw.table.SimpleCatalog"
151 cpp = "SimpleCatalog"
154class SourceCatalogPersistenceType(SimpleCatalogPersistenceType):
155 python = "lsst.afw.table.SourceCatalog"
156 cpp = "SourceCatalog"
159class ExposureCatalogPersistenceType(CatalogPersistenceType):
160 python = "lsst.afw.table.ExposureCatalog"
161 cpp = "ExposureCatalog"
164class PeakCatalogPersistenceType(CatalogPersistenceType):
165 python = "lsst.afw.detection.PeakCatalog"
166 cpp = "PeakCatalog"
169class SimpleMapping:
170 """Mapping object used to implement SimpleMapper, similar in intent to lsst.daf.peristence.Mapping.
171 """
173 template = None
174 keys = {}
176 def __init__(self, persistence, template=None, keys=None):
177 self.persistence = persistence
178 if template is not None:
179 self.template = template
180 if keys is not None:
181 self.keys = keys
183 def map(self, dataset, root, dataId, mapper, suffix=None, storage=None):
184 if self.template is not None:
185 path = self.template.format(dataset=dataset, ext=self.persistence.ext, **dataId)
186 else:
187 path = None
188 return self.persistence.makeButlerLocation(path, dataId, suffix=suffix, mapper=mapper,
189 storage=storage)
192class RawMapping(SimpleMapping):
193 """Mapping for dataset types that are organized the same way as raw data (i.e. by CCD)."""
195 template = "{dataset}-{visit:04d}-{ccd:01d}{ext}"
196 keys = dict(visit=int, ccd=int)
198 def query(self, dataset, index, level, format, dataId):
199 dictList = index[dataset][level]
200 results = [list(d.values()) for d in dictList[dataId.get(level, None)]]
201 return results
204class SkyMapping(SimpleMapping):
205 """Mapping for dataset types that are organized according to a SkyMap subdivision of the sky."""
207 template = "{dataset}-{filter}-{tract:02d}-{patch}{ext}"
208 keys = dict(filter=str, tract=int, patch=str)
211class TempExpMapping(SimpleMapping):
212 """Mapping for CoaddTempExp datasets."""
214 template = "{dataset}-{tract:02d}-{patch}-{visit:04d}{ext}"
215 keys = dict(tract=int, patch=str, visit=int)
218class ForcedSrcMapping(RawMapping):
219 """Mapping for forced_src datasets."""
221 template = "{dataset}-{tract:02d}-{visit:04d}-{ccd:01d}{ext}"
222 keys = dict(tract=int, ccd=int, visit=int)
225class MapperMeta(type):
226 """Metaclass for SimpleMapper that creates map_ and query_ methods for everything found in the
227 'mappings' class variable.
228 """
230 @staticmethod
231 def _makeMapClosure(dataset, mapping, suffix=None):
232 def mapClosure(self, dataId, write=False):
233 return mapping.map(dataset, self.root, dataId, self, suffix=suffix, storage=self.storage)
234 return mapClosure
236 @staticmethod
237 def _makeQueryClosure(dataset, mapping):
238 def queryClosure(self, level, format, dataId):
239 return mapping.query(dataset, self.index, level, format, dataId)
240 return queryClosure
242 def __init__(cls, name, bases, dict_): # noqa allow "cls" instead of "self"
243 type.__init__(cls, name, bases, dict_)
244 cls.keyDict = dict()
245 for dataset, mapping in cls.mappings.items():
246 setattr(cls, "map_" + dataset, MapperMeta._makeMapClosure(dataset, mapping, suffix=None))
247 for suffix in mapping.persistence.suffixes:
248 setattr(cls, "map_" + dataset + suffix,
249 MapperMeta._makeMapClosure(dataset, mapping, suffix=suffix))
250 if hasattr(mapping, "query"):
251 setattr(cls, "query_" + dataset, MapperMeta._makeQueryClosure(dataset, mapping))
252 cls.keyDict.update(mapping.keys)
255class SimpleMapper(lsst.daf.persistence.Mapper, metaclass=MapperMeta):
256 """
257 An extremely simple mapper for an imaginary camera for use in integration tests.
259 As SimpleMapper does not inherit from obs.base.CameraMapper, it does not
260 use a policy file to set mappings or a registry; all the information is here
261 (in the map_* and query_* methods).
263 The imaginary camera's raw data format has only 'visit' and 'ccd' keys, with
264 two CCDs per visit (by default).
265 """
267 mappings = dict(
268 calexp=RawMapping(ExposurePersistenceType),
269 forced_src=ForcedSrcMapping(SourceCatalogPersistenceType),
270 forced_src_schema=SimpleMapping(SourceCatalogPersistenceType,
271 template="{dataset}{ext}", keys={}),
272 truth=SimpleMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
273 keys={"tract": int}),
274 simsrc=RawMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
275 keys={"tract": int}),
276 observations=SimpleMapping(ExposureCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
277 keys={"tract": int}),
278 ccdExposureId=RawMapping(BypassPersistenceType),
279 ccdExposureId_bits=SimpleMapping(BypassPersistenceType),
280 deepCoaddId=SkyMapping(BypassPersistenceType),
281 deepCoaddId_bits=SimpleMapping(BypassPersistenceType),
282 deepMergedCoaddId=SkyMapping(BypassPersistenceType),
283 deepMergedCoaddId_bits=SimpleMapping(BypassPersistenceType),
284 deepCoadd_skyMap=SimpleMapping(SkyMapPersistenceType, template="{dataset}{ext}", keys={}),
285 deepCoadd=SkyMapping(ExposurePersistenceType),
286 deepCoaddPsfMatched=SkyMapping(ExposurePersistenceType),
287 deepCoadd_calexp=SkyMapping(ExposurePersistenceType),
288 deepCoadd_calexp_background=SkyMapping(CatalogPersistenceType),
289 deepCoadd_icSrc=SkyMapping(SourceCatalogPersistenceType),
290 deepCoadd_icSrc_schema=SimpleMapping(SourceCatalogPersistenceType,
291 template="{dataset}{ext}", keys={}),
292 deepCoadd_src=SkyMapping(SourceCatalogPersistenceType),
293 deepCoadd_src_schema=SimpleMapping(SourceCatalogPersistenceType,
294 template="{dataset}{ext}", keys={}),
295 deepCoadd_peak_schema=SimpleMapping(PeakCatalogPersistenceType,
296 template="{dataset}{ext}", keys={}),
297 deepCoadd_ref=SkyMapping(SourceCatalogPersistenceType),
298 deepCoadd_ref_schema=SimpleMapping(SourceCatalogPersistenceType,
299 template="{dataset}{ext}", keys={}),
300 deepCoadd_det=SkyMapping(SourceCatalogPersistenceType),
301 deepCoadd_det_schema=SimpleMapping(SourceCatalogPersistenceType,
302 template="{dataset}{ext}", keys={}),
303 deepCoadd_mergeDet=SkyMapping(SourceCatalogPersistenceType),
304 deepCoadd_mergeDet_schema=SimpleMapping(SourceCatalogPersistenceType,
305 template="{dataset}{ext}", keys={}),
306 deepCoadd_deblendedFlux=SkyMapping(SourceCatalogPersistenceType),
307 deepCoadd_deblendedFlux_schema=SimpleMapping(SourceCatalogPersistenceType,
308 template="{dataset}{ext}", keys={}),
309 deepCoadd_deblendedModel=SkyMapping(SourceCatalogPersistenceType),
310 deepCoadd_deblendedModel_schema=SimpleMapping(SourceCatalogPersistenceType,
311 template="{dataset}{ext}", keys={}),
312 deepCoadd_meas=SkyMapping(SourceCatalogPersistenceType),
313 deepCoadd_meas_schema=SimpleMapping(SourceCatalogPersistenceType,
314 template="{dataset}{ext}", keys={}),
315 deepCoadd_forced_src=SkyMapping(SourceCatalogPersistenceType),
316 deepCoadd_forced_src_schema=SimpleMapping(SourceCatalogPersistenceType,
317 template="{dataset}{ext}", keys={}),
318 deepCoadd_mock=SkyMapping(ExposurePersistenceType),
319 deepCoaddPsfMatched_mock=SkyMapping(ExposurePersistenceType),
320 deepCoadd_directWarp=TempExpMapping(ExposurePersistenceType),
321 deepCoadd_directWarp_mock=TempExpMapping(ExposurePersistenceType),
322 deepCoadd_psfMatchedWarp=TempExpMapping(ExposurePersistenceType),
323 deepCoadd_psfMatchedWarp_mock=TempExpMapping(ExposurePersistenceType),
324 )
326 levels = dict(
327 visit=['ccd'],
328 ccd=[],
329 )
331 def __init__(self, root, **kwargs):
332 self.storage = lsst.daf.persistence.Storage.makeFromURI(root)
333 super(SimpleMapper, self).__init__(**kwargs)
334 self.root = root
335 self.camera = makeSimpleCamera(nX=1, nY=2, sizeX=400, sizeY=200, gapX=2, gapY=2)
336 afwImageUtils.defineFilter('r', 619.42)
337 self.update()
339 def getDefaultLevel(self):
340 return "ccd"
342 def getKeys(self, datasetType, level):
343 if datasetType is None:
344 keyDict = self.keyDict
345 else:
346 keyDict = self.mappings[datasetType].keys
347 if level is not None and level in self.levels:
348 keyDict = dict(keyDict)
349 for l in self.levels[level]:
350 if l in keyDict:
351 del keyDict[l]
352 return keyDict
354 def update(self):
355 filenames = os.listdir(self.root)
356 rawRegex = re.compile(r"(?P<dataset>\w+)-(?P<visit>\d+)-(?P<ccd>\d).*")
357 self.index = {}
358 for filename in filenames:
359 m = rawRegex.match(filename)
360 if not m:
361 continue
362 index = self.index.setdefault(m.group('dataset'), dict(ccd={None: []}, visit={None: []}))
363 visit = int(m.group('visit'))
364 ccd = int(m.group('ccd'))
365 d1 = dict(visit=visit, ccd=ccd)
366 d2 = dict(visit=visit)
367 index['ccd'].setdefault(visit, []).append(d1)
368 index['ccd'][None].append(d1)
369 index['visit'][visit] = [d2]
370 index['visit'][None].append(d1)
372 def keys(self):
373 return self.keyDict
375 def bypass_camera(self, datasetType, pythonType, location, dataId):
376 return self.camera
378 def map_camera(self, dataId, write=False):
379 return lsst.daf.persistence.ButlerLocation(
380 "lsst.afw.cameraGeom.Camera", "Camera", None, [], dataId, mapper=self, storage=self.storage
381 )
383 def std_calexp(self, item, dataId):
384 detectorId = dataId["ccd"]
385 detector = self.camera[detectorId]
386 item.setDetector(detector)
387 item.setFilter(afwImage.Filter("r"))
388 return item
390 def _computeCcdExposureId(self, dataId):
391 return int(dataId["visit"]) * 10 + int(dataId["ccd"])
393 def _computeCoaddId(self, dataId):
394 # Note: for real IDs, we'd want to include filter here, but it doesn't actually matter
395 # for any of the tests we've done so far, which all assume filter='r'
396 tract = int(dataId['tract'])
397 if tract < 0 or tract >= 128:
398 raise RuntimeError('tract not in range [0,128)')
399 patchX, patchY = (int(c) for c in dataId['patch'].split(','))
400 for p in (patchX, patchY):
401 if p < 0 or p >= 2**13:
402 raise RuntimeError('patch component not in range [0, 8192)')
403 return (tract * 2**13 + patchX) * 2**13 + patchY
405 @staticmethod
406 def splitCcdExposureId(ccdExposureId):
407 return dict(visit=(int(ccdExposureId) // 10), ccd=(int(ccdExposureId) % 10))
409 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
410 return self._computeCcdExposureId(dataId)
412 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
413 return 32
415 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId):
416 return self._computeCoaddId(dataId)
418 def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId):
419 return 1 + 7 + 13*2 + 3
421 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId):
422 return self._computeCoaddId(dataId)
424 def bypass_deepMergedCoaddId_bits(self, datasetType, pythonType, location, dataId):
425 return 1 + 7 + 13*2 + 3
428def makeSimpleCamera(
429 nX, nY,
430 sizeX, sizeY,
431 gapX, gapY,
432 pixelSize=1.0,
433 plateScale=20.0,
434 radialDistortion=0.925,
435):
436 """Create a camera
438 @param[in] nx: number of detectors in x
439 @param[in] ny: number of detectors in y
440 @param[in] sizeX: detector size in x (pixels)
441 @param[in] sizeY: detector size in y (pixels)
442 @param[in] gapX: gap between detectors in x (mm)
443 @param[in] gapY: gap between detectors in y (mm)
444 @param[in] pixelSize: pixel size (mm) (a float)
445 @param[in] plateScale: plate scale in arcsec/mm; 20.0 is for LSST
446 @param[in] radialDistortion: radial distortion, in mm/rad^2
447 (the r^3 coefficient of the radial distortion polynomial
448 that converts FIELD_ANGLE in radians to FOCAL_PLANE in mm);
449 0.925 is the value Dave Monet measured for lsstSim data
451 Each detector will have one amplifier (with no raw information).
452 """
453 pScaleRad = lsst.geom.arcsecToRad(plateScale)
454 radialDistortCoeffs = [0.0, 1.0/pScaleRad, 0.0, radialDistortion/pScaleRad]
455 focalPlaneToFieldAngle = lsst.afw.geom.makeRadialTransform(radialDistortCoeffs)
457 ccdBBox = lsst.geom.Box2I(lsst.geom.Point2I(), lsst.geom.Extent2I(sizeX, sizeY))
459 cameraBuilder = lsst.afw.cameraGeom.Camera.Builder("Simple Camera")
461 detectorId = 0
462 for iY in range(nY):
463 cY = (iY - 0.5 * (nY - 1)) * (pixelSize * sizeY + gapY)
464 for iX in range(nX):
465 cX = (iX - 0.5 * (nX - 1)) * (pixelSize * sizeY + gapX)
466 fpPos = lsst.geom.Point2D(cX, cY)
467 detectorName = "detector %d,%d" % (iX, iY)
469 detectorBuilder = cameraBuilder.add(detectorName, detectorId)
470 detectorBuilder.setSerial(detectorName + " serial")
471 detectorBuilder.setBBox(ccdBBox)
472 detectorBuilder.setOrientation(lsst.afw.cameraGeom.Orientation(fpPos))
473 detectorBuilder.setPixelSize(lsst.geom.Extent2D(pixelSize, pixelSize))
475 ampBuilder = lsst.afw.cameraGeom.Amplifier.Builder()
476 ampName = "amp"
477 ampBuilder.setName(ampName)
478 ampBuilder.setBBox(ccdBBox)
479 ampBuilder.setGain(1.0)
480 ampBuilder.setReadNoise(5.0)
482 detectorBuilder.append(ampBuilder)
484 detectorId += 1
486 cameraBuilder.setTransformFromFocalPlaneTo(lsst.afw.cameraGeom.FIELD_ANGLE, focalPlaneToFieldAngle)
487 return cameraBuilder.finish()
490def makeDataRepo(root):
491 """
492 Create a data repository for SimpleMapper and return a butler for it.
494 Clobbers anything already in the given path.
495 """
496 if os.path.exists(root):
497 shutil.rmtree(root)
498 os.makedirs(root)
499 with open(os.path.join(root, "_mapper"), "w") as f:
500 f.write("lsst.pipe.tasks.mocks.SimpleMapper\n")
501 return lsst.daf.persistence.Butler(root=root)