Coverage for python/lsst/pipe/tasks/mocks/simpleMapper.py: 86%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
23"""Mapper and cameraGeom definition for extremely simple mock data.
25SimpleMapper inherits directly from Mapper, not CameraMapper. This means
26we can avoid any problems with paf files at the expense of reimplementing
27some parts of CameraMapper here. Jim is not sure this was the best
28possible approach, but it gave him an opportunity to play around with
29prototyping a future paf-free mapper class, and it does everything it
30needs to do right now.
31"""
32import os
33import shutil
34import re
36import lsst.geom
37import lsst.daf.persistence
38import lsst.afw.cameraGeom
39import lsst.afw.geom
40import lsst.afw.image as afwImage
42__all__ = ("SimpleMapper", "makeSimpleCamera", "makeDataRepo")
45class PersistenceType:
46 """Base class of a hierarchy used by SimpleMapper to defined different kinds of types of objects
47 to persist.
49 PersistenceType objects are never instantiated; only the type objects are used (we needed a
50 simple singleton struct that could be inherited, which is exactly what a Python type is).
51 """
52 python = None
53 cpp = "ignored"
54 storage = None
55 ext = ""
56 suffixes = ()
58 @classmethod
59 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
60 """Method called by SimpleMapping to implement a map_ method."""
61 return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [path], dataId,
62 mapper=mapper,
63 storage=storage)
65 def canStandardize(self, datasetType):
66 return False
69class BypassPersistenceType(PersistenceType):
70 """Persistence type for things that don't actually use daf_persistence.
71 """
73 python = "lsst.daf.base.PropertySet" # something to import even when we don't need to
75 @classmethod
76 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
77 """Method called by SimpleMapping to implement a map_ method; overridden to not use the path."""
78 return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [], dataId,
79 mapper=mapper, storage=storage)
82class ExposurePersistenceType(PersistenceType):
83 """Persistence type of Exposure images.
84 """
86 python = "lsst.afw.image.ExposureF"
87 cpp = "ExposureF"
88 storage = "FitsStorage"
89 ext = ".fits"
90 suffixes = ("_sub",)
92 @classmethod
93 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
94 """Method called by SimpleMapping to implement a map_ method; overridden to support subimages."""
95 if suffix is None:
96 loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, dataId, mapper, suffix=None,
97 storage=storage)
98 # Write options are never applicable for _sub, since that's only
99 # for read. None of the values aside from the "NONE"s matter, but
100 # writing explicit meaningless values for all of them to appease
101 # afw is the price we pay for trying to write a non-CameraMapper
102 # Mapper. It'll all get better with Gen3 (TM).
103 options = {
104 "compression.algorithm": "NONE",
105 "compression.columns": 0,
106 "compression.rows": 0,
107 "compression.quantizeLevel": 0.0,
108 "scaling.algorithm": "NONE",
109 "scaling.bzero": 0.0,
110 "scaling.bscale": 0.0,
111 "scaling.bitpix": 0,
112 "scaling.quantizeLevel": 0.0,
113 "scaling.quantizePad": 0.0,
114 "scaling.fuzz": False,
115 "scaling.seed": 0,
116 }
117 for prefix in ("image", "mask", "variance"):
118 for k, v in options.items():
119 loc.additionalData.set("{}.{}".format(prefix, k), v)
120 elif suffix == "_sub": 120 ↛ 132line 120 didn't jump to line 132, because the condition on line 120 was never false
121 subId = dataId.copy()
122 bbox = subId.pop('bbox')
123 loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, subId, mapper, suffix=None,
124 storage=storage)
125 loc.additionalData.set('llcX', bbox.getMinX())
126 loc.additionalData.set('llcY', bbox.getMinY())
127 loc.additionalData.set('width', bbox.getWidth())
128 loc.additionalData.set('height', bbox.getHeight())
129 if 'imageOrigin' in dataId: 129 ↛ 130line 129 didn't jump to line 130, because the condition on line 129 was never true
130 loc.additionalData.set('imageOrigin',
131 dataId['imageOrigin'])
132 return loc
135class SkyMapPersistenceType(PersistenceType):
136 python = "lsst.skymap.BaseSkyMap"
137 storage = "PickleStorage"
138 ext = ".pickle"
141class CatalogPersistenceType(PersistenceType):
142 python = "lsst.afw.table.BaseCatalog"
143 cpp = "BaseCatalog"
144 storage = "FitsCatalogStorage"
145 ext = ".fits"
148class SimpleCatalogPersistenceType(CatalogPersistenceType):
149 python = "lsst.afw.table.SimpleCatalog"
150 cpp = "SimpleCatalog"
153class SourceCatalogPersistenceType(SimpleCatalogPersistenceType):
154 python = "lsst.afw.table.SourceCatalog"
155 cpp = "SourceCatalog"
158class ExposureCatalogPersistenceType(CatalogPersistenceType):
159 python = "lsst.afw.table.ExposureCatalog"
160 cpp = "ExposureCatalog"
163class PeakCatalogPersistenceType(CatalogPersistenceType):
164 python = "lsst.afw.detection.PeakCatalog"
165 cpp = "PeakCatalog"
168class SimpleMapping:
169 """Mapping object used to implement SimpleMapper, similar in intent to lsst.daf.peristence.Mapping.
170 """
172 template = None
173 keys = {}
175 def __init__(self, persistence, template=None, keys=None):
176 self.persistence = persistence
177 if template is not None:
178 self.template = template
179 if keys is not None:
180 self.keys = keys
182 def map(self, dataset, root, dataId, mapper, suffix=None, storage=None):
183 if self.template is not None:
184 path = self.template.format(dataset=dataset, ext=self.persistence.ext, **dataId)
185 else:
186 path = None
187 return self.persistence.makeButlerLocation(path, dataId, suffix=suffix, mapper=mapper,
188 storage=storage)
191class RawMapping(SimpleMapping):
192 """Mapping for dataset types that are organized the same way as raw data (i.e. by CCD)."""
194 template = "{dataset}-{visit:04d}-{ccd:01d}{ext}"
195 keys = dict(visit=int, ccd=int)
197 def query(self, dataset, index, level, format, dataId):
198 dictList = index[dataset][level]
199 results = [list(d.values()) for d in dictList[dataId.get(level, None)]]
200 return results
203class SkyMapping(SimpleMapping):
204 """Mapping for dataset types that are organized according to a SkyMap subdivision of the sky."""
206 template = "{dataset}-{filter}-{tract:02d}-{patch}{ext}"
207 keys = dict(filter=str, tract=int, patch=str)
210class TempExpMapping(SimpleMapping):
211 """Mapping for CoaddTempExp datasets."""
213 template = "{dataset}-{tract:02d}-{patch}-{visit:04d}{ext}"
214 keys = dict(tract=int, patch=str, visit=int)
217class ForcedSrcMapping(RawMapping):
218 """Mapping for forced_src datasets."""
220 template = "{dataset}-{tract:02d}-{visit:04d}-{ccd:01d}{ext}"
221 keys = dict(tract=int, ccd=int, visit=int)
224class MapperMeta(type):
225 """Metaclass for SimpleMapper that creates map_ and query_ methods for everything found in the
226 'mappings' class variable.
227 """
229 @staticmethod
230 def _makeMapClosure(dataset, mapping, suffix=None):
231 def mapClosure(self, dataId, write=False):
232 return mapping.map(dataset, self.root, dataId, self, suffix=suffix, storage=self.storage)
233 return mapClosure
235 @staticmethod
236 def _makeQueryClosure(dataset, mapping):
237 def queryClosure(self, level, format, dataId):
238 return mapping.query(dataset, self.index, level, format, dataId)
239 return queryClosure
241 def __init__(cls, name, bases, dict_): # noqa allow "cls" instead of "self"
242 type.__init__(cls, name, bases, dict_)
243 cls.keyDict = dict()
244 for dataset, mapping in cls.mappings.items():
245 setattr(cls, "map_" + dataset, MapperMeta._makeMapClosure(dataset, mapping, suffix=None))
246 for suffix in mapping.persistence.suffixes:
247 setattr(cls, "map_" + dataset + suffix,
248 MapperMeta._makeMapClosure(dataset, mapping, suffix=suffix))
249 if hasattr(mapping, "query"):
250 setattr(cls, "query_" + dataset, MapperMeta._makeQueryClosure(dataset, mapping))
251 cls.keyDict.update(mapping.keys)
254class SimpleMapper(lsst.daf.persistence.Mapper, metaclass=MapperMeta):
255 """
256 An extremely simple mapper for an imaginary camera for use in integration tests.
258 As SimpleMapper does not inherit from obs.base.CameraMapper, it does not
259 use a policy file to set mappings or a registry; all the information is here
260 (in the map_* and query_* methods).
262 The imaginary camera's raw data format has only 'visit' and 'ccd' keys, with
263 two CCDs per visit (by default).
264 """
266 mappings = dict(
267 calexp=RawMapping(ExposurePersistenceType),
268 forced_src=ForcedSrcMapping(SourceCatalogPersistenceType),
269 forced_src_schema=SimpleMapping(SourceCatalogPersistenceType,
270 template="{dataset}{ext}", keys={}),
271 truth=SimpleMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
272 keys={"tract": int}),
273 simsrc=RawMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
274 keys={"tract": int}),
275 observations=SimpleMapping(ExposureCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
276 keys={"tract": int}),
277 ccdExposureId=RawMapping(BypassPersistenceType),
278 ccdExposureId_bits=SimpleMapping(BypassPersistenceType),
279 deepCoaddId=SkyMapping(BypassPersistenceType),
280 deepCoaddId_bits=SimpleMapping(BypassPersistenceType),
281 deepCoadd_band=SimpleMapping(BypassPersistenceType),
282 deepMergedCoaddId=SkyMapping(BypassPersistenceType),
283 deepMergedCoaddId_bits=SimpleMapping(BypassPersistenceType),
284 deepCoadd_skyMap=SimpleMapping(SkyMapPersistenceType, template="{dataset}{ext}", keys={}),
285 deepCoadd=SkyMapping(ExposurePersistenceType),
286 deepCoadd_filterLabel=SkyMapping(ExposurePersistenceType),
287 deepCoaddPsfMatched=SkyMapping(ExposurePersistenceType),
288 deepCoadd_calexp=SkyMapping(ExposurePersistenceType),
289 deepCoadd_calexp_background=SkyMapping(CatalogPersistenceType),
290 deepCoadd_icSrc=SkyMapping(SourceCatalogPersistenceType),
291 deepCoadd_icSrc_schema=SimpleMapping(SourceCatalogPersistenceType,
292 template="{dataset}{ext}", keys={}),
293 deepCoadd_src=SkyMapping(SourceCatalogPersistenceType),
294 deepCoadd_src_schema=SimpleMapping(SourceCatalogPersistenceType,
295 template="{dataset}{ext}", keys={}),
296 deepCoadd_peak_schema=SimpleMapping(PeakCatalogPersistenceType,
297 template="{dataset}{ext}", keys={}),
298 deepCoadd_ref=SkyMapping(SourceCatalogPersistenceType),
299 deepCoadd_ref_schema=SimpleMapping(SourceCatalogPersistenceType,
300 template="{dataset}{ext}", keys={}),
301 deepCoadd_det=SkyMapping(SourceCatalogPersistenceType),
302 deepCoadd_det_schema=SimpleMapping(SourceCatalogPersistenceType,
303 template="{dataset}{ext}", keys={}),
304 deepCoadd_mergeDet=SkyMapping(SourceCatalogPersistenceType),
305 deepCoadd_mergeDet_schema=SimpleMapping(SourceCatalogPersistenceType,
306 template="{dataset}{ext}", keys={}),
307 deepCoadd_deblendedFlux=SkyMapping(SourceCatalogPersistenceType),
308 deepCoadd_deblendedFlux_schema=SimpleMapping(SourceCatalogPersistenceType,
309 template="{dataset}{ext}", keys={}),
310 deepCoadd_deblendedModel=SkyMapping(SourceCatalogPersistenceType),
311 deepCoadd_deblendedModel_schema=SimpleMapping(SourceCatalogPersistenceType,
312 template="{dataset}{ext}", keys={}),
313 deepCoadd_meas=SkyMapping(SourceCatalogPersistenceType),
314 deepCoadd_meas_schema=SimpleMapping(SourceCatalogPersistenceType,
315 template="{dataset}{ext}", keys={}),
316 deepCoadd_forced_src=SkyMapping(SourceCatalogPersistenceType),
317 deepCoadd_forced_src_schema=SimpleMapping(SourceCatalogPersistenceType,
318 template="{dataset}{ext}", keys={}),
319 deepCoadd_mock=SkyMapping(ExposurePersistenceType),
320 deepCoaddPsfMatched_mock=SkyMapping(ExposurePersistenceType),
321 deepCoadd_directWarp=TempExpMapping(ExposurePersistenceType),
322 deepCoadd_directWarp_mock=TempExpMapping(ExposurePersistenceType),
323 deepCoadd_psfMatchedWarp=TempExpMapping(ExposurePersistenceType),
324 deepCoadd_psfMatchedWarp_mock=TempExpMapping(ExposurePersistenceType),
325 )
327 levels = dict(
328 visit=['ccd'],
329 ccd=[],
330 )
332 def __init__(self, root, **kwargs):
333 self.storage = lsst.daf.persistence.Storage.makeFromURI(root)
334 super(SimpleMapper, self).__init__(**kwargs)
335 self.root = root
336 self.camera = makeSimpleCamera(nX=1, nY=2, sizeX=400, sizeY=200, gapX=2, gapY=2)
337 # NOTE: we set band/physical here because CoaddsTestCase.testCoaddInputs()
338 # expects "r" for both the input dataIds (gen2 dataIds could be either
339 # physical or band) and the output CoaddInputRecords.
340 # The original data had just `Filter("r")`, so this has always assumed
341 # that physicalLabel==bandLabel, even though CoaddInputRecords are physical.
342 self.filterLabel = afwImage.FilterLabel(band="r", physical="r")
343 self.update()
345 def getDefaultLevel(self):
346 return "ccd"
348 def getKeys(self, datasetType, level):
349 if datasetType is None: 349 ↛ 350line 349 didn't jump to line 350, because the condition on line 349 was never true
350 keyDict = self.keyDict
351 else:
352 keyDict = self.mappings[datasetType].keys
353 if level is not None and level in self.levels: 353 ↛ 354line 353 didn't jump to line 354, because the condition on line 353 was never true
354 keyDict = dict(keyDict)
355 for lev in self.levels[level]:
356 if lev in keyDict:
357 del keyDict[lev]
358 return keyDict
360 def update(self):
361 filenames = os.listdir(self.root)
362 rawRegex = re.compile(r"(?P<dataset>\w+)-(?P<visit>\d+)-(?P<ccd>\d).*")
363 self.index = {}
364 for filename in filenames:
365 m = rawRegex.match(filename)
366 if not m:
367 continue
368 index = self.index.setdefault(m.group('dataset'), dict(ccd={None: []}, visit={None: []}))
369 visit = int(m.group('visit'))
370 ccd = int(m.group('ccd'))
371 d1 = dict(visit=visit, ccd=ccd)
372 d2 = dict(visit=visit)
373 index['ccd'].setdefault(visit, []).append(d1)
374 index['ccd'][None].append(d1)
375 index['visit'][visit] = [d2]
376 index['visit'][None].append(d1)
378 def keys(self):
379 return self.keyDict
381 def bypass_camera(self, datasetType, pythonType, location, dataId):
382 return self.camera
384 def map_camera(self, dataId, write=False):
385 return lsst.daf.persistence.ButlerLocation(
386 "lsst.afw.cameraGeom.Camera", "Camera", None, [], dataId, mapper=self, storage=self.storage
387 )
389 def std_calexp(self, item, dataId):
390 detectorId = dataId["ccd"]
391 detector = self.camera[detectorId]
392 item.setDetector(detector)
393 item.setFilterLabel(self.filterLabel)
394 return item
396 def _computeCcdExposureId(self, dataId):
397 return int(dataId["visit"]) * 10 + int(dataId["ccd"])
399 def _computeCoaddId(self, dataId):
400 # Note: for real IDs, we'd want to include filter here, but it doesn't actually matter
401 # for any of the tests we've done so far, which all assume filter='r'
402 tract = int(dataId['tract'])
403 if tract < 0 or tract >= 128:
404 raise RuntimeError('tract not in range [0,128)')
405 patchX, patchY = (int(c) for c in dataId['patch'].split(','))
406 for p in (patchX, patchY):
407 if p < 0 or p >= 2**13:
408 raise RuntimeError('patch component not in range [0, 8192)')
409 return (tract * 2**13 + patchX) * 2**13 + patchY
411 @staticmethod
412 def splitCcdExposureId(ccdExposureId):
413 return dict(visit=(int(ccdExposureId) // 10), ccd=(int(ccdExposureId) % 10))
415 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
416 return self._computeCcdExposureId(dataId)
418 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
419 return 32
421 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId):
422 return self._computeCoaddId(dataId)
424 def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId):
425 return 1 + 7 + 13*2 + 3
427 def bypass_deepCoadd_band(self, datasetType, pythonType, location, dataId):
428 return self.filterLabel.bandLabel
430 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId):
431 return self._computeCoaddId(dataId)
433 def bypass_deepMergedCoaddId_bits(self, datasetType, pythonType, location, dataId):
434 return 1 + 7 + 13*2 + 3
436 def bypass_deepCoadd_filterLabel(self, *args, **kwargs):
437 """To return a useful filterLabel for MergeDetectionsTask."""
438 return afwImage.FilterLabel(band=self.filterLabel.bandLabel)
441def makeSimpleCamera(
442 nX, nY,
443 sizeX, sizeY,
444 gapX, gapY,
445 pixelSize=1.0,
446 plateScale=20.0,
447 radialDistortion=0.925,
448):
449 """Create a camera
451 @param[in] nx: number of detectors in x
452 @param[in] ny: number of detectors in y
453 @param[in] sizeX: detector size in x (pixels)
454 @param[in] sizeY: detector size in y (pixels)
455 @param[in] gapX: gap between detectors in x (mm)
456 @param[in] gapY: gap between detectors in y (mm)
457 @param[in] pixelSize: pixel size (mm) (a float)
458 @param[in] plateScale: plate scale in arcsec/mm; 20.0 is for LSST
459 @param[in] radialDistortion: radial distortion, in mm/rad^2
460 (the r^3 coefficient of the radial distortion polynomial
461 that converts FIELD_ANGLE in radians to FOCAL_PLANE in mm);
462 0.925 is the value Dave Monet measured for lsstSim data
464 Each detector will have one amplifier (with no raw information).
465 """
466 pScaleRad = lsst.geom.arcsecToRad(plateScale)
467 radialDistortCoeffs = [0.0, 1.0/pScaleRad, 0.0, radialDistortion/pScaleRad]
468 focalPlaneToFieldAngle = lsst.afw.geom.makeRadialTransform(radialDistortCoeffs)
470 ccdBBox = lsst.geom.Box2I(lsst.geom.Point2I(), lsst.geom.Extent2I(sizeX, sizeY))
472 cameraBuilder = lsst.afw.cameraGeom.Camera.Builder("Simple Camera")
474 detectorId = 0
475 for iY in range(nY):
476 cY = (iY - 0.5 * (nY - 1)) * (pixelSize * sizeY + gapY)
477 for iX in range(nX):
478 cX = (iX - 0.5 * (nX - 1)) * (pixelSize * sizeY + gapX)
479 fpPos = lsst.geom.Point2D(cX, cY)
480 detectorName = "detector %d,%d" % (iX, iY)
482 detectorBuilder = cameraBuilder.add(detectorName, detectorId)
483 detectorBuilder.setSerial(detectorName + " serial")
484 detectorBuilder.setBBox(ccdBBox)
485 detectorBuilder.setOrientation(lsst.afw.cameraGeom.Orientation(fpPos))
486 detectorBuilder.setPixelSize(lsst.geom.Extent2D(pixelSize, pixelSize))
488 ampBuilder = lsst.afw.cameraGeom.Amplifier.Builder()
489 ampName = "amp"
490 ampBuilder.setName(ampName)
491 ampBuilder.setBBox(ccdBBox)
492 ampBuilder.setGain(1.0)
493 ampBuilder.setReadNoise(5.0)
495 detectorBuilder.append(ampBuilder)
497 detectorId += 1
499 cameraBuilder.setTransformFromFocalPlaneTo(lsst.afw.cameraGeom.FIELD_ANGLE, focalPlaneToFieldAngle)
500 return cameraBuilder.finish()
503def makeDataRepo(root):
504 """
505 Create a data repository for SimpleMapper and return a butler for it.
507 Clobbers anything already in the given path.
508 """
509 if os.path.exists(root): 509 ↛ 510line 509 didn't jump to line 510, because the condition on line 509 was never true
510 shutil.rmtree(root)
511 os.makedirs(root)
512 with open(os.path.join(root, "_mapper"), "w") as f:
513 f.write("lsst.pipe.tasks.mocks.SimpleMapper\n")
514 return lsst.daf.persistence.Butler(root=root)