lsst.pipe.tasks gb4d8b8e895+fcaa6a0601
simpleMapper.py
Go to the documentation of this file.
2# LSST Data Management System
3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22
23"""Mapper and cameraGeom definition for extremely simple mock data.
24
25SimpleMapper inherits directly from Mapper, not CameraMapper. This means
26we can avoid any problems with paf files at the expense of reimplementing
27some parts of CameraMapper here. Jim is not sure this was the best
28possible approach, but it gave him an opportunity to play around with
29prototyping a future paf-free mapper class, and it does everything it
30needs to do right now.
31"""
32import os
33import shutil
34import re
35
36import lsst.geom
37import lsst.daf.persistence
38import lsst.afw.cameraGeom
39import lsst.afw.geom
40import lsst.afw.image as afwImage
41
42__all__ = ("SimpleMapper", "makeSimpleCamera", "makeDataRepo")
43
44
45class PersistenceType:
46 """Base class of a hierarchy used by SimpleMapper to defined different kinds of types of objects
47 to persist.
48
49 PersistenceType objects are never instantiated; only the type objects are used (we needed a
50 simple singleton struct that could be inherited, which is exactly what a Python type is).
51 """
52 python = None
53 cpp = "ignored"
54 storage = None
55 ext = ""
56 suffixes = ()
57
58 @classmethod
59 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
60 """Method called by SimpleMapping to implement a map_ method."""
61 return lsst.daf.persistence.ButlerLocation(cls.pythonpython, cls.cppcpp, cls.storagestorage, [path], dataId,
62 mapper=mapper,
63 storage=storage)
64
65 def canStandardize(self, datasetType):
66 return False
67
68
70 """Persistence type for things that don't actually use daf_persistence.
71 """
72
73 python = "lsst.daf.base.PropertySet" # something to import even when we don't need to
74
75 @classmethod
76 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
77 """Method called by SimpleMapping to implement a map_ method; overridden to not use the path."""
78 return lsst.daf.persistence.ButlerLocation(cls.pythonpythonpython, cls.cppcpp, cls.storagestorage, [], dataId,
79 mapper=mapper, storage=storage)
80
81
83 """Persistence type of Exposure images.
84 """
85
86 python = "lsst.afw.image.ExposureF"
87 cpp = "ExposureF"
88 storage = "FitsStorage"
89 ext = ".fits"
90 suffixes = ("_sub",)
91
92 @classmethod
93 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
94 """Method called by SimpleMapping to implement a map_ method; overridden to support subimages."""
95 if suffix is None:
96 loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, dataId, mapper, suffix=None,
97 storage=storage)
98 # Write options are never applicable for _sub, since that's only
99 # for read. None of the values aside from the "NONE"s matter, but
100 # writing explicit meaningless values for all of them to appease
101 # afw is the price we pay for trying to write a non-CameraMapper
102 # Mapper. It'll all get better with Gen3 (TM).
103 options = {
104 "compression.algorithm": "NONE",
105 "compression.columns": 0,
106 "compression.rows": 0,
107 "compression.quantizeLevel": 0.0,
108 "scaling.algorithm": "NONE",
109 "scaling.bzero": 0.0,
110 "scaling.bscale": 0.0,
111 "scaling.bitpix": 0,
112 "scaling.quantizeLevel": 0.0,
113 "scaling.quantizePad": 0.0,
114 "scaling.fuzz": False,
115 "scaling.seed": 0,
116 }
117 for prefix in ("image", "mask", "variance"):
118 for k, v in options.items():
119 loc.additionalData.set("{}.{}".format(prefix, k), v)
120 elif suffix == "_sub":
121 subId = dataId.copy()
122 bbox = subId.pop('bbox')
123 loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, subId, mapper, suffix=None,
124 storage=storage)
125 loc.additionalData.set('llcX', bbox.getMinX())
126 loc.additionalData.set('llcY', bbox.getMinY())
127 loc.additionalData.set('width', bbox.getWidth())
128 loc.additionalData.set('height', bbox.getHeight())
129 if 'imageOrigin' in dataId:
130 loc.additionalData.set('imageOrigin',
131 dataId['imageOrigin'])
132 return loc
133
134
136 python = "lsst.skymap.BaseSkyMap"
137 storage = "PickleStorage"
138 ext = ".pickle"
139
140
142 python = "lsst.afw.table.BaseCatalog"
143 cpp = "BaseCatalog"
144 storage = "FitsCatalogStorage"
145 ext = ".fits"
146
147
149 python = "lsst.afw.table.SimpleCatalog"
150 cpp = "SimpleCatalog"
151
152
154 python = "lsst.afw.table.SourceCatalog"
155 cpp = "SourceCatalog"
156
157
159 python = "lsst.afw.table.ExposureCatalog"
160 cpp = "ExposureCatalog"
161
162
164 python = "lsst.afw.detection.PeakCatalog"
165 cpp = "PeakCatalog"
166
167
169 """Mapping object used to implement SimpleMapper, similar in intent to lsst.daf.peristence.Mapping.
170 """
171
172 template = None
173 keys = {}
174
175 def __init__(self, persistence, template=None, keys=None):
176 self.persistencepersistence = persistence
177 if template is not None:
178 self.templatetemplate = template
179 if keys is not None:
180 self.keyskeyskeys = keys
181
182 def map(self, dataset, root, dataId, mapper, suffix=None, storage=None):
183 if self.templatetemplate is not None:
184 path = self.templatetemplate.format(dataset=dataset, ext=self.persistencepersistence.ext, **dataId)
185 else:
186 path = None
187 return self.persistencepersistence.makeButlerLocation(path, dataId, suffix=suffix, mapper=mapper,
188 storage=storage)
189
190
192 """Mapping for dataset types that are organized the same way as raw data (i.e. by CCD)."""
193
194 template = "{dataset}-{visit:04d}-{ccd:01d}{ext}"
195 keys = dict(visit=int, ccd=int)
196
197 def query(self, dataset, index, level, format, dataId):
198 dictList = index[dataset][level]
199 results = [list(d.values()) for d in dictList[dataId.get(level, None)]]
200 return results
201
202
204 """Mapping for dataset types that are organized according to a SkyMap subdivision of the sky."""
205
206 template = "{dataset}-{filter}-{tract:02d}-{patch}{ext}"
207 keys = dict(filter=str, tract=int, patch=str)
208
209
211 """Mapping for CoaddTempExp datasets."""
212
213 template = "{dataset}-{tract:02d}-{patch}-{visit:04d}{ext}"
214 keys = dict(tract=int, patch=str, visit=int)
215
216
218 """Mapping for forced_src datasets."""
219
220 template = "{dataset}-{tract:02d}-{visit:04d}-{ccd:01d}{ext}"
221 keys = dict(tract=int, ccd=int, visit=int)
222
223
224class MapperMeta(type):
225 """Metaclass for SimpleMapper that creates map_ and query_ methods for everything found in the
226 'mappings' class variable.
227 """
228
229 @staticmethod
230 def _makeMapClosure(dataset, mapping, suffix=None):
231 def mapClosure(self, dataId, write=False):
232 return mapping.map(dataset, self.root, dataId, self, suffix=suffix, storage=self.storage)
233 return mapClosure
234
235 @staticmethod
236 def _makeQueryClosure(dataset, mapping):
237 def queryClosure(self, level, format, dataId):
238 return mapping.query(dataset, self.index, level, format, dataId)
239 return queryClosure
240
241 def __init__(cls, name, bases, dict_): # noqa allow "cls" instead of "self"
242 type.__init__(cls, name, bases, dict_)
243 cls.keyDictkeyDict = dict()
244 for dataset, mapping in cls.mappings.items():
245 setattr(cls, "map_" + dataset, MapperMeta._makeMapClosure(dataset, mapping, suffix=None))
246 for suffix in mapping.persistence.suffixes:
247 setattr(cls, "map_" + dataset + suffix,
248 MapperMeta._makeMapClosure(dataset, mapping, suffix=suffix))
249 if hasattr(mapping, "query"):
250 setattr(cls, "query_" + dataset, MapperMeta._makeQueryClosure(dataset, mapping))
251 cls.keyDictkeyDict.update(mapping.keys)
252
253
254class SimpleMapper(lsst.daf.persistence.Mapper, metaclass=MapperMeta):
255 """
256 An extremely simple mapper for an imaginary camera for use in integration tests.
257
258 As SimpleMapper does not inherit from obs.base.CameraMapper, it does not
259 use a policy file to set mappings or a registry; all the information is here
260 (in the map_* and query_* methods).
261
262 The imaginary camera's raw data format has only 'visit' and 'ccd' keys, with
263 two CCDs per visit (by default).
264 """
265
266 mappings = dict(
267 calexp=RawMapping(ExposurePersistenceType),
268 forced_src=ForcedSrcMapping(SourceCatalogPersistenceType),
269 forced_src_schema=SimpleMapping(SourceCatalogPersistenceType,
270 template="{dataset}{ext}", keys={}),
271 truth=SimpleMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
272 keys={"tract": int}),
273 simsrc=RawMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
274 keys={"tract": int}),
275 observations=SimpleMapping(ExposureCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
276 keys={"tract": int}),
277 ccdExposureId=RawMapping(BypassPersistenceType),
278 ccdExposureId_bits=SimpleMapping(BypassPersistenceType),
279 deepCoaddId=SkyMapping(BypassPersistenceType),
280 deepCoaddId_bits=SimpleMapping(BypassPersistenceType),
281 deepCoadd_band=SimpleMapping(BypassPersistenceType),
282 deepMergedCoaddId=SkyMapping(BypassPersistenceType),
283 deepMergedCoaddId_bits=SimpleMapping(BypassPersistenceType),
284 deepCoadd_skyMap=SimpleMapping(SkyMapPersistenceType, template="{dataset}{ext}", keys={}),
285 deepCoadd=SkyMapping(ExposurePersistenceType),
286 deepCoadd_filterLabel=SkyMapping(ExposurePersistenceType),
287 deepCoaddPsfMatched=SkyMapping(ExposurePersistenceType),
288 deepCoadd_calexp=SkyMapping(ExposurePersistenceType),
289 deepCoadd_calexp_background=SkyMapping(CatalogPersistenceType),
290 deepCoadd_icSrc=SkyMapping(SourceCatalogPersistenceType),
291 deepCoadd_icSrc_schema=SimpleMapping(SourceCatalogPersistenceType,
292 template="{dataset}{ext}", keys={}),
293 deepCoadd_src=SkyMapping(SourceCatalogPersistenceType),
294 deepCoadd_src_schema=SimpleMapping(SourceCatalogPersistenceType,
295 template="{dataset}{ext}", keys={}),
296 deepCoadd_peak_schema=SimpleMapping(PeakCatalogPersistenceType,
297 template="{dataset}{ext}", keys={}),
298 deepCoadd_ref=SkyMapping(SourceCatalogPersistenceType),
299 deepCoadd_ref_schema=SimpleMapping(SourceCatalogPersistenceType,
300 template="{dataset}{ext}", keys={}),
301 deepCoadd_det=SkyMapping(SourceCatalogPersistenceType),
302 deepCoadd_det_schema=SimpleMapping(SourceCatalogPersistenceType,
303 template="{dataset}{ext}", keys={}),
304 deepCoadd_mergeDet=SkyMapping(SourceCatalogPersistenceType),
305 deepCoadd_mergeDet_schema=SimpleMapping(SourceCatalogPersistenceType,
306 template="{dataset}{ext}", keys={}),
307 deepCoadd_deblendedFlux=SkyMapping(SourceCatalogPersistenceType),
308 deepCoadd_deblendedFlux_schema=SimpleMapping(SourceCatalogPersistenceType,
309 template="{dataset}{ext}", keys={}),
310 deepCoadd_deblendedModel=SkyMapping(SourceCatalogPersistenceType),
311 deepCoadd_deblendedModel_schema=SimpleMapping(SourceCatalogPersistenceType,
312 template="{dataset}{ext}", keys={}),
313 deepCoadd_meas=SkyMapping(SourceCatalogPersistenceType),
314 deepCoadd_meas_schema=SimpleMapping(SourceCatalogPersistenceType,
315 template="{dataset}{ext}", keys={}),
316 deepCoadd_forced_src=SkyMapping(SourceCatalogPersistenceType),
317 deepCoadd_forced_src_schema=SimpleMapping(SourceCatalogPersistenceType,
318 template="{dataset}{ext}", keys={}),
319 deepCoadd_mock=SkyMapping(ExposurePersistenceType),
320 deepCoaddPsfMatched_mock=SkyMapping(ExposurePersistenceType),
321 deepCoadd_directWarp=TempExpMapping(ExposurePersistenceType),
322 deepCoadd_directWarp_mock=TempExpMapping(ExposurePersistenceType),
323 deepCoadd_psfMatchedWarp=TempExpMapping(ExposurePersistenceType),
324 deepCoadd_psfMatchedWarp_mock=TempExpMapping(ExposurePersistenceType),
325 )
326
327 levels = dict(
328 visit=['ccd'],
329 ccd=[],
330 )
331
332 def __init__(self, root, **kwargs):
333 self.storagestorage = lsst.daf.persistence.Storage.makeFromURI(root)
334 super(SimpleMapper, self).__init__(**kwargs)
335 self.rootroot = root
336 self.cameracamera = makeSimpleCamera(nX=1, nY=2, sizeX=400, sizeY=200, gapX=2, gapY=2)
337 # NOTE: we set band/physical here because CoaddsTestCase.testCoaddInputs()
338 # expects "r" for both the input dataIds (gen2 dataIds could be either
339 # physical or band) and the output CoaddInputRecords.
340 # The original data had just `Filter("r")`, so this has always assumed
341 # that physicalLabel==bandLabel, even though CoaddInputRecords are physical.
342 self.filterLabelfilterLabel = afwImage.FilterLabel(band="r", physical="r")
343 self.updateupdate()
344
346 return "ccd"
347
348 def getKeys(self, datasetType, level):
349 if datasetType is None:
350 keyDict = self.keyDictkeyDict
351 else:
352 keyDict = self.mappingsmappings[datasetType].keys
353 if level is not None and level in self.levelslevels:
354 keyDict = dict(keyDict)
355 for lev in self.levelslevels[level]:
356 if lev in keyDict:
357 del keyDict[lev]
358 return keyDict
359
360 def update(self):
361 filenames = os.listdir(self.rootroot)
362 rawRegex = re.compile(r"(?P<dataset>\w+)-(?P<visit>\d+)-(?P<ccd>\d).*")
363 self.indexindex = {}
364 for filename in filenames:
365 m = rawRegex.match(filename)
366 if not m:
367 continue
368 index = self.indexindex.setdefault(m.group('dataset'), dict(ccd={None: []}, visit={None: []}))
369 visit = int(m.group('visit'))
370 ccd = int(m.group('ccd'))
371 d1 = dict(visit=visit, ccd=ccd)
372 d2 = dict(visit=visit)
373 index['ccd'].setdefault(visit, []).append(d1)
374 index['ccd'][None].append(d1)
375 index['visit'][visit] = [d2]
376 index['visit'][None].append(d1)
377
378 def keys(self):
379 return self.keyDictkeyDict
380
381 def bypass_camera(self, datasetType, pythonType, location, dataId):
382 return self.cameracamera
383
384 def map_camera(self, dataId, write=False):
385 return lsst.daf.persistence.ButlerLocation(
386 "lsst.afw.cameraGeom.Camera", "Camera", None, [], dataId, mapper=self, storage=self.storagestorage
387 )
388
389 def std_calexp(self, item, dataId):
390 detectorId = dataId["ccd"]
391 detector = self.cameracamera[detectorId]
392 item.setDetector(detector)
393 item.setFilterLabel(self.filterLabelfilterLabel)
394 return item
395
396 def _computeCcdExposureId(self, dataId):
397 return int(dataId["visit"]) * 10 + int(dataId["ccd"])
398
399 def _computeCoaddId(self, dataId):
400 # Note: for real IDs, we'd want to include filter here, but it doesn't actually matter
401 # for any of the tests we've done so far, which all assume filter='r'
402 tract = int(dataId['tract'])
403 if tract < 0 or tract >= 128:
404 raise RuntimeError('tract not in range [0,128)')
405 patchX, patchY = (int(c) for c in dataId['patch'].split(','))
406 for p in (patchX, patchY):
407 if p < 0 or p >= 2**13:
408 raise RuntimeError('patch component not in range [0, 8192)')
409 return (tract * 2**13 + patchX) * 2**13 + patchY
410
411 @staticmethod
412 def splitCcdExposureId(ccdExposureId):
413 return dict(visit=(int(ccdExposureId) // 10), ccd=(int(ccdExposureId) % 10))
414
415 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
416 return self._computeCcdExposureId_computeCcdExposureId(dataId)
417
418 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
419 return 32
420
421 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId):
422 return self._computeCoaddId_computeCoaddId(dataId)
423
424 def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId):
425 return 1 + 7 + 13*2 + 3
426
427 def bypass_deepCoadd_band(self, datasetType, pythonType, location, dataId):
428 return self.filterLabelfilterLabel.bandLabel
429
430 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId):
431 return self._computeCoaddId_computeCoaddId(dataId)
432
433 def bypass_deepMergedCoaddId_bits(self, datasetType, pythonType, location, dataId):
434 return 1 + 7 + 13*2 + 3
435
436 def bypass_deepCoadd_filterLabel(self, *args, **kwargs):
437 """To return a useful filterLabel for MergeDetectionsTask."""
438 return afwImage.FilterLabel(band=self.filterLabelfilterLabel.bandLabel)
439
440
442 nX, nY,
443 sizeX, sizeY,
444 gapX, gapY,
445 pixelSize=1.0,
446 plateScale=20.0,
447 radialDistortion=0.925,
448):
449 """Create a camera
450
451 @param[in] nx: number of detectors in x
452 @param[in] ny: number of detectors in y
453 @param[in] sizeX: detector size in x (pixels)
454 @param[in] sizeY: detector size in y (pixels)
455 @param[in] gapX: gap between detectors in x (mm)
456 @param[in] gapY: gap between detectors in y (mm)
457 @param[in] pixelSize: pixel size (mm) (a float)
458 @param[in] plateScale: plate scale in arcsec/mm; 20.0 is for LSST
459 @param[in] radialDistortion: radial distortion, in mm/rad^2
460 (the r^3 coefficient of the radial distortion polynomial
461 that converts FIELD_ANGLE in radians to FOCAL_PLANE in mm);
462 0.925 is the value Dave Monet measured for lsstSim data
463
464 Each detector will have one amplifier (with no raw information).
465 """
466 pScaleRad = lsst.geom.arcsecToRad(plateScale)
467 radialDistortCoeffs = [0.0, 1.0/pScaleRad, 0.0, radialDistortion/pScaleRad]
468 focalPlaneToFieldAngle = lsst.afw.geom.makeRadialTransform(radialDistortCoeffs)
469
470 ccdBBox = lsst.geom.Box2I(lsst.geom.Point2I(), lsst.geom.Extent2I(sizeX, sizeY))
471
472 cameraBuilder = lsst.afw.cameraGeom.Camera.Builder("Simple Camera")
473
474 detectorId = 0
475 for iY in range(nY):
476 cY = (iY - 0.5 * (nY - 1)) * (pixelSize * sizeY + gapY)
477 for iX in range(nX):
478 cX = (iX - 0.5 * (nX - 1)) * (pixelSize * sizeY + gapX)
479 fpPos = lsst.geom.Point2D(cX, cY)
480 detectorName = "detector %d,%d" % (iX, iY)
481
482 detectorBuilder = cameraBuilder.add(detectorName, detectorId)
483 detectorBuilder.setSerial(detectorName + " serial")
484 detectorBuilder.setBBox(ccdBBox)
485 detectorBuilder.setOrientation(lsst.afw.cameraGeom.Orientation(fpPos))
486 detectorBuilder.setPixelSize(lsst.geom.Extent2D(pixelSize, pixelSize))
487
489 ampName = "amp"
490 ampBuilder.setName(ampName)
491 ampBuilder.setBBox(ccdBBox)
492 ampBuilder.setGain(1.0)
493 ampBuilder.setReadNoise(5.0)
494
495 detectorBuilder.append(ampBuilder)
496
497 detectorId += 1
498
499 cameraBuilder.setTransformFromFocalPlaneTo(lsst.afw.cameraGeom.FIELD_ANGLE, focalPlaneToFieldAngle)
500 return cameraBuilder.finish()
501
502
503def makeDataRepo(root):
504 """
505 Create a data repository for SimpleMapper and return a butler for it.
506
507 Clobbers anything already in the given path.
508 """
509 if os.path.exists(root):
510 shutil.rmtree(root)
511 os.makedirs(root)
512 with open(os.path.join(root, "_mapper"), "w") as f:
513 f.write("lsst.pipe.tasks.mocks.SimpleMapper\n")
514 return lsst.daf.persistence.Butler(root=root)
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:76
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:93
def __init__(cls, name, bases, dict_)
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:59
def query(self, dataset, index, level, format, dataId)
def bypass_deepMergedCoaddId_bits(self, datasetType, pythonType, location, dataId)
def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId)
def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId)
def bypass_deepCoadd_filterLabel(self, *args, **kwargs)
def bypass_deepCoadd_band(self, datasetType, pythonType, location, dataId)
def map_camera(self, dataId, write=False)
def bypass_camera(self, datasetType, pythonType, location, dataId)
def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId)
def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId)
def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId)
def map(self, dataset, root, dataId, mapper, suffix=None, storage=None)
def __init__(self, persistence, template=None, keys=None)
std::shared_ptr< TransformPoint2ToPoint2 > makeRadialTransform(std::vector< double > const &coeffs)
constexpr double arcsecToRad(double x) noexcept
def makeSimpleCamera(nX, nY, sizeX, sizeY, gapX, gapY, pixelSize=1.0, plateScale=20.0, radialDistortion=0.925)