lsst.pipe.tasks g0f82ab2f21+b4c33f426f
simpleMapper.py
Go to the documentation of this file.
2# LSST Data Management System
3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22
23"""Mapper and cameraGeom definition for extremely simple mock data.
24
25SimpleMapper inherits directly from Mapper, not CameraMapper. This means
26we can avoid any problems with paf files at the expense of reimplementing
27some parts of CameraMapper here. Jim is not sure this was the best
28possible approach, but it gave him an opportunity to play around with
29prototyping a future paf-free mapper class, and it does everything it
30needs to do right now.
31"""
32import os
33import shutil
34import re
35
36import lsst.geom
37import lsst.daf.persistence
38import lsst.afw.cameraGeom
39import lsst.afw.geom
40import lsst.afw.image as afwImage
41from lsst.daf.base import PropertySet
42
43__all__ = ("SimpleMapper", "makeSimpleCamera", "makeDataRepo")
44
45
46# None of the values aside from the "NONE"s matter, but
47# writing explicit meaningless values for all of them to appease
48# afw is the price we pay for trying to write a non-CameraMapper
49# Mapper. It'll all get better with Gen3 (TM).
50# Cache the values in a PropertySet since this code is called thousands
51# of times.
52_write_options = PropertySet.from_mapping({
53 "compression.algorithm": "NONE",
54 "compression.columns": 0,
55 "compression.rows": 0,
56 "compression.quantizeLevel": 0.0,
57 "scaling.algorithm": "NONE",
58 "scaling.bzero": 0.0,
59 "scaling.bscale": 0.0,
60 "scaling.bitpix": 0,
61 "scaling.quantizeLevel": 0.0,
62 "scaling.quantizePad": 0.0,
63 "scaling.fuzz": False,
64 "scaling.seed": 0,
65})
66
67
69 """Base class of a hierarchy used by SimpleMapper to defined different kinds of types of objects
70 to persist.
71
72 PersistenceType objects are never instantiated; only the type objects are used (we needed a
73 simple singleton struct that could be inherited, which is exactly what a Python type is).
74 """
75 python = None
76 cpp = "ignored"
77 storage = None
78 ext = ""
79 suffixes = ()
80
81 @classmethod
82 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
83 """Method called by SimpleMapping to implement a map_ method."""
84 return lsst.daf.persistence.ButlerLocation(cls.pythonpython, cls.cppcpp, cls.storagestorage, [path], dataId,
85 mapper=mapper,
86 storage=storage)
87
88 def canStandardize(self, datasetType):
89 return False
90
91
93 """Persistence type for things that don't actually use daf_persistence.
94 """
95
96 python = "lsst.daf.base.PropertySet" # something to import even when we don't need to
97
98 @classmethod
99 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
100 """Method called by SimpleMapping to implement a map_ method; overridden to not use the path."""
101 return lsst.daf.persistence.ButlerLocation(cls.pythonpythonpython, cls.cppcpp, cls.storagestorage, [], dataId,
102 mapper=mapper, storage=storage)
103
104
106 """Persistence type of Exposure images.
107 """
108
109 python = "lsst.afw.image.ExposureF"
110 cpp = "ExposureF"
111 storage = "FitsStorage"
112 ext = ".fits"
113 suffixes = ("_sub",)
114
115 @classmethod
116 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
117 """Method called by SimpleMapping to implement a map_ method; overridden to support subimages."""
118 if suffix is None:
119 loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, dataId, mapper, suffix=None,
120 storage=storage)
121 # Write options are never applicable for _sub, since that's only
122 # for read.
123 for prefix in ("image", "mask", "variance"):
124 loc.additionalData.setPropertySet(prefix, _write_options)
125 elif suffix == "_sub":
126 subId = dataId.copy()
127 bbox = subId.pop('bbox')
128 loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, subId, mapper, suffix=None,
129 storage=storage)
130 loc.additionalData.set('llcX', bbox.getMinX())
131 loc.additionalData.set('llcY', bbox.getMinY())
132 loc.additionalData.set('width', bbox.getWidth())
133 loc.additionalData.set('height', bbox.getHeight())
134 if 'imageOrigin' in dataId:
135 loc.additionalData.set('imageOrigin',
136 dataId['imageOrigin'])
137 return loc
138
139
141 python = "lsst.skymap.BaseSkyMap"
142 storage = "PickleStorage"
143 ext = ".pickle"
144
145
147 python = "lsst.afw.table.BaseCatalog"
148 cpp = "BaseCatalog"
149 storage = "FitsCatalogStorage"
150 ext = ".fits"
151
152
154 python = "lsst.afw.table.SimpleCatalog"
155 cpp = "SimpleCatalog"
156
157
159 python = "lsst.afw.table.SourceCatalog"
160 cpp = "SourceCatalog"
161
162
164 python = "lsst.afw.table.ExposureCatalog"
165 cpp = "ExposureCatalog"
166
167
169 python = "lsst.afw.detection.PeakCatalog"
170 cpp = "PeakCatalog"
171
172
174 """Mapping object used to implement SimpleMapper, similar in intent to lsst.daf.peristence.Mapping.
175 """
176
177 template = None
178 keys = {}
179
180 def __init__(self, persistence, template=None, keys=None):
181 self.persistencepersistence = persistence
182 if template is not None:
183 self.templatetemplate = template
184 if keys is not None:
185 self.keyskeyskeys = keys
186
187 def map(self, dataset, root, dataId, mapper, suffix=None, storage=None):
188 if self.templatetemplate is not None:
189 path = self.templatetemplate.format(dataset=dataset, ext=self.persistencepersistence.ext, **dataId)
190 else:
191 path = None
192 return self.persistencepersistence.makeButlerLocation(path, dataId, suffix=suffix, mapper=mapper,
193 storage=storage)
194
195
197 """Mapping for dataset types that are organized the same way as raw data (i.e. by CCD)."""
198
199 template = "{dataset}-{visit:04d}-{ccd:01d}{ext}"
200 keys = dict(visit=int, ccd=int)
201
202 def query(self, dataset, index, level, format, dataId):
203 dictList = index[dataset][level]
204 results = [list(d.values()) for d in dictList[dataId.get(level, None)]]
205 return results
206
207
209 """Mapping for dataset types that are organized according to a SkyMap subdivision of the sky."""
210
211 template = "{dataset}-{filter}-{tract:02d}-{patch}{ext}"
212 keys = dict(filter=str, tract=int, patch=str)
213
214
216 """Mapping for CoaddTempExp datasets."""
217
218 template = "{dataset}-{tract:02d}-{patch}-{visit:04d}{ext}"
219 keys = dict(tract=int, patch=str, visit=int)
220
221
223 """Mapping for forced_src datasets."""
224
225 template = "{dataset}-{tract:02d}-{visit:04d}-{ccd:01d}{ext}"
226 keys = dict(tract=int, ccd=int, visit=int)
227
228
229class MapperMeta(type):
230 """Metaclass for SimpleMapper that creates map_ and query_ methods for everything found in the
231 'mappings' class variable.
232 """
233
234 @staticmethod
235 def _makeMapClosure(dataset, mapping, suffix=None):
236 def mapClosure(self, dataId, write=False):
237 return mapping.map(dataset, self.root, dataId, self, suffix=suffix, storage=self.storage)
238 return mapClosure
239
240 @staticmethod
241 def _makeQueryClosure(dataset, mapping):
242 def queryClosure(self, level, format, dataId):
243 return mapping.query(dataset, self.index, level, format, dataId)
244 return queryClosure
245
246 def __init__(cls, name, bases, dict_): # noqa allow "cls" instead of "self"
247 type.__init__(cls, name, bases, dict_)
248 cls.keyDictkeyDict = dict()
249 for dataset, mapping in cls.mappings.items():
250 setattr(cls, "map_" + dataset, MapperMeta._makeMapClosure(dataset, mapping, suffix=None))
251 for suffix in mapping.persistence.suffixes:
252 setattr(cls, "map_" + dataset + suffix,
253 MapperMeta._makeMapClosure(dataset, mapping, suffix=suffix))
254 if hasattr(mapping, "query"):
255 setattr(cls, "query_" + dataset, MapperMeta._makeQueryClosure(dataset, mapping))
256 cls.keyDictkeyDict.update(mapping.keys)
257
258
259class SimpleMapper(lsst.daf.persistence.Mapper, metaclass=MapperMeta):
260 """
261 An extremely simple mapper for an imaginary camera for use in integration tests.
262
263 As SimpleMapper does not inherit from obs.base.CameraMapper, it does not
264 use a policy file to set mappings or a registry; all the information is here
265 (in the map_* and query_* methods).
266
267 The imaginary camera's raw data format has only 'visit' and 'ccd' keys, with
268 two CCDs per visit (by default).
269 """
270
271 mappings = dict(
272 calexp=RawMapping(ExposurePersistenceType),
273 forced_src=ForcedSrcMapping(SourceCatalogPersistenceType),
274 forced_src_schema=SimpleMapping(SourceCatalogPersistenceType,
275 template="{dataset}{ext}", keys={}),
276 truth=SimpleMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
277 keys={"tract": int}),
278 simsrc=RawMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
279 keys={"tract": int}),
280 observations=SimpleMapping(ExposureCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
281 keys={"tract": int}),
282 ccdExposureId=RawMapping(BypassPersistenceType),
283 ccdExposureId_bits=SimpleMapping(BypassPersistenceType),
284 deepCoaddId=SkyMapping(BypassPersistenceType),
285 deepCoaddId_bits=SimpleMapping(BypassPersistenceType),
286 deepCoadd_band=SimpleMapping(BypassPersistenceType),
287 deepMergedCoaddId=SkyMapping(BypassPersistenceType),
288 deepMergedCoaddId_bits=SimpleMapping(BypassPersistenceType),
289 deepCoadd_skyMap=SimpleMapping(SkyMapPersistenceType, template="{dataset}{ext}", keys={}),
290 deepCoadd=SkyMapping(ExposurePersistenceType),
291 deepCoadd_filterLabel=SkyMapping(ExposurePersistenceType),
292 deepCoaddPsfMatched=SkyMapping(ExposurePersistenceType),
293 deepCoadd_calexp=SkyMapping(ExposurePersistenceType),
294 deepCoadd_calexp_background=SkyMapping(CatalogPersistenceType),
295 deepCoadd_icSrc=SkyMapping(SourceCatalogPersistenceType),
296 deepCoadd_icSrc_schema=SimpleMapping(SourceCatalogPersistenceType,
297 template="{dataset}{ext}", keys={}),
298 deepCoadd_src=SkyMapping(SourceCatalogPersistenceType),
299 deepCoadd_src_schema=SimpleMapping(SourceCatalogPersistenceType,
300 template="{dataset}{ext}", keys={}),
301 deepCoadd_peak_schema=SimpleMapping(PeakCatalogPersistenceType,
302 template="{dataset}{ext}", keys={}),
303 deepCoadd_ref=SkyMapping(SourceCatalogPersistenceType),
304 deepCoadd_ref_schema=SimpleMapping(SourceCatalogPersistenceType,
305 template="{dataset}{ext}", keys={}),
306 deepCoadd_det=SkyMapping(SourceCatalogPersistenceType),
307 deepCoadd_det_schema=SimpleMapping(SourceCatalogPersistenceType,
308 template="{dataset}{ext}", keys={}),
309 deepCoadd_mergeDet=SkyMapping(SourceCatalogPersistenceType),
310 deepCoadd_mergeDet_schema=SimpleMapping(SourceCatalogPersistenceType,
311 template="{dataset}{ext}", keys={}),
312 deepCoadd_deblendedFlux=SkyMapping(SourceCatalogPersistenceType),
313 deepCoadd_deblendedFlux_schema=SimpleMapping(SourceCatalogPersistenceType,
314 template="{dataset}{ext}", keys={}),
315 deepCoadd_deblendedModel=SkyMapping(SourceCatalogPersistenceType),
316 deepCoadd_deblendedModel_schema=SimpleMapping(SourceCatalogPersistenceType,
317 template="{dataset}{ext}", keys={}),
318 deepCoadd_meas=SkyMapping(SourceCatalogPersistenceType),
319 deepCoadd_meas_schema=SimpleMapping(SourceCatalogPersistenceType,
320 template="{dataset}{ext}", keys={}),
321 deepCoadd_forced_src=SkyMapping(SourceCatalogPersistenceType),
322 deepCoadd_forced_src_schema=SimpleMapping(SourceCatalogPersistenceType,
323 template="{dataset}{ext}", keys={}),
324 deepCoadd_mock=SkyMapping(ExposurePersistenceType),
325 deepCoaddPsfMatched_mock=SkyMapping(ExposurePersistenceType),
326 deepCoadd_directWarp=TempExpMapping(ExposurePersistenceType),
327 deepCoadd_directWarp_mock=TempExpMapping(ExposurePersistenceType),
328 deepCoadd_psfMatchedWarp=TempExpMapping(ExposurePersistenceType),
329 deepCoadd_psfMatchedWarp_mock=TempExpMapping(ExposurePersistenceType),
330 )
331
332 levels = dict(
333 visit=['ccd'],
334 ccd=[],
335 )
336
337 def __init__(self, root, **kwargs):
338 self.storagestorage = lsst.daf.persistence.Storage.makeFromURI(root)
339 super(SimpleMapper, self).__init__(**kwargs)
340 self.rootroot = root
341 self.cameracamera = makeSimpleCamera(nX=1, nY=2, sizeX=400, sizeY=200, gapX=2, gapY=2)
342 # NOTE: we set band/physical here because CoaddsTestCase.testCoaddInputs()
343 # expects "r" for both the input dataIds (gen2 dataIds could be either
344 # physical or band) and the output CoaddInputRecords.
345 # The original data had just `Filter("r")`, so this has always assumed
346 # that physicalLabel==bandLabel, even though CoaddInputRecords are physical.
347 self.filterLabelfilterLabel = afwImage.FilterLabel(band="r", physical="r")
348 self.updateupdate()
349
351 return "ccd"
352
353 def getKeys(self, datasetType, level):
354 if datasetType is None:
355 keyDict = self.keyDictkeyDict
356 else:
357 keyDict = self.mappingsmappings[datasetType].keys
358 if level is not None and level in self.levelslevels:
359 keyDict = dict(keyDict)
360 for lev in self.levelslevels[level]:
361 if lev in keyDict:
362 del keyDict[lev]
363 return keyDict
364
365 def update(self):
366 filenames = os.listdir(self.rootroot)
367 rawRegex = re.compile(r"(?P<dataset>\w+)-(?P<visit>\d+)-(?P<ccd>\d).*")
368 self.indexindex = {}
369 for filename in filenames:
370 m = rawRegex.match(filename)
371 if not m:
372 continue
373 index = self.indexindex.setdefault(m.group('dataset'), dict(ccd={None: []}, visit={None: []}))
374 visit = int(m.group('visit'))
375 ccd = int(m.group('ccd'))
376 d1 = dict(visit=visit, ccd=ccd)
377 d2 = dict(visit=visit)
378 index['ccd'].setdefault(visit, []).append(d1)
379 index['ccd'][None].append(d1)
380 index['visit'][visit] = [d2]
381 index['visit'][None].append(d1)
382
383 def keys(self):
384 return self.keyDictkeyDict
385
386 def bypass_camera(self, datasetType, pythonType, location, dataId):
387 return self.cameracamera
388
389 def map_camera(self, dataId, write=False):
390 return lsst.daf.persistence.ButlerLocation(
391 "lsst.afw.cameraGeom.Camera", "Camera", None, [], dataId, mapper=self, storage=self.storagestorage
392 )
393
394 def std_calexp(self, item, dataId):
395 detectorId = dataId["ccd"]
396 detector = self.cameracamera[detectorId]
397 item.setDetector(detector)
398 item.setFilterLabel(self.filterLabelfilterLabel)
399 return item
400
401 def _computeCcdExposureId(self, dataId):
402 return int(dataId["visit"]) * 10 + int(dataId["ccd"])
403
404 def _computeCoaddId(self, dataId):
405 # Note: for real IDs, we'd want to include filter here, but it doesn't actually matter
406 # for any of the tests we've done so far, which all assume filter='r'
407 tract = int(dataId['tract'])
408 if tract < 0 or tract >= 128:
409 raise RuntimeError('tract not in range [0,128)')
410 patchX, patchY = (int(c) for c in dataId['patch'].split(','))
411 for p in (patchX, patchY):
412 if p < 0 or p >= 2**13:
413 raise RuntimeError('patch component not in range [0, 8192)')
414 return (tract * 2**13 + patchX) * 2**13 + patchY
415
416 @staticmethod
417 def splitCcdExposureId(ccdExposureId):
418 return dict(visit=(int(ccdExposureId) // 10), ccd=(int(ccdExposureId) % 10))
419
420 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
421 return self._computeCcdExposureId_computeCcdExposureId(dataId)
422
423 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
424 return 32
425
426 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId):
427 return self._computeCoaddId_computeCoaddId(dataId)
428
429 def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId):
430 return 1 + 7 + 13*2 + 3
431
432 def bypass_deepCoadd_band(self, datasetType, pythonType, location, dataId):
433 return self.filterLabelfilterLabel.bandLabel
434
435 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId):
436 return self._computeCoaddId_computeCoaddId(dataId)
437
438 def bypass_deepMergedCoaddId_bits(self, datasetType, pythonType, location, dataId):
439 return 1 + 7 + 13*2 + 3
440
441 def bypass_deepCoadd_filterLabel(self, *args, **kwargs):
442 """To return a useful filterLabel for MergeDetectionsTask."""
443 return afwImage.FilterLabel(band=self.filterLabelfilterLabel.bandLabel)
444
445
447 nX, nY,
448 sizeX, sizeY,
449 gapX, gapY,
450 pixelSize=1.0,
451 plateScale=20.0,
452 radialDistortion=0.925,
453):
454 """Create a camera
455
456 @param[in] nx: number of detectors in x
457 @param[in] ny: number of detectors in y
458 @param[in] sizeX: detector size in x (pixels)
459 @param[in] sizeY: detector size in y (pixels)
460 @param[in] gapX: gap between detectors in x (mm)
461 @param[in] gapY: gap between detectors in y (mm)
462 @param[in] pixelSize: pixel size (mm) (a float)
463 @param[in] plateScale: plate scale in arcsec/mm; 20.0 is for LSST
464 @param[in] radialDistortion: radial distortion, in mm/rad^2
465 (the r^3 coefficient of the radial distortion polynomial
466 that converts FIELD_ANGLE in radians to FOCAL_PLANE in mm);
467 0.925 is the value Dave Monet measured for lsstSim data
468
469 Each detector will have one amplifier (with no raw information).
470 """
471 pScaleRad = lsst.geom.arcsecToRad(plateScale)
472 radialDistortCoeffs = [0.0, 1.0/pScaleRad, 0.0, radialDistortion/pScaleRad]
473 focalPlaneToFieldAngle = lsst.afw.geom.makeRadialTransform(radialDistortCoeffs)
474
475 ccdBBox = lsst.geom.Box2I(lsst.geom.Point2I(), lsst.geom.Extent2I(sizeX, sizeY))
476
477 cameraBuilder = lsst.afw.cameraGeom.Camera.Builder("Simple Camera")
478
479 detectorId = 0
480 for iY in range(nY):
481 cY = (iY - 0.5 * (nY - 1)) * (pixelSize * sizeY + gapY)
482 for iX in range(nX):
483 cX = (iX - 0.5 * (nX - 1)) * (pixelSize * sizeY + gapX)
484 fpPos = lsst.geom.Point2D(cX, cY)
485 detectorName = "detector %d,%d" % (iX, iY)
486
487 detectorBuilder = cameraBuilder.add(detectorName, detectorId)
488 detectorBuilder.setSerial(detectorName + " serial")
489 detectorBuilder.setBBox(ccdBBox)
490 detectorBuilder.setOrientation(lsst.afw.cameraGeom.Orientation(fpPos))
491 detectorBuilder.setPixelSize(lsst.geom.Extent2D(pixelSize, pixelSize))
492
494 ampName = "amp"
495 ampBuilder.setName(ampName)
496 ampBuilder.setBBox(ccdBBox)
497 ampBuilder.setGain(1.0)
498 ampBuilder.setReadNoise(5.0)
499
500 detectorBuilder.append(ampBuilder)
501
502 detectorId += 1
503
504 cameraBuilder.setTransformFromFocalPlaneTo(lsst.afw.cameraGeom.FIELD_ANGLE, focalPlaneToFieldAngle)
505 return cameraBuilder.finish()
506
507
508def makeDataRepo(root):
509 """
510 Create a data repository for SimpleMapper and return a butler for it.
511
512 Clobbers anything already in the given path.
513 """
514 if os.path.exists(root):
515 shutil.rmtree(root)
516 os.makedirs(root)
517 with open(os.path.join(root, "_mapper"), "w") as f:
518 f.write("lsst.pipe.tasks.mocks.SimpleMapper\n")
519 return lsst.daf.persistence.Butler(root=root)
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:99
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
def __init__(cls, name, bases, dict_)
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:82
def query(self, dataset, index, level, format, dataId)
def bypass_deepMergedCoaddId_bits(self, datasetType, pythonType, location, dataId)
def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId)
def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId)
def bypass_deepCoadd_filterLabel(self, *args, **kwargs)
def bypass_deepCoadd_band(self, datasetType, pythonType, location, dataId)
def map_camera(self, dataId, write=False)
def bypass_camera(self, datasetType, pythonType, location, dataId)
def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId)
def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId)
def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId)
def map(self, dataset, root, dataId, mapper, suffix=None, storage=None)
def __init__(self, persistence, template=None, keys=None)
std::shared_ptr< TransformPoint2ToPoint2 > makeRadialTransform(std::vector< double > const &coeffs)
constexpr double arcsecToRad(double x) noexcept
def makeSimpleCamera(nX, nY, sizeX, sizeY, gapX, gapY, pixelSize=1.0, plateScale=20.0, radialDistortion=0.925)