lsst.pipe.tasks v23.0.x-g1e964bd5bd+f2fbba1123
simpleMapper.py
Go to the documentation of this file.
2# LSST Data Management System
3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22
23"""Mapper and cameraGeom definition for extremely simple mock data.
24
25SimpleMapper inherits directly from Mapper, not CameraMapper. This means
26we can avoid any problems with paf files at the expense of reimplementing
27some parts of CameraMapper here. Jim is not sure this was the best
28possible approach, but it gave him an opportunity to play around with
29prototyping a future paf-free mapper class, and it does everything it
30needs to do right now.
31"""
32import os
33import shutil
34import re
35
36import lsst.geom
37import lsst.daf.persistence
38import lsst.afw.cameraGeom
39import lsst.afw.geom
40import lsst.afw.image as afwImage
41
42__all__ = ("SimpleMapper", "makeSimpleCamera", "makeDataRepo")
43
44
45class PersistenceType:
46 """Base class of a hierarchy used by SimpleMapper to defined different kinds of types of objects
47 to persist.
48
49 PersistenceType objects are never instantiated; only the type objects are used (we needed a
50 simple singleton struct that could be inherited, which is exactly what a Python type is).
51 """
52 python = None
53 cpp = "ignored"
54 storage = None
55 ext = ""
56 suffixes = ()
57
58 @classmethod
59 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
60 """Method called by SimpleMapping to implement a map_ method."""
61 return lsst.daf.persistence.ButlerLocation(cls.pythonpython, cls.cppcpp, cls.storagestorage, [path], dataId,
62 mapper=mapper,
63 storage=storage)
64
65 def canStandardize(self, datasetType):
66 return False
67
68
70 """Persistence type for things that don't actually use daf_persistence.
71 """
72
73 python = "lsst.daf.base.PropertySet" # something to import even when we don't need to
74
75 @classmethod
76 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
77 """Method called by SimpleMapping to implement a map_ method; overridden to not use the path."""
78 return lsst.daf.persistence.ButlerLocation(cls.pythonpythonpython, cls.cppcpp, cls.storagestorage, [], dataId,
79 mapper=mapper, storage=storage)
80
81
83 """Persistence type of Exposure images.
84 """
85
86 python = "lsst.afw.image.ExposureF"
87 cpp = "ExposureF"
88 storage = "FitsStorage"
89 ext = ".fits"
90 suffixes = ("_sub",)
91
92 @classmethod
93 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None):
94 """Method called by SimpleMapping to implement a map_ method; overridden to support subimages."""
95 if suffix is None:
96 loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, dataId, mapper, suffix=None,
97 storage=storage)
98 # Write options are never applicable for _sub, since that's only
99 # for read. None of the values aside from the "NONE"s matter, but
100 # writing explicit meaningless values for all of them to appease
101 # afw is the price we pay for trying to write a non-CameraMapper
102 # Mapper. It'll all get better with Gen3 (TM).
103 options = {
104 "compression.algorithm": "NONE",
105 "compression.columns": 0,
106 "compression.rows": 0,
107 "compression.quantizeLevel": 0.0,
108 "scaling.algorithm": "NONE",
109 "scaling.bzero": 0.0,
110 "scaling.bscale": 0.0,
111 "scaling.bitpix": 0,
112 "scaling.quantizeLevel": 0.0,
113 "scaling.quantizePad": 0.0,
114 "scaling.fuzz": False,
115 "scaling.seed": 0,
116 }
117 for prefix in ("image", "mask", "variance"):
118 for k, v in options.items():
119 loc.additionalData.set("{}.{}".format(prefix, k), v)
120 elif suffix == "_sub":
121 subId = dataId.copy()
122 bbox = subId.pop('bbox')
123 loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, subId, mapper, suffix=None,
124 storage=storage)
125 loc.additionalData.set('llcX', bbox.getMinX())
126 loc.additionalData.set('llcY', bbox.getMinY())
127 loc.additionalData.set('width', bbox.getWidth())
128 loc.additionalData.set('height', bbox.getHeight())
129 if 'imageOrigin' in dataId:
130 loc.additionalData.set('imageOrigin',
131 dataId['imageOrigin'])
132 return loc
133
134
136 python = "lsst.skymap.BaseSkyMap"
137 storage = "PickleStorage"
138 ext = ".pickle"
139
140
142 python = "lsst.afw.table.BaseCatalog"
143 cpp = "BaseCatalog"
144 storage = "FitsCatalogStorage"
145 ext = ".fits"
146
147
149 python = "lsst.afw.table.SimpleCatalog"
150 cpp = "SimpleCatalog"
151
152
154 python = "lsst.afw.table.SourceCatalog"
155 cpp = "SourceCatalog"
156
157
159 python = "lsst.afw.table.ExposureCatalog"
160 cpp = "ExposureCatalog"
161
162
164 python = "lsst.afw.detection.PeakCatalog"
165 cpp = "PeakCatalog"
166
167
169 """Mapping object used to implement SimpleMapper, similar in intent to lsst.daf.peristence.Mapping.
170 """
171
172 template = None
173 keys = {}
174
175 def __init__(self, persistence, template=None, keys=None):
176 self.persistencepersistence = persistence
177 if template is not None:
178 self.templatetemplate = template
179 if keys is not None:
180 self.keyskeyskeys = keys
181
182 def map(self, dataset, root, dataId, mapper, suffix=None, storage=None):
183 if self.templatetemplate is not None:
184 path = self.templatetemplate.format(dataset=dataset, ext=self.persistencepersistence.ext, **dataId)
185 else:
186 path = None
187 return self.persistencepersistence.makeButlerLocation(path, dataId, suffix=suffix, mapper=mapper,
188 storage=storage)
189
190
192 """Mapping for dataset types that are organized the same way as raw data (i.e. by CCD)."""
193
194 template = "{dataset}-{visit:04d}-{ccd:01d}{ext}"
195 keys = dict(visit=int, ccd=int)
196
197 def query(self, dataset, index, level, format, dataId):
198 dictList = index[dataset][level]
199 results = [list(d.values()) for d in dictList[dataId.get(level, None)]]
200 return results
201
202
204 """Mapping for dataset types that are organized according to a SkyMap subdivision of the sky."""
205
206 template = "{dataset}-{filter}-{tract:02d}-{patch}{ext}"
207 keys = dict(filter=str, tract=int, patch=str)
208
209
211 """Mapping for CoaddTempExp datasets."""
212
213 template = "{dataset}-{tract:02d}-{patch}-{visit:04d}{ext}"
214 keys = dict(tract=int, patch=str, visit=int)
215
216
218 """Mapping for forced_src datasets."""
219
220 template = "{dataset}-{tract:02d}-{visit:04d}-{ccd:01d}{ext}"
221 keys = dict(tract=int, ccd=int, visit=int)
222
223
224class MapperMeta(type):
225 """Metaclass for SimpleMapper that creates map_ and query_ methods for everything found in the
226 'mappings' class variable.
227 """
228
229 @staticmethod
230 def _makeMapClosure(dataset, mapping, suffix=None):
231 def mapClosure(self, dataId, write=False):
232 return mapping.map(dataset, self.root, dataId, self, suffix=suffix, storage=self.storage)
233 return mapClosure
234
235 @staticmethod
236 def _makeQueryClosure(dataset, mapping):
237 def queryClosure(self, level, format, dataId):
238 return mapping.query(dataset, self.index, level, format, dataId)
239 return queryClosure
240
241 def __init__(cls, name, bases, dict_): # noqa allow "cls" instead of "self"
242 type.__init__(cls, name, bases, dict_)
243 cls.keyDictkeyDict = dict()
244 for dataset, mapping in cls.mappings.items():
245 setattr(cls, "map_" + dataset, MapperMeta._makeMapClosure(dataset, mapping, suffix=None))
246 for suffix in mapping.persistence.suffixes:
247 setattr(cls, "map_" + dataset + suffix,
248 MapperMeta._makeMapClosure(dataset, mapping, suffix=suffix))
249 if hasattr(mapping, "query"):
250 setattr(cls, "query_" + dataset, MapperMeta._makeQueryClosure(dataset, mapping))
251 cls.keyDictkeyDict.update(mapping.keys)
252
253
254class SimpleMapper(lsst.daf.persistence.Mapper, metaclass=MapperMeta):
255 """
256 An extremely simple mapper for an imaginary camera for use in integration tests.
257
258 As SimpleMapper does not inherit from obs.base.CameraMapper, it does not
259 use a policy file to set mappings or a registry; all the information is here
260 (in the map_* and query_* methods).
261
262 The imaginary camera's raw data format has only 'visit' and 'ccd' keys, with
263 two CCDs per visit (by default).
264 """
265
266 mappings = dict(
267 calexp=RawMapping(ExposurePersistenceType),
268 forced_src=ForcedSrcMapping(SourceCatalogPersistenceType),
269 forced_src_schema=SimpleMapping(SourceCatalogPersistenceType,
270 template="{dataset}{ext}", keys={}),
271 truth=SimpleMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
272 keys={"tract": int}),
273 simsrc=RawMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
274 keys={"tract": int}),
275 observations=SimpleMapping(ExposureCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}",
276 keys={"tract": int}),
277 ccdExposureId=RawMapping(BypassPersistenceType),
278 ccdExposureId_bits=SimpleMapping(BypassPersistenceType),
279 deepCoaddId=SkyMapping(BypassPersistenceType),
280 deepCoaddId_bits=SimpleMapping(BypassPersistenceType),
281 deepMergedCoaddId=SkyMapping(BypassPersistenceType),
282 deepMergedCoaddId_bits=SimpleMapping(BypassPersistenceType),
283 deepCoadd_skyMap=SimpleMapping(SkyMapPersistenceType, template="{dataset}{ext}", keys={}),
284 deepCoadd=SkyMapping(ExposurePersistenceType),
285 deepCoadd_filterLabel=SkyMapping(ExposurePersistenceType),
286 deepCoaddPsfMatched=SkyMapping(ExposurePersistenceType),
287 deepCoadd_calexp=SkyMapping(ExposurePersistenceType),
288 deepCoadd_calexp_background=SkyMapping(CatalogPersistenceType),
289 deepCoadd_icSrc=SkyMapping(SourceCatalogPersistenceType),
290 deepCoadd_icSrc_schema=SimpleMapping(SourceCatalogPersistenceType,
291 template="{dataset}{ext}", keys={}),
292 deepCoadd_src=SkyMapping(SourceCatalogPersistenceType),
293 deepCoadd_src_schema=SimpleMapping(SourceCatalogPersistenceType,
294 template="{dataset}{ext}", keys={}),
295 deepCoadd_peak_schema=SimpleMapping(PeakCatalogPersistenceType,
296 template="{dataset}{ext}", keys={}),
297 deepCoadd_ref=SkyMapping(SourceCatalogPersistenceType),
298 deepCoadd_ref_schema=SimpleMapping(SourceCatalogPersistenceType,
299 template="{dataset}{ext}", keys={}),
300 deepCoadd_det=SkyMapping(SourceCatalogPersistenceType),
301 deepCoadd_det_schema=SimpleMapping(SourceCatalogPersistenceType,
302 template="{dataset}{ext}", keys={}),
303 deepCoadd_mergeDet=SkyMapping(SourceCatalogPersistenceType),
304 deepCoadd_mergeDet_schema=SimpleMapping(SourceCatalogPersistenceType,
305 template="{dataset}{ext}", keys={}),
306 deepCoadd_deblendedFlux=SkyMapping(SourceCatalogPersistenceType),
307 deepCoadd_deblendedFlux_schema=SimpleMapping(SourceCatalogPersistenceType,
308 template="{dataset}{ext}", keys={}),
309 deepCoadd_deblendedModel=SkyMapping(SourceCatalogPersistenceType),
310 deepCoadd_deblendedModel_schema=SimpleMapping(SourceCatalogPersistenceType,
311 template="{dataset}{ext}", keys={}),
312 deepCoadd_meas=SkyMapping(SourceCatalogPersistenceType),
313 deepCoadd_meas_schema=SimpleMapping(SourceCatalogPersistenceType,
314 template="{dataset}{ext}", keys={}),
315 deepCoadd_forced_src=SkyMapping(SourceCatalogPersistenceType),
316 deepCoadd_forced_src_schema=SimpleMapping(SourceCatalogPersistenceType,
317 template="{dataset}{ext}", keys={}),
318 deepCoadd_mock=SkyMapping(ExposurePersistenceType),
319 deepCoaddPsfMatched_mock=SkyMapping(ExposurePersistenceType),
320 deepCoadd_directWarp=TempExpMapping(ExposurePersistenceType),
321 deepCoadd_directWarp_mock=TempExpMapping(ExposurePersistenceType),
322 deepCoadd_psfMatchedWarp=TempExpMapping(ExposurePersistenceType),
323 deepCoadd_psfMatchedWarp_mock=TempExpMapping(ExposurePersistenceType),
324 )
325
326 levels = dict(
327 visit=['ccd'],
328 ccd=[],
329 )
330
331 def __init__(self, root, **kwargs):
332 self.storagestorage = lsst.daf.persistence.Storage.makeFromURI(root)
333 super(SimpleMapper, self).__init__(**kwargs)
334 self.rootroot = root
335 self.cameracamera = makeSimpleCamera(nX=1, nY=2, sizeX=400, sizeY=200, gapX=2, gapY=2)
336 # NOTE: we set band/physical here because CoaddsTestCase.testCoaddInputs()
337 # expects "r" for both the input dataIds (gen2 dataIds could be either
338 # physical or band) and the output CoaddInputRecords.
339 # The original data had just `Filter("r")`, so this has always assumed
340 # that physicalLabel==bandLabel, even though CoaddInputRecords are physical.
341 self.filterLabelfilterLabel = afwImage.FilterLabel(band="r", physical="r")
342 self.updateupdate()
343
345 return "ccd"
346
347 def getKeys(self, datasetType, level):
348 if datasetType is None:
349 keyDict = self.keyDictkeyDict
350 else:
351 keyDict = self.mappingsmappings[datasetType].keys
352 if level is not None and level in self.levelslevels:
353 keyDict = dict(keyDict)
354 for lev in self.levelslevels[level]:
355 if lev in keyDict:
356 del keyDict[lev]
357 return keyDict
358
359 def update(self):
360 filenames = os.listdir(self.rootroot)
361 rawRegex = re.compile(r"(?P<dataset>\w+)-(?P<visit>\d+)-(?P<ccd>\d).*")
362 self.indexindex = {}
363 for filename in filenames:
364 m = rawRegex.match(filename)
365 if not m:
366 continue
367 index = self.indexindex.setdefault(m.group('dataset'), dict(ccd={None: []}, visit={None: []}))
368 visit = int(m.group('visit'))
369 ccd = int(m.group('ccd'))
370 d1 = dict(visit=visit, ccd=ccd)
371 d2 = dict(visit=visit)
372 index['ccd'].setdefault(visit, []).append(d1)
373 index['ccd'][None].append(d1)
374 index['visit'][visit] = [d2]
375 index['visit'][None].append(d1)
376
377 def keys(self):
378 return self.keyDictkeyDict
379
380 def bypass_camera(self, datasetType, pythonType, location, dataId):
381 return self.cameracamera
382
383 def map_camera(self, dataId, write=False):
384 return lsst.daf.persistence.ButlerLocation(
385 "lsst.afw.cameraGeom.Camera", "Camera", None, [], dataId, mapper=self, storage=self.storagestorage
386 )
387
388 def std_calexp(self, item, dataId):
389 detectorId = dataId["ccd"]
390 detector = self.cameracamera[detectorId]
391 item.setDetector(detector)
392 item.setFilterLabel(self.filterLabelfilterLabel)
393 return item
394
395 def _computeCcdExposureId(self, dataId):
396 return int(dataId["visit"]) * 10 + int(dataId["ccd"])
397
398 def _computeCoaddId(self, dataId):
399 # Note: for real IDs, we'd want to include filter here, but it doesn't actually matter
400 # for any of the tests we've done so far, which all assume filter='r'
401 tract = int(dataId['tract'])
402 if tract < 0 or tract >= 128:
403 raise RuntimeError('tract not in range [0,128)')
404 patchX, patchY = (int(c) for c in dataId['patch'].split(','))
405 for p in (patchX, patchY):
406 if p < 0 or p >= 2**13:
407 raise RuntimeError('patch component not in range [0, 8192)')
408 return (tract * 2**13 + patchX) * 2**13 + patchY
409
410 @staticmethod
411 def splitCcdExposureId(ccdExposureId):
412 return dict(visit=(int(ccdExposureId) // 10), ccd=(int(ccdExposureId) % 10))
413
414 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
415 return self._computeCcdExposureId_computeCcdExposureId(dataId)
416
417 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
418 return 32
419
420 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId):
421 return self._computeCoaddId_computeCoaddId(dataId)
422
423 def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId):
424 return 1 + 7 + 13*2 + 3
425
426 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId):
427 return self._computeCoaddId_computeCoaddId(dataId)
428
429 def bypass_deepMergedCoaddId_bits(self, datasetType, pythonType, location, dataId):
430 return 1 + 7 + 13*2 + 3
431
432 def bypass_deepCoadd_filterLabel(self, *args, **kwargs):
433 """To return a useful filterLabel for MergeDetectionsTask."""
434 return afwImage.FilterLabel(band=self.filterLabelfilterLabel.bandLabel)
435
436
438 nX, nY,
439 sizeX, sizeY,
440 gapX, gapY,
441 pixelSize=1.0,
442 plateScale=20.0,
443 radialDistortion=0.925,
444):
445 """Create a camera
446
447 @param[in] nx: number of detectors in x
448 @param[in] ny: number of detectors in y
449 @param[in] sizeX: detector size in x (pixels)
450 @param[in] sizeY: detector size in y (pixels)
451 @param[in] gapX: gap between detectors in x (mm)
452 @param[in] gapY: gap between detectors in y (mm)
453 @param[in] pixelSize: pixel size (mm) (a float)
454 @param[in] plateScale: plate scale in arcsec/mm; 20.0 is for LSST
455 @param[in] radialDistortion: radial distortion, in mm/rad^2
456 (the r^3 coefficient of the radial distortion polynomial
457 that converts FIELD_ANGLE in radians to FOCAL_PLANE in mm);
458 0.925 is the value Dave Monet measured for lsstSim data
459
460 Each detector will have one amplifier (with no raw information).
461 """
462 pScaleRad = lsst.geom.arcsecToRad(plateScale)
463 radialDistortCoeffs = [0.0, 1.0/pScaleRad, 0.0, radialDistortion/pScaleRad]
464 focalPlaneToFieldAngle = lsst.afw.geom.makeRadialTransform(radialDistortCoeffs)
465
466 ccdBBox = lsst.geom.Box2I(lsst.geom.Point2I(), lsst.geom.Extent2I(sizeX, sizeY))
467
468 cameraBuilder = lsst.afw.cameraGeom.Camera.Builder("Simple Camera")
469
470 detectorId = 0
471 for iY in range(nY):
472 cY = (iY - 0.5 * (nY - 1)) * (pixelSize * sizeY + gapY)
473 for iX in range(nX):
474 cX = (iX - 0.5 * (nX - 1)) * (pixelSize * sizeY + gapX)
475 fpPos = lsst.geom.Point2D(cX, cY)
476 detectorName = "detector %d,%d" % (iX, iY)
477
478 detectorBuilder = cameraBuilder.add(detectorName, detectorId)
479 detectorBuilder.setSerial(detectorName + " serial")
480 detectorBuilder.setBBox(ccdBBox)
481 detectorBuilder.setOrientation(lsst.afw.cameraGeom.Orientation(fpPos))
482 detectorBuilder.setPixelSize(lsst.geom.Extent2D(pixelSize, pixelSize))
483
485 ampName = "amp"
486 ampBuilder.setName(ampName)
487 ampBuilder.setBBox(ccdBBox)
488 ampBuilder.setGain(1.0)
489 ampBuilder.setReadNoise(5.0)
490
491 detectorBuilder.append(ampBuilder)
492
493 detectorId += 1
494
495 cameraBuilder.setTransformFromFocalPlaneTo(lsst.afw.cameraGeom.FIELD_ANGLE, focalPlaneToFieldAngle)
496 return cameraBuilder.finish()
497
498
499def makeDataRepo(root):
500 """
501 Create a data repository for SimpleMapper and return a butler for it.
502
503 Clobbers anything already in the given path.
504 """
505 if os.path.exists(root):
506 shutil.rmtree(root)
507 os.makedirs(root)
508 with open(os.path.join(root, "_mapper"), "w") as f:
509 f.write("lsst.pipe.tasks.mocks.SimpleMapper\n")
510 return lsst.daf.persistence.Butler(root=root)
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:76
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:93
def __init__(cls, name, bases, dict_)
def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None)
Definition: simpleMapper.py:59
def query(self, dataset, index, level, format, dataId)
def bypass_deepMergedCoaddId_bits(self, datasetType, pythonType, location, dataId)
def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId)
def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId)
def bypass_deepCoadd_filterLabel(self, *args, **kwargs)
def map_camera(self, dataId, write=False)
def bypass_camera(self, datasetType, pythonType, location, dataId)
def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId)
def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId)
def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId)
def map(self, dataset, root, dataId, mapper, suffix=None, storage=None)
def __init__(self, persistence, template=None, keys=None)
std::shared_ptr< TransformPoint2ToPoint2 > makeRadialTransform(std::vector< double > const &coeffs)
constexpr double arcsecToRad(double x) noexcept
def makeSimpleCamera(nX, nY, sizeX, sizeY, gapX, gapY, pixelSize=1.0, plateScale=20.0, radialDistortion=0.925)