1# 

2# LSST Data Management System 

3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23"""Mapper and cameraGeom definition for extremely simple mock data. 

24 

25SimpleMapper inherits directly from Mapper, not CameraMapper. This means 

26we can avoid any problems with paf files at the expense of reimplementing 

27some parts of CameraMapper here. Jim is not sure this was the best 

28possible approach, but it gave him an opportunity to play around with 

29prototyping a future paf-free mapper class, and it does everything it 

30needs to do right now. 

31""" 

32import os 

33import shutil 

34import re 

35 

36import lsst.geom 

37import lsst.daf.persistence 

38import lsst.afw.cameraGeom 

39import lsst.afw.geom 

40import lsst.afw.image.utils as afwImageUtils 

41import lsst.afw.image as afwImage 

42 

43__all__ = ("SimpleMapper", "makeSimpleCamera", "makeDataRepo") 

44 

45 

46class PersistenceType: 

47 """Base class of a hierarchy used by SimpleMapper to defined different kinds of types of objects 

48 to persist. 

49 

50 PersistenceType objects are never instantiated; only the type objects are used (we needed a 

51 simple singleton struct that could be inherited, which is exactly what a Python type is). 

52 """ 

53 python = None 

54 cpp = "ignored" 

55 storage = None 

56 ext = "" 

57 suffixes = () 

58 

59 @classmethod 

60 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None): 

61 """Method called by SimpleMapping to implement a map_ method.""" 

62 return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [path], dataId, 

63 mapper=mapper, 

64 storage=storage) 

65 

66 def canStandardize(self, datasetType): 

67 return False 

68 

69 

70class BypassPersistenceType(PersistenceType): 

71 """Persistence type for things that don't actually use daf_persistence. 

72 """ 

73 

74 python = "lsst.daf.base.PropertySet" # something to import even when we don't need to 

75 

76 @classmethod 

77 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None): 

78 """Method called by SimpleMapping to implement a map_ method; overridden to not use the path.""" 

79 return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [], dataId, 

80 mapper=mapper, storage=storage) 

81 

82 

83class ExposurePersistenceType(PersistenceType): 

84 """Persistence type of Exposure images. 

85 """ 

86 

87 python = "lsst.afw.image.ExposureF" 

88 cpp = "ExposureF" 

89 storage = "FitsStorage" 

90 ext = ".fits" 

91 suffixes = ("_sub",) 

92 

93 @classmethod 

94 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None): 

95 """Method called by SimpleMapping to implement a map_ method; overridden to support subimages.""" 

96 if suffix is None: 

97 loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, dataId, mapper, suffix=None, 

98 storage=storage) 

99 # Write options are never applicable for _sub, since that's only 

100 # for read. None of the values aside from the "NONE"s matter, but 

101 # writing explicit meaningless values for all of them to appease 

102 # afw is the price we pay for trying to write a non-CameraMapper 

103 # Mapper. It'll all get better with Gen3 (TM). 

104 options = { 

105 "compression.algorithm": "NONE", 

106 "compression.columns": 0, 

107 "compression.rows": 0, 

108 "compression.quantizeLevel": 0.0, 

109 "scaling.algorithm": "NONE", 

110 "scaling.bzero": 0.0, 

111 "scaling.bscale": 0.0, 

112 "scaling.bitpix": 0, 

113 "scaling.quantizeLevel": 0.0, 

114 "scaling.quantizePad": 0.0, 

115 "scaling.fuzz": False, 

116 "scaling.seed": 0, 

117 } 

118 for prefix in ("image", "mask", "variance"): 

119 for k, v in options.items(): 

120 loc.additionalData.set("{}.{}".format(prefix, k), v) 

121 elif suffix == "_sub": 121 ↛ 133line 121 didn't jump to line 133, because the condition on line 121 was never false

122 subId = dataId.copy() 

123 bbox = subId.pop('bbox') 

124 loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, subId, mapper, suffix=None, 

125 storage=storage) 

126 loc.additionalData.set('llcX', bbox.getMinX()) 

127 loc.additionalData.set('llcY', bbox.getMinY()) 

128 loc.additionalData.set('width', bbox.getWidth()) 

129 loc.additionalData.set('height', bbox.getHeight()) 

130 if 'imageOrigin' in dataId: 130 ↛ 131line 130 didn't jump to line 131, because the condition on line 130 was never true

131 loc.additionalData.set('imageOrigin', 

132 dataId['imageOrigin']) 

133 return loc 

134 

135 

136class SkyMapPersistenceType(PersistenceType): 

137 python = "lsst.skymap.BaseSkyMap" 

138 storage = "PickleStorage" 

139 ext = ".pickle" 

140 

141 

142class CatalogPersistenceType(PersistenceType): 

143 python = "lsst.afw.table.BaseCatalog" 

144 cpp = "BaseCatalog" 

145 storage = "FitsCatalogStorage" 

146 ext = ".fits" 

147 

148 

149class SimpleCatalogPersistenceType(CatalogPersistenceType): 

150 python = "lsst.afw.table.SimpleCatalog" 

151 cpp = "SimpleCatalog" 

152 

153 

154class SourceCatalogPersistenceType(SimpleCatalogPersistenceType): 

155 python = "lsst.afw.table.SourceCatalog" 

156 cpp = "SourceCatalog" 

157 

158 

159class ExposureCatalogPersistenceType(CatalogPersistenceType): 

160 python = "lsst.afw.table.ExposureCatalog" 

161 cpp = "ExposureCatalog" 

162 

163 

164class PeakCatalogPersistenceType(CatalogPersistenceType): 

165 python = "lsst.afw.detection.PeakCatalog" 

166 cpp = "PeakCatalog" 

167 

168 

169class SimpleMapping: 

170 """Mapping object used to implement SimpleMapper, similar in intent to lsst.daf.peristence.Mapping. 

171 """ 

172 

173 template = None 

174 keys = {} 

175 

176 def __init__(self, persistence, template=None, keys=None): 

177 self.persistence = persistence 

178 if template is not None: 

179 self.template = template 

180 if keys is not None: 

181 self.keys = keys 

182 

183 def map(self, dataset, root, dataId, mapper, suffix=None, storage=None): 

184 if self.template is not None: 

185 path = self.template.format(dataset=dataset, ext=self.persistence.ext, **dataId) 

186 else: 

187 path = None 

188 return self.persistence.makeButlerLocation(path, dataId, suffix=suffix, mapper=mapper, 

189 storage=storage) 

190 

191 

192class RawMapping(SimpleMapping): 

193 """Mapping for dataset types that are organized the same way as raw data (i.e. by CCD).""" 

194 

195 template = "{dataset}-{visit:04d}-{ccd:01d}{ext}" 

196 keys = dict(visit=int, ccd=int) 

197 

198 def query(self, dataset, index, level, format, dataId): 

199 dictList = index[dataset][level] 

200 results = [list(d.values()) for d in dictList[dataId.get(level, None)]] 

201 return results 

202 

203 

204class SkyMapping(SimpleMapping): 

205 """Mapping for dataset types that are organized according to a SkyMap subdivision of the sky.""" 

206 

207 template = "{dataset}-{filter}-{tract:02d}-{patch}{ext}" 

208 keys = dict(filter=str, tract=int, patch=str) 

209 

210 

211class TempExpMapping(SimpleMapping): 

212 """Mapping for CoaddTempExp datasets.""" 

213 

214 template = "{dataset}-{tract:02d}-{patch}-{visit:04d}{ext}" 

215 keys = dict(tract=int, patch=str, visit=int) 

216 

217 

218class ForcedSrcMapping(RawMapping): 

219 """Mapping for forced_src datasets.""" 

220 

221 template = "{dataset}-{tract:02d}-{visit:04d}-{ccd:01d}{ext}" 

222 keys = dict(tract=int, ccd=int, visit=int) 

223 

224 

225class MapperMeta(type): 

226 """Metaclass for SimpleMapper that creates map_ and query_ methods for everything found in the 

227 'mappings' class variable. 

228 """ 

229 

230 @staticmethod 

231 def _makeMapClosure(dataset, mapping, suffix=None): 

232 def mapClosure(self, dataId, write=False): 

233 return mapping.map(dataset, self.root, dataId, self, suffix=suffix, storage=self.storage) 

234 return mapClosure 

235 

236 @staticmethod 

237 def _makeQueryClosure(dataset, mapping): 

238 def queryClosure(self, level, format, dataId): 

239 return mapping.query(dataset, self.index, level, format, dataId) 

240 return queryClosure 

241 

242 def __init__(cls, name, bases, dict_): # noqa allow "cls" instead of "self" 

243 type.__init__(cls, name, bases, dict_) 

244 cls.keyDict = dict() 

245 for dataset, mapping in cls.mappings.items(): 

246 setattr(cls, "map_" + dataset, MapperMeta._makeMapClosure(dataset, mapping, suffix=None)) 

247 for suffix in mapping.persistence.suffixes: 

248 setattr(cls, "map_" + dataset + suffix, 

249 MapperMeta._makeMapClosure(dataset, mapping, suffix=suffix)) 

250 if hasattr(mapping, "query"): 

251 setattr(cls, "query_" + dataset, MapperMeta._makeQueryClosure(dataset, mapping)) 

252 cls.keyDict.update(mapping.keys) 

253 

254 

255class SimpleMapper(lsst.daf.persistence.Mapper, metaclass=MapperMeta): 

256 """ 

257 An extremely simple mapper for an imaginary camera for use in integration tests. 

258 

259 As SimpleMapper does not inherit from obs.base.CameraMapper, it does not 

260 use a policy file to set mappings or a registry; all the information is here 

261 (in the map_* and query_* methods). 

262 

263 The imaginary camera's raw data format has only 'visit' and 'ccd' keys, with 

264 two CCDs per visit (by default). 

265 """ 

266 

267 mappings = dict( 

268 calexp=RawMapping(ExposurePersistenceType), 

269 forced_src=ForcedSrcMapping(SourceCatalogPersistenceType), 

270 forced_src_schema=SimpleMapping(SourceCatalogPersistenceType, 

271 template="{dataset}{ext}", keys={}), 

272 truth=SimpleMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}", 

273 keys={"tract": int}), 

274 simsrc=RawMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}", 

275 keys={"tract": int}), 

276 observations=SimpleMapping(ExposureCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}", 

277 keys={"tract": int}), 

278 ccdExposureId=RawMapping(BypassPersistenceType), 

279 ccdExposureId_bits=SimpleMapping(BypassPersistenceType), 

280 deepCoaddId=SkyMapping(BypassPersistenceType), 

281 deepCoaddId_bits=SimpleMapping(BypassPersistenceType), 

282 deepMergedCoaddId=SkyMapping(BypassPersistenceType), 

283 deepMergedCoaddId_bits=SimpleMapping(BypassPersistenceType), 

284 deepCoadd_skyMap=SimpleMapping(SkyMapPersistenceType, template="{dataset}{ext}", keys={}), 

285 deepCoadd=SkyMapping(ExposurePersistenceType), 

286 deepCoaddPsfMatched=SkyMapping(ExposurePersistenceType), 

287 deepCoadd_calexp=SkyMapping(ExposurePersistenceType), 

288 deepCoadd_calexp_background=SkyMapping(CatalogPersistenceType), 

289 deepCoadd_icSrc=SkyMapping(SourceCatalogPersistenceType), 

290 deepCoadd_icSrc_schema=SimpleMapping(SourceCatalogPersistenceType, 

291 template="{dataset}{ext}", keys={}), 

292 deepCoadd_src=SkyMapping(SourceCatalogPersistenceType), 

293 deepCoadd_src_schema=SimpleMapping(SourceCatalogPersistenceType, 

294 template="{dataset}{ext}", keys={}), 

295 deepCoadd_peak_schema=SimpleMapping(PeakCatalogPersistenceType, 

296 template="{dataset}{ext}", keys={}), 

297 deepCoadd_ref=SkyMapping(SourceCatalogPersistenceType), 

298 deepCoadd_ref_schema=SimpleMapping(SourceCatalogPersistenceType, 

299 template="{dataset}{ext}", keys={}), 

300 deepCoadd_det=SkyMapping(SourceCatalogPersistenceType), 

301 deepCoadd_det_schema=SimpleMapping(SourceCatalogPersistenceType, 

302 template="{dataset}{ext}", keys={}), 

303 deepCoadd_mergeDet=SkyMapping(SourceCatalogPersistenceType), 

304 deepCoadd_mergeDet_schema=SimpleMapping(SourceCatalogPersistenceType, 

305 template="{dataset}{ext}", keys={}), 

306 deepCoadd_deblendedFlux=SkyMapping(SourceCatalogPersistenceType), 

307 deepCoadd_deblendedFlux_schema=SimpleMapping(SourceCatalogPersistenceType, 

308 template="{dataset}{ext}", keys={}), 

309 deepCoadd_deblendedModel=SkyMapping(SourceCatalogPersistenceType), 

310 deepCoadd_deblendedModel_schema=SimpleMapping(SourceCatalogPersistenceType, 

311 template="{dataset}{ext}", keys={}), 

312 deepCoadd_meas=SkyMapping(SourceCatalogPersistenceType), 

313 deepCoadd_meas_schema=SimpleMapping(SourceCatalogPersistenceType, 

314 template="{dataset}{ext}", keys={}), 

315 deepCoadd_forced_src=SkyMapping(SourceCatalogPersistenceType), 

316 deepCoadd_forced_src_schema=SimpleMapping(SourceCatalogPersistenceType, 

317 template="{dataset}{ext}", keys={}), 

318 deepCoadd_mock=SkyMapping(ExposurePersistenceType), 

319 deepCoaddPsfMatched_mock=SkyMapping(ExposurePersistenceType), 

320 deepCoadd_directWarp=TempExpMapping(ExposurePersistenceType), 

321 deepCoadd_directWarp_mock=TempExpMapping(ExposurePersistenceType), 

322 deepCoadd_psfMatchedWarp=TempExpMapping(ExposurePersistenceType), 

323 deepCoadd_psfMatchedWarp_mock=TempExpMapping(ExposurePersistenceType), 

324 ) 

325 

326 levels = dict( 

327 visit=['ccd'], 

328 ccd=[], 

329 ) 

330 

331 def __init__(self, root, **kwargs): 

332 self.storage = lsst.daf.persistence.Storage.makeFromURI(root) 

333 super(SimpleMapper, self).__init__(**kwargs) 

334 self.root = root 

335 self.camera = makeSimpleCamera(nX=1, nY=2, sizeX=400, sizeY=200, gapX=2, gapY=2) 

336 afwImageUtils.defineFilter('r', 619.42) 

337 self.update() 

338 

339 def getDefaultLevel(self): 

340 return "ccd" 

341 

342 def getKeys(self, datasetType, level): 

343 if datasetType is None: 343 ↛ 344line 343 didn't jump to line 344, because the condition on line 343 was never true

344 keyDict = self.keyDict 

345 else: 

346 keyDict = self.mappings[datasetType].keys 

347 if level is not None and level in self.levels: 347 ↛ 348line 347 didn't jump to line 348, because the condition on line 347 was never true

348 keyDict = dict(keyDict) 

349 for lev in self.levels[level]: 

350 if lev in keyDict: 

351 del keyDict[lev] 

352 return keyDict 

353 

354 def update(self): 

355 filenames = os.listdir(self.root) 

356 rawRegex = re.compile(r"(?P<dataset>\w+)-(?P<visit>\d+)-(?P<ccd>\d).*") 

357 self.index = {} 

358 for filename in filenames: 

359 m = rawRegex.match(filename) 

360 if not m: 

361 continue 

362 index = self.index.setdefault(m.group('dataset'), dict(ccd={None: []}, visit={None: []})) 

363 visit = int(m.group('visit')) 

364 ccd = int(m.group('ccd')) 

365 d1 = dict(visit=visit, ccd=ccd) 

366 d2 = dict(visit=visit) 

367 index['ccd'].setdefault(visit, []).append(d1) 

368 index['ccd'][None].append(d1) 

369 index['visit'][visit] = [d2] 

370 index['visit'][None].append(d1) 

371 

372 def keys(self): 

373 return self.keyDict 

374 

375 def bypass_camera(self, datasetType, pythonType, location, dataId): 

376 return self.camera 

377 

378 def map_camera(self, dataId, write=False): 

379 return lsst.daf.persistence.ButlerLocation( 

380 "lsst.afw.cameraGeom.Camera", "Camera", None, [], dataId, mapper=self, storage=self.storage 

381 ) 

382 

383 def std_calexp(self, item, dataId): 

384 detectorId = dataId["ccd"] 

385 detector = self.camera[detectorId] 

386 item.setDetector(detector) 

387 item.setFilter(afwImage.Filter("r")) 

388 return item 

389 

390 def _computeCcdExposureId(self, dataId): 

391 return int(dataId["visit"]) * 10 + int(dataId["ccd"]) 

392 

393 def _computeCoaddId(self, dataId): 

394 # Note: for real IDs, we'd want to include filter here, but it doesn't actually matter 

395 # for any of the tests we've done so far, which all assume filter='r' 

396 tract = int(dataId['tract']) 

397 if tract < 0 or tract >= 128: 397 ↛ 398line 397 didn't jump to line 398, because the condition on line 397 was never true

398 raise RuntimeError('tract not in range [0,128)') 

399 patchX, patchY = (int(c) for c in dataId['patch'].split(',')) 

400 for p in (patchX, patchY): 

401 if p < 0 or p >= 2**13: 401 ↛ 402line 401 didn't jump to line 402, because the condition on line 401 was never true

402 raise RuntimeError('patch component not in range [0, 8192)') 

403 return (tract * 2**13 + patchX) * 2**13 + patchY 

404 

405 @staticmethod 

406 def splitCcdExposureId(ccdExposureId): 

407 return dict(visit=(int(ccdExposureId) // 10), ccd=(int(ccdExposureId) % 10)) 

408 

409 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId): 

410 return self._computeCcdExposureId(dataId) 

411 

412 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId): 

413 return 32 

414 

415 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId): 

416 return self._computeCoaddId(dataId) 

417 

418 def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId): 

419 return 1 + 7 + 13*2 + 3 

420 

421 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId): 

422 return self._computeCoaddId(dataId) 

423 

424 def bypass_deepMergedCoaddId_bits(self, datasetType, pythonType, location, dataId): 

425 return 1 + 7 + 13*2 + 3 

426 

427 

428def makeSimpleCamera( 

429 nX, nY, 

430 sizeX, sizeY, 

431 gapX, gapY, 

432 pixelSize=1.0, 

433 plateScale=20.0, 

434 radialDistortion=0.925, 

435): 

436 """Create a camera 

437 

438 @param[in] nx: number of detectors in x 

439 @param[in] ny: number of detectors in y 

440 @param[in] sizeX: detector size in x (pixels) 

441 @param[in] sizeY: detector size in y (pixels) 

442 @param[in] gapX: gap between detectors in x (mm) 

443 @param[in] gapY: gap between detectors in y (mm) 

444 @param[in] pixelSize: pixel size (mm) (a float) 

445 @param[in] plateScale: plate scale in arcsec/mm; 20.0 is for LSST 

446 @param[in] radialDistortion: radial distortion, in mm/rad^2 

447 (the r^3 coefficient of the radial distortion polynomial 

448 that converts FIELD_ANGLE in radians to FOCAL_PLANE in mm); 

449 0.925 is the value Dave Monet measured for lsstSim data 

450 

451 Each detector will have one amplifier (with no raw information). 

452 """ 

453 pScaleRad = lsst.geom.arcsecToRad(plateScale) 

454 radialDistortCoeffs = [0.0, 1.0/pScaleRad, 0.0, radialDistortion/pScaleRad] 

455 focalPlaneToFieldAngle = lsst.afw.geom.makeRadialTransform(radialDistortCoeffs) 

456 

457 ccdBBox = lsst.geom.Box2I(lsst.geom.Point2I(), lsst.geom.Extent2I(sizeX, sizeY)) 

458 

459 cameraBuilder = lsst.afw.cameraGeom.Camera.Builder("Simple Camera") 

460 

461 detectorId = 0 

462 for iY in range(nY): 

463 cY = (iY - 0.5 * (nY - 1)) * (pixelSize * sizeY + gapY) 

464 for iX in range(nX): 

465 cX = (iX - 0.5 * (nX - 1)) * (pixelSize * sizeY + gapX) 

466 fpPos = lsst.geom.Point2D(cX, cY) 

467 detectorName = "detector %d,%d" % (iX, iY) 

468 

469 detectorBuilder = cameraBuilder.add(detectorName, detectorId) 

470 detectorBuilder.setSerial(detectorName + " serial") 

471 detectorBuilder.setBBox(ccdBBox) 

472 detectorBuilder.setOrientation(lsst.afw.cameraGeom.Orientation(fpPos)) 

473 detectorBuilder.setPixelSize(lsst.geom.Extent2D(pixelSize, pixelSize)) 

474 

475 ampBuilder = lsst.afw.cameraGeom.Amplifier.Builder() 

476 ampName = "amp" 

477 ampBuilder.setName(ampName) 

478 ampBuilder.setBBox(ccdBBox) 

479 ampBuilder.setGain(1.0) 

480 ampBuilder.setReadNoise(5.0) 

481 

482 detectorBuilder.append(ampBuilder) 

483 

484 detectorId += 1 

485 

486 cameraBuilder.setTransformFromFocalPlaneTo(lsst.afw.cameraGeom.FIELD_ANGLE, focalPlaneToFieldAngle) 

487 return cameraBuilder.finish() 

488 

489 

490def makeDataRepo(root): 

491 """ 

492 Create a data repository for SimpleMapper and return a butler for it. 

493 

494 Clobbers anything already in the given path. 

495 """ 

496 if os.path.exists(root): 496 ↛ 497line 496 didn't jump to line 497, because the condition on line 496 was never true

497 shutil.rmtree(root) 

498 os.makedirs(root) 

499 with open(os.path.join(root, "_mapper"), "w") as f: 

500 f.write("lsst.pipe.tasks.mocks.SimpleMapper\n") 

501 return lsst.daf.persistence.Butler(root=root)