1# 

2# LSST Data Management System 

3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23"""Mapper and cameraGeom definition for extremely simple mock data. 

24 

25SimpleMapper inherits directly from Mapper, not CameraMapper. This means 

26we can avoid any problems with paf files at the expense of reimplementing 

27some parts of CameraMapper here. Jim is not sure this was the best 

28possible approach, but it gave him an opportunity to play around with 

29prototyping a future paf-free mapper class, and it does everything it 

30needs to do right now. 

31""" 

32import os 

33import shutil 

34import re 

35 

36import lsst.geom 

37import lsst.daf.persistence 

38import lsst.afw.cameraGeom 

39import lsst.afw.geom 

40import lsst.afw.image as afwImage 

41 

42__all__ = ("SimpleMapper", "makeSimpleCamera", "makeDataRepo") 

43 

44 

45class PersistenceType: 

46 """Base class of a hierarchy used by SimpleMapper to defined different kinds of types of objects 

47 to persist. 

48 

49 PersistenceType objects are never instantiated; only the type objects are used (we needed a 

50 simple singleton struct that could be inherited, which is exactly what a Python type is). 

51 """ 

52 python = None 

53 cpp = "ignored" 

54 storage = None 

55 ext = "" 

56 suffixes = () 

57 

58 @classmethod 

59 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None): 

60 """Method called by SimpleMapping to implement a map_ method.""" 

61 return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [path], dataId, 

62 mapper=mapper, 

63 storage=storage) 

64 

65 def canStandardize(self, datasetType): 

66 return False 

67 

68 

69class BypassPersistenceType(PersistenceType): 

70 """Persistence type for things that don't actually use daf_persistence. 

71 """ 

72 

73 python = "lsst.daf.base.PropertySet" # something to import even when we don't need to 

74 

75 @classmethod 

76 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None): 

77 """Method called by SimpleMapping to implement a map_ method; overridden to not use the path.""" 

78 return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [], dataId, 

79 mapper=mapper, storage=storage) 

80 

81 

82class ExposurePersistenceType(PersistenceType): 

83 """Persistence type of Exposure images. 

84 """ 

85 

86 python = "lsst.afw.image.ExposureF" 

87 cpp = "ExposureF" 

88 storage = "FitsStorage" 

89 ext = ".fits" 

90 suffixes = ("_sub",) 

91 

92 @classmethod 

93 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None): 

94 """Method called by SimpleMapping to implement a map_ method; overridden to support subimages.""" 

95 if suffix is None: 

96 loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, dataId, mapper, suffix=None, 

97 storage=storage) 

98 # Write options are never applicable for _sub, since that's only 

99 # for read. None of the values aside from the "NONE"s matter, but 

100 # writing explicit meaningless values for all of them to appease 

101 # afw is the price we pay for trying to write a non-CameraMapper 

102 # Mapper. It'll all get better with Gen3 (TM). 

103 options = { 

104 "compression.algorithm": "NONE", 

105 "compression.columns": 0, 

106 "compression.rows": 0, 

107 "compression.quantizeLevel": 0.0, 

108 "scaling.algorithm": "NONE", 

109 "scaling.bzero": 0.0, 

110 "scaling.bscale": 0.0, 

111 "scaling.bitpix": 0, 

112 "scaling.quantizeLevel": 0.0, 

113 "scaling.quantizePad": 0.0, 

114 "scaling.fuzz": False, 

115 "scaling.seed": 0, 

116 } 

117 for prefix in ("image", "mask", "variance"): 

118 for k, v in options.items(): 

119 loc.additionalData.set("{}.{}".format(prefix, k), v) 

120 elif suffix == "_sub": 120 ↛ 132line 120 didn't jump to line 132, because the condition on line 120 was never false

121 subId = dataId.copy() 

122 bbox = subId.pop('bbox') 

123 loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, subId, mapper, suffix=None, 

124 storage=storage) 

125 loc.additionalData.set('llcX', bbox.getMinX()) 

126 loc.additionalData.set('llcY', bbox.getMinY()) 

127 loc.additionalData.set('width', bbox.getWidth()) 

128 loc.additionalData.set('height', bbox.getHeight()) 

129 if 'imageOrigin' in dataId: 129 ↛ 130line 129 didn't jump to line 130, because the condition on line 129 was never true

130 loc.additionalData.set('imageOrigin', 

131 dataId['imageOrigin']) 

132 return loc 

133 

134 

135class SkyMapPersistenceType(PersistenceType): 

136 python = "lsst.skymap.BaseSkyMap" 

137 storage = "PickleStorage" 

138 ext = ".pickle" 

139 

140 

141class CatalogPersistenceType(PersistenceType): 

142 python = "lsst.afw.table.BaseCatalog" 

143 cpp = "BaseCatalog" 

144 storage = "FitsCatalogStorage" 

145 ext = ".fits" 

146 

147 

148class SimpleCatalogPersistenceType(CatalogPersistenceType): 

149 python = "lsst.afw.table.SimpleCatalog" 

150 cpp = "SimpleCatalog" 

151 

152 

153class SourceCatalogPersistenceType(SimpleCatalogPersistenceType): 

154 python = "lsst.afw.table.SourceCatalog" 

155 cpp = "SourceCatalog" 

156 

157 

158class ExposureCatalogPersistenceType(CatalogPersistenceType): 

159 python = "lsst.afw.table.ExposureCatalog" 

160 cpp = "ExposureCatalog" 

161 

162 

163class PeakCatalogPersistenceType(CatalogPersistenceType): 

164 python = "lsst.afw.detection.PeakCatalog" 

165 cpp = "PeakCatalog" 

166 

167 

168class SimpleMapping: 

169 """Mapping object used to implement SimpleMapper, similar in intent to lsst.daf.peristence.Mapping. 

170 """ 

171 

172 template = None 

173 keys = {} 

174 

175 def __init__(self, persistence, template=None, keys=None): 

176 self.persistence = persistence 

177 if template is not None: 

178 self.template = template 

179 if keys is not None: 

180 self.keys = keys 

181 

182 def map(self, dataset, root, dataId, mapper, suffix=None, storage=None): 

183 if self.template is not None: 

184 path = self.template.format(dataset=dataset, ext=self.persistence.ext, **dataId) 

185 else: 

186 path = None 

187 return self.persistence.makeButlerLocation(path, dataId, suffix=suffix, mapper=mapper, 

188 storage=storage) 

189 

190 

191class RawMapping(SimpleMapping): 

192 """Mapping for dataset types that are organized the same way as raw data (i.e. by CCD).""" 

193 

194 template = "{dataset}-{visit:04d}-{ccd:01d}{ext}" 

195 keys = dict(visit=int, ccd=int) 

196 

197 def query(self, dataset, index, level, format, dataId): 

198 dictList = index[dataset][level] 

199 results = [list(d.values()) for d in dictList[dataId.get(level, None)]] 

200 return results 

201 

202 

203class SkyMapping(SimpleMapping): 

204 """Mapping for dataset types that are organized according to a SkyMap subdivision of the sky.""" 

205 

206 template = "{dataset}-{filter}-{tract:02d}-{patch}{ext}" 

207 keys = dict(filter=str, tract=int, patch=str) 

208 

209 

210class TempExpMapping(SimpleMapping): 

211 """Mapping for CoaddTempExp datasets.""" 

212 

213 template = "{dataset}-{tract:02d}-{patch}-{visit:04d}{ext}" 

214 keys = dict(tract=int, patch=str, visit=int) 

215 

216 

217class ForcedSrcMapping(RawMapping): 

218 """Mapping for forced_src datasets.""" 

219 

220 template = "{dataset}-{tract:02d}-{visit:04d}-{ccd:01d}{ext}" 

221 keys = dict(tract=int, ccd=int, visit=int) 

222 

223 

224class MapperMeta(type): 

225 """Metaclass for SimpleMapper that creates map_ and query_ methods for everything found in the 

226 'mappings' class variable. 

227 """ 

228 

229 @staticmethod 

230 def _makeMapClosure(dataset, mapping, suffix=None): 

231 def mapClosure(self, dataId, write=False): 

232 return mapping.map(dataset, self.root, dataId, self, suffix=suffix, storage=self.storage) 

233 return mapClosure 

234 

235 @staticmethod 

236 def _makeQueryClosure(dataset, mapping): 

237 def queryClosure(self, level, format, dataId): 

238 return mapping.query(dataset, self.index, level, format, dataId) 

239 return queryClosure 

240 

241 def __init__(cls, name, bases, dict_): # noqa allow "cls" instead of "self" 

242 type.__init__(cls, name, bases, dict_) 

243 cls.keyDict = dict() 

244 for dataset, mapping in cls.mappings.items(): 

245 setattr(cls, "map_" + dataset, MapperMeta._makeMapClosure(dataset, mapping, suffix=None)) 

246 for suffix in mapping.persistence.suffixes: 

247 setattr(cls, "map_" + dataset + suffix, 

248 MapperMeta._makeMapClosure(dataset, mapping, suffix=suffix)) 

249 if hasattr(mapping, "query"): 

250 setattr(cls, "query_" + dataset, MapperMeta._makeQueryClosure(dataset, mapping)) 

251 cls.keyDict.update(mapping.keys) 

252 

253 

254class SimpleMapper(lsst.daf.persistence.Mapper, metaclass=MapperMeta): 

255 """ 

256 An extremely simple mapper for an imaginary camera for use in integration tests. 

257 

258 As SimpleMapper does not inherit from obs.base.CameraMapper, it does not 

259 use a policy file to set mappings or a registry; all the information is here 

260 (in the map_* and query_* methods). 

261 

262 The imaginary camera's raw data format has only 'visit' and 'ccd' keys, with 

263 two CCDs per visit (by default). 

264 """ 

265 

266 mappings = dict( 

267 calexp=RawMapping(ExposurePersistenceType), 

268 forced_src=ForcedSrcMapping(SourceCatalogPersistenceType), 

269 forced_src_schema=SimpleMapping(SourceCatalogPersistenceType, 

270 template="{dataset}{ext}", keys={}), 

271 truth=SimpleMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}", 

272 keys={"tract": int}), 

273 simsrc=RawMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}", 

274 keys={"tract": int}), 

275 observations=SimpleMapping(ExposureCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}", 

276 keys={"tract": int}), 

277 ccdExposureId=RawMapping(BypassPersistenceType), 

278 ccdExposureId_bits=SimpleMapping(BypassPersistenceType), 

279 deepCoaddId=SkyMapping(BypassPersistenceType), 

280 deepCoaddId_bits=SimpleMapping(BypassPersistenceType), 

281 deepMergedCoaddId=SkyMapping(BypassPersistenceType), 

282 deepMergedCoaddId_bits=SimpleMapping(BypassPersistenceType), 

283 deepCoadd_skyMap=SimpleMapping(SkyMapPersistenceType, template="{dataset}{ext}", keys={}), 

284 deepCoadd=SkyMapping(ExposurePersistenceType), 

285 deepCoadd_filterLabel=SkyMapping(ExposurePersistenceType), 

286 deepCoaddPsfMatched=SkyMapping(ExposurePersistenceType), 

287 deepCoadd_calexp=SkyMapping(ExposurePersistenceType), 

288 deepCoadd_calexp_background=SkyMapping(CatalogPersistenceType), 

289 deepCoadd_icSrc=SkyMapping(SourceCatalogPersistenceType), 

290 deepCoadd_icSrc_schema=SimpleMapping(SourceCatalogPersistenceType, 

291 template="{dataset}{ext}", keys={}), 

292 deepCoadd_src=SkyMapping(SourceCatalogPersistenceType), 

293 deepCoadd_src_schema=SimpleMapping(SourceCatalogPersistenceType, 

294 template="{dataset}{ext}", keys={}), 

295 deepCoadd_peak_schema=SimpleMapping(PeakCatalogPersistenceType, 

296 template="{dataset}{ext}", keys={}), 

297 deepCoadd_ref=SkyMapping(SourceCatalogPersistenceType), 

298 deepCoadd_ref_schema=SimpleMapping(SourceCatalogPersistenceType, 

299 template="{dataset}{ext}", keys={}), 

300 deepCoadd_det=SkyMapping(SourceCatalogPersistenceType), 

301 deepCoadd_det_schema=SimpleMapping(SourceCatalogPersistenceType, 

302 template="{dataset}{ext}", keys={}), 

303 deepCoadd_mergeDet=SkyMapping(SourceCatalogPersistenceType), 

304 deepCoadd_mergeDet_schema=SimpleMapping(SourceCatalogPersistenceType, 

305 template="{dataset}{ext}", keys={}), 

306 deepCoadd_deblendedFlux=SkyMapping(SourceCatalogPersistenceType), 

307 deepCoadd_deblendedFlux_schema=SimpleMapping(SourceCatalogPersistenceType, 

308 template="{dataset}{ext}", keys={}), 

309 deepCoadd_deblendedModel=SkyMapping(SourceCatalogPersistenceType), 

310 deepCoadd_deblendedModel_schema=SimpleMapping(SourceCatalogPersistenceType, 

311 template="{dataset}{ext}", keys={}), 

312 deepCoadd_meas=SkyMapping(SourceCatalogPersistenceType), 

313 deepCoadd_meas_schema=SimpleMapping(SourceCatalogPersistenceType, 

314 template="{dataset}{ext}", keys={}), 

315 deepCoadd_forced_src=SkyMapping(SourceCatalogPersistenceType), 

316 deepCoadd_forced_src_schema=SimpleMapping(SourceCatalogPersistenceType, 

317 template="{dataset}{ext}", keys={}), 

318 deepCoadd_mock=SkyMapping(ExposurePersistenceType), 

319 deepCoaddPsfMatched_mock=SkyMapping(ExposurePersistenceType), 

320 deepCoadd_directWarp=TempExpMapping(ExposurePersistenceType), 

321 deepCoadd_directWarp_mock=TempExpMapping(ExposurePersistenceType), 

322 deepCoadd_psfMatchedWarp=TempExpMapping(ExposurePersistenceType), 

323 deepCoadd_psfMatchedWarp_mock=TempExpMapping(ExposurePersistenceType), 

324 ) 

325 

326 levels = dict( 

327 visit=['ccd'], 

328 ccd=[], 

329 ) 

330 

331 def __init__(self, root, **kwargs): 

332 self.storage = lsst.daf.persistence.Storage.makeFromURI(root) 

333 super(SimpleMapper, self).__init__(**kwargs) 

334 self.root = root 

335 self.camera = makeSimpleCamera(nX=1, nY=2, sizeX=400, sizeY=200, gapX=2, gapY=2) 

336 # NOTE: we set band/physical here because CoaddsTestCase.testCoaddInputs() 

337 # expects "r" for both the input dataIds (gen2 dataIds could be either 

338 # physical or band) and the output CoaddInputRecords. 

339 # The original data had just `Filter("r")`, so this has always assumed 

340 # that physicalLabel==bandLabel, even though CoaddInputRecords are physical. 

341 self.filterLabel = afwImage.FilterLabel(band="r", physical="r") 

342 self.update() 

343 

344 def getDefaultLevel(self): 

345 return "ccd" 

346 

347 def getKeys(self, datasetType, level): 

348 if datasetType is None: 348 ↛ 349line 348 didn't jump to line 349, because the condition on line 348 was never true

349 keyDict = self.keyDict 

350 else: 

351 keyDict = self.mappings[datasetType].keys 

352 if level is not None and level in self.levels: 352 ↛ 353line 352 didn't jump to line 353, because the condition on line 352 was never true

353 keyDict = dict(keyDict) 

354 for lev in self.levels[level]: 

355 if lev in keyDict: 

356 del keyDict[lev] 

357 return keyDict 

358 

359 def update(self): 

360 filenames = os.listdir(self.root) 

361 rawRegex = re.compile(r"(?P<dataset>\w+)-(?P<visit>\d+)-(?P<ccd>\d).*") 

362 self.index = {} 

363 for filename in filenames: 

364 m = rawRegex.match(filename) 

365 if not m: 

366 continue 

367 index = self.index.setdefault(m.group('dataset'), dict(ccd={None: []}, visit={None: []})) 

368 visit = int(m.group('visit')) 

369 ccd = int(m.group('ccd')) 

370 d1 = dict(visit=visit, ccd=ccd) 

371 d2 = dict(visit=visit) 

372 index['ccd'].setdefault(visit, []).append(d1) 

373 index['ccd'][None].append(d1) 

374 index['visit'][visit] = [d2] 

375 index['visit'][None].append(d1) 

376 

377 def keys(self): 

378 return self.keyDict 

379 

380 def bypass_camera(self, datasetType, pythonType, location, dataId): 

381 return self.camera 

382 

383 def map_camera(self, dataId, write=False): 

384 return lsst.daf.persistence.ButlerLocation( 

385 "lsst.afw.cameraGeom.Camera", "Camera", None, [], dataId, mapper=self, storage=self.storage 

386 ) 

387 

388 def std_calexp(self, item, dataId): 

389 detectorId = dataId["ccd"] 

390 detector = self.camera[detectorId] 

391 item.setDetector(detector) 

392 item.setFilterLabel(self.filterLabel) 

393 return item 

394 

395 def _computeCcdExposureId(self, dataId): 

396 return int(dataId["visit"]) * 10 + int(dataId["ccd"]) 

397 

398 def _computeCoaddId(self, dataId): 

399 # Note: for real IDs, we'd want to include filter here, but it doesn't actually matter 

400 # for any of the tests we've done so far, which all assume filter='r' 

401 tract = int(dataId['tract']) 

402 if tract < 0 or tract >= 128: 402 ↛ 403line 402 didn't jump to line 403, because the condition on line 402 was never true

403 raise RuntimeError('tract not in range [0,128)') 

404 patchX, patchY = (int(c) for c in dataId['patch'].split(',')) 

405 for p in (patchX, patchY): 

406 if p < 0 or p >= 2**13: 406 ↛ 407line 406 didn't jump to line 407, because the condition on line 406 was never true

407 raise RuntimeError('patch component not in range [0, 8192)') 

408 return (tract * 2**13 + patchX) * 2**13 + patchY 

409 

410 @staticmethod 

411 def splitCcdExposureId(ccdExposureId): 

412 return dict(visit=(int(ccdExposureId) // 10), ccd=(int(ccdExposureId) % 10)) 

413 

414 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId): 

415 return self._computeCcdExposureId(dataId) 

416 

417 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId): 

418 return 32 

419 

420 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId): 

421 return self._computeCoaddId(dataId) 

422 

423 def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId): 

424 return 1 + 7 + 13*2 + 3 

425 

426 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId): 

427 return self._computeCoaddId(dataId) 

428 

429 def bypass_deepMergedCoaddId_bits(self, datasetType, pythonType, location, dataId): 

430 return 1 + 7 + 13*2 + 3 

431 

432 def bypass_deepCoadd_filterLabel(self, *args, **kwargs): 

433 """To return a useful filterLabel for MergeDetectionsTask.""" 

434 return afwImage.FilterLabel(band=self.filterLabel.bandLabel) 

435 

436 

437def makeSimpleCamera( 

438 nX, nY, 

439 sizeX, sizeY, 

440 gapX, gapY, 

441 pixelSize=1.0, 

442 plateScale=20.0, 

443 radialDistortion=0.925, 

444): 

445 """Create a camera 

446 

447 @param[in] nx: number of detectors in x 

448 @param[in] ny: number of detectors in y 

449 @param[in] sizeX: detector size in x (pixels) 

450 @param[in] sizeY: detector size in y (pixels) 

451 @param[in] gapX: gap between detectors in x (mm) 

452 @param[in] gapY: gap between detectors in y (mm) 

453 @param[in] pixelSize: pixel size (mm) (a float) 

454 @param[in] plateScale: plate scale in arcsec/mm; 20.0 is for LSST 

455 @param[in] radialDistortion: radial distortion, in mm/rad^2 

456 (the r^3 coefficient of the radial distortion polynomial 

457 that converts FIELD_ANGLE in radians to FOCAL_PLANE in mm); 

458 0.925 is the value Dave Monet measured for lsstSim data 

459 

460 Each detector will have one amplifier (with no raw information). 

461 """ 

462 pScaleRad = lsst.geom.arcsecToRad(plateScale) 

463 radialDistortCoeffs = [0.0, 1.0/pScaleRad, 0.0, radialDistortion/pScaleRad] 

464 focalPlaneToFieldAngle = lsst.afw.geom.makeRadialTransform(radialDistortCoeffs) 

465 

466 ccdBBox = lsst.geom.Box2I(lsst.geom.Point2I(), lsst.geom.Extent2I(sizeX, sizeY)) 

467 

468 cameraBuilder = lsst.afw.cameraGeom.Camera.Builder("Simple Camera") 

469 

470 detectorId = 0 

471 for iY in range(nY): 

472 cY = (iY - 0.5 * (nY - 1)) * (pixelSize * sizeY + gapY) 

473 for iX in range(nX): 

474 cX = (iX - 0.5 * (nX - 1)) * (pixelSize * sizeY + gapX) 

475 fpPos = lsst.geom.Point2D(cX, cY) 

476 detectorName = "detector %d,%d" % (iX, iY) 

477 

478 detectorBuilder = cameraBuilder.add(detectorName, detectorId) 

479 detectorBuilder.setSerial(detectorName + " serial") 

480 detectorBuilder.setBBox(ccdBBox) 

481 detectorBuilder.setOrientation(lsst.afw.cameraGeom.Orientation(fpPos)) 

482 detectorBuilder.setPixelSize(lsst.geom.Extent2D(pixelSize, pixelSize)) 

483 

484 ampBuilder = lsst.afw.cameraGeom.Amplifier.Builder() 

485 ampName = "amp" 

486 ampBuilder.setName(ampName) 

487 ampBuilder.setBBox(ccdBBox) 

488 ampBuilder.setGain(1.0) 

489 ampBuilder.setReadNoise(5.0) 

490 

491 detectorBuilder.append(ampBuilder) 

492 

493 detectorId += 1 

494 

495 cameraBuilder.setTransformFromFocalPlaneTo(lsst.afw.cameraGeom.FIELD_ANGLE, focalPlaneToFieldAngle) 

496 return cameraBuilder.finish() 

497 

498 

499def makeDataRepo(root): 

500 """ 

501 Create a data repository for SimpleMapper and return a butler for it. 

502 

503 Clobbers anything already in the given path. 

504 """ 

505 if os.path.exists(root): 505 ↛ 506line 505 didn't jump to line 506, because the condition on line 505 was never true

506 shutil.rmtree(root) 

507 os.makedirs(root) 

508 with open(os.path.join(root, "_mapper"), "w") as f: 

509 f.write("lsst.pipe.tasks.mocks.SimpleMapper\n") 

510 return lsst.daf.persistence.Butler(root=root)