Coverage for python/lsst/pipe/tasks/mocks/simpleMapper.py: 50%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

239 statements  

1# 

2# LSST Data Management System 

3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23"""Mapper and cameraGeom definition for extremely simple mock data. 

24 

25SimpleMapper inherits directly from Mapper, not CameraMapper. This means 

26we can avoid any problems with paf files at the expense of reimplementing 

27some parts of CameraMapper here. Jim is not sure this was the best 

28possible approach, but it gave him an opportunity to play around with 

29prototyping a future paf-free mapper class, and it does everything it 

30needs to do right now. 

31""" 

32import os 

33import shutil 

34import re 

35 

36import lsst.geom 

37import lsst.daf.persistence 

38import lsst.afw.cameraGeom 

39import lsst.afw.geom 

40import lsst.afw.image as afwImage 

41 

42__all__ = ("SimpleMapper", "makeSimpleCamera", "makeDataRepo") 

43 

44 

45class PersistenceType: 

46 """Base class of a hierarchy used by SimpleMapper to defined different kinds of types of objects 

47 to persist. 

48 

49 PersistenceType objects are never instantiated; only the type objects are used (we needed a 

50 simple singleton struct that could be inherited, which is exactly what a Python type is). 

51 """ 

52 python = None 

53 cpp = "ignored" 

54 storage = None 

55 ext = "" 

56 suffixes = () 

57 

58 @classmethod 

59 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None): 

60 """Method called by SimpleMapping to implement a map_ method.""" 

61 return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [path], dataId, 

62 mapper=mapper, 

63 storage=storage) 

64 

65 def canStandardize(self, datasetType): 

66 return False 

67 

68 

69class BypassPersistenceType(PersistenceType): 

70 """Persistence type for things that don't actually use daf_persistence. 

71 """ 

72 

73 python = "lsst.daf.base.PropertySet" # something to import even when we don't need to 

74 

75 @classmethod 

76 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None): 

77 """Method called by SimpleMapping to implement a map_ method; overridden to not use the path.""" 

78 return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [], dataId, 

79 mapper=mapper, storage=storage) 

80 

81 

82class ExposurePersistenceType(PersistenceType): 

83 """Persistence type of Exposure images. 

84 """ 

85 

86 python = "lsst.afw.image.ExposureF" 

87 cpp = "ExposureF" 

88 storage = "FitsStorage" 

89 ext = ".fits" 

90 suffixes = ("_sub",) 

91 

92 @classmethod 

93 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None): 

94 """Method called by SimpleMapping to implement a map_ method; overridden to support subimages.""" 

95 if suffix is None: 

96 loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, dataId, mapper, suffix=None, 

97 storage=storage) 

98 # Write options are never applicable for _sub, since that's only 

99 # for read. None of the values aside from the "NONE"s matter, but 

100 # writing explicit meaningless values for all of them to appease 

101 # afw is the price we pay for trying to write a non-CameraMapper 

102 # Mapper. It'll all get better with Gen3 (TM). 

103 options = { 

104 "compression.algorithm": "NONE", 

105 "compression.columns": 0, 

106 "compression.rows": 0, 

107 "compression.quantizeLevel": 0.0, 

108 "scaling.algorithm": "NONE", 

109 "scaling.bzero": 0.0, 

110 "scaling.bscale": 0.0, 

111 "scaling.bitpix": 0, 

112 "scaling.quantizeLevel": 0.0, 

113 "scaling.quantizePad": 0.0, 

114 "scaling.fuzz": False, 

115 "scaling.seed": 0, 

116 } 

117 for prefix in ("image", "mask", "variance"): 

118 for k, v in options.items(): 

119 loc.additionalData.set("{}.{}".format(prefix, k), v) 

120 elif suffix == "_sub": 

121 subId = dataId.copy() 

122 bbox = subId.pop('bbox') 

123 loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, subId, mapper, suffix=None, 

124 storage=storage) 

125 loc.additionalData.set('llcX', bbox.getMinX()) 

126 loc.additionalData.set('llcY', bbox.getMinY()) 

127 loc.additionalData.set('width', bbox.getWidth()) 

128 loc.additionalData.set('height', bbox.getHeight()) 

129 if 'imageOrigin' in dataId: 

130 loc.additionalData.set('imageOrigin', 

131 dataId['imageOrigin']) 

132 return loc 

133 

134 

135class SkyMapPersistenceType(PersistenceType): 

136 python = "lsst.skymap.BaseSkyMap" 

137 storage = "PickleStorage" 

138 ext = ".pickle" 

139 

140 

141class CatalogPersistenceType(PersistenceType): 

142 python = "lsst.afw.table.BaseCatalog" 

143 cpp = "BaseCatalog" 

144 storage = "FitsCatalogStorage" 

145 ext = ".fits" 

146 

147 

148class SimpleCatalogPersistenceType(CatalogPersistenceType): 

149 python = "lsst.afw.table.SimpleCatalog" 

150 cpp = "SimpleCatalog" 

151 

152 

153class SourceCatalogPersistenceType(SimpleCatalogPersistenceType): 

154 python = "lsst.afw.table.SourceCatalog" 

155 cpp = "SourceCatalog" 

156 

157 

158class ExposureCatalogPersistenceType(CatalogPersistenceType): 

159 python = "lsst.afw.table.ExposureCatalog" 

160 cpp = "ExposureCatalog" 

161 

162 

163class PeakCatalogPersistenceType(CatalogPersistenceType): 

164 python = "lsst.afw.detection.PeakCatalog" 

165 cpp = "PeakCatalog" 

166 

167 

168class SimpleMapping: 

169 """Mapping object used to implement SimpleMapper, similar in intent to lsst.daf.peristence.Mapping. 

170 """ 

171 

172 template = None 

173 keys = {} 

174 

175 def __init__(self, persistence, template=None, keys=None): 

176 self.persistence = persistence 

177 if template is not None: 

178 self.template = template 

179 if keys is not None: 

180 self.keys = keys 

181 

182 def map(self, dataset, root, dataId, mapper, suffix=None, storage=None): 

183 if self.template is not None: 

184 path = self.template.format(dataset=dataset, ext=self.persistence.ext, **dataId) 

185 else: 

186 path = None 

187 return self.persistence.makeButlerLocation(path, dataId, suffix=suffix, mapper=mapper, 

188 storage=storage) 

189 

190 

191class RawMapping(SimpleMapping): 

192 """Mapping for dataset types that are organized the same way as raw data (i.e. by CCD).""" 

193 

194 template = "{dataset}-{visit:04d}-{ccd:01d}{ext}" 

195 keys = dict(visit=int, ccd=int) 

196 

197 def query(self, dataset, index, level, format, dataId): 

198 dictList = index[dataset][level] 

199 results = [list(d.values()) for d in dictList[dataId.get(level, None)]] 

200 return results 

201 

202 

203class SkyMapping(SimpleMapping): 

204 """Mapping for dataset types that are organized according to a SkyMap subdivision of the sky.""" 

205 

206 template = "{dataset}-{filter}-{tract:02d}-{patch}{ext}" 

207 keys = dict(filter=str, tract=int, patch=str) 

208 

209 

210class TempExpMapping(SimpleMapping): 

211 """Mapping for CoaddTempExp datasets.""" 

212 

213 template = "{dataset}-{tract:02d}-{patch}-{visit:04d}{ext}" 

214 keys = dict(tract=int, patch=str, visit=int) 

215 

216 

217class ForcedSrcMapping(RawMapping): 

218 """Mapping for forced_src datasets.""" 

219 

220 template = "{dataset}-{tract:02d}-{visit:04d}-{ccd:01d}{ext}" 

221 keys = dict(tract=int, ccd=int, visit=int) 

222 

223 

224class MapperMeta(type): 

225 """Metaclass for SimpleMapper that creates map_ and query_ methods for everything found in the 

226 'mappings' class variable. 

227 """ 

228 

229 @staticmethod 

230 def _makeMapClosure(dataset, mapping, suffix=None): 

231 def mapClosure(self, dataId, write=False): 

232 return mapping.map(dataset, self.root, dataId, self, suffix=suffix, storage=self.storage) 

233 return mapClosure 

234 

235 @staticmethod 

236 def _makeQueryClosure(dataset, mapping): 

237 def queryClosure(self, level, format, dataId): 

238 return mapping.query(dataset, self.index, level, format, dataId) 

239 return queryClosure 

240 

241 def __init__(cls, name, bases, dict_): # noqa allow "cls" instead of "self" 

242 type.__init__(cls, name, bases, dict_) 

243 cls.keyDict = dict() 

244 for dataset, mapping in cls.mappings.items(): 

245 setattr(cls, "map_" + dataset, MapperMeta._makeMapClosure(dataset, mapping, suffix=None)) 

246 for suffix in mapping.persistence.suffixes: 

247 setattr(cls, "map_" + dataset + suffix, 

248 MapperMeta._makeMapClosure(dataset, mapping, suffix=suffix)) 

249 if hasattr(mapping, "query"): 

250 setattr(cls, "query_" + dataset, MapperMeta._makeQueryClosure(dataset, mapping)) 

251 cls.keyDict.update(mapping.keys) 

252 

253 

254class SimpleMapper(lsst.daf.persistence.Mapper, metaclass=MapperMeta): 

255 """ 

256 An extremely simple mapper for an imaginary camera for use in integration tests. 

257 

258 As SimpleMapper does not inherit from obs.base.CameraMapper, it does not 

259 use a policy file to set mappings or a registry; all the information is here 

260 (in the map_* and query_* methods). 

261 

262 The imaginary camera's raw data format has only 'visit' and 'ccd' keys, with 

263 two CCDs per visit (by default). 

264 """ 

265 

266 mappings = dict( 

267 calexp=RawMapping(ExposurePersistenceType), 

268 forced_src=ForcedSrcMapping(SourceCatalogPersistenceType), 

269 forced_src_schema=SimpleMapping(SourceCatalogPersistenceType, 

270 template="{dataset}{ext}", keys={}), 

271 truth=SimpleMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}", 

272 keys={"tract": int}), 

273 simsrc=RawMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}", 

274 keys={"tract": int}), 

275 observations=SimpleMapping(ExposureCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}", 

276 keys={"tract": int}), 

277 ccdExposureId=RawMapping(BypassPersistenceType), 

278 ccdExposureId_bits=SimpleMapping(BypassPersistenceType), 

279 deepCoaddId=SkyMapping(BypassPersistenceType), 

280 deepCoaddId_bits=SimpleMapping(BypassPersistenceType), 

281 deepCoadd_band=SimpleMapping(BypassPersistenceType), 

282 deepMergedCoaddId=SkyMapping(BypassPersistenceType), 

283 deepMergedCoaddId_bits=SimpleMapping(BypassPersistenceType), 

284 deepCoadd_skyMap=SimpleMapping(SkyMapPersistenceType, template="{dataset}{ext}", keys={}), 

285 deepCoadd=SkyMapping(ExposurePersistenceType), 

286 deepCoadd_filterLabel=SkyMapping(ExposurePersistenceType), 

287 deepCoaddPsfMatched=SkyMapping(ExposurePersistenceType), 

288 deepCoadd_calexp=SkyMapping(ExposurePersistenceType), 

289 deepCoadd_calexp_background=SkyMapping(CatalogPersistenceType), 

290 deepCoadd_icSrc=SkyMapping(SourceCatalogPersistenceType), 

291 deepCoadd_icSrc_schema=SimpleMapping(SourceCatalogPersistenceType, 

292 template="{dataset}{ext}", keys={}), 

293 deepCoadd_src=SkyMapping(SourceCatalogPersistenceType), 

294 deepCoadd_src_schema=SimpleMapping(SourceCatalogPersistenceType, 

295 template="{dataset}{ext}", keys={}), 

296 deepCoadd_peak_schema=SimpleMapping(PeakCatalogPersistenceType, 

297 template="{dataset}{ext}", keys={}), 

298 deepCoadd_ref=SkyMapping(SourceCatalogPersistenceType), 

299 deepCoadd_ref_schema=SimpleMapping(SourceCatalogPersistenceType, 

300 template="{dataset}{ext}", keys={}), 

301 deepCoadd_det=SkyMapping(SourceCatalogPersistenceType), 

302 deepCoadd_det_schema=SimpleMapping(SourceCatalogPersistenceType, 

303 template="{dataset}{ext}", keys={}), 

304 deepCoadd_mergeDet=SkyMapping(SourceCatalogPersistenceType), 

305 deepCoadd_mergeDet_schema=SimpleMapping(SourceCatalogPersistenceType, 

306 template="{dataset}{ext}", keys={}), 

307 deepCoadd_deblendedFlux=SkyMapping(SourceCatalogPersistenceType), 

308 deepCoadd_deblendedFlux_schema=SimpleMapping(SourceCatalogPersistenceType, 

309 template="{dataset}{ext}", keys={}), 

310 deepCoadd_deblendedModel=SkyMapping(SourceCatalogPersistenceType), 

311 deepCoadd_deblendedModel_schema=SimpleMapping(SourceCatalogPersistenceType, 

312 template="{dataset}{ext}", keys={}), 

313 deepCoadd_meas=SkyMapping(SourceCatalogPersistenceType), 

314 deepCoadd_meas_schema=SimpleMapping(SourceCatalogPersistenceType, 

315 template="{dataset}{ext}", keys={}), 

316 deepCoadd_forced_src=SkyMapping(SourceCatalogPersistenceType), 

317 deepCoadd_forced_src_schema=SimpleMapping(SourceCatalogPersistenceType, 

318 template="{dataset}{ext}", keys={}), 

319 deepCoadd_mock=SkyMapping(ExposurePersistenceType), 

320 deepCoaddPsfMatched_mock=SkyMapping(ExposurePersistenceType), 

321 deepCoadd_directWarp=TempExpMapping(ExposurePersistenceType), 

322 deepCoadd_directWarp_mock=TempExpMapping(ExposurePersistenceType), 

323 deepCoadd_psfMatchedWarp=TempExpMapping(ExposurePersistenceType), 

324 deepCoadd_psfMatchedWarp_mock=TempExpMapping(ExposurePersistenceType), 

325 ) 

326 

327 levels = dict( 

328 visit=['ccd'], 

329 ccd=[], 

330 ) 

331 

332 def __init__(self, root, **kwargs): 

333 self.storage = lsst.daf.persistence.Storage.makeFromURI(root) 

334 super(SimpleMapper, self).__init__(**kwargs) 

335 self.root = root 

336 self.camera = makeSimpleCamera(nX=1, nY=2, sizeX=400, sizeY=200, gapX=2, gapY=2) 

337 # NOTE: we set band/physical here because CoaddsTestCase.testCoaddInputs() 

338 # expects "r" for both the input dataIds (gen2 dataIds could be either 

339 # physical or band) and the output CoaddInputRecords. 

340 # The original data had just `Filter("r")`, so this has always assumed 

341 # that physicalLabel==bandLabel, even though CoaddInputRecords are physical. 

342 self.filterLabel = afwImage.FilterLabel(band="r", physical="r") 

343 self.update() 

344 

345 def getDefaultLevel(self): 

346 return "ccd" 

347 

348 def getKeys(self, datasetType, level): 

349 if datasetType is None: 

350 keyDict = self.keyDict 

351 else: 

352 keyDict = self.mappings[datasetType].keys 

353 if level is not None and level in self.levels: 

354 keyDict = dict(keyDict) 

355 for lev in self.levels[level]: 

356 if lev in keyDict: 

357 del keyDict[lev] 

358 return keyDict 

359 

360 def update(self): 

361 filenames = os.listdir(self.root) 

362 rawRegex = re.compile(r"(?P<dataset>\w+)-(?P<visit>\d+)-(?P<ccd>\d).*") 

363 self.index = {} 

364 for filename in filenames: 

365 m = rawRegex.match(filename) 

366 if not m: 

367 continue 

368 index = self.index.setdefault(m.group('dataset'), dict(ccd={None: []}, visit={None: []})) 

369 visit = int(m.group('visit')) 

370 ccd = int(m.group('ccd')) 

371 d1 = dict(visit=visit, ccd=ccd) 

372 d2 = dict(visit=visit) 

373 index['ccd'].setdefault(visit, []).append(d1) 

374 index['ccd'][None].append(d1) 

375 index['visit'][visit] = [d2] 

376 index['visit'][None].append(d1) 

377 

378 def keys(self): 

379 return self.keyDict 

380 

381 def bypass_camera(self, datasetType, pythonType, location, dataId): 

382 return self.camera 

383 

384 def map_camera(self, dataId, write=False): 

385 return lsst.daf.persistence.ButlerLocation( 

386 "lsst.afw.cameraGeom.Camera", "Camera", None, [], dataId, mapper=self, storage=self.storage 

387 ) 

388 

389 def std_calexp(self, item, dataId): 

390 detectorId = dataId["ccd"] 

391 detector = self.camera[detectorId] 

392 item.setDetector(detector) 

393 item.setFilterLabel(self.filterLabel) 

394 return item 

395 

396 def _computeCcdExposureId(self, dataId): 

397 return int(dataId["visit"]) * 10 + int(dataId["ccd"]) 

398 

399 def _computeCoaddId(self, dataId): 

400 # Note: for real IDs, we'd want to include filter here, but it doesn't actually matter 

401 # for any of the tests we've done so far, which all assume filter='r' 

402 tract = int(dataId['tract']) 

403 if tract < 0 or tract >= 128: 

404 raise RuntimeError('tract not in range [0,128)') 

405 patchX, patchY = (int(c) for c in dataId['patch'].split(',')) 

406 for p in (patchX, patchY): 

407 if p < 0 or p >= 2**13: 

408 raise RuntimeError('patch component not in range [0, 8192)') 

409 return (tract * 2**13 + patchX) * 2**13 + patchY 

410 

411 @staticmethod 

412 def splitCcdExposureId(ccdExposureId): 

413 return dict(visit=(int(ccdExposureId) // 10), ccd=(int(ccdExposureId) % 10)) 

414 

415 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId): 

416 return self._computeCcdExposureId(dataId) 

417 

418 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId): 

419 return 32 

420 

421 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId): 

422 return self._computeCoaddId(dataId) 

423 

424 def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId): 

425 return 1 + 7 + 13*2 + 3 

426 

427 def bypass_deepCoadd_band(self, datasetType, pythonType, location, dataId): 

428 return self.filterLabel.bandLabel 

429 

430 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId): 

431 return self._computeCoaddId(dataId) 

432 

433 def bypass_deepMergedCoaddId_bits(self, datasetType, pythonType, location, dataId): 

434 return 1 + 7 + 13*2 + 3 

435 

436 def bypass_deepCoadd_filterLabel(self, *args, **kwargs): 

437 """To return a useful filterLabel for MergeDetectionsTask.""" 

438 return afwImage.FilterLabel(band=self.filterLabel.bandLabel) 

439 

440 

441def makeSimpleCamera( 

442 nX, nY, 

443 sizeX, sizeY, 

444 gapX, gapY, 

445 pixelSize=1.0, 

446 plateScale=20.0, 

447 radialDistortion=0.925, 

448): 

449 """Create a camera 

450 

451 @param[in] nx: number of detectors in x 

452 @param[in] ny: number of detectors in y 

453 @param[in] sizeX: detector size in x (pixels) 

454 @param[in] sizeY: detector size in y (pixels) 

455 @param[in] gapX: gap between detectors in x (mm) 

456 @param[in] gapY: gap between detectors in y (mm) 

457 @param[in] pixelSize: pixel size (mm) (a float) 

458 @param[in] plateScale: plate scale in arcsec/mm; 20.0 is for LSST 

459 @param[in] radialDistortion: radial distortion, in mm/rad^2 

460 (the r^3 coefficient of the radial distortion polynomial 

461 that converts FIELD_ANGLE in radians to FOCAL_PLANE in mm); 

462 0.925 is the value Dave Monet measured for lsstSim data 

463 

464 Each detector will have one amplifier (with no raw information). 

465 """ 

466 pScaleRad = lsst.geom.arcsecToRad(plateScale) 

467 radialDistortCoeffs = [0.0, 1.0/pScaleRad, 0.0, radialDistortion/pScaleRad] 

468 focalPlaneToFieldAngle = lsst.afw.geom.makeRadialTransform(radialDistortCoeffs) 

469 

470 ccdBBox = lsst.geom.Box2I(lsst.geom.Point2I(), lsst.geom.Extent2I(sizeX, sizeY)) 

471 

472 cameraBuilder = lsst.afw.cameraGeom.Camera.Builder("Simple Camera") 

473 

474 detectorId = 0 

475 for iY in range(nY): 

476 cY = (iY - 0.5 * (nY - 1)) * (pixelSize * sizeY + gapY) 

477 for iX in range(nX): 

478 cX = (iX - 0.5 * (nX - 1)) * (pixelSize * sizeY + gapX) 

479 fpPos = lsst.geom.Point2D(cX, cY) 

480 detectorName = "detector %d,%d" % (iX, iY) 

481 

482 detectorBuilder = cameraBuilder.add(detectorName, detectorId) 

483 detectorBuilder.setSerial(detectorName + " serial") 

484 detectorBuilder.setBBox(ccdBBox) 

485 detectorBuilder.setOrientation(lsst.afw.cameraGeom.Orientation(fpPos)) 

486 detectorBuilder.setPixelSize(lsst.geom.Extent2D(pixelSize, pixelSize)) 

487 

488 ampBuilder = lsst.afw.cameraGeom.Amplifier.Builder() 

489 ampName = "amp" 

490 ampBuilder.setName(ampName) 

491 ampBuilder.setBBox(ccdBBox) 

492 ampBuilder.setGain(1.0) 

493 ampBuilder.setReadNoise(5.0) 

494 

495 detectorBuilder.append(ampBuilder) 

496 

497 detectorId += 1 

498 

499 cameraBuilder.setTransformFromFocalPlaneTo(lsst.afw.cameraGeom.FIELD_ANGLE, focalPlaneToFieldAngle) 

500 return cameraBuilder.finish() 

501 

502 

503def makeDataRepo(root): 

504 """ 

505 Create a data repository for SimpleMapper and return a butler for it. 

506 

507 Clobbers anything already in the given path. 

508 """ 

509 if os.path.exists(root): 

510 shutil.rmtree(root) 

511 os.makedirs(root) 

512 with open(os.path.join(root, "_mapper"), "w") as f: 

513 f.write("lsst.pipe.tasks.mocks.SimpleMapper\n") 

514 return lsst.daf.persistence.Butler(root=root)