Coverage for python/lsst/pipe/tasks/mocks/simpleMapper.py: 51%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

239 statements  

1# 

2# LSST Data Management System 

3# Copyright 2008, 2009, 2010, 2011, 2012 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23"""Mapper and cameraGeom definition for extremely simple mock data. 

24 

25SimpleMapper inherits directly from Mapper, not CameraMapper. This means 

26we can avoid any problems with paf files at the expense of reimplementing 

27some parts of CameraMapper here. Jim is not sure this was the best 

28possible approach, but it gave him an opportunity to play around with 

29prototyping a future paf-free mapper class, and it does everything it 

30needs to do right now. 

31""" 

32import os 

33import shutil 

34import re 

35 

36import lsst.geom 

37import lsst.daf.persistence 

38import lsst.afw.cameraGeom 

39import lsst.afw.geom 

40import lsst.afw.image as afwImage 

41from lsst.daf.base import PropertySet 

42 

43__all__ = ("SimpleMapper", "makeSimpleCamera", "makeDataRepo") 

44 

45 

46# None of the values aside from the "NONE"s matter, but 

47# writing explicit meaningless values for all of them to appease 

48# afw is the price we pay for trying to write a non-CameraMapper 

49# Mapper. It'll all get better with Gen3 (TM). 

50# Cache the values in a PropertySet since this code is called thousands 

51# of times. 

52_write_options = PropertySet.from_mapping({ 

53 "compression.algorithm": "NONE", 

54 "compression.columns": 0, 

55 "compression.rows": 0, 

56 "compression.quantizeLevel": 0.0, 

57 "scaling.algorithm": "NONE", 

58 "scaling.bzero": 0.0, 

59 "scaling.bscale": 0.0, 

60 "scaling.bitpix": 0, 

61 "scaling.quantizeLevel": 0.0, 

62 "scaling.quantizePad": 0.0, 

63 "scaling.fuzz": False, 

64 "scaling.seed": 0, 

65}) 

66 

67 

68class PersistenceType: 

69 """Base class of a hierarchy used by SimpleMapper to defined different kinds of types of objects 

70 to persist. 

71 

72 PersistenceType objects are never instantiated; only the type objects are used (we needed a 

73 simple singleton struct that could be inherited, which is exactly what a Python type is). 

74 """ 

75 python = None 

76 cpp = "ignored" 

77 storage = None 

78 ext = "" 

79 suffixes = () 

80 

81 @classmethod 

82 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None): 

83 """Method called by SimpleMapping to implement a map_ method.""" 

84 return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [path], dataId, 

85 mapper=mapper, 

86 storage=storage) 

87 

88 def canStandardize(self, datasetType): 

89 return False 

90 

91 

92class BypassPersistenceType(PersistenceType): 

93 """Persistence type for things that don't actually use daf_persistence. 

94 """ 

95 

96 python = "lsst.daf.base.PropertySet" # something to import even when we don't need to 

97 

98 @classmethod 

99 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None): 

100 """Method called by SimpleMapping to implement a map_ method; overridden to not use the path.""" 

101 return lsst.daf.persistence.ButlerLocation(cls.python, cls.cpp, cls.storage, [], dataId, 

102 mapper=mapper, storage=storage) 

103 

104 

105class ExposurePersistenceType(PersistenceType): 

106 """Persistence type of Exposure images. 

107 """ 

108 

109 python = "lsst.afw.image.ExposureF" 

110 cpp = "ExposureF" 

111 storage = "FitsStorage" 

112 ext = ".fits" 

113 suffixes = ("_sub",) 

114 

115 @classmethod 

116 def makeButlerLocation(cls, path, dataId, mapper, suffix=None, storage=None): 

117 """Method called by SimpleMapping to implement a map_ method; overridden to support subimages.""" 

118 if suffix is None: 

119 loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, dataId, mapper, suffix=None, 

120 storage=storage) 

121 # Write options are never applicable for _sub, since that's only 

122 # for read. 

123 for prefix in ("image", "mask", "variance"): 

124 loc.additionalData.setPropertySet(prefix, _write_options) 

125 elif suffix == "_sub": 

126 subId = dataId.copy() 

127 bbox = subId.pop('bbox') 

128 loc = super(ExposurePersistenceType, cls).makeButlerLocation(path, subId, mapper, suffix=None, 

129 storage=storage) 

130 loc.additionalData.set('llcX', bbox.getMinX()) 

131 loc.additionalData.set('llcY', bbox.getMinY()) 

132 loc.additionalData.set('width', bbox.getWidth()) 

133 loc.additionalData.set('height', bbox.getHeight()) 

134 if 'imageOrigin' in dataId: 

135 loc.additionalData.set('imageOrigin', 

136 dataId['imageOrigin']) 

137 return loc 

138 

139 

140class SkyMapPersistenceType(PersistenceType): 

141 python = "lsst.skymap.BaseSkyMap" 

142 storage = "PickleStorage" 

143 ext = ".pickle" 

144 

145 

146class CatalogPersistenceType(PersistenceType): 

147 python = "lsst.afw.table.BaseCatalog" 

148 cpp = "BaseCatalog" 

149 storage = "FitsCatalogStorage" 

150 ext = ".fits" 

151 

152 

153class SimpleCatalogPersistenceType(CatalogPersistenceType): 

154 python = "lsst.afw.table.SimpleCatalog" 

155 cpp = "SimpleCatalog" 

156 

157 

158class SourceCatalogPersistenceType(SimpleCatalogPersistenceType): 

159 python = "lsst.afw.table.SourceCatalog" 

160 cpp = "SourceCatalog" 

161 

162 

163class ExposureCatalogPersistenceType(CatalogPersistenceType): 

164 python = "lsst.afw.table.ExposureCatalog" 

165 cpp = "ExposureCatalog" 

166 

167 

168class PeakCatalogPersistenceType(CatalogPersistenceType): 

169 python = "lsst.afw.detection.PeakCatalog" 

170 cpp = "PeakCatalog" 

171 

172 

173class SimpleMapping: 

174 """Mapping object used to implement SimpleMapper, similar in intent to lsst.daf.peristence.Mapping. 

175 """ 

176 

177 template = None 

178 keys = {} 

179 

180 def __init__(self, persistence, template=None, keys=None): 

181 self.persistence = persistence 

182 if template is not None: 

183 self.template = template 

184 if keys is not None: 

185 self.keys = keys 

186 

187 def map(self, dataset, root, dataId, mapper, suffix=None, storage=None): 

188 if self.template is not None: 

189 path = self.template.format(dataset=dataset, ext=self.persistence.ext, **dataId) 

190 else: 

191 path = None 

192 return self.persistence.makeButlerLocation(path, dataId, suffix=suffix, mapper=mapper, 

193 storage=storage) 

194 

195 

196class RawMapping(SimpleMapping): 

197 """Mapping for dataset types that are organized the same way as raw data (i.e. by CCD).""" 

198 

199 template = "{dataset}-{visit:04d}-{ccd:01d}{ext}" 

200 keys = dict(visit=int, ccd=int) 

201 

202 def query(self, dataset, index, level, format, dataId): 

203 dictList = index[dataset][level] 

204 results = [list(d.values()) for d in dictList[dataId.get(level, None)]] 

205 return results 

206 

207 

208class SkyMapping(SimpleMapping): 

209 """Mapping for dataset types that are organized according to a SkyMap subdivision of the sky.""" 

210 

211 template = "{dataset}-{filter}-{tract:02d}-{patch}{ext}" 

212 keys = dict(filter=str, tract=int, patch=str) 

213 

214 

215class TempExpMapping(SimpleMapping): 

216 """Mapping for CoaddTempExp datasets.""" 

217 

218 template = "{dataset}-{tract:02d}-{patch}-{visit:04d}{ext}" 

219 keys = dict(tract=int, patch=str, visit=int) 

220 

221 

222class ForcedSrcMapping(RawMapping): 

223 """Mapping for forced_src datasets.""" 

224 

225 template = "{dataset}-{tract:02d}-{visit:04d}-{ccd:01d}{ext}" 

226 keys = dict(tract=int, ccd=int, visit=int) 

227 

228 

229class MapperMeta(type): 

230 """Metaclass for SimpleMapper that creates map_ and query_ methods for everything found in the 

231 'mappings' class variable. 

232 """ 

233 

234 @staticmethod 

235 def _makeMapClosure(dataset, mapping, suffix=None): 

236 def mapClosure(self, dataId, write=False): 

237 return mapping.map(dataset, self.root, dataId, self, suffix=suffix, storage=self.storage) 

238 return mapClosure 

239 

240 @staticmethod 

241 def _makeQueryClosure(dataset, mapping): 

242 def queryClosure(self, level, format, dataId): 

243 return mapping.query(dataset, self.index, level, format, dataId) 

244 return queryClosure 

245 

246 def __init__(cls, name, bases, dict_): # noqa allow "cls" instead of "self" 

247 type.__init__(cls, name, bases, dict_) 

248 cls.keyDict = dict() 

249 for dataset, mapping in cls.mappings.items(): 

250 setattr(cls, "map_" + dataset, MapperMeta._makeMapClosure(dataset, mapping, suffix=None)) 

251 for suffix in mapping.persistence.suffixes: 

252 setattr(cls, "map_" + dataset + suffix, 

253 MapperMeta._makeMapClosure(dataset, mapping, suffix=suffix)) 

254 if hasattr(mapping, "query"): 

255 setattr(cls, "query_" + dataset, MapperMeta._makeQueryClosure(dataset, mapping)) 

256 cls.keyDict.update(mapping.keys) 

257 

258 

259class SimpleMapper(lsst.daf.persistence.Mapper, metaclass=MapperMeta): 

260 """ 

261 An extremely simple mapper for an imaginary camera for use in integration tests. 

262 

263 As SimpleMapper does not inherit from obs.base.CameraMapper, it does not 

264 use a policy file to set mappings or a registry; all the information is here 

265 (in the map_* and query_* methods). 

266 

267 The imaginary camera's raw data format has only 'visit' and 'ccd' keys, with 

268 two CCDs per visit (by default). 

269 """ 

270 

271 mappings = dict( 

272 calexp=RawMapping(ExposurePersistenceType), 

273 forced_src=ForcedSrcMapping(SourceCatalogPersistenceType), 

274 forced_src_schema=SimpleMapping(SourceCatalogPersistenceType, 

275 template="{dataset}{ext}", keys={}), 

276 truth=SimpleMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}", 

277 keys={"tract": int}), 

278 simsrc=RawMapping(SimpleCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}", 

279 keys={"tract": int}), 

280 observations=SimpleMapping(ExposureCatalogPersistenceType, template="{dataset}-{tract:02d}{ext}", 

281 keys={"tract": int}), 

282 ccdExposureId=RawMapping(BypassPersistenceType), 

283 ccdExposureId_bits=SimpleMapping(BypassPersistenceType), 

284 deepCoaddId=SkyMapping(BypassPersistenceType), 

285 deepCoaddId_bits=SimpleMapping(BypassPersistenceType), 

286 deepCoadd_band=SimpleMapping(BypassPersistenceType), 

287 deepMergedCoaddId=SkyMapping(BypassPersistenceType), 

288 deepMergedCoaddId_bits=SimpleMapping(BypassPersistenceType), 

289 deepCoadd_skyMap=SimpleMapping(SkyMapPersistenceType, template="{dataset}{ext}", keys={}), 

290 deepCoadd=SkyMapping(ExposurePersistenceType), 

291 deepCoadd_filterLabel=SkyMapping(ExposurePersistenceType), 

292 deepCoaddPsfMatched=SkyMapping(ExposurePersistenceType), 

293 deepCoadd_calexp=SkyMapping(ExposurePersistenceType), 

294 deepCoadd_calexp_background=SkyMapping(CatalogPersistenceType), 

295 deepCoadd_icSrc=SkyMapping(SourceCatalogPersistenceType), 

296 deepCoadd_icSrc_schema=SimpleMapping(SourceCatalogPersistenceType, 

297 template="{dataset}{ext}", keys={}), 

298 deepCoadd_src=SkyMapping(SourceCatalogPersistenceType), 

299 deepCoadd_src_schema=SimpleMapping(SourceCatalogPersistenceType, 

300 template="{dataset}{ext}", keys={}), 

301 deepCoadd_peak_schema=SimpleMapping(PeakCatalogPersistenceType, 

302 template="{dataset}{ext}", keys={}), 

303 deepCoadd_ref=SkyMapping(SourceCatalogPersistenceType), 

304 deepCoadd_ref_schema=SimpleMapping(SourceCatalogPersistenceType, 

305 template="{dataset}{ext}", keys={}), 

306 deepCoadd_det=SkyMapping(SourceCatalogPersistenceType), 

307 deepCoadd_det_schema=SimpleMapping(SourceCatalogPersistenceType, 

308 template="{dataset}{ext}", keys={}), 

309 deepCoadd_mergeDet=SkyMapping(SourceCatalogPersistenceType), 

310 deepCoadd_mergeDet_schema=SimpleMapping(SourceCatalogPersistenceType, 

311 template="{dataset}{ext}", keys={}), 

312 deepCoadd_deblendedFlux=SkyMapping(SourceCatalogPersistenceType), 

313 deepCoadd_deblendedFlux_schema=SimpleMapping(SourceCatalogPersistenceType, 

314 template="{dataset}{ext}", keys={}), 

315 deepCoadd_deblendedModel=SkyMapping(SourceCatalogPersistenceType), 

316 deepCoadd_deblendedModel_schema=SimpleMapping(SourceCatalogPersistenceType, 

317 template="{dataset}{ext}", keys={}), 

318 deepCoadd_meas=SkyMapping(SourceCatalogPersistenceType), 

319 deepCoadd_meas_schema=SimpleMapping(SourceCatalogPersistenceType, 

320 template="{dataset}{ext}", keys={}), 

321 deepCoadd_forced_src=SkyMapping(SourceCatalogPersistenceType), 

322 deepCoadd_forced_src_schema=SimpleMapping(SourceCatalogPersistenceType, 

323 template="{dataset}{ext}", keys={}), 

324 deepCoadd_mock=SkyMapping(ExposurePersistenceType), 

325 deepCoaddPsfMatched_mock=SkyMapping(ExposurePersistenceType), 

326 deepCoadd_directWarp=TempExpMapping(ExposurePersistenceType), 

327 deepCoadd_directWarp_mock=TempExpMapping(ExposurePersistenceType), 

328 deepCoadd_psfMatchedWarp=TempExpMapping(ExposurePersistenceType), 

329 deepCoadd_psfMatchedWarp_mock=TempExpMapping(ExposurePersistenceType), 

330 ) 

331 

332 levels = dict( 

333 visit=['ccd'], 

334 ccd=[], 

335 ) 

336 

337 def __init__(self, root, **kwargs): 

338 self.storage = lsst.daf.persistence.Storage.makeFromURI(root) 

339 super(SimpleMapper, self).__init__(**kwargs) 

340 self.root = root 

341 self.camera = makeSimpleCamera(nX=1, nY=2, sizeX=400, sizeY=200, gapX=2, gapY=2) 

342 # NOTE: we set band/physical here because CoaddsTestCase.testCoaddInputs() 

343 # expects "r" for both the input dataIds (gen2 dataIds could be either 

344 # physical or band) and the output CoaddInputRecords. 

345 # The original data had just `Filter("r")`, so this has always assumed 

346 # that physicalLabel==bandLabel, even though CoaddInputRecords are physical. 

347 self.filterLabel = afwImage.FilterLabel(band="r", physical="r") 

348 self.update() 

349 

350 def getDefaultLevel(self): 

351 return "ccd" 

352 

353 def getKeys(self, datasetType, level): 

354 if datasetType is None: 

355 keyDict = self.keyDict 

356 else: 

357 keyDict = self.mappings[datasetType].keys 

358 if level is not None and level in self.levels: 

359 keyDict = dict(keyDict) 

360 for lev in self.levels[level]: 

361 if lev in keyDict: 

362 del keyDict[lev] 

363 return keyDict 

364 

365 def update(self): 

366 filenames = os.listdir(self.root) 

367 rawRegex = re.compile(r"(?P<dataset>\w+)-(?P<visit>\d+)-(?P<ccd>\d).*") 

368 self.index = {} 

369 for filename in filenames: 

370 m = rawRegex.match(filename) 

371 if not m: 

372 continue 

373 index = self.index.setdefault(m.group('dataset'), dict(ccd={None: []}, visit={None: []})) 

374 visit = int(m.group('visit')) 

375 ccd = int(m.group('ccd')) 

376 d1 = dict(visit=visit, ccd=ccd) 

377 d2 = dict(visit=visit) 

378 index['ccd'].setdefault(visit, []).append(d1) 

379 index['ccd'][None].append(d1) 

380 index['visit'][visit] = [d2] 

381 index['visit'][None].append(d1) 

382 

383 def keys(self): 

384 return self.keyDict 

385 

386 def bypass_camera(self, datasetType, pythonType, location, dataId): 

387 return self.camera 

388 

389 def map_camera(self, dataId, write=False): 

390 return lsst.daf.persistence.ButlerLocation( 

391 "lsst.afw.cameraGeom.Camera", "Camera", None, [], dataId, mapper=self, storage=self.storage 

392 ) 

393 

394 def std_calexp(self, item, dataId): 

395 detectorId = dataId["ccd"] 

396 detector = self.camera[detectorId] 

397 item.setDetector(detector) 

398 item.setFilterLabel(self.filterLabel) 

399 return item 

400 

401 def _computeCcdExposureId(self, dataId): 

402 return int(dataId["visit"]) * 10 + int(dataId["ccd"]) 

403 

404 def _computeCoaddId(self, dataId): 

405 # Note: for real IDs, we'd want to include filter here, but it doesn't actually matter 

406 # for any of the tests we've done so far, which all assume filter='r' 

407 tract = int(dataId['tract']) 

408 if tract < 0 or tract >= 128: 

409 raise RuntimeError('tract not in range [0,128)') 

410 patchX, patchY = (int(c) for c in dataId['patch'].split(',')) 

411 for p in (patchX, patchY): 

412 if p < 0 or p >= 2**13: 

413 raise RuntimeError('patch component not in range [0, 8192)') 

414 return (tract * 2**13 + patchX) * 2**13 + patchY 

415 

416 @staticmethod 

417 def splitCcdExposureId(ccdExposureId): 

418 return dict(visit=(int(ccdExposureId) // 10), ccd=(int(ccdExposureId) % 10)) 

419 

420 def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId): 

421 return self._computeCcdExposureId(dataId) 

422 

423 def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId): 

424 return 32 

425 

426 def bypass_deepCoaddId(self, datasetType, pythonType, location, dataId): 

427 return self._computeCoaddId(dataId) 

428 

429 def bypass_deepCoaddId_bits(self, datasetType, pythonType, location, dataId): 

430 return 1 + 7 + 13*2 + 3 

431 

432 def bypass_deepCoadd_band(self, datasetType, pythonType, location, dataId): 

433 return self.filterLabel.bandLabel 

434 

435 def bypass_deepMergedCoaddId(self, datasetType, pythonType, location, dataId): 

436 return self._computeCoaddId(dataId) 

437 

438 def bypass_deepMergedCoaddId_bits(self, datasetType, pythonType, location, dataId): 

439 return 1 + 7 + 13*2 + 3 

440 

441 def bypass_deepCoadd_filterLabel(self, *args, **kwargs): 

442 """To return a useful filterLabel for MergeDetectionsTask.""" 

443 return afwImage.FilterLabel(band=self.filterLabel.bandLabel) 

444 

445 

446def makeSimpleCamera( 

447 nX, nY, 

448 sizeX, sizeY, 

449 gapX, gapY, 

450 pixelSize=1.0, 

451 plateScale=20.0, 

452 radialDistortion=0.925, 

453): 

454 """Create a camera 

455 

456 @param[in] nx: number of detectors in x 

457 @param[in] ny: number of detectors in y 

458 @param[in] sizeX: detector size in x (pixels) 

459 @param[in] sizeY: detector size in y (pixels) 

460 @param[in] gapX: gap between detectors in x (mm) 

461 @param[in] gapY: gap between detectors in y (mm) 

462 @param[in] pixelSize: pixel size (mm) (a float) 

463 @param[in] plateScale: plate scale in arcsec/mm; 20.0 is for LSST 

464 @param[in] radialDistortion: radial distortion, in mm/rad^2 

465 (the r^3 coefficient of the radial distortion polynomial 

466 that converts FIELD_ANGLE in radians to FOCAL_PLANE in mm); 

467 0.925 is the value Dave Monet measured for lsstSim data 

468 

469 Each detector will have one amplifier (with no raw information). 

470 """ 

471 pScaleRad = lsst.geom.arcsecToRad(plateScale) 

472 radialDistortCoeffs = [0.0, 1.0/pScaleRad, 0.0, radialDistortion/pScaleRad] 

473 focalPlaneToFieldAngle = lsst.afw.geom.makeRadialTransform(radialDistortCoeffs) 

474 

475 ccdBBox = lsst.geom.Box2I(lsst.geom.Point2I(), lsst.geom.Extent2I(sizeX, sizeY)) 

476 

477 cameraBuilder = lsst.afw.cameraGeom.Camera.Builder("Simple Camera") 

478 

479 detectorId = 0 

480 for iY in range(nY): 

481 cY = (iY - 0.5 * (nY - 1)) * (pixelSize * sizeY + gapY) 

482 for iX in range(nX): 

483 cX = (iX - 0.5 * (nX - 1)) * (pixelSize * sizeY + gapX) 

484 fpPos = lsst.geom.Point2D(cX, cY) 

485 detectorName = "detector %d,%d" % (iX, iY) 

486 

487 detectorBuilder = cameraBuilder.add(detectorName, detectorId) 

488 detectorBuilder.setSerial(detectorName + " serial") 

489 detectorBuilder.setBBox(ccdBBox) 

490 detectorBuilder.setOrientation(lsst.afw.cameraGeom.Orientation(fpPos)) 

491 detectorBuilder.setPixelSize(lsst.geom.Extent2D(pixelSize, pixelSize)) 

492 

493 ampBuilder = lsst.afw.cameraGeom.Amplifier.Builder() 

494 ampName = "amp" 

495 ampBuilder.setName(ampName) 

496 ampBuilder.setBBox(ccdBBox) 

497 ampBuilder.setGain(1.0) 

498 ampBuilder.setReadNoise(5.0) 

499 

500 detectorBuilder.append(ampBuilder) 

501 

502 detectorId += 1 

503 

504 cameraBuilder.setTransformFromFocalPlaneTo(lsst.afw.cameraGeom.FIELD_ANGLE, focalPlaneToFieldAngle) 

505 return cameraBuilder.finish() 

506 

507 

508def makeDataRepo(root): 

509 """ 

510 Create a data repository for SimpleMapper and return a butler for it. 

511 

512 Clobbers anything already in the given path. 

513 """ 

514 if os.path.exists(root): 

515 shutil.rmtree(root) 

516 os.makedirs(root) 

517 with open(os.path.join(root, "_mapper"), "w") as f: 

518 f.write("lsst.pipe.tasks.mocks.SimpleMapper\n") 

519 return lsst.daf.persistence.Butler(root=root)