Coverage for tests/test_imageMapReduce.py: 15%

283 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-03-24 03:48 -0700

1# 

2# LSST Data Management System 

3# Copyright 2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21 

22import unittest 

23import numpy as np 

24 

25import lsst.utils.tests 

26import lsst.afw.image as afwImage 

27import lsst.afw.math as afwMath 

28import lsst.afw.geom as afwGeom 

29import lsst.daf.base as dafBase 

30import lsst.geom as geom 

31import lsst.meas.algorithms as measAlg 

32import lsst.pex.config as pexConfig 

33import lsst.pipe.base as pipeBase 

34 

35from lsst.ip.diffim.imageMapReduce import (ImageMapReduceTask, ImageMapReduceConfig, 

36 ImageMapper, ImageMapperConfig) 

37 

38 

39def setup_module(module): 

40 lsst.utils.tests.init() 

41 

42 

43def makeWcs(offset=0): 

44 # taken from $AFW_DIR/tests/testMakeWcs.py 

45 metadata = dafBase.PropertySet() 

46 metadata.set("SIMPLE", "T") 

47 metadata.set("BITPIX", -32) 

48 metadata.set("NAXIS", 2) 

49 metadata.set("NAXIS1", 1024) 

50 metadata.set("NAXIS2", 1153) 

51 metadata.set("RADESYS", 'FK5') 

52 metadata.set("EQUINOX", 2000.) 

53 metadata.setDouble("CRVAL1", 215.604025685476) 

54 metadata.setDouble("CRVAL2", 53.1595451514076) 

55 metadata.setDouble("CRPIX1", 1109.99981456774 + offset) 

56 metadata.setDouble("CRPIX2", 560.018167811613 + offset) 

57 metadata.set("CTYPE1", 'RA---SIN') 

58 metadata.set("CTYPE2", 'DEC--SIN') 

59 metadata.setDouble("CD1_1", 5.10808596133527E-05) 

60 metadata.setDouble("CD1_2", 1.85579539217196E-07) 

61 metadata.setDouble("CD2_2", -5.10281493481982E-05) 

62 metadata.setDouble("CD2_1", -8.27440751733828E-07) 

63 return afwGeom.makeSkyWcs(metadata) 

64 

65 

66def getPsfMoments(psfArray): 

67 # Borrowed and modified from meas_algorithms/testCoaddPsf 

68 sumx2 = sumy2 = sumy = sumx = sumf = 0.0 

69 for x in range(psfArray.shape[0]): 

70 for y in range(psfArray.shape[1]): 

71 f = psfArray[x, y] 

72 sumx2 += x*x*f 

73 sumy2 += y*y*f 

74 sumx += x*f 

75 sumy += y*f 

76 sumf += f 

77 xbar = sumx/sumf 

78 ybar = sumy/sumf 

79 mxx = sumx2 - 2*xbar*sumx + xbar*xbar*sumf 

80 myy = sumy2 - 2*ybar*sumy + ybar*ybar*sumf 

81 return sumf, xbar, ybar, mxx, myy 

82 

83 

84def getPsfSecondMoments(psfArray): 

85 sum, xbar, ybar, mxx, myy = getPsfMoments(psfArray) 

86 return mxx, myy 

87 

88 

89class AddAmountImageMapperConfig(ImageMapperConfig): 

90 """Configuration parameters for the AddAmountImageMapper 

91 """ 

92 addAmount = pexConfig.Field( 

93 dtype=float, 

94 doc="Amount to add to image", 

95 default=10. 

96 ) 

97 

98 

99class AddAmountImageMapper(ImageMapper): 

100 """Image mapper subTask that adds a constant value to the input subexposure 

101 """ 

102 ConfigClass = AddAmountImageMapperConfig 

103 _DefaultName = "ip_diffim_AddAmountImageMapper" 

104 

105 def run(self, subExposure, expandedSubExp, fullBBox, addNans=False, **kwargs): 

106 """Add `addAmount` to given `subExposure`. 

107 

108 Optionally add NaNs to check the NaN-safe 'copy' operation. 

109 

110 Parameters 

111 ---------- 

112 subExposure : `afwImage.Exposure` 

113 Input `subExposure` upon which to operate 

114 expandedSubExp : `afwImage.Exposure` 

115 Input expanded subExposure (not used here) 

116 fullBBox : `lsst.geom.Box2I` 

117 Bounding box of original exposure (not used here) 

118 addNaNs : boolean 

119 Set a single pixel of `subExposure` to `np.nan` 

120 kwargs 

121 Arbitrary keyword arguments (ignored) 

122 

123 Returns 

124 ------- 

125 `pipeBase.Struct` containing (with name 'subExposure') the 

126 copy of `subExposure` to which `addAmount` has been added 

127 """ 

128 subExp = subExposure.clone() 

129 img = subExp.getMaskedImage() 

130 img += self.config.addAmount 

131 if addNans: 

132 img.getImage().getArray()[0, 0] = np.nan 

133 return pipeBase.Struct(subExposure=subExp) 

134 

135 

136class AddAmountImageMapReduceConfig(ImageMapReduceConfig): 

137 """Configuration parameters for the AddAmountImageMapReduceTask 

138 """ 

139 mapper = pexConfig.ConfigurableField( 

140 doc="Mapper subtask to run on each subimage", 

141 target=AddAmountImageMapper, 

142 ) 

143 

144 

145class GetMeanImageMapper(ImageMapper): 

146 """ImageMapper subtask that computes and returns the mean value of the 

147 input sub-exposure 

148 """ 

149 ConfigClass = AddAmountImageMapperConfig # Doesn't need its own config 

150 _DefaultName = "ip_diffim_GetMeanImageMapper" 

151 

152 def run(self, subExposure, expandedSubExp, fullBBox, **kwargs): 

153 """Compute the mean of the given `subExposure` 

154 

155 Parameters 

156 ---------- 

157 subExposure : `afwImage.Exposure` 

158 Input `subExposure` upon which to operate 

159 expandedSubExp : `afwImage.Exposure` 

160 Input expanded subExposure (not used here) 

161 fullBBox : `lsst.geom.Box2I` 

162 Bounding box of original exposure (not used here) 

163 kwargs 

164 Arbitrary keyword arguments (ignored) 

165 

166 Returns 

167 ------- 

168 `pipeBase.Struct` containing the mean value of `subExposure` 

169 image plane. We name it 'subExposure' to enable the correct 

170 test in `testNotNoneReduceWithNonExposureMapper`. In real 

171 operations, use something like 'mean' for the name. 

172 """ 

173 subMI = subExposure.getMaskedImage() 

174 statObj = afwMath.makeStatistics(subMI, afwMath.MEAN) 

175 return pipeBase.Struct(subExposure=statObj.getValue()) 

176 

177 

178class GetMeanImageMapReduceConfig(ImageMapReduceConfig): 

179 """Configuration parameters for the GetMeanImageMapReduceTask 

180 """ 

181 mapper = pexConfig.ConfigurableField( 

182 doc="Mapper subtask to run on each subimage", 

183 target=GetMeanImageMapper, 

184 ) 

185 

186 

187class ImageMapReduceTest(lsst.utils.tests.TestCase): 

188 """A test case for the image gridded processing task 

189 """ 

190 def setUp(self): 

191 self.longMessage = True 

192 self._makeImage() 

193 

194 def tearDown(self): 

195 del self.exposure 

196 

197 def _makeImage(self): 

198 self.exposure = afwImage.ExposureF(128, 128) 

199 self.exposure.setPsf(measAlg.DoubleGaussianPsf(11, 11, 2.0, 3.7)) 

200 mi = self.exposure.getMaskedImage() 

201 mi.set(0.) 

202 self.exposure.setWcs(makeWcs()) # required for PSF construction via CoaddPsf 

203 

204 def testCopySumNoOverlaps(self): 

205 self._testCopySumNoOverlaps(reduceOp='copy', withNaNs=False) 

206 self._testCopySumNoOverlaps(reduceOp='copy', withNaNs=True) 

207 self._testCopySumNoOverlaps(reduceOp='sum', withNaNs=False) 

208 self._testCopySumNoOverlaps(reduceOp='sum', withNaNs=True) 

209 

210 def _testCopySumNoOverlaps(self, reduceOp='copy', withNaNs=False): 

211 """Test sample grid task that adds 5.0 to input image and uses 

212 `reduceOperation = 'copy'`. Optionally add NaNs to subimages. 

213 """ 

214 config = AddAmountImageMapReduceConfig() 

215 task = ImageMapReduceTask(config) 

216 config.mapper.addAmount = 5. 

217 config.reducer.reduceOperation = reduceOp 

218 newExp = task.run(self.exposure, addNans=withNaNs).exposure 

219 newMI = newExp.getMaskedImage() 

220 newArr = newMI.getImage().getArray() 

221 isnan = np.isnan(newArr) 

222 if not withNaNs: 

223 self.assertEqual(np.sum(isnan), 0, 

224 msg='Failed on withNaNs: %s' % str(withNaNs)) 

225 

226 mi = self.exposure.getMaskedImage().getImage().getArray() 

227 if reduceOp != 'sum': 

228 self.assertFloatsAlmostEqual(mi[~isnan], newArr[~isnan] - 5., 

229 msg='Failed on withNaNs: %s' % str(withNaNs)) 

230 else: # We don't construct a new PSF if reduceOperation == 'copy'. 

231 self._testCoaddPsf(newExp) 

232 

233 def testAverageWithOverlaps(self): 

234 self._testAverageWithOverlaps(withNaNs=False) 

235 self._testAverageWithOverlaps(withNaNs=True) 

236 

237 def _testAverageWithOverlaps(self, withNaNs=False): 

238 """Test sample grid task that adds 5.0 to input image and uses 

239 'average' `reduceOperation`. Optionally add NaNs to subimages. 

240 """ 

241 config = AddAmountImageMapReduceConfig() 

242 config.gridStepX = config.gridStepY = 8. 

243 config.reducer.reduceOperation = 'average' 

244 task = ImageMapReduceTask(config) 

245 config.mapper.addAmount = 5. 

246 newExp = task.run(self.exposure, addNans=withNaNs).exposure 

247 newMI = newExp.getMaskedImage() 

248 newArr = newMI.getImage().getArray() 

249 mi = self.exposure.getMaskedImage() 

250 isnan = np.isnan(newArr) 

251 if not withNaNs: 

252 self.assertEqual(np.sum(isnan), 0, 

253 msg='Failed on withNaNs: %s' % str(withNaNs)) 

254 

255 mi = self.exposure.getMaskedImage().getImage().getArray() 

256 self.assertFloatsAlmostEqual(mi[~isnan], newArr[~isnan] - 5., 

257 msg='Failed on withNaNs: %s' % str(withNaNs)) 

258 self._testCoaddPsf(newExp) 

259 

260 def _testCoaddPsf(self, newExposure): 

261 """Test that the new CoaddPsf of the `newExposure` returns PSF images 

262 ~identical to the input PSF of `self.exposure` across a grid 

263 covering the entire exposure bounding box. 

264 """ 

265 origPsf = self.exposure.getPsf() 

266 newPsf = newExposure.getPsf() 

267 self.assertTrue(isinstance(newPsf, measAlg.CoaddPsf)) 

268 extentX = int(self.exposure.getWidth()*0.05) 

269 extentY = int(self.exposure.getHeight()*0.05) 

270 for x in np.linspace(extentX, self.exposure.getWidth()-extentX, 10): 

271 for y in np.linspace(extentY, self.exposure.getHeight()-extentY, 10): 

272 point = geom.Point2D(np.rint(x), np.rint(y)) 

273 oPsf = origPsf.computeImage(point).getArray() 

274 nPsf = newPsf.computeImage(point).getArray() 

275 if oPsf.shape[0] < nPsf.shape[0]: # sometimes CoaddPsf does this. 

276 oPsf = np.pad(oPsf, ((1, 1), (0, 0)), mode='constant') 

277 elif oPsf.shape[0] > nPsf.shape[0]: 

278 nPsf = np.pad(nPsf, ((1, 1), (0, 0)), mode='constant') 

279 if oPsf.shape[1] < nPsf.shape[1]: # sometimes CoaddPsf does this. 

280 oPsf = np.pad(oPsf, ((0, 0), (1, 1)), mode='constant') 

281 elif oPsf.shape[1] > nPsf.shape[1]: 

282 nPsf = np.pad(nPsf, ((0, 0), (1, 1)), mode='constant') 

283 # pixel-wise comparison -- pretty stringent 

284 self.assertFloatsAlmostEqual(oPsf, nPsf, atol=1e-4, msg='Failed on Psf') 

285 

286 origMmts = np.array(getPsfSecondMoments(oPsf)) 

287 newMmts = np.array(getPsfSecondMoments(nPsf)) 

288 self.assertFloatsAlmostEqual(origMmts, newMmts, atol=1e-4, msg='Failed on Psf') 

289 

290 def testAverageVersusCopy(self): 

291 self._testAverageVersusCopy(withNaNs=False) 

292 self._testAverageVersusCopy(withNaNs=True) 

293 

294 def _testAverageVersusCopy(self, withNaNs=False): 

295 """Re-run `testExampleTaskNoOverlaps` and `testExampleTaskWithOverlaps` 

296 on a more complex image (with random noise). Ensure that the results are 

297 identical (within between 'copy' and 'average' reduceOperation. 

298 """ 

299 exposure1 = self.exposure.clone() 

300 img = exposure1.getMaskedImage().getImage() 

301 afwMath.randomGaussianImage(img, afwMath.Random()) 

302 exposure2 = exposure1.clone() 

303 

304 config = AddAmountImageMapReduceConfig() 

305 task = ImageMapReduceTask(config) 

306 config.mapper.addAmount = 5. 

307 newExp = task.run(exposure1, addNans=withNaNs).exposure 

308 newMI1 = newExp.getMaskedImage() 

309 

310 config.gridStepX = config.gridStepY = 8. 

311 config.reducer.reduceOperation = 'average' 

312 task = ImageMapReduceTask(config) 

313 newExp = task.run(exposure2, addNans=withNaNs).exposure 

314 newMI2 = newExp.getMaskedImage() 

315 

316 newMA1 = newMI1.getImage().getArray() 

317 isnan = np.isnan(newMA1) 

318 if not withNaNs: 

319 self.assertEqual(np.sum(isnan), 0) 

320 newMA2 = newMI2.getImage().getArray() 

321 

322 # Because the average uses a float accumulator, we can have differences, set a tolerance. 

323 # Turns out (in practice for this test), only 7 pixels seem to have a small difference. 

324 self.assertFloatsAlmostEqual(newMA1[~isnan], newMA2[~isnan], rtol=1e-7) 

325 

326 def testMean(self): 

327 """Test sample grid task that returns the mean of the subimages and uses 

328 'none' `reduceOperation`. 

329 """ 

330 config = GetMeanImageMapReduceConfig() 

331 config.reducer.reduceOperation = 'none' 

332 task = ImageMapReduceTask(config) 

333 testExposure = self.exposure.clone() 

334 testExposure.getMaskedImage().set(1.234) 

335 subMeans = task.run(testExposure).result 

336 subMeans = [x.subExposure for x in subMeans] 

337 

338 self.assertEqual(len(subMeans), len(task.boxes0)) 

339 firstPixel = testExposure.getMaskedImage().getImage().getArray()[0, 0] 

340 self.assertFloatsAlmostEqual(np.array(subMeans), firstPixel) 

341 

342 def testCellCentroids(self): 

343 """Test sample grid task which is provided a set of `cellCentroids` and 

344 returns the mean of the subimages surrounding those centroids using 'none' 

345 for `reduceOperation`. 

346 """ 

347 config = GetMeanImageMapReduceConfig() 

348 config.gridStepX = config.gridStepY = 8. 

349 config.reducer.reduceOperation = 'none' 

350 config.cellCentroidsX = [i for i in np.linspace(0, 128, 50)] 

351 config.cellCentroidsY = config.cellCentroidsX 

352 task = ImageMapReduceTask(config) 

353 testExposure = self.exposure.clone() 

354 testExposure.getMaskedImage().set(1.234) 

355 subMeans = task.run(testExposure).result 

356 subMeans = [x.subExposure for x in subMeans] 

357 

358 self.assertEqual(len(subMeans), len(config.cellCentroidsX)) 

359 firstPixel = testExposure.getMaskedImage().getImage().getArray()[0, 0] 

360 self.assertFloatsAlmostEqual(np.array(subMeans), firstPixel) 

361 

362 def testCellCentroidsWrongLength(self): 

363 """Test sample grid task which is provided a set of `cellCentroids` and 

364 returns the mean of the subimages surrounding those centroids using 'none' 

365 for `reduceOperation`. In this case, we ensure that len(task.boxes0) != 

366 len(task.boxes1) and check for ValueError. 

367 """ 

368 config = GetMeanImageMapReduceConfig() 

369 config.reducer.reduceOperation = 'none' 

370 config.cellCentroidsX = [i for i in np.linspace(0, 128, 50)] 

371 config.cellCentroidsY = [i for i in np.linspace(0, 128, 50)] 

372 task = ImageMapReduceTask(config) 

373 task._generateGrid(self.exposure) 

374 del task.boxes0[-1] # remove the last box 

375 with self.assertRaises(ValueError): 

376 task.run(self.exposure) 

377 

378 def testMasks(self): 

379 """Test the mask for an exposure produced by a sample grid task 

380 where we provide a set of `cellCentroids` and thus should have 

381 many invalid pixels. 

382 """ 

383 config = AddAmountImageMapReduceConfig() 

384 config.gridStepX = config.gridStepY = 8. 

385 config.cellCentroidsX = [i for i in np.linspace(0, 128, 50)] 

386 config.cellCentroidsY = config.cellCentroidsX 

387 config.reducer.reduceOperation = 'average' 

388 task = ImageMapReduceTask(config) 

389 config.mapper.addAmount = 5. 

390 newExp = task.run(self.exposure).exposure 

391 newMI = newExp.getMaskedImage() 

392 newArr = newMI.getImage().getArray() 

393 mi = self.exposure.getMaskedImage() 

394 isnan = np.isnan(newArr) 

395 self.assertGreater(np.sum(isnan), 1000) 

396 

397 mi = self.exposure.getMaskedImage().getImage().getArray() 

398 self.assertFloatsAlmostEqual(mi[~isnan], newArr[~isnan] - 5.) 

399 

400 mask = newMI.getMask() # Now check the mask 

401 self.assertGreater(mask.getMaskPlane('INVALID_MAPREDUCE'), 0) 

402 maskBit = mask.getPlaneBitMask('INVALID_MAPREDUCE') 

403 nMasked = np.sum(np.bitwise_and(mask.getArray(), maskBit) != 0) 

404 self.assertGreater(nMasked, 1000) 

405 self.assertEqual(np.sum(np.isnan(newArr)), nMasked) 

406 

407 def testNotNoneReduceWithNonExposureMapper(self): 

408 """Test that a combination of a mapper that returns a non-exposure 

409 cannot work correctly with a reducer with reduceOperation='none'. 

410 Should raise a TypeError. 

411 """ 

412 config = GetMeanImageMapReduceConfig() # mapper returns a float (mean) 

413 config.gridStepX = config.gridStepY = 8. 

414 config.reducer.reduceOperation = 'average' # not 'none'! 

415 task = ImageMapReduceTask(config) 

416 with self.assertRaises(TypeError): 

417 task.run(self.exposure) 

418 

419 def testGridValidity(self): 

420 """Test sample grids with various spacings and sizes and other options. 

421 """ 

422 expectedVal = 1. 

423 n_tests = 0 

424 

425 for reduceOp in ('copy', 'average'): 

426 for adjustGridOption in ('spacing', 'size', 'none'): 

427 for gstepx in range(11, 3, -4): 

428 for gsizex in gstepx + np.array([0, 1, 2]): 

429 for gstepy in range(11, 3, -4): 

430 for gsizey in gstepy + np.array([0, 1, 2]): 

431 config = AddAmountImageMapReduceConfig() 

432 config.reducer.reduceOperation = reduceOp 

433 n_tests += 1 

434 self._runGridValidity(config, gstepx, gsizex, 

435 gstepy, gsizey, adjustGridOption, 

436 expectedVal) 

437 print("Ran a total of %d grid validity tests." % n_tests) 

438 

439 def _runGridValidity(self, config, gstepx, gsizex, gstepy, gsizey, 

440 adjustGridOption, expectedVal=1.): 

441 """Method to test the grid validity given an input config. 

442 

443 Here we also iterate over scaleByFwhm in (True, False) and 

444 ensure that we get more `boxes` when `scaleByFwhm=False` than 

445 vice versa. 

446 

447 Parameters 

448 ---------- 

449 config : `ipDiffim.AddAmountImageMapReduceConfig` 

450 input AddAmountImageMapReduceConfig 

451 gstepx : `float` 

452 grid x-direction step size 

453 gsizex : `float` 

454 grid x-direction box size 

455 gstepy : `float` 

456 grid y-direction step size 

457 gsizey : `float` 

458 grid y-direction box size 

459 expectedVal : `float` 

460 float to add to exposure (to compare for testing) 

461 """ 

462 config.mapper.addAmount = expectedVal 

463 lenBoxes = [0, 0] 

464 for scaleByFwhm in (True, False): 

465 config.scaleByFwhm = scaleByFwhm 

466 if scaleByFwhm: 

467 config.gridStepX = float(gstepx) 

468 config.cellSizeX = float(gsizex) 

469 config.gridStepY = float(gstepy) 

470 config.cellSizeY = float(gsizey) 

471 else: # otherwise the grid is too fine and elements too small. 

472 config.gridStepX = gstepx * 3. 

473 config.cellSizeX = gsizex * 3. 

474 config.gridStepY = gstepy * 3. 

475 config.cellSizeY = gsizey * 3. 

476 config.adjustGridOption = adjustGridOption 

477 task = ImageMapReduceTask(config) 

478 task._generateGrid(self.exposure) 

479 ind = 0 if scaleByFwhm else 1 

480 lenBoxes[ind] = len(task.boxes0) 

481 newExp = task.run(self.exposure).exposure 

482 newMI = newExp.getMaskedImage() 

483 newArr = newMI.getImage().getArray() 

484 isnan = np.isnan(newArr) 

485 self.assertEqual(np.sum(isnan), 0, msg='Failed NaN (%d), on config: %s' % 

486 (np.sum(isnan), str(config))) 

487 

488 mi = self.exposure.getMaskedImage().getImage().getArray() 

489 self.assertFloatsAlmostEqual(mi[~isnan], newArr[~isnan] - expectedVal, 

490 msg='Failed on config: %s' % str(config)) 

491 

492 self.assertLess(lenBoxes[0], lenBoxes[1], msg='Failed lengths on config: %s' % 

493 str(config)) 

494 

495 

496class MemoryTester(lsst.utils.tests.MemoryTestCase): 

497 pass 

498 

499 

500if __name__ == "__main__": 500 ↛ 501line 500 didn't jump to line 501, because the condition on line 500 was never true

501 lsst.utils.tests.init() 

502 unittest.main()