Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2016-2017 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21import unittest 

22 

23import numpy as np 

24 

25import lsst.utils.tests 

26import lsst.afw.image as afwImage 

27import lsst.afw.geom as afwGeom 

28import lsst.afw.math as afwMath 

29import lsst.geom as geom 

30import lsst.meas.algorithms as measAlg 

31import lsst.daf.base as dafBase 

32 

33from lsst.ip.diffim.imageDecorrelation import (DecorrelateALKernelTask, 

34 DecorrelateALKernelMapReduceConfig, 

35 DecorrelateALKernelSpatialConfig, 

36 DecorrelateALKernelSpatialTask) 

37from lsst.ip.diffim.imageMapReduce import ImageMapReduceTask 

38 

39try: 

40 type(verbose) 

41except NameError: 

42 verbose = False 

43 

44 

45def setup_module(module): 

46 lsst.utils.tests.init() 

47 

48 

49def singleGaussian2d(x, y, xc, yc, sigma_x=1., sigma_y=1., theta=0., ampl=1.): 

50 """! Generate a 2-d Gaussian, possibly elongated and rotated, on a grid of pixel 

51 coordinates given by x,y. 

52 @param x,y each a 1-d numpy.array containing x- and y- coordinates for independent variables, 

53 for example `np.arange(-16, 15)`. 

54 @param xc,yc each a float giving the centroid of the gaussian 

55 @param sigma_x,sigma_y each a float giving the sigma of the gaussian 

56 @param theta a float giving the rotation of the gaussian (degrees) 

57 @param ampl a float giving the amplitude of the gaussian 

58 @return a 2-d numpy.array containing the normalized 2-d Gaussian 

59 

60 @Note this can be done in `astropy.modeling` but for now we have it explicitly here. 

61 """ 

62 theta = (theta/180.) * np.pi 

63 cos_theta2, sin_theta2 = np.cos(theta)**2., np.sin(theta)**2. 

64 sigma_x2, sigma_y2 = sigma_x**2., sigma_y**2. 

65 a = cos_theta2/(2.*sigma_x2) + sin_theta2/(2.*sigma_y2) 

66 b = -(np.sin(2.*theta))/(4.*sigma_x2) + (np.sin(2.*theta))/(4.*sigma_y2) 

67 c = sin_theta2/(2.*sigma_x2) + cos_theta2/(2.*sigma_y2) 

68 xxc, yyc = x-xc, y-yc 

69 out = np.exp(-(a*(xxc**2.) + 2.*b*xxc*yyc + c*(yyc**2.))) 

70 out /= out.sum() 

71 return out 

72 

73 

74def makeFakeImages(size=(256, 256), svar=0.04, tvar=0.04, psf1=3.3, psf2=2.2, offset=None, 

75 psf_yvary_factor=0., varSourceChange=1/50., theta1=0., theta2=0., 

76 n_sources=500, seed=66, verbose=False): 

77 """! Make two exposures: a template and a science exposure. 

78 Add random sources with randomly-distributed and identical fluxes and a given PSF, then add noise. 

79 In all cases below, index (1) is the science image, and (2) is the template. 

80 @param size tuple givein image pixel size. Pixel coordinates are set to 

81 (-size[0]//2:size[0]//2, -size[1]//2:size[1]//2) 

82 @param svar,tar variance of noise to be generated on science/template images. Default is 0.04 for both. 

83 @param psf1,psf2 std. dev. of (Gaussian) PSFs for the two images in x,y direction. Default is 

84 [3.3, 3.3] and [2.2, 2.2] for im1 and im2 respectively. 

85 @param offset add a constant (pixel) astrometric offset between the two images 

86 @param psf_yvary_factor vary the y-width of the PSF across the x-axis of the science image (zero, 

87 the default, means no variation) 

88 @param varSourceChange add this amount of fractional flux to a single source closest to 

89 the center of the science image 

90 @param n_sources the number of sources to add to the images 

91 @param seed the numpy random seed to set prior to image generation 

92 @param verbose be verbose 

93 

94 @return im1, im2: the science and template afwImage.Exposures 

95 

96 @note having sources near the edges really messes up the 

97 fitting (probably because of the convolution). So we make sure no 

98 sources are near the edge. 

99 @note also it seems that having the variable source with a large 

100 flux increase also messes up the fitting (seems to lead to 

101 overfitting -- perhaps to the source itself). This might be fixed by 

102 adding more constant sources. 

103 """ 

104 np.random.seed(seed) 

105 

106 psf1 = [3.3, 3.3] if psf1 is None else psf1 

107 if not hasattr(psf1, "__len__") and not isinstance(psf1, str): 

108 psf1 = [psf1, psf1] 

109 psf2 = [2.2, 2.2] if psf2 is None else psf2 

110 if not hasattr(psf2, "__len__") and not isinstance(psf2, str): 

111 psf2 = [psf2, psf2] 

112 offset = [0., 0.] if offset is None else offset # astrometric offset (pixels) between the two images 

113 if verbose: 

114 print('Science PSF:', psf1, theta1) 

115 print('Template PSF:', psf2, theta2) 

116 print(np.sqrt(psf1[0]**2 - psf2[0]**2)) 

117 print('Offset:', offset) 

118 

119 xim = np.arange(-size[0]//2, size[0]//2, 1) 

120 yim = np.arange(-size[1]//2, size[1]//2, 1) 

121 x0im, y0im = np.meshgrid(yim, xim) 

122 im1 = np.random.normal(scale=np.sqrt(svar), size=x0im.shape) # variance of science image 

123 im2 = np.random.normal(scale=np.sqrt(tvar), size=x0im.shape) # variance of template 

124 

125 fluxes = np.random.uniform(50, 30000, n_sources) 

126 xposns = np.random.uniform(xim.min()+16, xim.max()-5, n_sources) 

127 yposns = np.random.uniform(yim.min()+16, yim.max()-5, n_sources) 

128 

129 # Make the source closest to the center of the image the one that increases in flux 

130 ind = np.argmin(xposns**2. + yposns**2.) 

131 

132 # vary the y-width of psf across x-axis of science image (zero means no variation): 

133 psf1_yvary = psf_yvary_factor * (yim.mean() - yposns) / yim.max() 

134 if verbose: 

135 print('PSF y spatial-variation:', psf1_yvary.min(), psf1_yvary.max()) 

136 

137 for i in range(n_sources): 

138 flux = fluxes[i] 

139 tmp = flux * singleGaussian2d(x0im, y0im, xposns[i], yposns[i], psf2[0], psf2[1], theta=theta2) 

140 im2 += tmp 

141 if i == ind: 

142 flux += flux * varSourceChange 

143 tmp = flux * singleGaussian2d(x0im, y0im, xposns[i]+offset[0], yposns[i]+offset[1], 

144 psf1[0], psf1[1]+psf1_yvary[i], theta=theta1) 

145 im1 += tmp 

146 

147 im1_psf = singleGaussian2d(x0im, y0im, 0, 0, psf1[0], psf1[1], theta=theta1) 

148 im2_psf = singleGaussian2d(x0im, y0im, offset[0], offset[1], psf2[0], psf2[1], theta=theta2) 

149 

150 def makeWcs(offset=0): 

151 """ Make a fake Wcs 

152 

153 Parameters 

154 ---------- 

155 offset : float 

156 offset the Wcs by this many pixels. 

157 """ 

158 # taken from $AFW_DIR/tests/testMakeWcs.py 

159 metadata = dafBase.PropertySet() 

160 metadata.set("SIMPLE", "T") 

161 metadata.set("BITPIX", -32) 

162 metadata.set("NAXIS", 2) 

163 metadata.set("NAXIS1", 1024) 

164 metadata.set("NAXIS2", 1153) 

165 metadata.set("RADESYS", 'FK5') 

166 metadata.set("EQUINOX", 2000.) 

167 metadata.setDouble("CRVAL1", 215.604025685476) 

168 metadata.setDouble("CRVAL2", 53.1595451514076) 

169 metadata.setDouble("CRPIX1", 1109.99981456774 + offset) 

170 metadata.setDouble("CRPIX2", 560.018167811613 + offset) 

171 metadata.set("CTYPE1", 'RA---SIN') 

172 metadata.set("CTYPE2", 'DEC--SIN') 

173 metadata.setDouble("CD1_1", 5.10808596133527E-05) 

174 metadata.setDouble("CD1_2", 1.85579539217196E-07) 

175 metadata.setDouble("CD2_2", -5.10281493481982E-05) 

176 metadata.setDouble("CD2_1", -8.27440751733828E-07) 

177 return afwGeom.makeSkyWcs(metadata) 

178 

179 def makeExposure(imgArray, psfArray, imgVariance): 

180 """! Convert an image numpy.array and corresponding PSF numpy.array into an exposure. 

181 

182 Add the (constant) variance plane equal to `imgVariance`. 

183 

184 @param imgArray 2-d numpy.array containing the image 

185 @param psfArray 2-d numpy.array containing the PSF image 

186 @param imgVariance variance of input image 

187 @return a new exposure containing the image, PSF and desired variance plane 

188 """ 

189 # All this code to convert the template image array/psf array into an exposure. 

190 bbox = geom.Box2I(geom.Point2I(0, 0), geom.Point2I(imgArray.shape[1]-1, imgArray.shape[0]-1)) 

191 im1ex = afwImage.ExposureD(bbox) 

192 im1ex.getMaskedImage().getImage().getArray()[:, :] = imgArray 

193 im1ex.getMaskedImage().getVariance().getArray()[:, :] = imgVariance 

194 psfBox = geom.Box2I(geom.Point2I(-12, -12), geom.Point2I(12, 12)) # a 25x25 pixel psf 

195 psf = afwImage.ImageD(psfBox) 

196 psfBox.shift(geom.Extent2I(size[0]//2, size[1]//2)) 

197 im1_psf_sub = psfArray[psfBox.getMinX():psfBox.getMaxX()+1, psfBox.getMinY():psfBox.getMaxY()+1] 

198 psf.getArray()[:, :] = im1_psf_sub 

199 psfK = afwMath.FixedKernel(psf) 

200 psfNew = measAlg.KernelPsf(psfK) 

201 im1ex.setPsf(psfNew) 

202 wcs = makeWcs() 

203 im1ex.setWcs(wcs) 

204 return im1ex 

205 

206 im1ex = makeExposure(im1, im1_psf, svar) # Science image 

207 im2ex = makeExposure(im2, im2_psf, tvar) # Template 

208 

209 return im1ex, im2ex 

210 

211 

212class DiffimCorrectionTest(lsst.utils.tests.TestCase): 

213 """!A test case for the diffim image decorrelation algorithm. 

214 """ 

215 

216 def setUp(self): 

217 self.psf1_sigma = 3.3 # sigma of psf of science image 

218 self.psf2_sigma = 2.2 # sigma of psf of template image 

219 

220 self.statsControl = afwMath.StatisticsControl() 

221 self.statsControl.setNumSigmaClip(3.) 

222 self.statsControl.setNumIter(3) 

223 self.statsControl.setAndMask(afwImage.Mask 

224 .getPlaneBitMask(["INTRP", "EDGE", "SAT", "CR", 

225 "DETECTED", "BAD", 

226 "NO_DATA", "DETECTED_NEGATIVE"])) 

227 

228 def _setUpImages(self, svar=0.04, tvar=0.04, varyPsf=0.): 

229 """!Generate a fake aligned template and science image. 

230 """ 

231 

232 self.svar = svar # variance of noise in science image 

233 self.tvar = tvar # variance of noise in template image 

234 

235 self.im1ex, self.im2ex \ 

236 = makeFakeImages(svar=self.svar, tvar=self.tvar, psf1=self.psf1_sigma, psf2=self.psf2_sigma, 

237 n_sources=50, psf_yvary_factor=varyPsf, verbose=False) 

238 

239 def _computeVarianceMean(self, maskedIm): 

240 statObj = afwMath.makeStatistics(maskedIm.getVariance(), 

241 maskedIm.getMask(), afwMath.MEANCLIP, 

242 self.statsControl) 

243 mn = statObj.getValue(afwMath.MEANCLIP) 

244 return mn 

245 

246 def _computePixelVariance(self, maskedIm): 

247 statObj = afwMath.makeStatistics(maskedIm, afwMath.VARIANCECLIP, 

248 self.statsControl) 

249 var = statObj.getValue(afwMath.VARIANCECLIP) 

250 return var 

251 

252 def tearDown(self): 

253 del self.im1ex 

254 del self.im2ex 

255 

256 def _makeAndTestUncorrectedDiffim(self): 

257 """Create the (un-decorrelated) diffim, and verify that its variance is too low. 

258 """ 

259 # Create the matching kernel. We used Gaussian PSFs for im1 and im2, so we can compute the "expected" 

260 # matching kernel sigma. 

261 psf1_sig = self.im1ex.getPsf().computeShape().getDeterminantRadius() 

262 psf2_sig = self.im2ex.getPsf().computeShape().getDeterminantRadius() 

263 sig_match = np.sqrt((psf1_sig**2. - psf2_sig**2.)) 

264 # Sanity check - make sure PSFs are correct. 

265 self.assertFloatsAlmostEqual(sig_match, np.sqrt((self.psf1_sigma**2. - self.psf2_sigma**2.)), 

266 rtol=2e-5) 

267 # mKernel = measAlg.SingleGaussianPsf(31, 31, sig_match) 

268 x0 = np.arange(-16, 16, 1) 

269 y0 = x0.copy() 

270 x0im, y0im = np.meshgrid(x0, y0) 

271 matchingKernel = singleGaussian2d(x0im, y0im, -1., -1., sigma_x=sig_match, sigma_y=sig_match) 

272 kernelImg = afwImage.ImageD(matchingKernel.shape[0], matchingKernel.shape[1]) 

273 kernelImg.getArray()[:, :] = matchingKernel 

274 mKernel = afwMath.FixedKernel(kernelImg) 

275 

276 # Create the matched template by convolving the template with the matchingKernel 

277 matched_im2ex = self.im2ex.clone() 

278 convCntrl = afwMath.ConvolutionControl(False, True, 0) 

279 afwMath.convolve(matched_im2ex.getMaskedImage(), self.im2ex.getMaskedImage(), mKernel, convCntrl) 

280 

281 # Expected (ideal) variance of difference image 

282 expected_var = self.svar + self.tvar 

283 if verbose: 

284 print('EXPECTED VARIANCE:', expected_var) 

285 

286 # Create the diffim (uncorrected) 

287 # Uncorrected diffim exposure - variance plane is wrong (too low) 

288 tmp_diffExp = self.im1ex.getMaskedImage().clone() 

289 tmp_diffExp -= matched_im2ex.getMaskedImage() 

290 var = self._computeVarianceMean(tmp_diffExp) 

291 self.assertLess(var, expected_var) 

292 

293 # Uncorrected diffim exposure - variance is wrong (too low) - same as above but on pixels 

294 diffExp = self.im1ex.clone() 

295 tmp = diffExp.getMaskedImage() 

296 tmp -= matched_im2ex.getMaskedImage() 

297 var = self._computePixelVariance(diffExp.getMaskedImage()) 

298 self.assertLess(var, expected_var) 

299 

300 # Uncorrected diffim exposure - variance plane is wrong (too low) 

301 mn = self._computeVarianceMean(diffExp.getMaskedImage()) 

302 self.assertLess(mn, expected_var) 

303 if verbose: 

304 print('UNCORRECTED VARIANCE:', var, mn) 

305 

306 return diffExp, mKernel, expected_var 

307 

308 def _runDecorrelationTask(self, diffExp, mKernel): 

309 """ Run the decorrelation task on the given diffim with the given matching kernel 

310 """ 

311 task = DecorrelateALKernelTask() 

312 decorrResult = task.run(self.im1ex, self.im2ex, diffExp, mKernel) 

313 corrected_diffExp = decorrResult.correctedExposure 

314 return corrected_diffExp 

315 

316 def _testDecorrelation(self, expected_var, corrected_diffExp): 

317 """ Check that the variance of the corrected diffim matches the theoretical value. 

318 """ 

319 # Corrected diffim - variance should be close to expected. 

320 # We set the tolerance a bit higher here since the simulated images have many bright stars 

321 var = self._computePixelVariance(corrected_diffExp.getMaskedImage()) 

322 self.assertFloatsAlmostEqual(var, expected_var, rtol=0.05) 

323 

324 # Check statistics of variance plane in corrected diffim 

325 mn = self._computeVarianceMean(corrected_diffExp.getMaskedImage()) 

326 if verbose: 

327 print('CORRECTED VARIANCE:', var, mn) 

328 self.assertFloatsAlmostEqual(mn, expected_var, rtol=0.02) 

329 self.assertFloatsAlmostEqual(var, mn, rtol=0.05) 

330 return var, mn 

331 

332 def _testDiffimCorrection(self, svar, tvar): 

333 """ Run decorrelation and check the variance of the corrected diffim. 

334 """ 

335 self._setUpImages(svar=svar, tvar=tvar) 

336 diffExp, mKernel, expected_var = self._makeAndTestUncorrectedDiffim() 

337 corrected_diffExp = self._runDecorrelationTask(diffExp, mKernel) 

338 self._testDecorrelation(expected_var, corrected_diffExp) 

339 

340 def testDiffimCorrection(self): 

341 """Test decorrelated diffim from images with different combinations of variances. 

342 """ 

343 # Same variance 

344 self._testDiffimCorrection(svar=0.04, tvar=0.04) 

345 # Science image variance is higher than that of the template. 

346 self._testDiffimCorrection(svar=0.08, tvar=0.04) 

347 # Template variance is higher than that of the science img. 

348 self._testDiffimCorrection(svar=0.04, tvar=0.08) 

349 

350 def _runDecorrelationTaskMapReduced(self, diffExp, mKernel): 

351 """ Run decorrelation using the imageMapReducer. 

352 """ 

353 config = DecorrelateALKernelMapReduceConfig() 

354 config.borderSizeX = config.borderSizeY = 3 

355 config.reducer.reduceOperation = 'average' 

356 task = ImageMapReduceTask(config=config) 

357 decorrResult = task.run(diffExp, template=self.im2ex, science=self.im1ex, 

358 psfMatchingKernel=mKernel, forceEvenSized=True) 

359 corrected_diffExp = decorrResult.exposure 

360 return corrected_diffExp 

361 

362 def _testDiffimCorrection_mapReduced(self, svar, tvar, varyPsf=0.0): 

363 """ Run decorrelation using the imageMapReduce task, and check the variance of 

364 the corrected diffim. 

365 """ 

366 self._setUpImages(svar=svar, tvar=tvar, varyPsf=varyPsf) 

367 diffExp, mKernel, expected_var = self._makeAndTestUncorrectedDiffim() 

368 corrected_diffExp = self._runDecorrelationTaskMapReduced(diffExp, mKernel) 

369 self._testDecorrelation(expected_var, corrected_diffExp) 

370 # Also compare the diffim generated here vs. the non-ImageMapReduce one 

371 corrected_diffExp_OLD = self._runDecorrelationTask(diffExp, mKernel) 

372 self.assertMaskedImagesAlmostEqual(corrected_diffExp.getMaskedImage(), 

373 corrected_diffExp_OLD.getMaskedImage()) 

374 

375 def testDiffimCorrection_mapReduced(self): 

376 """ Test decorrelated diffim when using the imageMapReduce task. 

377 Compare results with those from the original DecorrelateALKernelTask. 

378 """ 

379 # Same variance 

380 self._testDiffimCorrection_mapReduced(svar=0.04, tvar=0.04) 

381 # Science image variance is higher than that of the template. 

382 self._testDiffimCorrection_mapReduced(svar=0.04, tvar=0.08) 

383 # Template variance is higher than that of the science img. 

384 self._testDiffimCorrection_mapReduced(svar=0.08, tvar=0.04) 

385 

386 def _runDecorrelationSpatialTask(self, diffExp, mKernel, spatiallyVarying=False): 

387 """ Run decorrelation using the DecorrelateALKernelSpatialTask. 

388 """ 

389 config = DecorrelateALKernelSpatialConfig() 

390 task = DecorrelateALKernelSpatialTask(config=config) 

391 decorrResult = task.run(scienceExposure=self.im1ex, templateExposure=self.im2ex, 

392 subtractedExposure=diffExp, psfMatchingKernel=mKernel, 

393 spatiallyVarying=spatiallyVarying) 

394 corrected_diffExp = decorrResult.correctedExposure 

395 return corrected_diffExp 

396 

397 def _testDiffimCorrection_spatialTask(self, svar, tvar, varyPsf=0.0): 

398 """Run decorrelation using the DecorrelateALKernelSpatialTask, and 

399 check the variance of the corrected diffim. Do it for `spatiallyVarying` both 

400 True and False. Also compare the variances between the two `spatiallyVarying` 

401 cases. 

402 """ 

403 self._setUpImages(svar=svar, tvar=tvar, varyPsf=varyPsf) 

404 diffExp, mKernel, expected_var = self._makeAndTestUncorrectedDiffim() 

405 variances = [] 

406 for spatiallyVarying in [False, True]: 

407 corrected_diffExp = self._runDecorrelationSpatialTask(diffExp, mKernel, 

408 spatiallyVarying) 

409 var, mn = self._testDecorrelation(expected_var, corrected_diffExp) 

410 variances.append(var) 

411 self.assertFloatsAlmostEqual(variances[0], variances[1], rtol=0.03) 

412 

413 def testDiffimCorrection_spatialTask(self): 

414 """Test decorrelated diffim when using the DecorrelateALKernelSpatialTask. 

415 Compare results with those from the original DecorrelateALKernelTask. 

416 """ 

417 # Same variance 

418 self._testDiffimCorrection_spatialTask(svar=0.04, tvar=0.04) 

419 # Science image variance is higher than that of the template. 

420 self._testDiffimCorrection_spatialTask(svar=0.04, tvar=0.08) 

421 # Template variance is higher than that of the science img. 

422 self._testDiffimCorrection_spatialTask(svar=0.08, tvar=0.04) 

423 

424 

425class MemoryTester(lsst.utils.tests.MemoryTestCase): 

426 pass 

427 

428 

429if __name__ == "__main__": 429 ↛ 430line 429 didn't jump to line 430, because the condition on line 429 was never true

430 lsst.utils.tests.init() 

431 unittest.main()