Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

407

408

409

410

411

412

413

414

415

416

417

418

419

420

421

422

423

424

425

426

427

428

429

430

# 

# LSST Data Management System 

# Copyright 2016-2017 AURA/LSST. 

# 

# This product includes software developed by the 

# LSST Project (http://www.lsst.org/). 

# 

# This program is free software: you can redistribute it and/or modify 

# it under the terms of the GNU General Public License as published by 

# the Free Software Foundation, either version 3 of the License, or 

# (at your option) any later version. 

# 

# This program is distributed in the hope that it will be useful, 

# but WITHOUT ANY WARRANTY; without even the implied warranty of 

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

# GNU General Public License for more details. 

# 

# You should have received a copy of the LSST License Statement and 

# the GNU General Public License along with this program. If not, 

# see <https://www.lsstcorp.org/LegalNotices/>. 

import unittest 

 

import numpy as np 

 

import lsst.utils.tests 

import lsst.afw.image as afwImage 

import lsst.afw.geom as afwGeom 

import lsst.afw.math as afwMath 

import lsst.meas.algorithms as measAlg 

import lsst.daf.base as dafBase 

 

from lsst.ip.diffim.imageDecorrelation import (DecorrelateALKernelTask, 

DecorrelateALKernelMapReduceConfig, 

DecorrelateALKernelSpatialConfig, 

DecorrelateALKernelSpatialTask) 

from lsst.ip.diffim.imageMapReduce import ImageMapReduceTask 

 

try: 

type(verbose) 

except NameError: 

verbose = False 

 

 

def setup_module(module): 

lsst.utils.tests.init() 

 

 

def singleGaussian2d(x, y, xc, yc, sigma_x=1., sigma_y=1., theta=0., ampl=1.): 

"""! Generate a 2-d Gaussian, possibly elongated and rotated, on a grid of pixel 

coordinates given by x,y. 

@param x,y each a 1-d numpy.array containing x- and y- coordinates for independent variables, 

for example `np.arange(-16, 15)`. 

@param xc,yc each a float giving the centroid of the gaussian 

@param sigma_x,sigma_y each a float giving the sigma of the gaussian 

@param theta a float giving the rotation of the gaussian (degrees) 

@param ampl a float giving the amplitude of the gaussian 

@return a 2-d numpy.array containing the normalized 2-d Gaussian 

 

@Note this can be done in `astropy.modeling` but for now we have it explicitly here. 

""" 

theta = (theta/180.) * np.pi 

cos_theta2, sin_theta2 = np.cos(theta)**2., np.sin(theta)**2. 

sigma_x2, sigma_y2 = sigma_x**2., sigma_y**2. 

a = cos_theta2/(2.*sigma_x2) + sin_theta2/(2.*sigma_y2) 

b = -(np.sin(2.*theta))/(4.*sigma_x2) + (np.sin(2.*theta))/(4.*sigma_y2) 

c = sin_theta2/(2.*sigma_x2) + cos_theta2/(2.*sigma_y2) 

xxc, yyc = x-xc, y-yc 

out = np.exp(-(a*(xxc**2.) + 2.*b*xxc*yyc + c*(yyc**2.))) 

out /= out.sum() 

return out 

 

 

def makeFakeImages(size=(256, 256), svar=0.04, tvar=0.04, psf1=3.3, psf2=2.2, offset=None, 

psf_yvary_factor=0., varSourceChange=1/50., theta1=0., theta2=0., 

n_sources=500, seed=66, verbose=False): 

"""! Make two exposures: a template and a science exposure. 

Add random sources with randomly-distributed and identical fluxes and a given PSF, then add noise. 

In all cases below, index (1) is the science image, and (2) is the template. 

@param size tuple givein image pixel size. Pixel coordinates are set to 

(-size[0]//2:size[0]//2, -size[1]//2:size[1]//2) 

@param svar,tar variance of noise to be generated on science/template images. Default is 0.04 for both. 

@param psf1,psf2 std. dev. of (Gaussian) PSFs for the two images in x,y direction. Default is 

[3.3, 3.3] and [2.2, 2.2] for im1 and im2 respectively. 

@param offset add a constant (pixel) astrometric offset between the two images 

@param psf_yvary_factor vary the y-width of the PSF across the x-axis of the science image (zero, 

the default, means no variation) 

@param varSourceChange add this amount of fractional flux to a single source closest to 

the center of the science image 

@param n_sources the number of sources to add to the images 

@param seed the numpy random seed to set prior to image generation 

@param verbose be verbose 

 

@return im1, im2: the science and template afwImage.Exposures 

 

@note having sources near the edges really messes up the 

fitting (probably because of the convolution). So we make sure no 

sources are near the edge. 

@note also it seems that having the variable source with a large 

flux increase also messes up the fitting (seems to lead to 

overfitting -- perhaps to the source itself). This might be fixed by 

adding more constant sources. 

""" 

np.random.seed(seed) 

 

psf1 = [3.3, 3.3] if psf1 is None else psf1 

106 ↛ 108line 106 didn't jump to line 108, because the condition on line 106 was never false if not hasattr(psf1, "__len__") and not isinstance(psf1, str): 

psf1 = [psf1, psf1] 

psf2 = [2.2, 2.2] if psf2 is None else psf2 

109 ↛ 111line 109 didn't jump to line 111, because the condition on line 109 was never false if not hasattr(psf2, "__len__") and not isinstance(psf2, str): 

psf2 = [psf2, psf2] 

offset = [0., 0.] if offset is None else offset # astrometric offset (pixels) between the two images 

112 ↛ 113line 112 didn't jump to line 113, because the condition on line 112 was never true if verbose: 

print('Science PSF:', psf1, theta1) 

print('Template PSF:', psf2, theta2) 

print(np.sqrt(psf1[0]**2 - psf2[0]**2)) 

print('Offset:', offset) 

 

xim = np.arange(-size[0]//2, size[0]//2, 1) 

yim = np.arange(-size[1]//2, size[1]//2, 1) 

x0im, y0im = np.meshgrid(yim, xim) 

im1 = np.random.normal(scale=np.sqrt(svar), size=x0im.shape) # variance of science image 

im2 = np.random.normal(scale=np.sqrt(tvar), size=x0im.shape) # variance of template 

 

fluxes = np.random.uniform(50, 30000, n_sources) 

xposns = np.random.uniform(xim.min()+16, xim.max()-5, n_sources) 

yposns = np.random.uniform(yim.min()+16, yim.max()-5, n_sources) 

 

# Make the source closest to the center of the image the one that increases in flux 

ind = np.argmin(xposns**2. + yposns**2.) 

 

# vary the y-width of psf across x-axis of science image (zero means no variation): 

psf1_yvary = psf_yvary_factor * (yim.mean() - yposns) / yim.max() 

133 ↛ 134line 133 didn't jump to line 134, because the condition on line 133 was never true if verbose: 

print('PSF y spatial-variation:', psf1_yvary.min(), psf1_yvary.max()) 

 

for i in range(n_sources): 

flux = fluxes[i] 

tmp = flux * singleGaussian2d(x0im, y0im, xposns[i], yposns[i], psf2[0], psf2[1], theta=theta2) 

im2 += tmp 

if i == ind: 

flux += flux * varSourceChange 

tmp = flux * singleGaussian2d(x0im, y0im, xposns[i]+offset[0], yposns[i]+offset[1], 

psf1[0], psf1[1]+psf1_yvary[i], theta=theta1) 

im1 += tmp 

 

im1_psf = singleGaussian2d(x0im, y0im, 0, 0, psf1[0], psf1[1], theta=theta1) 

im2_psf = singleGaussian2d(x0im, y0im, offset[0], offset[1], psf2[0], psf2[1], theta=theta2) 

 

def makeWcs(offset=0): 

""" Make a fake Wcs 

 

Parameters 

---------- 

offset : float 

offset the Wcs by this many pixels. 

""" 

# taken from $AFW_DIR/tests/testMakeWcs.py 

metadata = dafBase.PropertySet() 

metadata.set("SIMPLE", "T") 

metadata.set("BITPIX", -32) 

metadata.set("NAXIS", 2) 

metadata.set("NAXIS1", 1024) 

metadata.set("NAXIS2", 1153) 

metadata.set("RADESYS", 'FK5') 

metadata.set("EQUINOX", 2000.) 

metadata.setDouble("CRVAL1", 215.604025685476) 

metadata.setDouble("CRVAL2", 53.1595451514076) 

metadata.setDouble("CRPIX1", 1109.99981456774 + offset) 

metadata.setDouble("CRPIX2", 560.018167811613 + offset) 

metadata.set("CTYPE1", 'RA---SIN') 

metadata.set("CTYPE2", 'DEC--SIN') 

metadata.setDouble("CD1_1", 5.10808596133527E-05) 

metadata.setDouble("CD1_2", 1.85579539217196E-07) 

metadata.setDouble("CD2_2", -5.10281493481982E-05) 

metadata.setDouble("CD2_1", -8.27440751733828E-07) 

return afwGeom.makeSkyWcs(metadata) 

 

def makeExposure(imgArray, psfArray, imgVariance): 

"""! Convert an image numpy.array and corresponding PSF numpy.array into an exposure. 

 

Add the (constant) variance plane equal to `imgVariance`. 

 

@param imgArray 2-d numpy.array containing the image 

@param psfArray 2-d numpy.array containing the PSF image 

@param imgVariance variance of input image 

@return a new exposure containing the image, PSF and desired variance plane 

""" 

# All this code to convert the template image array/psf array into an exposure. 

bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Point2I(imgArray.shape[1]-1, imgArray.shape[0]-1)) 

im1ex = afwImage.ExposureD(bbox) 

im1ex.getMaskedImage().getImage().getArray()[:, :] = imgArray 

im1ex.getMaskedImage().getVariance().getArray()[:, :] = imgVariance 

psfBox = afwGeom.Box2I(afwGeom.Point2I(-12, -12), afwGeom.Point2I(12, 12)) # a 25x25 pixel psf 

psf = afwImage.ImageD(psfBox) 

psfBox.shift(afwGeom.Extent2I(size[0]//2, size[1]//2)) 

im1_psf_sub = psfArray[psfBox.getMinX():psfBox.getMaxX()+1, psfBox.getMinY():psfBox.getMaxY()+1] 

psf.getArray()[:, :] = im1_psf_sub 

psfK = afwMath.FixedKernel(psf) 

psfNew = measAlg.KernelPsf(psfK) 

im1ex.setPsf(psfNew) 

wcs = makeWcs() 

im1ex.setWcs(wcs) 

return im1ex 

 

im1ex = makeExposure(im1, im1_psf, svar) # Science image 

im2ex = makeExposure(im2, im2_psf, tvar) # Template 

 

return im1ex, im2ex 

 

 

class DiffimCorrectionTest(lsst.utils.tests.TestCase): 

"""!A test case for the diffim image decorrelation algorithm. 

""" 

 

def setUp(self): 

self.psf1_sigma = 3.3 # sigma of psf of science image 

self.psf2_sigma = 2.2 # sigma of psf of template image 

 

self.statsControl = afwMath.StatisticsControl() 

self.statsControl.setNumSigmaClip(3.) 

self.statsControl.setNumIter(3) 

self.statsControl.setAndMask(afwImage.Mask 

.getPlaneBitMask(["INTRP", "EDGE", "SAT", "CR", 

"DETECTED", "BAD", 

"NO_DATA", "DETECTED_NEGATIVE"])) 

 

def _setUpImages(self, svar=0.04, tvar=0.04, varyPsf=0.): 

"""!Generate a fake aligned template and science image. 

""" 

 

self.svar = svar # variance of noise in science image 

self.tvar = tvar # variance of noise in template image 

 

self.im1ex, self.im2ex \ 

= makeFakeImages(svar=self.svar, tvar=self.tvar, psf1=self.psf1_sigma, psf2=self.psf2_sigma, 

n_sources=50, psf_yvary_factor=varyPsf, verbose=False) 

 

def _computeVarianceMean(self, maskedIm): 

statObj = afwMath.makeStatistics(maskedIm.getVariance(), 

maskedIm.getMask(), afwMath.MEANCLIP, 

self.statsControl) 

mn = statObj.getValue(afwMath.MEANCLIP) 

return mn 

 

def _computePixelVariance(self, maskedIm): 

statObj = afwMath.makeStatistics(maskedIm, afwMath.VARIANCECLIP, 

self.statsControl) 

var = statObj.getValue(afwMath.VARIANCECLIP) 

return var 

 

def tearDown(self): 

del self.im1ex 

del self.im2ex 

 

def _makeAndTestUncorrectedDiffim(self): 

"""Create the (un-decorrelated) diffim, and verify that its variance is too low. 

""" 

# Create the matching kernel. We used Gaussian PSFs for im1 and im2, so we can compute the "expected" 

# matching kernel sigma. 

psf1_sig = self.im1ex.getPsf().computeShape().getDeterminantRadius() 

psf2_sig = self.im2ex.getPsf().computeShape().getDeterminantRadius() 

sig_match = np.sqrt((psf1_sig**2. - psf2_sig**2.)) 

# Sanity check - make sure PSFs are correct. 

self.assertFloatsAlmostEqual(sig_match, np.sqrt((self.psf1_sigma**2. - self.psf2_sigma**2.)), 

rtol=2e-5) 

# mKernel = measAlg.SingleGaussianPsf(31, 31, sig_match) 

x0 = np.arange(-16, 16, 1) 

y0 = x0.copy() 

x0im, y0im = np.meshgrid(x0, y0) 

matchingKernel = singleGaussian2d(x0im, y0im, -1., -1., sigma_x=sig_match, sigma_y=sig_match) 

kernelImg = afwImage.ImageD(matchingKernel.shape[0], matchingKernel.shape[1]) 

kernelImg.getArray()[:, :] = matchingKernel 

mKernel = afwMath.FixedKernel(kernelImg) 

 

# Create the matched template by convolving the template with the matchingKernel 

matched_im2ex = self.im2ex.clone() 

convCntrl = afwMath.ConvolutionControl(False, True, 0) 

afwMath.convolve(matched_im2ex.getMaskedImage(), self.im2ex.getMaskedImage(), mKernel, convCntrl) 

 

# Expected (ideal) variance of difference image 

expected_var = self.svar + self.tvar 

282 ↛ 283line 282 didn't jump to line 283, because the condition on line 282 was never true if verbose: 

print('EXPECTED VARIANCE:', expected_var) 

 

# Create the diffim (uncorrected) 

# Uncorrected diffim exposure - variance plane is wrong (too low) 

tmp_diffExp = self.im1ex.getMaskedImage().clone() 

tmp_diffExp -= matched_im2ex.getMaskedImage() 

var = self._computeVarianceMean(tmp_diffExp) 

self.assertLess(var, expected_var) 

 

# Uncorrected diffim exposure - variance is wrong (too low) - same as above but on pixels 

diffExp = self.im1ex.clone() 

tmp = diffExp.getMaskedImage() 

tmp -= matched_im2ex.getMaskedImage() 

var = self._computePixelVariance(diffExp.getMaskedImage()) 

self.assertLess(var, expected_var) 

 

# Uncorrected diffim exposure - variance plane is wrong (too low) 

mn = self._computeVarianceMean(diffExp.getMaskedImage()) 

self.assertLess(mn, expected_var) 

302 ↛ 303line 302 didn't jump to line 303, because the condition on line 302 was never true if verbose: 

print('UNCORRECTED VARIANCE:', var, mn) 

 

return diffExp, mKernel, expected_var 

 

def _runDecorrelationTask(self, diffExp, mKernel): 

""" Run the decorrelation task on the given diffim with the given matching kernel 

""" 

task = DecorrelateALKernelTask() 

decorrResult = task.run(self.im1ex, self.im2ex, diffExp, mKernel) 

corrected_diffExp = decorrResult.correctedExposure 

return corrected_diffExp 

 

def _testDecorrelation(self, expected_var, corrected_diffExp): 

""" Check that the variance of the corrected diffim matches the theoretical value. 

""" 

# Corrected diffim - variance should be close to expected. 

# We set the tolerance a bit higher here since the simulated images have many bright stars 

var = self._computePixelVariance(corrected_diffExp.getMaskedImage()) 

self.assertFloatsAlmostEqual(var, expected_var, rtol=0.05) 

 

# Check statistics of variance plane in corrected diffim 

mn = self._computeVarianceMean(corrected_diffExp.getMaskedImage()) 

325 ↛ 326line 325 didn't jump to line 326, because the condition on line 325 was never true if verbose: 

print('CORRECTED VARIANCE:', var, mn) 

self.assertFloatsAlmostEqual(mn, expected_var, rtol=0.02) 

self.assertFloatsAlmostEqual(var, mn, rtol=0.05) 

return var, mn 

 

def _testDiffimCorrection(self, svar, tvar): 

""" Run decorrelation and check the variance of the corrected diffim. 

""" 

self._setUpImages(svar=svar, tvar=tvar) 

diffExp, mKernel, expected_var = self._makeAndTestUncorrectedDiffim() 

corrected_diffExp = self._runDecorrelationTask(diffExp, mKernel) 

self._testDecorrelation(expected_var, corrected_diffExp) 

 

def testDiffimCorrection(self): 

"""Test decorrelated diffim from images with different combinations of variances. 

""" 

# Same variance 

self._testDiffimCorrection(svar=0.04, tvar=0.04) 

# Science image variance is higher than that of the template. 

self._testDiffimCorrection(svar=0.08, tvar=0.04) 

# Template variance is higher than that of the science img. 

self._testDiffimCorrection(svar=0.04, tvar=0.08) 

 

def _runDecorrelationTaskMapReduced(self, diffExp, mKernel): 

""" Run decorrelation using the imageMapReducer. 

""" 

config = DecorrelateALKernelMapReduceConfig() 

config.borderSizeX = config.borderSizeY = 3 

config.reducer.reduceOperation = 'average' 

task = ImageMapReduceTask(config=config) 

decorrResult = task.run(diffExp, template=self.im2ex, science=self.im1ex, 

psfMatchingKernel=mKernel, forceEvenSized=True) 

corrected_diffExp = decorrResult.exposure 

return corrected_diffExp 

 

def _testDiffimCorrection_mapReduced(self, svar, tvar, varyPsf=0.0): 

""" Run decorrelation using the imageMapReduce task, and check the variance of 

the corrected diffim. 

""" 

self._setUpImages(svar=svar, tvar=tvar, varyPsf=varyPsf) 

diffExp, mKernel, expected_var = self._makeAndTestUncorrectedDiffim() 

corrected_diffExp = self._runDecorrelationTaskMapReduced(diffExp, mKernel) 

self._testDecorrelation(expected_var, corrected_diffExp) 

# Also compare the diffim generated here vs. the non-ImageMapReduce one 

corrected_diffExp_OLD = self._runDecorrelationTask(diffExp, mKernel) 

self.assertMaskedImagesAlmostEqual(corrected_diffExp.getMaskedImage(), 

corrected_diffExp_OLD.getMaskedImage()) 

 

def testDiffimCorrection_mapReduced(self): 

""" Test decorrelated diffim when using the imageMapReduce task. 

Compare results with those from the original DecorrelateALKernelTask. 

""" 

# Same variance 

self._testDiffimCorrection_mapReduced(svar=0.04, tvar=0.04) 

# Science image variance is higher than that of the template. 

self._testDiffimCorrection_mapReduced(svar=0.04, tvar=0.08) 

# Template variance is higher than that of the science img. 

self._testDiffimCorrection_mapReduced(svar=0.08, tvar=0.04) 

 

def _runDecorrelationSpatialTask(self, diffExp, mKernel, spatiallyVarying=False): 

""" Run decorrelation using the DecorrelateALKernelSpatialTask. 

""" 

config = DecorrelateALKernelSpatialConfig() 

task = DecorrelateALKernelSpatialTask(config=config) 

decorrResult = task.run(scienceExposure=self.im1ex, templateExposure=self.im2ex, 

subtractedExposure=diffExp, psfMatchingKernel=mKernel, 

spatiallyVarying=spatiallyVarying) 

corrected_diffExp = decorrResult.correctedExposure 

return corrected_diffExp 

 

def _testDiffimCorrection_spatialTask(self, svar, tvar, varyPsf=0.0): 

"""Run decorrelation using the DecorrelateALKernelSpatialTask, and 

check the variance of the corrected diffim. Do it for `spatiallyVarying` both 

True and False. Also compare the variances between the two `spatiallyVarying` 

cases. 

""" 

self._setUpImages(svar=svar, tvar=tvar, varyPsf=varyPsf) 

diffExp, mKernel, expected_var = self._makeAndTestUncorrectedDiffim() 

variances = [] 

for spatiallyVarying in [False, True]: 

corrected_diffExp = self._runDecorrelationSpatialTask(diffExp, mKernel, 

spatiallyVarying) 

var, mn = self._testDecorrelation(expected_var, corrected_diffExp) 

variances.append(var) 

self.assertFloatsAlmostEqual(variances[0], variances[1], rtol=0.03) 

 

def testDiffimCorrection_spatialTask(self): 

"""Test decorrelated diffim when using the DecorrelateALKernelSpatialTask. 

Compare results with those from the original DecorrelateALKernelTask. 

""" 

# Same variance 

self._testDiffimCorrection_spatialTask(svar=0.04, tvar=0.04) 

# Science image variance is higher than that of the template. 

self._testDiffimCorrection_spatialTask(svar=0.04, tvar=0.08) 

# Template variance is higher than that of the science img. 

self._testDiffimCorrection_spatialTask(svar=0.08, tvar=0.04) 

 

 

class MemoryTester(lsst.utils.tests.MemoryTestCase): 

pass 

 

 

428 ↛ 429line 428 didn't jump to line 429, because the condition on line 428 was never trueif __name__ == "__main__": 

lsst.utils.tests.init() 

unittest.main()