Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of meas_algorithms. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import os 

23import unittest 

24import math 

25import numpy as np 

26 

27import lsst.geom 

28import lsst.afw.image as afwImage 

29import lsst.meas.algorithms as algorithms 

30import lsst.utils.tests 

31from lsst.daf.base import PropertyList 

32 

33try: 

34 type(display) 

35except NameError: 

36 display = False 

37else: 

38 import lsst.afw.display as afwDisplay 

39 afwDisplay.setDefaultMaskTransparency(75) 

40 

41# Determine if we have afwdata 

42try: 

43 afwdataDir = lsst.utils.getPackageDir('afwdata') 

44except Exception: 

45 afwdataDir = None 

46 

47TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

48 

49 

50class DefectsTestCase(lsst.utils.tests.TestCase): 

51 """Tests for collections of Defect.""" 

52 

53 def assertMetadata(self, first, second): 

54 """Compare the metadata associated with Defects""" 

55 

56 # Must strip out DATE metadata before comparison 

57 meta1 = first.getMetadata() 

58 meta2 = second.getMetadata() 

59 for d in (meta1, meta2): 

60 for k in ("DATE", "CALIB_CREATION_DATE", "CALIB_CREATION_TIME"): 

61 if k in d: 

62 del d[k] 

63 

64 self.assertEqual(meta1, meta2) 

65 meta1["NEW"] = "additional header" 

66 self.assertNotEqual(first.getMetadata(), second.getMetadata()) 

67 del meta1["NEW"] 

68 

69 def test_defects(self): 

70 defects = algorithms.Defects() 

71 

72 defects.append(algorithms.Defect(lsst.geom.Box2I(lsst.geom.Point2I(5, 6), 

73 lsst.geom.Point2I(41, 50)))) 

74 

75 defects.append(lsst.geom.Box2I(lsst.geom.Point2I(0, 0), 

76 lsst.geom.Point2I(4, 5))) 

77 defects.append(lsst.geom.Point2I(50, 50)) 

78 defects.append(afwImage.DefectBase(lsst.geom.Box2I(lsst.geom.Point2I(100, 200), 

79 lsst.geom.Extent2I(5, 5)))) 

80 self.assertEqual(len(defects), 4) 

81 

82 for d in defects: 

83 self.assertIsInstance(d, algorithms.Defect) 

84 

85 # Transposition 

86 transposed = defects.transpose() 

87 self.assertEqual(len(transposed), len(defects)) 

88 

89 # Check that an individual defect is found properly transposed within 

90 # the outputs. 

91 found = False 

92 for defect in transposed: 92 ↛ 96line 92 didn't jump to line 96, because the loop on line 92 didn't complete

93 if defect.getBBox() == lsst.geom.Box2I(lsst.geom.Point2I(6, 5), lsst.geom.Extent2I(45, 37)): 

94 found = True 

95 break 

96 self.assertTrue(found) 

97 

98 # Serialization round trip 

99 meta = PropertyList() 

100 meta["TESTHDR"] = "testing" 

101 defects.setMetadata(meta) 

102 

103 table = defects.toFitsRegionTable() 

104 defects2 = algorithms.Defects.fromTable(table) 

105 self.assertEqual(defects2, defects) 

106 

107 # via FITS 

108 with lsst.utils.tests.getTempFilePath(".fits") as tmpFile: 

109 defects.writeFits(tmpFile) 

110 defects2 = algorithms.Defects.readFits(tmpFile) 

111 

112 # Equality tests the bounding boxes so metadata is tested separately. 

113 self.assertEqual(defects2, defects) 

114 self.assertMetadata(defects2, defects) 

115 

116 # via text file 

117 with lsst.utils.tests.getTempFilePath(".ecsv") as tmpFile: 

118 defects.writeText(tmpFile) 

119 defects2 = algorithms.Defects.readText(tmpFile) 

120 

121 # Equality tests the bounding boxes so metadata is tested separately. 

122 self.assertEqual(defects2, defects) 

123 self.assertMetadata(defects2, defects) 

124 

125 # Check bad values 

126 with self.assertRaises(ValueError): 

127 defects.append(lsst.geom.Box2D(lsst.geom.Point2D(0., 0.), 

128 lsst.geom.Point2D(3.1, 3.1))) 

129 with self.assertRaises(ValueError): 

130 defects.append("defect") 

131 

132 def testAstropyRegion(self): 

133 """Read a FITS region file created by Astropy regions.""" 

134 # The file contains three regions: 

135 # 

136 # - Point2I(340, 344) 

137 # - Point2I(340, 344) 

138 # - Box2I(minimum=Point2I(5, -5), dimensions=Extent2I(10, 20)) 

139 # 

140 # The two coincident points are combined on read, so we end up with two defects. 

141 

142 with self.assertLogs(): 

143 defects = algorithms.Defects.readFits(os.path.join(TESTDIR, "data", "fits_region.fits"), 

144 normalize_on_init=True) 

145 

146 self.assertEqual(len(defects), 2) 

147 

148 def testLsstTextfile(self): 

149 """Read legacy LSST text file format""" 

150 with lsst.utils.tests.getTempFilePath(".txt") as tmpFile: 

151 with open(tmpFile, "w") as fh: 

152 print("""# X0 Y0 width height 

153 996 0 56 24 

154 0 4156 2048 20 

155 0 0 17 4176 

156 1998 4035 50 141 

157 1023 0 2 4176 

158 2027 0 21 4176 

159 0 4047 37 129 

160# Some rows without fixed column widths 

16114 20 2000 50 

16210 10 10 10 

163""", file=fh) 

164 

165 defects = algorithms.Defects.readLsstDefectsFile(tmpFile, normalize_on_init=True) 

166 

167 # Although there are 9 defects listed above, we record 11 after 

168 # normalization. This is due to non-optimal behaviour in 

169 # Defects.fromMask; see DM-24781. 

170 self.assertEqual(len(defects), 11) 

171 

172 def test_normalize_defects(self): 

173 """A test for the lsst.meas.algorithms.Defect.normalize() method. 

174 """ 

175 defects = algorithms.Defects() 

176 

177 # First series of 1-pixel contiguous defects 

178 for yPix in range(1, 6): 

179 defects.append(lsst.geom.Box2I(corner=lsst.geom.Point2I(15, yPix), 

180 dimensions=lsst.geom.Extent2I(1, 1))) 

181 

182 # Defects are normalized as they are added; check that the above have 

183 # been merged into a single bounding box. 

184 self.assertEqual(len(defects), 1) 

185 

186 # Second series of 1-pixel contiguous defects in bulk mode 

187 with defects.bulk_update(): 

188 for yPix in range(11, 16): 

189 defects.append(lsst.geom.Box2I(corner=lsst.geom.Point2I(20, yPix), 

190 dimensions=lsst.geom.Extent2I(1, 1))) 

191 # In bulk mode, defects are not normalized. 

192 self.assertEqual(len(defects), 6) 

193 

194 # Normalization applied on exiting bulk mode. 

195 self.assertEqual(len(defects), 2) 

196 

197 boxesMeasured = [] 

198 for defect in defects: 

199 boxesMeasured.append(defect.getBBox()) 

200 

201 # The normalizing function should have created the following two boxes out 

202 # of the individual 1-pixel defects from above 

203 expectedDefects = [lsst.geom.Box2I(corner=lsst.geom.Point2I(15, 1), 

204 dimensions=lsst.geom.Extent2I(1, 5)), 

205 lsst.geom.Box2I(corner=lsst.geom.Point2I(20, 11), 

206 dimensions=lsst.geom.Extent2I(1, 5))] 

207 

208 self.assertEqual(len(expectedDefects), len(boxesMeasured)) 

209 for expDef, measDef in zip(expectedDefects, boxesMeasured): 

210 self.assertEqual(expDef, measDef) 

211 

212 # Normalize two distinct sets of Defects and ensure they compare to the same thing 

213 defects = algorithms.Defects() 

214 # Set 1 

215 defects.append(lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 1), dimensions=lsst.geom.Extent2I(1, 1))) 

216 defects.append(lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 2), dimensions=lsst.geom.Extent2I(1, 1))) 

217 defects.append(lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 3), dimensions=lsst.geom.Extent2I(1, 1))) 

218 defects.append(lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 4), dimensions=lsst.geom.Extent2I(1, 1))) 

219 defects.append(lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 5), dimensions=lsst.geom.Extent2I(1, 1))) 

220 defects.append(lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 6), dimensions=lsst.geom.Extent2I(1, 1))) 

221 defects.append(lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 7), dimensions=lsst.geom.Extent2I(1, 1))) 

222 defects.append(lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 8), dimensions=lsst.geom.Extent2I(1, 1))) 

223 

224 # Set 2 

225 defects2 = algorithms.Defects() 

226 defects2.append(lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 1), dimensions=lsst.geom.Extent2I(1, 5))) 

227 defects2.append(lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 5), dimensions=lsst.geom.Extent2I(1, 4))) 

228 

229 self.assertEqual(defects, defects2) 

230 

231 boxesMeasured, boxesMeasured2 = [], [] 

232 for defect, defect2 in zip(defects, defects2): 

233 boxesMeasured.append(defect.getBBox()) 

234 boxesMeasured2.append(defect2.getBBox()) 

235 

236 expectedDefects = [lsst.geom.Box2I(corner=lsst.geom.Point2I(25, 1), 

237 dimensions=lsst.geom.Extent2I(1, 8))] 

238 

239 self.assertEqual(len(expectedDefects), len(boxesMeasured)) 

240 for expDef, measDef in zip(expectedDefects, boxesMeasured): 

241 self.assertEqual(expDef, measDef) 

242 

243 self.assertEqual(len(expectedDefects), len(boxesMeasured2)) 

244 for expDef, measDef in zip(expectedDefects, boxesMeasured2): 

245 self.assertEqual(expDef, measDef) 

246 

247 

248class InterpolationTestCase(lsst.utils.tests.TestCase): 

249 """A test case for interpolation.""" 

250 

251 def setUp(self): 

252 self.FWHM = 5 

253 self.psf = algorithms.DoubleGaussianPsf(15, 15, self.FWHM/(2*math.sqrt(2*math.log(2)))) 

254 maskedImageFile = os.path.join(afwdataDir, "CFHT", "D4", "cal-53535-i-797722_1.fits") 

255 

256 self.mi = afwImage.MaskedImageF(maskedImageFile) 

257 if False: # use sub-image? 

258 self.mi = self.mi.Factory(self.mi, afwImage.BBox(afwImage.PointI(760, 20), 256, 256)) 

259 self.mi.getMask().addMaskPlane("INTERP") 

260 

261 measAlgorithmsDir = lsst.utils.getPackageDir('meas_algorithms') 

262 self.badPixels = algorithms.Defects.readText(os.path.join(measAlgorithmsDir, 

263 "policy", "BadPixels.ecsv")) 

264 

265 def tearDown(self): 

266 del self.mi 

267 del self.psf 

268 del self.badPixels 

269 

270 @unittest.skipUnless(afwdataDir, "afwdata not available") 

271 def testDetection(self): 

272 """Test Interp algorithms.""" 

273 

274 if display: 

275 frame = 0 

276 afwDisplay.Display(frame=frame).mtv(self.mi, title=self._testMethodName + ": Original") 

277 

278 algorithms.interpolateOverDefects(self.mi, self.psf, self.badPixels) 

279 

280 if display: 

281 frame += 1 

282 afwDisplay.Display(frame=frame).mtv(self.mi, title=self._testMethodName + ": Interpolated") 

283 frame += 1 

284 afwDisplay.Display(frame=frame).mtv(self.mi.getVariance(), 

285 title=self._testMethodName + ": Variance") 

286 

287 @unittest.skipUnless(afwdataDir, "afwdata not available") 

288 def test818(self): 

289 """A test case for #818; the full test is in /lsst/DC3root/ticketFiles/818""" 

290 

291 badPixels = [] 

292 defects = [((82, 663), 6, 8), 

293 ((83, 659), 9, 6), 

294 ((85, 660), 10, 11), 

295 ((87, 669), 3, 3), 

296 ] 

297 

298 for xy0, width, height in defects: 

299 x0, y0 = xy0 

300 bbox = lsst.geom.BoxI(lsst.geom.PointI(x0, y0), lsst.geom.ExtentI(width, height)) 

301 badPixels.append(algorithms.Defect(bbox)) 

302 

303 mi = afwImage.MaskedImageF(517, 800) 

304 

305 algorithms.interpolateOverDefects(mi, self.psf, badPixels) 

306 

307 @unittest.skipUnless(afwdataDir, "afwdata not available") 

308 def test1295(self): 

309 """A test case for #1295 (failure to interpolate over groups of defects.""" 

310 im = afwImage.ImageF(lsst.geom.ExtentI(100, 100)) 

311 mi = afwImage.makeMaskedImage(im) 

312 mi.set(100) 

313 flat = afwImage.ImageF(im.getDimensions()) 

314 flat.set(1) 

315 flat[50:51, :, afwImage.LOCAL] = 0.0 

316 flat[55:56, :, afwImage.LOCAL] = 0.0 

317 flat[58:59, :, afwImage.LOCAL] = 0.0 

318 flat[51:60, 51:, afwImage.LOCAL] = 0.0 

319 

320 mi /= flat 

321 

322 if display: 

323 afwDisplay.Display(frame=0).mtv(mi, title=self._testMethodName + ": Raw") 

324 

325 defectList = algorithms.Defects() 

326 bbox = lsst.geom.BoxI(lsst.geom.PointI(50, 0), lsst.geom.ExtentI(1, 100)) 

327 defectList.append(algorithms.Defect(bbox)) 

328 bbox = lsst.geom.BoxI(lsst.geom.PointI(55, 0), lsst.geom.ExtentI(1, 100)) 

329 defectList.append(algorithms.Defect(bbox)) 

330 bbox = lsst.geom.BoxI(lsst.geom.PointI(58, 0), lsst.geom.ExtentI(1, 100)) 

331 defectList.append(algorithms.Defect(bbox)) 

332 bbox = lsst.geom.BoxI(lsst.geom.PointI(51, 51), lsst.geom.ExtentI(9, 49)) 

333 defectList.append(algorithms.Defect(bbox)) 

334 

335 psf = algorithms.DoubleGaussianPsf(15, 15, 1./(2*math.sqrt(2*math.log(2)))) 

336 algorithms.interpolateOverDefects(mi, psf, defectList, 50.) 

337 

338 if display: 

339 afwDisplay.Display(frame=1).mtv(mi, title=self._testMethodName + ": Interpolated") 

340 

341 self.assertTrue(np.isfinite(mi.image[56, 51, afwImage.LOCAL])) 

342 

343 @unittest.skipUnless(afwdataDir, "afwdata not available") 

344 def testEdge(self): 

345 """Test that we can interpolate to the edge""" 

346 mi = afwImage.MaskedImageF(80, 30) 

347 

348 ima = mi.getImage().getArray() 

349 # 

350 # Loop over number of bad columns at left or right edge of image 

351 # 

352 for nBadCol in range(0, 20): 

353 mi.set((0, 0x0, 0)) 

354 

355 np.random.seed(666) 

356 ima[:] = np.random.uniform(-1, 1, ima.shape) 

357 

358 defects = [] 

359 

360 if nBadCol > 0: 

361 # 

362 # Bad left edge 

363 # 

364 ima[:, 0:nBadCol] = 10 

365 defects.append(lsst.geom.BoxI(lsst.geom.PointI(0, 0), 

366 lsst.geom.ExtentI(nBadCol, mi.getHeight()))) 

367 # 

368 # With another bad set of columns next to bad left edge 

369 # 

370 ima[:, -nBadCol:] = 10 

371 defects.append(lsst.geom.BoxI(lsst.geom.PointI(mi.getWidth() - nBadCol, 0), 

372 lsst.geom.ExtentI(nBadCol, mi.getHeight()))) 

373 # 

374 # Bad right edge 

375 # 

376 ima[0:10, nBadCol+1:nBadCol+4] = 100 

377 defects.append(lsst.geom.BoxI(lsst.geom.PointI(nBadCol+1, 0), 

378 lsst.geom.ExtentI(3, 10))) 

379 # 

380 # With another bad set of columns next to bad right edge 

381 # 

382 ima[0:10, -nBadCol-4:-nBadCol-1] = 100 

383 defects.append((lsst.geom.BoxI(lsst.geom.PointI(mi.getWidth() - nBadCol - 4, 0), 

384 lsst.geom.ExtentI(3, 10)))) 

385 # 

386 # Test cases that left and right bad patches nearly (or do) coalesce 

387 # 

388 ima[-3:, 0:mi.getWidth()//2-1] = 100 

389 defects.append(lsst.geom.BoxI(lsst.geom.PointI(0, mi.getHeight() - 3), 

390 lsst.geom.ExtentI(mi.getWidth()//2-1, 1))) 

391 

392 ima[-3:, mi.getWidth()//2+1:] = 100 

393 defects.append(lsst.geom.BoxI(lsst.geom.PointI(mi.getWidth()//2 + 1, mi.getHeight() - 3), 

394 lsst.geom.ExtentI(mi.getWidth()//2 - 1, 1))) 

395 

396 ima[-2:, 0:mi.getWidth()//2] = 100 

397 defects.append(lsst.geom.BoxI(lsst.geom.PointI(0, mi.getHeight() - 2), 

398 lsst.geom.ExtentI(mi.getWidth()//2, 1))) 

399 

400 ima[-2:, mi.getWidth()//2+1:] = 100 

401 defects.append(lsst.geom.BoxI(lsst.geom.PointI(mi.getWidth()//2 + 1, mi.getHeight() - 2), 

402 lsst.geom.ExtentI(mi.getWidth()//2 - 1, 1))) 

403 

404 ima[-1:, :] = 100 

405 defects.append(lsst.geom.BoxI(lsst.geom.PointI(0, mi.getHeight() - 1), 

406 lsst.geom.ExtentI(mi.getWidth(), 1))) 

407 

408 # Test fix for HSC-978: long defect stops one pixel shy of the edge (when nBadCol == 0) 

409 ima[13, :-1] = 100 

410 defects.append(lsst.geom.BoxI(lsst.geom.PointI(0, 13), lsst.geom.ExtentI(mi.getWidth() - 1, 1))) 

411 ima[14, 1:] = 100 

412 defects.append(lsst.geom.BoxI(lsst.geom.PointI(1, 14), lsst.geom.ExtentI(mi.getWidth() - 1, 1))) 

413 

414 # 

415 # Build list of defects to interpolate over 

416 # 

417 defectList = algorithms.Defects() 

418 

419 for bbox in defects: 

420 defectList.append(algorithms.Defect(bbox)) 

421 # 

422 # Guess a PSF and do the work 

423 # 

424 if display: 

425 afwDisplay.Display(frame=2).mtv(mi, title=self._testMethodName + ": image") 

426 

427 psf = algorithms.DoubleGaussianPsf(15, 15, 1./(2*math.sqrt(2*math.log(2)))) 

428 algorithms.interpolateOverDefects(mi, psf, defectList, 0, True) 

429 

430 if display: 

431 afwDisplay.Display(frame=3).mtv(mi, title=self._testMethodName + ": image") 

432 

433 self.assertGreater(np.min(ima), -2) 

434 self.assertGreater(2, np.max(ima)) 

435 

436 

437class TestMemory(lsst.utils.tests.MemoryTestCase): 

438 pass 

439 

440 

441def setup_module(module): 

442 lsst.utils.tests.init() 

443 

444 

445if __name__ == "__main__": 445 ↛ 446line 445 didn't jump to line 446, because the condition on line 445 was never true

446 lsst.utils.tests.init() 

447 unittest.main()