Coverage for tests/test_actions.py: 16%

262 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-09-17 01:32 -0700

1# This file is part of analysis_tools. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import unittest 

23 

24import astropy.units as u 

25import numpy as np 

26import pandas as pd 

27from lsst.analysis.tools.actions.scalar.scalarActions import ( 

28 ApproxFloor, 

29 CountAction, 

30 MeanAction, 

31 MedianAction, 

32 SigmaMadAction, 

33 StdevAction, 

34) 

35from lsst.analysis.tools.actions.vector.calcBinnedStats import CalcBinnedStatsAction 

36from lsst.analysis.tools.actions.vector.calcShapeSize import CalcShapeSize 

37from lsst.analysis.tools.actions.vector.selectors import ( 

38 CoaddPlotFlagSelector, 

39 FlagSelector, 

40 GalaxySelector, 

41 RangeSelector, 

42 SkyObjectSelector, 

43 SnSelector, 

44 StarSelector, 

45 VectorSelector, 

46) 

47from lsst.analysis.tools.actions.vector.vectorActions import ( 

48 DivideVector, 

49 DownselectVector, 

50 ExtinctionCorrectedMagDiff, 

51 FractionalDifference, 

52 LoadVector, 

53 MagColumnNanoJansky, 

54 MagDiff, 

55 SubtractVector, 

56) 

57 

58 

59class TestScalarActions(unittest.TestCase): 

60 """ "Test ScalarActions""" 

61 

62 def setUp(self): 

63 x = np.arange(100, dtype=float) 

64 x[31] = np.nan 

65 x[41] = np.nan 

66 x[59] = np.nan 

67 y = x**2 

68 self.data = {"r_y": x, "i_y": y} 

69 self.mask = np.zeros(100, dtype=bool) 

70 self.mask[1:50] = 1 

71 

72 def _testScalarActionAlmostEqual(self, cls, truth, maskedTruth): 

73 action = cls(vectorKey="{band}_y") 

74 schema = [col for col, colType in action.getInputSchema()] 

75 self.assertEqual(schema, ["{band}_y"]) 

76 result = action(self.data, band="i") 

77 self.assertAlmostEqual(result, truth) 

78 result = action(self.data, mask=self.mask, band="i") 

79 self.assertAlmostEqual(result, maskedTruth) 

80 

81 def testMedianAction(self): 

82 self._testScalarActionAlmostEqual(MedianAction, 2500, 576) 

83 

84 def testMeanAction(self): 

85 self._testScalarActionAlmostEqual(MeanAction, 3321.9278350515465, 803.8936170212766) 

86 

87 def testStdevAction(self): 

88 self._testScalarActionAlmostEqual(StdevAction, 2984.5855649976297, 733.5989754407842) 

89 

90 def testSigmaMadAction(self): 

91 self._testScalarActionAlmostEqual(SigmaMadAction, 3278.033505115886, 759.0923358748682) 

92 

93 def testCountAction(self): 

94 self._testScalarActionAlmostEqual(CountAction, 97, 47) 

95 

96 def testApproxFloorAction(self): 

97 self._testScalarActionAlmostEqual(ApproxFloor, 9216.0, 2352.5) 

98 

99 

100class TestVectorActions(unittest.TestCase): 

101 """Test VectorActions""" 

102 

103 def setUp(self): 

104 self.size = 5 

105 r = np.arange(self.size) + 1 

106 i = r**2 

107 rFlag = np.ones(self.size) 

108 rFlag[1] = 0 

109 iFlag = np.ones(self.size) 

110 iFlag[3] = 0 

111 data = { 

112 "r_vector": r, 

113 "i_vector": i, 

114 "r_flag": rFlag, 

115 "i_flag": iFlag, 

116 "g_vector": 3 * r, 

117 "E(B-V)": r[::-1], 

118 "r_ixx": r**2, 

119 "r_iyy": r[::-1] ** 2, 

120 "r_ixy": r * r[::-1], 

121 "g_iixx": r**2, 

122 "g_iiyy": r[::-1] ** 2, 

123 "g_iixy": r * r[::-1], 

124 } 

125 

126 self.data = pd.DataFrame.from_dict(data) 

127 

128 def _checkSchema(self, action, truth): 

129 schema = sorted([col for col, colType in action.getInputSchema()]) 

130 self.assertEqual(schema, truth) 

131 

132 def testDownselectVector(self): 

133 selector = FlagSelector(selectWhenTrue=["{band}_flag"]) 

134 action = DownselectVector(vectorKey="{band}_vector", selector=selector) 

135 result = action(self.data, band="r") 

136 self._checkSchema(action, ["{band}_flag", "{band}_vector"]) 

137 np.testing.assert_array_equal(result, np.array([1, 3, 4, 5])) 

138 

139 def testMagColumnNanoJansky(self): 

140 truth = [ 

141 31.4, 

142 29.89485002168, 

143 29.0143937264, 

144 28.38970004336, 

145 27.90514997832, 

146 ] 

147 action = MagColumnNanoJansky(vectorKey="{band}_vector") 

148 result = action(self.data, band="i") 

149 self._checkSchema(action, ["{band}_vector"]) 

150 np.testing.assert_array_almost_equal(result, truth) 

151 

152 def testFractionalDifference(self): 

153 actionA = LoadVector(vectorKey="{band1}_vector") 

154 actionB = LoadVector(vectorKey="{band2}_vector") 

155 truth = [0.0, -0.5, -0.6666666666666666, -0.75, -0.8] 

156 diff = FractionalDifference(actionA=actionA, actionB=actionB) 

157 result = diff(self.data, band1="r", band2="i") 

158 self._checkSchema(diff, ["{band1}_vector", "{band2}_vector"]) 

159 np.testing.assert_array_almost_equal(result, truth) 

160 

161 def testSubtract(self): 

162 actionA = LoadVector(vectorKey="{band1}_vector") 

163 actionB = LoadVector(vectorKey="{band2}_vector") 

164 truth = [0.0, -2.0, -6.0, -12.0, -20.0] 

165 diff = SubtractVector(actionA=actionA, actionB=actionB) 

166 result = diff(self.data, band1="r", band2="i") 

167 self._checkSchema(diff, ["{band1}_vector", "{band2}_vector"]) 

168 np.testing.assert_array_almost_equal(result, truth) 

169 

170 def testDivide(self): 

171 actionA = LoadVector(vectorKey="{band1}_vector") 

172 actionB = LoadVector(vectorKey="{band2}_vector") 

173 truth = 1 / np.arange(1, 6) 

174 diff = DivideVector(actionA=actionA, actionB=actionB) 

175 result = diff(self.data, band1="r", band2="i") 

176 self._checkSchema(diff, ["{band1}_vector", "{band2}_vector"]) 

177 np.testing.assert_array_almost_equal(result, truth) 

178 

179 def testMagDiff(self): 

180 # Use the same units as the data so that we know the difference exactly 

181 # without conversions. 

182 # testExtinctionCorrectedMagDiff will test the conversions 

183 truth = np.array(2 * self.data["r_vector"]) * u.ABmag 

184 action = MagDiff( 

185 col1="{band1}_vector", 

186 col2="{band2}_vector", 

187 fluxUnits1="mag(AB)", 

188 fluxUnits2="mag(AB)", 

189 returnMillimags=False, 

190 ) 

191 result = action(self.data, band1="g", band2="r") 

192 self._checkSchema(action, ["{band1}_vector", "{band2}_vector"]) 

193 np.testing.assert_array_almost_equal(result, truth.value) 

194 

195 def testExtinctionCorrectedMagDiff(self): 

196 for returnMillimags in (True, False): 

197 # Check that conversions are working properly 

198 magDiff = MagDiff( 

199 col1="{band1}_vector", 

200 col2="{band2}_vector", 

201 fluxUnits1="jansky", 

202 fluxUnits2="jansky", 

203 returnMillimags=returnMillimags, 

204 ) 

205 action = ExtinctionCorrectedMagDiff( 

206 magDiff=magDiff, 

207 band1="g", 

208 band2="r", 

209 ebvCol="E(B-V)", 

210 extinctionCoeffs={"g": 0.2, "r": 1.5}, 

211 ) 

212 

213 result = action(self.data, band1="g", band2="r") 

214 lhs = (np.array(self.data["g_vector"]) * u.jansky).to(u.ABmag) 

215 rhs = (np.array(self.data["r_vector"]) * u.jansky).to(u.ABmag) 

216 diff = lhs - rhs 

217 correction = np.array((0.2 - 1.5) * self.data["E(B-V)"]) * u.mag 

218 if returnMillimags: 

219 diff = diff.to(u.mmag) 

220 correction = correction.to(u.mmag) 

221 truth = diff - correction 

222 self._checkSchema(action, ["E(B-V)", "{band1}_vector", "{band2}_vector"]) 

223 np.testing.assert_array_almost_equal(result, truth.value) 

224 

225 # Test with hard coded bands 

226 magDiff = MagDiff( 

227 col1="g_vector", 

228 col2="r_vector", 

229 fluxUnits1="jansky", 

230 fluxUnits2="jansky", 

231 returnMillimags=False, 

232 ) 

233 action = ExtinctionCorrectedMagDiff( 

234 magDiff=magDiff, 

235 ebvCol="E(B-V)", 

236 extinctionCoeffs={"g": 0.2, "r": 1.5}, 

237 ) 

238 result = action(self.data) 

239 lhs = (np.array(self.data["g_vector"]) * u.jansky).to(u.ABmag) 

240 rhs = (np.array(self.data["r_vector"]) * u.jansky).to(u.ABmag) 

241 diff = lhs - rhs 

242 correction = np.array((0.2 - 1.5) * self.data["E(B-V)"]) * u.mag 

243 truth = diff - correction 

244 self._checkSchema(action, ["E(B-V)", "g_vector", "r_vector"]) 

245 np.testing.assert_array_almost_equal(result, truth.value) 

246 

247 def testCalcBinnedStats(self): 

248 selector = RangeSelector(column="r_vector", minimum=0, maximum=self.size + 1) 

249 prefix = "a" 

250 stats = CalcBinnedStatsAction(name_prefix=prefix, selector_range=selector, key_vector="r_vector") 

251 result = stats(self.data) 

252 median = (1 + self.size) / 2.0 

253 truth = { 

254 stats.name_mask: np.ones(self.size), 

255 stats.name_median: median, 

256 stats.name_sigmaMad: 1.482602218505602 * np.median(np.abs(self.data["r_vector"] - median)), 

257 stats.name_count: self.size, 

258 stats.name_select_maximum: self.size, 

259 stats.name_select_median: median, 

260 stats.name_select_minimum: 1, 

261 "range_maximum": self.size + 1, 

262 "range_minimum": 0, 

263 } 

264 self.assertEqual(list(result.keys()), list(truth.keys())) 

265 

266 self.assertAlmostEqual(result[stats.name_sigmaMad], truth[stats.name_sigmaMad]) 

267 del truth[stats.name_sigmaMad] 

268 

269 np.testing.assert_array_equal(result[stats.name_mask], truth[stats.name_mask]) 

270 del truth[stats.name_mask] 

271 

272 for key, value in truth.items(): 

273 self.assertEqual(result[key], value, key) 

274 

275 def testCalcShapeSize(self): 

276 xx = self.data["r_ixx"] 

277 yy = self.data["r_iyy"] 

278 xy = self.data["r_ixy"] 

279 

280 # Test determinant with defaults 

281 action = CalcShapeSize() 

282 result = action(self.data, band="r") 

283 schema = [col for col, colType in action.getInputSchema()] 

284 self.assertEqual(sorted(schema), ["{band}_ixx", "{band}_ixy", "{band}_iyy"]) 

285 truth = 0.25 * (xx * yy - xy**2) 

286 np.testing.assert_array_almost_equal(result, truth) 

287 

288 # Test trace with columns specified 

289 action = CalcShapeSize( 

290 colXx="{band}_iixx", 

291 colYy="{band}_iiyy", 

292 colXy="{band}_iixy", 

293 sizeType="trace", 

294 ) 

295 result = action(self.data, band="g") 

296 schema = [col for col, colType in action.getInputSchema()] 

297 self.assertEqual(sorted(schema), ["{band}_iixx", "{band}_iiyy"]) 

298 truth = np.sqrt(0.5 * (xx + yy)) 

299 np.testing.assert_array_almost_equal(result, truth) 

300 

301 

302class TestVectorSelectors(unittest.TestCase): 

303 def setUp(self): 

304 self.size = 20 

305 falseFlags = { 

306 "{band}_psfFlux_flag": [1], 

307 "{band}_pixelFlags_saturatedCenter": [3], 

308 "{band}_extendedness_flag": [5], 

309 "xy_flag": [7], 

310 "i_pixelFlags_edge": [13], 

311 "r_pixelFlags_edge": [15], 

312 "sky_object": [13, 15, 17], 

313 } 

314 

315 trueFlags = { 

316 "detect_isPatchInner": [9], 

317 "detect_isDeblendedSource": [11], 

318 } 

319 

320 flux = np.arange(self.size) * 10 

321 fluxErr = np.ones(self.size) * 0.1 

322 extendedness = np.arange(20) / 20 - 0.1 

323 

324 self.data = { 

325 "r_psfFlux": flux, 

326 "r_psfFluxErr": fluxErr, 

327 "i_cmodelFlux": flux[::-1], 

328 "i_cmodelFluxError": fluxErr[::-1], 

329 "r_cmodelFlux": flux, 

330 "r_cmodelFluxError": fluxErr, 

331 "i_extendedness": extendedness, 

332 "i_extended": extendedness, 

333 } 

334 bands = ("r", "i") 

335 for band in bands: 

336 for flag, bits in falseFlags.items(): 

337 vector = np.zeros(self.size, dtype=bool) 

338 for bit in bits: 

339 vector[bit] = 1 

340 self.data[flag.format(band=band)] = vector 

341 

342 for flag, bits in trueFlags.items(): 

343 vector = np.ones(self.size, dtype=bool) 

344 for bit in bits: 

345 vector[bit] = 0 

346 self.data[flag.format(band=band)] = vector 

347 

348 def _checkSchema(self, action, truth): 

349 schema = [col for col, colType in action.getInputSchema()] 

350 self.assertEqual(sorted(schema), sorted(truth)) 

351 

352 def testFlagSelector(self): 

353 selector = FlagSelector( 

354 selectWhenFalse=["{band}_psfFlux_flag"], selectWhenTrue=["detect_isPatchInner"] 

355 ) 

356 self._checkSchema(selector, ["detect_isPatchInner", "{band}_psfFlux_flag"]) 

357 result = selector(self.data, band="r") 

358 truth = np.ones(self.size, dtype=bool) 

359 truth[1] = False 

360 truth[9] = False 

361 np.testing.assert_array_equal(result, truth) 

362 

363 def testCoaddPlotFlagSelector(self): 

364 # Test defaults 

365 selector = CoaddPlotFlagSelector() 

366 keys = [ 

367 "{band}_psfFlux_flag", 

368 "{band}_pixelFlags_saturatedCenter", 

369 "{band}_extendedness_flag", 

370 "xy_flag", 

371 "detect_isPatchInner", 

372 "detect_isDeblendedSource", 

373 ] 

374 self._checkSchema(selector, keys) 

375 

376 result = selector(self.data) 

377 truth = np.ones(self.size, dtype=bool) 

378 for bit in (1, 3, 5, 7, 9, 11): 

379 truth[bit] = 0 

380 np.testing.assert_array_equal(result, truth) 

381 

382 # Test bands override 

383 selector = CoaddPlotFlagSelector( 

384 bands=["i", "r"], 

385 selectWhenFalse=["{band}_psfFlux_flag"], 

386 selectWhenTrue=["detect_isDeblendedSource"], 

387 ) 

388 self._checkSchema(selector, ["{band}_psfFlux_flag", "detect_isDeblendedSource"]) 

389 result = selector(self.data) 

390 truth = np.ones(self.size, dtype=bool) 

391 for bit in (1, 11): 

392 truth[bit] = 0 

393 np.testing.assert_array_equal(result, truth) 

394 

395 def testRangeSelector(self): 

396 selector = RangeSelector(column="r_psfFlux", minimum=np.nextafter(20, 30), maximum=50) 

397 self._checkSchema(selector, ["r_psfFlux"]) 

398 result = self.data["r_psfFlux"][selector(self.data)] 

399 truth = [30, 40] 

400 np.testing.assert_array_equal(result, truth) 

401 

402 def testSnSelector(self): 

403 # test defaults 

404 selector = SnSelector() 

405 keys = [ 

406 "{band}_psfFlux", 

407 "{band}_psfFluxErr", 

408 ] 

409 self._checkSchema(selector, keys) 

410 result = selector(self.data, bands=["r"]) 

411 truth = np.ones(self.size, dtype=bool) 

412 truth[:6] = 0 

413 np.testing.assert_array_equal(result, truth) 

414 

415 # test overrides 

416 selector = SnSelector( 

417 fluxType="{band}_cmodelFlux", 

418 threshold=200.0, 

419 uncertaintySuffix="Error", 

420 bands=["r", "i"], 

421 ) 

422 keys = [ 

423 "{band}_cmodelFlux", 

424 "{band}_cmodelFluxError", 

425 ] 

426 self._checkSchema(selector, keys) 

427 result = selector(self.data) 

428 truth = np.ones(self.size, dtype=bool) 

429 truth[:3] = 0 

430 truth[-3:] = 0 

431 np.testing.assert_array_equal(result, truth) 

432 

433 def testSkyObjectSelector(self): 

434 # Test default 

435 selector = SkyObjectSelector() 

436 keys = ["{band}_pixelFlags_edge", "sky_object"] 

437 self._checkSchema(selector, keys) 

438 result = selector(self.data) 

439 truth = np.zeros(self.size, dtype=bool) 

440 truth[15] = 1 

441 truth[17] = 1 

442 np.testing.assert_array_equal(result, truth) 

443 

444 # Test overrides 

445 selector = SkyObjectSelector(bands=["i", "r"]) 

446 self._checkSchema(selector, keys) 

447 result = selector(self.data) 

448 truth = np.zeros(self.size, dtype=bool) 

449 truth[17] = 1 

450 np.testing.assert_array_equal(result, truth) 

451 

452 def testStarSelector(self): 

453 # test default 

454 selector = StarSelector() 

455 self._checkSchema(selector, ["{band}_extendedness"]) 

456 result = selector(self.data, band="i") 

457 truth = (self.data["i_extendedness"] >= 0) & (self.data["i_extendedness"] < 0.5) 

458 np.testing.assert_array_almost_equal(result, truth) 

459 

460 # Test overrides 

461 selector = StarSelector(vectorKey="i_extended", extendedness_maximum=0.3) 

462 result = selector(self.data, band="i") 

463 truth = (self.data["i_extendedness"] >= 0) & (self.data["i_extendedness"] < 0.3) 

464 np.testing.assert_array_almost_equal(result, truth) 

465 

466 def testGalaxySelector(self): 

467 # test default 

468 selector = GalaxySelector() 

469 self._checkSchema(selector, ["{band}_extendedness"]) 

470 result = selector(self.data, band="i") 

471 truth = self.data["i_extendedness"] > 0.5 

472 np.testing.assert_array_almost_equal(result, truth) 

473 

474 # Test overrides 

475 selector = GalaxySelector(vectorKey="i_extended", extendedness_minimum=0.3) 

476 result = selector(self.data, band="i") 

477 truth = self.data["i_extendedness"] > 0.3 

478 np.testing.assert_array_almost_equal(result, truth) 

479 

480 def testVectorSelector(self): 

481 selector = VectorSelector(vectorKey="{band}_psfFlux_flag") 

482 self._checkSchema(selector, ["{band}_psfFlux_flag"]) 

483 result = selector(self.data, band="i") 

484 truth = np.zeros(self.size, dtype=bool) 

485 truth[1] = True 

486 np.testing.assert_array_equal(result, truth)