Coverage for tests/test_actions.py: 15%

274 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-18 03:21 -0700

1# This file is part of analysis_tools. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import unittest 

23 

24import astropy.units as u 

25import numpy as np 

26import pandas as pd 

27from lsst.analysis.tools.actions.scalar.scalarActions import ( 

28 ApproxFloor, 

29 CountAction, 

30 MeanAction, 

31 MedianAction, 

32 SigmaMadAction, 

33 StdevAction, 

34) 

35from lsst.analysis.tools.actions.vector.calcBinnedStats import CalcBinnedStatsAction 

36from lsst.analysis.tools.actions.vector.calcShapeSize import CalcShapeSize 

37from lsst.analysis.tools.actions.vector.mathActions import ( 

38 AddVector, 

39 ConstantValue, 

40 DivideVector, 

41 FractionalDifference, 

42 MultiplyVector, 

43 SubtractVector, 

44) 

45from lsst.analysis.tools.actions.vector.selectors import ( 

46 CoaddPlotFlagSelector, 

47 FlagSelector, 

48 GalaxySelector, 

49 RangeSelector, 

50 SkyObjectSelector, 

51 SnSelector, 

52 StarSelector, 

53 VectorSelector, 

54) 

55from lsst.analysis.tools.actions.vector.vectorActions import ( 

56 ConvertFluxToMag, 

57 DownselectVector, 

58 ExtinctionCorrectedMagDiff, 

59 LoadVector, 

60 MagDiff, 

61) 

62 

63 

64class TestScalarActions(unittest.TestCase): 

65 """ "Test ScalarActions""" 

66 

67 def setUp(self): 

68 x = np.arange(100, dtype=float) 

69 x[31] = np.nan 

70 x[41] = np.nan 

71 x[59] = np.nan 

72 y = x**2 

73 self.data = {"r_y": x, "i_y": y} 

74 self.mask = np.zeros(100, dtype=bool) 

75 self.mask[1:50] = 1 

76 

77 def _testScalarActionAlmostEqual(self, cls, truth, maskedTruth): 

78 action = cls(vectorKey="{band}_y") 

79 schema = [col for col, colType in action.getInputSchema()] 

80 self.assertEqual(schema, ["{band}_y"]) 

81 result = action(self.data, band="i") 

82 self.assertAlmostEqual(result, truth) 

83 result = action(self.data, mask=self.mask, band="i") 

84 self.assertAlmostEqual(result, maskedTruth) 

85 

86 def testMedianAction(self): 

87 self._testScalarActionAlmostEqual(MedianAction, 2500, 576) 

88 

89 def testMeanAction(self): 

90 self._testScalarActionAlmostEqual(MeanAction, 3321.9278350515465, 803.8936170212766) 

91 

92 def testStdevAction(self): 

93 self._testScalarActionAlmostEqual(StdevAction, 2984.5855649976297, 733.5989754407842) 

94 

95 def testSigmaMadAction(self): 

96 self._testScalarActionAlmostEqual(SigmaMadAction, 3278.033505115886, 759.0923358748682) 

97 

98 def testCountAction(self): 

99 self._testScalarActionAlmostEqual(CountAction, 97, 47) 

100 

101 def testApproxFloorAction(self): 

102 self._testScalarActionAlmostEqual(ApproxFloor, 9216.0, 2352.5) 

103 

104 

105class TestVectorActions(unittest.TestCase): 

106 """Test VectorActions""" 

107 

108 def setUp(self): 

109 self.size = 5 

110 r = np.arange(self.size) + 1 

111 i = r**2 

112 rFlag = np.ones(self.size) 

113 rFlag[1] = 0 

114 iFlag = np.ones(self.size) 

115 iFlag[3] = 0 

116 data = { 

117 "r_vector": r, 

118 "i_vector": i, 

119 "r_flag": rFlag, 

120 "i_flag": iFlag, 

121 "g_vector": 3 * r, 

122 "E(B-V)": r[::-1], 

123 "r_ixx": r**2, 

124 "r_iyy": r[::-1] ** 2, 

125 "r_ixy": r * r[::-1], 

126 "g_iixx": r**2, 

127 "g_iiyy": r[::-1] ** 2, 

128 "g_iixy": r * r[::-1], 

129 } 

130 

131 self.data = pd.DataFrame.from_dict(data) 

132 

133 def _checkSchema(self, action, truth): 

134 schema = sorted([col for col, colType in action.getInputSchema()]) 

135 self.assertEqual(schema, truth) 

136 

137 # VectorActions with their own files 

138 

139 def testCalcBinnedStats(self): 

140 selector = RangeSelector(key="r_vector", minimum=0, maximum=self.size + 1) 

141 prefix = "a_" 

142 stats = CalcBinnedStatsAction(name_prefix=prefix, selector_range=selector, key_vector="r_vector") 

143 result = stats(self.data) 

144 median = (1 + self.size) / 2.0 

145 truth = { 

146 stats.name_mask: np.ones(self.size), 

147 stats.name_median: median, 

148 stats.name_sigmaMad: 1.482602218505602 * np.median(np.abs(self.data["r_vector"] - median)), 

149 stats.name_count: self.size, 

150 stats.name_select_maximum: self.size, 

151 stats.name_select_median: median, 

152 stats.name_select_minimum: 1, 

153 "range_maximum": self.size + 1, 

154 "range_minimum": 0, 

155 } 

156 self.assertEqual(list(result.keys()), list(truth.keys())) 

157 

158 self.assertAlmostEqual(result[stats.name_sigmaMad], truth[stats.name_sigmaMad]) 

159 del truth[stats.name_sigmaMad] 

160 

161 np.testing.assert_array_equal(result[stats.name_mask], truth[stats.name_mask]) 

162 del truth[stats.name_mask] 

163 

164 for key, value in truth.items(): 

165 self.assertEqual(result[key], value, key) 

166 

167 # def testCalcRhoStatistics(self): TODO: implement 

168 

169 def testCalcShapeSize(self): 

170 xx = self.data["r_ixx"] 

171 yy = self.data["r_iyy"] 

172 xy = self.data["r_ixy"] 

173 

174 # Test determinant with defaults 

175 action = CalcShapeSize() 

176 result = action(self.data, band="r") 

177 schema = [col for col, colType in action.getInputSchema()] 

178 self.assertEqual(sorted(schema), ["{band}_ixx", "{band}_ixy", "{band}_iyy"]) 

179 truth = 0.25 * (xx * yy - xy**2) 

180 np.testing.assert_array_almost_equal(result, truth) 

181 

182 # Test trace with columns specified 

183 action = CalcShapeSize( 

184 colXx="{band}_iixx", 

185 colYy="{band}_iiyy", 

186 colXy="{band}_iixy", 

187 sizeType="trace", 

188 ) 

189 result = action(self.data, band="g") 

190 schema = [col for col, colType in action.getInputSchema()] 

191 self.assertEqual(sorted(schema), ["{band}_iixx", "{band}_iiyy"]) 

192 truth = np.sqrt(0.5 * (xx + yy)) 

193 np.testing.assert_array_almost_equal(result, truth) 

194 

195 # def testCalcE(self): TODO: implement 

196 

197 # def testCalcEDiff(self): TODO: implement 

198 

199 # def testCalcE1(self): TODO: implement 

200 

201 # def testCalcE2(self): TODO: implement 

202 

203 # MathActions 

204 

205 def _testMath(self, ActionType, truth, compare_exact: bool = False): 

206 actionA = LoadVector(vectorKey="{band1}_vector") 

207 actionB = LoadVector(vectorKey="{band2}_vector") 

208 action = ActionType(actionA=actionA, actionB=actionB) 

209 result = action(self.data, band1="r", band2="i") 

210 self._checkSchema(action, [actionA.vectorKey, actionB.vectorKey]) 

211 if compare_exact: 

212 np.testing.assert_array_equal(result, truth) 

213 else: 

214 np.testing.assert_array_almost_equal(result, truth) 

215 

216 def testConstant(self): 

217 truth = [42.0] 

218 action = ConstantValue(value=truth[0]) 

219 self._checkSchema(action, []) 

220 result = action({}) 

221 np.testing.assert_array_equal(result, truth) 

222 

223 def testAdd(self): 

224 truth = [2.0, 6.0, 12.0, 20.0, 30.0] 

225 self._testMath(AddVector, truth, True) 

226 

227 def testSubtract(self): 

228 truth = [0.0, -2.0, -6.0, -12.0, -20.0] 

229 self._testMath(SubtractVector, truth, True) 

230 

231 def testMultiply(self): 

232 truth = [1.0, 8.0, 27.0, 64.0, 125.0] 

233 self._testMath(MultiplyVector, truth, False) 

234 

235 def testDivide(self): 

236 truth = 1 / np.arange(1, 6) 

237 self._testMath(DivideVector, truth, False) 

238 

239 def testFractionalDifference(self): 

240 actionA = LoadVector(vectorKey="{band1}_vector") 

241 actionB = LoadVector(vectorKey="{band2}_vector") 

242 truth = [0.0, -0.5, -0.6666666666666666, -0.75, -0.8] 

243 diff = FractionalDifference(actionA=actionA, actionB=actionB) 

244 result = diff(self.data, band1="r", band2="i") 

245 self._checkSchema(diff, ["{band1}_vector", "{band2}_vector"]) 

246 np.testing.assert_array_almost_equal(result, truth) 

247 

248 # Basic vectorActions 

249 

250 # def testLoadVector(self): TODO: implement 

251 

252 def testDownselectVector(self): 

253 selector = FlagSelector(selectWhenTrue=["{band}_flag"]) 

254 action = DownselectVector(vectorKey="{band}_vector", selector=selector) 

255 result = action(self.data, band="r") 

256 self._checkSchema(action, ["{band}_flag", "{band}_vector"]) 

257 np.testing.assert_array_equal(result, np.array([1, 3, 4, 5])) 

258 

259 # def testMultiCriteriaDownselectVector(self): TODO: implement 

260 

261 # Astronomical vectorActions 

262 

263 # def testCalcSn(self): TODO: implement 

264 

265 def testConvertFluxToMag(self): 

266 truth = [ 

267 31.4, 

268 29.89485002168, 

269 29.0143937264, 

270 28.38970004336, 

271 27.90514997832, 

272 ] 

273 action = ConvertFluxToMag(vectorKey="{band}_vector") 

274 result = action(self.data, band="i") 

275 self._checkSchema(action, ["{band}_vector"]) 

276 np.testing.assert_array_almost_equal(result, truth) 

277 

278 # def testConvertUnits(self): TODO: implement 

279 

280 def testMagDiff(self): 

281 # Use the same units as the data so that we know the difference exactly 

282 # without conversions. 

283 # testExtinctionCorrectedMagDiff will test the conversions 

284 truth = np.array(2 * self.data["r_vector"]) * u.ABmag 

285 action = MagDiff( 

286 col1="{band1}_vector", 

287 col2="{band2}_vector", 

288 fluxUnits1="mag(AB)", 

289 fluxUnits2="mag(AB)", 

290 returnMillimags=False, 

291 ) 

292 result = action(self.data, band1="g", band2="r") 

293 self._checkSchema(action, ["{band1}_vector", "{band2}_vector"]) 

294 np.testing.assert_array_almost_equal(result, truth.value) 

295 

296 def testExtinctionCorrectedMagDiff(self): 

297 for returnMillimags in (True, False): 

298 # Check that conversions are working properly 

299 magDiff = MagDiff( 

300 col1="{band1}_vector", 

301 col2="{band2}_vector", 

302 fluxUnits1="jansky", 

303 fluxUnits2="jansky", 

304 returnMillimags=returnMillimags, 

305 ) 

306 action = ExtinctionCorrectedMagDiff( 

307 magDiff=magDiff, 

308 band1="g", 

309 band2="r", 

310 ebvCol="E(B-V)", 

311 extinctionCoeffs={"g": 0.2, "r": 1.5}, 

312 ) 

313 

314 result = action(self.data, band1="g", band2="r") 

315 lhs = (np.array(self.data["g_vector"]) * u.jansky).to(u.ABmag) 

316 rhs = (np.array(self.data["r_vector"]) * u.jansky).to(u.ABmag) 

317 diff = lhs - rhs 

318 correction = np.array((0.2 - 1.5) * self.data["E(B-V)"]) * u.mag 

319 if returnMillimags: 

320 diff = diff.to(u.mmag) 

321 correction = correction.to(u.mmag) 

322 truth = diff - correction 

323 self._checkSchema(action, ["E(B-V)", "{band1}_vector", "{band2}_vector"]) 

324 np.testing.assert_array_almost_equal(result, truth.value) 

325 

326 # Test with hard coded bands 

327 magDiff = MagDiff( 

328 col1="g_vector", 

329 col2="r_vector", 

330 fluxUnits1="jansky", 

331 fluxUnits2="jansky", 

332 returnMillimags=False, 

333 ) 

334 action = ExtinctionCorrectedMagDiff( 

335 magDiff=magDiff, 

336 ebvCol="E(B-V)", 

337 extinctionCoeffs={"g": 0.2, "r": 1.5}, 

338 ) 

339 result = action(self.data) 

340 lhs = (np.array(self.data["g_vector"]) * u.jansky).to(u.ABmag) 

341 rhs = (np.array(self.data["r_vector"]) * u.jansky).to(u.ABmag) 

342 diff = lhs - rhs 

343 correction = np.array((0.2 - 1.5) * self.data["E(B-V)"]) * u.mag 

344 truth = diff - correction 

345 self._checkSchema(action, ["E(B-V)", "g_vector", "r_vector"]) 

346 np.testing.assert_array_almost_equal(result, truth.value) 

347 

348 # def testRAcosDec(self): TODO: implement 

349 

350 # Statistical vectorActions 

351 

352 # def testPerGroupStatistic(self): TODO: implement 

353 

354 # def testResidualWithPerGroupStatistic(self): TODO: implement 

355 

356 

357class TestVectorSelectors(unittest.TestCase): 

358 def setUp(self): 

359 self.size = 20 

360 falseFlags = { 

361 "{band}_psfFlux_flag": [1], 

362 "{band}_pixelFlags_saturatedCenter": [3], 

363 "{band}_extendedness_flag": [5], 

364 "xy_flag": [7], 

365 "i_pixelFlags_edge": [13], 

366 "r_pixelFlags_edge": [15], 

367 "sky_object": [13, 15, 17], 

368 } 

369 

370 trueFlags = { 

371 "detect_isPatchInner": [9], 

372 "detect_isDeblendedSource": [11], 

373 } 

374 

375 flux = np.arange(self.size) * 10 

376 fluxErr = np.ones(self.size) * 0.1 

377 extendedness = np.arange(20) / 20 - 0.1 

378 

379 self.data = { 

380 "r_psfFlux": flux, 

381 "r_psfFluxErr": fluxErr, 

382 "i_cmodelFlux": flux[::-1], 

383 "i_cmodelFluxError": fluxErr[::-1], 

384 "r_cmodelFlux": flux, 

385 "r_cmodelFluxError": fluxErr, 

386 "i_extendedness": extendedness, 

387 "i_extended": extendedness, 

388 } 

389 bands = ("r", "i") 

390 for band in bands: 

391 for flag, bits in falseFlags.items(): 

392 vector = np.zeros(self.size, dtype=bool) 

393 for bit in bits: 

394 vector[bit] = 1 

395 self.data[flag.format(band=band)] = vector 

396 

397 for flag, bits in trueFlags.items(): 

398 vector = np.ones(self.size, dtype=bool) 

399 for bit in bits: 

400 vector[bit] = 0 

401 self.data[flag.format(band=band)] = vector 

402 

403 def _checkSchema(self, action, truth): 

404 schema = [col for col, colType in action.getInputSchema()] 

405 self.assertEqual(sorted(schema), sorted(truth)) 

406 

407 def testFlagSelector(self): 

408 selector = FlagSelector( 

409 selectWhenFalse=["{band}_psfFlux_flag"], selectWhenTrue=["detect_isPatchInner"] 

410 ) 

411 self._checkSchema(selector, ["detect_isPatchInner", "{band}_psfFlux_flag"]) 

412 result = selector(self.data, band="r") 

413 truth = np.ones(self.size, dtype=bool) 

414 truth[1] = False 

415 truth[9] = False 

416 np.testing.assert_array_equal(result, truth) 

417 

418 def testCoaddPlotFlagSelector(self): 

419 # Test defaults 

420 # Bands needs to be set to something otherwise it 

421 # will crash as the default value looks it up 

422 # elsewhere. 

423 selector = CoaddPlotFlagSelector(bands=["i"]) 

424 keys = [ 

425 "{band}_psfFlux_flag", 

426 "{band}_pixelFlags_saturatedCenter", 

427 "{band}_extendedness_flag", 

428 "xy_flag", 

429 "detect_isPatchInner", 

430 "detect_isDeblendedSource", 

431 ] 

432 self._checkSchema(selector, keys) 

433 

434 result = selector(self.data) 

435 truth = np.ones(self.size, dtype=bool) 

436 for bit in (1, 3, 5, 7, 9, 11): 

437 truth[bit] = 0 

438 np.testing.assert_array_equal(result, truth) 

439 

440 # Test bands override 

441 selector = CoaddPlotFlagSelector( 

442 bands=["i", "r"], 

443 selectWhenFalse=["{band}_psfFlux_flag"], 

444 selectWhenTrue=["detect_isDeblendedSource"], 

445 ) 

446 self._checkSchema(selector, ["{band}_psfFlux_flag", "detect_isDeblendedSource"]) 

447 result = selector(self.data) 

448 truth = np.ones(self.size, dtype=bool) 

449 for bit in (1, 11): 

450 truth[bit] = 0 

451 np.testing.assert_array_equal(result, truth) 

452 

453 def testRangeSelector(self): 

454 selector = RangeSelector(key="r_psfFlux", minimum=np.nextafter(20, 30), maximum=50) 

455 self._checkSchema(selector, ["r_psfFlux"]) 

456 result = self.data["r_psfFlux"][selector(self.data)] 

457 truth = [30, 40] 

458 np.testing.assert_array_equal(result, truth) 

459 

460 def testSnSelector(self): 

461 # test defaults 

462 selector = SnSelector() 

463 keys = [ 

464 "{band}_psfFlux", 

465 "{band}_psfFluxErr", 

466 ] 

467 self._checkSchema(selector, keys) 

468 result = selector(self.data, bands=["r"]) 

469 truth = np.ones(self.size, dtype=bool) 

470 truth[:6] = 0 

471 np.testing.assert_array_equal(result, truth) 

472 

473 # test overrides 

474 selector = SnSelector( 

475 fluxType="{band}_cmodelFlux", 

476 threshold=200.0, 

477 uncertaintySuffix="Error", 

478 bands=["r", "i"], 

479 ) 

480 keys = [ 

481 "{band}_cmodelFlux", 

482 "{band}_cmodelFluxError", 

483 ] 

484 self._checkSchema(selector, keys) 

485 result = selector(self.data) 

486 truth = np.ones(self.size, dtype=bool) 

487 truth[:3] = 0 

488 truth[-3:] = 0 

489 np.testing.assert_array_equal(result, truth) 

490 

491 def testSkyObjectSelector(self): 

492 # Test default 

493 selector = SkyObjectSelector() 

494 keys = ["{band}_pixelFlags_edge", "sky_object"] 

495 self._checkSchema(selector, keys) 

496 result = selector(self.data) 

497 truth = np.zeros(self.size, dtype=bool) 

498 truth[15] = 1 

499 truth[17] = 1 

500 np.testing.assert_array_equal(result, truth) 

501 

502 # Test overrides 

503 selector = SkyObjectSelector(bands=["i", "r"]) 

504 self._checkSchema(selector, keys) 

505 result = selector(self.data) 

506 truth = np.zeros(self.size, dtype=bool) 

507 truth[17] = 1 

508 np.testing.assert_array_equal(result, truth) 

509 

510 def testStarSelector(self): 

511 # test default 

512 selector = StarSelector() 

513 self._checkSchema(selector, ["{band}_extendedness"]) 

514 result = selector(self.data, band="i") 

515 truth = (self.data["i_extendedness"] >= 0) & (self.data["i_extendedness"] < 0.5) 

516 np.testing.assert_array_almost_equal(result, truth) 

517 

518 # Test overrides 

519 selector = StarSelector(vectorKey="i_extended", extendedness_maximum=0.3) 

520 result = selector(self.data, band="i") 

521 truth = (self.data["i_extendedness"] >= 0) & (self.data["i_extendedness"] < 0.3) 

522 np.testing.assert_array_almost_equal(result, truth) 

523 

524 def testGalaxySelector(self): 

525 # test default 

526 selector = GalaxySelector() 

527 self._checkSchema(selector, ["{band}_extendedness"]) 

528 result = selector(self.data, band="i") 

529 truth = self.data["i_extendedness"] > 0.5 

530 np.testing.assert_array_almost_equal(result, truth) 

531 

532 # Test overrides 

533 selector = GalaxySelector(vectorKey="i_extended", extendedness_minimum=0.3) 

534 result = selector(self.data, band="i") 

535 truth = self.data["i_extendedness"] > 0.3 

536 np.testing.assert_array_almost_equal(result, truth) 

537 

538 def testVectorSelector(self): 

539 selector = VectorSelector(vectorKey="{band}_psfFlux_flag") 

540 self._checkSchema(selector, ["{band}_psfFlux_flag"]) 

541 result = selector(self.data, band="i") 

542 truth = np.zeros(self.size, dtype=bool) 

543 truth[1] = True 

544 np.testing.assert_array_equal(result, truth)