Coverage for tests/test_actions.py: 15%

276 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-07 03:18 -0700

1# This file is part of analysis_tools. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import unittest 

23 

24import astropy.units as u 

25import numpy as np 

26import pandas as pd 

27from lsst.analysis.tools.actions.scalar.scalarActions import ( 

28 ApproxFloor, 

29 CountAction, 

30 MeanAction, 

31 MedianAction, 

32 SigmaMadAction, 

33 StdevAction, 

34) 

35from lsst.analysis.tools.actions.vector.calcBinnedStats import CalcBinnedStatsAction 

36from lsst.analysis.tools.actions.vector.calcShapeSize import CalcShapeSize 

37from lsst.analysis.tools.actions.vector.mathActions import ( 

38 AddVector, 

39 ConstantValue, 

40 DivideVector, 

41 FractionalDifference, 

42 MultiplyVector, 

43 SubtractVector, 

44) 

45from lsst.analysis.tools.actions.vector.selectors import ( 

46 CoaddPlotFlagSelector, 

47 FlagSelector, 

48 GalaxySelector, 

49 RangeSelector, 

50 SkyObjectSelector, 

51 SnSelector, 

52 StarSelector, 

53 VectorSelector, 

54) 

55from lsst.analysis.tools.actions.vector.vectorActions import ( 

56 ConvertFluxToMag, 

57 DownselectVector, 

58 ExtinctionCorrectedMagDiff, 

59 LoadVector, 

60 MagDiff, 

61) 

62 

63 

64class TestScalarActions(unittest.TestCase): 

65 """ "Test ScalarActions""" 

66 

67 def setUp(self): 

68 x = np.arange(100, dtype=float) 

69 x[31] = np.nan 

70 x[41] = np.nan 

71 x[59] = np.nan 

72 y = x**2 

73 self.data = {"r_y": x, "i_y": y} 

74 self.mask = np.zeros(100, dtype=bool) 

75 self.mask[1:50] = 1 

76 

77 def _testScalarActionAlmostEqual(self, cls, truth, maskedTruth): 

78 action = cls(vectorKey="{band}_y") 

79 schema = [col for col, colType in action.getInputSchema()] 

80 self.assertEqual(schema, ["{band}_y"]) 

81 result = action(self.data, band="i") 

82 self.assertAlmostEqual(result, truth) 

83 result = action(self.data, mask=self.mask, band="i") 

84 self.assertAlmostEqual(result, maskedTruth) 

85 

86 def testMedianAction(self): 

87 self._testScalarActionAlmostEqual(MedianAction, 2500, 576) 

88 

89 def testMeanAction(self): 

90 self._testScalarActionAlmostEqual(MeanAction, 3321.9278350515465, 803.8936170212766) 

91 

92 def testStdevAction(self): 

93 self._testScalarActionAlmostEqual(StdevAction, 2984.5855649976297, 733.5989754407842) 

94 

95 def testSigmaMadAction(self): 

96 self._testScalarActionAlmostEqual(SigmaMadAction, 3278.033505115886, 759.0923358748682) 

97 

98 def testCountAction(self): 

99 self._testScalarActionAlmostEqual(CountAction, 97, 47) 

100 

101 def testApproxFloorAction(self): 

102 self._testScalarActionAlmostEqual(ApproxFloor, 9216.0, 2352.5) 

103 

104 

105class TestVectorActions(unittest.TestCase): 

106 """Test VectorActions""" 

107 

108 def setUp(self): 

109 self.size = 5 

110 r = np.arange(self.size) + 1 

111 i = r**2 

112 rFlag = np.ones(self.size) 

113 rFlag[1] = 0 

114 iFlag = np.ones(self.size) 

115 iFlag[3] = 0 

116 data = { 

117 "r_vector": r, 

118 "i_vector": i, 

119 "r_flag": rFlag, 

120 "i_flag": iFlag, 

121 "g_vector": 3 * r, 

122 "E(B-V)": r[::-1], 

123 "r_ixx": r**2, 

124 "r_iyy": r[::-1] ** 2, 

125 "r_ixy": r * r[::-1], 

126 "g_iixx": r**2, 

127 "g_iiyy": r[::-1] ** 2, 

128 "g_iixy": r * r[::-1], 

129 } 

130 

131 self.data = pd.DataFrame.from_dict(data) 

132 

133 def _checkSchema(self, action, truth): 

134 schema = sorted([col for col, colType in action.getInputSchema()]) 

135 self.assertEqual(schema, truth) 

136 

137 # VectorActions with their own files 

138 

139 def testCalcBinnedStats(self): 

140 selector = RangeSelector(key="r_vector", minimum=0, maximum=self.size) 

141 prefix = "a_" 

142 stats = CalcBinnedStatsAction(name_prefix=prefix, selector_range=selector, key_vector="r_vector") 

143 result = stats(self.data) 

144 mask = selector(self.data) 

145 values = self.data["r_vector"][mask] 

146 median = np.median(values) 

147 truth = { 

148 stats.name_mask: mask, 

149 stats.name_median: median, 

150 stats.name_sigmaMad: 1.482602218505602 * np.median(np.abs(values - median)), 

151 stats.name_count: np.sum(mask), 

152 stats.name_select_maximum: np.max(values), 

153 stats.name_select_median: median, 

154 stats.name_select_minimum: np.min(values), 

155 "range_maximum": self.size, 

156 "range_minimum": 0, 

157 } 

158 self.assertEqual(list(result.keys()), list(truth.keys())) 

159 

160 np.testing.assert_array_almost_equal(result[stats.name_sigmaMad], truth[stats.name_sigmaMad]) 

161 del truth[stats.name_sigmaMad] 

162 

163 np.testing.assert_array_equal(result[stats.name_mask], truth[stats.name_mask]) 

164 del truth[stats.name_mask] 

165 

166 for key, value in truth.items(): 

167 self.assertEqual(result[key], value, key) 

168 

169 # def testCalcRhoStatistics(self): TODO: implement 

170 

171 def testCalcShapeSize(self): 

172 xx = self.data["r_ixx"] 

173 yy = self.data["r_iyy"] 

174 xy = self.data["r_ixy"] 

175 

176 # Test determinant with defaults 

177 action = CalcShapeSize() 

178 result = action(self.data, band="r") 

179 schema = [col for col, colType in action.getInputSchema()] 

180 self.assertEqual(sorted(schema), ["{band}_ixx", "{band}_ixy", "{band}_iyy"]) 

181 truth = 0.25 * (xx * yy - xy**2) 

182 np.testing.assert_array_almost_equal(result, truth) 

183 

184 # Test trace with columns specified 

185 action = CalcShapeSize( 

186 colXx="{band}_iixx", 

187 colYy="{band}_iiyy", 

188 colXy="{band}_iixy", 

189 sizeType="trace", 

190 ) 

191 result = action(self.data, band="g") 

192 schema = [col for col, colType in action.getInputSchema()] 

193 self.assertEqual(sorted(schema), ["{band}_iixx", "{band}_iiyy"]) 

194 truth = np.sqrt(0.5 * (xx + yy)) 

195 np.testing.assert_array_almost_equal(result, truth) 

196 

197 # def testCalcE(self): TODO: implement 

198 

199 # def testCalcEDiff(self): TODO: implement 

200 

201 # def testCalcE1(self): TODO: implement 

202 

203 # def testCalcE2(self): TODO: implement 

204 

205 # MathActions 

206 

207 def _testMath(self, ActionType, truth, compare_exact: bool = False): 

208 actionA = LoadVector(vectorKey="{band1}_vector") 

209 actionB = LoadVector(vectorKey="{band2}_vector") 

210 action = ActionType(actionA=actionA, actionB=actionB) 

211 result = action(self.data, band1="r", band2="i") 

212 self._checkSchema(action, [actionA.vectorKey, actionB.vectorKey]) 

213 if compare_exact: 

214 np.testing.assert_array_equal(result, truth) 

215 else: 

216 np.testing.assert_array_almost_equal(result, truth) 

217 

218 def testConstant(self): 

219 truth = [42.0] 

220 action = ConstantValue(value=truth[0]) 

221 self._checkSchema(action, []) 

222 result = action({}) 

223 np.testing.assert_array_equal(result, truth) 

224 

225 def testAdd(self): 

226 truth = [2.0, 6.0, 12.0, 20.0, 30.0] 

227 self._testMath(AddVector, truth, True) 

228 

229 def testSubtract(self): 

230 truth = [0.0, -2.0, -6.0, -12.0, -20.0] 

231 self._testMath(SubtractVector, truth, True) 

232 

233 def testMultiply(self): 

234 truth = [1.0, 8.0, 27.0, 64.0, 125.0] 

235 self._testMath(MultiplyVector, truth, False) 

236 

237 def testDivide(self): 

238 truth = 1 / np.arange(1, 6) 

239 self._testMath(DivideVector, truth, False) 

240 

241 def testFractionalDifference(self): 

242 actionA = LoadVector(vectorKey="{band1}_vector") 

243 actionB = LoadVector(vectorKey="{band2}_vector") 

244 truth = [0.0, -0.5, -0.6666666666666666, -0.75, -0.8] 

245 diff = FractionalDifference(actionA=actionA, actionB=actionB) 

246 result = diff(self.data, band1="r", band2="i") 

247 self._checkSchema(diff, ["{band1}_vector", "{band2}_vector"]) 

248 np.testing.assert_array_almost_equal(result, truth) 

249 

250 # Basic vectorActions 

251 

252 # def testLoadVector(self): TODO: implement 

253 

254 def testDownselectVector(self): 

255 selector = FlagSelector(selectWhenTrue=["{band}_flag"]) 

256 action = DownselectVector(vectorKey="{band}_vector", selector=selector) 

257 result = action(self.data, band="r") 

258 self._checkSchema(action, ["{band}_flag", "{band}_vector"]) 

259 np.testing.assert_array_equal(result, np.array([1, 3, 4, 5])) 

260 

261 # def testMultiCriteriaDownselectVector(self): TODO: implement 

262 

263 # Astronomical vectorActions 

264 

265 # def testCalcSn(self): TODO: implement 

266 

267 def testConvertFluxToMag(self): 

268 truth = [ 

269 31.4, 

270 29.89485002168, 

271 29.0143937264, 

272 28.38970004336, 

273 27.90514997832, 

274 ] 

275 action = ConvertFluxToMag(vectorKey="{band}_vector") 

276 result = action(self.data, band="i") 

277 self._checkSchema(action, ["{band}_vector"]) 

278 np.testing.assert_array_almost_equal(result, truth) 

279 

280 # def testConvertUnits(self): TODO: implement 

281 

282 def testMagDiff(self): 

283 # Use the same units as the data so that we know the difference exactly 

284 # without conversions. 

285 # testExtinctionCorrectedMagDiff will test the conversions 

286 truth = np.array(2 * self.data["r_vector"]) * u.ABmag 

287 action = MagDiff( 

288 col1="{band1}_vector", 

289 col2="{band2}_vector", 

290 fluxUnits1="mag(AB)", 

291 fluxUnits2="mag(AB)", 

292 returnMillimags=False, 

293 ) 

294 result = action(self.data, band1="g", band2="r") 

295 self._checkSchema(action, ["{band1}_vector", "{band2}_vector"]) 

296 np.testing.assert_array_almost_equal(result, truth.value) 

297 

298 def testExtinctionCorrectedMagDiff(self): 

299 for returnMillimags in (True, False): 

300 # Check that conversions are working properly 

301 magDiff = MagDiff( 

302 col1="{band1}_vector", 

303 col2="{band2}_vector", 

304 fluxUnits1="jansky", 

305 fluxUnits2="jansky", 

306 returnMillimags=returnMillimags, 

307 ) 

308 action = ExtinctionCorrectedMagDiff( 

309 magDiff=magDiff, 

310 band1="g", 

311 band2="r", 

312 ebvCol="E(B-V)", 

313 extinctionCoeffs={"g": 0.2, "r": 1.5}, 

314 ) 

315 

316 result = action(self.data, band1="g", band2="r") 

317 lhs = (np.array(self.data["g_vector"]) * u.jansky).to(u.ABmag) 

318 rhs = (np.array(self.data["r_vector"]) * u.jansky).to(u.ABmag) 

319 diff = lhs - rhs 

320 correction = np.array((0.2 - 1.5) * self.data["E(B-V)"]) * u.mag 

321 if returnMillimags: 

322 diff = diff.to(u.mmag) 

323 correction = correction.to(u.mmag) 

324 truth = diff - correction 

325 self._checkSchema(action, ["E(B-V)", "{band1}_vector", "{band2}_vector"]) 

326 np.testing.assert_array_almost_equal(result, truth.value) 

327 

328 # Test with hard coded bands 

329 magDiff = MagDiff( 

330 col1="g_vector", 

331 col2="r_vector", 

332 fluxUnits1="jansky", 

333 fluxUnits2="jansky", 

334 returnMillimags=False, 

335 ) 

336 action = ExtinctionCorrectedMagDiff( 

337 magDiff=magDiff, 

338 ebvCol="E(B-V)", 

339 extinctionCoeffs={"g": 0.2, "r": 1.5}, 

340 ) 

341 result = action(self.data) 

342 lhs = (np.array(self.data["g_vector"]) * u.jansky).to(u.ABmag) 

343 rhs = (np.array(self.data["r_vector"]) * u.jansky).to(u.ABmag) 

344 diff = lhs - rhs 

345 correction = np.array((0.2 - 1.5) * self.data["E(B-V)"]) * u.mag 

346 truth = diff - correction 

347 self._checkSchema(action, ["E(B-V)", "g_vector", "r_vector"]) 

348 np.testing.assert_array_almost_equal(result, truth.value) 

349 

350 # def testRAcosDec(self): TODO: implement 

351 

352 # Statistical vectorActions 

353 

354 # def testPerGroupStatistic(self): TODO: implement 

355 

356 # def testResidualWithPerGroupStatistic(self): TODO: implement 

357 

358 

359class TestVectorSelectors(unittest.TestCase): 

360 def setUp(self): 

361 self.size = 20 

362 falseFlags = { 

363 "{band}_psfFlux_flag": [1], 

364 "{band}_pixelFlags_saturatedCenter": [3], 

365 "{band}_extendedness_flag": [5], 

366 "xy_flag": [7], 

367 "i_pixelFlags_edge": [13], 

368 "r_pixelFlags_edge": [15], 

369 "sky_object": [13, 15, 17], 

370 } 

371 

372 trueFlags = { 

373 "detect_isPatchInner": [9], 

374 "detect_isDeblendedSource": [11], 

375 } 

376 

377 flux = np.arange(self.size) * 10 

378 fluxErr = np.ones(self.size) * 0.1 

379 extendedness = np.arange(20) / 20 - 0.1 

380 

381 self.data = { 

382 "r_psfFlux": flux, 

383 "r_psfFluxErr": fluxErr, 

384 "i_cmodelFlux": flux[::-1], 

385 "i_cmodelFluxError": fluxErr[::-1], 

386 "r_cmodelFlux": flux, 

387 "r_cmodelFluxError": fluxErr, 

388 "i_extendedness": extendedness, 

389 "i_extended": extendedness, 

390 } 

391 bands = ("r", "i") 

392 for band in bands: 

393 for flag, bits in falseFlags.items(): 

394 vector = np.zeros(self.size, dtype=bool) 

395 for bit in bits: 

396 vector[bit] = 1 

397 self.data[flag.format(band=band)] = vector 

398 

399 for flag, bits in trueFlags.items(): 

400 vector = np.ones(self.size, dtype=bool) 

401 for bit in bits: 

402 vector[bit] = 0 

403 self.data[flag.format(band=band)] = vector 

404 

405 def _checkSchema(self, action, truth): 

406 schema = [col for col, colType in action.getInputSchema()] 

407 self.assertEqual(sorted(schema), sorted(truth)) 

408 

409 def testFlagSelector(self): 

410 selector = FlagSelector( 

411 selectWhenFalse=["{band}_psfFlux_flag"], selectWhenTrue=["detect_isPatchInner"] 

412 ) 

413 self._checkSchema(selector, ["detect_isPatchInner", "{band}_psfFlux_flag"]) 

414 result = selector(self.data, band="r") 

415 truth = np.ones(self.size, dtype=bool) 

416 truth[1] = False 

417 truth[9] = False 

418 np.testing.assert_array_equal(result, truth) 

419 

420 def testCoaddPlotFlagSelector(self): 

421 # Test defaults 

422 # Bands needs to be set to something otherwise it 

423 # will crash as the default value looks it up 

424 # elsewhere. 

425 selector = CoaddPlotFlagSelector(bands=["i"]) 

426 keys = [ 

427 "{band}_psfFlux_flag", 

428 "{band}_pixelFlags_saturatedCenter", 

429 "{band}_extendedness_flag", 

430 "xy_flag", 

431 "detect_isPatchInner", 

432 "detect_isDeblendedSource", 

433 ] 

434 self._checkSchema(selector, keys) 

435 

436 result = selector(self.data) 

437 truth = np.ones(self.size, dtype=bool) 

438 for bit in (1, 3, 5, 7, 9, 11): 

439 truth[bit] = 0 

440 np.testing.assert_array_equal(result, truth) 

441 

442 # Test bands override 

443 selector = CoaddPlotFlagSelector( 

444 bands=["i", "r"], 

445 selectWhenFalse=["{band}_psfFlux_flag"], 

446 selectWhenTrue=["detect_isDeblendedSource"], 

447 ) 

448 self._checkSchema(selector, ["{band}_psfFlux_flag", "detect_isDeblendedSource"]) 

449 result = selector(self.data) 

450 truth = np.ones(self.size, dtype=bool) 

451 for bit in (1, 11): 

452 truth[bit] = 0 

453 np.testing.assert_array_equal(result, truth) 

454 

455 def testRangeSelector(self): 

456 selector = RangeSelector(key="r_psfFlux", minimum=np.nextafter(20, 30), maximum=50) 

457 self._checkSchema(selector, ["r_psfFlux"]) 

458 result = self.data["r_psfFlux"][selector(self.data)] 

459 truth = [30, 40] 

460 np.testing.assert_array_equal(result, truth) 

461 

462 def testSnSelector(self): 

463 # test defaults 

464 selector = SnSelector() 

465 keys = [ 

466 "{band}_psfFlux", 

467 "{band}_psfFluxErr", 

468 ] 

469 self._checkSchema(selector, keys) 

470 result = selector(self.data, bands=["r"]) 

471 truth = np.ones(self.size, dtype=bool) 

472 truth[:6] = 0 

473 np.testing.assert_array_equal(result, truth) 

474 

475 # test overrides 

476 selector = SnSelector( 

477 fluxType="{band}_cmodelFlux", 

478 threshold=200.0, 

479 uncertaintySuffix="Error", 

480 bands=["r", "i"], 

481 ) 

482 keys = [ 

483 "{band}_cmodelFlux", 

484 "{band}_cmodelFluxError", 

485 ] 

486 self._checkSchema(selector, keys) 

487 result = selector(self.data) 

488 truth = np.ones(self.size, dtype=bool) 

489 truth[:3] = 0 

490 truth[-3:] = 0 

491 np.testing.assert_array_equal(result, truth) 

492 

493 def testSkyObjectSelector(self): 

494 # Test default 

495 selector = SkyObjectSelector() 

496 keys = ["{band}_pixelFlags_edge", "sky_object"] 

497 self._checkSchema(selector, keys) 

498 result = selector(self.data) 

499 truth = np.zeros(self.size, dtype=bool) 

500 truth[15] = 1 

501 truth[17] = 1 

502 np.testing.assert_array_equal(result, truth) 

503 

504 # Test overrides 

505 selector = SkyObjectSelector(bands=["i", "r"]) 

506 self._checkSchema(selector, keys) 

507 result = selector(self.data) 

508 truth = np.zeros(self.size, dtype=bool) 

509 truth[17] = 1 

510 np.testing.assert_array_equal(result, truth) 

511 

512 def testStarSelector(self): 

513 # test default 

514 selector = StarSelector() 

515 self._checkSchema(selector, ["{band}_extendedness"]) 

516 result = selector(self.data, band="i") 

517 truth = (self.data["i_extendedness"] >= 0) & (self.data["i_extendedness"] < 0.5) 

518 np.testing.assert_array_almost_equal(result, truth) 

519 

520 # Test overrides 

521 selector = StarSelector(vectorKey="i_extended", extendedness_maximum=0.3) 

522 result = selector(self.data, band="i") 

523 truth = (self.data["i_extendedness"] >= 0) & (self.data["i_extendedness"] < 0.3) 

524 np.testing.assert_array_almost_equal(result, truth) 

525 

526 def testGalaxySelector(self): 

527 # test default 

528 selector = GalaxySelector() 

529 self._checkSchema(selector, ["{band}_extendedness"]) 

530 result = selector(self.data, band="i") 

531 truth = self.data["i_extendedness"] > 0.5 

532 np.testing.assert_array_almost_equal(result, truth) 

533 

534 # Test overrides 

535 selector = GalaxySelector(vectorKey="i_extended", extendedness_minimum=0.3) 

536 result = selector(self.data, band="i") 

537 truth = self.data["i_extendedness"] > 0.3 

538 np.testing.assert_array_almost_equal(result, truth) 

539 

540 def testVectorSelector(self): 

541 selector = VectorSelector(vectorKey="{band}_psfFlux_flag") 

542 self._checkSchema(selector, ["{band}_psfFlux_flag"]) 

543 result = selector(self.data, band="i") 

544 truth = np.zeros(self.size, dtype=bool) 

545 truth[1] = True 

546 np.testing.assert_array_equal(result, truth)