Coverage for tests/test_actions.py: 14%

268 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-04 11:09 +0000

1# This file is part of analysis_tools. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import unittest 

23 

24import astropy.units as u 

25import numpy as np 

26import pandas as pd 

27from lsst.analysis.tools.actions.scalar.scalarActions import ( 

28 ApproxFloor, 

29 CountAction, 

30 MeanAction, 

31 MedianAction, 

32 SigmaMadAction, 

33 StdevAction, 

34) 

35from lsst.analysis.tools.actions.vector.calcBinnedStats import CalcBinnedStatsAction 

36from lsst.analysis.tools.actions.vector.calcShapeSize import CalcShapeSize 

37from lsst.analysis.tools.actions.vector.selectors import ( 

38 CoaddPlotFlagSelector, 

39 FlagSelector, 

40 GalaxySelector, 

41 RangeSelector, 

42 SkyObjectSelector, 

43 SnSelector, 

44 StarSelector, 

45 VectorSelector, 

46) 

47from lsst.analysis.tools.actions.vector.vectorActions import ( 

48 ConstantValue, 

49 DivideVector, 

50 DownselectVector, 

51 ExtinctionCorrectedMagDiff, 

52 FractionalDifference, 

53 LoadVector, 

54 MagColumnNanoJansky, 

55 MagDiff, 

56 SubtractVector, 

57) 

58 

59 

60class TestScalarActions(unittest.TestCase): 

61 """ "Test ScalarActions""" 

62 

63 def setUp(self): 

64 x = np.arange(100, dtype=float) 

65 x[31] = np.nan 

66 x[41] = np.nan 

67 x[59] = np.nan 

68 y = x**2 

69 self.data = {"r_y": x, "i_y": y} 

70 self.mask = np.zeros(100, dtype=bool) 

71 self.mask[1:50] = 1 

72 

73 def _testScalarActionAlmostEqual(self, cls, truth, maskedTruth): 

74 action = cls(vectorKey="{band}_y") 

75 schema = [col for col, colType in action.getInputSchema()] 

76 self.assertEqual(schema, ["{band}_y"]) 

77 result = action(self.data, band="i") 

78 self.assertAlmostEqual(result, truth) 

79 result = action(self.data, mask=self.mask, band="i") 

80 self.assertAlmostEqual(result, maskedTruth) 

81 

82 def testMedianAction(self): 

83 self._testScalarActionAlmostEqual(MedianAction, 2500, 576) 

84 

85 def testMeanAction(self): 

86 self._testScalarActionAlmostEqual(MeanAction, 3321.9278350515465, 803.8936170212766) 

87 

88 def testStdevAction(self): 

89 self._testScalarActionAlmostEqual(StdevAction, 2984.5855649976297, 733.5989754407842) 

90 

91 def testSigmaMadAction(self): 

92 self._testScalarActionAlmostEqual(SigmaMadAction, 3278.033505115886, 759.0923358748682) 

93 

94 def testCountAction(self): 

95 self._testScalarActionAlmostEqual(CountAction, 97, 47) 

96 

97 def testApproxFloorAction(self): 

98 self._testScalarActionAlmostEqual(ApproxFloor, 9216.0, 2352.5) 

99 

100 

101class TestVectorActions(unittest.TestCase): 

102 """Test VectorActions""" 

103 

104 def setUp(self): 

105 self.size = 5 

106 r = np.arange(self.size) + 1 

107 i = r**2 

108 rFlag = np.ones(self.size) 

109 rFlag[1] = 0 

110 iFlag = np.ones(self.size) 

111 iFlag[3] = 0 

112 data = { 

113 "r_vector": r, 

114 "i_vector": i, 

115 "r_flag": rFlag, 

116 "i_flag": iFlag, 

117 "g_vector": 3 * r, 

118 "E(B-V)": r[::-1], 

119 "r_ixx": r**2, 

120 "r_iyy": r[::-1] ** 2, 

121 "r_ixy": r * r[::-1], 

122 "g_iixx": r**2, 

123 "g_iiyy": r[::-1] ** 2, 

124 "g_iixy": r * r[::-1], 

125 } 

126 

127 self.data = pd.DataFrame.from_dict(data) 

128 

129 def _checkSchema(self, action, truth): 

130 schema = sorted([col for col, colType in action.getInputSchema()]) 

131 self.assertEqual(schema, truth) 

132 

133 def testDownselectVector(self): 

134 selector = FlagSelector(selectWhenTrue=["{band}_flag"]) 

135 action = DownselectVector(vectorKey="{band}_vector", selector=selector) 

136 result = action(self.data, band="r") 

137 self._checkSchema(action, ["{band}_flag", "{band}_vector"]) 

138 np.testing.assert_array_equal(result, np.array([1, 3, 4, 5])) 

139 

140 def testMagColumnNanoJansky(self): 

141 truth = [ 

142 31.4, 

143 29.89485002168, 

144 29.0143937264, 

145 28.38970004336, 

146 27.90514997832, 

147 ] 

148 action = MagColumnNanoJansky(vectorKey="{band}_vector") 

149 result = action(self.data, band="i") 

150 self._checkSchema(action, ["{band}_vector"]) 

151 np.testing.assert_array_almost_equal(result, truth) 

152 

153 def testFractionalDifference(self): 

154 actionA = LoadVector(vectorKey="{band1}_vector") 

155 actionB = LoadVector(vectorKey="{band2}_vector") 

156 truth = [0.0, -0.5, -0.6666666666666666, -0.75, -0.8] 

157 diff = FractionalDifference(actionA=actionA, actionB=actionB) 

158 result = diff(self.data, band1="r", band2="i") 

159 self._checkSchema(diff, ["{band1}_vector", "{band2}_vector"]) 

160 np.testing.assert_array_almost_equal(result, truth) 

161 

162 def testConstant(self): 

163 truth = [42.0] 

164 action = ConstantValue(value=truth[0]) 

165 self._checkSchema(action, []) 

166 result = action({}) 

167 np.testing.assert_array_equal(result, truth) 

168 

169 def testSubtract(self): 

170 actionA = LoadVector(vectorKey="{band1}_vector") 

171 actionB = LoadVector(vectorKey="{band2}_vector") 

172 truth = [0.0, -2.0, -6.0, -12.0, -20.0] 

173 diff = SubtractVector(actionA=actionA, actionB=actionB) 

174 result = diff(self.data, band1="r", band2="i") 

175 self._checkSchema(diff, ["{band1}_vector", "{band2}_vector"]) 

176 np.testing.assert_array_almost_equal(result, truth) 

177 

178 def testDivide(self): 

179 actionA = LoadVector(vectorKey="{band1}_vector") 

180 actionB = LoadVector(vectorKey="{band2}_vector") 

181 truth = 1 / np.arange(1, 6) 

182 diff = DivideVector(actionA=actionA, actionB=actionB) 

183 result = diff(self.data, band1="r", band2="i") 

184 self._checkSchema(diff, ["{band1}_vector", "{band2}_vector"]) 

185 np.testing.assert_array_almost_equal(result, truth) 

186 

187 def testMagDiff(self): 

188 # Use the same units as the data so that we know the difference exactly 

189 # without conversions. 

190 # testExtinctionCorrectedMagDiff will test the conversions 

191 truth = np.array(2 * self.data["r_vector"]) * u.ABmag 

192 action = MagDiff( 

193 col1="{band1}_vector", 

194 col2="{band2}_vector", 

195 fluxUnits1="mag(AB)", 

196 fluxUnits2="mag(AB)", 

197 returnMillimags=False, 

198 ) 

199 result = action(self.data, band1="g", band2="r") 

200 self._checkSchema(action, ["{band1}_vector", "{band2}_vector"]) 

201 np.testing.assert_array_almost_equal(result, truth.value) 

202 

203 def testExtinctionCorrectedMagDiff(self): 

204 for returnMillimags in (True, False): 

205 # Check that conversions are working properly 

206 magDiff = MagDiff( 

207 col1="{band1}_vector", 

208 col2="{band2}_vector", 

209 fluxUnits1="jansky", 

210 fluxUnits2="jansky", 

211 returnMillimags=returnMillimags, 

212 ) 

213 action = ExtinctionCorrectedMagDiff( 

214 magDiff=magDiff, 

215 band1="g", 

216 band2="r", 

217 ebvCol="E(B-V)", 

218 extinctionCoeffs={"g": 0.2, "r": 1.5}, 

219 ) 

220 

221 result = action(self.data, band1="g", band2="r") 

222 lhs = (np.array(self.data["g_vector"]) * u.jansky).to(u.ABmag) 

223 rhs = (np.array(self.data["r_vector"]) * u.jansky).to(u.ABmag) 

224 diff = lhs - rhs 

225 correction = np.array((0.2 - 1.5) * self.data["E(B-V)"]) * u.mag 

226 if returnMillimags: 

227 diff = diff.to(u.mmag) 

228 correction = correction.to(u.mmag) 

229 truth = diff - correction 

230 self._checkSchema(action, ["E(B-V)", "{band1}_vector", "{band2}_vector"]) 

231 np.testing.assert_array_almost_equal(result, truth.value) 

232 

233 # Test with hard coded bands 

234 magDiff = MagDiff( 

235 col1="g_vector", 

236 col2="r_vector", 

237 fluxUnits1="jansky", 

238 fluxUnits2="jansky", 

239 returnMillimags=False, 

240 ) 

241 action = ExtinctionCorrectedMagDiff( 

242 magDiff=magDiff, 

243 ebvCol="E(B-V)", 

244 extinctionCoeffs={"g": 0.2, "r": 1.5}, 

245 ) 

246 result = action(self.data) 

247 lhs = (np.array(self.data["g_vector"]) * u.jansky).to(u.ABmag) 

248 rhs = (np.array(self.data["r_vector"]) * u.jansky).to(u.ABmag) 

249 diff = lhs - rhs 

250 correction = np.array((0.2 - 1.5) * self.data["E(B-V)"]) * u.mag 

251 truth = diff - correction 

252 self._checkSchema(action, ["E(B-V)", "g_vector", "r_vector"]) 

253 np.testing.assert_array_almost_equal(result, truth.value) 

254 

255 def testCalcBinnedStats(self): 

256 selector = RangeSelector(key="r_vector", minimum=0, maximum=self.size + 1) 

257 prefix = "a_" 

258 stats = CalcBinnedStatsAction(name_prefix=prefix, selector_range=selector, key_vector="r_vector") 

259 result = stats(self.data) 

260 median = (1 + self.size) / 2.0 

261 truth = { 

262 stats.name_mask: np.ones(self.size), 

263 stats.name_median: median, 

264 stats.name_sigmaMad: 1.482602218505602 * np.median(np.abs(self.data["r_vector"] - median)), 

265 stats.name_count: self.size, 

266 stats.name_select_maximum: self.size, 

267 stats.name_select_median: median, 

268 stats.name_select_minimum: 1, 

269 "range_maximum": self.size + 1, 

270 "range_minimum": 0, 

271 } 

272 self.assertEqual(list(result.keys()), list(truth.keys())) 

273 

274 self.assertAlmostEqual(result[stats.name_sigmaMad], truth[stats.name_sigmaMad]) 

275 del truth[stats.name_sigmaMad] 

276 

277 np.testing.assert_array_equal(result[stats.name_mask], truth[stats.name_mask]) 

278 del truth[stats.name_mask] 

279 

280 for key, value in truth.items(): 

281 self.assertEqual(result[key], value, key) 

282 

283 def testCalcShapeSize(self): 

284 xx = self.data["r_ixx"] 

285 yy = self.data["r_iyy"] 

286 xy = self.data["r_ixy"] 

287 

288 # Test determinant with defaults 

289 action = CalcShapeSize() 

290 result = action(self.data, band="r") 

291 schema = [col for col, colType in action.getInputSchema()] 

292 self.assertEqual(sorted(schema), ["{band}_ixx", "{band}_ixy", "{band}_iyy"]) 

293 truth = 0.25 * (xx * yy - xy**2) 

294 np.testing.assert_array_almost_equal(result, truth) 

295 

296 # Test trace with columns specified 

297 action = CalcShapeSize( 

298 colXx="{band}_iixx", 

299 colYy="{band}_iiyy", 

300 colXy="{band}_iixy", 

301 sizeType="trace", 

302 ) 

303 result = action(self.data, band="g") 

304 schema = [col for col, colType in action.getInputSchema()] 

305 self.assertEqual(sorted(schema), ["{band}_iixx", "{band}_iiyy"]) 

306 truth = np.sqrt(0.5 * (xx + yy)) 

307 np.testing.assert_array_almost_equal(result, truth) 

308 

309 

310class TestVectorSelectors(unittest.TestCase): 

311 def setUp(self): 

312 self.size = 20 

313 falseFlags = { 

314 "{band}_psfFlux_flag": [1], 

315 "{band}_pixelFlags_saturatedCenter": [3], 

316 "{band}_extendedness_flag": [5], 

317 "xy_flag": [7], 

318 "i_pixelFlags_edge": [13], 

319 "r_pixelFlags_edge": [15], 

320 "sky_object": [13, 15, 17], 

321 } 

322 

323 trueFlags = { 

324 "detect_isPatchInner": [9], 

325 "detect_isDeblendedSource": [11], 

326 } 

327 

328 flux = np.arange(self.size) * 10 

329 fluxErr = np.ones(self.size) * 0.1 

330 extendedness = np.arange(20) / 20 - 0.1 

331 

332 self.data = { 

333 "r_psfFlux": flux, 

334 "r_psfFluxErr": fluxErr, 

335 "i_cmodelFlux": flux[::-1], 

336 "i_cmodelFluxError": fluxErr[::-1], 

337 "r_cmodelFlux": flux, 

338 "r_cmodelFluxError": fluxErr, 

339 "i_extendedness": extendedness, 

340 "i_extended": extendedness, 

341 } 

342 bands = ("r", "i") 

343 for band in bands: 

344 for flag, bits in falseFlags.items(): 

345 vector = np.zeros(self.size, dtype=bool) 

346 for bit in bits: 

347 vector[bit] = 1 

348 self.data[flag.format(band=band)] = vector 

349 

350 for flag, bits in trueFlags.items(): 

351 vector = np.ones(self.size, dtype=bool) 

352 for bit in bits: 

353 vector[bit] = 0 

354 self.data[flag.format(band=band)] = vector 

355 

356 def _checkSchema(self, action, truth): 

357 schema = [col for col, colType in action.getInputSchema()] 

358 self.assertEqual(sorted(schema), sorted(truth)) 

359 

360 def testFlagSelector(self): 

361 selector = FlagSelector( 

362 selectWhenFalse=["{band}_psfFlux_flag"], selectWhenTrue=["detect_isPatchInner"] 

363 ) 

364 self._checkSchema(selector, ["detect_isPatchInner", "{band}_psfFlux_flag"]) 

365 result = selector(self.data, band="r") 

366 truth = np.ones(self.size, dtype=bool) 

367 truth[1] = False 

368 truth[9] = False 

369 np.testing.assert_array_equal(result, truth) 

370 

371 def testCoaddPlotFlagSelector(self): 

372 # Test defaults 

373 # Bands needs to be set to something otherwise it 

374 # will crash as the default value looks it up 

375 # elsewhere. 

376 selector = CoaddPlotFlagSelector(bands=["i"]) 

377 keys = [ 

378 "{band}_psfFlux_flag", 

379 "{band}_pixelFlags_saturatedCenter", 

380 "{band}_extendedness_flag", 

381 "xy_flag", 

382 "detect_isPatchInner", 

383 "detect_isDeblendedSource", 

384 ] 

385 self._checkSchema(selector, keys) 

386 

387 result = selector(self.data) 

388 truth = np.ones(self.size, dtype=bool) 

389 for bit in (1, 3, 5, 7, 9, 11): 

390 truth[bit] = 0 

391 np.testing.assert_array_equal(result, truth) 

392 

393 # Test bands override 

394 selector = CoaddPlotFlagSelector( 

395 bands=["i", "r"], 

396 selectWhenFalse=["{band}_psfFlux_flag"], 

397 selectWhenTrue=["detect_isDeblendedSource"], 

398 ) 

399 self._checkSchema(selector, ["{band}_psfFlux_flag", "detect_isDeblendedSource"]) 

400 result = selector(self.data) 

401 truth = np.ones(self.size, dtype=bool) 

402 for bit in (1, 11): 

403 truth[bit] = 0 

404 np.testing.assert_array_equal(result, truth) 

405 

406 def testRangeSelector(self): 

407 selector = RangeSelector(key="r_psfFlux", minimum=np.nextafter(20, 30), maximum=50) 

408 self._checkSchema(selector, ["r_psfFlux"]) 

409 result = self.data["r_psfFlux"][selector(self.data)] 

410 truth = [30, 40] 

411 np.testing.assert_array_equal(result, truth) 

412 

413 def testSnSelector(self): 

414 # test defaults 

415 selector = SnSelector() 

416 keys = [ 

417 "{band}_psfFlux", 

418 "{band}_psfFluxErr", 

419 ] 

420 self._checkSchema(selector, keys) 

421 result = selector(self.data, bands=["r"]) 

422 truth = np.ones(self.size, dtype=bool) 

423 truth[:6] = 0 

424 np.testing.assert_array_equal(result, truth) 

425 

426 # test overrides 

427 selector = SnSelector( 

428 fluxType="{band}_cmodelFlux", 

429 threshold=200.0, 

430 uncertaintySuffix="Error", 

431 bands=["r", "i"], 

432 ) 

433 keys = [ 

434 "{band}_cmodelFlux", 

435 "{band}_cmodelFluxError", 

436 ] 

437 self._checkSchema(selector, keys) 

438 result = selector(self.data) 

439 truth = np.ones(self.size, dtype=bool) 

440 truth[:3] = 0 

441 truth[-3:] = 0 

442 np.testing.assert_array_equal(result, truth) 

443 

444 def testSkyObjectSelector(self): 

445 # Test default 

446 selector = SkyObjectSelector() 

447 keys = ["{band}_pixelFlags_edge", "sky_object"] 

448 self._checkSchema(selector, keys) 

449 result = selector(self.data) 

450 truth = np.zeros(self.size, dtype=bool) 

451 truth[15] = 1 

452 truth[17] = 1 

453 np.testing.assert_array_equal(result, truth) 

454 

455 # Test overrides 

456 selector = SkyObjectSelector(bands=["i", "r"]) 

457 self._checkSchema(selector, keys) 

458 result = selector(self.data) 

459 truth = np.zeros(self.size, dtype=bool) 

460 truth[17] = 1 

461 np.testing.assert_array_equal(result, truth) 

462 

463 def testStarSelector(self): 

464 # test default 

465 selector = StarSelector() 

466 self._checkSchema(selector, ["{band}_extendedness"]) 

467 result = selector(self.data, band="i") 

468 truth = (self.data["i_extendedness"] >= 0) & (self.data["i_extendedness"] < 0.5) 

469 np.testing.assert_array_almost_equal(result, truth) 

470 

471 # Test overrides 

472 selector = StarSelector(vectorKey="i_extended", extendedness_maximum=0.3) 

473 result = selector(self.data, band="i") 

474 truth = (self.data["i_extendedness"] >= 0) & (self.data["i_extendedness"] < 0.3) 

475 np.testing.assert_array_almost_equal(result, truth) 

476 

477 def testGalaxySelector(self): 

478 # test default 

479 selector = GalaxySelector() 

480 self._checkSchema(selector, ["{band}_extendedness"]) 

481 result = selector(self.data, band="i") 

482 truth = self.data["i_extendedness"] > 0.5 

483 np.testing.assert_array_almost_equal(result, truth) 

484 

485 # Test overrides 

486 selector = GalaxySelector(vectorKey="i_extended", extendedness_minimum=0.3) 

487 result = selector(self.data, band="i") 

488 truth = self.data["i_extendedness"] > 0.3 

489 np.testing.assert_array_almost_equal(result, truth) 

490 

491 def testVectorSelector(self): 

492 selector = VectorSelector(vectorKey="{band}_psfFlux_flag") 

493 self._checkSchema(selector, ["{band}_psfFlux_flag"]) 

494 result = selector(self.data, band="i") 

495 truth = np.zeros(self.size, dtype=bool) 

496 truth[1] = True 

497 np.testing.assert_array_equal(result, truth)