Coverage for tests/test_actions.py: 15%

305 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-06-09 11:40 +0000

1# This file is part of analysis_tools. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import unittest 

23 

24import astropy.units as u 

25import lsst.utils.tests 

26import numpy as np 

27import pandas as pd 

28from lsst.analysis.tools.actions.keyedData.calcDistances import CalcRelativeDistances 

29from lsst.analysis.tools.actions.scalar.scalarActions import ( 

30 ApproxFloor, 

31 CountAction, 

32 MeanAction, 

33 MedianAction, 

34 SigmaMadAction, 

35 StdevAction, 

36) 

37from lsst.analysis.tools.actions.vector.calcBinnedStats import CalcBinnedStatsAction 

38from lsst.analysis.tools.actions.vector.calcShapeSize import CalcShapeSize 

39from lsst.analysis.tools.actions.vector.mathActions import ( 

40 AddVector, 

41 ConstantValue, 

42 DivideVector, 

43 FractionalDifference, 

44 MultiplyVector, 

45 SubtractVector, 

46) 

47from lsst.analysis.tools.actions.vector.selectors import ( 

48 CoaddPlotFlagSelector, 

49 FlagSelector, 

50 GalaxySelector, 

51 RangeSelector, 

52 SkyObjectSelector, 

53 SnSelector, 

54 StarSelector, 

55 VectorSelector, 

56) 

57from lsst.analysis.tools.actions.vector.vectorActions import ( 

58 ConvertFluxToMag, 

59 DownselectVector, 

60 ExtinctionCorrectedMagDiff, 

61 LoadVector, 

62 MagDiff, 

63) 

64 

65 

66class TestScalarActions(unittest.TestCase): 

67 """ "Test ScalarActions""" 

68 

69 def setUp(self): 

70 x = np.arange(100, dtype=float) 

71 x[31] = np.nan 

72 x[41] = np.nan 

73 x[59] = np.nan 

74 y = x**2 

75 self.data = {"r_y": x, "i_y": y} 

76 self.mask = np.zeros(100, dtype=bool) 

77 self.mask[1:50] = 1 

78 

79 def _testScalarActionAlmostEqual(self, cls, truth, maskedTruth): 

80 action = cls(vectorKey="{band}_y") 

81 schema = [col for col, colType in action.getInputSchema()] 

82 self.assertEqual(schema, ["{band}_y"]) 

83 result = action(self.data, band="i") 

84 self.assertAlmostEqual(result, truth) 

85 result = action(self.data, mask=self.mask, band="i") 

86 self.assertAlmostEqual(result, maskedTruth) 

87 

88 def testMedianAction(self): 

89 self._testScalarActionAlmostEqual(MedianAction, 2500, 576) 

90 

91 def testMeanAction(self): 

92 self._testScalarActionAlmostEqual(MeanAction, 3321.9278350515465, 803.8936170212766) 

93 

94 def testStdevAction(self): 

95 self._testScalarActionAlmostEqual(StdevAction, 2984.5855649976297, 733.5989754407842) 

96 

97 def testSigmaMadAction(self): 

98 self._testScalarActionAlmostEqual(SigmaMadAction, 3278.033505115886, 759.0923358748682) 

99 

100 def testCountAction(self): 

101 self._testScalarActionAlmostEqual(CountAction, 97, 47) 

102 

103 def testApproxFloorAction(self): 

104 self._testScalarActionAlmostEqual(ApproxFloor, 9216.0, 2352.5) 

105 

106 

107class TestVectorActions(unittest.TestCase): 

108 """Test VectorActions""" 

109 

110 def setUp(self): 

111 self.size = 5 

112 r = np.arange(self.size) + 1 

113 i = r**2 

114 rFlag = np.ones(self.size) 

115 rFlag[1] = 0 

116 iFlag = np.ones(self.size) 

117 iFlag[3] = 0 

118 data = { 

119 "r_vector": r, 

120 "i_vector": i, 

121 "r_flag": rFlag, 

122 "i_flag": iFlag, 

123 "g_vector": 3 * r, 

124 "E(B-V)": r[::-1], 

125 "r_ixx": r**2, 

126 "r_iyy": r[::-1] ** 2, 

127 "r_ixy": r * r[::-1], 

128 "g_iixx": r**2, 

129 "g_iiyy": r[::-1] ** 2, 

130 "g_iixy": r * r[::-1], 

131 } 

132 

133 self.data = pd.DataFrame.from_dict(data) 

134 

135 def _checkSchema(self, action, truth): 

136 schema = sorted([col for col, colType in action.getInputSchema()]) 

137 self.assertEqual(schema, truth) 

138 

139 # VectorActions with their own files 

140 

141 def testCalcBinnedStats(self): 

142 selector = RangeSelector(key="r_vector", minimum=0, maximum=self.size) 

143 prefix = "a_" 

144 stats = CalcBinnedStatsAction(name_prefix=prefix, selector_range=selector, key_vector="r_vector") 

145 result = stats(self.data) 

146 mask = selector(self.data) 

147 values = self.data["r_vector"][mask] 

148 median = np.median(values) 

149 truth = { 

150 stats.name_mask: mask, 

151 stats.name_median: median, 

152 stats.name_sigmaMad: 1.482602218505602 * np.median(np.abs(values - median)), 

153 stats.name_count: np.sum(mask), 

154 stats.name_select_maximum: np.max(values), 

155 stats.name_select_median: median, 

156 stats.name_select_minimum: np.min(values), 

157 "range_maximum": self.size, 

158 "range_minimum": 0, 

159 } 

160 self.assertEqual(list(result.keys()), list(truth.keys())) 

161 

162 np.testing.assert_array_almost_equal(result[stats.name_sigmaMad], truth[stats.name_sigmaMad]) 

163 del truth[stats.name_sigmaMad] 

164 

165 np.testing.assert_array_equal(result[stats.name_mask], truth[stats.name_mask]) 

166 del truth[stats.name_mask] 

167 

168 for key, value in truth.items(): 

169 self.assertEqual(result[key], value, key) 

170 

171 # def testCalcRhoStatistics(self): TODO: implement 

172 

173 def testCalcShapeSize(self): 

174 xx = self.data["r_ixx"] 

175 yy = self.data["r_iyy"] 

176 xy = self.data["r_ixy"] 

177 

178 # Test determinant with defaults 

179 action = CalcShapeSize() 

180 result = action(self.data, band="r") 

181 schema = [col for col, colType in action.getInputSchema()] 

182 self.assertEqual(sorted(schema), ["{band}_ixx", "{band}_ixy", "{band}_iyy"]) 

183 truth = 0.25 * (xx * yy - xy**2) 

184 np.testing.assert_array_almost_equal(result, truth) 

185 

186 # Test trace with columns specified 

187 action = CalcShapeSize( 

188 colXx="{band}_iixx", 

189 colYy="{band}_iiyy", 

190 colXy="{band}_iixy", 

191 sizeType="trace", 

192 ) 

193 result = action(self.data, band="g") 

194 schema = [col for col, colType in action.getInputSchema()] 

195 self.assertEqual(sorted(schema), ["{band}_iixx", "{band}_iiyy"]) 

196 truth = np.sqrt(0.5 * (xx + yy)) 

197 np.testing.assert_array_almost_equal(result, truth) 

198 

199 # def testCalcE(self): TODO: implement 

200 

201 # def testCalcEDiff(self): TODO: implement 

202 

203 # def testCalcE1(self): TODO: implement 

204 

205 # def testCalcE2(self): TODO: implement 

206 

207 # MathActions 

208 

209 def _testMath(self, ActionType, truth, compare_exact: bool = False): 

210 actionA = LoadVector(vectorKey="{band1}_vector") 

211 actionB = LoadVector(vectorKey="{band2}_vector") 

212 action = ActionType(actionA=actionA, actionB=actionB) 

213 result = action(self.data, band1="r", band2="i") 

214 self._checkSchema(action, [actionA.vectorKey, actionB.vectorKey]) 

215 if compare_exact: 

216 np.testing.assert_array_equal(result, truth) 

217 else: 

218 np.testing.assert_array_almost_equal(result, truth) 

219 

220 def testConstant(self): 

221 truth = [42.0] 

222 action = ConstantValue(value=truth[0]) 

223 self._checkSchema(action, []) 

224 result = action({}) 

225 np.testing.assert_array_equal(result, truth) 

226 

227 def testAdd(self): 

228 truth = [2.0, 6.0, 12.0, 20.0, 30.0] 

229 self._testMath(AddVector, truth, True) 

230 

231 def testSubtract(self): 

232 truth = [0.0, -2.0, -6.0, -12.0, -20.0] 

233 self._testMath(SubtractVector, truth, True) 

234 

235 def testMultiply(self): 

236 truth = [1.0, 8.0, 27.0, 64.0, 125.0] 

237 self._testMath(MultiplyVector, truth, False) 

238 

239 def testDivide(self): 

240 truth = 1 / np.arange(1, 6) 

241 self._testMath(DivideVector, truth, False) 

242 

243 def testFractionalDifference(self): 

244 actionA = LoadVector(vectorKey="{band1}_vector") 

245 actionB = LoadVector(vectorKey="{band2}_vector") 

246 truth = [0.0, -0.5, -0.6666666666666666, -0.75, -0.8] 

247 diff = FractionalDifference(actionA=actionA, actionB=actionB) 

248 result = diff(self.data, band1="r", band2="i") 

249 self._checkSchema(diff, ["{band1}_vector", "{band2}_vector"]) 

250 np.testing.assert_array_almost_equal(result, truth) 

251 

252 # Basic vectorActions 

253 

254 # def testLoadVector(self): TODO: implement 

255 

256 def testDownselectVector(self): 

257 selector = FlagSelector(selectWhenTrue=["{band}_flag"]) 

258 action = DownselectVector(vectorKey="{band}_vector", selector=selector) 

259 result = action(self.data, band="r") 

260 self._checkSchema(action, ["{band}_flag", "{band}_vector"]) 

261 np.testing.assert_array_equal(result, np.array([1, 3, 4, 5])) 

262 

263 # def testMultiCriteriaDownselectVector(self): TODO: implement 

264 

265 # Astronomical vectorActions 

266 

267 # def testCalcSn(self): TODO: implement 

268 

269 def testConvertFluxToMag(self): 

270 truth = [ 

271 31.4, 

272 29.89485002168, 

273 29.0143937264, 

274 28.38970004336, 

275 27.90514997832, 

276 ] 

277 action = ConvertFluxToMag(vectorKey="{band}_vector") 

278 result = action(self.data, band="i") 

279 self._checkSchema(action, ["{band}_vector"]) 

280 np.testing.assert_array_almost_equal(result, truth) 

281 

282 # def testConvertUnits(self): TODO: implement 

283 

284 def testMagDiff(self): 

285 # Use the same units as the data so that we know the difference exactly 

286 # without conversions. 

287 # testExtinctionCorrectedMagDiff will test the conversions 

288 truth = np.array(2 * self.data["r_vector"]) * u.ABmag 

289 action = MagDiff( 

290 col1="{band1}_vector", 

291 col2="{band2}_vector", 

292 fluxUnits1="mag(AB)", 

293 fluxUnits2="mag(AB)", 

294 returnMillimags=False, 

295 ) 

296 result = action(self.data, band1="g", band2="r") 

297 self._checkSchema(action, ["{band1}_vector", "{band2}_vector"]) 

298 np.testing.assert_array_almost_equal(result, truth.value) 

299 

300 def testExtinctionCorrectedMagDiff(self): 

301 for returnMillimags in (True, False): 

302 # Check that conversions are working properly 

303 magDiff = MagDiff( 

304 col1="{band1}_vector", 

305 col2="{band2}_vector", 

306 fluxUnits1="jansky", 

307 fluxUnits2="jansky", 

308 returnMillimags=returnMillimags, 

309 ) 

310 action = ExtinctionCorrectedMagDiff( 

311 magDiff=magDiff, 

312 band1="g", 

313 band2="r", 

314 ebvCol="E(B-V)", 

315 extinctionCoeffs={"g": 0.2, "r": 1.5}, 

316 ) 

317 

318 result = action(self.data, band1="g", band2="r") 

319 lhs = (np.array(self.data["g_vector"]) * u.jansky).to(u.ABmag) 

320 rhs = (np.array(self.data["r_vector"]) * u.jansky).to(u.ABmag) 

321 diff = lhs - rhs 

322 correction = np.array((0.2 - 1.5) * self.data["E(B-V)"]) * u.mag 

323 if returnMillimags: 

324 diff = diff.to(u.mmag) 

325 correction = correction.to(u.mmag) 

326 truth = diff - correction 

327 self._checkSchema(action, ["E(B-V)", "{band1}_vector", "{band2}_vector"]) 

328 np.testing.assert_array_almost_equal(result, truth.value) 

329 

330 # Test with hard coded bands 

331 magDiff = MagDiff( 

332 col1="g_vector", 

333 col2="r_vector", 

334 fluxUnits1="jansky", 

335 fluxUnits2="jansky", 

336 returnMillimags=False, 

337 ) 

338 action = ExtinctionCorrectedMagDiff( 

339 magDiff=magDiff, 

340 ebvCol="E(B-V)", 

341 extinctionCoeffs={"g": 0.2, "r": 1.5}, 

342 ) 

343 result = action(self.data) 

344 lhs = (np.array(self.data["g_vector"]) * u.jansky).to(u.ABmag) 

345 rhs = (np.array(self.data["r_vector"]) * u.jansky).to(u.ABmag) 

346 diff = lhs - rhs 

347 correction = np.array((0.2 - 1.5) * self.data["E(B-V)"]) * u.mag 

348 truth = diff - correction 

349 self._checkSchema(action, ["E(B-V)", "g_vector", "r_vector"]) 

350 np.testing.assert_array_almost_equal(result, truth.value) 

351 

352 # def testRAcosDec(self): TODO: implement 

353 

354 # Statistical vectorActions 

355 

356 # def testPerGroupStatistic(self): TODO: implement 

357 

358 # def testResidualWithPerGroupStatistic(self): TODO: implement 

359 

360 

361class TestVectorSelectors(unittest.TestCase): 

362 def setUp(self): 

363 self.size = 20 

364 falseFlags = { 

365 "{band}_psfFlux_flag": [1], 

366 "{band}_pixelFlags_saturatedCenter": [3], 

367 "{band}_extendedness_flag": [5], 

368 "xy_flag": [7], 

369 "i_pixelFlags_edge": [13], 

370 "r_pixelFlags_edge": [15], 

371 "sky_object": [13, 15, 17], 

372 } 

373 

374 trueFlags = { 

375 "detect_isPatchInner": [9], 

376 "detect_isDeblendedSource": [11], 

377 } 

378 

379 flux = np.arange(self.size) * 10 

380 fluxErr = np.ones(self.size) * 0.1 

381 extendedness = np.arange(20) / 20 - 0.1 

382 

383 self.data = { 

384 "r_psfFlux": flux, 

385 "r_psfFluxErr": fluxErr, 

386 "i_cmodelFlux": flux[::-1], 

387 "i_cmodelFluxError": fluxErr[::-1], 

388 "r_cmodelFlux": flux, 

389 "r_cmodelFluxError": fluxErr, 

390 "i_extendedness": extendedness, 

391 "i_extended": extendedness, 

392 } 

393 bands = ("r", "i") 

394 for band in bands: 

395 for flag, bits in falseFlags.items(): 

396 vector = np.zeros(self.size, dtype=bool) 

397 for bit in bits: 

398 vector[bit] = 1 

399 self.data[flag.format(band=band)] = vector 

400 

401 for flag, bits in trueFlags.items(): 

402 vector = np.ones(self.size, dtype=bool) 

403 for bit in bits: 

404 vector[bit] = 0 

405 self.data[flag.format(band=band)] = vector 

406 

407 def _checkSchema(self, action, truth): 

408 schema = [col for col, colType in action.getInputSchema()] 

409 self.assertEqual(sorted(schema), sorted(truth)) 

410 

411 def testFlagSelector(self): 

412 selector = FlagSelector( 

413 selectWhenFalse=["{band}_psfFlux_flag"], selectWhenTrue=["detect_isPatchInner"] 

414 ) 

415 self._checkSchema(selector, ["detect_isPatchInner", "{band}_psfFlux_flag"]) 

416 result = selector(self.data, band="r") 

417 truth = np.ones(self.size, dtype=bool) 

418 truth[1] = False 

419 truth[9] = False 

420 np.testing.assert_array_equal(result, truth) 

421 

422 def testCoaddPlotFlagSelector(self): 

423 # Test defaults 

424 # Bands needs to be set to something otherwise it 

425 # will crash as the default value looks it up 

426 # elsewhere. 

427 selector = CoaddPlotFlagSelector(bands=["i"]) 

428 keys = [ 

429 "{band}_psfFlux_flag", 

430 "{band}_pixelFlags_saturatedCenter", 

431 "{band}_extendedness_flag", 

432 "xy_flag", 

433 "detect_isPatchInner", 

434 "detect_isDeblendedSource", 

435 ] 

436 self._checkSchema(selector, keys) 

437 

438 result = selector(self.data) 

439 truth = np.ones(self.size, dtype=bool) 

440 for bit in (1, 3, 5, 7, 9, 11): 

441 truth[bit] = 0 

442 np.testing.assert_array_equal(result, truth) 

443 

444 # Test bands override 

445 selector = CoaddPlotFlagSelector( 

446 bands=["i", "r"], 

447 selectWhenFalse=["{band}_psfFlux_flag"], 

448 selectWhenTrue=["detect_isDeblendedSource"], 

449 ) 

450 self._checkSchema(selector, ["{band}_psfFlux_flag", "detect_isDeblendedSource"]) 

451 result = selector(self.data) 

452 truth = np.ones(self.size, dtype=bool) 

453 for bit in (1, 11): 

454 truth[bit] = 0 

455 np.testing.assert_array_equal(result, truth) 

456 

457 def testRangeSelector(self): 

458 selector = RangeSelector(key="r_psfFlux", minimum=np.nextafter(20, 30), maximum=50) 

459 self._checkSchema(selector, ["r_psfFlux"]) 

460 result = self.data["r_psfFlux"][selector(self.data)] 

461 truth = [30, 40] 

462 np.testing.assert_array_equal(result, truth) 

463 

464 def testSnSelector(self): 

465 # test defaults 

466 selector = SnSelector() 

467 keys = [ 

468 "{band}_psfFlux", 

469 "{band}_psfFluxErr", 

470 ] 

471 self._checkSchema(selector, keys) 

472 result = selector(self.data, bands=["r"]) 

473 truth = np.ones(self.size, dtype=bool) 

474 truth[:6] = 0 

475 np.testing.assert_array_equal(result, truth) 

476 

477 # test overrides 

478 selector = SnSelector( 

479 fluxType="{band}_cmodelFlux", 

480 threshold=200.0, 

481 uncertaintySuffix="Error", 

482 bands=["r", "i"], 

483 ) 

484 keys = [ 

485 "{band}_cmodelFlux", 

486 "{band}_cmodelFluxError", 

487 ] 

488 self._checkSchema(selector, keys) 

489 result = selector(self.data) 

490 truth = np.ones(self.size, dtype=bool) 

491 truth[:3] = 0 

492 truth[-3:] = 0 

493 np.testing.assert_array_equal(result, truth) 

494 

495 def testSkyObjectSelector(self): 

496 # Test default 

497 selector = SkyObjectSelector() 

498 keys = ["{band}_pixelFlags_edge", "sky_object"] 

499 self._checkSchema(selector, keys) 

500 result = selector(self.data) 

501 truth = np.zeros(self.size, dtype=bool) 

502 truth[15] = 1 

503 truth[17] = 1 

504 np.testing.assert_array_equal(result, truth) 

505 

506 # Test overrides 

507 selector = SkyObjectSelector(bands=["i", "r"]) 

508 self._checkSchema(selector, keys) 

509 result = selector(self.data) 

510 truth = np.zeros(self.size, dtype=bool) 

511 truth[17] = 1 

512 np.testing.assert_array_equal(result, truth) 

513 

514 def testStarSelector(self): 

515 # test default 

516 selector = StarSelector() 

517 self._checkSchema(selector, ["{band}_extendedness"]) 

518 result = selector(self.data, band="i") 

519 truth = (self.data["i_extendedness"] >= 0) & (self.data["i_extendedness"] < 0.5) 

520 np.testing.assert_array_almost_equal(result, truth) 

521 

522 # Test overrides 

523 selector = StarSelector(vectorKey="i_extended", extendedness_maximum=0.3) 

524 result = selector(self.data, band="i") 

525 truth = (self.data["i_extendedness"] >= 0) & (self.data["i_extendedness"] < 0.3) 

526 np.testing.assert_array_almost_equal(result, truth) 

527 

528 def testGalaxySelector(self): 

529 # test default 

530 selector = GalaxySelector() 

531 self._checkSchema(selector, ["{band}_extendedness"]) 

532 result = selector(self.data, band="i") 

533 truth = self.data["i_extendedness"] > 0.5 

534 np.testing.assert_array_almost_equal(result, truth) 

535 

536 # Test overrides 

537 selector = GalaxySelector(vectorKey="i_extended", extendedness_minimum=0.3) 

538 result = selector(self.data, band="i") 

539 truth = self.data["i_extendedness"] > 0.3 

540 np.testing.assert_array_almost_equal(result, truth) 

541 

542 def testVectorSelector(self): 

543 selector = VectorSelector(vectorKey="{band}_psfFlux_flag") 

544 self._checkSchema(selector, ["{band}_psfFlux_flag"]) 

545 result = selector(self.data, band="i") 

546 truth = np.zeros(self.size, dtype=bool) 

547 truth[1] = True 

548 np.testing.assert_array_equal(result, truth) 

549 

550 

551class TestKeyedDataActions(unittest.TestCase): 

552 def testCalcRelativeDistances(self): 

553 # To test CalcRelativeDistances, make a matched visit catalog with 

554 # objects in a box slightly larger than the annulus used in calculating 

555 # relative distances. 

556 num_visits = 15 

557 scatter_in_degrees = (5 * u.milliarcsecond).to(u.degree).value 

558 obj_id = 0 

559 visit_id = range(num_visits) 

560 all_ras, all_decs, all_objs, all_visits = [], [], [], [] 

561 for ra in np.linspace(0, 6, 10): 

562 for dec in np.linspace(0, 6, 10): 

563 ra_degrees = (ra * u.arcmin).to(u.degree).value 

564 dec_degrees = (dec * u.arcmin).to(u.degree).value 

565 ra_meas = ra_degrees + np.random.rand(num_visits) * scatter_in_degrees 

566 dec_meas = dec_degrees + np.random.rand(num_visits) * scatter_in_degrees 

567 all_ras.append(ra_meas) 

568 all_decs.append(dec_meas) 

569 all_objs.append(np.ones(num_visits) * obj_id) 

570 all_visits.append(visit_id) 

571 obj_id += 1 

572 data = pd.DataFrame( 

573 { 

574 "coord_ra": np.concatenate(all_ras), 

575 "coord_dec": np.concatenate(all_decs), 

576 "obj_index": np.concatenate(all_objs), 

577 "visit": np.concatenate(all_visits), 

578 } 

579 ) 

580 

581 task = CalcRelativeDistances() 

582 res = task(data) 

583 

584 self.assertNotEqual(res["AMx"], np.nan) 

585 self.assertNotEqual(res["ADx"], np.nan) 

586 self.assertNotEqual(res["AFx"], np.nan) 

587 

588 

589if __name__ == "__main__": 589 ↛ 590line 589 didn't jump to line 590, because the condition on line 589 was never true

590 lsst.utils.tests.init() 

591 unittest.main()