Coverage for tests/test_diaCalculationPlugins.py: 14%

343 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-01-25 03:07 -0800

1# This file is part of ap_association. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22from astropy.stats import median_absolute_deviation 

23import numpy as np 

24import pandas as pd 

25from scipy.stats import skew 

26import unittest 

27 

28from lsst.meas.base import ( 

29 MeanDiaPosition, MeanDiaPositionConfig, 

30 HTMIndexDiaPosition, HTMIndexDiaPositionConfig, 

31 NumDiaSourcesDiaPlugin, NumDiaSourcesDiaPluginConfig, 

32 SimpleSourceFlagDiaPlugin, SimpleSourceFlagDiaPluginConfig, 

33 WeightedMeanDiaPsFlux, WeightedMeanDiaPsFluxConfig, 

34 PercentileDiaPsFlux, PercentileDiaPsFluxConfig, 

35 SigmaDiaPsFlux, SigmaDiaPsFluxConfig, 

36 Chi2DiaPsFlux, Chi2DiaPsFluxConfig, 

37 MadDiaPsFlux, MadDiaPsFluxConfig, 

38 SkewDiaPsFlux, SkewDiaPsFluxConfig, 

39 MinMaxDiaPsFlux, MinMaxDiaPsFluxConfig, 

40 MaxSlopeDiaPsFlux, MaxSlopeDiaPsFluxConfig, 

41 ErrMeanDiaPsFlux, ErrMeanDiaPsFluxConfig, 

42 LinearFitDiaPsFlux, LinearFitDiaPsFluxConfig, 

43 StetsonJDiaPsFlux, StetsonJDiaPsFluxConfig, 

44 WeightedMeanDiaTotFlux, WeightedMeanDiaTotFluxConfig, 

45 SigmaDiaTotFlux, SigmaDiaTotFluxConfig) 

46import lsst.utils.tests 

47 

48 

49def run_single_plugin(diaObjectCat, 

50 diaObjectId, 

51 diaSourceCat, 

52 filterName, 

53 plugin): 

54 """Wrapper for running single plugins. 

55 

56 Reproduces some of the behavior of `lsst.ap.association.DiaCalcuation.run` 

57 

58 Parameters 

59 ---------- 

60 diaObjectCat : `pandas.DataFrame` 

61 Input object catalog to store data into and read from. 

62 diaSourcesCat : `pandas.DataFrame` 

63 DiaSource catalog to read data from and groupby on. 

64 fitlerName : `str` 

65 String name of the filter to process. 

66 plugin : `lsst.ap.association.DiaCalculationPlugin` 

67 Plugin to run. 

68 """ 

69 diaObjectCat.set_index("diaObjectId", inplace=True, drop=False) 

70 diaSourceCat.set_index( 

71 ["diaObjectId", "filterName", "diaSourceId"], 

72 inplace=True, 

73 drop=False) 

74 

75 objDiaSources = diaSourceCat.loc[diaObjectId] 

76 updatingFilterDiaSources = diaSourceCat.loc[ 

77 (diaObjectId, filterName), : 

78 ] 

79 

80 plugin.calculate(diaObjects=diaObjectCat, 

81 diaObjectId=diaObjectId, 

82 diaSources=objDiaSources, 

83 filterDiaSources=updatingFilterDiaSources, 

84 filterName=filterName) 

85 

86 

87def run_multi_plugin(diaObjectCat, diaSourceCat, filterName, plugin): 

88 """Wrapper for running multi plugins. 

89 

90 Reproduces some of the behavior of `lsst.ap.association.DiaCalcuation.run` 

91 

92 Parameters 

93 ---------- 

94 diaObjectCat : `pandas.DataFrame` 

95 Input object catalog to store data into and read from. 

96 diaSourcesCat : `pandas.DataFrame` 

97 DiaSource catalog to read data from and groupby on. 

98 fitlerName : `str` 

99 String name of the filter to process. 

100 plugin : `lsst.ap.association.DiaCalculationPlugin` 

101 Plugin to run. 

102 """ 

103 diaObjectCat.set_index("diaObjectId", inplace=True, drop=False) 

104 diaSourceCat.set_index( 

105 ["diaObjectId", "filterName", "diaSourceId"], 

106 inplace=True, 

107 drop=False) 

108 

109 updatingFilterDiaSources = diaSourceCat.loc[ 

110 (slice(None), filterName), : 

111 ] 

112 

113 diaSourcesGB = diaSourceCat.groupby(level=0) 

114 filterDiaSourcesGB = updatingFilterDiaSources.groupby(level=0) 

115 

116 plugin.calculate(diaObjects=diaObjectCat, 

117 diaSources=diaSourcesGB, 

118 filterDiaSources=filterDiaSourcesGB, 

119 filterName=filterName) 

120 

121 

122class TestMeanPosition(unittest.TestCase): 

123 

124 def testCalculate(self): 

125 """Test mean position calculation. 

126 """ 

127 n_sources = 10 

128 objId = 0 

129 

130 plug = MeanDiaPosition(MeanDiaPositionConfig(), 

131 "ap_meanPosition", 

132 None) 

133 

134 # Test expected means in RA. 

135 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

136 diaSources = pd.DataFrame(data={"ra": np.linspace(-1, 1, n_sources), 

137 "decl": np.zeros(n_sources), 

138 "midPointTai": np.linspace(0, 

139 n_sources, 

140 n_sources), 

141 "diaObjectId": n_sources * [objId], 

142 "filterName": n_sources * ["g"], 

143 "diaSourceId": np.arange(n_sources, 

144 dtype=int)}) 

145 run_multi_plugin(diaObjects, diaSources, "g", plug) 

146 

147 self.assertAlmostEqual(diaObjects.loc[objId, "ra"], 0.0) 

148 self.assertAlmostEqual(diaObjects.loc[objId, "decl"], 0.0) 

149 self.assertEqual(diaObjects.loc[objId, "radecTai"], 10) 

150 

151 # Test expected means in DEC. 

152 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

153 diaSources = pd.DataFrame(data={"ra": np.zeros(n_sources), 

154 "decl": np.linspace(-1, 1, n_sources), 

155 "midPointTai": np.linspace(0, 

156 n_sources, 

157 n_sources), 

158 "diaObjectId": n_sources * [objId], 

159 "filterName": n_sources * ["g"], 

160 "diaSourceId": np.arange(n_sources, 

161 dtype=int)}) 

162 run_multi_plugin(diaObjects, diaSources, "g", plug) 

163 

164 self.assertAlmostEqual(diaObjects.loc[objId, "ra"], 0.0) 

165 self.assertAlmostEqual(diaObjects.loc[objId, "decl"], 0.0) 

166 self.assertEqual(diaObjects.loc[objId, "radecTai"], 10) 

167 

168 # Test failure mode RA is nan. 

169 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

170 diaSources = pd.DataFrame(data={"ra": np.full(n_sources, np.nan), 

171 "decl": np.zeros(n_sources), 

172 "midPointTai": np.linspace(0, 

173 n_sources, 

174 n_sources), 

175 "diaObjectId": n_sources * [objId], 

176 "filterName": n_sources * ["g"], 

177 "diaSourceId": np.arange(n_sources, 

178 dtype=int)}) 

179 run_multi_plugin(diaObjects, diaSources, "g", plug) 

180 

181 self.assertTrue(np.isnan(diaObjects.loc[objId, "ra"])) 

182 self.assertTrue(np.isnan(diaObjects.loc[objId, "decl"])) 

183 self.assertTrue(np.isnan(diaObjects.loc[objId, "radecTai"])) 

184 

185 # Test failure mode DEC is nan. 

186 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

187 diaSources = pd.DataFrame(data={"ra": np.zeros(n_sources), 

188 "decl": np.full(n_sources, np.nan), 

189 "midPointTai": np.linspace(0, 

190 n_sources, 

191 n_sources), 

192 "diaObjectId": n_sources * [objId], 

193 "filterName": n_sources * ["g"], 

194 "diaSourceId": np.arange(n_sources, 

195 dtype=int)}) 

196 run_multi_plugin(diaObjects, diaSources, "g", plug) 

197 

198 self.assertTrue(np.isnan(diaObjects.loc[objId, "ra"])) 

199 self.assertTrue(np.isnan(diaObjects.loc[objId, "decl"])) 

200 self.assertTrue(np.isnan(diaObjects.loc[objId, "radecTai"])) 

201 

202 

203class TestHTMIndexPosition(unittest.TestCase): 

204 

205 def testCalculate(self): 

206 """Test HTMPixel assignment calculation. 

207 """ 

208 # Test expected pixelId at RA, DEC = 0 

209 objId = 0 

210 n_sources = 10 

211 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

212 diaObjects.loc[objId, "ra"] = 0. 

213 diaObjects.loc[objId, "decl"] = 0. 

214 diaSources = pd.DataFrame( 

215 data={"diaObjectId": n_sources * [objId], 

216 "filterName": n_sources * ["g"], 

217 "diaSourceId": np.arange(n_sources, dtype=int)}) 

218 plug = HTMIndexDiaPosition(HTMIndexDiaPositionConfig(), 

219 "ap_HTMIndex", 

220 None) 

221 

222 run_single_plugin(diaObjectCat=diaObjects, 

223 diaObjectId=objId, 

224 diaSourceCat=diaSources, 

225 filterName="g", 

226 plugin=plug) 

227 self.assertEqual(diaObjects.at[objId, "pixelId"], 

228 17042430230528) 

229 

230 # Test expected pixelId at some value of RA and DEC. 

231 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

232 diaObjects.loc[objId, "ra"] = 45.37 

233 diaObjects.loc[objId, "decl"] = 13.67 

234 diaSources = pd.DataFrame( 

235 data={"diaObjectId": n_sources * [objId], 

236 "filterName": n_sources * ["g"], 

237 "diaSourceId": np.arange(n_sources, dtype=int)}) 

238 run_single_plugin(diaObjectCat=diaObjects, 

239 diaObjectId=objId, 

240 diaSourceCat=diaSources, 

241 filterName="g", 

242 plugin=plug) 

243 self.assertEqual(diaObjects.at[objId, "pixelId"], 

244 17450571968473) 

245 

246 

247class TestNDiaSourcesDiaPlugin(unittest.TestCase): 

248 

249 def testCalculate(self): 

250 """Test that the number of DiaSources is correct. 

251 """ 

252 

253 for n_sources in [1, 8, 10]: 

254 # Test expected number of sources per object. 

255 objId = 0 

256 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

257 diaSources = pd.DataFrame( 

258 data={"diaObjectId": n_sources * [objId], 

259 "filterName": n_sources * ["g"], 

260 "diaSourceId": np.arange(n_sources, dtype=int)}) 

261 plug = NumDiaSourcesDiaPlugin(NumDiaSourcesDiaPluginConfig(), 

262 "ap_nDiaSources", 

263 None) 

264 run_multi_plugin(diaObjects, diaSources, "g", plug) 

265 

266 self.assertEqual(n_sources, diaObjects.at[objId, "nDiaSources"]) 

267 

268 

269class TestSimpleSourceFlagDiaPlugin(unittest.TestCase): 

270 

271 def testCalculate(self): 

272 """Test that DiaObject flags are set. 

273 """ 

274 objId = 0 

275 n_sources = 10 

276 

277 # Test expected flags, no flags set. 

278 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

279 diaSources = pd.DataFrame( 

280 data={"diaObjectId": n_sources * [objId], 

281 "filterName": n_sources * ["g"], 

282 "diaSourceId": np.arange(n_sources, dtype=int), 

283 "flags": np.zeros(n_sources, dtype=np.uint64)}) 

284 plug = SimpleSourceFlagDiaPlugin(SimpleSourceFlagDiaPluginConfig(), 

285 "ap_diaObjectFlag", 

286 None) 

287 run_multi_plugin(diaObjects, diaSources, "g", plug) 

288 self.assertEqual(diaObjects.at[objId, "flags"], 0) 

289 

290 # Test expected flags, all flags set. 

291 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

292 diaSources = pd.DataFrame( 

293 data={"diaObjectId": n_sources * [objId], 

294 "filterName": n_sources * ["g"], 

295 "diaSourceId": np.arange(n_sources, dtype=int), 

296 "flags": np.ones(n_sources, dtype=np.uint64)}) 

297 run_multi_plugin(diaObjects, diaSources, "g", plug) 

298 self.assertEqual(diaObjects.at[objId, "flags"], 1) 

299 

300 # Test expected flags, random flags. 

301 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

302 diaSources = pd.DataFrame( 

303 data={"diaObjectId": n_sources * [objId], 

304 "filterName": n_sources * ["g"], 

305 "diaSourceId": np.arange(n_sources, dtype=int), 

306 "flags": np.random.randint(0, 2 ** 16, size=n_sources)}) 

307 run_multi_plugin(diaObjects, diaSources, "g", plug) 

308 self.assertEqual(diaObjects.at[objId, "flags"], 1) 

309 

310 # Test expected flags, one flag set. 

311 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

312 flag_array = np.zeros(n_sources, dtype=np.uint64) 

313 flag_array[4] = 256 

314 diaSources = pd.DataFrame( 

315 data={"diaObjectId": n_sources * [objId], 

316 "filterName": n_sources * ["g"], 

317 "diaSourceId": np.arange(n_sources, dtype=int), 

318 "flags": flag_array}) 

319 run_multi_plugin(diaObjects, diaSources, "g", plug) 

320 self.assertEqual(diaObjects.at[objId, "flags"], 1) 

321 

322 

323class TestWeightedMeanDiaPsFlux(unittest.TestCase): 

324 

325 def testCalculate(self): 

326 """Test mean value calculation. 

327 """ 

328 n_sources = 10 

329 objId = 0 

330 

331 # Test expected mean. 

332 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

333 diaSources = pd.DataFrame( 

334 data={"diaObjectId": n_sources * [objId], 

335 "filterName": n_sources * ["u"], 

336 "diaSourceId": np.arange(n_sources, dtype=int), 

337 "psFlux": np.linspace(-1, 1, n_sources), 

338 "psFluxErr": np.ones(n_sources)}) 

339 

340 plug = WeightedMeanDiaPsFlux(WeightedMeanDiaPsFluxConfig(), 

341 "ap_meanFlux", 

342 None) 

343 run_multi_plugin(diaObjects, diaSources, "u", plug) 

344 

345 self.assertAlmostEqual(diaObjects.loc[objId, "uPSFluxMean"], 0.0) 

346 self.assertAlmostEqual(diaObjects.loc[objId, "uPSFluxMeanErr"], 

347 np.sqrt(1 / n_sources)) 

348 self.assertEqual(diaObjects.loc[objId, "uPSFluxNdata"], n_sources) 

349 

350 # Test expected mean with a nan value. 

351 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

352 fluxes = np.linspace(-1, 1, n_sources) 

353 fluxes[4] = np.nan 

354 diaSources = pd.DataFrame( 

355 data={"diaObjectId": n_sources * [objId], 

356 "filterName": n_sources * ["r"], 

357 "diaSourceId": np.arange(n_sources, dtype=int), 

358 "psFlux": fluxes, 

359 "psFluxErr": np.ones(n_sources)}) 

360 run_multi_plugin(diaObjects, diaSources, "r", plug) 

361 

362 self.assertAlmostEqual(diaObjects.at[objId, "rPSFluxMean"], 

363 np.nanmean(fluxes)) 

364 self.assertAlmostEqual(diaObjects.at[objId, "rPSFluxMeanErr"], 

365 np.sqrt(1 / (n_sources - 1))) 

366 self.assertEqual(diaObjects.loc[objId, "rPSFluxNdata"], n_sources - 1) 

367 

368 

369class TestPercentileDiaPsFlux(unittest.TestCase): 

370 

371 def testCalculate(self): 

372 """Test flux percentile calculation. 

373 """ 

374 n_sources = 10 

375 objId = 0 

376 

377 # Test expected percentile values. 

378 fluxes = np.linspace(-1, 1, n_sources) 

379 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

380 diaSources = pd.DataFrame( 

381 data={"diaObjectId": n_sources * [objId], 

382 "filterName": n_sources * ["u"], 

383 "diaSourceId": np.arange(n_sources, dtype=int), 

384 "psFlux": fluxes, 

385 "psFluxErr": np.ones(n_sources)}) 

386 

387 plug = PercentileDiaPsFlux(PercentileDiaPsFluxConfig(), 

388 "ap_percentileFlux", 

389 None) 

390 run_multi_plugin(diaObjects, diaSources, "u", plug) 

391 for pTile, testVal in zip(plug.config.percentiles, 

392 np.nanpercentile( 

393 fluxes, 

394 plug.config.percentiles)): 

395 self.assertAlmostEqual( 

396 diaObjects.at[objId, "uPSFluxPercentile{:02d}".format(pTile)], 

397 testVal) 

398 

399 # Test expected percentile values with a nan value. 

400 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

401 fluxes[4] = np.nan 

402 diaSources = pd.DataFrame( 

403 data={"diaObjectId": n_sources * [objId], 

404 "filterName": n_sources * ["r"], 

405 "diaSourceId": np.arange(n_sources, dtype=int), 

406 "psFlux": fluxes, 

407 "psFluxErr": np.ones(n_sources)}) 

408 run_multi_plugin(diaObjects, diaSources, "r", plug) 

409 for pTile, testVal in zip(plug.config.percentiles, 

410 np.nanpercentile( 

411 fluxes, 

412 plug.config.percentiles)): 

413 self.assertAlmostEqual( 

414 diaObjects.at[objId, "rPSFluxPercentile{:02d}".format(pTile)], 

415 testVal) 

416 

417 

418class TestSigmaDiaPsFlux(unittest.TestCase): 

419 

420 def testCalculate(self): 

421 """Test flux scatter calculation. 

422 """ 

423 n_sources = 10 

424 objId = 0 

425 

426 # Test expected sigma scatter of fluxes. 

427 fluxes = np.linspace(-1, 1, n_sources) 

428 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

429 diaSources = pd.DataFrame( 

430 data={"diaObjectId": n_sources * [objId], 

431 "filterName": n_sources * ["u"], 

432 "diaSourceId": np.arange(n_sources, dtype=int), 

433 "psFlux": fluxes, 

434 "psFluxErr": np.ones(n_sources)}) 

435 

436 plug = SigmaDiaPsFlux(SigmaDiaPsFluxConfig(), 

437 "ap_sigmaFlux", 

438 None) 

439 run_multi_plugin(diaObjects, diaSources, "u", plug) 

440 self.assertAlmostEqual(diaObjects.at[objId, "uPSFluxSigma"], 

441 np.nanstd(fluxes, ddof=1)) 

442 

443 # test one input, returns nan. 

444 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

445 diaSources = pd.DataFrame( 

446 data={"diaObjectId": 1 * [objId], 

447 "filterName": 1 * ["g"], 

448 "diaSourceId": [0], 

449 "psFlux": [fluxes[0]], 

450 "psFluxErr": [1.]}) 

451 run_multi_plugin(diaObjects, diaSources, "g", plug) 

452 self.assertTrue(np.isnan(diaObjects.at[objId, "gPSFluxSigma"])) 

453 

454 # Test expected sigma scatter of fluxes with a nan value. 

455 fluxes[4] = np.nan 

456 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

457 diaSources = pd.DataFrame( 

458 data={"diaObjectId": n_sources * [objId], 

459 "filterName": n_sources * ["r"], 

460 "diaSourceId": np.arange(n_sources, dtype=int), 

461 "psFlux": fluxes, 

462 "psFluxErr": np.ones(n_sources)}) 

463 run_multi_plugin(diaObjects, diaSources, "r", plug) 

464 self.assertAlmostEqual(diaObjects.at[objId, "rPSFluxSigma"], 

465 np.nanstd(fluxes, ddof=1)) 

466 

467 

468class TestChi2DiaPsFlux(unittest.TestCase): 

469 

470 def testCalculate(self): 

471 """Test flux chi2 calculation. 

472 """ 

473 n_sources = 10 

474 objId = 0 

475 

476 # Test expected chi^2 value. 

477 fluxes = np.linspace(-1, 1, n_sources) 

478 diaObjects = pd.DataFrame({"diaObjectId": [objId], 

479 "uPSFluxMean": [0.0]}) 

480 diaSources = pd.DataFrame( 

481 data={"diaObjectId": n_sources * [objId], 

482 "filterName": n_sources * ["u"], 

483 "diaSourceId": np.arange(n_sources, dtype=int), 

484 "psFlux": fluxes, 

485 "psFluxErr": np.ones(n_sources)}) 

486 

487 plug = Chi2DiaPsFlux(Chi2DiaPsFluxConfig(), 

488 "ap_chi2Flux", 

489 None) 

490 run_multi_plugin(diaObjects, diaSources, "u", plug) 

491 self.assertAlmostEqual( 

492 diaObjects.loc[objId, "uPSFluxChi2"], 

493 np.nansum(((diaSources["psFlux"] 

494 - np.nanmean(diaSources["psFlux"])) 

495 / diaSources["psFluxErr"]) ** 2)) 

496 

497 # Test expected chi^2 value with a nan value set. 

498 fluxes[4] = np.nan 

499 diaObjects = pd.DataFrame({"diaObjectId": [objId], 

500 "rPSFluxMean": [np.nanmean(fluxes)]}) 

501 diaSources = pd.DataFrame( 

502 data={"diaObjectId": n_sources * [objId], 

503 "filterName": n_sources * ["r"], 

504 "diaSourceId": np.arange(n_sources, dtype=int), 

505 "psFlux": fluxes, 

506 "psFluxErr": np.ones(n_sources)}) 

507 run_multi_plugin(diaObjects, diaSources, "r", plug) 

508 self.assertAlmostEqual( 

509 diaObjects.loc[objId, "rPSFluxChi2"], 

510 np.nansum(((diaSources["psFlux"] 

511 - np.nanmean(diaSources["psFlux"])) 

512 / diaSources["psFluxErr"]) ** 2)) 

513 

514 

515class TestMadDiaPsFlux(unittest.TestCase): 

516 

517 def testCalculate(self): 

518 """Test flux median absolute deviation calculation. 

519 """ 

520 n_sources = 10 

521 objId = 0 

522 

523 # Test expected MAD value. 

524 fluxes = np.linspace(-1, 1, n_sources) 

525 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

526 diaSources = pd.DataFrame( 

527 data={"diaObjectId": n_sources * [objId], 

528 "filterName": n_sources * ["u"], 

529 "diaSourceId": np.arange(n_sources, dtype=int), 

530 "psFlux": fluxes, 

531 "psFluxErr": np.ones(n_sources)}) 

532 

533 plug = MadDiaPsFlux(MadDiaPsFluxConfig(), 

534 "ap_madFlux", 

535 None) 

536 run_multi_plugin(diaObjects, diaSources, "u", plug) 

537 self.assertAlmostEqual(diaObjects.at[objId, "uPSFluxMAD"], 

538 median_absolute_deviation(fluxes, 

539 ignore_nan=True)) 

540 

541 # Test expected MAD value with a nan set. 

542 fluxes[4] = np.nan 

543 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

544 diaSources = pd.DataFrame( 

545 data={"diaObjectId": n_sources * [objId], 

546 "filterName": n_sources * ["r"], 

547 "diaSourceId": np.arange(n_sources, dtype=int), 

548 "psFlux": fluxes, 

549 "psFluxErr": np.ones(n_sources)}) 

550 run_multi_plugin(diaObjects, diaSources, "r", plug) 

551 self.assertAlmostEqual(diaObjects.at[objId, "rPSFluxMAD"], 

552 median_absolute_deviation(fluxes, 

553 ignore_nan=True)) 

554 

555 

556class TestSkewDiaPsFlux(unittest.TestCase): 

557 

558 def testCalculate(self): 

559 """Test flux skew calculation. 

560 """ 

561 n_sources = 10 

562 objId = 0 

563 

564 # Test expected skew value. 

565 fluxes = np.linspace(-1, 1, n_sources) 

566 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

567 diaSources = pd.DataFrame( 

568 data={"diaObjectId": n_sources * [objId], 

569 "filterName": n_sources * ["u"], 

570 "diaSourceId": np.arange(n_sources, dtype=int), 

571 "psFlux": fluxes, 

572 "psFluxErr": np.ones(n_sources)}) 

573 

574 plug = SkewDiaPsFlux(SkewDiaPsFluxConfig(), 

575 "ap_skewFlux", 

576 None) 

577 run_multi_plugin(diaObjects, diaSources, "u", plug) 

578 self.assertAlmostEqual( 

579 diaObjects.loc[objId, "uPSFluxSkew"], 

580 skew_wrapper(fluxes)) 

581 

582 # Test expected skew value with a nan set. 

583 fluxes[4] = np.nan 

584 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

585 diaSources = pd.DataFrame( 

586 data={"diaObjectId": n_sources * [objId], 

587 "filterName": n_sources * ["r"], 

588 "diaSourceId": np.arange(n_sources, dtype=int), 

589 "psFlux": fluxes, 

590 "psFluxErr": np.ones(n_sources)}) 

591 run_multi_plugin(diaObjects, diaSources, "r", plug) 

592 

593 self.assertAlmostEqual( 

594 diaObjects.at[objId, "rPSFluxSkew"], 

595 skew_wrapper(fluxes)) 

596 

597 

598class TestMinMaxDiaPsFlux(unittest.TestCase): 

599 

600 def testCalculate(self): 

601 """Test flux min/max calculation. 

602 """ 

603 n_sources = 10 

604 objId = 0 

605 

606 # Test expected MinMax fluxes. 

607 fluxes = np.linspace(-1, 1, n_sources) 

608 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

609 diaSources = pd.DataFrame( 

610 data={"diaObjectId": n_sources * [objId], 

611 "filterName": n_sources * ["u"], 

612 "diaSourceId": np.arange(n_sources, dtype=int), 

613 "psFlux": fluxes, 

614 "psFluxErr": np.ones(n_sources)}) 

615 

616 plug = MinMaxDiaPsFlux(MinMaxDiaPsFluxConfig(), 

617 "ap_minMaxFlux", 

618 None) 

619 run_multi_plugin(diaObjects, diaSources, "u", plug) 

620 self.assertEqual(diaObjects.loc[objId, "uPSFluxMin"], -1) 

621 self.assertEqual(diaObjects.loc[objId, "uPSFluxMax"], 1) 

622 

623 # Test expected MinMax fluxes with a nan set. 

624 fluxes[4] = np.nan 

625 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

626 diaSources = pd.DataFrame( 

627 data={"diaObjectId": n_sources * [objId], 

628 "filterName": n_sources * ["r"], 

629 "diaSourceId": np.arange(n_sources, dtype=int), 

630 "psFlux": fluxes, 

631 "psFluxErr": np.ones(n_sources)}) 

632 run_multi_plugin(diaObjects, diaSources, "r", plug) 

633 self.assertEqual(diaObjects.loc[objId, "rPSFluxMin"], -1) 

634 self.assertEqual(diaObjects.loc[objId, "rPSFluxMax"], 1) 

635 

636 

637class TestMaxSlopeDiaPsFlux(unittest.TestCase): 

638 

639 def testCalculate(self): 

640 """Test flux maximum slope. 

641 """ 

642 n_sources = 10 

643 objId = 0 

644 

645 # Test max slope value. 

646 fluxes = np.linspace(-1, 1, n_sources) 

647 times = np.concatenate([np.linspace(0, 1, n_sources)[:-1], [1 - 1/90]]) 

648 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

649 diaSources = pd.DataFrame( 

650 data={"diaObjectId": n_sources * [objId], 

651 "filterName": n_sources * ["u"], 

652 "diaSourceId": np.arange(n_sources, dtype=int), 

653 "psFlux": fluxes, 

654 "psFluxErr": np.ones(n_sources), 

655 "midPointTai": times}) 

656 

657 plug = MaxSlopeDiaPsFlux(MaxSlopeDiaPsFluxConfig(), 

658 "ap_maxSlopeFlux", 

659 None) 

660 run_multi_plugin(diaObjects, diaSources, "u", plug) 

661 self.assertAlmostEqual(diaObjects.at[objId, "uPSFluxMaxSlope"], 2 + 2/9) 

662 

663 # Test max slope value returns nan on 1 input. 

664 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

665 diaSources = pd.DataFrame( 

666 data={"diaObjectId": 1 * [objId], 

667 "filterName": 1 * ["g"], 

668 "diaSourceId": np.arange(1, dtype=int), 

669 "psFlux": fluxes[0], 

670 "psFluxErr": np.ones(1), 

671 "midPointTai": times[0]}) 

672 run_multi_plugin(diaObjects, diaSources, "g", plug) 

673 self.assertTrue(np.isnan(diaObjects.at[objId, "gPSFluxMaxSlope"])) 

674 

675 # Test max slope value inputing nan values. 

676 fluxes[4] = np.nan 

677 times[7] = np.nan 

678 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

679 diaSources = pd.DataFrame( 

680 data={"diaObjectId": n_sources * [objId], 

681 "filterName": n_sources * ["r"], 

682 "diaSourceId": np.arange(n_sources, dtype=int), 

683 "psFlux": fluxes, 

684 "psFluxErr": np.ones(n_sources), 

685 "midPointTai": times}) 

686 run_multi_plugin(diaObjects, diaSources, "r", plug) 

687 self.assertAlmostEqual(diaObjects.at[objId, "rPSFluxMaxSlope"], 2 + 2 / 9) 

688 

689 

690class TestErrMeanDiaPsFlux(unittest.TestCase): 

691 

692 def testCalculate(self): 

693 """Test error mean calculation. 

694 """ 

695 n_sources = 10 

696 objId = 0 

697 

698 # Test mean of the errors. 

699 fluxes = np.linspace(-1, 1, n_sources) 

700 errors = np.linspace(1, 2, n_sources) 

701 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

702 diaSources = pd.DataFrame( 

703 data={"diaObjectId": n_sources * [objId], 

704 "filterName": n_sources * ["u"], 

705 "diaSourceId": np.arange(n_sources, dtype=int), 

706 "psFlux": fluxes, 

707 "psFluxErr": errors}) 

708 

709 plug = ErrMeanDiaPsFlux(ErrMeanDiaPsFluxConfig(), 

710 "ap_errMeanFlux", 

711 None) 

712 run_multi_plugin(diaObjects, diaSources, "u", plug) 

713 self.assertAlmostEqual(diaObjects.at[objId, "uPSFluxErrMean"], 

714 np.nanmean(errors)) 

715 

716 # Test mean of the errors with input nan value. 

717 errors[4] = np.nan 

718 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

719 diaSources = pd.DataFrame( 

720 data={"diaObjectId": n_sources * [objId], 

721 "filterName": n_sources * ["r"], 

722 "diaSourceId": np.arange(n_sources, dtype=int), 

723 "psFlux": fluxes, 

724 "psFluxErr": errors}) 

725 run_multi_plugin(diaObjects, diaSources, "r", plug) 

726 self.assertAlmostEqual(diaObjects.at[objId, "rPSFluxErrMean"], 

727 np.nanmean(errors)) 

728 

729 

730class TestLinearFitDiaPsFlux(unittest.TestCase): 

731 

732 def testCalculate(self): 

733 """Test a linear fit to flux vs time. 

734 """ 

735 n_sources = 10 

736 objId = 0 

737 

738 # Test best fit linear model. 

739 fluxes = np.linspace(-1, 1, n_sources) 

740 errors = np.linspace(1, 2, n_sources) 

741 times = np.linspace(0, 1, n_sources) 

742 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

743 diaSources = pd.DataFrame( 

744 data={"diaObjectId": n_sources * [objId], 

745 "filterName": n_sources * ["u"], 

746 "diaSourceId": np.arange(n_sources, dtype=int), 

747 "psFlux": fluxes, 

748 "psFluxErr": errors, 

749 "midPointTai": times}) 

750 

751 plug = LinearFitDiaPsFlux(LinearFitDiaPsFluxConfig(), 

752 "ap_LinearFit", 

753 None) 

754 run_multi_plugin(diaObjects, diaSources, "u", plug) 

755 self.assertAlmostEqual(diaObjects.loc[objId, "uPSFluxLinearSlope"], 

756 2.) 

757 self.assertAlmostEqual(diaObjects.loc[objId, "uPSFluxLinearIntercept"], 

758 -1.) 

759 

760 # Test best fit linear model with input nans. 

761 fluxes[7] = np.nan 

762 errors[4] = np.nan 

763 times[2] = np.nan 

764 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

765 diaSources = pd.DataFrame( 

766 data={"diaObjectId": n_sources * [objId], 

767 "filterName": n_sources * ["r"], 

768 "diaSourceId": np.arange(n_sources, dtype=int), 

769 "psFlux": fluxes, 

770 "psFluxErr": errors, 

771 "midPointTai": times}) 

772 run_multi_plugin(diaObjects, diaSources, "r", plug) 

773 self.assertAlmostEqual(diaObjects.loc[objId, "rPSFluxLinearSlope"], 2.) 

774 self.assertAlmostEqual(diaObjects.loc[objId, "rPSFluxLinearIntercept"], 

775 -1.) 

776 

777 

778class TestStetsonJDiaPsFlux(unittest.TestCase): 

779 

780 def testCalculate(self): 

781 """Test the stetsonJ statistic. 

782 """ 

783 n_sources = 10 

784 objId = 0 

785 

786 # Test stetsonJ calculation. 

787 fluxes = np.linspace(-1, 1, n_sources) 

788 errors = np.ones(n_sources) 

789 diaObjects = pd.DataFrame({"diaObjectId": [objId], 

790 "uPSFluxMean": [np.nanmean(fluxes)]}) 

791 diaSources = pd.DataFrame( 

792 data={"diaObjectId": n_sources * [objId], 

793 "filterName": n_sources * ["u"], 

794 "diaSourceId": np.arange(n_sources, dtype=int), 

795 "psFlux": fluxes, 

796 "psFluxErr": errors}) 

797 

798 plug = StetsonJDiaPsFlux(StetsonJDiaPsFluxConfig(), 

799 "ap_StetsonJ", 

800 None) 

801 run_multi_plugin(diaObjects, diaSources, "u", plug) 

802 # Expected StetsonJ for the values created. Confirmed using Cesimum's 

803 # implementation. http://github.com/cesium-ml/cesium 

804 self.assertAlmostEqual(diaObjects.loc[objId, "uPSFluxStetsonJ"], 

805 -0.5958393936080928) 

806 

807 # Test stetsonJ calculation returns nan on single input. 

808 diaObjects = pd.DataFrame({"diaObjectId": [objId], 

809 "gPSFluxMean": [np.nanmean(fluxes)]}) 

810 diaSources = pd.DataFrame( 

811 data={"diaObjectId": 1 * [objId], 

812 "filterName": 1 * ["g"], 

813 "diaSourceId": np.arange(1, dtype=int), 

814 "psFlux": fluxes[0], 

815 "psFluxErr": errors[0]}) 

816 run_multi_plugin(diaObjects, diaSources, "g", plug) 

817 self.assertTrue(np.isnan(diaObjects.at[objId, "gPSFluxStetsonJ"])) 

818 

819 # Test stetsonJ calculation returns when nans are input. 

820 fluxes[7] = np.nan 

821 errors[4] = np.nan 

822 nonNanMask = np.logical_and(~np.isnan(fluxes), 

823 ~np.isnan(errors)) 

824 diaObjects = pd.DataFrame( 

825 {"diaObjectId": [objId], 

826 "rPSFluxMean": [np.average(fluxes[nonNanMask], 

827 weights=errors[nonNanMask])]}) 

828 diaSources = pd.DataFrame( 

829 data={"diaObjectId": n_sources * [objId], 

830 "filterName": n_sources * ["r"], 

831 "diaSourceId": np.arange(n_sources, dtype=int), 

832 "psFlux": fluxes, 

833 "psFluxErr": errors}) 

834 run_multi_plugin(diaObjects, diaSources, "r", plug) 

835 self.assertAlmostEqual(diaObjects.at[objId, "rPSFluxStetsonJ"], 

836 -0.5412797916187173) 

837 

838 

839class TestWeightedMeanDiaTotFlux(unittest.TestCase): 

840 

841 def testCalculate(self): 

842 """Test mean value calculation. 

843 """ 

844 n_sources = 10 

845 objId = 0 

846 

847 # Test test mean on totFlux. 

848 fluxes = np.linspace(-1, 1, n_sources) 

849 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

850 diaSources = pd.DataFrame( 

851 data={"diaObjectId": n_sources * [objId], 

852 "filterName": n_sources * ["u"], 

853 "diaSourceId": np.arange(n_sources, dtype=int), 

854 "totFlux": fluxes, 

855 "totFluxErr": np.ones(n_sources)}) 

856 

857 plug = WeightedMeanDiaTotFlux(WeightedMeanDiaTotFluxConfig(), 

858 "ap_meanTotFlux", 

859 None) 

860 run_multi_plugin(diaObjects, diaSources, "u", plug) 

861 

862 self.assertAlmostEqual(diaObjects.at[objId, "uTOTFluxMean"], 0.0) 

863 self.assertAlmostEqual(diaObjects.at[objId, "uTOTFluxMeanErr"], 

864 np.sqrt(1 / n_sources)) 

865 

866 # Test test mean on totFlux with input nans 

867 fluxes[4] = np.nan 

868 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

869 diaSources = pd.DataFrame( 

870 data={"diaObjectId": n_sources * [objId], 

871 "filterName": n_sources * ["r"], 

872 "diaSourceId": np.arange(n_sources, dtype=int), 

873 "totFlux": fluxes, 

874 "totFluxErr": np.ones(n_sources)}) 

875 run_multi_plugin(diaObjects, diaSources, "r", plug) 

876 

877 self.assertAlmostEqual(diaObjects.at[objId, "rTOTFluxMean"], 

878 np.nanmean(fluxes)) 

879 self.assertAlmostEqual(diaObjects.at[objId, "rTOTFluxMeanErr"], 

880 np.sqrt(1 / (n_sources - 1))) 

881 

882 

883class TestSigmaDiaTotFlux(unittest.TestCase): 

884 

885 def testCalculate(self): 

886 """Test flux scatter calculation. 

887 """ 

888 n_sources = 10 

889 objId = 0 

890 

891 # Test test scatter on totFlux. 

892 fluxes = np.linspace(-1, 1, n_sources) 

893 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

894 diaSources = pd.DataFrame( 

895 data={"diaObjectId": n_sources * [objId], 

896 "filterName": n_sources * ["u"], 

897 "diaSourceId": np.arange(n_sources, dtype=int), 

898 "totFlux": fluxes, 

899 "totFluxErr": np.ones(n_sources)}) 

900 

901 plug = SigmaDiaTotFlux(SigmaDiaTotFluxConfig(), 

902 "ap_sigmaTotFlux", 

903 None) 

904 run_multi_plugin(diaObjects, diaSources, "u", plug) 

905 self.assertAlmostEqual(diaObjects.at[objId, "uTOTFluxSigma"], 

906 np.nanstd(fluxes, ddof=1)) 

907 

908 # Test test scatter on totFlux returns nan on 1 input. 

909 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

910 diaSources = pd.DataFrame( 

911 data={"diaObjectId": 1 * [objId], 

912 "filterName": 1 * ["g"], 

913 "diaSourceId": np.arange(1, dtype=int), 

914 "totFlux": fluxes[0], 

915 "totFluxErr": np.ones(1)}) 

916 run_multi_plugin(diaObjects, diaSources, "g", plug) 

917 self.assertTrue(np.isnan(diaObjects.at[objId, "gTOTFluxSigma"])) 

918 

919 # Test test scatter on totFlux takes input nans. 

920 fluxes[4] = np.nan 

921 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

922 diaSources = pd.DataFrame( 

923 data={"diaObjectId": n_sources * [objId], 

924 "filterName": n_sources * ["r"], 

925 "diaSourceId": np.arange(n_sources, dtype=int), 

926 "totFlux": fluxes, 

927 "totFluxErr": np.ones(n_sources)}) 

928 run_multi_plugin(diaObjects, diaSources, "r", plug) 

929 self.assertAlmostEqual(diaObjects.at[objId, "rTOTFluxSigma"], 

930 np.nanstd(fluxes, ddof=1)) 

931 

932 

933def skew_wrapper(values): 

934 """Compute scipy skew, omitting nans. 

935 

936 This version works with both scipy<1.9 (where it erroneously returns a 

937 masked array) and scipy>=1.9 (where it correctly returns a float). 

938 

939 Parameters 

940 ---------- 

941 values : `np.ndarray` 

942 

943 Returns 

944 ------- 

945 skew_value : `float` 

946 """ 

947 value = skew(values, bias=False, nan_policy="omit") 

948 if isinstance(value, np.ma.masked_array): 

949 return value.data 

950 else: 

951 return value 

952 

953 

954class MemoryTester(lsst.utils.tests.MemoryTestCase): 

955 pass 

956 

957 

958def setup_module(module): 

959 lsst.utils.tests.init() 

960 

961 

962if __name__ == "__main__": 962 ↛ 963line 962 didn't jump to line 963, because the condition on line 962 was never true

963 lsst.utils.tests.init() 

964 unittest.main()