Coverage for tests/test_diaCalculationPlugins.py: 22%

338 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-06-18 10:52 +0000

1# This file is part of ap_association. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22from astropy.stats import median_absolute_deviation 

23import numpy as np 

24import pandas as pd 

25from scipy.stats import skew 

26import unittest 

27 

28from lsst.meas.base import ( 

29 MeanDiaPosition, MeanDiaPositionConfig, 

30 HTMIndexDiaPosition, HTMIndexDiaPositionConfig, 

31 NumDiaSourcesDiaPlugin, NumDiaSourcesDiaPluginConfig, 

32 SimpleSourceFlagDiaPlugin, SimpleSourceFlagDiaPluginConfig, 

33 WeightedMeanDiaPsFlux, WeightedMeanDiaPsFluxConfig, 

34 PercentileDiaPsFlux, PercentileDiaPsFluxConfig, 

35 SigmaDiaPsFlux, SigmaDiaPsFluxConfig, 

36 Chi2DiaPsFlux, Chi2DiaPsFluxConfig, 

37 MadDiaPsFlux, MadDiaPsFluxConfig, 

38 SkewDiaPsFlux, SkewDiaPsFluxConfig, 

39 MinMaxDiaPsFlux, MinMaxDiaPsFluxConfig, 

40 MaxSlopeDiaPsFlux, MaxSlopeDiaPsFluxConfig, 

41 ErrMeanDiaPsFlux, ErrMeanDiaPsFluxConfig, 

42 LinearFitDiaPsFlux, LinearFitDiaPsFluxConfig, 

43 StetsonJDiaPsFlux, StetsonJDiaPsFluxConfig, 

44 WeightedMeanDiaTotFlux, WeightedMeanDiaTotFluxConfig, 

45 SigmaDiaTotFlux, SigmaDiaTotFluxConfig) 

46import lsst.utils.tests 

47 

48 

49def run_single_plugin(diaObjectCat, 

50 diaObjectId, 

51 diaSourceCat, 

52 filterName, 

53 plugin): 

54 """Wrapper for running single plugins. 

55 

56 Reproduces some of the behavior of `lsst.ap.association.DiaCalcuation.run` 

57 

58 Parameters 

59 ---------- 

60 diaObjectCat : `pandas.DataFrame` 

61 Input object catalog to store data into and read from. 

62 diaSourcesCat : `pandas.DataFrame` 

63 DiaSource catalog to read data from and groupby on. 

64 fitlerName : `str` 

65 String name of the filter to process. 

66 plugin : `lsst.ap.association.DiaCalculationPlugin` 

67 Plugin to run. 

68 """ 

69 diaObjectCat.set_index("diaObjectId", inplace=True, drop=False) 

70 diaSourceCat.set_index( 

71 ["diaObjectId", "filterName", "diaSourceId"], 

72 inplace=True, 

73 drop=False) 

74 

75 objDiaSources = diaSourceCat.loc[diaObjectId] 

76 updatingFilterDiaSources = diaSourceCat.loc[ 

77 (diaObjectId, filterName), : 

78 ] 

79 

80 plugin.calculate(diaObjects=diaObjectCat, 

81 diaObjectId=diaObjectId, 

82 diaSources=objDiaSources, 

83 filterDiaSources=updatingFilterDiaSources, 

84 filterName=filterName) 

85 

86 

87def run_multi_plugin(diaObjectCat, diaSourceCat, filterName, plugin): 

88 """Wrapper for running multi plugins. 

89 

90 Reproduces some of the behavior of `lsst.ap.association.DiaCalcuation.run` 

91 

92 Parameters 

93 ---------- 

94 diaObjectCat : `pandas.DataFrame` 

95 Input object catalog to store data into and read from. 

96 diaSourcesCat : `pandas.DataFrame` 

97 DiaSource catalog to read data from and groupby on. 

98 fitlerName : `str` 

99 String name of the filter to process. 

100 plugin : `lsst.ap.association.DiaCalculationPlugin` 

101 Plugin to run. 

102 """ 

103 diaObjectCat.set_index("diaObjectId", inplace=True, drop=False) 

104 diaSourceCat.set_index( 

105 ["diaObjectId", "filterName", "diaSourceId"], 

106 inplace=True, 

107 drop=False) 

108 

109 updatingFilterDiaSources = diaSourceCat.loc[ 

110 (slice(None), filterName), : 

111 ] 

112 

113 diaSourcesGB = diaSourceCat.groupby(level=0) 

114 filterDiaSourcesGB = updatingFilterDiaSources.groupby(level=0) 

115 

116 plugin.calculate(diaObjects=diaObjectCat, 

117 diaSources=diaSourcesGB, 

118 filterDiaSources=filterDiaSourcesGB, 

119 filterName=filterName) 

120 

121 

122class TestMeanPosition(unittest.TestCase): 

123 

124 def testCalculate(self): 

125 """Test mean position calculation. 

126 """ 

127 n_sources = 10 

128 objId = 0 

129 

130 plug = MeanDiaPosition(MeanDiaPositionConfig(), 

131 "ap_meanPosition", 

132 None) 

133 

134 # Test expected means in RA. 

135 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

136 diaSources = pd.DataFrame(data={"ra": np.linspace(-1, 1, n_sources), 

137 "decl": np.zeros(n_sources), 

138 "midPointTai": np.linspace(0, 

139 n_sources, 

140 n_sources), 

141 "diaObjectId": n_sources * [objId], 

142 "filterName": n_sources * ["g"], 

143 "diaSourceId": np.arange(n_sources, 

144 dtype=int)}) 

145 run_multi_plugin(diaObjects, diaSources, "g", plug) 

146 

147 self.assertAlmostEqual(diaObjects.loc[objId, "ra"], 0.0) 

148 self.assertAlmostEqual(diaObjects.loc[objId, "decl"], 0.0) 

149 self.assertEqual(diaObjects.loc[objId, "radecTai"], 10) 

150 

151 # Test expected means in DEC. 

152 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

153 diaSources = pd.DataFrame(data={"ra": np.zeros(n_sources), 

154 "decl": np.linspace(-1, 1, n_sources), 

155 "midPointTai": np.linspace(0, 

156 n_sources, 

157 n_sources), 

158 "diaObjectId": n_sources * [objId], 

159 "filterName": n_sources * ["g"], 

160 "diaSourceId": np.arange(n_sources, 

161 dtype=int)}) 

162 run_multi_plugin(diaObjects, diaSources, "g", plug) 

163 

164 self.assertAlmostEqual(diaObjects.loc[objId, "ra"], 0.0) 

165 self.assertAlmostEqual(diaObjects.loc[objId, "decl"], 0.0) 

166 self.assertEqual(diaObjects.loc[objId, "radecTai"], 10) 

167 

168 # Test failure mode RA is nan. 

169 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

170 diaSources = pd.DataFrame(data={"ra": np.full(n_sources, np.nan), 

171 "decl": np.zeros(n_sources), 

172 "midPointTai": np.linspace(0, 

173 n_sources, 

174 n_sources), 

175 "diaObjectId": n_sources * [objId], 

176 "filterName": n_sources * ["g"], 

177 "diaSourceId": np.arange(n_sources, 

178 dtype=int)}) 

179 run_multi_plugin(diaObjects, diaSources, "g", plug) 

180 

181 self.assertTrue(np.isnan(diaObjects.loc[objId, "ra"])) 

182 self.assertTrue(np.isnan(diaObjects.loc[objId, "decl"])) 

183 self.assertTrue(np.isnan(diaObjects.loc[objId, "radecTai"])) 

184 

185 # Test failure mode DEC is nan. 

186 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

187 diaSources = pd.DataFrame(data={"ra": np.zeros(n_sources), 

188 "decl": np.full(n_sources, np.nan), 

189 "midPointTai": np.linspace(0, 

190 n_sources, 

191 n_sources), 

192 "diaObjectId": n_sources * [objId], 

193 "filterName": n_sources * ["g"], 

194 "diaSourceId": np.arange(n_sources, 

195 dtype=int)}) 

196 run_multi_plugin(diaObjects, diaSources, "g", plug) 

197 

198 self.assertTrue(np.isnan(diaObjects.loc[objId, "ra"])) 

199 self.assertTrue(np.isnan(diaObjects.loc[objId, "decl"])) 

200 self.assertTrue(np.isnan(diaObjects.loc[objId, "radecTai"])) 

201 

202 

203class TestHTMIndexPosition(unittest.TestCase): 

204 

205 def testCalculate(self): 

206 """Test HTMPixel assignment calculation. 

207 """ 

208 # Test expected pixelId at RA, DEC = 0 

209 objId = 0 

210 n_sources = 10 

211 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

212 diaObjects.loc[objId, "ra"] = 0. 

213 diaObjects.loc[objId, "decl"] = 0. 

214 diaSources = pd.DataFrame( 

215 data={"diaObjectId": n_sources * [objId], 

216 "filterName": n_sources * ["g"], 

217 "diaSourceId": np.arange(n_sources, dtype=int)}) 

218 plug = HTMIndexDiaPosition(HTMIndexDiaPositionConfig(), 

219 "ap_HTMIndex", 

220 None) 

221 

222 run_single_plugin(diaObjectCat=diaObjects, 

223 diaObjectId=objId, 

224 diaSourceCat=diaSources, 

225 filterName="g", 

226 plugin=plug) 

227 self.assertEqual(diaObjects.at[objId, "pixelId"], 

228 17042430230528) 

229 

230 # Test expected pixelId at some value of RA and DEC. 

231 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

232 diaObjects.loc[objId, "ra"] = 45.37 

233 diaObjects.loc[objId, "decl"] = 13.67 

234 diaSources = pd.DataFrame( 

235 data={"diaObjectId": n_sources * [objId], 

236 "filterName": n_sources * ["g"], 

237 "diaSourceId": np.arange(n_sources, dtype=int)}) 

238 run_single_plugin(diaObjectCat=diaObjects, 

239 diaObjectId=objId, 

240 diaSourceCat=diaSources, 

241 filterName="g", 

242 plugin=plug) 

243 self.assertEqual(diaObjects.at[objId, "pixelId"], 

244 17450571968473) 

245 

246 

247class TestNDiaSourcesDiaPlugin(unittest.TestCase): 

248 

249 def testCalculate(self): 

250 """Test that the number of DiaSources is correct. 

251 """ 

252 

253 for n_sources in [1, 8, 10]: 

254 # Test expected number of sources per object. 

255 objId = 0 

256 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

257 diaSources = pd.DataFrame( 

258 data={"diaObjectId": n_sources * [objId], 

259 "filterName": n_sources * ["g"], 

260 "diaSourceId": np.arange(n_sources, dtype=int)}) 

261 plug = NumDiaSourcesDiaPlugin(NumDiaSourcesDiaPluginConfig(), 

262 "ap_nDiaSources", 

263 None) 

264 run_multi_plugin(diaObjects, diaSources, "g", plug) 

265 

266 self.assertEqual(n_sources, diaObjects.at[objId, "nDiaSources"]) 

267 

268 

269class TestSimpleSourceFlagDiaPlugin(unittest.TestCase): 

270 

271 def testCalculate(self): 

272 """Test that DiaObject flags are set. 

273 """ 

274 objId = 0 

275 n_sources = 10 

276 

277 # Test expected flags, no flags set. 

278 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

279 diaSources = pd.DataFrame( 

280 data={"diaObjectId": n_sources * [objId], 

281 "filterName": n_sources * ["g"], 

282 "diaSourceId": np.arange(n_sources, dtype=int), 

283 "flags": np.zeros(n_sources, dtype=np.uint64)}) 

284 plug = SimpleSourceFlagDiaPlugin(SimpleSourceFlagDiaPluginConfig(), 

285 "ap_diaObjectFlag", 

286 None) 

287 run_multi_plugin(diaObjects, diaSources, "g", plug) 

288 self.assertEqual(diaObjects.at[objId, "flags"], 0) 

289 

290 # Test expected flags, all flags set. 

291 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

292 diaSources = pd.DataFrame( 

293 data={"diaObjectId": n_sources * [objId], 

294 "filterName": n_sources * ["g"], 

295 "diaSourceId": np.arange(n_sources, dtype=int), 

296 "flags": np.ones(n_sources, dtype=np.uint64)}) 

297 run_multi_plugin(diaObjects, diaSources, "g", plug) 

298 self.assertEqual(diaObjects.at[objId, "flags"], 1) 

299 

300 # Test expected flags, random flags. 

301 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

302 diaSources = pd.DataFrame( 

303 data={"diaObjectId": n_sources * [objId], 

304 "filterName": n_sources * ["g"], 

305 "diaSourceId": np.arange(n_sources, dtype=int), 

306 "flags": np.random.randint(0, 2 ** 16, size=n_sources)}) 

307 run_multi_plugin(diaObjects, diaSources, "g", plug) 

308 self.assertEqual(diaObjects.at[objId, "flags"], 1) 

309 

310 # Test expected flags, one flag set. 

311 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

312 flag_array = np.zeros(n_sources, dtype=np.uint64) 

313 flag_array[4] = 256 

314 diaSources = pd.DataFrame( 

315 data={"diaObjectId": n_sources * [objId], 

316 "filterName": n_sources * ["g"], 

317 "diaSourceId": np.arange(n_sources, dtype=int), 

318 "flags": flag_array}) 

319 run_multi_plugin(diaObjects, diaSources, "g", plug) 

320 self.assertEqual(diaObjects.at[objId, "flags"], 1) 

321 

322 

323class TestWeightedMeanDiaPsFlux(unittest.TestCase): 

324 

325 def testCalculate(self): 

326 """Test mean value calculation. 

327 """ 

328 n_sources = 10 

329 objId = 0 

330 

331 # Test expected mean. 

332 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

333 diaSources = pd.DataFrame( 

334 data={"diaObjectId": n_sources * [objId], 

335 "filterName": n_sources * ["u"], 

336 "diaSourceId": np.arange(n_sources, dtype=int), 

337 "psFlux": np.linspace(-1, 1, n_sources), 

338 "psFluxErr": np.ones(n_sources)}) 

339 

340 plug = WeightedMeanDiaPsFlux(WeightedMeanDiaPsFluxConfig(), 

341 "ap_meanFlux", 

342 None) 

343 run_multi_plugin(diaObjects, diaSources, "u", plug) 

344 

345 self.assertAlmostEqual(diaObjects.loc[objId, "uPSFluxMean"], 0.0) 

346 self.assertAlmostEqual(diaObjects.loc[objId, "uPSFluxMeanErr"], 

347 np.sqrt(1 / n_sources)) 

348 self.assertEqual(diaObjects.loc[objId, "uPSFluxNdata"], n_sources) 

349 

350 # Test expected mean with a nan value. 

351 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

352 fluxes = np.linspace(-1, 1, n_sources) 

353 fluxes[4] = np.nan 

354 diaSources = pd.DataFrame( 

355 data={"diaObjectId": n_sources * [objId], 

356 "filterName": n_sources * ["r"], 

357 "diaSourceId": np.arange(n_sources, dtype=int), 

358 "psFlux": fluxes, 

359 "psFluxErr": np.ones(n_sources)}) 

360 run_multi_plugin(diaObjects, diaSources, "r", plug) 

361 

362 self.assertAlmostEqual(diaObjects.at[objId, "rPSFluxMean"], 

363 np.nanmean(fluxes)) 

364 self.assertAlmostEqual(diaObjects.at[objId, "rPSFluxMeanErr"], 

365 np.sqrt(1 / (n_sources - 1))) 

366 self.assertEqual(diaObjects.loc[objId, "rPSFluxNdata"], n_sources - 1) 

367 

368 

369class TestPercentileDiaPsFlux(unittest.TestCase): 

370 

371 def testCalculate(self): 

372 """Test flux percentile calculation. 

373 """ 

374 n_sources = 10 

375 objId = 0 

376 

377 # Test expected percentile values. 

378 fluxes = np.linspace(-1, 1, n_sources) 

379 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

380 diaSources = pd.DataFrame( 

381 data={"diaObjectId": n_sources * [objId], 

382 "filterName": n_sources * ["u"], 

383 "diaSourceId": np.arange(n_sources, dtype=int), 

384 "psFlux": fluxes, 

385 "psFluxErr": np.ones(n_sources)}) 

386 

387 plug = PercentileDiaPsFlux(PercentileDiaPsFluxConfig(), 

388 "ap_percentileFlux", 

389 None) 

390 run_multi_plugin(diaObjects, diaSources, "u", plug) 

391 for pTile, testVal in zip(plug.config.percentiles, 

392 np.nanpercentile( 

393 fluxes, 

394 plug.config.percentiles)): 

395 self.assertAlmostEqual( 

396 diaObjects.at[objId, "uPSFluxPercentile{:02d}".format(pTile)], 

397 testVal) 

398 

399 # Test expected percentile values with a nan value. 

400 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

401 fluxes[4] = np.nan 

402 diaSources = pd.DataFrame( 

403 data={"diaObjectId": n_sources * [objId], 

404 "filterName": n_sources * ["r"], 

405 "diaSourceId": np.arange(n_sources, dtype=int), 

406 "psFlux": fluxes, 

407 "psFluxErr": np.ones(n_sources)}) 

408 run_multi_plugin(diaObjects, diaSources, "r", plug) 

409 for pTile, testVal in zip(plug.config.percentiles, 

410 np.nanpercentile( 

411 fluxes, 

412 plug.config.percentiles)): 

413 self.assertAlmostEqual( 

414 diaObjects.at[objId, "rPSFluxPercentile{:02d}".format(pTile)], 

415 testVal) 

416 

417 

418class TestSigmaDiaPsFlux(unittest.TestCase): 

419 

420 def testCalculate(self): 

421 """Test flux scatter calculation. 

422 """ 

423 n_sources = 10 

424 objId = 0 

425 

426 # Test expected sigma scatter of fluxes. 

427 fluxes = np.linspace(-1, 1, n_sources) 

428 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

429 diaSources = pd.DataFrame( 

430 data={"diaObjectId": n_sources * [objId], 

431 "filterName": n_sources * ["u"], 

432 "diaSourceId": np.arange(n_sources, dtype=int), 

433 "psFlux": fluxes, 

434 "psFluxErr": np.ones(n_sources)}) 

435 

436 plug = SigmaDiaPsFlux(SigmaDiaPsFluxConfig(), 

437 "ap_sigmaFlux", 

438 None) 

439 run_multi_plugin(diaObjects, diaSources, "u", plug) 

440 self.assertAlmostEqual(diaObjects.at[objId, "uPSFluxSigma"], 

441 np.nanstd(fluxes, ddof=1)) 

442 

443 # test one input, returns nan. 

444 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

445 diaSources = pd.DataFrame( 

446 data={"diaObjectId": 1 * [objId], 

447 "filterName": 1 * ["g"], 

448 "diaSourceId": [0], 

449 "psFlux": [fluxes[0]], 

450 "psFluxErr": [1.]}) 

451 run_multi_plugin(diaObjects, diaSources, "g", plug) 

452 self.assertTrue(np.isnan(diaObjects.at[objId, "gPSFluxSigma"])) 

453 

454 # Test expected sigma scatter of fluxes with a nan value. 

455 fluxes[4] = np.nan 

456 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

457 diaSources = pd.DataFrame( 

458 data={"diaObjectId": n_sources * [objId], 

459 "filterName": n_sources * ["r"], 

460 "diaSourceId": np.arange(n_sources, dtype=int), 

461 "psFlux": fluxes, 

462 "psFluxErr": np.ones(n_sources)}) 

463 run_multi_plugin(diaObjects, diaSources, "r", plug) 

464 self.assertAlmostEqual(diaObjects.at[objId, "rPSFluxSigma"], 

465 np.nanstd(fluxes, ddof=1)) 

466 

467 

468class TestChi2DiaPsFlux(unittest.TestCase): 

469 

470 def testCalculate(self): 

471 """Test flux chi2 calculation. 

472 """ 

473 n_sources = 10 

474 objId = 0 

475 

476 # Test expected chi^2 value. 

477 fluxes = np.linspace(-1, 1, n_sources) 

478 diaObjects = pd.DataFrame({"diaObjectId": [objId], 

479 "uPSFluxMean": [0.0]}) 

480 diaSources = pd.DataFrame( 

481 data={"diaObjectId": n_sources * [objId], 

482 "filterName": n_sources * ["u"], 

483 "diaSourceId": np.arange(n_sources, dtype=int), 

484 "psFlux": fluxes, 

485 "psFluxErr": np.ones(n_sources)}) 

486 

487 plug = Chi2DiaPsFlux(Chi2DiaPsFluxConfig(), 

488 "ap_chi2Flux", 

489 None) 

490 run_multi_plugin(diaObjects, diaSources, "u", plug) 

491 self.assertAlmostEqual( 

492 diaObjects.loc[objId, "uPSFluxChi2"], 

493 np.nansum(((diaSources["psFlux"] 

494 - np.nanmean(diaSources["psFlux"])) 

495 / diaSources["psFluxErr"]) ** 2)) 

496 

497 # Test expected chi^2 value with a nan value set. 

498 fluxes[4] = np.nan 

499 diaObjects = pd.DataFrame({"diaObjectId": [objId], 

500 "rPSFluxMean": [np.nanmean(fluxes)]}) 

501 diaSources = pd.DataFrame( 

502 data={"diaObjectId": n_sources * [objId], 

503 "filterName": n_sources * ["r"], 

504 "diaSourceId": np.arange(n_sources, dtype=int), 

505 "psFlux": fluxes, 

506 "psFluxErr": np.ones(n_sources)}) 

507 run_multi_plugin(diaObjects, diaSources, "r", plug) 

508 self.assertAlmostEqual( 

509 diaObjects.loc[objId, "rPSFluxChi2"], 

510 np.nansum(((diaSources["psFlux"] 

511 - np.nanmean(diaSources["psFlux"])) 

512 / diaSources["psFluxErr"]) ** 2)) 

513 

514 

515class TestMadDiaPsFlux(unittest.TestCase): 

516 

517 def testCalculate(self): 

518 """Test flux median absolute deviation calculation. 

519 """ 

520 n_sources = 10 

521 objId = 0 

522 

523 # Test expected MAD value. 

524 fluxes = np.linspace(-1, 1, n_sources) 

525 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

526 diaSources = pd.DataFrame( 

527 data={"diaObjectId": n_sources * [objId], 

528 "filterName": n_sources * ["u"], 

529 "diaSourceId": np.arange(n_sources, dtype=int), 

530 "psFlux": fluxes, 

531 "psFluxErr": np.ones(n_sources)}) 

532 

533 plug = MadDiaPsFlux(MadDiaPsFluxConfig(), 

534 "ap_madFlux", 

535 None) 

536 run_multi_plugin(diaObjects, diaSources, "u", plug) 

537 self.assertAlmostEqual(diaObjects.at[objId, "uPSFluxMAD"], 

538 median_absolute_deviation(fluxes, 

539 ignore_nan=True)) 

540 

541 # Test expected MAD value with a nan set. 

542 fluxes[4] = np.nan 

543 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

544 diaSources = pd.DataFrame( 

545 data={"diaObjectId": n_sources * [objId], 

546 "filterName": n_sources * ["r"], 

547 "diaSourceId": np.arange(n_sources, dtype=int), 

548 "psFlux": fluxes, 

549 "psFluxErr": np.ones(n_sources)}) 

550 run_multi_plugin(diaObjects, diaSources, "r", plug) 

551 self.assertAlmostEqual(diaObjects.at[objId, "rPSFluxMAD"], 

552 median_absolute_deviation(fluxes, 

553 ignore_nan=True)) 

554 

555 

556class TestSkewDiaPsFlux(unittest.TestCase): 

557 

558 def testCalculate(self): 

559 """Test flux skew calculation. 

560 """ 

561 n_sources = 10 

562 objId = 0 

563 

564 # Test expected skew value. 

565 fluxes = np.linspace(-1, 1, n_sources) 

566 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

567 diaSources = pd.DataFrame( 

568 data={"diaObjectId": n_sources * [objId], 

569 "filterName": n_sources * ["u"], 

570 "diaSourceId": np.arange(n_sources, dtype=int), 

571 "psFlux": fluxes, 

572 "psFluxErr": np.ones(n_sources)}) 

573 

574 plug = SkewDiaPsFlux(SkewDiaPsFluxConfig(), 

575 "ap_skewFlux", 

576 None) 

577 run_multi_plugin(diaObjects, diaSources, "u", plug) 

578 self.assertAlmostEqual( 

579 diaObjects.loc[objId, "uPSFluxSkew"], 

580 skew(fluxes, bias=False, nan_policy="omit")) 

581 

582 # Test expected skew value with a nan set. 

583 fluxes[4] = np.nan 

584 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

585 diaSources = pd.DataFrame( 

586 data={"diaObjectId": n_sources * [objId], 

587 "filterName": n_sources * ["r"], 

588 "diaSourceId": np.arange(n_sources, dtype=int), 

589 "psFlux": fluxes, 

590 "psFluxErr": np.ones(n_sources)}) 

591 run_multi_plugin(diaObjects, diaSources, "r", plug) 

592 # Skew returns a named tuple when called on an array 

593 # with nan values. 

594 self.assertAlmostEqual( 

595 diaObjects.at[objId, "rPSFluxSkew"], 

596 skew(fluxes, bias=False, nan_policy="omit").data) 

597 

598 

599class TestMinMaxDiaPsFlux(unittest.TestCase): 

600 

601 def testCalculate(self): 

602 """Test flux min/max calculation. 

603 """ 

604 n_sources = 10 

605 objId = 0 

606 

607 # Test expected MinMax fluxes. 

608 fluxes = np.linspace(-1, 1, n_sources) 

609 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

610 diaSources = pd.DataFrame( 

611 data={"diaObjectId": n_sources * [objId], 

612 "filterName": n_sources * ["u"], 

613 "diaSourceId": np.arange(n_sources, dtype=int), 

614 "psFlux": fluxes, 

615 "psFluxErr": np.ones(n_sources)}) 

616 

617 plug = MinMaxDiaPsFlux(MinMaxDiaPsFluxConfig(), 

618 "ap_minMaxFlux", 

619 None) 

620 run_multi_plugin(diaObjects, diaSources, "u", plug) 

621 self.assertEqual(diaObjects.loc[objId, "uPSFluxMin"], -1) 

622 self.assertEqual(diaObjects.loc[objId, "uPSFluxMax"], 1) 

623 

624 # Test expected MinMax fluxes with a nan set. 

625 fluxes[4] = np.nan 

626 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

627 diaSources = pd.DataFrame( 

628 data={"diaObjectId": n_sources * [objId], 

629 "filterName": n_sources * ["r"], 

630 "diaSourceId": np.arange(n_sources, dtype=int), 

631 "psFlux": fluxes, 

632 "psFluxErr": np.ones(n_sources)}) 

633 run_multi_plugin(diaObjects, diaSources, "r", plug) 

634 self.assertEqual(diaObjects.loc[objId, "rPSFluxMin"], -1) 

635 self.assertEqual(diaObjects.loc[objId, "rPSFluxMax"], 1) 

636 

637 

638class TestMaxSlopeDiaPsFlux(unittest.TestCase): 

639 

640 def testCalculate(self): 

641 """Test flux maximum slope. 

642 """ 

643 n_sources = 10 

644 objId = 0 

645 

646 # Test max slope value. 

647 fluxes = np.linspace(-1, 1, n_sources) 

648 times = np.concatenate([np.linspace(0, 1, n_sources)[:-1], [1 - 1/90]]) 

649 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

650 diaSources = pd.DataFrame( 

651 data={"diaObjectId": n_sources * [objId], 

652 "filterName": n_sources * ["u"], 

653 "diaSourceId": np.arange(n_sources, dtype=int), 

654 "psFlux": fluxes, 

655 "psFluxErr": np.ones(n_sources), 

656 "midPointTai": times}) 

657 

658 plug = MaxSlopeDiaPsFlux(MaxSlopeDiaPsFluxConfig(), 

659 "ap_maxSlopeFlux", 

660 None) 

661 run_multi_plugin(diaObjects, diaSources, "u", plug) 

662 self.assertAlmostEqual(diaObjects.at[objId, "uPSFluxMaxSlope"], 2 + 2/9) 

663 

664 # Test max slope value returns nan on 1 input. 

665 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

666 diaSources = pd.DataFrame( 

667 data={"diaObjectId": 1 * [objId], 

668 "filterName": 1 * ["g"], 

669 "diaSourceId": np.arange(1, dtype=int), 

670 "psFlux": fluxes[0], 

671 "psFluxErr": np.ones(1), 

672 "midPointTai": times[0]}) 

673 run_multi_plugin(diaObjects, diaSources, "g", plug) 

674 self.assertTrue(np.isnan(diaObjects.at[objId, "gPSFluxMaxSlope"])) 

675 

676 # Test max slope value inputing nan values. 

677 fluxes[4] = np.nan 

678 times[7] = np.nan 

679 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

680 diaSources = pd.DataFrame( 

681 data={"diaObjectId": n_sources * [objId], 

682 "filterName": n_sources * ["r"], 

683 "diaSourceId": np.arange(n_sources, dtype=int), 

684 "psFlux": fluxes, 

685 "psFluxErr": np.ones(n_sources), 

686 "midPointTai": times}) 

687 run_multi_plugin(diaObjects, diaSources, "r", plug) 

688 self.assertAlmostEqual(diaObjects.at[objId, "rPSFluxMaxSlope"], 2 + 2 / 9) 

689 

690 

691class TestErrMeanDiaPsFlux(unittest.TestCase): 

692 

693 def testCalculate(self): 

694 """Test error mean calculation. 

695 """ 

696 n_sources = 10 

697 objId = 0 

698 

699 # Test mean of the errors. 

700 fluxes = np.linspace(-1, 1, n_sources) 

701 errors = np.linspace(1, 2, n_sources) 

702 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

703 diaSources = pd.DataFrame( 

704 data={"diaObjectId": n_sources * [objId], 

705 "filterName": n_sources * ["u"], 

706 "diaSourceId": np.arange(n_sources, dtype=int), 

707 "psFlux": fluxes, 

708 "psFluxErr": errors}) 

709 

710 plug = ErrMeanDiaPsFlux(ErrMeanDiaPsFluxConfig(), 

711 "ap_errMeanFlux", 

712 None) 

713 run_multi_plugin(diaObjects, diaSources, "u", plug) 

714 self.assertAlmostEqual(diaObjects.at[objId, "uPSFluxErrMean"], 

715 np.nanmean(errors)) 

716 

717 # Test mean of the errors with input nan value. 

718 errors[4] = np.nan 

719 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

720 diaSources = pd.DataFrame( 

721 data={"diaObjectId": n_sources * [objId], 

722 "filterName": n_sources * ["r"], 

723 "diaSourceId": np.arange(n_sources, dtype=int), 

724 "psFlux": fluxes, 

725 "psFluxErr": errors}) 

726 run_multi_plugin(diaObjects, diaSources, "r", plug) 

727 self.assertAlmostEqual(diaObjects.at[objId, "rPSFluxErrMean"], 

728 np.nanmean(errors)) 

729 

730 

731class TestLinearFitDiaPsFlux(unittest.TestCase): 

732 

733 def testCalculate(self): 

734 """Test a linear fit to flux vs time. 

735 """ 

736 n_sources = 10 

737 objId = 0 

738 

739 # Test best fit linear model. 

740 fluxes = np.linspace(-1, 1, n_sources) 

741 errors = np.linspace(1, 2, n_sources) 

742 times = np.linspace(0, 1, n_sources) 

743 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

744 diaSources = pd.DataFrame( 

745 data={"diaObjectId": n_sources * [objId], 

746 "filterName": n_sources * ["u"], 

747 "diaSourceId": np.arange(n_sources, dtype=int), 

748 "psFlux": fluxes, 

749 "psFluxErr": errors, 

750 "midPointTai": times}) 

751 

752 plug = LinearFitDiaPsFlux(LinearFitDiaPsFluxConfig(), 

753 "ap_LinearFit", 

754 None) 

755 run_multi_plugin(diaObjects, diaSources, "u", plug) 

756 self.assertAlmostEqual(diaObjects.loc[objId, "uPSFluxLinearSlope"], 

757 2.) 

758 self.assertAlmostEqual(diaObjects.loc[objId, "uPSFluxLinearIntercept"], 

759 -1.) 

760 

761 # Test best fit linear model with input nans. 

762 fluxes[7] = np.nan 

763 errors[4] = np.nan 

764 times[2] = np.nan 

765 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

766 diaSources = pd.DataFrame( 

767 data={"diaObjectId": n_sources * [objId], 

768 "filterName": n_sources * ["r"], 

769 "diaSourceId": np.arange(n_sources, dtype=int), 

770 "psFlux": fluxes, 

771 "psFluxErr": errors, 

772 "midPointTai": times}) 

773 run_multi_plugin(diaObjects, diaSources, "r", plug) 

774 self.assertAlmostEqual(diaObjects.loc[objId, "rPSFluxLinearSlope"], 2.) 

775 self.assertAlmostEqual(diaObjects.loc[objId, "rPSFluxLinearIntercept"], 

776 -1.) 

777 

778 

779class TestStetsonJDiaPsFlux(unittest.TestCase): 

780 

781 def testCalculate(self): 

782 """Test the stetsonJ statistic. 

783 """ 

784 n_sources = 10 

785 objId = 0 

786 

787 # Test stetsonJ calculation. 

788 fluxes = np.linspace(-1, 1, n_sources) 

789 errors = np.ones(n_sources) 

790 diaObjects = pd.DataFrame({"diaObjectId": [objId], 

791 "uPSFluxMean": [np.nanmean(fluxes)]}) 

792 diaSources = pd.DataFrame( 

793 data={"diaObjectId": n_sources * [objId], 

794 "filterName": n_sources * ["u"], 

795 "diaSourceId": np.arange(n_sources, dtype=int), 

796 "psFlux": fluxes, 

797 "psFluxErr": errors}) 

798 

799 plug = StetsonJDiaPsFlux(StetsonJDiaPsFluxConfig(), 

800 "ap_StetsonJ", 

801 None) 

802 run_multi_plugin(diaObjects, diaSources, "u", plug) 

803 # Expected StetsonJ for the values created. Confirmed using Cesimum's 

804 # implementation. http://github.com/cesium-ml/cesium 

805 self.assertAlmostEqual(diaObjects.loc[objId, "uPSFluxStetsonJ"], 

806 -0.5958393936080928) 

807 

808 # Test stetsonJ calculation returns nan on single input. 

809 diaObjects = pd.DataFrame({"diaObjectId": [objId], 

810 "gPSFluxMean": [np.nanmean(fluxes)]}) 

811 diaSources = pd.DataFrame( 

812 data={"diaObjectId": 1 * [objId], 

813 "filterName": 1 * ["g"], 

814 "diaSourceId": np.arange(1, dtype=int), 

815 "psFlux": fluxes[0], 

816 "psFluxErr": errors[0]}) 

817 run_multi_plugin(diaObjects, diaSources, "g", plug) 

818 self.assertTrue(np.isnan(diaObjects.at[objId, "gPSFluxStetsonJ"])) 

819 

820 # Test stetsonJ calculation returns when nans are input. 

821 fluxes[7] = np.nan 

822 errors[4] = np.nan 

823 nonNanMask = np.logical_and(~np.isnan(fluxes), 

824 ~np.isnan(errors)) 

825 diaObjects = pd.DataFrame( 

826 {"diaObjectId": [objId], 

827 "rPSFluxMean": [np.average(fluxes[nonNanMask], 

828 weights=errors[nonNanMask])]}) 

829 diaSources = pd.DataFrame( 

830 data={"diaObjectId": n_sources * [objId], 

831 "filterName": n_sources * ["r"], 

832 "diaSourceId": np.arange(n_sources, dtype=int), 

833 "psFlux": fluxes, 

834 "psFluxErr": errors}) 

835 run_multi_plugin(diaObjects, diaSources, "r", plug) 

836 self.assertAlmostEqual(diaObjects.at[objId, "rPSFluxStetsonJ"], 

837 -0.5412797916187173) 

838 

839 

840class TestWeightedMeanDiaTotFlux(unittest.TestCase): 

841 

842 def testCalculate(self): 

843 """Test mean value calculation. 

844 """ 

845 n_sources = 10 

846 objId = 0 

847 

848 # Test test mean on totFlux. 

849 fluxes = np.linspace(-1, 1, n_sources) 

850 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

851 diaSources = pd.DataFrame( 

852 data={"diaObjectId": n_sources * [objId], 

853 "filterName": n_sources * ["u"], 

854 "diaSourceId": np.arange(n_sources, dtype=int), 

855 "totFlux": fluxes, 

856 "totFluxErr": np.ones(n_sources)}) 

857 

858 plug = WeightedMeanDiaTotFlux(WeightedMeanDiaTotFluxConfig(), 

859 "ap_meanTotFlux", 

860 None) 

861 run_multi_plugin(diaObjects, diaSources, "u", plug) 

862 

863 self.assertAlmostEqual(diaObjects.at[objId, "uTOTFluxMean"], 0.0) 

864 self.assertAlmostEqual(diaObjects.at[objId, "uTOTFluxMeanErr"], 

865 np.sqrt(1 / n_sources)) 

866 

867 # Test test mean on totFlux with input nans 

868 fluxes[4] = np.nan 

869 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

870 diaSources = pd.DataFrame( 

871 data={"diaObjectId": n_sources * [objId], 

872 "filterName": n_sources * ["r"], 

873 "diaSourceId": np.arange(n_sources, dtype=int), 

874 "totFlux": fluxes, 

875 "totFluxErr": np.ones(n_sources)}) 

876 run_multi_plugin(diaObjects, diaSources, "r", plug) 

877 

878 self.assertAlmostEqual(diaObjects.at[objId, "rTOTFluxMean"], 

879 np.nanmean(fluxes)) 

880 self.assertAlmostEqual(diaObjects.at[objId, "rTOTFluxMeanErr"], 

881 np.sqrt(1 / (n_sources - 1))) 

882 

883 

884class TestSigmaDiaTotFlux(unittest.TestCase): 

885 

886 def testCalculate(self): 

887 """Test flux scatter calculation. 

888 """ 

889 n_sources = 10 

890 objId = 0 

891 

892 # Test test scatter on totFlux. 

893 fluxes = np.linspace(-1, 1, n_sources) 

894 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

895 diaSources = pd.DataFrame( 

896 data={"diaObjectId": n_sources * [objId], 

897 "filterName": n_sources * ["u"], 

898 "diaSourceId": np.arange(n_sources, dtype=int), 

899 "totFlux": fluxes, 

900 "totFluxErr": np.ones(n_sources)}) 

901 

902 plug = SigmaDiaTotFlux(SigmaDiaTotFluxConfig(), 

903 "ap_sigmaTotFlux", 

904 None) 

905 run_multi_plugin(diaObjects, diaSources, "u", plug) 

906 self.assertAlmostEqual(diaObjects.at[objId, "uTOTFluxSigma"], 

907 np.nanstd(fluxes, ddof=1)) 

908 

909 # Test test scatter on totFlux returns nan on 1 input. 

910 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

911 diaSources = pd.DataFrame( 

912 data={"diaObjectId": 1 * [objId], 

913 "filterName": 1 * ["g"], 

914 "diaSourceId": np.arange(1, dtype=int), 

915 "totFlux": fluxes[0], 

916 "totFluxErr": np.ones(1)}) 

917 run_multi_plugin(diaObjects, diaSources, "g", plug) 

918 self.assertTrue(np.isnan(diaObjects.at[objId, "gTOTFluxSigma"])) 

919 

920 # Test test scatter on totFlux takes input nans. 

921 fluxes[4] = np.nan 

922 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

923 diaSources = pd.DataFrame( 

924 data={"diaObjectId": n_sources * [objId], 

925 "filterName": n_sources * ["r"], 

926 "diaSourceId": np.arange(n_sources, dtype=int), 

927 "totFlux": fluxes, 

928 "totFluxErr": np.ones(n_sources)}) 

929 run_multi_plugin(diaObjects, diaSources, "r", plug) 

930 self.assertAlmostEqual(diaObjects.at[objId, "rTOTFluxSigma"], 

931 np.nanstd(fluxes, ddof=1)) 

932 

933 

934class MemoryTester(lsst.utils.tests.MemoryTestCase): 

935 pass 

936 

937 

938def setup_module(module): 

939 lsst.utils.tests.init() 

940 

941 

942if __name__ == "__main__": 942 ↛ 943line 942 didn't jump to line 943, because the condition on line 942 was never true

943 lsst.utils.tests.init() 

944 unittest.main()