Coverage for tests/test_diaCalculationPlugins.py: 14%

343 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-06 03:06 -0700

1# This file is part of ap_association. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22from astropy.stats import median_absolute_deviation 

23import numpy as np 

24import pandas as pd 

25from scipy.stats import skew 

26import unittest 

27 

28from lsst.meas.base import ( 

29 MeanDiaPosition, MeanDiaPositionConfig, 

30 HTMIndexDiaPosition, HTMIndexDiaPositionConfig, 

31 NumDiaSourcesDiaPlugin, NumDiaSourcesDiaPluginConfig, 

32 SimpleSourceFlagDiaPlugin, SimpleSourceFlagDiaPluginConfig, 

33 WeightedMeanDiaPsfFlux, WeightedMeanDiaPsfFluxConfig, 

34 PercentileDiaPsfFlux, PercentileDiaPsfFluxConfig, 

35 SigmaDiaPsfFlux, SigmaDiaPsfFluxConfig, 

36 Chi2DiaPsfFlux, Chi2DiaPsfFluxConfig, 

37 MadDiaPsfFlux, MadDiaPsfFluxConfig, 

38 SkewDiaPsfFlux, SkewDiaPsfFluxConfig, 

39 MinMaxDiaPsfFlux, MinMaxDiaPsfFluxConfig, 

40 MaxSlopeDiaPsfFlux, MaxSlopeDiaPsfFluxConfig, 

41 ErrMeanDiaPsfFlux, ErrMeanDiaPsfFluxConfig, 

42 LinearFitDiaPsfFlux, LinearFitDiaPsfFluxConfig, 

43 StetsonJDiaPsfFlux, StetsonJDiaPsfFluxConfig, 

44 WeightedMeanDiaTotFlux, WeightedMeanDiaTotFluxConfig, 

45 SigmaDiaTotFlux, SigmaDiaTotFluxConfig) 

46import lsst.utils.tests 

47 

48 

49def run_single_plugin(diaObjectCat, 

50 diaObjectId, 

51 diaSourceCat, 

52 band, 

53 plugin): 

54 """Wrapper for running single plugins. 

55 

56 Reproduces some of the behavior of `lsst.ap.association.DiaCalcuation.run` 

57 

58 Parameters 

59 ---------- 

60 diaObjectCat : `pandas.DataFrame` 

61 Input object catalog to store data into and read from. 

62 diaSourcesCat : `pandas.DataFrame` 

63 DiaSource catalog to read data from and groupby on. 

64 fitlerName : `str` 

65 String name of the filter to process. 

66 plugin : `lsst.ap.association.DiaCalculationPlugin` 

67 Plugin to run. 

68 """ 

69 diaObjectCat.set_index("diaObjectId", inplace=True, drop=False) 

70 diaSourceCat.set_index( 

71 ["diaObjectId", "band", "diaSourceId"], 

72 inplace=True, 

73 drop=False) 

74 

75 objDiaSources = diaSourceCat.loc[diaObjectId] 

76 updatingFilterDiaSources = diaSourceCat.loc[ 

77 (diaObjectId, band), : 

78 ] 

79 

80 plugin.calculate(diaObjects=diaObjectCat, 

81 diaObjectId=diaObjectId, 

82 diaSources=objDiaSources, 

83 filterDiaSources=updatingFilterDiaSources, 

84 band=band) 

85 

86 

87def run_multi_plugin(diaObjectCat, diaSourceCat, band, plugin): 

88 """Wrapper for running multi plugins. 

89 

90 Reproduces some of the behavior of `lsst.ap.association.DiaCalcuation.run` 

91 

92 Parameters 

93 ---------- 

94 diaObjectCat : `pandas.DataFrame` 

95 Input object catalog to store data into and read from. 

96 diaSourcesCat : `pandas.DataFrame` 

97 DiaSource catalog to read data from and groupby on. 

98 fitlerName : `str` 

99 String name of the filter to process. 

100 plugin : `lsst.ap.association.DiaCalculationPlugin` 

101 Plugin to run. 

102 """ 

103 diaObjectCat.set_index("diaObjectId", inplace=True, drop=False) 

104 diaSourceCat.set_index( 

105 ["diaObjectId", "band", "diaSourceId"], 

106 inplace=True, 

107 drop=False) 

108 

109 updatingFilterDiaSources = diaSourceCat.loc[ 

110 (slice(None), band), : 

111 ] 

112 

113 diaSourcesGB = diaSourceCat.groupby(level=0) 

114 filterDiaSourcesGB = updatingFilterDiaSources.groupby(level=0) 

115 

116 plugin.calculate(diaObjects=diaObjectCat, 

117 diaSources=diaSourcesGB, 

118 filterDiaSources=filterDiaSourcesGB, 

119 band=band) 

120 

121 

122class TestMeanPosition(unittest.TestCase): 

123 

124 def testCalculate(self): 

125 """Test mean position calculation. 

126 """ 

127 n_sources = 10 

128 objId = 0 

129 

130 plug = MeanDiaPosition(MeanDiaPositionConfig(), 

131 "ap_meanPosition", 

132 None) 

133 

134 # Test expected means in RA. 

135 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

136 diaSources = pd.DataFrame(data={"ra": np.linspace(-1, 1, n_sources), 

137 "dec": np.zeros(n_sources), 

138 "midpointMjdTai": np.linspace(0, n_sources, n_sources), 

139 "diaObjectId": n_sources * [objId], 

140 "band": n_sources * ["g"], 

141 "diaSourceId": np.arange(n_sources, 

142 dtype=int)}) 

143 run_multi_plugin(diaObjects, diaSources, "g", plug) 

144 

145 self.assertAlmostEqual(diaObjects.loc[objId, "ra"], 0.0) 

146 self.assertAlmostEqual(diaObjects.loc[objId, "dec"], 0.0) 

147 self.assertEqual(diaObjects.loc[objId, "radecMjdTai"], 10) 

148 

149 # Test expected means in DEC. 

150 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

151 diaSources = pd.DataFrame(data={"ra": np.zeros(n_sources), 

152 "dec": np.linspace(-1, 1, n_sources), 

153 "midpointMjdTai": np.linspace(0, n_sources, n_sources), 

154 "diaObjectId": n_sources * [objId], 

155 "band": n_sources * ["g"], 

156 "diaSourceId": np.arange(n_sources, 

157 dtype=int)}) 

158 run_multi_plugin(diaObjects, diaSources, "g", plug) 

159 

160 self.assertAlmostEqual(diaObjects.loc[objId, "ra"], 0.0) 

161 self.assertAlmostEqual(diaObjects.loc[objId, "dec"], 0.0) 

162 self.assertEqual(diaObjects.loc[objId, "radecMjdTai"], 10) 

163 

164 # Test failure mode RA is nan. 

165 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

166 diaSources = pd.DataFrame(data={"ra": np.full(n_sources, np.nan), 

167 "dec": np.zeros(n_sources), 

168 "midpointMjdTai": np.linspace(0, n_sources, n_sources), 

169 "diaObjectId": n_sources * [objId], 

170 "band": n_sources * ["g"], 

171 "diaSourceId": np.arange(n_sources, 

172 dtype=int)}) 

173 run_multi_plugin(diaObjects, diaSources, "g", plug) 

174 

175 self.assertTrue(np.isnan(diaObjects.loc[objId, "ra"])) 

176 self.assertTrue(np.isnan(diaObjects.loc[objId, "dec"])) 

177 self.assertTrue(np.isnan(diaObjects.loc[objId, "radecMjdTai"])) 

178 

179 # Test failure mode DEC is nan. 

180 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

181 diaSources = pd.DataFrame(data={"ra": np.zeros(n_sources), 

182 "dec": np.full(n_sources, np.nan), 

183 "midpointMjdTai": np.linspace(0, n_sources, n_sources), 

184 "diaObjectId": n_sources * [objId], 

185 "band": n_sources * ["g"], 

186 "diaSourceId": np.arange(n_sources, 

187 dtype=int)}) 

188 run_multi_plugin(diaObjects, diaSources, "g", plug) 

189 

190 self.assertTrue(np.isnan(diaObjects.loc[objId, "ra"])) 

191 self.assertTrue(np.isnan(diaObjects.loc[objId, "dec"])) 

192 self.assertTrue(np.isnan(diaObjects.loc[objId, "radecMjdTai"])) 

193 

194 

195class TestHTMIndexPosition(unittest.TestCase): 

196 

197 def testCalculate(self): 

198 """Test HTMPixel assignment calculation. 

199 """ 

200 # Test expected pixelId at RA, DEC = 0 

201 objId = 0 

202 n_sources = 10 

203 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

204 diaObjects.loc[objId, "ra"] = 0. 

205 diaObjects.loc[objId, "dec"] = 0. 

206 diaSources = pd.DataFrame( 

207 data={"diaObjectId": n_sources * [objId], 

208 "band": n_sources * ["g"], 

209 "diaSourceId": np.arange(n_sources, dtype=int)}) 

210 plug = HTMIndexDiaPosition(HTMIndexDiaPositionConfig(), 

211 "ap_HTMIndex", 

212 None) 

213 

214 run_single_plugin(diaObjectCat=diaObjects, 

215 diaObjectId=objId, 

216 diaSourceCat=diaSources, 

217 band="g", 

218 plugin=plug) 

219 self.assertEqual(diaObjects.at[objId, "pixelId"], 

220 17042430230528) 

221 

222 # Test expected pixelId at some value of RA and DEC. 

223 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

224 diaObjects.loc[objId, "ra"] = 45.37 

225 diaObjects.loc[objId, "dec"] = 13.67 

226 diaSources = pd.DataFrame( 

227 data={"diaObjectId": n_sources * [objId], 

228 "band": n_sources * ["g"], 

229 "diaSourceId": np.arange(n_sources, dtype=int)}) 

230 run_single_plugin(diaObjectCat=diaObjects, 

231 diaObjectId=objId, 

232 diaSourceCat=diaSources, 

233 band="g", 

234 plugin=plug) 

235 self.assertEqual(diaObjects.at[objId, "pixelId"], 

236 17450571968473) 

237 

238 

239class TestNDiaSourcesDiaPlugin(unittest.TestCase): 

240 

241 def testCalculate(self): 

242 """Test that the number of DiaSources is correct. 

243 """ 

244 

245 for n_sources in [1, 8, 10]: 

246 # Test expected number of sources per object. 

247 objId = 0 

248 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

249 diaSources = pd.DataFrame( 

250 data={"diaObjectId": n_sources * [objId], 

251 "band": n_sources * ["g"], 

252 "diaSourceId": np.arange(n_sources, dtype=int)}) 

253 plug = NumDiaSourcesDiaPlugin(NumDiaSourcesDiaPluginConfig(), 

254 "ap_nDiaSources", 

255 None) 

256 run_multi_plugin(diaObjects, diaSources, "g", plug) 

257 

258 self.assertEqual(n_sources, diaObjects.at[objId, "nDiaSources"]) 

259 

260 

261class TestSimpleSourceFlagDiaPlugin(unittest.TestCase): 

262 

263 def testCalculate(self): 

264 """Test that DiaObject flags are set. 

265 """ 

266 objId = 0 

267 n_sources = 10 

268 

269 # Test expected flags, no flags set. 

270 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

271 diaSources = pd.DataFrame( 

272 data={"diaObjectId": n_sources * [objId], 

273 "band": n_sources * ["g"], 

274 "diaSourceId": np.arange(n_sources, dtype=int), 

275 "flags": np.zeros(n_sources, dtype=np.uint64)}) 

276 plug = SimpleSourceFlagDiaPlugin(SimpleSourceFlagDiaPluginConfig(), 

277 "ap_diaObjectFlag", 

278 None) 

279 run_multi_plugin(diaObjects, diaSources, "g", plug) 

280 self.assertEqual(diaObjects.at[objId, "flags"], 0) 

281 

282 # Test expected flags, all flags set. 

283 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

284 diaSources = pd.DataFrame( 

285 data={"diaObjectId": n_sources * [objId], 

286 "band": n_sources * ["g"], 

287 "diaSourceId": np.arange(n_sources, dtype=int), 

288 "flags": np.ones(n_sources, dtype=np.uint64)}) 

289 run_multi_plugin(diaObjects, diaSources, "g", plug) 

290 self.assertEqual(diaObjects.at[objId, "flags"], 1) 

291 

292 # Test expected flags, random flags. 

293 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

294 diaSources = pd.DataFrame( 

295 data={"diaObjectId": n_sources * [objId], 

296 "band": n_sources * ["g"], 

297 "diaSourceId": np.arange(n_sources, dtype=int), 

298 "flags": np.random.randint(0, 2 ** 16, size=n_sources)}) 

299 run_multi_plugin(diaObjects, diaSources, "g", plug) 

300 self.assertEqual(diaObjects.at[objId, "flags"], 1) 

301 

302 # Test expected flags, one flag set. 

303 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

304 flag_array = np.zeros(n_sources, dtype=np.uint64) 

305 flag_array[4] = 256 

306 diaSources = pd.DataFrame( 

307 data={"diaObjectId": n_sources * [objId], 

308 "band": n_sources * ["g"], 

309 "diaSourceId": np.arange(n_sources, dtype=int), 

310 "flags": flag_array}) 

311 run_multi_plugin(diaObjects, diaSources, "g", plug) 

312 self.assertEqual(diaObjects.at[objId, "flags"], 1) 

313 

314 

315class TestWeightedMeanDiaPsfFlux(unittest.TestCase): 

316 

317 def testCalculate(self): 

318 """Test mean value calculation. 

319 """ 

320 n_sources = 10 

321 objId = 0 

322 

323 # Test expected mean. 

324 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

325 diaSources = pd.DataFrame( 

326 data={"diaObjectId": n_sources * [objId], 

327 "band": n_sources * ["u"], 

328 "diaSourceId": np.arange(n_sources, dtype=int), 

329 "psfFlux": np.linspace(-1, 1, n_sources), 

330 "psfFluxErr": np.ones(n_sources)}) 

331 

332 plug = WeightedMeanDiaPsfFlux(WeightedMeanDiaPsfFluxConfig(), 

333 "ap_meanFlux", 

334 None) 

335 run_multi_plugin(diaObjects, diaSources, "u", plug) 

336 

337 self.assertAlmostEqual(diaObjects.loc[objId, "u_psfFluxMean"], 0.0) 

338 self.assertAlmostEqual(diaObjects.loc[objId, "u_psfFluxMeanErr"], 

339 np.sqrt(1 / n_sources)) 

340 self.assertEqual(diaObjects.loc[objId, "u_psfFluxNdata"], n_sources) 

341 

342 # Test expected mean with a nan value. 

343 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

344 fluxes = np.linspace(-1, 1, n_sources) 

345 fluxes[4] = np.nan 

346 diaSources = pd.DataFrame( 

347 data={"diaObjectId": n_sources * [objId], 

348 "band": n_sources * ["r"], 

349 "diaSourceId": np.arange(n_sources, dtype=int), 

350 "psfFlux": fluxes, 

351 "psfFluxErr": np.ones(n_sources)}) 

352 run_multi_plugin(diaObjects, diaSources, "r", plug) 

353 

354 self.assertAlmostEqual(diaObjects.at[objId, "r_psfFluxMean"], 

355 np.nanmean(fluxes)) 

356 self.assertAlmostEqual(diaObjects.at[objId, "r_psfFluxMeanErr"], 

357 np.sqrt(1 / (n_sources - 1))) 

358 self.assertEqual(diaObjects.loc[objId, "r_psfFluxNdata"], n_sources - 1) 

359 

360 

361class TestPercentileDiaPsfFlux(unittest.TestCase): 

362 

363 def testCalculate(self): 

364 """Test flux percentile calculation. 

365 """ 

366 n_sources = 10 

367 objId = 0 

368 

369 # Test expected percentile values. 

370 fluxes = np.linspace(-1, 1, n_sources) 

371 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

372 diaSources = pd.DataFrame( 

373 data={"diaObjectId": n_sources * [objId], 

374 "band": n_sources * ["u"], 

375 "diaSourceId": np.arange(n_sources, dtype=int), 

376 "psfFlux": fluxes, 

377 "psfFluxErr": np.ones(n_sources)}) 

378 

379 plug = PercentileDiaPsfFlux(PercentileDiaPsfFluxConfig(), 

380 "ap_percentileFlux", 

381 None) 

382 run_multi_plugin(diaObjects, diaSources, "u", plug) 

383 for pTile, testVal in zip(plug.config.percentiles, 

384 np.nanpercentile( 

385 fluxes, 

386 plug.config.percentiles)): 

387 self.assertAlmostEqual( 

388 diaObjects.at[objId, "u_psfFluxPercentile{:02d}".format(pTile)], 

389 testVal) 

390 

391 # Test expected percentile values with a nan value. 

392 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

393 fluxes[4] = np.nan 

394 diaSources = pd.DataFrame( 

395 data={"diaObjectId": n_sources * [objId], 

396 "band": n_sources * ["r"], 

397 "diaSourceId": np.arange(n_sources, dtype=int), 

398 "psfFlux": fluxes, 

399 "psfFluxErr": np.ones(n_sources)}) 

400 run_multi_plugin(diaObjects, diaSources, "r", plug) 

401 for pTile, testVal in zip(plug.config.percentiles, 

402 np.nanpercentile( 

403 fluxes, 

404 plug.config.percentiles)): 

405 self.assertAlmostEqual( 

406 diaObjects.at[objId, "r_psfFluxPercentile{:02d}".format(pTile)], 

407 testVal) 

408 

409 

410class TestSigmaDiaPsfFlux(unittest.TestCase): 

411 

412 def testCalculate(self): 

413 """Test flux scatter calculation. 

414 """ 

415 n_sources = 10 

416 objId = 0 

417 

418 # Test expected sigma scatter of fluxes. 

419 fluxes = np.linspace(-1, 1, n_sources) 

420 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

421 diaSources = pd.DataFrame( 

422 data={"diaObjectId": n_sources * [objId], 

423 "band": n_sources * ["u"], 

424 "diaSourceId": np.arange(n_sources, dtype=int), 

425 "psfFlux": fluxes, 

426 "psfFluxErr": np.ones(n_sources)}) 

427 

428 plug = SigmaDiaPsfFlux(SigmaDiaPsfFluxConfig(), 

429 "ap_sigmaFlux", 

430 None) 

431 run_multi_plugin(diaObjects, diaSources, "u", plug) 

432 self.assertAlmostEqual(diaObjects.at[objId, "u_psfFluxSigma"], 

433 np.nanstd(fluxes, ddof=1)) 

434 

435 # test one input, returns nan. 

436 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

437 diaSources = pd.DataFrame( 

438 data={"diaObjectId": 1 * [objId], 

439 "band": 1 * ["g"], 

440 "diaSourceId": [0], 

441 "psfFlux": [fluxes[0]], 

442 "psfFluxErr": [1.]}) 

443 run_multi_plugin(diaObjects, diaSources, "g", plug) 

444 self.assertTrue(np.isnan(diaObjects.at[objId, "g_psfFluxSigma"])) 

445 

446 # Test expected sigma scatter of fluxes with a nan value. 

447 fluxes[4] = np.nan 

448 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

449 diaSources = pd.DataFrame( 

450 data={"diaObjectId": n_sources * [objId], 

451 "band": n_sources * ["r"], 

452 "diaSourceId": np.arange(n_sources, dtype=int), 

453 "psfFlux": fluxes, 

454 "psfFluxErr": np.ones(n_sources)}) 

455 run_multi_plugin(diaObjects, diaSources, "r", plug) 

456 self.assertAlmostEqual(diaObjects.at[objId, "r_psfFluxSigma"], 

457 np.nanstd(fluxes, ddof=1)) 

458 

459 

460class TestChi2DiaPsfFlux(unittest.TestCase): 

461 

462 def testCalculate(self): 

463 """Test flux chi2 calculation. 

464 """ 

465 n_sources = 10 

466 objId = 0 

467 

468 # Test expected chi^2 value. 

469 fluxes = np.linspace(-1, 1, n_sources) 

470 diaObjects = pd.DataFrame({"diaObjectId": [objId], 

471 "u_psfFluxMean": [0.0]}) 

472 diaSources = pd.DataFrame( 

473 data={"diaObjectId": n_sources * [objId], 

474 "band": n_sources * ["u"], 

475 "diaSourceId": np.arange(n_sources, dtype=int), 

476 "psfFlux": fluxes, 

477 "psfFluxErr": np.ones(n_sources)}) 

478 

479 plug = Chi2DiaPsfFlux(Chi2DiaPsfFluxConfig(), 

480 "ap_chi2Flux", 

481 None) 

482 run_multi_plugin(diaObjects, diaSources, "u", plug) 

483 self.assertAlmostEqual( 

484 diaObjects.loc[objId, "u_psfFluxChi2"], 

485 np.nansum(((diaSources["psfFlux"] 

486 - np.nanmean(diaSources["psfFlux"])) 

487 / diaSources["psfFluxErr"]) ** 2)) 

488 

489 # Test expected chi^2 value with a nan value set. 

490 fluxes[4] = np.nan 

491 diaObjects = pd.DataFrame({"diaObjectId": [objId], 

492 "r_psfFluxMean": [np.nanmean(fluxes)]}) 

493 diaSources = pd.DataFrame( 

494 data={"diaObjectId": n_sources * [objId], 

495 "band": n_sources * ["r"], 

496 "diaSourceId": np.arange(n_sources, dtype=int), 

497 "psfFlux": fluxes, 

498 "psfFluxErr": np.ones(n_sources)}) 

499 run_multi_plugin(diaObjects, diaSources, "r", plug) 

500 self.assertAlmostEqual( 

501 diaObjects.loc[objId, "r_psfFluxChi2"], 

502 np.nansum(((diaSources["psfFlux"] 

503 - np.nanmean(diaSources["psfFlux"])) 

504 / diaSources["psfFluxErr"]) ** 2)) 

505 

506 

507class TestMadDiaPsfFlux(unittest.TestCase): 

508 

509 def testCalculate(self): 

510 """Test flux median absolute deviation calculation. 

511 """ 

512 n_sources = 10 

513 objId = 0 

514 

515 # Test expected MAD value. 

516 fluxes = np.linspace(-1, 1, n_sources) 

517 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

518 diaSources = pd.DataFrame( 

519 data={"diaObjectId": n_sources * [objId], 

520 "band": n_sources * ["u"], 

521 "diaSourceId": np.arange(n_sources, dtype=int), 

522 "psfFlux": fluxes, 

523 "psfFluxErr": np.ones(n_sources)}) 

524 

525 plug = MadDiaPsfFlux(MadDiaPsfFluxConfig(), 

526 "ap_madFlux", 

527 None) 

528 run_multi_plugin(diaObjects, diaSources, "u", plug) 

529 self.assertAlmostEqual(diaObjects.at[objId, "u_psfFluxMAD"], 

530 median_absolute_deviation(fluxes, 

531 ignore_nan=True)) 

532 

533 # Test expected MAD value with a nan set. 

534 fluxes[4] = np.nan 

535 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

536 diaSources = pd.DataFrame( 

537 data={"diaObjectId": n_sources * [objId], 

538 "band": n_sources * ["r"], 

539 "diaSourceId": np.arange(n_sources, dtype=int), 

540 "psfFlux": fluxes, 

541 "psfFluxErr": np.ones(n_sources)}) 

542 run_multi_plugin(diaObjects, diaSources, "r", plug) 

543 self.assertAlmostEqual(diaObjects.at[objId, "r_psfFluxMAD"], 

544 median_absolute_deviation(fluxes, 

545 ignore_nan=True)) 

546 

547 

548class TestSkewDiaPsfFlux(unittest.TestCase): 

549 

550 def testCalculate(self): 

551 """Test flux skew calculation. 

552 """ 

553 n_sources = 10 

554 objId = 0 

555 

556 # Test expected skew value. 

557 fluxes = np.linspace(-1, 1, n_sources) 

558 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

559 diaSources = pd.DataFrame( 

560 data={"diaObjectId": n_sources * [objId], 

561 "band": n_sources * ["u"], 

562 "diaSourceId": np.arange(n_sources, dtype=int), 

563 "psfFlux": fluxes, 

564 "psfFluxErr": np.ones(n_sources)}) 

565 

566 plug = SkewDiaPsfFlux(SkewDiaPsfFluxConfig(), 

567 "ap_skewFlux", 

568 None) 

569 run_multi_plugin(diaObjects, diaSources, "u", plug) 

570 self.assertAlmostEqual( 

571 diaObjects.loc[objId, "u_psfFluxSkew"], 

572 skew_wrapper(fluxes)) 

573 

574 # Test expected skew value with a nan set. 

575 fluxes[4] = np.nan 

576 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

577 diaSources = pd.DataFrame( 

578 data={"diaObjectId": n_sources * [objId], 

579 "band": n_sources * ["r"], 

580 "diaSourceId": np.arange(n_sources, dtype=int), 

581 "psfFlux": fluxes, 

582 "psfFluxErr": np.ones(n_sources)}) 

583 run_multi_plugin(diaObjects, diaSources, "r", plug) 

584 

585 self.assertAlmostEqual( 

586 diaObjects.at[objId, "r_psfFluxSkew"], 

587 skew_wrapper(fluxes)) 

588 

589 

590class TestMinMaxDiaPsfFlux(unittest.TestCase): 

591 

592 def testCalculate(self): 

593 """Test flux min/max calculation. 

594 """ 

595 n_sources = 10 

596 objId = 0 

597 

598 # Test expected MinMax fluxes. 

599 fluxes = np.linspace(-1, 1, n_sources) 

600 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

601 diaSources = pd.DataFrame( 

602 data={"diaObjectId": n_sources * [objId], 

603 "band": n_sources * ["u"], 

604 "diaSourceId": np.arange(n_sources, dtype=int), 

605 "psfFlux": fluxes, 

606 "psfFluxErr": np.ones(n_sources)}) 

607 

608 plug = MinMaxDiaPsfFlux(MinMaxDiaPsfFluxConfig(), 

609 "ap_minMaxFlux", 

610 None) 

611 run_multi_plugin(diaObjects, diaSources, "u", plug) 

612 self.assertEqual(diaObjects.loc[objId, "u_psfFluxMin"], -1) 

613 self.assertEqual(diaObjects.loc[objId, "u_psfFluxMax"], 1) 

614 

615 # Test expected MinMax fluxes with a nan set. 

616 fluxes[4] = np.nan 

617 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

618 diaSources = pd.DataFrame( 

619 data={"diaObjectId": n_sources * [objId], 

620 "band": n_sources * ["r"], 

621 "diaSourceId": np.arange(n_sources, dtype=int), 

622 "psfFlux": fluxes, 

623 "psfFluxErr": np.ones(n_sources)}) 

624 run_multi_plugin(diaObjects, diaSources, "r", plug) 

625 self.assertEqual(diaObjects.loc[objId, "r_psfFluxMin"], -1) 

626 self.assertEqual(diaObjects.loc[objId, "r_psfFluxMax"], 1) 

627 

628 

629class TestMaxSlopeDiaPsfFlux(unittest.TestCase): 

630 

631 def testCalculate(self): 

632 """Test flux maximum slope. 

633 """ 

634 n_sources = 10 

635 objId = 0 

636 

637 # Test max slope value. 

638 fluxes = np.linspace(-1, 1, n_sources) 

639 times = np.concatenate([np.linspace(0, 1, n_sources)[:-1], [1 - 1/90]]) 

640 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

641 diaSources = pd.DataFrame( 

642 data={"diaObjectId": n_sources * [objId], 

643 "band": n_sources * ["u"], 

644 "diaSourceId": np.arange(n_sources, dtype=int), 

645 "psfFlux": fluxes, 

646 "psfFluxErr": np.ones(n_sources), 

647 "midpointMjdTai": times}) 

648 

649 plug = MaxSlopeDiaPsfFlux(MaxSlopeDiaPsfFluxConfig(), 

650 "ap_maxSlopeFlux", 

651 None) 

652 run_multi_plugin(diaObjects, diaSources, "u", plug) 

653 self.assertAlmostEqual(diaObjects.at[objId, "u_psfFluxMaxSlope"], 2 + 2/9) 

654 

655 # Test max slope value returns nan on 1 input. 

656 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

657 diaSources = pd.DataFrame( 

658 data={"diaObjectId": 1 * [objId], 

659 "band": 1 * ["g"], 

660 "diaSourceId": np.arange(1, dtype=int), 

661 "psfFlux": fluxes[0], 

662 "psfFluxErr": np.ones(1), 

663 "midpointMjdTai": times[0]}) 

664 run_multi_plugin(diaObjects, diaSources, "g", plug) 

665 self.assertTrue(np.isnan(diaObjects.at[objId, "g_psfFluxMaxSlope"])) 

666 

667 # Test max slope value inputing nan values. 

668 fluxes[4] = np.nan 

669 times[7] = np.nan 

670 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

671 diaSources = pd.DataFrame( 

672 data={"diaObjectId": n_sources * [objId], 

673 "band": n_sources * ["r"], 

674 "diaSourceId": np.arange(n_sources, dtype=int), 

675 "psfFlux": fluxes, 

676 "psfFluxErr": np.ones(n_sources), 

677 "midpointMjdTai": times}) 

678 run_multi_plugin(diaObjects, diaSources, "r", plug) 

679 self.assertAlmostEqual(diaObjects.at[objId, "r_psfFluxMaxSlope"], 2 + 2 / 9) 

680 

681 

682class TestErrMeanDiaPsfFlux(unittest.TestCase): 

683 

684 def testCalculate(self): 

685 """Test error mean calculation. 

686 """ 

687 n_sources = 10 

688 objId = 0 

689 

690 # Test mean of the errors. 

691 fluxes = np.linspace(-1, 1, n_sources) 

692 errors = np.linspace(1, 2, n_sources) 

693 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

694 diaSources = pd.DataFrame( 

695 data={"diaObjectId": n_sources * [objId], 

696 "band": n_sources * ["u"], 

697 "diaSourceId": np.arange(n_sources, dtype=int), 

698 "psfFlux": fluxes, 

699 "psfFluxErr": errors}) 

700 

701 plug = ErrMeanDiaPsfFlux(ErrMeanDiaPsfFluxConfig(), 

702 "ap_errMeanFlux", 

703 None) 

704 run_multi_plugin(diaObjects, diaSources, "u", plug) 

705 self.assertAlmostEqual(diaObjects.at[objId, "u_psfFluxErrMean"], 

706 np.nanmean(errors)) 

707 

708 # Test mean of the errors with input nan value. 

709 errors[4] = np.nan 

710 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

711 diaSources = pd.DataFrame( 

712 data={"diaObjectId": n_sources * [objId], 

713 "band": n_sources * ["r"], 

714 "diaSourceId": np.arange(n_sources, dtype=int), 

715 "psfFlux": fluxes, 

716 "psfFluxErr": errors}) 

717 run_multi_plugin(diaObjects, diaSources, "r", plug) 

718 self.assertAlmostEqual(diaObjects.at[objId, "r_psfFluxErrMean"], 

719 np.nanmean(errors)) 

720 

721 

722class TestLinearFitDiaPsfFlux(unittest.TestCase): 

723 

724 def testCalculate(self): 

725 """Test a linear fit to flux vs time. 

726 """ 

727 n_sources = 10 

728 objId = 0 

729 

730 # Test best fit linear model. 

731 fluxes = np.linspace(-1, 1, n_sources) 

732 errors = np.linspace(1, 2, n_sources) 

733 times = np.linspace(0, 1, n_sources) 

734 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

735 diaSources = pd.DataFrame( 

736 data={"diaObjectId": n_sources * [objId], 

737 "band": n_sources * ["u"], 

738 "diaSourceId": np.arange(n_sources, dtype=int), 

739 "psfFlux": fluxes, 

740 "psfFluxErr": errors, 

741 "midpointMjdTai": times}) 

742 

743 plug = LinearFitDiaPsfFlux(LinearFitDiaPsfFluxConfig(), 

744 "ap_LinearFit", 

745 None) 

746 run_multi_plugin(diaObjects, diaSources, "u", plug) 

747 self.assertAlmostEqual(diaObjects.loc[objId, "u_psfFluxLinearSlope"], 

748 2.) 

749 self.assertAlmostEqual(diaObjects.loc[objId, "u_psfFluxLinearIntercept"], 

750 -1.) 

751 

752 # Test best fit linear model with input nans. 

753 fluxes[7] = np.nan 

754 errors[4] = np.nan 

755 times[2] = np.nan 

756 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

757 diaSources = pd.DataFrame( 

758 data={"diaObjectId": n_sources * [objId], 

759 "band": n_sources * ["r"], 

760 "diaSourceId": np.arange(n_sources, dtype=int), 

761 "psfFlux": fluxes, 

762 "psfFluxErr": errors, 

763 "midpointMjdTai": times}) 

764 run_multi_plugin(diaObjects, diaSources, "r", plug) 

765 self.assertAlmostEqual(diaObjects.loc[objId, "r_psfFluxLinearSlope"], 2.) 

766 self.assertAlmostEqual(diaObjects.loc[objId, "r_psfFluxLinearIntercept"], 

767 -1.) 

768 

769 

770class TestStetsonJDiaPsfFlux(unittest.TestCase): 

771 

772 def testCalculate(self): 

773 """Test the stetsonJ statistic. 

774 """ 

775 n_sources = 10 

776 objId = 0 

777 

778 # Test stetsonJ calculation. 

779 fluxes = np.linspace(-1, 1, n_sources) 

780 errors = np.ones(n_sources) 

781 diaObjects = pd.DataFrame({"diaObjectId": [objId], 

782 "u_psfFluxMean": [np.nanmean(fluxes)]}) 

783 diaSources = pd.DataFrame( 

784 data={"diaObjectId": n_sources * [objId], 

785 "band": n_sources * ["u"], 

786 "diaSourceId": np.arange(n_sources, dtype=int), 

787 "psfFlux": fluxes, 

788 "psfFluxErr": errors}) 

789 

790 plug = StetsonJDiaPsfFlux(StetsonJDiaPsfFluxConfig(), 

791 "ap_StetsonJ", 

792 None) 

793 run_multi_plugin(diaObjects, diaSources, "u", plug) 

794 # Expected StetsonJ for the values created. Confirmed using Cesimum's 

795 # implementation. http://github.com/cesium-ml/cesium 

796 self.assertAlmostEqual(diaObjects.loc[objId, "u_psfFluxStetsonJ"], 

797 -0.5958393936080928) 

798 

799 # Test stetsonJ calculation returns nan on single input. 

800 diaObjects = pd.DataFrame({"diaObjectId": [objId], 

801 "g_psfFluxMean": [np.nanmean(fluxes)]}) 

802 diaSources = pd.DataFrame( 

803 data={"diaObjectId": 1 * [objId], 

804 "band": 1 * ["g"], 

805 "diaSourceId": np.arange(1, dtype=int), 

806 "psfFlux": fluxes[0], 

807 "psfFluxErr": errors[0]}) 

808 run_multi_plugin(diaObjects, diaSources, "g", plug) 

809 self.assertTrue(np.isnan(diaObjects.at[objId, "g_psfFluxStetsonJ"])) 

810 

811 # Test stetsonJ calculation returns when nans are input. 

812 fluxes[7] = np.nan 

813 errors[4] = np.nan 

814 nonNanMask = np.logical_and(~np.isnan(fluxes), 

815 ~np.isnan(errors)) 

816 diaObjects = pd.DataFrame( 

817 {"diaObjectId": [objId], 

818 "r_psfFluxMean": [np.average(fluxes[nonNanMask], 

819 weights=errors[nonNanMask])]}) 

820 diaSources = pd.DataFrame( 

821 data={"diaObjectId": n_sources * [objId], 

822 "band": n_sources * ["r"], 

823 "diaSourceId": np.arange(n_sources, dtype=int), 

824 "psfFlux": fluxes, 

825 "psfFluxErr": errors}) 

826 run_multi_plugin(diaObjects, diaSources, "r", plug) 

827 self.assertAlmostEqual(diaObjects.at[objId, "r_psfFluxStetsonJ"], 

828 -0.5412797916187173) 

829 

830 

831class TestWeightedMeanDiaTotFlux(unittest.TestCase): 

832 

833 def testCalculate(self): 

834 """Test mean value calculation. 

835 """ 

836 n_sources = 10 

837 objId = 0 

838 

839 # Test test mean on scienceFlux. 

840 fluxes = np.linspace(-1, 1, n_sources) 

841 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

842 diaSources = pd.DataFrame( 

843 data={"diaObjectId": n_sources * [objId], 

844 "band": n_sources * ["u"], 

845 "diaSourceId": np.arange(n_sources, dtype=int), 

846 "scienceFlux": fluxes, 

847 "scienceFluxErr": np.ones(n_sources)}) 

848 

849 plug = WeightedMeanDiaTotFlux(WeightedMeanDiaTotFluxConfig(), 

850 "ap_meanTotFlux", 

851 None) 

852 run_multi_plugin(diaObjects, diaSources, "u", plug) 

853 

854 self.assertAlmostEqual(diaObjects.at[objId, "u_scienceFluxMean"], 0.0) 

855 self.assertAlmostEqual(diaObjects.at[objId, "u_scienceFluxMeanErr"], 

856 np.sqrt(1 / n_sources)) 

857 

858 # Test test mean on scienceFlux with input nans 

859 fluxes[4] = np.nan 

860 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

861 diaSources = pd.DataFrame( 

862 data={"diaObjectId": n_sources * [objId], 

863 "band": n_sources * ["r"], 

864 "diaSourceId": np.arange(n_sources, dtype=int), 

865 "scienceFlux": fluxes, 

866 "scienceFluxErr": np.ones(n_sources)}) 

867 run_multi_plugin(diaObjects, diaSources, "r", plug) 

868 

869 self.assertAlmostEqual(diaObjects.at[objId, "r_scienceFluxMean"], 

870 np.nanmean(fluxes)) 

871 self.assertAlmostEqual(diaObjects.at[objId, "r_scienceFluxMeanErr"], 

872 np.sqrt(1 / (n_sources - 1))) 

873 

874 

875class TestSigmaDiaTotFlux(unittest.TestCase): 

876 

877 def testCalculate(self): 

878 """Test flux scatter calculation. 

879 """ 

880 n_sources = 10 

881 objId = 0 

882 

883 # Test test scatter on scienceFlux. 

884 fluxes = np.linspace(-1, 1, n_sources) 

885 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

886 diaSources = pd.DataFrame( 

887 data={"diaObjectId": n_sources * [objId], 

888 "band": n_sources * ["u"], 

889 "diaSourceId": np.arange(n_sources, dtype=int), 

890 "scienceFlux": fluxes, 

891 "scienceFluxErr": np.ones(n_sources)}) 

892 

893 plug = SigmaDiaTotFlux(SigmaDiaTotFluxConfig(), 

894 "ap_sigmaTotFlux", 

895 None) 

896 run_multi_plugin(diaObjects, diaSources, "u", plug) 

897 self.assertAlmostEqual(diaObjects.at[objId, "u_scienceFluxSigma"], 

898 np.nanstd(fluxes, ddof=1)) 

899 

900 # Test test scatter on scienceFlux returns nan on 1 input. 

901 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

902 diaSources = pd.DataFrame( 

903 data={"diaObjectId": 1 * [objId], 

904 "band": 1 * ["g"], 

905 "diaSourceId": np.arange(1, dtype=int), 

906 "scienceFlux": fluxes[0], 

907 "scienceFluxErr": np.ones(1)}) 

908 run_multi_plugin(diaObjects, diaSources, "g", plug) 

909 self.assertTrue(np.isnan(diaObjects.at[objId, "g_scienceFluxSigma"])) 

910 

911 # Test test scatter on scienceFlux takes input nans. 

912 fluxes[4] = np.nan 

913 diaObjects = pd.DataFrame({"diaObjectId": [objId]}) 

914 diaSources = pd.DataFrame( 

915 data={"diaObjectId": n_sources * [objId], 

916 "band": n_sources * ["r"], 

917 "diaSourceId": np.arange(n_sources, dtype=int), 

918 "scienceFlux": fluxes, 

919 "scienceFluxErr": np.ones(n_sources)}) 

920 run_multi_plugin(diaObjects, diaSources, "r", plug) 

921 self.assertAlmostEqual(diaObjects.at[objId, "r_scienceFluxSigma"], 

922 np.nanstd(fluxes, ddof=1)) 

923 

924 

925def skew_wrapper(values): 

926 """Compute scipy skew, omitting nans. 

927 

928 This version works with both scipy<1.9 (where it erroneously returns a 

929 masked array) and scipy>=1.9 (where it correctly returns a float). 

930 

931 Parameters 

932 ---------- 

933 values : `np.ndarray` 

934 

935 Returns 

936 ------- 

937 skew_value : `float` 

938 """ 

939 value = skew(values, bias=False, nan_policy="omit") 

940 if isinstance(value, np.ma.masked_array): 

941 return value.data 

942 else: 

943 return value 

944 

945 

946class MemoryTester(lsst.utils.tests.MemoryTestCase): 

947 pass 

948 

949 

950def setup_module(module): 

951 lsst.utils.tests.init() 

952 

953 

954if __name__ == "__main__": 954 ↛ 955line 954 didn't jump to line 955, because the condition on line 954 was never true

955 lsst.utils.tests.init() 

956 unittest.main()