Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import astropy.units as u 

23import copy 

24import functools 

25import numpy as np 

26import os 

27import pandas as pd 

28import unittest 

29 

30import lsst.utils.tests 

31 

32# TODO: Remove skipUnless and this try block DM-22256 

33try: 

34 from lsst.pipe.tasks.parquetTable import MultilevelParquetTable 

35 from lsst.pipe.tasks.functors import (CompositeFunctor, CustomFunctor, Column, RAColumn, 

36 DecColumn, Mag, MagDiff, Color, StarGalaxyLabeller, 

37 DeconvolvedMoments, SdssTraceSize, PsfSdssTraceSizeDiff, 

38 HsmTraceSize, PsfHsmTraceSizeDiff, HsmFwhm, 

39 LocalPhotometry, LocalNanojansky, LocalNanojanskyErr, 

40 LocalMagnitude, LocalMagnitudeErr) 

41 havePyArrow = True 

42except ImportError: 

43 havePyArrow = False 

44 

45ROOT = os.path.abspath(os.path.dirname(__file__)) 

46 

47 

48@unittest.skipUnless(havePyArrow, "Requires pyarrow") 

49class FunctorTestCase(unittest.TestCase): 

50 

51 def simulateMultiParquet(self, dataDict): 

52 """Create a simple test MultilevelParquetTable 

53 """ 

54 simpleDF = pd.DataFrame(dataDict) 

55 dfFilterDSCombos = [] 

56 for ds in self.datasets: 

57 for filterName in self.filters: 

58 df = copy.copy(simpleDF) 

59 df.reindex(sorted(df.columns), axis=1) 

60 df['dataset'] = ds 

61 df['filter'] = filterName 

62 df.columns = pd.MultiIndex.from_tuples( 

63 [(ds, filterName, c) for c in df.columns], 

64 names=('dataset', 'filter', 'column')) 

65 dfFilterDSCombos.append(df) 

66 

67 df = functools.reduce(lambda d1, d2: d1.join(d2), dfFilterDSCombos) 

68 

69 return MultilevelParquetTable(dataFrame=df) 

70 

71 def setUp(self): 

72 np.random.seed(1234) 

73 self.datasets = ['forced_src', 'meas', 'ref'] 

74 self.filters = ['HSC-G', 'HSC-R'] 

75 self.columns = ['coord_ra', 'coord_dec'] 

76 self.nRecords = 5 

77 self.dataDict = { 

78 "coord_ra": [3.77654137, 3.77643059, 3.77621148, 3.77611944, 3.77610396], 

79 "coord_dec": [0.01127624, 0.01127787, 0.01127543, 0.01127543, 0.01127543]} 

80 

81 def _funcVal(self, functor, parq): 

82 self.assertIsInstance(functor.name, str) 

83 self.assertIsInstance(functor.shortname, str) 

84 

85 val = functor(parq) 

86 self.assertIsInstance(val, pd.Series) 

87 

88 val = functor(parq, dropna=True) 

89 self.assertEqual(val.isnull().sum(), 0) 

90 

91 return val 

92 

93 def testColumn(self): 

94 self.columns.append("base_FootprintArea_value") 

95 self.dataDict["base_FootprintArea_value"] = \ 

96 np.full(self.nRecords, 1) 

97 parq = self.simulateMultiParquet(self.dataDict) 

98 func = Column('base_FootprintArea_value', filt='HSC-G') 

99 self._funcVal(func, parq) 

100 

101 def testCustom(self): 

102 self.columns.append("base_FootprintArea_value") 

103 self.dataDict["base_FootprintArea_value"] = \ 

104 np.random.rand(self.nRecords) 

105 parq = self.simulateMultiParquet(self.dataDict) 

106 func = CustomFunctor('2*base_FootprintArea_value', filt='HSC-G') 

107 val = self._funcVal(func, parq) 

108 

109 func2 = Column('base_FootprintArea_value', filt='HSC-G') 

110 

111 np.allclose(val.values, 2*func2(parq).values, atol=1e-13, rtol=0) 

112 

113 def testCoords(self): 

114 parq = self.simulateMultiParquet(self.dataDict) 

115 ra = self._funcVal(RAColumn(), parq) 

116 dec = self._funcVal(DecColumn(), parq) 

117 

118 columnDict = {'dataset': 'ref', 'filter': 'HSC-G', 

119 'column': ['coord_ra', 'coord_dec']} 

120 coords = parq.toDataFrame(columns=columnDict, droplevels=True) / np.pi * 180. 

121 

122 self.assertTrue(np.allclose(ra, coords[('ref', 'HSC-G', 'coord_ra')], atol=1e-13, rtol=0)) 

123 self.assertTrue(np.allclose(dec, coords[('ref', 'HSC-G', 'coord_dec')], atol=1e-13, rtol=0)) 

124 

125 def testMag(self): 

126 self.columns.extend(["base_PsfFlux_instFlux", "base_PsfFlux_instFluxErr"]) 

127 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1000) 

128 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords, 10) 

129 parq = self.simulateMultiParquet(self.dataDict) 

130 # Change one dataset filter combinations value. 

131 parq._df[("meas", "HSC-G", "base_PsfFlux_instFlux")] -= 1 

132 

133 fluxName = 'base_PsfFlux' 

134 

135 # Check that things work when you provide dataset explicitly 

136 for dataset in ['forced_src', 'meas']: 

137 psfMag_G = self._funcVal(Mag(fluxName, dataset=dataset, 

138 filt='HSC-G'), 

139 parq) 

140 psfMag_R = self._funcVal(Mag(fluxName, dataset=dataset, 

141 filt='HSC-R'), 

142 parq) 

143 

144 psfColor_GR = self._funcVal(Color(fluxName, 'HSC-G', 'HSC-R', 

145 dataset=dataset), 

146 parq) 

147 

148 self.assertTrue(np.allclose((psfMag_G - psfMag_R).dropna(), psfColor_GR, rtol=0, atol=1e-13)) 

149 

150 # Check that behavior as expected when dataset not provided; 

151 # that is, that the color comes from forced and default Mag is meas 

152 psfMag_G = self._funcVal(Mag(fluxName, filt='HSC-G'), parq) 

153 psfMag_R = self._funcVal(Mag(fluxName, filt='HSC-R'), parq) 

154 

155 psfColor_GR = self._funcVal(Color(fluxName, 'HSC-G', 'HSC-R'), parq) 

156 

157 # These should *not* be equal. 

158 self.assertFalse(np.allclose((psfMag_G - psfMag_R).dropna(), psfColor_GR)) 

159 

160 def testMagDiff(self): 

161 self.columns.extend(["base_PsfFlux_instFlux", "base_PsfFlux_instFluxErr", 

162 "modelfit_CModel_instFlux", "modelfit_CModel_instFluxErr"]) 

163 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1000) 

164 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords, 10) 

165 self.dataDict["modelfit_CModel_instFlux"] = np.full(self.nRecords, 1000) 

166 self.dataDict["modelfit_CModel_instFluxErr"] = np.full(self.nRecords, 10) 

167 parq = self.simulateMultiParquet(self.dataDict) 

168 

169 for filt in self.filters: 

170 filt = 'HSC-G' 

171 val = self._funcVal(MagDiff('base_PsfFlux', 'modelfit_CModel', filt=filt), parq) 

172 

173 mag1 = self._funcVal(Mag('modelfit_CModel', filt=filt), parq) 

174 mag2 = self._funcVal(Mag('base_PsfFlux', filt=filt), parq) 

175 self.assertTrue(np.allclose((mag2 - mag1).dropna(), val, rtol=0, atol=1e-13)) 

176 

177 def testLabeller(self): 

178 # Covering the code is better than nothing 

179 self.columns.append("base_ClassificationExtendedness_value") 

180 self.dataDict["base_ClassificationExtendedness_value"] = np.full(self.nRecords, 1) 

181 parq = self.simulateMultiParquet(self.dataDict) 

182 labels = self._funcVal(StarGalaxyLabeller(), parq) # noqa 

183 

184 def testOther(self): 

185 self.columns.extend(["ext_shapeHSM_HsmSourceMoments_xx", "ext_shapeHSM_HsmSourceMoments_yy", 

186 "base_SdssShape_xx", "base_SdssShape_yy", 

187 "ext_shapeHSM_HsmPsfMoments_xx", "ext_shapeHSM_HsmPsfMoments_yy", 

188 "base_SdssShape_psf_xx", "base_SdssShape_psf_yy"]) 

189 self.dataDict["ext_shapeHSM_HsmSourceMoments_xx"] = np.full(self.nRecords, 1 / np.sqrt(2)) 

190 self.dataDict["ext_shapeHSM_HsmSourceMoments_yy"] = np.full(self.nRecords, 1 / np.sqrt(2)) 

191 self.dataDict["base_SdssShape_xx"] = np.full(self.nRecords, 1 / np.sqrt(2)) 

192 self.dataDict["base_SdssShape_yy"] = np.full(self.nRecords, 1 / np.sqrt(2)) 

193 self.dataDict["ext_shapeHSM_HsmPsfMoments_xx"] = np.full(self.nRecords, 1 / np.sqrt(2)) 

194 self.dataDict["ext_shapeHSM_HsmPsfMoments_yy"] = np.full(self.nRecords, 1 / np.sqrt(2)) 

195 self.dataDict["base_SdssShape_psf_xx"] = np.full(self.nRecords, 1) 

196 self.dataDict["base_SdssShape_psf_yy"] = np.full(self.nRecords, 1) 

197 parq = self.simulateMultiParquet(self.dataDict) 

198 # Covering the code is better than nothing 

199 for filt in self.filters: 

200 for Func in [DeconvolvedMoments, 

201 SdssTraceSize, 

202 PsfSdssTraceSizeDiff, 

203 HsmTraceSize, PsfHsmTraceSizeDiff, HsmFwhm]: 

204 val = self._funcVal(Func(filt=filt), parq) # noqa 

205 

206 def _compositeFuncVal(self, functor, parq): 

207 self.assertIsInstance(functor, CompositeFunctor) 

208 

209 df = functor(parq) 

210 

211 self.assertIsInstance(df, pd.DataFrame) 

212 self.assertTrue(np.all([k in df.columns for k in functor.funcDict.keys()])) 

213 

214 df = functor(parq, dropna=True) 

215 

216 # Check that there are no nulls 

217 self.assertFalse(df.isnull().any(axis=None)) 

218 

219 return df 

220 

221 def testComposite(self): 

222 self.columns.extend(["modelfit_CModel_instFlux", "base_PsfFlux_instFlux"]) 

223 self.dataDict["modelfit_CModel_instFlux"] = np.full(self.nRecords, 1) 

224 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1) 

225 parq = self.simulateMultiParquet(self.dataDict) 

226 # Modify r band value slightly. 

227 parq._df[("meas", "HSC-R", "base_PsfFlux_instFlux")] -= 0.1 

228 

229 filt = 'HSC-G' 

230 funcDict = {'psfMag_ref': Mag('base_PsfFlux', dataset='ref'), 

231 'ra': RAColumn(), 

232 'dec': DecColumn(), 

233 'psfMag': Mag('base_PsfFlux', filt=filt), 

234 'cmodel_magDiff': MagDiff('base_PsfFlux', 

235 'modelfit_CModel', filt=filt)} 

236 func = CompositeFunctor(funcDict) 

237 df = self._compositeFuncVal(func, parq) 

238 

239 # Repeat same, but define filter globally instead of individually 

240 funcDict2 = {'psfMag_ref': Mag('base_PsfFlux', dataset='ref'), 

241 'ra': RAColumn(), 

242 'dec': DecColumn(), 

243 'psfMag': Mag('base_PsfFlux'), 

244 'cmodel_magDiff': MagDiff('base_PsfFlux', 

245 'modelfit_CModel')} 

246 

247 func2 = CompositeFunctor(funcDict2, filt=filt) 

248 df2 = self._compositeFuncVal(func2, parq) 

249 self.assertTrue(df.equals(df2)) 

250 

251 func2.filt = 'HSC-R' 

252 df3 = self._compositeFuncVal(func2, parq) 

253 # Because we modified the R filter this should fail. 

254 self.assertFalse(df2.equals(df3)) 

255 

256 # Make sure things work with passing list instead of dict 

257 funcs = [Mag('base_PsfFlux', dataset='ref'), 

258 RAColumn(), 

259 DecColumn(), 

260 Mag('base_PsfFlux', filt=filt), 

261 MagDiff('base_PsfFlux', 'modelfit_CModel', filt=filt)] 

262 

263 df = self._compositeFuncVal(CompositeFunctor(funcs), parq) 

264 

265 def testCompositeColor(self): 

266 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1000) 

267 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords, 10) 

268 parq = self.simulateMultiParquet(self.dataDict) 

269 funcDict = {'a': Mag('base_PsfFlux', dataset='meas', filt='HSC-G'), 

270 'b': Mag('base_PsfFlux', dataset='forced_src', filt='HSC-G'), 

271 'c': Color('base_PsfFlux', 'HSC-G', 'HSC-R')} 

272 # Covering the code is better than nothing 

273 df = self._compositeFuncVal(CompositeFunctor(funcDict), parq) # noqa 

274 

275 def testLocalPhotometry(self): 

276 """Test the local photometry functors. 

277 """ 

278 flux = 1000 

279 fluxErr = 10 

280 calib = 10 

281 calibErr = 1 

282 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, flux) 

283 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords, 

284 fluxErr) 

285 self.dataDict["base_LocalPhotoCalib"] = np.full(self.nRecords, calib) 

286 self.dataDict["base_LocalPhotoCalibErr"] = np.full(self.nRecords, 

287 calibErr) 

288 parq = self.simulateMultiParquet(self.dataDict) 

289 func = LocalPhotometry("base_PsfFlux_instFlux", 

290 "base_PsfFlux_instFluxErr", 

291 "base_LocalPhotoCalib", 

292 "base_LocalPhotoCalibErr") 

293 df = parq.toDataFrame(columns={"dataset": "meas", 

294 "filter": "HSC-G", 

295 "columns": ["base_PsfFlux_instFlux", 

296 "base_PsfFlux_instFluxErr", 

297 "base_LocalPhotoCalib", 

298 "base_LocalPhotoCalibErr"]}) 

299 nanoJansky = func.instFluxToNanojansky( 

300 df[("meas", "HSC-G", "base_PsfFlux_instFlux")], 

301 df[("meas", "HSC-G", "base_LocalPhotoCalib")]) 

302 mag = func.instFluxToMagnitude( 

303 df[("meas", "HSC-G", "base_PsfFlux_instFlux")], 

304 df[("meas", "HSC-G", "base_LocalPhotoCalib")]) 

305 nanoJanskyErr = func.instFluxErrToNanojanskyErr( 

306 df[("meas", "HSC-G", "base_PsfFlux_instFlux")], 

307 df[("meas", "HSC-G", "base_PsfFlux_instFluxErr")], 

308 df[("meas", "HSC-G", "base_LocalPhotoCalib")], 

309 df[("meas", "HSC-G", "base_LocalPhotoCalibErr")]) 

310 magErr = func.instFluxErrToMagnitudeErr( 

311 df[("meas", "HSC-G", "base_PsfFlux_instFlux")], 

312 df[("meas", "HSC-G", "base_PsfFlux_instFluxErr")], 

313 df[("meas", "HSC-G", "base_LocalPhotoCalib")], 

314 df[("meas", "HSC-G", "base_LocalPhotoCalibErr")]) 

315 

316 self.assertTrue(np.allclose(nanoJansky.values, 

317 flux * calib, 

318 atol=1e-13, 

319 rtol=0)) 

320 self.assertTrue(np.allclose(mag.values, 

321 (flux * calib * u.nJy).to_value(u.ABmag), 

322 atol=1e-13, 

323 rtol=0)) 

324 self.assertTrue(np.allclose(nanoJanskyErr.values, 

325 np.hypot(fluxErr * calib, flux * calibErr), 

326 atol=1e-13, 

327 rtol=0)) 

328 self.assertTrue(np.allclose( 

329 magErr.values, 

330 2.5 / np.log(10) * nanoJanskyErr.values / nanoJansky.values, 

331 atol=1e-13, 

332 rtol=0)) 

333 

334 # Test functors against the values computed above. 

335 self._testLocalPhotometryFunctors(LocalNanojansky, 

336 parq, 

337 nanoJansky) 

338 self._testLocalPhotometryFunctors(LocalNanojanskyErr, 

339 parq, 

340 nanoJanskyErr) 

341 self._testLocalPhotometryFunctors(LocalMagnitude, 

342 parq, 

343 mag) 

344 self._testLocalPhotometryFunctors(LocalMagnitudeErr, 

345 parq, 

346 magErr) 

347 

348 def _testLocalPhotometryFunctors(self, functor, parq, testValues): 

349 func = functor("base_PsfFlux_instFlux", 

350 "base_PsfFlux_instFluxErr", 

351 "base_LocalPhotoCalib", 

352 "base_LocalPhotoCalibErr") 

353 val = self._funcVal(func, parq) 

354 self.assertTrue(np.allclose(testValues.values, 

355 val.values, 

356 atol=1e-13, 

357 rtol=0)) 

358 

359 

360class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

361 pass 

362 

363 

364def setup_module(module): 

365 lsst.utils.tests.init() 

366 

367 

368if __name__ == "__main__": 368 ↛ 369line 368 didn't jump to line 369, because the condition on line 368 was never true

369 lsst.utils.tests.init() 

370 unittest.main()