Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import astropy.units as u 

23import copy 

24import functools 

25import numpy as np 

26import os 

27import pandas as pd 

28import unittest 

29 

30import lsst.daf.base as dafBase 

31import lsst.afw.geom as afwGeom 

32import lsst.utils.tests 

33 

34# TODO: Remove skipUnless and this try block DM-22256 

35try: 

36 from lsst.pipe.tasks.parquetTable import MultilevelParquetTable 

37 from lsst.pipe.tasks.functors import (CompositeFunctor, CustomFunctor, Column, RAColumn, 

38 DecColumn, Mag, MagDiff, Color, StarGalaxyLabeller, 

39 DeconvolvedMoments, SdssTraceSize, PsfSdssTraceSizeDiff, 

40 HsmTraceSize, PsfHsmTraceSizeDiff, HsmFwhm, 

41 LocalPhotometry, LocalNanojansky, LocalNanojanskyErr, 

42 LocalMagnitude, LocalMagnitudeErr, 

43 ComputePixelScale, ConvertPixelToArcseconds) 

44 havePyArrow = True 

45except ImportError: 

46 havePyArrow = False 

47 

48ROOT = os.path.abspath(os.path.dirname(__file__)) 

49 

50 

51@unittest.skipUnless(havePyArrow, "Requires pyarrow") 

52class FunctorTestCase(unittest.TestCase): 

53 

54 def simulateMultiParquet(self, dataDict): 

55 """Create a simple test MultilevelParquetTable 

56 """ 

57 simpleDF = pd.DataFrame(dataDict) 

58 dfFilterDSCombos = [] 

59 for ds in self.datasets: 

60 for filterName in self.filters: 

61 df = copy.copy(simpleDF) 

62 df.reindex(sorted(df.columns), axis=1) 

63 df['dataset'] = ds 

64 df['filter'] = filterName 

65 df.columns = pd.MultiIndex.from_tuples( 

66 [(ds, filterName, c) for c in df.columns], 

67 names=('dataset', 'filter', 'column')) 

68 dfFilterDSCombos.append(df) 

69 

70 df = functools.reduce(lambda d1, d2: d1.join(d2), dfFilterDSCombos) 

71 

72 return MultilevelParquetTable(dataFrame=df) 

73 

74 def setUp(self): 

75 np.random.seed(1234) 

76 self.datasets = ['forced_src', 'meas', 'ref'] 

77 self.filters = ['HSC-G', 'HSC-R'] 

78 self.columns = ['coord_ra', 'coord_dec'] 

79 self.nRecords = 5 

80 self.dataDict = { 

81 "coord_ra": [3.77654137, 3.77643059, 3.77621148, 3.77611944, 3.77610396], 

82 "coord_dec": [0.01127624, 0.01127787, 0.01127543, 0.01127543, 0.01127543]} 

83 

84 def _funcVal(self, functor, parq): 

85 self.assertIsInstance(functor.name, str) 

86 self.assertIsInstance(functor.shortname, str) 

87 

88 val = functor(parq) 

89 self.assertIsInstance(val, pd.Series) 

90 

91 val = functor(parq, dropna=True) 

92 self.assertEqual(val.isnull().sum(), 0) 

93 

94 return val 

95 

96 def testColumn(self): 

97 self.columns.append("base_FootprintArea_value") 

98 self.dataDict["base_FootprintArea_value"] = \ 

99 np.full(self.nRecords, 1) 

100 parq = self.simulateMultiParquet(self.dataDict) 

101 func = Column('base_FootprintArea_value', filt='HSC-G') 

102 self._funcVal(func, parq) 

103 

104 def testCustom(self): 

105 self.columns.append("base_FootprintArea_value") 

106 self.dataDict["base_FootprintArea_value"] = \ 

107 np.random.rand(self.nRecords) 

108 parq = self.simulateMultiParquet(self.dataDict) 

109 func = CustomFunctor('2*base_FootprintArea_value', filt='HSC-G') 

110 val = self._funcVal(func, parq) 

111 

112 func2 = Column('base_FootprintArea_value', filt='HSC-G') 

113 

114 np.allclose(val.values, 2*func2(parq).values, atol=1e-13, rtol=0) 

115 

116 def testCoords(self): 

117 parq = self.simulateMultiParquet(self.dataDict) 

118 ra = self._funcVal(RAColumn(), parq) 

119 dec = self._funcVal(DecColumn(), parq) 

120 

121 columnDict = {'dataset': 'ref', 'filter': 'HSC-G', 

122 'column': ['coord_ra', 'coord_dec']} 

123 coords = parq.toDataFrame(columns=columnDict, droplevels=True) / np.pi * 180. 

124 

125 self.assertTrue(np.allclose(ra, coords[('ref', 'HSC-G', 'coord_ra')], atol=1e-13, rtol=0)) 

126 self.assertTrue(np.allclose(dec, coords[('ref', 'HSC-G', 'coord_dec')], atol=1e-13, rtol=0)) 

127 

128 def testMag(self): 

129 self.columns.extend(["base_PsfFlux_instFlux", "base_PsfFlux_instFluxErr"]) 

130 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1000) 

131 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords, 10) 

132 parq = self.simulateMultiParquet(self.dataDict) 

133 # Change one dataset filter combinations value. 

134 parq._df[("meas", "HSC-G", "base_PsfFlux_instFlux")] -= 1 

135 

136 fluxName = 'base_PsfFlux' 

137 

138 # Check that things work when you provide dataset explicitly 

139 for dataset in ['forced_src', 'meas']: 

140 psfMag_G = self._funcVal(Mag(fluxName, dataset=dataset, 

141 filt='HSC-G'), 

142 parq) 

143 psfMag_R = self._funcVal(Mag(fluxName, dataset=dataset, 

144 filt='HSC-R'), 

145 parq) 

146 

147 psfColor_GR = self._funcVal(Color(fluxName, 'HSC-G', 'HSC-R', 

148 dataset=dataset), 

149 parq) 

150 

151 self.assertTrue(np.allclose((psfMag_G - psfMag_R).dropna(), psfColor_GR, rtol=0, atol=1e-13)) 

152 

153 # Check that behavior as expected when dataset not provided; 

154 # that is, that the color comes from forced and default Mag is meas 

155 psfMag_G = self._funcVal(Mag(fluxName, filt='HSC-G'), parq) 

156 psfMag_R = self._funcVal(Mag(fluxName, filt='HSC-R'), parq) 

157 

158 psfColor_GR = self._funcVal(Color(fluxName, 'HSC-G', 'HSC-R'), parq) 

159 

160 # These should *not* be equal. 

161 self.assertFalse(np.allclose((psfMag_G - psfMag_R).dropna(), psfColor_GR)) 

162 

163 def testMagDiff(self): 

164 self.columns.extend(["base_PsfFlux_instFlux", "base_PsfFlux_instFluxErr", 

165 "modelfit_CModel_instFlux", "modelfit_CModel_instFluxErr"]) 

166 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1000) 

167 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords, 10) 

168 self.dataDict["modelfit_CModel_instFlux"] = np.full(self.nRecords, 1000) 

169 self.dataDict["modelfit_CModel_instFluxErr"] = np.full(self.nRecords, 10) 

170 parq = self.simulateMultiParquet(self.dataDict) 

171 

172 for filt in self.filters: 

173 filt = 'HSC-G' 

174 val = self._funcVal(MagDiff('base_PsfFlux', 'modelfit_CModel', filt=filt), parq) 

175 

176 mag1 = self._funcVal(Mag('modelfit_CModel', filt=filt), parq) 

177 mag2 = self._funcVal(Mag('base_PsfFlux', filt=filt), parq) 

178 self.assertTrue(np.allclose((mag2 - mag1).dropna(), val, rtol=0, atol=1e-13)) 

179 

180 def testLabeller(self): 

181 # Covering the code is better than nothing 

182 self.columns.append("base_ClassificationExtendedness_value") 

183 self.dataDict["base_ClassificationExtendedness_value"] = np.full(self.nRecords, 1) 

184 parq = self.simulateMultiParquet(self.dataDict) 

185 labels = self._funcVal(StarGalaxyLabeller(), parq) # noqa 

186 

187 def testPixelScale(self): 

188 # Test that the pixel scale and pix->arcsec calculations perform as 

189 # expected. 

190 pass 

191 

192 def testOther(self): 

193 self.columns.extend(["ext_shapeHSM_HsmSourceMoments_xx", "ext_shapeHSM_HsmSourceMoments_yy", 

194 "base_SdssShape_xx", "base_SdssShape_yy", 

195 "ext_shapeHSM_HsmPsfMoments_xx", "ext_shapeHSM_HsmPsfMoments_yy", 

196 "base_SdssShape_psf_xx", "base_SdssShape_psf_yy"]) 

197 self.dataDict["ext_shapeHSM_HsmSourceMoments_xx"] = np.full(self.nRecords, 1 / np.sqrt(2)) 

198 self.dataDict["ext_shapeHSM_HsmSourceMoments_yy"] = np.full(self.nRecords, 1 / np.sqrt(2)) 

199 self.dataDict["base_SdssShape_xx"] = np.full(self.nRecords, 1 / np.sqrt(2)) 

200 self.dataDict["base_SdssShape_yy"] = np.full(self.nRecords, 1 / np.sqrt(2)) 

201 self.dataDict["ext_shapeHSM_HsmPsfMoments_xx"] = np.full(self.nRecords, 1 / np.sqrt(2)) 

202 self.dataDict["ext_shapeHSM_HsmPsfMoments_yy"] = np.full(self.nRecords, 1 / np.sqrt(2)) 

203 self.dataDict["base_SdssShape_psf_xx"] = np.full(self.nRecords, 1) 

204 self.dataDict["base_SdssShape_psf_yy"] = np.full(self.nRecords, 1) 

205 parq = self.simulateMultiParquet(self.dataDict) 

206 # Covering the code is better than nothing 

207 for filt in self.filters: 

208 for Func in [DeconvolvedMoments, 

209 SdssTraceSize, 

210 PsfSdssTraceSizeDiff, 

211 HsmTraceSize, PsfHsmTraceSizeDiff, HsmFwhm]: 

212 val = self._funcVal(Func(filt=filt), parq) # noqa 

213 

214 def _compositeFuncVal(self, functor, parq): 

215 self.assertIsInstance(functor, CompositeFunctor) 

216 

217 df = functor(parq) 

218 

219 self.assertIsInstance(df, pd.DataFrame) 

220 self.assertTrue(np.all([k in df.columns for k in functor.funcDict.keys()])) 

221 

222 df = functor(parq, dropna=True) 

223 

224 # Check that there are no nulls 

225 self.assertFalse(df.isnull().any(axis=None)) 

226 

227 return df 

228 

229 def testComposite(self): 

230 self.columns.extend(["modelfit_CModel_instFlux", "base_PsfFlux_instFlux"]) 

231 self.dataDict["modelfit_CModel_instFlux"] = np.full(self.nRecords, 1) 

232 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1) 

233 parq = self.simulateMultiParquet(self.dataDict) 

234 # Modify r band value slightly. 

235 parq._df[("meas", "HSC-R", "base_PsfFlux_instFlux")] -= 0.1 

236 

237 filt = 'HSC-G' 

238 funcDict = {'psfMag_ref': Mag('base_PsfFlux', dataset='ref'), 

239 'ra': RAColumn(), 

240 'dec': DecColumn(), 

241 'psfMag': Mag('base_PsfFlux', filt=filt), 

242 'cmodel_magDiff': MagDiff('base_PsfFlux', 

243 'modelfit_CModel', filt=filt)} 

244 func = CompositeFunctor(funcDict) 

245 df = self._compositeFuncVal(func, parq) 

246 

247 # Repeat same, but define filter globally instead of individually 

248 funcDict2 = {'psfMag_ref': Mag('base_PsfFlux', dataset='ref'), 

249 'ra': RAColumn(), 

250 'dec': DecColumn(), 

251 'psfMag': Mag('base_PsfFlux'), 

252 'cmodel_magDiff': MagDiff('base_PsfFlux', 

253 'modelfit_CModel')} 

254 

255 func2 = CompositeFunctor(funcDict2, filt=filt) 

256 df2 = self._compositeFuncVal(func2, parq) 

257 self.assertTrue(df.equals(df2)) 

258 

259 func2.filt = 'HSC-R' 

260 df3 = self._compositeFuncVal(func2, parq) 

261 # Because we modified the R filter this should fail. 

262 self.assertFalse(df2.equals(df3)) 

263 

264 # Make sure things work with passing list instead of dict 

265 funcs = [Mag('base_PsfFlux', dataset='ref'), 

266 RAColumn(), 

267 DecColumn(), 

268 Mag('base_PsfFlux', filt=filt), 

269 MagDiff('base_PsfFlux', 'modelfit_CModel', filt=filt)] 

270 

271 df = self._compositeFuncVal(CompositeFunctor(funcs), parq) 

272 

273 def testCompositeColor(self): 

274 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1000) 

275 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords, 10) 

276 parq = self.simulateMultiParquet(self.dataDict) 

277 funcDict = {'a': Mag('base_PsfFlux', dataset='meas', filt='HSC-G'), 

278 'b': Mag('base_PsfFlux', dataset='forced_src', filt='HSC-G'), 

279 'c': Color('base_PsfFlux', 'HSC-G', 'HSC-R')} 

280 # Covering the code is better than nothing 

281 df = self._compositeFuncVal(CompositeFunctor(funcDict), parq) # noqa 

282 

283 def testLocalPhotometry(self): 

284 """Test the local photometry functors. 

285 """ 

286 flux = 1000 

287 fluxErr = 10 

288 calib = 10 

289 calibErr = 1 

290 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, flux) 

291 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords, 

292 fluxErr) 

293 self.dataDict["base_LocalPhotoCalib"] = np.full(self.nRecords, calib) 

294 self.dataDict["base_LocalPhotoCalibErr"] = np.full(self.nRecords, 

295 calibErr) 

296 parq = self.simulateMultiParquet(self.dataDict) 

297 func = LocalPhotometry("base_PsfFlux_instFlux", 

298 "base_PsfFlux_instFluxErr", 

299 "base_LocalPhotoCalib", 

300 "base_LocalPhotoCalibErr") 

301 df = parq.toDataFrame(columns={"dataset": "meas", 

302 "filter": "HSC-G", 

303 "columns": ["base_PsfFlux_instFlux", 

304 "base_PsfFlux_instFluxErr", 

305 "base_LocalPhotoCalib", 

306 "base_LocalPhotoCalibErr"]}) 

307 nanoJansky = func.instFluxToNanojansky( 

308 df[("meas", "HSC-G", "base_PsfFlux_instFlux")], 

309 df[("meas", "HSC-G", "base_LocalPhotoCalib")]) 

310 mag = func.instFluxToMagnitude( 

311 df[("meas", "HSC-G", "base_PsfFlux_instFlux")], 

312 df[("meas", "HSC-G", "base_LocalPhotoCalib")]) 

313 nanoJanskyErr = func.instFluxErrToNanojanskyErr( 

314 df[("meas", "HSC-G", "base_PsfFlux_instFlux")], 

315 df[("meas", "HSC-G", "base_PsfFlux_instFluxErr")], 

316 df[("meas", "HSC-G", "base_LocalPhotoCalib")], 

317 df[("meas", "HSC-G", "base_LocalPhotoCalibErr")]) 

318 magErr = func.instFluxErrToMagnitudeErr( 

319 df[("meas", "HSC-G", "base_PsfFlux_instFlux")], 

320 df[("meas", "HSC-G", "base_PsfFlux_instFluxErr")], 

321 df[("meas", "HSC-G", "base_LocalPhotoCalib")], 

322 df[("meas", "HSC-G", "base_LocalPhotoCalibErr")]) 

323 

324 self.assertTrue(np.allclose(nanoJansky.values, 

325 flux * calib, 

326 atol=1e-13, 

327 rtol=0)) 

328 self.assertTrue(np.allclose(mag.values, 

329 (flux * calib * u.nJy).to_value(u.ABmag), 

330 atol=1e-13, 

331 rtol=0)) 

332 self.assertTrue(np.allclose(nanoJanskyErr.values, 

333 np.hypot(fluxErr * calib, flux * calibErr), 

334 atol=1e-13, 

335 rtol=0)) 

336 self.assertTrue(np.allclose( 

337 magErr.values, 

338 2.5 / np.log(10) * nanoJanskyErr.values / nanoJansky.values, 

339 atol=1e-13, 

340 rtol=0)) 

341 

342 # Test functors against the values computed above. 

343 self._testLocalPhotometryFunctors(LocalNanojansky, 

344 parq, 

345 nanoJansky) 

346 self._testLocalPhotometryFunctors(LocalNanojanskyErr, 

347 parq, 

348 nanoJanskyErr) 

349 self._testLocalPhotometryFunctors(LocalMagnitude, 

350 parq, 

351 mag) 

352 self._testLocalPhotometryFunctors(LocalMagnitudeErr, 

353 parq, 

354 magErr) 

355 

356 def _testLocalPhotometryFunctors(self, functor, parq, testValues): 

357 func = functor("base_PsfFlux_instFlux", 

358 "base_PsfFlux_instFluxErr", 

359 "base_LocalPhotoCalib", 

360 "base_LocalPhotoCalibErr") 

361 val = self._funcVal(func, parq) 

362 self.assertTrue(np.allclose(testValues.values, 

363 val.values, 

364 atol=1e-13, 

365 rtol=0)) 

366 

367 def testConvertPixelToArcseconds(self): 

368 """Test calculations of the pixel scale and conversions of pixel to 

369 arcseconds. 

370 """ 

371 wcs = self._makeWcs() 

372 cdMatrix = wcs.getCdMatrix(wcs.getPixelOrigin()) 

373 self.dataDict["dipoleSep"] = np.full(self.nRecords, 10) 

374 self.dataDict["base_LocalWcs_CDMatrix_1_1"] = np.full(self.nRecords, 

375 cdMatrix[0, 0]) 

376 self.dataDict["base_LocalWcs_CDMatrix_1_2"] = np.full(self.nRecords, 

377 cdMatrix[0, 1]) 

378 self.dataDict["base_LocalWcs_CDMatrix_2_1"] = np.full(self.nRecords, 

379 cdMatrix[1, 0]) 

380 self.dataDict["base_LocalWcs_CDMatrix_2_2"] = np.full(self.nRecords, 

381 cdMatrix[1, 1]) 

382 parq = self.simulateMultiParquet(self.dataDict) 

383 func = ComputePixelScale("base_LocalWcs_CDMatrix_1_1", 

384 "base_LocalWcs_CDMatrix_1_2", 

385 "base_LocalWcs_CDMatrix_2_1", 

386 "base_LocalWcs_CDMatrix_2_2") 

387 df = parq.toDataFrame(columns={"dataset": "meas", 

388 "filter": "HSC-G", 

389 "columns": ["dipoleSep", 

390 "base_LocalWcs_CDMatrix_1_1", 

391 "base_LocalWcs_CDMatrix_1_2", 

392 "base_LocalWcs_CDMatrix_2_1", 

393 "base_LocalWcs_CDMatrix_2_2"]}) 

394 pixelScale = func.pixelScale(df[("meas", "HSC-G", "base_LocalWcs_CDMatrix_1_1")], 

395 df[("meas", "HSC-G", "base_LocalWcs_CDMatrix_1_2")], 

396 df[("meas", "HSC-G", "base_LocalWcs_CDMatrix_2_1")], 

397 df[("meas", "HSC-G", "base_LocalWcs_CDMatrix_2_2")]) 

398 self.assertTrue(np.allclose(pixelScale.values, 

399 wcs.getPixelScale().asArcseconds(), 

400 rtol=0, 

401 atol=1e-10)) 

402 

403 # Test functors 

404 val = self._funcVal(func, parq) 

405 self.assertTrue(np.allclose(pixelScale.values, 

406 val.values, 

407 atol=1e-13, 

408 rtol=0)) 

409 

410 func = ConvertPixelToArcseconds("dipoleSep", 

411 "base_LocalWcs_CDMatrix_1_1", 

412 "base_LocalWcs_CDMatrix_1_2", 

413 "base_LocalWcs_CDMatrix_2_1", 

414 "base_LocalWcs_CDMatrix_2_2") 

415 val = self._funcVal(func, parq) 

416 self.assertTrue(np.allclose(pixelScale.values * 10, 

417 val.values, 

418 atol=1e-13, 

419 rtol=0)) 

420 

421 def _makeWcs(self): 

422 """Create a wcs from real CFHT values. 

423 

424 Returns 

425 ------- 

426 wcs : `lsst.afw.geom` 

427 Created wcs. 

428 """ 

429 metadata = dafBase.PropertySet() 

430 

431 metadata.set("SIMPLE", "T") 

432 metadata.set("BITPIX", -32) 

433 metadata.set("NAXIS", 2) 

434 metadata.set("NAXIS1", 1024) 

435 metadata.set("NAXIS2", 1153) 

436 metadata.set("RADECSYS", 'FK5') 

437 metadata.set("EQUINOX", 2000.) 

438 

439 metadata.setDouble("CRVAL1", 215.604025685476) 

440 metadata.setDouble("CRVAL2", 53.1595451514076) 

441 metadata.setDouble("CRPIX1", 1109.99981456774) 

442 metadata.setDouble("CRPIX2", 560.018167811613) 

443 metadata.set("CTYPE1", 'RA---SIN') 

444 metadata.set("CTYPE2", 'DEC--SIN') 

445 

446 metadata.setDouble("CD1_1", 5.10808596133527E-05) 

447 metadata.setDouble("CD1_2", 1.85579539217196E-07) 

448 metadata.setDouble("CD2_2", -5.10281493481982E-05) 

449 metadata.setDouble("CD2_1", -8.27440751733828E-07) 

450 

451 return afwGeom.makeSkyWcs(metadata) 

452 

453 

454class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

455 pass 

456 

457 

458def setup_module(module): 

459 lsst.utils.tests.init() 

460 

461 

462if __name__ == "__main__": 462 ↛ 463line 462 didn't jump to line 463, because the condition on line 462 was never true

463 lsst.utils.tests.init() 

464 unittest.main()