Coverage for tests/test_functors.py: 13%

395 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-11-06 13:40 -0800

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import astropy.units as u 

23import copy 

24import functools 

25import numpy as np 

26import os 

27import pandas as pd 

28import unittest 

29import tempfile 

30import shutil 

31 

32import lsst.daf.base as dafBase 

33import lsst.afw.geom as afwGeom 

34import lsst.geom as geom 

35from lsst.sphgeom import HtmPixelization 

36import lsst.meas.base as measBase 

37import lsst.utils.tests 

38from lsst.pipe.tasks.parquetTable import MultilevelParquetTable, ParquetTable 

39from lsst.daf.butler import Butler, DatasetType 

40from lsst.pipe.tasks.functors import (CompositeFunctor, CustomFunctor, Column, RAColumn, 

41 DecColumn, Mag, MagDiff, Color, StarGalaxyLabeller, 

42 DeconvolvedMoments, SdssTraceSize, PsfSdssTraceSizeDiff, 

43 HsmTraceSize, PsfHsmTraceSizeDiff, HsmFwhm, 

44 LocalPhotometry, LocalNanojansky, LocalNanojanskyErr, 

45 LocalMagnitude, LocalMagnitudeErr, 

46 LocalDipoleMeanFlux, LocalDipoleMeanFluxErr, 

47 LocalDipoleDiffFlux, LocalDipoleDiffFluxErr, 

48 LocalWcs, ComputePixelScale, ConvertPixelToArcseconds, 

49 ConvertPixelSqToArcsecondsSq, Ratio, HtmIndex20) 

50 

51ROOT = os.path.abspath(os.path.dirname(__file__)) 

52 

53 

54class FunctorTestCase(unittest.TestCase): 

55 

56 def simulateMultiParquet(self, dataDict): 

57 """Create a simple test MultilevelParquetTable 

58 """ 

59 simpleDF = pd.DataFrame(dataDict) 

60 dfFilterDSCombos = [] 

61 for ds in self.datasets: 

62 for band in self.bands: 

63 df = copy.copy(simpleDF) 

64 df.reindex(sorted(df.columns), axis=1) 

65 df['dataset'] = ds 

66 df['band'] = band 

67 df.columns = pd.MultiIndex.from_tuples( 

68 [(ds, band, c) for c in df.columns], 

69 names=('dataset', 'band', 'column')) 

70 dfFilterDSCombos.append(df) 

71 

72 df = functools.reduce(lambda d1, d2: d1.join(d2), dfFilterDSCombos) 

73 

74 return MultilevelParquetTable(dataFrame=df) 

75 

76 def simulateParquet(self, dataDict): 

77 df = pd.DataFrame(dataDict) 

78 return ParquetTable(dataFrame=df) 

79 

80 def getDatasetHandle(self, parq): 

81 df = parq._df 

82 lo, hi = HtmPixelization(7).universe().ranges()[0] 

83 value = np.random.randint(lo, hi) 

84 ref = self.butler.put(df, self.datasetType, dataId={'htm7': value}) 

85 return self.butler.getDeferred(ref) 

86 

87 def setUp(self): 

88 np.random.seed(12345) 

89 self.datasets = ['forced_src', 'meas', 'ref'] 

90 self.bands = ['g', 'r'] 

91 self.columns = ['coord_ra', 'coord_dec'] 

92 self.nRecords = 5 

93 self.dataDict = { 

94 "coord_ra": [3.77654137, 3.77643059, 3.77621148, 3.77611944, 3.77610396], 

95 "coord_dec": [0.01127624, 0.01127787, 0.01127543, 0.01127543, 0.01127543]} 

96 

97 # Set up butler 

98 self.root = tempfile.mkdtemp(dir=ROOT) 

99 Butler.makeRepo(self.root) 

100 self.butler = Butler(self.root, run="test_run") 

101 self.datasetType = DatasetType("data", dimensions=('htm7',), storageClass="DataFrame", 

102 universe=self.butler.registry.dimensions) 

103 self.butler.registry.registerDatasetType(self.datasetType) 

104 

105 def tearDown(self): 

106 if os.path.exists(self.root): 

107 shutil.rmtree(self.root, ignore_errors=True) 

108 

109 def _funcVal(self, functor, parq): 

110 self.assertIsInstance(functor.name, str) 

111 self.assertIsInstance(functor.shortname, str) 

112 

113 handle = self.getDatasetHandle(parq) 

114 

115 val = functor(parq) 

116 val2 = functor(handle) 

117 self.assertTrue((val == val2).all()) 

118 self.assertIsInstance(val, pd.Series) 

119 

120 val = functor(parq, dropna=True) 

121 val2 = functor(handle, dropna=True) 

122 self.assertTrue((val == val2).all()) 

123 self.assertEqual(val.isnull().sum(), 0) 

124 

125 return val 

126 

127 def _differenceVal(self, functor, parq1, parq2): 

128 self.assertIsInstance(functor.name, str) 

129 self.assertIsInstance(functor.shortname, str) 

130 

131 handle1 = self.getDatasetHandle(parq1) 

132 handle2 = self.getDatasetHandle(parq2) 

133 

134 val = functor.difference(parq1, parq2) 

135 val2 = functor.difference(handle1, handle2) 

136 self.assertTrue(val.equals(val2)) 

137 self.assertIsInstance(val, pd.Series) 

138 

139 val = functor.difference(parq1, parq2, dropna=True) 

140 val2 = functor.difference(handle1, handle2, dropna=True) 

141 self.assertTrue(val.equals(val2)) 

142 self.assertEqual(val.isnull().sum(), 0) 

143 

144 val1 = self._funcVal(functor, parq1) 

145 val2 = self._funcVal(functor, parq2) 

146 

147 self.assertTrue(np.allclose(val, val1 - val2)) 

148 

149 return val 

150 

151 def testColumn(self): 

152 self.columns.append("base_FootprintArea_value") 

153 self.dataDict["base_FootprintArea_value"] = \ 

154 np.full(self.nRecords, 1) 

155 parq = self.simulateMultiParquet(self.dataDict) 

156 func = Column('base_FootprintArea_value', filt='g') 

157 self._funcVal(func, parq) 

158 

159 parq = self.simulateParquet(self.dataDict) 

160 func = Column('base_FootprintArea_value') 

161 self._funcVal(func, parq) 

162 

163 def testCustom(self): 

164 self.columns.append("base_FootprintArea_value") 

165 self.dataDict["base_FootprintArea_value"] = \ 

166 np.random.rand(self.nRecords) 

167 parq = self.simulateMultiParquet(self.dataDict) 

168 func = CustomFunctor('2*base_FootprintArea_value', filt='g') 

169 val = self._funcVal(func, parq) 

170 

171 func2 = Column('base_FootprintArea_value', filt='g') 

172 

173 np.allclose(val.values, 2*func2(parq).values, atol=1e-13, rtol=0) 

174 

175 parq = self.simulateParquet(self.dataDict) 

176 func = CustomFunctor('2 * base_FootprintArea_value') 

177 val = self._funcVal(func, parq) 

178 func2 = Column('base_FootprintArea_value') 

179 

180 np.allclose(val.values, 2*func2(parq).values, atol=1e-13, rtol=0) 

181 

182 def testCoords(self): 

183 parq = self.simulateMultiParquet(self.dataDict) 

184 ra = self._funcVal(RAColumn(), parq) 

185 dec = self._funcVal(DecColumn(), parq) 

186 

187 columnDict = {'dataset': 'ref', 'band': 'g', 

188 'column': ['coord_ra', 'coord_dec']} 

189 

190 coords = parq.toDataFrame(columns=columnDict, droplevels=True) / np.pi * 180. 

191 

192 self.assertTrue(np.allclose(ra, coords[('ref', 'g', 'coord_ra')], atol=1e-13, rtol=0)) 

193 self.assertTrue(np.allclose(dec, coords[('ref', 'g', 'coord_dec')], atol=1e-13, rtol=0)) 

194 

195 # single-level column index table 

196 parq = self.simulateParquet(self.dataDict) 

197 ra = self._funcVal(RAColumn(), parq) 

198 dec = self._funcVal(DecColumn(), parq) 

199 

200 coords = parq.toDataFrame(columns=['coord_ra', 'coord_dec']) / np.pi * 180. 

201 

202 self.assertTrue(np.allclose(ra, coords['coord_ra'], atol=1e-13, rtol=0)) 

203 self.assertTrue(np.allclose(dec, coords['coord_dec'], atol=1e-13, rtol=0)) 

204 

205 def testMag(self): 

206 self.columns.extend(["base_PsfFlux_instFlux", "base_PsfFlux_instFluxErr"]) 

207 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1000) 

208 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords, 10) 

209 parq = self.simulateMultiParquet(self.dataDict) 

210 # Change one dataset filter combinations value. 

211 parq._df[("meas", "g", "base_PsfFlux_instFlux")] -= 1 

212 

213 fluxName = 'base_PsfFlux' 

214 

215 # Check that things work when you provide dataset explicitly 

216 for dataset in ['forced_src', 'meas']: 

217 psfMag_G = self._funcVal(Mag(fluxName, dataset=dataset, 

218 filt='g'), 

219 parq) 

220 psfMag_R = self._funcVal(Mag(fluxName, dataset=dataset, 

221 filt='r'), 

222 parq) 

223 

224 psfColor_GR = self._funcVal(Color(fluxName, 'g', 'r', 

225 dataset=dataset), 

226 parq) 

227 

228 self.assertTrue(np.allclose((psfMag_G - psfMag_R).dropna(), psfColor_GR, rtol=0, atol=1e-13)) 

229 

230 # Check that behavior as expected when dataset not provided; 

231 # that is, that the color comes from forced and default Mag is meas 

232 psfMag_G = self._funcVal(Mag(fluxName, filt='g'), parq) 

233 psfMag_R = self._funcVal(Mag(fluxName, filt='r'), parq) 

234 

235 psfColor_GR = self._funcVal(Color(fluxName, 'g', 'r'), parq) 

236 

237 # These should *not* be equal. 

238 self.assertFalse(np.allclose((psfMag_G - psfMag_R).dropna(), psfColor_GR)) 

239 

240 def testMagDiff(self): 

241 self.columns.extend(["base_PsfFlux_instFlux", "base_PsfFlux_instFluxErr", 

242 "modelfit_CModel_instFlux", "modelfit_CModel_instFluxErr"]) 

243 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1000) 

244 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords, 10) 

245 self.dataDict["modelfit_CModel_instFlux"] = np.full(self.nRecords, 1000) 

246 self.dataDict["modelfit_CModel_instFluxErr"] = np.full(self.nRecords, 10) 

247 parq = self.simulateMultiParquet(self.dataDict) 

248 

249 for filt in self.bands: 

250 filt = 'g' 

251 val = self._funcVal(MagDiff('base_PsfFlux', 'modelfit_CModel', filt=filt), parq) 

252 

253 mag1 = self._funcVal(Mag('modelfit_CModel', filt=filt), parq) 

254 mag2 = self._funcVal(Mag('base_PsfFlux', filt=filt), parq) 

255 self.assertTrue(np.allclose((mag2 - mag1).dropna(), val, rtol=0, atol=1e-13)) 

256 

257 def testDifference(self): 

258 """Test .difference method using MagDiff as the example. 

259 """ 

260 self.columns.extend(["base_PsfFlux_instFlux", "base_PsfFlux_instFluxErr", 

261 "modelfit_CModel_instFlux", "modelfit_CModel_instFluxErr"]) 

262 

263 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1000) 

264 self.dataDict["modelfit_CModel_instFlux"] = np.full(self.nRecords, 1000) 

265 parq1 = self.simulateMultiParquet(self.dataDict) 

266 

267 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 999) 

268 self.dataDict["modelfit_CModel_instFlux"] = np.full(self.nRecords, 999) 

269 parq2 = self.simulateMultiParquet(self.dataDict) 

270 

271 magDiff = MagDiff('base_PsfFlux', 'modelfit_CModel', filt='g') 

272 

273 # Asserts that differences computed properly 

274 self._differenceVal(magDiff, parq1, parq2) 

275 

276 def testLabeller(self): 

277 # Covering the code is better than nothing 

278 self.columns.append("base_ClassificationExtendedness_value") 

279 self.dataDict["base_ClassificationExtendedness_value"] = np.full(self.nRecords, 1) 

280 parq = self.simulateMultiParquet(self.dataDict) 

281 labels = self._funcVal(StarGalaxyLabeller(), parq) # noqa 

282 

283 def testPixelScale(self): 

284 # Test that the pixel scale and pix->arcsec calculations perform as 

285 # expected. 

286 pass 

287 

288 def testOther(self): 

289 self.columns.extend(["ext_shapeHSM_HsmSourceMoments_xx", "ext_shapeHSM_HsmSourceMoments_yy", 

290 "base_SdssShape_xx", "base_SdssShape_yy", 

291 "ext_shapeHSM_HsmPsfMoments_xx", "ext_shapeHSM_HsmPsfMoments_yy", 

292 "base_SdssShape_psf_xx", "base_SdssShape_psf_yy"]) 

293 self.dataDict["ext_shapeHSM_HsmSourceMoments_xx"] = np.full(self.nRecords, 1 / np.sqrt(2)) 

294 self.dataDict["ext_shapeHSM_HsmSourceMoments_yy"] = np.full(self.nRecords, 1 / np.sqrt(2)) 

295 self.dataDict["base_SdssShape_xx"] = np.full(self.nRecords, 1 / np.sqrt(2)) 

296 self.dataDict["base_SdssShape_yy"] = np.full(self.nRecords, 1 / np.sqrt(2)) 

297 self.dataDict["ext_shapeHSM_HsmPsfMoments_xx"] = np.full(self.nRecords, 1 / np.sqrt(2)) 

298 self.dataDict["ext_shapeHSM_HsmPsfMoments_yy"] = np.full(self.nRecords, 1 / np.sqrt(2)) 

299 self.dataDict["base_SdssShape_psf_xx"] = np.full(self.nRecords, 1) 

300 self.dataDict["base_SdssShape_psf_yy"] = np.full(self.nRecords, 1) 

301 parq = self.simulateMultiParquet(self.dataDict) 

302 # Covering the code is better than nothing 

303 for filt in self.bands: 

304 for Func in [DeconvolvedMoments, 

305 SdssTraceSize, 

306 PsfSdssTraceSizeDiff, 

307 HsmTraceSize, PsfHsmTraceSizeDiff, HsmFwhm]: 

308 val = self._funcVal(Func(filt=filt), parq) # noqa 

309 

310 def _compositeFuncVal(self, functor, parq): 

311 self.assertIsInstance(functor, CompositeFunctor) 

312 

313 handle = self.getDatasetHandle(parq) 

314 

315 df = functor(parq) 

316 df2 = functor(handle) 

317 self.assertTrue(df.equals(df2)) 

318 

319 self.assertIsInstance(df, pd.DataFrame) 

320 self.assertTrue(np.all([k in df.columns for k in functor.funcDict.keys()])) 

321 

322 df = functor(parq, dropna=True) 

323 df2 = functor(handle, dropna=True) 

324 self.assertTrue(df.equals(df2)) 

325 

326 # Check that there are no nulls 

327 self.assertFalse(df.isnull().any(axis=None)) 

328 

329 return df 

330 

331 def _compositeDifferenceVal(self, functor, parq1, parq2): 

332 self.assertIsInstance(functor, CompositeFunctor) 

333 

334 handle1 = self.getDatasetHandle(parq1) 

335 handle2 = self.getDatasetHandle(parq2) 

336 

337 df = functor.difference(parq1, parq2) 

338 df2 = functor.difference(handle1, handle2) 

339 self.assertTrue(df.equals(df2)) 

340 

341 self.assertIsInstance(df, pd.DataFrame) 

342 self.assertTrue(np.all([k in df.columns for k in functor.funcDict.keys()])) 

343 

344 df = functor.difference(parq1, parq2, dropna=True) 

345 df2 = functor.difference(handle1, handle2, dropna=True) 

346 self.assertTrue(df.equals(df2)) 

347 

348 # Check that there are no nulls 

349 self.assertFalse(df.isnull().any(axis=None)) 

350 

351 df1 = functor(parq1) 

352 df2 = functor(parq2) 

353 

354 self.assertTrue(np.allclose(df.values, df1.values - df2.values)) 

355 

356 return df 

357 

358 def testComposite(self): 

359 self.columns.extend(["modelfit_CModel_instFlux", "base_PsfFlux_instFlux"]) 

360 self.dataDict["modelfit_CModel_instFlux"] = np.full(self.nRecords, 1) 

361 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1) 

362 

363 parq = self.simulateMultiParquet(self.dataDict) 

364 # Modify r band value slightly. 

365 parq._df[("meas", "r", "base_PsfFlux_instFlux")] -= 0.1 

366 

367 filt = 'g' 

368 funcDict = {'psfMag_ref': Mag('base_PsfFlux', dataset='ref'), 

369 'ra': RAColumn(), 

370 'dec': DecColumn(), 

371 'psfMag': Mag('base_PsfFlux', filt=filt), 

372 'cmodel_magDiff': MagDiff('base_PsfFlux', 

373 'modelfit_CModel', filt=filt)} 

374 func = CompositeFunctor(funcDict) 

375 df = self._compositeFuncVal(func, parq) 

376 

377 # Repeat same, but define filter globally instead of individually 

378 funcDict2 = {'psfMag_ref': Mag('base_PsfFlux', dataset='ref'), 

379 'ra': RAColumn(), 

380 'dec': DecColumn(), 

381 'psfMag': Mag('base_PsfFlux'), 

382 'cmodel_magDiff': MagDiff('base_PsfFlux', 

383 'modelfit_CModel')} 

384 

385 func2 = CompositeFunctor(funcDict2, filt=filt) 

386 df2 = self._compositeFuncVal(func2, parq) 

387 self.assertTrue(df.equals(df2)) 

388 

389 func2.filt = 'r' 

390 df3 = self._compositeFuncVal(func2, parq) 

391 # Because we modified the R filter this should fail. 

392 self.assertFalse(df2.equals(df3)) 

393 

394 # Make sure things work with passing list instead of dict 

395 funcs = [Mag('base_PsfFlux', dataset='ref'), 

396 RAColumn(), 

397 DecColumn(), 

398 Mag('base_PsfFlux', filt=filt), 

399 MagDiff('base_PsfFlux', 'modelfit_CModel', filt=filt)] 

400 

401 df = self._compositeFuncVal(CompositeFunctor(funcs), parq) 

402 

403 def testCompositeSimple(self): 

404 """Test single-level composite functor for functionality 

405 """ 

406 self.columns.extend(["modelfit_CModel_instFlux", "base_PsfFlux_instFlux"]) 

407 self.dataDict["modelfit_CModel_instFlux"] = np.full(self.nRecords, 1) 

408 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1) 

409 

410 parq = self.simulateParquet(self.dataDict) 

411 funcDict = {'ra': RAColumn(), 

412 'dec': DecColumn(), 

413 'psfMag': Mag('base_PsfFlux'), 

414 'cmodel_magDiff': MagDiff('base_PsfFlux', 

415 'modelfit_CModel')} 

416 func = CompositeFunctor(funcDict) 

417 df = self._compositeFuncVal(func, parq) # noqa 

418 

419 def testCompositeColor(self): 

420 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1000) 

421 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords, 10) 

422 parq = self.simulateMultiParquet(self.dataDict) 

423 funcDict = {'a': Mag('base_PsfFlux', dataset='meas', filt='g'), 

424 'b': Mag('base_PsfFlux', dataset='forced_src', filt='g'), 

425 'c': Color('base_PsfFlux', 'g', 'r')} 

426 # Covering the code is better than nothing 

427 df = self._compositeFuncVal(CompositeFunctor(funcDict), parq) # noqa 

428 

429 def testCompositeDifference(self): 

430 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1000) 

431 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords, 10) 

432 parq1 = self.simulateMultiParquet(self.dataDict) 

433 

434 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 999) 

435 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords, 9) 

436 parq2 = self.simulateMultiParquet(self.dataDict) 

437 

438 funcDict = {'a': Mag('base_PsfFlux', dataset='meas', filt='g'), 

439 'b': Mag('base_PsfFlux', dataset='forced_src', filt='g'), 

440 'c': Color('base_PsfFlux', 'g', 'r')} 

441 # Covering the code is better than nothing 

442 df = self._compositeDifferenceVal(CompositeFunctor(funcDict), parq1, parq2) # noqa 

443 

444 def testCompositeFail(self): 

445 """Test a composite functor where one of the functors should be junk. 

446 """ 

447 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1000) 

448 parq = self.simulateMultiParquet(self.dataDict) 

449 

450 funcDict = {'good': Column("base_PsfFlux_instFlux"), 

451 'bad': Column('not_a_column')} 

452 

453 df = self._compositeFuncVal(CompositeFunctor(funcDict), parq) # noqa 

454 

455 def testLocalPhotometry(self): 

456 """Test the local photometry functors. 

457 """ 

458 flux = 1000 

459 fluxErr = 10 

460 calib = 10 

461 calibErr = 1 

462 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, flux) 

463 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords, 

464 fluxErr) 

465 self.dataDict["base_LocalPhotoCalib"] = np.full(self.nRecords, calib) 

466 self.dataDict["base_LocalPhotoCalibErr"] = np.full(self.nRecords, 

467 calibErr) 

468 parq = self.simulateMultiParquet(self.dataDict) 

469 func = LocalPhotometry("base_PsfFlux_instFlux", 

470 "base_PsfFlux_instFluxErr", 

471 "base_LocalPhotoCalib", 

472 "base_LocalPhotoCalibErr") 

473 df = parq.toDataFrame(columns={"dataset": "meas", 

474 "band": "g", 

475 "columns": ["base_PsfFlux_instFlux", 

476 "base_PsfFlux_instFluxErr", 

477 "base_LocalPhotoCalib", 

478 "base_LocalPhotoCalibErr"]}) 

479 nanoJansky = func.instFluxToNanojansky( 

480 df[("meas", "g", "base_PsfFlux_instFlux")], 

481 df[("meas", "g", "base_LocalPhotoCalib")]) 

482 mag = func.instFluxToMagnitude( 

483 df[("meas", "g", "base_PsfFlux_instFlux")], 

484 df[("meas", "g", "base_LocalPhotoCalib")]) 

485 nanoJanskyErr = func.instFluxErrToNanojanskyErr( 

486 df[("meas", "g", "base_PsfFlux_instFlux")], 

487 df[("meas", "g", "base_PsfFlux_instFluxErr")], 

488 df[("meas", "g", "base_LocalPhotoCalib")], 

489 df[("meas", "g", "base_LocalPhotoCalibErr")]) 

490 magErr = func.instFluxErrToMagnitudeErr( 

491 df[("meas", "g", "base_PsfFlux_instFlux")], 

492 df[("meas", "g", "base_PsfFlux_instFluxErr")], 

493 df[("meas", "g", "base_LocalPhotoCalib")], 

494 df[("meas", "g", "base_LocalPhotoCalibErr")]) 

495 

496 self.assertTrue(np.allclose(nanoJansky.values, 

497 flux * calib, 

498 atol=1e-13, 

499 rtol=0)) 

500 self.assertTrue(np.allclose(mag.values, 

501 (flux * calib * u.nJy).to_value(u.ABmag), 

502 atol=1e-13, 

503 rtol=0)) 

504 self.assertTrue(np.allclose(nanoJanskyErr.values, 

505 np.hypot(fluxErr * calib, flux * calibErr), 

506 atol=1e-13, 

507 rtol=0)) 

508 self.assertTrue(np.allclose( 

509 magErr.values, 

510 2.5 / np.log(10) * nanoJanskyErr.values / nanoJansky.values, 

511 atol=1e-13, 

512 rtol=0)) 

513 

514 # Test functors against the values computed above. 

515 self._testLocalPhotometryFunctors(LocalNanojansky, 

516 parq, 

517 nanoJansky) 

518 self._testLocalPhotometryFunctors(LocalNanojanskyErr, 

519 parq, 

520 nanoJanskyErr) 

521 self._testLocalPhotometryFunctors(LocalMagnitude, 

522 parq, 

523 mag) 

524 self._testLocalPhotometryFunctors(LocalMagnitudeErr, 

525 parq, 

526 magErr) 

527 

528 def _testLocalPhotometryFunctors(self, functor, parq, testValues): 

529 func = functor("base_PsfFlux_instFlux", 

530 "base_PsfFlux_instFluxErr", 

531 "base_LocalPhotoCalib", 

532 "base_LocalPhotoCalibErr") 

533 val = self._funcVal(func, parq) 

534 self.assertTrue(np.allclose(testValues.values, 

535 val.values, 

536 atol=1e-13, 

537 rtol=0)) 

538 

539 def testDipPhotometry(self): 

540 """Test calibrated flux calculations for dipoles.""" 

541 fluxNeg = -100 

542 fluxPos = 101 

543 fluxErr = 10 

544 calib = 10 

545 calibErr = 1 

546 

547 # compute expected values. 

548 absMean = 0.5*(fluxPos - fluxNeg)*calib 

549 absDiff = (fluxNeg + fluxPos)*calib 

550 absMeanErr = 0.5*np.sqrt(2*(fluxErr*calib)**2 

551 + ((fluxPos - fluxNeg)*calibErr)**2) 

552 absDiffErr = np.sqrt(2*(fluxErr*calib)**2 

553 + ((fluxPos + fluxNeg)*calibErr)**2) 

554 

555 self.dataDict["ip_diffim_DipoleFluxNeg_instFlux"] = np.full(self.nRecords, fluxNeg) 

556 self.dataDict["ip_diffim_DipoleFluxNeg_instFluxErr"] = np.full(self.nRecords, fluxErr) 

557 self.dataDict["ip_diffim_DipoleFluxPos_instFlux"] = np.full(self.nRecords, fluxPos) 

558 self.dataDict["ip_diffim_DipoleFluxPos_instFluxErr"] = np.full(self.nRecords, fluxErr) 

559 self.dataDict["base_LocalPhotoCalib"] = np.full(self.nRecords, calib) 

560 self.dataDict["base_LocalPhotoCalibErr"] = np.full(self.nRecords, 

561 calibErr) 

562 

563 parq = self.simulateMultiParquet(self.dataDict) 

564 func = LocalDipoleMeanFlux("ip_diffim_DipoleFluxPos_instFlux", 

565 "ip_diffim_DipoleFluxNeg_instFlux", 

566 "ip_diffim_DipoleFluxPos_instFluxErr", 

567 "ip_diffim_DipoleFluxNeg_instFluxErr", 

568 "base_LocalPhotoCalib", 

569 "base_LocalPhotoCalibErr") 

570 val = self._funcVal(func, parq) 

571 self.assertTrue(np.allclose(val.values, 

572 absMean, 

573 atol=1e-13, 

574 rtol=0)) 

575 

576 func = LocalDipoleMeanFluxErr("ip_diffim_DipoleFluxPos_instFlux", 

577 "ip_diffim_DipoleFluxNeg_instFlux", 

578 "ip_diffim_DipoleFluxPos_instFluxErr", 

579 "ip_diffim_DipoleFluxNeg_instFluxErr", 

580 "base_LocalPhotoCalib", 

581 "base_LocalPhotoCalibErr") 

582 val = self._funcVal(func, parq) 

583 self.assertTrue(np.allclose(val.values, 

584 absMeanErr, 

585 atol=1e-13, 

586 rtol=0)) 

587 

588 func = LocalDipoleDiffFlux("ip_diffim_DipoleFluxPos_instFlux", 

589 "ip_diffim_DipoleFluxNeg_instFlux", 

590 "ip_diffim_DipoleFluxPos_instFluxErr", 

591 "ip_diffim_DipoleFluxNeg_instFluxErr", 

592 "base_LocalPhotoCalib", 

593 "base_LocalPhotoCalibErr") 

594 val = self._funcVal(func, parq) 

595 self.assertTrue(np.allclose(val.values, 

596 absDiff, 

597 atol=1e-13, 

598 rtol=0)) 

599 

600 func = LocalDipoleDiffFluxErr("ip_diffim_DipoleFluxPos_instFlux", 

601 "ip_diffim_DipoleFluxNeg_instFlux", 

602 "ip_diffim_DipoleFluxPos_instFluxErr", 

603 "ip_diffim_DipoleFluxNeg_instFluxErr", 

604 "base_LocalPhotoCalib", 

605 "base_LocalPhotoCalibErr") 

606 val = self._funcVal(func, parq) 

607 print(val.values[0]) 

608 self.assertTrue(np.allclose(val.values, 

609 absDiffErr, 

610 atol=1e-13, 

611 rtol=0)) 

612 

613 def testConvertPixelToArcseconds(self): 

614 """Test calculations of the pixel scale and conversions of pixel to 

615 arcseconds. 

616 """ 

617 dipoleSep = 10 

618 ixx = 10 

619 testPixelDeltas = np.random.uniform(-100, 100, size=(self.nRecords, 2)) 

620 import lsst.afw.table as afwTable 

621 localWcsPlugin = measBase.EvaluateLocalWcsPlugin( 

622 None, 

623 "base_LocalWcs", 

624 afwTable.SourceTable.makeMinimalSchema(), 

625 None) 

626 for dec in np.linspace(-90, 90, 10): 

627 for x, y in zip(np.random.uniform(2 * 1109.99981456774, size=10), 

628 np.random.uniform(2 * 560.018167811613, size=10)): 

629 center = geom.Point2D(x, y) 

630 wcs = self._makeWcs(dec) 

631 skyOrigin = wcs.pixelToSky(center) 

632 

633 linAffMatrix = localWcsPlugin.makeLocalTransformMatrix(wcs, 

634 center) 

635 self.dataDict["dipoleSep"] = np.full(self.nRecords, dipoleSep) 

636 self.dataDict["ixx"] = np.full(self.nRecords, ixx) 

637 self.dataDict["slot_Centroid_x"] = np.full(self.nRecords, x) 

638 self.dataDict["slot_Centroid_y"] = np.full(self.nRecords, y) 

639 self.dataDict["someCentroid_x"] = x + testPixelDeltas[:, 0] 

640 self.dataDict["someCentroid_y"] = y + testPixelDeltas[:, 1] 

641 self.dataDict["base_LocalWcs_CDMatrix_1_1"] = np.full(self.nRecords, 

642 linAffMatrix[0, 0]) 

643 self.dataDict["base_LocalWcs_CDMatrix_1_2"] = np.full(self.nRecords, 

644 linAffMatrix[0, 1]) 

645 self.dataDict["base_LocalWcs_CDMatrix_2_1"] = np.full(self.nRecords, 

646 linAffMatrix[1, 0]) 

647 self.dataDict["base_LocalWcs_CDMatrix_2_2"] = np.full(self.nRecords, 

648 linAffMatrix[1, 1]) 

649 parq = self.simulateMultiParquet(self.dataDict) 

650 func = LocalWcs("base_LocalWcs_CDMatrix_1_1", 

651 "base_LocalWcs_CDMatrix_1_2", 

652 "base_LocalWcs_CDMatrix_2_1", 

653 "base_LocalWcs_CDMatrix_2_2") 

654 df = parq.toDataFrame(columns={"dataset": "meas", 

655 "band": "g", 

656 "columns": ["dipoleSep", 

657 "slot_Centroid_x", 

658 "slot_Centroid_y", 

659 "someCentroid_x", 

660 "someCentroid_y", 

661 "base_LocalWcs_CDMatrix_1_1", 

662 "base_LocalWcs_CDMatrix_1_2", 

663 "base_LocalWcs_CDMatrix_2_1", 

664 "base_LocalWcs_CDMatrix_2_2"]}) 

665 

666 # Exercise the full set of functions in LocalWcs. 

667 sepRadians = func.getSkySeperationFromPixel( 

668 df[("meas", "g", "someCentroid_x")] - df[("meas", "g", "slot_Centroid_x")], 

669 df[("meas", "g", "someCentroid_y")] - df[("meas", "g", "slot_Centroid_y")], 

670 0.0, 

671 0.0, 

672 df[("meas", "g", "base_LocalWcs_CDMatrix_1_1")], 

673 df[("meas", "g", "base_LocalWcs_CDMatrix_1_2")], 

674 df[("meas", "g", "base_LocalWcs_CDMatrix_2_1")], 

675 df[("meas", "g", "base_LocalWcs_CDMatrix_2_2")]) 

676 

677 # Test functor values against afw SkyWcs computations. 

678 for centX, centY, sep in zip(testPixelDeltas[:, 0], 

679 testPixelDeltas[:, 1], 

680 sepRadians.values): 

681 afwSepRadians = skyOrigin.separation( 

682 wcs.pixelToSky(x + centX, y + centY)).asRadians() 

683 self.assertAlmostEqual(1 - sep / afwSepRadians, 0, places=6) 

684 

685 # Test the pixel scale computation. 

686 func = ComputePixelScale("base_LocalWcs_CDMatrix_1_1", 

687 "base_LocalWcs_CDMatrix_1_2", 

688 "base_LocalWcs_CDMatrix_2_1", 

689 "base_LocalWcs_CDMatrix_2_2") 

690 pixelScale = self._funcVal(func, parq) 

691 self.assertTrue(np.allclose( 

692 wcs.getPixelScale(center).asArcseconds(), 

693 pixelScale.values, 

694 rtol=1e-8, 

695 atol=0)) 

696 

697 # Test pixel -> arcsec conversion. 

698 func = ConvertPixelToArcseconds("dipoleSep", 

699 "base_LocalWcs_CDMatrix_1_1", 

700 "base_LocalWcs_CDMatrix_1_2", 

701 "base_LocalWcs_CDMatrix_2_1", 

702 "base_LocalWcs_CDMatrix_2_2") 

703 val = self._funcVal(func, parq) 

704 self.assertTrue(np.allclose(pixelScale.values * dipoleSep, 

705 val.values, 

706 atol=1e-16, 

707 rtol=1e-16)) 

708 

709 # Test pixel^2 -> arcsec^2 conversion. 

710 func = ConvertPixelSqToArcsecondsSq("ixx", 

711 "base_LocalWcs_CDMatrix_1_1", 

712 "base_LocalWcs_CDMatrix_1_2", 

713 "base_LocalWcs_CDMatrix_2_1", 

714 "base_LocalWcs_CDMatrix_2_2") 

715 val = self._funcVal(func, parq) 

716 self.assertTrue(np.allclose(pixelScale.values ** 2 * dipoleSep, 

717 val.values, 

718 atol=1e-16, 

719 rtol=1e-16)) 

720 

721 def _makeWcs(self, dec=53.1595451514076): 

722 """Create a wcs from real CFHT values. 

723 

724 Returns 

725 ------- 

726 wcs : `lsst.afw.geom` 

727 Created wcs. 

728 """ 

729 metadata = dafBase.PropertySet() 

730 

731 metadata.set("SIMPLE", "T") 

732 metadata.set("BITPIX", -32) 

733 metadata.set("NAXIS", 2) 

734 metadata.set("NAXIS1", 1024) 

735 metadata.set("NAXIS2", 1153) 

736 metadata.set("RADECSYS", 'FK5') 

737 metadata.set("EQUINOX", 2000.) 

738 

739 metadata.setDouble("CRVAL1", 215.604025685476) 

740 metadata.setDouble("CRVAL2", dec) 

741 metadata.setDouble("CRPIX1", 1109.99981456774) 

742 metadata.setDouble("CRPIX2", 560.018167811613) 

743 metadata.set("CTYPE1", 'RA---SIN') 

744 metadata.set("CTYPE2", 'DEC--SIN') 

745 

746 metadata.setDouble("CD1_1", 5.10808596133527E-05) 

747 metadata.setDouble("CD1_2", 1.85579539217196E-07) 

748 metadata.setDouble("CD2_2", -5.10281493481982E-05) 

749 metadata.setDouble("CD2_1", -8.27440751733828E-07) 

750 

751 return afwGeom.makeSkyWcs(metadata) 

752 

753 def testRatio(self): 

754 """Test the ratio functor where one of the functors should be junk. 

755 """ 

756 self.dataDict["base_PsfFlux_instFlux"] = np.full(self.nRecords, 1000) 

757 self.dataDict["base_PsfFlux_instFluxErr"] = np.full(self.nRecords, 100) 

758 parq = self.simulateMultiParquet(self.dataDict) 

759 

760 func = Ratio("base_PsfFlux_instFlux", "base_PsfFlux_instFluxErr") 

761 

762 val = self._funcVal(func, parq) 

763 self.assertTrue(np.allclose(np.full(self.nRecords, 10), 

764 val.values, 

765 atol=1e-16, 

766 rtol=1e-16)) 

767 

768 def testHtm(self): 

769 """Test that HtmIndxes are created as expected. 

770 """ 

771 parq = self.simulateMultiParquet(self.dataDict) 

772 func = HtmIndex20("coord_ra", "coord_dec") 

773 

774 val = self._funcVal(func, parq) 

775 # Test that the HtmIds come out as the ra/dec in dataDict. 

776 self.assertTrue(np.all(np.equal( 

777 val.values, 

778 [14924528684992, 14924528689697, 14924528501716, 14924526434259, 

779 14924526433879]))) 

780 

781 

782class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

783 pass 

784 

785 

786def setup_module(module): 

787 lsst.utils.tests.init() 

788 

789 

790if __name__ == "__main__": 790 ↛ 791line 790 didn't jump to line 791, because the condition on line 790 was never true

791 lsst.utils.tests.init() 

792 unittest.main()