Coverage for tests/test_isolatedStarAssociation.py: 14%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

215 statements  

1# This file is part of pipe_tasks. 

2# 

3# LSST Data Management System 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# See COPYRIGHT file at the top of the source tree. 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22"""Test IsolatedStarAssociationTask. 

23""" 

24import unittest 

25import numpy as np 

26import pandas as pd 

27 

28import lsst.utils.tests 

29import lsst.daf.butler 

30import lsst.skymap 

31 

32from lsst.pipe.tasks.isolatedStarAssociation import (IsolatedStarAssociationConfig, 

33 IsolatedStarAssociationTask) 

34from smatch.matcher import Matcher 

35 

36 

37class MockSourceTableReference(lsst.daf.butler.DeferredDatasetHandle): 

38 """Very simple object that looks like a Gen3 data reference to 

39 a sourceTable_visit. 

40 

41 Parameters 

42 ---------- 

43 source_table : `pandas.DataFrame` 

44 Dataframe of the source table. 

45 """ 

46 def __init__(self, source_table): 

47 self.source_table = source_table 

48 

49 def get(self, parameters={}, **kwargs): 

50 """Retrieve the specified dataset using the API of the Gen3 Butler. 

51 

52 Parameters 

53 ---------- 

54 parameters : `dict`, optional 

55 Parameter dictionary. Supported key is ``columns``. 

56 

57 Returns 

58 ------- 

59 dataframe : `pandas.DataFrame` 

60 dataframe, cut to the specified columns. 

61 """ 

62 if 'columns' in parameters: 

63 _columns = parameters['columns'] 

64 if 'sourceId' in parameters['columns']: 

65 # Treat the index separately 

66 _columns.remove('sourceId') 

67 

68 return self.source_table[_columns] 

69 else: 

70 return self.source_table.copy() 

71 

72 

73class IsolatedStarAssociationTestCase(lsst.utils.tests.TestCase): 

74 """Tests of IsolatedStarAssociationTask. 

75 

76 These tests bypass the middleware used for accessing data and 

77 managing Task execution. 

78 """ 

79 def setUp(self): 

80 self.skymap = self._make_skymap() 

81 self.tract = 9813 

82 self.data_refs = self._make_simdata(self.tract) 

83 self.visits = np.arange(len(self.data_refs)) + 1 

84 

85 self.data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits, 

86 self.data_refs)} 

87 

88 config = IsolatedStarAssociationConfig() 

89 config.band_order = ['i', 'r'] 

90 config.extra_columns = ['extra_column'] 

91 config.source_selector['science'].doFlags = False 

92 config.source_selector['science'].doIsolated = False 

93 

94 self.isolatedStarAssociationTask = IsolatedStarAssociationTask(config=config) 

95 

96 def _make_skymap(self): 

97 """Make a testing skymap.""" 

98 skymap_config = lsst.skymap.ringsSkyMap.RingsSkyMapConfig() 

99 skymap_config.numRings = 120 

100 skymap_config.projection = "TAN" 

101 skymap_config.tractOverlap = 1.0/60 

102 skymap_config.pixelScale = 0.168 

103 return lsst.skymap.ringsSkyMap.RingsSkyMap(skymap_config) 

104 

105 def _make_simdata(self, 

106 tract, 

107 only_neighbors=False, 

108 only_out_of_tract=False, 

109 only_out_of_inner_tract=False): 

110 """Make simulated data tables and references. 

111 

112 Parameters 

113 ---------- 

114 only_neighbors : `bool`, optional 

115 Only put in neighbors. 

116 only_out_of_tract : `bool`, optional 

117 All stars are out of the tract. 

118 only_out_of_inner_tract : `bool`, optional 

119 All stars are out of the inner tract. 

120 

121 Returns 

122 ------- 

123 data_refs : `list` [`MockSourceTableReference`] 

124 List of mock references. 

125 """ 

126 np.random.seed(12345) 

127 

128 n_visit_per_band = 5 

129 n_star_both = 50 

130 n_star_just_one = 5 

131 

132 tract_info = self.skymap[tract] 

133 ctr = tract_info.ctr_coord 

134 ctr_ra = ctr.getRa().asDegrees() 

135 ctr_dec = ctr.getDec().asDegrees() 

136 

137 ra_both = np.linspace(ctr_ra - 1.5, ctr_ra + 1.5, n_star_both) 

138 dec_both = np.linspace(ctr_dec - 1.5, ctr_dec + 1.5, n_star_both) 

139 

140 ra_just_r = np.linspace(ctr_ra - 0.5, ctr_ra + 0.5, n_star_just_one) 

141 dec_just_r = np.linspace(ctr_dec + 0.2, ctr_dec + 0.2, n_star_just_one) 

142 ra_just_i = np.linspace(ctr_ra - 0.5, ctr_ra + 0.5, n_star_just_one) 

143 dec_just_i = np.linspace(ctr_dec - 0.2, ctr_dec - 0.2, n_star_just_one) 

144 

145 ra_neighbor = np.array([ra_both[n_star_both//2] + 1./3600.]) 

146 dec_neighbor = np.array([dec_both[n_star_both//2] + 1./3600.]) 

147 

148 # Create the r-band datarefs 

149 dtype = [('sourceId', 'i8'), 

150 ('ra', 'f8'), 

151 ('decl', 'f8'), 

152 ('apFlux_12_0_instFlux', 'f4'), 

153 ('apFlux_12_0_instFluxErr', 'f4'), 

154 ('apFlux_12_0_instFlux_flag', '?'), 

155 ('extendedness', 'f4'), 

156 ('visit', 'i4'), 

157 ('detector', 'i4'), 

158 ('physical_filter', 'U10'), 

159 ('band', 'U2'), 

160 ('extra_column', 'f4')] 

161 

162 id_counter = 0 

163 visit_counter = 1 

164 

165 data_refs = [] 

166 for band in ['r', 'i']: 

167 if band == 'r': 

168 filtername = 'R FILTER' 

169 ra_just = ra_just_r 

170 dec_just = dec_just_r 

171 else: 

172 filtername = 'I FILTER' 

173 ra_just = ra_just_i 

174 dec_just = dec_just_i 

175 

176 if only_neighbors: 

177 star_ra = np.concatenate(([ra_both[n_star_both//2]], ra_neighbor)) 

178 star_dec = np.concatenate(([dec_both[n_star_both//2]], dec_neighbor)) 

179 else: 

180 star_ra = np.concatenate((ra_both, ra_neighbor, ra_just)) 

181 star_dec = np.concatenate((dec_both, dec_neighbor, dec_just)) 

182 

183 if only_out_of_tract: 

184 poly = self.skymap[self.tract].outer_sky_polygon 

185 use = ~poly.contains(np.deg2rad(star_ra), np.deg2rad(star_dec)) 

186 star_ra = star_ra[use] 

187 star_dec = star_dec[use] 

188 elif only_out_of_inner_tract: 

189 inner_tract_ids = self.skymap.findTractIdArray(star_ra, star_dec, degrees=True) 

190 use = (inner_tract_ids != self.tract) 

191 star_ra = star_ra[use] 

192 star_dec = star_dec[use] 

193 

194 nstar = len(star_ra) 

195 

196 for i in range(n_visit_per_band): 

197 ras = np.random.normal(loc=star_ra, scale=0.2/3600.) 

198 decs = np.random.normal(loc=star_dec, scale=0.2/3600.) 

199 

200 table = np.zeros(nstar, dtype=dtype) 

201 table['sourceId'] = np.arange(nstar) + id_counter 

202 table['ra'] = ras 

203 table['decl'] = decs 

204 table['apFlux_12_0_instFlux'] = 100.0 

205 table['apFlux_12_0_instFluxErr'] = 1.0 

206 table['physical_filter'] = filtername 

207 table['band'] = band 

208 table['extra_column'] = np.ones(nstar) 

209 table['visit'] = visit_counter 

210 table['detector'] = 1 

211 

212 if i == 0: 

213 # Make one star have low s/n 

214 table['apFlux_12_0_instFlux'][0] = 1.0 

215 

216 df = pd.DataFrame(table) 

217 df.set_index('sourceId', inplace=True) 

218 data_refs.append(MockSourceTableReference(df)) 

219 

220 id_counter += nstar 

221 visit_counter += 1 

222 

223 self.n_visit_per_band = n_visit_per_band 

224 self.n_star_both = n_star_both 

225 self.n_star_just_one = n_star_just_one 

226 if only_neighbors: 

227 self.star_ras = np.concatenate(([ra_both[n_star_both//2]], ra_neighbor)) 

228 self.star_decs = np.concatenate(([dec_both[n_star_both//2]], dec_neighbor)) 

229 else: 

230 self.star_ras = np.concatenate((ra_both, ra_just_r, ra_just_i, ra_neighbor)) 

231 self.star_decs = np.concatenate((dec_both, dec_just_r, dec_just_i, dec_neighbor)) 

232 

233 return data_refs 

234 

235 def test_compute_unique_ids(self): 

236 """Test computation of unique ids.""" 

237 ids1 = self.isolatedStarAssociationTask._compute_unique_ids(self.skymap, 

238 9813, 

239 10000) 

240 ids2 = self.isolatedStarAssociationTask._compute_unique_ids(self.skymap, 

241 9814, 

242 5000) 

243 ids = np.concatenate((ids1, ids2)) 

244 self.assertEqual(len(np.unique(ids)), len(ids)) 

245 

246 def test_remove_neighbors(self): 

247 """Test removing close neighbors.""" 

248 primary_star_cat = np.zeros(3, dtype=[('ra', 'f8'), 

249 ('decl', 'f8')]) 

250 

251 # Put two stars < 2" apart, and across the 0/360 boundary. 

252 primary_star_cat['ra'] = [0.7/3600., 360.0 - 0.7/3600., 1.0] 

253 primary_star_cat['decl'] = [5.0, 5.0, 5.0] 

254 

255 cut_cat = self.isolatedStarAssociationTask._remove_neighbors(primary_star_cat) 

256 

257 self.assertEqual(len(cut_cat), 1) 

258 np.testing.assert_almost_equal(1.0, cut_cat['ra'][0]) 

259 

260 def test_match_primary_stars(self): 

261 """Test matching primary stars.""" 

262 # Stack all the sources; we do not want any cutting here. 

263 tables = [] 

264 for data_ref in self.data_refs: 

265 df = data_ref.get() 

266 tables.append(df.to_records()) 

267 source_cat = np.concatenate(tables) 

268 

269 primary_star_cat = self.isolatedStarAssociationTask._match_primary_stars(['i', 'r'], 

270 source_cat) 

271 

272 # Ensure we found the right number of stars in each p 

273 test_i = (primary_star_cat['primary_band'] == 'i') 

274 self.assertEqual(test_i.sum(), self.n_star_both + self.n_star_just_one + 1) 

275 test_r = (primary_star_cat['primary_band'] == 'r') 

276 self.assertEqual(test_r.sum(), self.n_star_just_one) 

277 

278 # Ensure that these stars all match to input stars within 1 arcsec. 

279 with Matcher(self.star_ras, self.star_decs) as matcher: 

280 idx, i1, i2, d = matcher.query_radius(primary_star_cat['ra'], 

281 primary_star_cat['decl'], 

282 1./3600., 

283 return_indices=True) 

284 self.assertEqual(i1.size, self.star_ras.size) 

285 

286 def test_get_source_table_visit_columns(self): 

287 """Test making of source table visit columns.""" 

288 all_columns, persist_columns = self.isolatedStarAssociationTask._get_source_table_visit_column_names() 

289 

290 # Make sure all persisted columns are in all columns. 

291 for col in persist_columns: 

292 self.assertTrue(col in all_columns) 

293 

294 # And make sure extendedness is not in persisted columns. 

295 self.assertTrue('extendedness' not in persist_columns) 

296 

297 def test_match_sources(self): 

298 """Test _match_sources source to primary matching.""" 

299 # Stack all the sources; we do not want any cutting here. 

300 tables = [] 

301 for data_ref in self.data_refs: 

302 df = data_ref.get() 

303 tables.append(df.to_records()) 

304 source_cat = np.concatenate(tables) 

305 

306 source_cat = np.lib.recfunctions.append_fields(source_cat, 

307 ['obj_index'], 

308 [np.zeros(source_cat.size, dtype=np.int32)], 

309 dtypes=['i4'], 

310 usemask=False) 

311 

312 primary_bands = ['i', 'r'] 

313 

314 primary_cat = np.zeros(self.star_ras.size, 

315 dtype=self.isolatedStarAssociationTask._get_primary_dtype(primary_bands)) 

316 primary_cat['ra'] = self.star_ras 

317 primary_cat['decl'] = self.star_decs 

318 

319 source_cat_sorted, primary_star_cat = self.isolatedStarAssociationTask._match_sources(['i', 'r'], 

320 source_cat, 

321 primary_cat) 

322 # All the star sources should be matched 

323 self.assertEqual(source_cat_sorted.size, source_cat.size) 

324 

325 # Full index tests are performed in test_run_isolated_star_association_task 

326 

327 def test_make_all_star_sources(self): 

328 """Test appending all the star sources.""" 

329 source_cat = self.isolatedStarAssociationTask._make_all_star_sources(self.skymap[self.tract], 

330 self.data_ref_dict) 

331 

332 # Make sure we don't have any low s/n sources. 

333 sn_min = np.min(source_cat['apFlux_12_0_instFlux']/source_cat['apFlux_12_0_instFluxErr']) 

334 self.assertGreater(sn_min, 10.0) 

335 

336 # And make sure they are all within the tract outer boundary. 

337 poly = self.skymap[self.tract].outer_sky_polygon 

338 use = poly.contains(np.deg2rad(source_cat['ra']), np.deg2rad(source_cat['decl'])) 

339 self.assertEqual(use.sum(), len(source_cat)) 

340 

341 def test_run_isolated_star_association_task(self): 

342 """Test running the full task.""" 

343 struct = self.isolatedStarAssociationTask.run(self.skymap, 

344 self.tract, 

345 self.data_ref_dict) 

346 

347 star_source_cat = struct.star_source_cat 

348 star_cat = struct.star_cat 

349 

350 # Check that sources are all unique ids 

351 self.assertEqual(np.unique(star_source_cat['sourceId']).size, star_source_cat.size) 

352 

353 inner_tract_ids = self.skymap.findTractIdArray(self.star_ras, 

354 self.star_decs, 

355 degrees=True) 

356 inner_stars = (inner_tract_ids == self.tract) 

357 

358 # There should be the same number of stars in the inner tract region, 

359 # taking away the 2 close neighbors. 

360 self.assertEqual(star_cat.size, np.sum(inner_stars) - 2) 

361 

362 # Check the star indices 

363 for i in range(len(star_cat)): 

364 all_source_star = star_source_cat[star_cat['source_cat_index'][i]: 

365 star_cat['source_cat_index'][i] + star_cat['nsource'][i]] 

366 

367 # Check that these all point to the correct object 

368 np.testing.assert_array_equal(all_source_star['obj_index'], i) 

369 

370 # Check these are all pointing to the same star position 

371 with Matcher(np.atleast_1d(star_cat['ra'][i]), 

372 np.atleast_1d(star_cat['decl'][i])) as matcher: 

373 idx = matcher.query_radius(all_source_star['ra'], 

374 all_source_star['decl'], 

375 1./3600.) 

376 self.assertEqual(len(idx[0]), star_cat['nsource'][i]) 

377 

378 # Check per band indices 

379 for band in ['r', 'i']: 

380 band_source_star = star_source_cat[star_cat[f'source_cat_index_{band}'][i]: 

381 star_cat[f'source_cat_index_{band}'][i] 

382 + star_cat[f'nsource_{band}'][i]] 

383 with Matcher(np.atleast_1d(star_cat['ra'][i]), 

384 np.atleast_1d(star_cat['decl'][i])) as matcher: 

385 idx = matcher.query_radius(band_source_star['ra'], 

386 band_source_star['decl'], 

387 1./3600.) 

388 self.assertEqual(len(idx[0]), star_cat[f'nsource_{band}'][i]) 

389 

390 def test_run_task_all_neighbors(self): 

391 """Test running the task when all the stars are rejected as neighbors.""" 

392 data_refs = self._make_simdata(self.tract, only_neighbors=True) 

393 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits, 

394 data_refs)} 

395 

396 struct = self.isolatedStarAssociationTask.run(self.skymap, 

397 self.tract, 

398 data_ref_dict) 

399 

400 # These should ber zero length. 

401 self.assertEqual(len(struct.star_source_cat), 0) 

402 self.assertEqual(len(struct.star_cat), 0) 

403 # And spot-check a couple of expected fields to make sure they have the right type. 

404 self.assertTrue('physical_filter' in struct.star_source_cat.dtype.names) 

405 self.assertTrue('nsource_i' in struct.star_cat.dtype.names) 

406 

407 def test_run_task_all_out_of_tract(self): 

408 """Test running the task when all the sources are out of the tract.""" 

409 data_refs = self._make_simdata(self.tract, only_out_of_tract=True) 

410 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits, 

411 data_refs)} 

412 

413 struct = self.isolatedStarAssociationTask.run(self.skymap, 

414 self.tract, 

415 data_ref_dict) 

416 

417 # These should ber zero length. 

418 self.assertEqual(len(struct.star_source_cat), 0) 

419 self.assertEqual(len(struct.star_cat), 0) 

420 # And spot-check a couple of expected fields to make sure they have the right type. 

421 self.assertTrue('physical_filter' in struct.star_source_cat.dtype.names) 

422 self.assertTrue('nsource_i' in struct.star_cat.dtype.names) 

423 

424 def test_run_task_all_out_of_inner_tract(self): 

425 """Test running the task when all the sources are out of the inner tract.""" 

426 data_refs = self._make_simdata(self.tract, only_out_of_inner_tract=True) 

427 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits, 

428 data_refs)} 

429 

430 struct = self.isolatedStarAssociationTask.run(self.skymap, 

431 self.tract, 

432 data_ref_dict) 

433 

434 # These should ber zero length. 

435 self.assertEqual(len(struct.star_source_cat), 0) 

436 self.assertEqual(len(struct.star_cat), 0) 

437 # And spot-check a couple of expected fields to make sure they have the right type. 

438 self.assertTrue('physical_filter' in struct.star_source_cat.dtype.names) 

439 self.assertTrue('nsource_i' in struct.star_cat.dtype.names) 

440 

441 

442class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

443 pass 

444 

445 

446def setup_module(module): 

447 lsst.utils.tests.init() 

448 

449 

450if __name__ == "__main__": 450 ↛ 451line 450 didn't jump to line 451, because the condition on line 450 was never true

451 lsst.utils.tests.init() 

452 unittest.main()