Coverage for tests/test_isolatedStarAssociation.py: 14%

224 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-08-26 03:18 -0700

1# This file is part of pipe_tasks. 

2# 

3# LSST Data Management System 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# See COPYRIGHT file at the top of the source tree. 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22"""Test IsolatedStarAssociationTask. 

23""" 

24import unittest 

25import numpy as np 

26import pandas as pd 

27 

28import lsst.utils.tests 

29import lsst.daf.butler 

30import lsst.skymap 

31 

32from lsst.pipe.tasks.isolatedStarAssociation import (IsolatedStarAssociationConfig, 

33 IsolatedStarAssociationTask) 

34from smatch.matcher import Matcher 

35 

36 

37class MockSourceTableReference(lsst.daf.butler.DeferredDatasetHandle): 

38 """Very simple object that looks like a Gen3 data reference to 

39 a sourceTable_visit. 

40 

41 Parameters 

42 ---------- 

43 source_table : `pandas.DataFrame` 

44 Dataframe of the source table. 

45 """ 

46 def __init__(self, source_table): 

47 self.source_table = source_table 

48 

49 def get(self, parameters={}, **kwargs): 

50 """Retrieve the specified dataset using the API of the Gen3 Butler. 

51 

52 Parameters 

53 ---------- 

54 parameters : `dict`, optional 

55 Parameter dictionary. Supported key is ``columns``. 

56 

57 Returns 

58 ------- 

59 dataframe : `pandas.DataFrame` 

60 dataframe, cut to the specified columns. 

61 """ 

62 if 'columns' in parameters: 

63 _columns = parameters['columns'] 

64 if 'sourceId' in parameters['columns']: 

65 # Treat the index separately 

66 _columns.remove('sourceId') 

67 

68 return self.source_table[_columns] 

69 else: 

70 return self.source_table.copy() 

71 

72 

73class IsolatedStarAssociationTestCase(lsst.utils.tests.TestCase): 

74 """Tests of IsolatedStarAssociationTask. 

75 

76 These tests bypass the middleware used for accessing data and 

77 managing Task execution. 

78 """ 

79 def setUp(self): 

80 self.skymap = self._make_skymap() 

81 self.tract = 9813 

82 self.data_refs = self._make_simdata(self.tract) 

83 self.visits = np.arange(len(self.data_refs)) + 1 

84 

85 self.data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits, 

86 self.data_refs)} 

87 

88 config = IsolatedStarAssociationConfig() 

89 config.band_order = ['i', 'r'] 

90 config.extra_columns = ['extra_column'] 

91 config.source_selector['science'].doFlags = False 

92 config.source_selector['science'].doIsolated = False 

93 

94 self.isolatedStarAssociationTask = IsolatedStarAssociationTask(config=config) 

95 

96 def _make_skymap(self): 

97 """Make a testing skymap.""" 

98 skymap_config = lsst.skymap.ringsSkyMap.RingsSkyMapConfig() 

99 skymap_config.numRings = 120 

100 skymap_config.projection = "TAN" 

101 skymap_config.tractOverlap = 1.0/60 

102 skymap_config.pixelScale = 0.168 

103 return lsst.skymap.ringsSkyMap.RingsSkyMap(skymap_config) 

104 

105 def _make_simdata(self, 

106 tract, 

107 only_neighbors=False, 

108 only_out_of_tract=False, 

109 only_out_of_inner_tract=False, 

110 no_secondary_overlap=False): 

111 """Make simulated data tables and references. 

112 

113 Parameters 

114 ---------- 

115 only_neighbors : `bool`, optional 

116 Only put in neighbors. 

117 only_out_of_tract : `bool`, optional 

118 All stars are out of the tract. 

119 only_out_of_inner_tract : `bool`, optional 

120 All stars are out of the inner tract. 

121 no_secondary_overlap : `bool`, optional 

122 Secondary band has no overlap with the primary band. 

123 

124 Returns 

125 ------- 

126 data_refs : `list` [`MockSourceTableReference`] 

127 List of mock references. 

128 """ 

129 np.random.seed(12345) 

130 

131 n_visit_per_band = 5 

132 n_star_both = 50 

133 n_star_just_one = 5 

134 

135 tract_info = self.skymap[tract] 

136 ctr = tract_info.ctr_coord 

137 ctr_ra = ctr.getRa().asDegrees() 

138 ctr_dec = ctr.getDec().asDegrees() 

139 

140 ra_both = np.linspace(ctr_ra - 1.5, ctr_ra + 1.5, n_star_both) 

141 dec_both = np.linspace(ctr_dec - 1.5, ctr_dec + 1.5, n_star_both) 

142 

143 ra_just_r = np.linspace(ctr_ra - 0.5, ctr_ra + 0.5, n_star_just_one) 

144 dec_just_r = np.linspace(ctr_dec + 0.2, ctr_dec + 0.2, n_star_just_one) 

145 ra_just_i = np.linspace(ctr_ra - 0.5, ctr_ra + 0.5, n_star_just_one) 

146 dec_just_i = np.linspace(ctr_dec - 0.2, ctr_dec - 0.2, n_star_just_one) 

147 

148 ra_neighbor = np.array([ra_both[n_star_both//2] + 1./3600.]) 

149 dec_neighbor = np.array([dec_both[n_star_both//2] + 1./3600.]) 

150 

151 # Create the r-band datarefs 

152 dtype = [('sourceId', 'i8'), 

153 ('ra', 'f8'), 

154 ('decl', 'f8'), 

155 ('apFlux_12_0_instFlux', 'f4'), 

156 ('apFlux_12_0_instFluxErr', 'f4'), 

157 ('apFlux_12_0_instFlux_flag', '?'), 

158 ('extendedness', 'f4'), 

159 ('visit', 'i4'), 

160 ('detector', 'i4'), 

161 ('physical_filter', 'U10'), 

162 ('band', 'U2'), 

163 ('extra_column', 'f4')] 

164 

165 id_counter = 0 

166 visit_counter = 1 

167 

168 data_refs = [] 

169 for band in ['r', 'i']: 

170 if band == 'r': 

171 filtername = 'R FILTER' 

172 ra_just = ra_just_r 

173 dec_just = dec_just_r 

174 else: 

175 filtername = 'I FILTER' 

176 ra_just = ra_just_i 

177 dec_just = dec_just_i 

178 

179 if only_neighbors: 

180 star_ra = np.concatenate(([ra_both[n_star_both//2]], ra_neighbor)) 

181 star_dec = np.concatenate(([dec_both[n_star_both//2]], dec_neighbor)) 

182 elif no_secondary_overlap: 

183 star_ra = np.concatenate((ra_just,)) 

184 star_dec = np.concatenate((dec_just,)) 

185 else: 

186 star_ra = np.concatenate((ra_both, ra_neighbor, ra_just)) 

187 star_dec = np.concatenate((dec_both, dec_neighbor, dec_just)) 

188 

189 if only_out_of_tract: 

190 poly = self.skymap[self.tract].outer_sky_polygon 

191 use = ~poly.contains(np.deg2rad(star_ra), np.deg2rad(star_dec)) 

192 star_ra = star_ra[use] 

193 star_dec = star_dec[use] 

194 elif only_out_of_inner_tract: 

195 inner_tract_ids = self.skymap.findTractIdArray(star_ra, star_dec, degrees=True) 

196 use = (inner_tract_ids != self.tract) 

197 star_ra = star_ra[use] 

198 star_dec = star_dec[use] 

199 

200 nstar = len(star_ra) 

201 

202 for i in range(n_visit_per_band): 

203 ras = np.random.normal(loc=star_ra, scale=0.2/3600.) 

204 decs = np.random.normal(loc=star_dec, scale=0.2/3600.) 

205 

206 table = np.zeros(nstar, dtype=dtype) 

207 table['sourceId'] = np.arange(nstar) + id_counter 

208 table['ra'] = ras 

209 table['decl'] = decs 

210 table['apFlux_12_0_instFlux'] = 100.0 

211 table['apFlux_12_0_instFluxErr'] = 1.0 

212 table['physical_filter'] = filtername 

213 table['band'] = band 

214 table['extra_column'] = np.ones(nstar) 

215 table['visit'] = visit_counter 

216 table['detector'] = 1 

217 

218 if i == 0: 

219 # Make one star have low s/n 

220 table['apFlux_12_0_instFlux'][0] = 1.0 

221 

222 df = pd.DataFrame(table) 

223 df.set_index('sourceId', inplace=True) 

224 data_refs.append(MockSourceTableReference(df)) 

225 

226 id_counter += nstar 

227 visit_counter += 1 

228 

229 self.n_visit_per_band = n_visit_per_band 

230 self.n_star_both = n_star_both 

231 self.n_star_just_one = n_star_just_one 

232 if only_neighbors: 

233 self.star_ras = np.concatenate(([ra_both[n_star_both//2]], ra_neighbor)) 

234 self.star_decs = np.concatenate(([dec_both[n_star_both//2]], dec_neighbor)) 

235 else: 

236 self.star_ras = np.concatenate((ra_both, ra_just_r, ra_just_i, ra_neighbor)) 

237 self.star_decs = np.concatenate((dec_both, dec_just_r, dec_just_i, dec_neighbor)) 

238 

239 return data_refs 

240 

241 def test_compute_unique_ids(self): 

242 """Test computation of unique ids.""" 

243 ids1 = self.isolatedStarAssociationTask._compute_unique_ids(self.skymap, 

244 9813, 

245 10000) 

246 ids2 = self.isolatedStarAssociationTask._compute_unique_ids(self.skymap, 

247 9814, 

248 5000) 

249 ids = np.concatenate((ids1, ids2)) 

250 self.assertEqual(len(np.unique(ids)), len(ids)) 

251 

252 def test_remove_neighbors(self): 

253 """Test removing close neighbors.""" 

254 primary_star_cat = np.zeros(3, dtype=[('ra', 'f8'), 

255 ('decl', 'f8')]) 

256 

257 # Put two stars < 2" apart, and across the 0/360 boundary. 

258 primary_star_cat['ra'] = [0.7/3600., 360.0 - 0.7/3600., 1.0] 

259 primary_star_cat['decl'] = [5.0, 5.0, 5.0] 

260 

261 cut_cat = self.isolatedStarAssociationTask._remove_neighbors(primary_star_cat) 

262 

263 self.assertEqual(len(cut_cat), 1) 

264 np.testing.assert_almost_equal(1.0, cut_cat['ra'][0]) 

265 

266 def test_match_primary_stars(self): 

267 """Test matching primary stars.""" 

268 # Stack all the sources; we do not want any cutting here. 

269 tables = [] 

270 for data_ref in self.data_refs: 

271 df = data_ref.get() 

272 tables.append(df.to_records()) 

273 source_cat = np.concatenate(tables) 

274 

275 primary_star_cat = self.isolatedStarAssociationTask._match_primary_stars(['i', 'r'], 

276 source_cat) 

277 

278 # Ensure we found the right number of stars in each p 

279 test_i = (primary_star_cat['primary_band'] == 'i') 

280 self.assertEqual(test_i.sum(), self.n_star_both + self.n_star_just_one + 1) 

281 test_r = (primary_star_cat['primary_band'] == 'r') 

282 self.assertEqual(test_r.sum(), self.n_star_just_one) 

283 

284 # Ensure that these stars all match to input stars within 1 arcsec. 

285 with Matcher(self.star_ras, self.star_decs) as matcher: 

286 idx, i1, i2, d = matcher.query_radius(primary_star_cat['ra'], 

287 primary_star_cat['decl'], 

288 1./3600., 

289 return_indices=True) 

290 self.assertEqual(i1.size, self.star_ras.size) 

291 

292 def test_get_source_table_visit_columns(self): 

293 """Test making of source table visit columns.""" 

294 all_columns, persist_columns = self.isolatedStarAssociationTask._get_source_table_visit_column_names() 

295 

296 # Make sure all persisted columns are in all columns. 

297 for col in persist_columns: 

298 self.assertTrue(col in all_columns) 

299 

300 # And make sure extendedness is not in persisted columns. 

301 self.assertTrue('extendedness' not in persist_columns) 

302 

303 def test_match_sources(self): 

304 """Test _match_sources source to primary matching.""" 

305 # Stack all the sources; we do not want any cutting here. 

306 tables = [] 

307 for data_ref in self.data_refs: 

308 df = data_ref.get() 

309 tables.append(df.to_records()) 

310 source_cat = np.concatenate(tables) 

311 

312 source_cat = np.lib.recfunctions.append_fields(source_cat, 

313 ['obj_index'], 

314 [np.zeros(source_cat.size, dtype=np.int32)], 

315 dtypes=['i4'], 

316 usemask=False) 

317 

318 primary_bands = ['i', 'r'] 

319 

320 primary_cat = np.zeros(self.star_ras.size, 

321 dtype=self.isolatedStarAssociationTask._get_primary_dtype(primary_bands)) 

322 primary_cat['ra'] = self.star_ras 

323 primary_cat['decl'] = self.star_decs 

324 

325 source_cat_sorted, primary_star_cat = self.isolatedStarAssociationTask._match_sources(['i', 'r'], 

326 source_cat, 

327 primary_cat) 

328 # All the star sources should be matched 

329 self.assertEqual(source_cat_sorted.size, source_cat.size) 

330 

331 # Full index tests are performed in test_run_isolated_star_association_task 

332 

333 def test_make_all_star_sources(self): 

334 """Test appending all the star sources.""" 

335 source_cat = self.isolatedStarAssociationTask._make_all_star_sources(self.skymap[self.tract], 

336 self.data_ref_dict) 

337 

338 # Make sure we don't have any low s/n sources. 

339 sn_min = np.min(source_cat['apFlux_12_0_instFlux']/source_cat['apFlux_12_0_instFluxErr']) 

340 self.assertGreater(sn_min, 10.0) 

341 

342 # And make sure they are all within the tract outer boundary. 

343 poly = self.skymap[self.tract].outer_sky_polygon 

344 use = poly.contains(np.deg2rad(source_cat['ra']), np.deg2rad(source_cat['decl'])) 

345 self.assertEqual(use.sum(), len(source_cat)) 

346 

347 def test_run_isolated_star_association_task(self): 

348 """Test running the full task.""" 

349 struct = self.isolatedStarAssociationTask.run(self.skymap, 

350 self.tract, 

351 self.data_ref_dict) 

352 

353 star_source_cat = struct.star_source_cat 

354 star_cat = struct.star_cat 

355 

356 # Check that sources are all unique ids 

357 self.assertEqual(np.unique(star_source_cat['sourceId']).size, star_source_cat.size) 

358 

359 inner_tract_ids = self.skymap.findTractIdArray(self.star_ras, 

360 self.star_decs, 

361 degrees=True) 

362 inner_stars = (inner_tract_ids == self.tract) 

363 

364 # There should be the same number of stars in the inner tract region, 

365 # taking away the 2 close neighbors. 

366 self.assertEqual(star_cat.size, np.sum(inner_stars) - 2) 

367 

368 # Check the star indices 

369 for i in range(len(star_cat)): 

370 all_source_star = star_source_cat[star_cat['source_cat_index'][i]: 

371 star_cat['source_cat_index'][i] + star_cat['nsource'][i]] 

372 

373 # Check that these all point to the correct object 

374 np.testing.assert_array_equal(all_source_star['obj_index'], i) 

375 

376 # Check these are all pointing to the same star position 

377 with Matcher(np.atleast_1d(star_cat['ra'][i]), 

378 np.atleast_1d(star_cat['decl'][i])) as matcher: 

379 idx = matcher.query_radius(all_source_star['ra'], 

380 all_source_star['decl'], 

381 1./3600.) 

382 self.assertEqual(len(idx[0]), star_cat['nsource'][i]) 

383 

384 # Check per band indices 

385 for band in ['r', 'i']: 

386 band_source_star = star_source_cat[star_cat[f'source_cat_index_{band}'][i]: 

387 star_cat[f'source_cat_index_{band}'][i] 

388 + star_cat[f'nsource_{band}'][i]] 

389 with Matcher(np.atleast_1d(star_cat['ra'][i]), 

390 np.atleast_1d(star_cat['decl'][i])) as matcher: 

391 idx = matcher.query_radius(band_source_star['ra'], 

392 band_source_star['decl'], 

393 1./3600.) 

394 self.assertEqual(len(idx[0]), star_cat[f'nsource_{band}'][i]) 

395 

396 def test_run_task_all_neighbors(self): 

397 """Test running the task when all the stars are rejected as neighbors.""" 

398 data_refs = self._make_simdata(self.tract, only_neighbors=True) 

399 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits, 

400 data_refs)} 

401 

402 struct = self.isolatedStarAssociationTask.run(self.skymap, 

403 self.tract, 

404 data_ref_dict) 

405 

406 # These should ber zero length. 

407 self.assertEqual(len(struct.star_source_cat), 0) 

408 self.assertEqual(len(struct.star_cat), 0) 

409 # And spot-check a couple of expected fields to make sure they have the right type. 

410 self.assertTrue('physical_filter' in struct.star_source_cat.dtype.names) 

411 self.assertTrue('nsource_i' in struct.star_cat.dtype.names) 

412 

413 def test_run_task_all_out_of_tract(self): 

414 """Test running the task when all the sources are out of the tract.""" 

415 data_refs = self._make_simdata(self.tract, only_out_of_tract=True) 

416 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits, 

417 data_refs)} 

418 

419 struct = self.isolatedStarAssociationTask.run(self.skymap, 

420 self.tract, 

421 data_ref_dict) 

422 

423 # These should ber zero length. 

424 self.assertEqual(len(struct.star_source_cat), 0) 

425 self.assertEqual(len(struct.star_cat), 0) 

426 # And spot-check a couple of expected fields to make sure they have the right type. 

427 self.assertTrue('physical_filter' in struct.star_source_cat.dtype.names) 

428 self.assertTrue('nsource_i' in struct.star_cat.dtype.names) 

429 

430 def test_run_task_all_out_of_inner_tract(self): 

431 """Test running the task when all the sources are out of the inner tract.""" 

432 data_refs = self._make_simdata(self.tract, only_out_of_inner_tract=True) 

433 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits, 

434 data_refs)} 

435 

436 struct = self.isolatedStarAssociationTask.run(self.skymap, 

437 self.tract, 

438 data_ref_dict) 

439 

440 # These should ber zero length. 

441 self.assertEqual(len(struct.star_source_cat), 0) 

442 self.assertEqual(len(struct.star_cat), 0) 

443 # And spot-check a couple of expected fields to make sure they have the right type. 

444 self.assertTrue('physical_filter' in struct.star_source_cat.dtype.names) 

445 self.assertTrue('nsource_i' in struct.star_cat.dtype.names) 

446 

447 def test_run_task_secondary_no_overlap(self): 

448 """Test running the task when the secondary band has no overlaps. 

449 

450 This tests DM-34834. 

451 """ 

452 data_refs = self._make_simdata(self.tract, no_secondary_overlap=True) 

453 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits, 

454 data_refs)} 

455 

456 struct = self.isolatedStarAssociationTask.run(self.skymap, 

457 self.tract, 

458 data_ref_dict) 

459 

460 # Add a sanity check that we got a catalog out. 

461 self.assertGreater(len(struct.star_source_cat), 0) 

462 self.assertGreater(len(struct.star_cat), 0) 

463 

464 

465class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

466 pass 

467 

468 

469def setup_module(module): 

470 lsst.utils.tests.init() 

471 

472 

473if __name__ == "__main__": 473 ↛ 474line 473 didn't jump to line 474, because the condition on line 473 was never true

474 lsst.utils.tests.init() 

475 unittest.main()