Coverage for tests/test_isolatedStarAssociation.py: 11%

215 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-04-27 03:59 -0700

1# This file is part of pipe_tasks. 

2# 

3# LSST Data Management System 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# See COPYRIGHT file at the top of the source tree. 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22"""Test IsolatedStarAssociationTask. 

23""" 

24import unittest 

25import numpy as np 

26import pandas as pd 

27 

28import lsst.utils.tests 

29import lsst.pipe.base 

30import lsst.skymap 

31 

32from lsst.pipe.tasks.isolatedStarAssociation import (IsolatedStarAssociationConfig, 

33 IsolatedStarAssociationTask) 

34from smatch.matcher import Matcher 

35 

36 

37class IsolatedStarAssociationTestCase(lsst.utils.tests.TestCase): 

38 """Tests of IsolatedStarAssociationTask. 

39 

40 These tests bypass the middleware used for accessing data and 

41 managing Task execution. 

42 """ 

43 def setUp(self): 

44 self.skymap = self._make_skymap() 

45 self.tract = 9813 

46 self.data_refs = self._make_simdata(self.tract) 

47 self.visits = np.arange(len(self.data_refs)) + 1 

48 

49 self.data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits, 

50 self.data_refs)} 

51 

52 config = IsolatedStarAssociationConfig() 

53 config.band_order = ['i', 'r'] 

54 config.extra_columns = ['extra_column'] 

55 config.source_selector['science'].doFlags = False 

56 config.source_selector['science'].doIsolated = False 

57 

58 self.isolatedStarAssociationTask = IsolatedStarAssociationTask(config=config) 

59 

60 def _make_skymap(self): 

61 """Make a testing skymap.""" 

62 skymap_config = lsst.skymap.ringsSkyMap.RingsSkyMapConfig() 

63 skymap_config.numRings = 120 

64 skymap_config.projection = "TAN" 

65 skymap_config.tractOverlap = 1.0/60 

66 skymap_config.pixelScale = 0.168 

67 return lsst.skymap.ringsSkyMap.RingsSkyMap(skymap_config) 

68 

69 def _make_simdata(self, 

70 tract, 

71 only_neighbors=False, 

72 only_out_of_tract=False, 

73 only_out_of_inner_tract=False, 

74 no_secondary_overlap=False): 

75 """Make simulated data tables and references. 

76 

77 Parameters 

78 ---------- 

79 only_neighbors : `bool`, optional 

80 Only put in neighbors. 

81 only_out_of_tract : `bool`, optional 

82 All stars are out of the tract. 

83 only_out_of_inner_tract : `bool`, optional 

84 All stars are out of the inner tract. 

85 no_secondary_overlap : `bool`, optional 

86 Secondary band has no overlap with the primary band. 

87 

88 Returns 

89 ------- 

90 data_refs : `list` [`InMemoryDatasetHandle`] 

91 List of mock references. 

92 """ 

93 np.random.seed(12345) 

94 

95 n_visit_per_band = 5 

96 n_star_both = 50 

97 n_star_just_one = 5 

98 

99 tract_info = self.skymap[tract] 

100 ctr = tract_info.ctr_coord 

101 ctr_ra = ctr.getRa().asDegrees() 

102 ctr_dec = ctr.getDec().asDegrees() 

103 

104 ra_both = np.linspace(ctr_ra - 1.5, ctr_ra + 1.5, n_star_both) 

105 dec_both = np.linspace(ctr_dec - 1.5, ctr_dec + 1.5, n_star_both) 

106 

107 ra_just_r = np.linspace(ctr_ra - 0.5, ctr_ra + 0.5, n_star_just_one) 

108 dec_just_r = np.linspace(ctr_dec + 0.2, ctr_dec + 0.2, n_star_just_one) 

109 ra_just_i = np.linspace(ctr_ra - 0.5, ctr_ra + 0.5, n_star_just_one) 

110 dec_just_i = np.linspace(ctr_dec - 0.2, ctr_dec - 0.2, n_star_just_one) 

111 

112 ra_neighbor = np.array([ra_both[n_star_both//2] + 1./3600.]) 

113 dec_neighbor = np.array([dec_both[n_star_both//2] + 1./3600.]) 

114 

115 # Create the r-band datarefs 

116 dtype = [('sourceId', 'i8'), 

117 ('ra', 'f8'), 

118 ('dec', 'f8'), 

119 ('apFlux_12_0_instFlux', 'f4'), 

120 ('apFlux_12_0_instFluxErr', 'f4'), 

121 ('apFlux_12_0_instFlux_flag', '?'), 

122 ('extendedness', 'f4'), 

123 ('sizeExtendedness', 'f4'), 

124 ('detect_isPrimary', bool), 

125 ('visit', 'i4'), 

126 ('detector', 'i4'), 

127 ('physical_filter', 'U10'), 

128 ('band', 'U2'), 

129 ('extra_column', 'f4')] 

130 

131 id_counter = 0 

132 visit_counter = 1 

133 

134 data_refs = [] 

135 for band in ['r', 'i']: 

136 if band == 'r': 

137 filtername = 'R FILTER' 

138 ra_just = ra_just_r 

139 dec_just = dec_just_r 

140 else: 

141 filtername = 'I FILTER' 

142 ra_just = ra_just_i 

143 dec_just = dec_just_i 

144 

145 if only_neighbors: 

146 star_ra = np.concatenate(([ra_both[n_star_both//2]], ra_neighbor)) 

147 star_dec = np.concatenate(([dec_both[n_star_both//2]], dec_neighbor)) 

148 elif no_secondary_overlap: 

149 star_ra = np.concatenate((ra_just,)) 

150 star_dec = np.concatenate((dec_just,)) 

151 else: 

152 star_ra = np.concatenate((ra_both, ra_neighbor, ra_just)) 

153 star_dec = np.concatenate((dec_both, dec_neighbor, dec_just)) 

154 

155 if only_out_of_tract: 

156 poly = self.skymap[self.tract].outer_sky_polygon 

157 use = ~poly.contains(np.deg2rad(star_ra), np.deg2rad(star_dec)) 

158 star_ra = star_ra[use] 

159 star_dec = star_dec[use] 

160 elif only_out_of_inner_tract: 

161 inner_tract_ids = self.skymap.findTractIdArray(star_ra, star_dec, degrees=True) 

162 use = (inner_tract_ids != self.tract) 

163 star_ra = star_ra[use] 

164 star_dec = star_dec[use] 

165 

166 nstar = len(star_ra) 

167 

168 for i in range(n_visit_per_band): 

169 ras = np.random.normal(loc=star_ra, scale=0.2/3600.) 

170 decs = np.random.normal(loc=star_dec, scale=0.2/3600.) 

171 

172 table = np.zeros(nstar, dtype=dtype) 

173 table['sourceId'] = np.arange(nstar) + id_counter 

174 table['ra'] = ras 

175 table['dec'] = decs 

176 table['apFlux_12_0_instFlux'] = 100.0 

177 table['apFlux_12_0_instFluxErr'] = 1.0 

178 table['physical_filter'] = filtername 

179 table['band'] = band 

180 table['extra_column'] = np.ones(nstar) 

181 table['visit'] = visit_counter 

182 table['detector'] = 1 

183 table['detect_isPrimary'] = True 

184 

185 if i == 0: 

186 # Make one star have low s/n 

187 table['apFlux_12_0_instFlux'][0] = 1.0 

188 

189 df = pd.DataFrame(table) 

190 df.set_index('sourceId', inplace=True) 

191 data_refs.append(lsst.pipe.base.InMemoryDatasetHandle(df, storageClass="DataFrame")) 

192 

193 id_counter += nstar 

194 visit_counter += 1 

195 

196 self.n_visit_per_band = n_visit_per_band 

197 self.n_star_both = n_star_both 

198 self.n_star_just_one = n_star_just_one 

199 if only_neighbors: 

200 self.star_ras = np.concatenate(([ra_both[n_star_both//2]], ra_neighbor)) 

201 self.star_decs = np.concatenate(([dec_both[n_star_both//2]], dec_neighbor)) 

202 else: 

203 self.star_ras = np.concatenate((ra_both, ra_just_r, ra_just_i, ra_neighbor)) 

204 self.star_decs = np.concatenate((dec_both, dec_just_r, dec_just_i, dec_neighbor)) 

205 

206 return data_refs 

207 

208 def test_compute_unique_ids(self): 

209 """Test computation of unique ids.""" 

210 ids1 = self.isolatedStarAssociationTask._compute_unique_ids(self.skymap, 

211 9813, 

212 10000) 

213 ids2 = self.isolatedStarAssociationTask._compute_unique_ids(self.skymap, 

214 9814, 

215 5000) 

216 ids = np.concatenate((ids1, ids2)) 

217 self.assertEqual(len(np.unique(ids)), len(ids)) 

218 

219 def test_remove_neighbors(self): 

220 """Test removing close neighbors.""" 

221 primary_star_cat = np.zeros(3, dtype=[('ra', 'f8'), 

222 ('dec', 'f8')]) 

223 

224 # Put two stars < 2" apart, and across the 0/360 boundary. 

225 primary_star_cat['ra'] = [0.7/3600., 360.0 - 0.7/3600., 1.0] 

226 primary_star_cat['dec'] = [5.0, 5.0, 5.0] 

227 

228 cut_cat = self.isolatedStarAssociationTask._remove_neighbors(primary_star_cat) 

229 

230 self.assertEqual(len(cut_cat), 1) 

231 np.testing.assert_almost_equal(1.0, cut_cat['ra'][0]) 

232 

233 def test_match_primary_stars(self): 

234 """Test matching primary stars.""" 

235 # Stack all the sources; we do not want any cutting here. 

236 tables = [] 

237 for data_ref in self.data_refs: 

238 df = data_ref.get() 

239 tables.append(df.to_records()) 

240 source_cat = np.concatenate(tables) 

241 

242 primary_star_cat = self.isolatedStarAssociationTask._match_primary_stars(['i', 'r'], 

243 source_cat) 

244 

245 # Ensure we found the right number of stars in each p 

246 test_i = (primary_star_cat['primary_band'] == 'i') 

247 self.assertEqual(test_i.sum(), self.n_star_both + self.n_star_just_one + 1) 

248 test_r = (primary_star_cat['primary_band'] == 'r') 

249 self.assertEqual(test_r.sum(), self.n_star_just_one) 

250 

251 # Ensure that these stars all match to input stars within 1 arcsec. 

252 with Matcher(self.star_ras, self.star_decs) as matcher: 

253 idx, i1, i2, d = matcher.query_radius(primary_star_cat['ra'], 

254 primary_star_cat['dec'], 

255 1./3600., 

256 return_indices=True) 

257 self.assertEqual(i1.size, self.star_ras.size) 

258 

259 def test_get_source_table_visit_columns(self): 

260 """Test making of source table visit columns.""" 

261 all_columns, persist_columns = self.isolatedStarAssociationTask._get_source_table_visit_column_names() 

262 

263 # Make sure all persisted columns are in all columns. 

264 for col in persist_columns: 

265 self.assertTrue(col in all_columns) 

266 

267 # And make sure extendedness is not in persisted columns. 

268 self.assertTrue('extendedness' not in persist_columns) 

269 

270 def test_match_sources(self): 

271 """Test _match_sources source to primary matching.""" 

272 # Stack all the sources; we do not want any cutting here. 

273 tables = [] 

274 for data_ref in self.data_refs: 

275 df = data_ref.get() 

276 tables.append(df.to_records()) 

277 source_cat = np.concatenate(tables) 

278 

279 source_cat = np.lib.recfunctions.append_fields(source_cat, 

280 ['obj_index'], 

281 [np.zeros(source_cat.size, dtype=np.int32)], 

282 dtypes=['i4'], 

283 usemask=False) 

284 

285 primary_bands = ['i', 'r'] 

286 

287 primary_cat = np.zeros(self.star_ras.size, 

288 dtype=self.isolatedStarAssociationTask._get_primary_dtype(primary_bands)) 

289 primary_cat['ra'] = self.star_ras 

290 primary_cat['dec'] = self.star_decs 

291 

292 source_cat_sorted, primary_star_cat = self.isolatedStarAssociationTask._match_sources(['i', 'r'], 

293 source_cat, 

294 primary_cat) 

295 # All the star sources should be matched 

296 self.assertEqual(source_cat_sorted.size, source_cat.size) 

297 

298 # Full index tests are performed in test_run_isolated_star_association_task 

299 

300 def test_make_all_star_sources(self): 

301 """Test appending all the star sources.""" 

302 source_cat = self.isolatedStarAssociationTask._make_all_star_sources(self.skymap[self.tract], 

303 self.data_ref_dict) 

304 

305 # Make sure we don't have any low s/n sources. 

306 sn_min = np.min(source_cat['apFlux_12_0_instFlux']/source_cat['apFlux_12_0_instFluxErr']) 

307 self.assertGreater(sn_min, 10.0) 

308 

309 # And make sure they are all within the tract outer boundary. 

310 poly = self.skymap[self.tract].outer_sky_polygon 

311 use = poly.contains(np.deg2rad(source_cat['ra']), np.deg2rad(source_cat['dec'])) 

312 self.assertEqual(use.sum(), len(source_cat)) 

313 

314 def test_run_isolated_star_association_task(self): 

315 """Test running the full task.""" 

316 struct = self.isolatedStarAssociationTask.run(self.skymap, 

317 self.tract, 

318 self.data_ref_dict) 

319 

320 star_source_cat = struct.star_source_cat 

321 star_cat = struct.star_cat 

322 

323 # Check that sources are all unique ids 

324 self.assertEqual(np.unique(star_source_cat['sourceId']).size, star_source_cat.size) 

325 

326 inner_tract_ids = self.skymap.findTractIdArray(self.star_ras, 

327 self.star_decs, 

328 degrees=True) 

329 inner_stars = (inner_tract_ids == self.tract) 

330 

331 # There should be the same number of stars in the inner tract region, 

332 # taking away the 2 close neighbors. 

333 self.assertEqual(star_cat.size, np.sum(inner_stars) - 2) 

334 

335 # Check the star indices 

336 for i in range(len(star_cat)): 

337 all_source_star = star_source_cat[star_cat['source_cat_index'][i]: 

338 star_cat['source_cat_index'][i] + star_cat['nsource'][i]] 

339 

340 # Check that these all point to the correct object 

341 np.testing.assert_array_equal(all_source_star['obj_index'], i) 

342 

343 # Check these are all pointing to the same star position 

344 with Matcher(np.atleast_1d(star_cat['ra'][i]), 

345 np.atleast_1d(star_cat['dec'][i])) as matcher: 

346 idx = matcher.query_radius(all_source_star['ra'], 

347 all_source_star['dec'], 

348 1./3600.) 

349 self.assertEqual(len(idx[0]), star_cat['nsource'][i]) 

350 

351 # Check per band indices 

352 for band in ['r', 'i']: 

353 band_source_star = star_source_cat[star_cat[f'source_cat_index_{band}'][i]: 

354 star_cat[f'source_cat_index_{band}'][i] 

355 + star_cat[f'nsource_{band}'][i]] 

356 with Matcher(np.atleast_1d(star_cat['ra'][i]), 

357 np.atleast_1d(star_cat['dec'][i])) as matcher: 

358 idx = matcher.query_radius(band_source_star['ra'], 

359 band_source_star['dec'], 

360 1./3600.) 

361 self.assertEqual(len(idx[0]), star_cat[f'nsource_{band}'][i]) 

362 

363 def test_run_task_all_neighbors(self): 

364 """Test running the task when all the stars are rejected as neighbors.""" 

365 data_refs = self._make_simdata(self.tract, only_neighbors=True) 

366 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits, 

367 data_refs)} 

368 

369 struct = self.isolatedStarAssociationTask.run(self.skymap, 

370 self.tract, 

371 data_ref_dict) 

372 

373 # These should ber zero length. 

374 self.assertEqual(len(struct.star_source_cat), 0) 

375 self.assertEqual(len(struct.star_cat), 0) 

376 # And spot-check a couple of expected fields to make sure they have the right type. 

377 self.assertTrue('physical_filter' in struct.star_source_cat.dtype.names) 

378 self.assertTrue('nsource_i' in struct.star_cat.dtype.names) 

379 

380 def test_run_task_all_out_of_tract(self): 

381 """Test running the task when all the sources are out of the tract.""" 

382 data_refs = self._make_simdata(self.tract, only_out_of_tract=True) 

383 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits, 

384 data_refs)} 

385 

386 struct = self.isolatedStarAssociationTask.run(self.skymap, 

387 self.tract, 

388 data_ref_dict) 

389 

390 # These should ber zero length. 

391 self.assertEqual(len(struct.star_source_cat), 0) 

392 self.assertEqual(len(struct.star_cat), 0) 

393 # And spot-check a couple of expected fields to make sure they have the right type. 

394 self.assertTrue('physical_filter' in struct.star_source_cat.dtype.names) 

395 self.assertTrue('nsource_i' in struct.star_cat.dtype.names) 

396 

397 def test_run_task_all_out_of_inner_tract(self): 

398 """Test running the task when all the sources are out of the inner tract.""" 

399 data_refs = self._make_simdata(self.tract, only_out_of_inner_tract=True) 

400 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits, 

401 data_refs)} 

402 

403 struct = self.isolatedStarAssociationTask.run(self.skymap, 

404 self.tract, 

405 data_ref_dict) 

406 

407 # These should ber zero length. 

408 self.assertEqual(len(struct.star_source_cat), 0) 

409 self.assertEqual(len(struct.star_cat), 0) 

410 # And spot-check a couple of expected fields to make sure they have the right type. 

411 self.assertTrue('physical_filter' in struct.star_source_cat.dtype.names) 

412 self.assertTrue('nsource_i' in struct.star_cat.dtype.names) 

413 

414 def test_run_task_secondary_no_overlap(self): 

415 """Test running the task when the secondary band has no overlaps. 

416 

417 This tests DM-34834. 

418 """ 

419 data_refs = self._make_simdata(self.tract, no_secondary_overlap=True) 

420 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits, 

421 data_refs)} 

422 

423 struct = self.isolatedStarAssociationTask.run(self.skymap, 

424 self.tract, 

425 data_ref_dict) 

426 

427 # Add a sanity check that we got a catalog out. 

428 self.assertGreater(len(struct.star_source_cat), 0) 

429 self.assertGreater(len(struct.star_cat), 0) 

430 

431 

432class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

433 pass 

434 

435 

436def setup_module(module): 

437 lsst.utils.tests.init() 

438 

439 

440if __name__ == "__main__": 440 ↛ 441line 440 didn't jump to line 441, because the condition on line 440 was never true

441 lsst.utils.tests.init() 

442 unittest.main()