Coverage for tests/test_isolatedStarAssociation.py: 11%

214 statements  

« prev     ^ index     » next       coverage.py v7.2.3, created at 2023-04-27 03:36 -0700

1# This file is part of pipe_tasks. 

2# 

3# LSST Data Management System 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# See COPYRIGHT file at the top of the source tree. 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22"""Test IsolatedStarAssociationTask. 

23""" 

24import unittest 

25import numpy as np 

26import pandas as pd 

27 

28import lsst.utils.tests 

29import lsst.pipe.base 

30import lsst.skymap 

31 

32from lsst.pipe.tasks.isolatedStarAssociation import (IsolatedStarAssociationConfig, 

33 IsolatedStarAssociationTask) 

34from smatch.matcher import Matcher 

35 

36 

37class IsolatedStarAssociationTestCase(lsst.utils.tests.TestCase): 

38 """Tests of IsolatedStarAssociationTask. 

39 

40 These tests bypass the middleware used for accessing data and 

41 managing Task execution. 

42 """ 

43 def setUp(self): 

44 self.skymap = self._make_skymap() 

45 self.tract = 9813 

46 self.data_refs = self._make_simdata(self.tract) 

47 self.visits = np.arange(len(self.data_refs)) + 1 

48 

49 self.data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits, 

50 self.data_refs)} 

51 

52 config = IsolatedStarAssociationConfig() 

53 config.band_order = ['i', 'r'] 

54 config.extra_columns = ['extra_column'] 

55 config.source_selector['science'].doFlags = False 

56 config.source_selector['science'].doIsolated = False 

57 

58 self.isolatedStarAssociationTask = IsolatedStarAssociationTask(config=config) 

59 

60 def _make_skymap(self): 

61 """Make a testing skymap.""" 

62 skymap_config = lsst.skymap.ringsSkyMap.RingsSkyMapConfig() 

63 skymap_config.numRings = 120 

64 skymap_config.projection = "TAN" 

65 skymap_config.tractOverlap = 1.0/60 

66 skymap_config.pixelScale = 0.168 

67 return lsst.skymap.ringsSkyMap.RingsSkyMap(skymap_config) 

68 

69 def _make_simdata(self, 

70 tract, 

71 only_neighbors=False, 

72 only_out_of_tract=False, 

73 only_out_of_inner_tract=False, 

74 no_secondary_overlap=False): 

75 """Make simulated data tables and references. 

76 

77 Parameters 

78 ---------- 

79 only_neighbors : `bool`, optional 

80 Only put in neighbors. 

81 only_out_of_tract : `bool`, optional 

82 All stars are out of the tract. 

83 only_out_of_inner_tract : `bool`, optional 

84 All stars are out of the inner tract. 

85 no_secondary_overlap : `bool`, optional 

86 Secondary band has no overlap with the primary band. 

87 

88 Returns 

89 ------- 

90 data_refs : `list` [`InMemoryDatasetHandle`] 

91 List of mock references. 

92 """ 

93 np.random.seed(12345) 

94 

95 n_visit_per_band = 5 

96 n_star_both = 50 

97 n_star_just_one = 5 

98 

99 tract_info = self.skymap[tract] 

100 ctr = tract_info.ctr_coord 

101 ctr_ra = ctr.getRa().asDegrees() 

102 ctr_dec = ctr.getDec().asDegrees() 

103 

104 ra_both = np.linspace(ctr_ra - 1.5, ctr_ra + 1.5, n_star_both) 

105 dec_both = np.linspace(ctr_dec - 1.5, ctr_dec + 1.5, n_star_both) 

106 

107 ra_just_r = np.linspace(ctr_ra - 0.5, ctr_ra + 0.5, n_star_just_one) 

108 dec_just_r = np.linspace(ctr_dec + 0.2, ctr_dec + 0.2, n_star_just_one) 

109 ra_just_i = np.linspace(ctr_ra - 0.5, ctr_ra + 0.5, n_star_just_one) 

110 dec_just_i = np.linspace(ctr_dec - 0.2, ctr_dec - 0.2, n_star_just_one) 

111 

112 ra_neighbor = np.array([ra_both[n_star_both//2] + 1./3600.]) 

113 dec_neighbor = np.array([dec_both[n_star_both//2] + 1./3600.]) 

114 

115 # Create the r-band datarefs 

116 dtype = [('sourceId', 'i8'), 

117 ('ra', 'f8'), 

118 ('decl', 'f8'), 

119 ('apFlux_12_0_instFlux', 'f4'), 

120 ('apFlux_12_0_instFluxErr', 'f4'), 

121 ('apFlux_12_0_instFlux_flag', '?'), 

122 ('extendedness', 'f4'), 

123 ('visit', 'i4'), 

124 ('detector', 'i4'), 

125 ('physical_filter', 'U10'), 

126 ('band', 'U2'), 

127 ('extra_column', 'f4')] 

128 

129 id_counter = 0 

130 visit_counter = 1 

131 

132 data_refs = [] 

133 for band in ['r', 'i']: 

134 if band == 'r': 

135 filtername = 'R FILTER' 

136 ra_just = ra_just_r 

137 dec_just = dec_just_r 

138 else: 

139 filtername = 'I FILTER' 

140 ra_just = ra_just_i 

141 dec_just = dec_just_i 

142 

143 if only_neighbors: 

144 star_ra = np.concatenate(([ra_both[n_star_both//2]], ra_neighbor)) 

145 star_dec = np.concatenate(([dec_both[n_star_both//2]], dec_neighbor)) 

146 elif no_secondary_overlap: 

147 star_ra = np.concatenate((ra_just,)) 

148 star_dec = np.concatenate((dec_just,)) 

149 else: 

150 star_ra = np.concatenate((ra_both, ra_neighbor, ra_just)) 

151 star_dec = np.concatenate((dec_both, dec_neighbor, dec_just)) 

152 

153 if only_out_of_tract: 

154 poly = self.skymap[self.tract].outer_sky_polygon 

155 use = ~poly.contains(np.deg2rad(star_ra), np.deg2rad(star_dec)) 

156 star_ra = star_ra[use] 

157 star_dec = star_dec[use] 

158 elif only_out_of_inner_tract: 

159 inner_tract_ids = self.skymap.findTractIdArray(star_ra, star_dec, degrees=True) 

160 use = (inner_tract_ids != self.tract) 

161 star_ra = star_ra[use] 

162 star_dec = star_dec[use] 

163 

164 nstar = len(star_ra) 

165 

166 for i in range(n_visit_per_band): 

167 ras = np.random.normal(loc=star_ra, scale=0.2/3600.) 

168 decs = np.random.normal(loc=star_dec, scale=0.2/3600.) 

169 

170 table = np.zeros(nstar, dtype=dtype) 

171 table['sourceId'] = np.arange(nstar) + id_counter 

172 table['ra'] = ras 

173 table['decl'] = decs 

174 table['apFlux_12_0_instFlux'] = 100.0 

175 table['apFlux_12_0_instFluxErr'] = 1.0 

176 table['physical_filter'] = filtername 

177 table['band'] = band 

178 table['extra_column'] = np.ones(nstar) 

179 table['visit'] = visit_counter 

180 table['detector'] = 1 

181 

182 if i == 0: 

183 # Make one star have low s/n 

184 table['apFlux_12_0_instFlux'][0] = 1.0 

185 

186 df = pd.DataFrame(table) 

187 df.set_index('sourceId', inplace=True) 

188 data_refs.append(lsst.pipe.base.InMemoryDatasetHandle(df, storageClass="DataFrame")) 

189 

190 id_counter += nstar 

191 visit_counter += 1 

192 

193 self.n_visit_per_band = n_visit_per_band 

194 self.n_star_both = n_star_both 

195 self.n_star_just_one = n_star_just_one 

196 if only_neighbors: 

197 self.star_ras = np.concatenate(([ra_both[n_star_both//2]], ra_neighbor)) 

198 self.star_decs = np.concatenate(([dec_both[n_star_both//2]], dec_neighbor)) 

199 else: 

200 self.star_ras = np.concatenate((ra_both, ra_just_r, ra_just_i, ra_neighbor)) 

201 self.star_decs = np.concatenate((dec_both, dec_just_r, dec_just_i, dec_neighbor)) 

202 

203 return data_refs 

204 

205 def test_compute_unique_ids(self): 

206 """Test computation of unique ids.""" 

207 ids1 = self.isolatedStarAssociationTask._compute_unique_ids(self.skymap, 

208 9813, 

209 10000) 

210 ids2 = self.isolatedStarAssociationTask._compute_unique_ids(self.skymap, 

211 9814, 

212 5000) 

213 ids = np.concatenate((ids1, ids2)) 

214 self.assertEqual(len(np.unique(ids)), len(ids)) 

215 

216 def test_remove_neighbors(self): 

217 """Test removing close neighbors.""" 

218 primary_star_cat = np.zeros(3, dtype=[('ra', 'f8'), 

219 ('decl', 'f8')]) 

220 

221 # Put two stars < 2" apart, and across the 0/360 boundary. 

222 primary_star_cat['ra'] = [0.7/3600., 360.0 - 0.7/3600., 1.0] 

223 primary_star_cat['decl'] = [5.0, 5.0, 5.0] 

224 

225 cut_cat = self.isolatedStarAssociationTask._remove_neighbors(primary_star_cat) 

226 

227 self.assertEqual(len(cut_cat), 1) 

228 np.testing.assert_almost_equal(1.0, cut_cat['ra'][0]) 

229 

230 def test_match_primary_stars(self): 

231 """Test matching primary stars.""" 

232 # Stack all the sources; we do not want any cutting here. 

233 tables = [] 

234 for data_ref in self.data_refs: 

235 df = data_ref.get() 

236 tables.append(df.to_records()) 

237 source_cat = np.concatenate(tables) 

238 

239 primary_star_cat = self.isolatedStarAssociationTask._match_primary_stars(['i', 'r'], 

240 source_cat) 

241 

242 # Ensure we found the right number of stars in each p 

243 test_i = (primary_star_cat['primary_band'] == 'i') 

244 self.assertEqual(test_i.sum(), self.n_star_both + self.n_star_just_one + 1) 

245 test_r = (primary_star_cat['primary_band'] == 'r') 

246 self.assertEqual(test_r.sum(), self.n_star_just_one) 

247 

248 # Ensure that these stars all match to input stars within 1 arcsec. 

249 with Matcher(self.star_ras, self.star_decs) as matcher: 

250 idx, i1, i2, d = matcher.query_radius(primary_star_cat['ra'], 

251 primary_star_cat['decl'], 

252 1./3600., 

253 return_indices=True) 

254 self.assertEqual(i1.size, self.star_ras.size) 

255 

256 def test_get_source_table_visit_columns(self): 

257 """Test making of source table visit columns.""" 

258 all_columns, persist_columns = self.isolatedStarAssociationTask._get_source_table_visit_column_names() 

259 

260 # Make sure all persisted columns are in all columns. 

261 for col in persist_columns: 

262 self.assertTrue(col in all_columns) 

263 

264 # And make sure extendedness is not in persisted columns. 

265 self.assertTrue('extendedness' not in persist_columns) 

266 

267 def test_match_sources(self): 

268 """Test _match_sources source to primary matching.""" 

269 # Stack all the sources; we do not want any cutting here. 

270 tables = [] 

271 for data_ref in self.data_refs: 

272 df = data_ref.get() 

273 tables.append(df.to_records()) 

274 source_cat = np.concatenate(tables) 

275 

276 source_cat = np.lib.recfunctions.append_fields(source_cat, 

277 ['obj_index'], 

278 [np.zeros(source_cat.size, dtype=np.int32)], 

279 dtypes=['i4'], 

280 usemask=False) 

281 

282 primary_bands = ['i', 'r'] 

283 

284 primary_cat = np.zeros(self.star_ras.size, 

285 dtype=self.isolatedStarAssociationTask._get_primary_dtype(primary_bands)) 

286 primary_cat['ra'] = self.star_ras 

287 primary_cat['decl'] = self.star_decs 

288 

289 source_cat_sorted, primary_star_cat = self.isolatedStarAssociationTask._match_sources(['i', 'r'], 

290 source_cat, 

291 primary_cat) 

292 # All the star sources should be matched 

293 self.assertEqual(source_cat_sorted.size, source_cat.size) 

294 

295 # Full index tests are performed in test_run_isolated_star_association_task 

296 

297 def test_make_all_star_sources(self): 

298 """Test appending all the star sources.""" 

299 source_cat = self.isolatedStarAssociationTask._make_all_star_sources(self.skymap[self.tract], 

300 self.data_ref_dict) 

301 

302 # Make sure we don't have any low s/n sources. 

303 sn_min = np.min(source_cat['apFlux_12_0_instFlux']/source_cat['apFlux_12_0_instFluxErr']) 

304 self.assertGreater(sn_min, 10.0) 

305 

306 # And make sure they are all within the tract outer boundary. 

307 poly = self.skymap[self.tract].outer_sky_polygon 

308 use = poly.contains(np.deg2rad(source_cat['ra']), np.deg2rad(source_cat['decl'])) 

309 self.assertEqual(use.sum(), len(source_cat)) 

310 

311 def test_run_isolated_star_association_task(self): 

312 """Test running the full task.""" 

313 struct = self.isolatedStarAssociationTask.run(self.skymap, 

314 self.tract, 

315 self.data_ref_dict) 

316 

317 star_source_cat = struct.star_source_cat 

318 star_cat = struct.star_cat 

319 

320 # Check that sources are all unique ids 

321 self.assertEqual(np.unique(star_source_cat['sourceId']).size, star_source_cat.size) 

322 

323 inner_tract_ids = self.skymap.findTractIdArray(self.star_ras, 

324 self.star_decs, 

325 degrees=True) 

326 inner_stars = (inner_tract_ids == self.tract) 

327 

328 # There should be the same number of stars in the inner tract region, 

329 # taking away the 2 close neighbors. 

330 self.assertEqual(star_cat.size, np.sum(inner_stars) - 2) 

331 

332 # Check the star indices 

333 for i in range(len(star_cat)): 

334 all_source_star = star_source_cat[star_cat['source_cat_index'][i]: 

335 star_cat['source_cat_index'][i] + star_cat['nsource'][i]] 

336 

337 # Check that these all point to the correct object 

338 np.testing.assert_array_equal(all_source_star['obj_index'], i) 

339 

340 # Check these are all pointing to the same star position 

341 with Matcher(np.atleast_1d(star_cat['ra'][i]), 

342 np.atleast_1d(star_cat['decl'][i])) as matcher: 

343 idx = matcher.query_radius(all_source_star['ra'], 

344 all_source_star['decl'], 

345 1./3600.) 

346 self.assertEqual(len(idx[0]), star_cat['nsource'][i]) 

347 

348 # Check per band indices 

349 for band in ['r', 'i']: 

350 band_source_star = star_source_cat[star_cat[f'source_cat_index_{band}'][i]: 

351 star_cat[f'source_cat_index_{band}'][i] 

352 + star_cat[f'nsource_{band}'][i]] 

353 with Matcher(np.atleast_1d(star_cat['ra'][i]), 

354 np.atleast_1d(star_cat['decl'][i])) as matcher: 

355 idx = matcher.query_radius(band_source_star['ra'], 

356 band_source_star['decl'], 

357 1./3600.) 

358 self.assertEqual(len(idx[0]), star_cat[f'nsource_{band}'][i]) 

359 

360 def test_run_task_all_neighbors(self): 

361 """Test running the task when all the stars are rejected as neighbors.""" 

362 data_refs = self._make_simdata(self.tract, only_neighbors=True) 

363 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits, 

364 data_refs)} 

365 

366 struct = self.isolatedStarAssociationTask.run(self.skymap, 

367 self.tract, 

368 data_ref_dict) 

369 

370 # These should ber zero length. 

371 self.assertEqual(len(struct.star_source_cat), 0) 

372 self.assertEqual(len(struct.star_cat), 0) 

373 # And spot-check a couple of expected fields to make sure they have the right type. 

374 self.assertTrue('physical_filter' in struct.star_source_cat.dtype.names) 

375 self.assertTrue('nsource_i' in struct.star_cat.dtype.names) 

376 

377 def test_run_task_all_out_of_tract(self): 

378 """Test running the task when all the sources are out of the tract.""" 

379 data_refs = self._make_simdata(self.tract, only_out_of_tract=True) 

380 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits, 

381 data_refs)} 

382 

383 struct = self.isolatedStarAssociationTask.run(self.skymap, 

384 self.tract, 

385 data_ref_dict) 

386 

387 # These should ber zero length. 

388 self.assertEqual(len(struct.star_source_cat), 0) 

389 self.assertEqual(len(struct.star_cat), 0) 

390 # And spot-check a couple of expected fields to make sure they have the right type. 

391 self.assertTrue('physical_filter' in struct.star_source_cat.dtype.names) 

392 self.assertTrue('nsource_i' in struct.star_cat.dtype.names) 

393 

394 def test_run_task_all_out_of_inner_tract(self): 

395 """Test running the task when all the sources are out of the inner tract.""" 

396 data_refs = self._make_simdata(self.tract, only_out_of_inner_tract=True) 

397 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits, 

398 data_refs)} 

399 

400 struct = self.isolatedStarAssociationTask.run(self.skymap, 

401 self.tract, 

402 data_ref_dict) 

403 

404 # These should ber zero length. 

405 self.assertEqual(len(struct.star_source_cat), 0) 

406 self.assertEqual(len(struct.star_cat), 0) 

407 # And spot-check a couple of expected fields to make sure they have the right type. 

408 self.assertTrue('physical_filter' in struct.star_source_cat.dtype.names) 

409 self.assertTrue('nsource_i' in struct.star_cat.dtype.names) 

410 

411 def test_run_task_secondary_no_overlap(self): 

412 """Test running the task when the secondary band has no overlaps. 

413 

414 This tests DM-34834. 

415 """ 

416 data_refs = self._make_simdata(self.tract, no_secondary_overlap=True) 

417 data_ref_dict = {visit: data_ref for visit, data_ref in zip(self.visits, 

418 data_refs)} 

419 

420 struct = self.isolatedStarAssociationTask.run(self.skymap, 

421 self.tract, 

422 data_ref_dict) 

423 

424 # Add a sanity check that we got a catalog out. 

425 self.assertGreater(len(struct.star_source_cat), 0) 

426 self.assertGreater(len(struct.star_cat), 0) 

427 

428 

429class MyMemoryTestCase(lsst.utils.tests.MemoryTestCase): 

430 pass 

431 

432 

433def setup_module(module): 

434 lsst.utils.tests.init() 

435 

436 

437if __name__ == "__main__": 437 ↛ 438line 437 didn't jump to line 438, because the condition on line 437 was never true

438 lsst.utils.tests.init() 

439 unittest.main()