Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import unittest 

2import numpy as np 

3import sqlite3 

4import shutil 

5import tempfile 

6import os 

7 

8import lsst.utils.tests 

9from lsst.utils import getPackageDir 

10 

11import lsst.sims.utils.htmModule as htm 

12from lsst.sims.utils import xyz_from_ra_dec 

13from lsst.sims.utils import ra_dec_from_xyz 

14from lsst.sims.utils import angularSeparation 

15from lsst.sims.utils import ObservationMetaData 

16from lsst.sims.catUtils.baseCatalogModels import UWGalaxyModels as UWGal 

17 

18from lsst.sims.catalogs.definitions import InstanceCatalog 

19from lsst.sims.catalogs.decorators import cached 

20 

21from lsst.sims.utils.CodeUtilities import sims_clean_up 

22 

23ROOT = os.path.abspath(os.path.dirname(__file__)) 

24 

25 

26def setup_module(module): 

27 lsst.utils.tests.init() 

28 

29 

30class GalaxyTileObjTestCase(unittest.TestCase): 

31 

32 @classmethod 

33 def setUpClass(cls): 

34 cls._tile_radius = angularSeparation(0.0, 0.0, 2.0, 2.0) 

35 

36 dir_name = os.path.join(getPackageDir('sims_catUtils'), 'tests') 

37 cls._tmpdir = tempfile.mkdtemp(dir=dir_name) 

38 

39 cls._temp_gal_db = os.path.join(cls._tmpdir, 'test_galaxies.db') 

40 

41 ra_min = -2.25 

42 cls._ra_min = ra_min 

43 ra_max = 2.251 

44 cls._d_ra = 0.05 

45 ra_grid = np.arange(ra_min, ra_max, cls._d_ra) 

46 dec_grid = np.arange(ra_min, ra_max, cls._d_ra) 

47 print('raw grid %d' % len(ra_grid)) 

48 ra_dec_mesh = np.meshgrid(ra_grid, dec_grid) 

49 ra_grid = ra_dec_mesh[0].flatten() 

50 dec_grid = ra_dec_mesh[1].flatten() 

51 

52 # add a very small offset so that numerical precision 

53 # does not foul things up on the tile boundaries 

54 rng = np.random.RandomState(7163) 

55 ra_grid += 1.0e-5*(rng.random_sample(len(ra_grid))-0.5) 

56 dec_grid += 1.0e-5*(rng.random_sample(len(dec_grid))-0.5) 

57 

58 

59 galtag = (100*np.round(45 + ra_grid/cls._d_ra) + 

60 np.round(45+dec_grid/cls._d_ra)).astype(int) 

61 assert len(galtag) == len(np.unique(galtag)) 

62 htmid_grid = htm.findHtmid(ra_grid, dec_grid, 21) 

63 print('got htmid %d' % len(htmid_grid)) 

64 print(htm.levelFromHtmid(htmid_grid.min())) 

65 print(htm.levelFromHtmid(htmid_grid.max())) 

66 assert htm.levelFromHtmid(htmid_grid.min())==21 

67 assert htm.levelFromHtmid(htmid_grid.max())==21 

68 gid = np.arange(len(ra_grid), dtype=int) 

69 assert len(galtag) == len(np.unique(galtag)) 

70 print(ra_grid.max(),ra_grid.min()) 

71 

72 with sqlite3.connect(cls._temp_gal_db) as conn: 

73 c = conn.cursor() 

74 query = '''CREATE TABLE galaxy(htmid int, id int, 

75 galid text, ra real, dec real, galtag int, redshift)''' 

76 c.execute(query).fetchall() 

77 values = ((int(hh), int(ii), str(ii), r, d, int(g), 0.5) for hh, ii, r, d, g in 

78 zip(htmid_grid, gid, ra_grid, dec_grid, galtag)) 

79 c.executemany('INSERT INTO galaxy VALUES (?,?,?,?,?,?,?)', values) 

80 

81 @classmethod 

82 def tearDownClass(cls): 

83 sims_clean_up() 

84 if os.path.isdir(cls._tmpdir): 

85 shutil.rmtree(cls._tmpdir, ignore_errors=True) 

86 

87 

88 def test_equatorial_tile(self): 

89 """ 

90 Query an observation at the center of an equatorial tile; make sure the right galaxies 

91 get returned 

92 """ 

93 

94 # select all of the galaxies that should be in the tile 

95 with sqlite3.connect(self._temp_gal_db) as db_conn: 

96 c = db_conn.cursor() 

97 results = c.execute("SELECT galtag FROM galaxy WHERE ra>=-2.0 AND ra<=2.0 " 

98 "AND dec>=-2.0 AND dec<=2.0") 

99 

100 expected_galtag = set(int(r[0]) for r in results) 

101 

102 dbobj = UWGal.UWGalaxyTileObj(database=self._temp_gal_db, driver='sqlite') 

103 obs = ObservationMetaData(pointingRA=34.0, pointingDec=0.0, 

104 boundType='circle', 

105 boundLength=self._tile_radius) 

106 

107 data_iter = dbobj.query_columns(['galtileid', 'ra', 'dec', 'galtag'], 

108 obs_metadata=obs) 

109 

110 n_data = 0 

111 n_valid = 0 

112 for chunk in data_iter: 

113 n_data += len(chunk) 

114 

115 # select only those galaxies that are inside the tile on which we 

116 # are centered 

117 valid = np.where(np.logical_and(chunk['ra']>=obs.pointingRA-2.0, 

118 np.logical_and(chunk['ra']<=obs.pointingRA+2.0, 

119 np.logical_and(chunk['dec']>=obs.pointingDec-2.0, 

120 chunk['dec']<=obs.pointingDec+2.0)))) 

121 n_valid += len(valid[0]) 

122 if len(valid[0])==0: 

123 continue 

124 

125 # validate that the correct galaxies were rotated to the correct position 

126 # when the galaxy database was rotated into the field of view 

127 valid_data = chunk[valid] 

128 tag_shld = 100*np.round(45+(valid_data['ra']-obs.pointingRA)/self._d_ra).astype(int) 

129 tag_shld += np.round(45+(valid_data['dec']-obs.pointingDec)/self._d_ra).astype(int) 

130 np.testing.assert_array_equal(tag_shld, valid_data['galtag']) 

131 for tag in valid_data['galtag']: 

132 self.assertIn(tag, expected_galtag) 

133 for tag in expected_galtag: 

134 self.assertIn(tag, valid_data['galtag']) 

135 

136 self.assertGreater(n_data, 10) 

137 self.assertGreater(n_valid, 1000) 

138 

139 def test_two_equatorial_tiles(self): 

140 """ 

141 Query a field of view directly between two equatorial tiles and make sure that 

142 all expected galaxies are returned where we think they ought to be 

143 """ 

144 radius = 1.0 

145 expected_galaxies_left = [] # galaxies that come from left edge of original tile 

146 expected_galaxies_right = [] # galaxies that come from right edge of original tile 

147 with sqlite3.connect(self._temp_gal_db) as db_conn: 

148 c = db_conn.cursor() 

149 query = "SELECT ra, dec, galtag FROM galaxy " 

150 query += "WHERE ra>=-2.0 AND ra<=2.0 " 

151 query += "AND dec>=-2.0 AND dec<=2.0" 

152 results = c.execute(query).fetchall() 

153 for gal in results: 

154 dd_left = angularSeparation(-2.0, 0.0, gal[0], gal[1]) 

155 if dd_left <= radius: 

156 expected_galaxies_left.append(gal) 

157 dd_right = angularSeparation(2.0, 0.0, gal[0], gal[1]) 

158 if dd_right <= radius: 

159 expected_galaxies_right.append(gal) 

160 

161 # create sets of the expected galtag values 

162 gal_tag_left = set(g[2] for g in expected_galaxies_left) 

163 gal_tag_right = set(g[2] for g in expected_galaxies_right) 

164 self.assertGreater(len(gal_tag_left), 100) 

165 self.assertGreater(len(gal_tag_right), 100) 

166 

167 # construct field of view on the border between two equatorial tiles 

168 ra1 = 34.0 

169 ra2 = 38.0 

170 obs = ObservationMetaData(pointingRA=0.5*(ra1+ra2), pointingDec=0.0, 

171 boundType='circle', boundLength=radius) 

172 

173 dbobj = UWGal.UWGalaxyTileObj(database=self._temp_gal_db, driver='sqlite') 

174 data_iter = dbobj.query_columns(['galtileid', 'ra', 'dec', 'galtag'], 

175 obs_metadata=obs) 

176 

177 found_left = set() 

178 found_right = set() 

179 for chunk in data_iter: 

180 valid_left = np.where(chunk['ra']>obs.pointingRA) 

181 valid_right = np.where(chunk['ra']<obs.pointingRA) 

182 self.assertEqual(len(valid_left[0])+len(valid_right[0]), len(chunk)) 

183 data_left = chunk[valid_left] 

184 tag_left = 100*np.round(45+(data_left['ra']-ra2)/self._d_ra) 

185 tag_left += np.round(45+(data_left['dec']-obs.pointingDec)/self._d_ra) 

186 tag_left = tag_left.astype(int) 

187 for tag in tag_left: 

188 self.assertIn(tag, gal_tag_left) 

189 found_left.add(tag) 

190 

191 data_right = chunk[valid_right] 

192 tag_right = 100*np.round(45+(data_right['ra']-ra1)/self._d_ra) 

193 tag_right += np.round(45+(data_right['dec']-obs.pointingDec)/self._d_ra) 

194 tag_right = tag_right.astype(int) 

195 for tag in tag_right: 

196 self.assertIn(tag, gal_tag_right) 

197 found_right.add(tag) 

198 

199 # make sure that the galaxies returned by the query 

200 # match what we expected 

201 for tag in found_left: 

202 self.assertIn(tag, gal_tag_left) 

203 for tag in gal_tag_left: 

204 self.assertIn(tag, found_left) 

205 for tag in found_right: 

206 self.assertIn(tag, gal_tag_right) 

207 for tag in gal_tag_right: 

208 self.assertIn(tag, found_right) 

209 

210 def test_four_corners(self): 

211 """ 

212 Test field of view centered at a corner between four tiles 

213 (not on the equator) 

214 """ 

215 ra_obs = 44.0 

216 dec_obs = 34.0 

217 radius = 2.8 # this should be enough to get galaxies to appear 

218 # in more than one tile 

219 

220 # construct bounding half spaces for the four quadrants 

221 quadrant_half_spaces = [] 

222 quadrant_centers = [] # prime tile coords of tile corners 

223 for ra_q, dec_q in zip([46.0, 42.0, 42.0, 46.0], 

224 [36.0, 36.0, 32.0, 32.0]): 

225 

226 cos_ra = np.cos(np.radians(ra_q)) 

227 sin_ra = np.sin(np.radians(ra_q)) 

228 cos_dec = np.cos(np.radians(dec_q)) 

229 sin_dec = np.sin(np.radians(dec_q)) 

230 

231 m_ra = np.array([[cos_ra, sin_ra, 0.0], 

232 [-sin_ra, cos_ra, 0.0], 

233 [0.0, 0.0, 1.0]]) 

234 

235 m_dec = np.array([[cos_dec, 0.0, sin_dec], 

236 [0.0, 1.0, 0.0], 

237 [-sin_dec, 0.0, cos_dec]]) 

238 

239 upper_hs_vv = np.array([0.0, 0.0, -1.0]) 

240 upper_hs_vv = np.dot(m_dec, 

241 np.dot(m_ra, upper_hs_vv)) 

242 rr, dd = ra_dec_from_xyz(upper_hs_vv[0], 

243 upper_hs_vv[1], 

244 upper_hs_vv[2]) 

245 upper_hs = htm.halfSpaceFromRaDec(rr, dd, 90.0+dec_q+2.0) 

246 

247 lower_hs_vv = np.array([0.0, 0.0, 1.0]) 

248 lower_hs_vv = np.dot(m_dec, 

249 np.dot(m_ra, lower_hs_vv)) 

250 rr, dd = ra_dec_from_xyz(lower_hs_vv[0], 

251 lower_hs_vv[1], 

252 lower_hs_vv[2]) 

253 

254 lower_hs = htm.halfSpaceFromRaDec(rr, dd, 92.0-dec_q) 

255 

256 left_hs_vv = xyz_from_ra_dec(ra_q+88.0, 0.0) 

257 left_hs_vv = np.dot(m_dec, 

258 np.dot(m_ra, left_hs_vv)) 

259 rr, dd = ra_dec_from_xyz(left_hs_vv[0], 

260 left_hs_vv[1], 

261 left_hs_vv[2]) 

262 left_hs = htm.halfSpaceFromRaDec(rr, dd, 90.0) 

263 

264 right_hs_vv = xyz_from_ra_dec(ra_q-88.0, 0.0) 

265 right_hs_vv = np.dot(m_dec, 

266 np.dot(m_ra, right_hs_vv)) 

267 rr, dd = ra_dec_from_xyz(right_hs_vv[0], 

268 right_hs_vv[1], 

269 right_hs_vv[2]) 

270 right_hs = htm.halfSpaceFromRaDec(rr, dd, 90.0) 

271 quadrant_half_spaces.append((upper_hs, lower_hs, 

272 left_hs, right_hs)) 

273 

274 corner = xyz_from_ra_dec(44.0, 34.0) 

275 rot_corner = np.dot(m_dec, 

276 np.dot(m_ra, corner)) 

277 quadrant_centers.append(ra_dec_from_xyz(rot_corner[0], 

278 rot_corner[1], 

279 rot_corner[2])) 

280 

281 

282 gal_tag_1st_quad = set() 

283 gal_tag_2nd_quad = set() 

284 gal_tag_3rd_quad = set() 

285 gal_tag_4th_quad = set() 

286 n_multiple_quad = 0 

287 with sqlite3.connect(self._temp_gal_db) as db_conn: 

288 c = db_conn.cursor() 

289 query = "SELECT ra, dec, galtag FROM galaxy " 

290 results = c.execute(query).fetchall() 

291 for gal in results: 

292 n_quad = 0 

293 vv = xyz_from_ra_dec(gal[0], gal[1]) 

294 dd = list([angularSeparation(c[0], c[1], gal[0], gal[1]) 

295 for c in quadrant_centers]) 

296 

297 if gal[2]== 6785: 

298 for hs in quadrant_half_spaces[3]: 

299 print('6785 contained ',hs.contains_pt(vv)) 

300 

301 if (dd[0] <= radius and 

302 quadrant_half_spaces[0][0].contains_pt(vv) and 

303 quadrant_half_spaces[0][1].contains_pt(vv) and 

304 quadrant_half_spaces[0][2].contains_pt(vv) and 

305 quadrant_half_spaces[0][3].contains_pt(vv)): 

306 n_quad += 1 

307 gal_tag_1st_quad.add(gal[2]) 

308 

309 if (dd[1] <= radius and 

310 quadrant_half_spaces[1][0].contains_pt(vv) and 

311 quadrant_half_spaces[1][1].contains_pt(vv) and 

312 quadrant_half_spaces[1][2].contains_pt(vv) and 

313 quadrant_half_spaces[1][3].contains_pt(vv)): 

314 n_quad += 1 

315 gal_tag_2nd_quad.add(gal[2]) 

316 

317 if (dd[2] <= radius and 

318 quadrant_half_spaces[2][0].contains_pt(vv) and 

319 quadrant_half_spaces[2][1].contains_pt(vv) and 

320 quadrant_half_spaces[2][2].contains_pt(vv) and 

321 quadrant_half_spaces[2][3].contains_pt(vv)): 

322 n_quad += 1 

323 gal_tag_3rd_quad.add(gal[2]) 

324 

325 if (dd[3] <= radius and 

326 quadrant_half_spaces[3][0].contains_pt(vv) and 

327 quadrant_half_spaces[3][1].contains_pt(vv) and 

328 quadrant_half_spaces[3][2].contains_pt(vv) and 

329 quadrant_half_spaces[3][3].contains_pt(vv)): 

330 n_quad += 1 

331 gal_tag_4th_quad.add(gal[2]) 

332 

333 if n_quad>1: 

334 n_multiple_quad += 1 

335 

336 self.assertGreater(n_multiple_quad, 100) 

337 

338 gal_found_1st_quad = set() 

339 gal_found_2nd_quad = set() 

340 gal_found_3rd_quad = set() 

341 ra_dec_3 = {} 

342 gal_found_4th_quad = set() 

343 ra_dec_4 = {} 

344 

345 obs = ObservationMetaData(pointingRA=ra_obs, pointingDec=dec_obs, 

346 boundType='circle', boundLength=radius) 

347 

348 dbobj = UWGal.UWGalaxyTileObj(database=self._temp_gal_db, driver='sqlite') 

349 data_iter = dbobj.query_columns(['galtileid', 'ra', 'dec', 'galtag'], 

350 obs_metadata=obs) 

351 

352 for chunk in data_iter: 

353 for gal in chunk: 

354 if gal['ra'] >= ra_obs and gal['dec'] >= dec_obs: 

355 quad_set = gal_found_1st_quad 

356 elif gal['ra'] < ra_obs and gal['dec'] >= dec_obs: 

357 quad_set = gal_found_2nd_quad 

358 elif gal['ra'] < ra_obs and gal['dec'] < dec_obs: 

359 quad_set = gal_found_3rd_quad 

360 ra_dec_3[gal['galtag']] = (gal['ra'], gal['dec']) 

361 elif gal['ra'] >= ra_obs and gal['dec'] < dec_obs: 

362 quad_set = gal_found_4th_quad 

363 ra_dec_4[gal['galtag']] = (gal['ra'], gal['dec']) 

364 else: 

365 raise RuntimeError("Unsure what quadrant galaxy belongs in") 

366 assert gal['galtag'] not in quad_set 

367 quad_set.add(gal['galtag']) 

368 

369 test_sum = 0 

370 control_sum = 0 

371 for test, control, ra_dec in zip([gal_found_1st_quad, gal_found_2nd_quad, 

372 gal_found_3rd_quad, gal_found_4th_quad], 

373 [gal_tag_1st_quad, gal_tag_2nd_quad, 

374 gal_tag_3rd_quad, gal_tag_4th_quad], 

375 [None, None, ra_dec_3, ra_dec_4]): 

376 

377 n_erroneous_test = 0 

378 for tag in test: 

379 if tag not in control: 

380 n_erroneous_test += 1 

381 dd = angularSeparation(ra_obs, dec_obs, 

382 ra_dec[tag][0], ra_dec[tag][1]) 

383 print(' bad test %d %e -- %.4f %.4f' % (tag, dd, 

384 ra_dec[tag][0], ra_dec[tag][1])) 

385 n_erroneous_control = 0 

386 for tag in control: 

387 if tag not in test: 

388 n_erroneous_control += 1 

389 print('n_test %d bad %d' % (len(test), n_erroneous_test)) 

390 print('n_control %d bad %d\n' % (len(control), n_erroneous_control)) 

391 test_sum += len(test) 

392 control_sum += len(control) 

393 print('test_sum %d' % test_sum) 

394 print('control_sum %d' % control_sum) 

395 

396 self.assertEqual(len(gal_found_1st_quad), len(gal_tag_1st_quad)) 

397 self.assertEqual(len(gal_found_2nd_quad), len(gal_tag_2nd_quad)) 

398 self.assertEqual(len(gal_found_3rd_quad), len(gal_tag_3rd_quad)) 

399 self.assertEqual(len(gal_found_4th_quad), len(gal_tag_4th_quad)) 

400 

401 for tag in gal_found_1st_quad: 

402 self.assertIn(tag, gal_tag_1st_quad) 

403 for tag in gal_found_2nd_quad: 

404 self.assertIn(tag, gal_tag_2nd_quad) 

405 for tag in gal_found_3rd_quad: 

406 self.assertIn(tag, gal_tag_3rd_quad) 

407 for tag in gal_found_4th_quad: 

408 self.assertIn(tag, gal_tag_4th_quad) 

409 

410 def test_local_instance_catalog(self): 

411 """ 

412 Test that the local galaxy obj can interface with the 

413 InstanceCatalog as expected 

414 """ 

415 

416 class LocalGalDummyICat(InstanceCatalog): 

417 column_outputs=['raJ2000', 'decJ2000', 

418 'ra', 'dec', 'galtag', 

419 'someFloat'] 

420 

421 delimiter = ' ' 

422 default_formats = {'f': '%.9g'} 

423 

424 @cached 

425 def get_someFloat(self): 

426 t = self.column_by_name('galtag') 

427 r = self.column_by_name('ra') 

428 d = self.column_by_name('dec') 

429 return r**2+d+0.5*t 

430 

431 obs = ObservationMetaData(pointingRA=34.0, pointingDec=44.0, 

432 boundType='circle', boundLength=3.0) 

433 

434 dbobj = UWGal.UWGalaxyTileObj(database=self._temp_gal_db, driver='sqlite') 

435 

436 cat = LocalGalDummyICat(dbobj, obs_metadata=obs) 

437 scratch_dir = tempfile.mkdtemp(dir=ROOT, prefix='local_gal') 

438 cat_name = tempfile.mktemp(dir=scratch_dir, 

439 prefix='local_gal', 

440 suffix='.txt') 

441 

442 cat.write_catalog(cat_name) 

443 dt = np.dtype([('raJ2000', float), ('decJ2000', float), 

444 ('ra', float), ('dec', float), ('galtag', int), 

445 ('someFloat', float)]) 

446 

447 data = np.genfromtxt(cat_name, dtype=dt) 

448 dd = angularSeparation(data['ra'], data['dec'], 

449 np.degrees(data['raJ2000']), 

450 np.degrees(data['decJ2000'])) 

451 self.assertLess(dd.max(), 0.001/3600.0) 

452 ff = data['ra']**2+data['dec']+0.5*data['galtag'] 

453 np.testing.assert_array_almost_equal(ff, data['someFloat'], decimal=5) 

454 

455 if os.path.exists(cat_name): 

456 os.unlink(cat_name) 

457 shutil.rmtree(scratch_dir) 

458 

459 

460class MemoryTestClass(lsst.utils.tests.MemoryTestCase): 

461 pass 

462 

463if __name__ == "__main__": 463 ↛ 464line 463 didn't jump to line 464, because the condition on line 463 was never true

464 lsst.utils.tests.init() 

465 unittest.main()