Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import os 

2import numpy as np 

3import numpy.lib.recfunctions as np_recfn 

4import json 

5import copy 

6from sqlalchemy import text 

7from lsst.utils import getPackageDir 

8from lsst.sims.utils import ObservationMetaData 

9from lsst.sims.utils import HalfSpace, levelFromHtmid 

10from lsst.sims.utils import halfSpaceFromRaDec 

11from lsst.sims.utils import halfSpaceFromPoints 

12from lsst.sims.utils import intersectHalfSpaces 

13from lsst.sims.utils import cartesianFromSpherical, sphericalFromCartesian 

14from lsst.sims.catalogs.db import ChunkIterator 

15from lsst.sims.catUtils.baseCatalogModels import GalaxyObj 

16from lsst.sims.catUtils.baseCatalogModels import GalaxyBulgeObj 

17from lsst.sims.catUtils.baseCatalogModels import GalaxyDiskObj 

18from lsst.sims.catUtils.baseCatalogModels import GalaxyAgnObj 

19 

20__all__ = ["Tile", "UWGalaxyTiles", "UWGalaxyTileObj", 

21 "UWGalaxyDiskObj", "UWGalaxyBulgeObj", 

22 "UWGalaxyAgnObj"] 

23 

24 

25class Tile(object): 

26 

27 def __init__(self, box_corners): 

28 self._trixel_bounds = None 

29 self._trixel_bound_level = None 

30 self._hs_list = [] 

31 if len(box_corners) == 0: 

32 return 

33 self._init_from_corners(box_corners) 

34 

35 def _init_from_corners(self, box_corners): 

36 ra_range = [c[0] for c in box_corners] 

37 ra_min = min(ra_range) 

38 ra_max = max(ra_range) 

39 dec_range = [c[1] for c in box_corners] 

40 dec_min = min(dec_range) 

41 dec_max = max(dec_range) 

42 tol = 1.0e-10 

43 for i_c1 in range(len(box_corners)): 

44 c1 = box_corners[i_c1] 

45 pt1 = cartesianFromSpherical(np.degrees(c1[0]), np.degrees(c1[1])) 

46 for i_c2 in range(i_c1+1, len(box_corners), 1): 

47 hs = None 

48 c2 = box_corners[i_c2] 

49 pt2 = cartesianFromSpherical(np.degrees(c2[0]), np.degrees(c2[1])) 

50 if np.abs(1.0-np.dot(pt1, pt2))<tol: 

51 continue 

52 

53 dra = np.abs(c1[0]-c2[0]) 

54 ddec = np.abs(c1[1]-c2[1]) 

55 

56 if dra<tol and ddec>tol: 

57 # The RAs of the two corners is identical, but the Decs are 

58 # different; this Half Space is defined by a Great Circle 

59 if np.abs(c1[0]-ra_min)<tol: 

60 inner_pt = (ra_min+0.001, dec_min+0.001) 

61 else: 

62 inner_pt = (ra_max-0.001, dec_min+0.001) 

63 hs = halfSpaceFromPoints(c1, c2, inner_pt) 

64 elif ddec<tol and dra>tol: 

65 # The Decs of the two corners is identical, bu the RAs are 

66 # different; this Half Space is defined by a line of constant 

67 # Dec and should be centered at one of the poles 

68 if np.abs(c1[1]-dec_min)<tol: 

69 hs = halfSpaceFromRaDec(0.0, 90.0, 90.0-dec_min) 

70 else: 

71 hs = halfSpaceFromRaDec(0.0, -90.0, 90.0+dec_max) 

72 else: 

73 continue 

74 

75 if hs is None: 

76 raise RuntimeError("Somehow Half Space == None") 

77 self._hs_list.append(hs) 

78 

79 def contains_many_pts(self, pts): 

80 result = None 

81 for hs in self.half_space_list: 

82 valid = hs.contains_many_pts(pts) 

83 if result is None: 

84 result = valid 

85 else: 

86 result &= valid 

87 return result 

88 

89 @property 

90 def half_space_list(self): 

91 return self._hs_list 

92 

93 def rotate(self, matrix): 

94 new_tile = Tile([]) 

95 for hs in self.half_space_list: 

96 vv = np.dot(matrix, hs.vector) 

97 new_hs = HalfSpace(vv, hs.dd) 

98 new_tile._hs_list.append(new_hs) 

99 return new_tile 

100 

101 def intersects_circle(self, center_pt, radius_rad): 

102 gross_is_contained = True 

103 for hs in self.half_space_list: 

104 if not hs.intersects_circle(center_pt, radius_rad): 

105 gross_is_contained = False 

106 break 

107 if not gross_is_contained: 

108 return False 

109 

110 hs_interest = HalfSpace(center_pt, np.cos(radius_rad)) 

111 for i_h1 in range(len(self.half_space_list)): 

112 hs1 = self.half_space_list[i_h1] 

113 roots = intersectHalfSpaces(hs1, hs_interest) 

114 if len(roots) == 0: 

115 continue 

116 

117 if len(roots.shape)==1: 

118 roots = [roots] 

119 

120 for i_h2 in range(len(self.half_space_list)): 

121 if i_h1 == i_h2: 

122 continue 

123 hs2 = self.half_space_list[i_h2] 

124 local_contained = False 

125 for rr in roots: 

126 if hs2.contains_pt(rr, tol=1.0e-10): 

127 local_contained = True 

128 break 

129 if not local_contained: 

130 return False 

131 return True 

132 

133 def _generate_all_trixels(self, level): 

134 output = None 

135 for hs in self.half_space_list: 

136 local_limits = hs.findAllTrixels(level) 

137 if output is None: 

138 output = local_limits 

139 else: 

140 output = HalfSpace.join_trixel_bound_sets(output, local_limits) 

141 self._trixel_bounds = output 

142 self._trixel_bound_level = level 

143 return None 

144 

145 @property 

146 def trixel_bound_level(self): 

147 return self._trixel_bound_level 

148 

149 @property 

150 def trixel_bounds(self): 

151 return self._trixel_bounds 

152 

153 def find_all_trixels(self, level): 

154 if self._trixel_bounds is None or self.trixel_bound_level != level: 

155 self._generate_all_trixels(level) 

156 return self._trixel_bounds 

157 

158 

159class UWGalaxyTiles(object): 

160 """ 

161 A class to store the UW CatSim server galaxy tiles as a series of 

162 Half Spaces 

163 """ 

164 

165 def __init__(self): 

166 data_dir = os.path.join(getPackageDir('sims_catUtils'), 'data') 

167 data_file = os.path.join(data_dir, 'tile_data.txt') 

168 dtype = np.dtype([('id', int), ('ra', float), ('dec', float), 

169 ('box', str, 500)]) 

170 tile_data = np.genfromtxt(data_file, dtype=dtype, delimiter=';') 

171 

172 self._tile_id = tile_data['id'] 

173 self._tile_ra = {} 

174 self._tile_dec = {} 

175 self._rotation_matrix_dict = {} 

176 for ii, rr, dd, in zip(tile_data['id'], tile_data['ra'], tile_data['dec']): 

177 self._tile_ra[ii] = rr 

178 self._tile_dec[ii] = dd 

179 ra_rad = np.radians(rr) 

180 dec_rad = np.radians(dd) 

181 

182 ra_mat = np.array([[np.cos(ra_rad), np.sin(ra_rad), 0.0], 

183 [-np.sin(ra_rad), np.cos(ra_rad), 0.0], 

184 [0.0, 0.0, 1.0]]) 

185 

186 dec_mat = np.array([[np.cos(dec_rad), 0.0, np.sin(dec_rad)], 

187 [0.0, 1.0, 0.0], 

188 [-np.sin(dec_rad), 0.0, np.cos(dec_rad)]]) 

189 

190 full_mat = np.dot(dec_mat, ra_mat) 

191 self._rotation_matrix_dict[ii] = full_mat 

192 

193 self._tile_dict = {} 

194 for tile_id, box in zip(tile_data['id'], tile_data['box']): 

195 box_corners = json.loads(box) 

196 self._tile_dict[tile_id] = Tile(box_corners) 

197 

198 

199 def tile_ra(self, tile_idx): 

200 return self._tile_ra[tile_idx] 

201 

202 def tile_dec(self, tile_idx): 

203 return self._tile_dec[tile_idx] 

204 

205 def rotation_matrix(self, tile_idx): 

206 return self._rotation_matrix_dict[tile_idx] 

207 

208 def tile(self, tile_idx): 

209 return self._tile_dict[tile_idx] 

210 

211 def find_all_tiles(self, ra, dec, radius): 

212 """ 

213 ra, dec, radius are all in degrees 

214 

215 returns a numpy array of tile IDs that intersect the circle 

216 """ 

217 valid_id = [] 

218 radius_rad = np.radians(radius) 

219 center_pt = cartesianFromSpherical(np.radians(ra), np.radians(dec)) 

220 for tile_id in self._tile_dict: 

221 tile = self._tile_dict[tile_id] 

222 is_contained = tile.intersects_circle(center_pt, radius_rad) 

223 if is_contained: 

224 valid_id.append(tile_id) 

225 

226 return np.array(valid_id) 

227 

228 

229class UWGalaxyChunkIterator(ChunkIterator): 

230 

231 def __init__(self, dbobj, colnames, obs_metadata, chunk_size, constraint): 

232 """ 

233 Parameters 

234 ---------- 

235 dbobj -- a CatalogDBObject connected to the 'galaxies' table on 

236 the UW CatSim server 

237 

238 colnames -- a list of the columns to query 

239 

240 chunk_size -- size of chunks to return 

241 

242 constraint -- a string specifying a SQL 'WHERE' clause 

243 """ 

244 self.arbitrarySQL = False 

245 self.dbobj = dbobj 

246 if 'ra' not in colnames: 

247 query_colnames = ['htmid', 'galid', 'ra', 'dec'] + colnames 

248 else: 

249 query_colnames = ['htmid', 'galid'] + colnames 

250 self._column_query = dbobj._get_column_query(query_colnames) 

251 self.chunk_size = chunk_size 

252 tile_idx_list = np.sort(self._find_tiles(obs_metadata)) 

253 self._trixel_search_level = 9 

254 self.obs_metadata = obs_metadata 

255 total_trixel_bounds = [] 

256 self._00_bounds = [] 

257 self._rotate_to_sky = [] 

258 self._sky_tile = [] 

259 self._tile_idx = [] 

260 

261 # construct a HalfSpace based on obs_metadata 

262 self.obs_hs = halfSpaceFromRaDec(obs_metadata.pointingRA, 

263 obs_metadata.pointingDec, 

264 obs_metadata.boundLength) 

265 

266 obs_where_clause = "(" 

267 for tile_idx in tile_idx_list: 

268 rotate_to_00 = self.uwgalaxy_tiles.rotation_matrix(tile_idx) 

269 

270 # find the bounds for trixels contained by the field of view 

271 # when rotated from the current tile to RA=Dec=0 

272 new_vv = np.dot(rotate_to_00, self.obs_hs.vector) 

273 new_ra, new_dec = sphericalFromCartesian(new_vv) 

274 new_obs = ObservationMetaData(pointingRA=np.degrees(new_ra), 

275 pointingDec=np.degrees(new_dec), 

276 boundType='circle', 

277 boundLength=self.obs_metadata.boundLength) 

278 

279 if obs_where_clause != "(": 

280 obs_where_clause += " OR (" 

281 else: 

282 obs_where_clause += "(" 

283 obs_where_clause += new_obs.bounds.to_SQL('ra', 'dec') 

284 obs_where_clause += ")" 

285 

286 obs_hs_00 = HalfSpace(new_vv, self.obs_hs.dd) 

287 obs_hs_00_trixels = obs_hs_00.findAllTrixels(self._trixel_search_level) 

288 

289 # find the trixels in the current tile when it is rotated 

290 # to RA=Dec=0 

291 sky_tile = self.uwgalaxy_tiles.tile(tile_idx) 

292 single_tile = sky_tile.rotate(rotate_to_00) 

293 local_bounds = single_tile.find_all_trixels(self._trixel_search_level) 

294 local_bounds = HalfSpace.join_trixel_bound_sets(local_bounds, obs_hs_00_trixels) 

295 

296 total_trixel_bounds += local_bounds 

297 

298 self._sky_tile.append(sky_tile) 

299 self._00_bounds.append(local_bounds) 

300 self._rotate_to_sky.append(np.linalg.inv(rotate_to_00)) 

301 self._tile_idx.append(tile_idx) 

302 obs_where_clause += ")" 

303 

304 total_trixel_bounds = HalfSpace.merge_trixel_bounds(total_trixel_bounds) 

305 

306 where_clause = "(" 

307 for i_bound, bound in enumerate(total_trixel_bounds): 

308 if i_bound>0: 

309 where_clause += " OR " 

310 htmid_min = bound[0] << 2*(21-self._trixel_search_level) 

311 htmid_max = (bound[1]+1) << 2*(21-self._trixel_search_level) 

312 assert levelFromHtmid(htmid_min) == 21 

313 assert levelFromHtmid(htmid_max) == 21 

314 assert htmid_min<htmid_max 

315 where_clause += "(htmid>=%d AND htmid<=%d)" % (htmid_min, htmid_max) 

316 where_clause += ")" 

317 where_clause += " AND " 

318 where_clause += obs_where_clause 

319 

320 if constraint is not None: 

321 where_clause += " AND (%s)" % text(constraint) 

322 

323 query = self._column_query 

324 query = query.filter(text(where_clause)) 

325 query = query.order_by('redshift') 

326 

327 self._galaxy_query = dbobj.connection.session.execute(query) 

328 self._tile_to_do = 0 

329 

330 self._has_J2000 = False 

331 if 'raJ2000' in colnames: 

332 self._has_J2000 = True 

333 self._valid_tiles = 0 

334 self._n_chunks = 0 

335 self._n_rows = 0 

336 self._rows_kept = 0 

337 

338 

339 def __next__(self): 

340 if self._tile_to_do == 0: 

341 self._valid_tiles = 0 

342 self._n_chunks += 1 

343 if self.chunk_size is None and not self._galaxy_query.closed: 

344 results = self._galaxy_query.fetchall() 

345 elif self.chunk_size is not None: 

346 results = self._galaxy_query.fetchmany(self.chunk_size) 

347 else: 

348 raise StopIteration 

349 self._galaxy_cache = self.dbobj._convert_results_to_numpy_recarray_catalogDBObj(results) 

350 self._n_rows += len(self._galaxy_cache) 

351 if len(self._galaxy_cache) == 0: 

352 raise StopIteration 

353 

354 current_chunk = copy.deepcopy(self._galaxy_cache) 

355 rot_mat = self._rotate_to_sky[self._tile_to_do] 

356 bounds = self._00_bounds[self._tile_to_do] 

357 sky_tile = self._sky_tile[self._tile_to_do] 

358 tile_idx = self._tile_idx[self._tile_to_do] 

359 

360 make_the_cut = None 

361 for bb in bounds: 

362 htmid_min = bb[0] << 2*(21-self._trixel_search_level) 

363 htmid_max = (bb[1]+1) << 2*(21-self._trixel_search_level) 

364 valid = ((current_chunk['htmid']>=htmid_min) & (current_chunk['htmid']<=htmid_max)) 

365 if make_the_cut is None: 

366 make_the_cut = valid 

367 else: 

368 make_the_cut |= valid 

369 

370 good_dexes = np.where(make_the_cut)[0] 

371 if len(good_dexes) < len(current_chunk): 

372 current_chunk = current_chunk[good_dexes] 

373 

374 self._tile_to_do += 1 

375 if self._tile_to_do >= len(self._rotate_to_sky): 

376 self._tile_to_do = 0 

377 

378 if len(current_chunk) == 0: 

379 return self.__next__() 

380 

381 xyz = cartesianFromSpherical(np.radians(current_chunk['ra']), 

382 np.radians(current_chunk['dec'])) 

383 

384 xyz_sky = np.dot(rot_mat, xyz.transpose()).transpose() 

385 

386 final_cut = sky_tile.contains_many_pts(xyz_sky) 

387 final_cut &= self.obs_hs.contains_many_pts(xyz_sky) 

388 final_cut = np.where(final_cut) 

389 

390 xyz_sky = xyz_sky[final_cut] 

391 current_chunk = current_chunk[final_cut] 

392 if len(current_chunk) == 0: 

393 return self.__next__() 

394 

395 ra_dec_sky = sphericalFromCartesian(xyz_sky) 

396 current_chunk['ra'] = np.degrees(ra_dec_sky[0]) % 360.0 

397 current_chunk['dec'] = np.degrees(ra_dec_sky[1]) % 360.0 

398 current_chunk['dec'] = np.where(current_chunk['dec']<270.0, 

399 current_chunk['dec'], 

400 current_chunk['dec']-360.0) 

401 current_chunk['dec'] = np.where(np.abs(current_chunk['dec'])<=90.0, 

402 current_chunk['dec'], 

403 180.0-current_chunk['dec']) 

404 if self._has_J2000: 

405 current_chunk['raJ2000'] = ra_dec_sky[0] % (2.0*np.pi) 

406 _dec = ra_dec_sky[1] % (2.0*np.pi) 

407 current_chunk['decJ2000'] = np.where(_dec<1.5*np.pi, 

408 _dec, 

409 _dec-2.0*np.pi) 

410 current_chunk['decJ2000'] = np.where(np.abs(current_chunk['decJ2000'])<=0.5*np.pi, 

411 current_chunk['decJ2000'], 

412 np.pi-current_chunk['decJ2000']) 

413 

414 

415 #>>> r2 = recfunc.append_fields(r,['d','e'],d,dtypes=[float, int], usemask=False, asrecarray=True) 

416 

417 galtileid = tile_idx*100000000+current_chunk['id'] 

418 current_chunk = np_recfn.append_fields(current_chunk, ['galtileid'], [galtileid], 

419 dtypes=[int], usemask=False, asrecarray=True) 

420 

421 self._valid_tiles += 1 

422 self._rows_kept += len(current_chunk) 

423 return self._postprocess_results(current_chunk) 

424 

425 @property 

426 def uwgalaxy_tiles(self): 

427 if not hasattr(self, '_uwgalaxy_tiles'): 

428 self._uwgalaxy_tiles = UWGalaxyTiles() 

429 return self._uwgalaxy_tiles 

430 

431 def _find_tiles(self, obs_metadata): 

432 

433 if obs_metadata.boundType != 'circle': 

434 raise RuntimeError("Cannot use ObservationMetaData with " 

435 "boundType == %s in UWGalaxyTileObj" % obs_metadata.boundType) 

436 

437 return self.uwgalaxy_tiles.find_all_tiles(obs_metadata.pointingRA, 

438 obs_metadata.pointingDec, 

439 obs_metadata.boundLength) 

440 

441 

442 

443class UWGalaxyTileObj(GalaxyObj): 

444 _class_constraint = None 

445 

446 def query_columns(self, colnames=None, chunk_size=None, obs_metadata=None, constraint=None, 

447 limit=None): 

448 """Execute a query 

449 

450 **Parameters** 

451 

452 * colnames : list or None 

453 a list of valid column names, corresponding to entries in the 

454 `columns` class attribute. If not specified, all columns are 

455 queried. 

456 * chunksize : int (optional) 

457 if specified, then return an iterator object to query the database, 

458 each time returning the next `chunksize` elements. If not 

459 specified, all matching results will be returned. 

460 * obs_metadata : object (optional) 

461 object containing information on the observation including the region of the sky 

462 to query and time of the observation. 

463 * constraint : string (optional) 

464 if specified, the predicate is added to the query verbatim using AND 

465 * limit: 

466 This kwarg is not actually used. It exists to preserve the same interface 

467 as other definitions of query_columns elsewhere in CatSim. If not None, 

468 a warning will be emitted, pointing out to the user that 'limit' is not used. 

469 

470 **Returns** 

471 

472 * result : structured array or iterator 

473 If chunksize is not specified, then result is a structured array of all 

474 items which match the specified query with columns named by the column 

475 names in the columns class attribute. If chunksize is specified, 

476 then result is an iterator over structured arrays of the given size. 

477 

478 """ 

479 

480 if colnames is None: 

481 colnames = [k for k in self.columnMap.keys()] 

482 

483 # We know that galtileid comes back with the query, but we don't want 

484 # to add it to the query since it's generated on the fly. 

485 # 

486 # 25 August 2015 

487 # The code below has been modified to remove all column names 

488 # that contain 'galtileid.' This is to accommodate the 

489 # CompoundInstanceCatalog and CompoundDBObject classes, which 

490 # mangle column names such that they include the objid of the 

491 # specific CatalogDBObject that is asking for them. 

492 query_colnames = copy.deepcopy(colnames) 

493 

494 for name in query_colnames: 

495 if 'galtileid' in name: 

496 query_colnames.remove(name) 

497 

498 if limit is not None: 

499 warnings.warn("You specified a row number limit in your query of a UWGalaxyTileObj " 

500 "daughter class. Because of the way UWGalaxyTileObj is searched, row " 

501 "number limits are not possible. If you really want to limit the number " 

502 "of rows returned by you query, consider using GalaxyObj (note that you " 

503 "will have to you limit your search to -2.5<RA<2.5 -2.25<Dec<2.25 -- both in " 

504 "degrees -- as this is the only region where galaxies exist in GalaxyObj).") 

505 

506 # should probably write a new ChunkIterator that will do the query once 

507 # and then selectively munge the outputs per relevant tile 

508 

509 if constraint is not None and self._class_constraint is not None: 

510 constraint = '(%s AND %s)' % (constraint, text(self._class_constraint)) 

511 elif constraint is None and self._class_constraint is not None: 

512 constraint = '(%s)' % text(self._class_constraint) 

513 

514 return UWGalaxyChunkIterator(self, query_colnames, obs_metadata, 

515 chunk_size, constraint) 

516 

517 

518class UWGalaxyBulgeObj(UWGalaxyTileObj): 

519 columns = GalaxyBulgeObj.columns 

520 tableid = 'galaxy_disk' 

521 

522 

523class UWGalaxyDiskObj(UWGalaxyTileObj): 

524 columns = GalaxyDiskObj.columns 

525 tableid = 'galaxy_disk' 

526 

527 

528class UWGalaxyAgnObj(UWGalaxyTileObj): 

529 columns = GalaxyAgnObj.columns 

530 tableid = 'galaxy_agn'