Coverage for python/lsst/sims/catUtils/baseCatalogModels/UWGalaxyModels.py : 12%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import os
2import numpy as np
3import numpy.lib.recfunctions as np_recfn
4import json
5import copy
6from sqlalchemy import text
7from lsst.utils import getPackageDir
8from lsst.sims.utils import ObservationMetaData
9from lsst.sims.utils import HalfSpace, levelFromHtmid
10from lsst.sims.utils import halfSpaceFromRaDec
11from lsst.sims.utils import halfSpaceFromPoints
12from lsst.sims.utils import intersectHalfSpaces
13from lsst.sims.utils import cartesianFromSpherical, sphericalFromCartesian
14from lsst.sims.catalogs.db import ChunkIterator
15from lsst.sims.catUtils.baseCatalogModels import GalaxyObj
16from lsst.sims.catUtils.baseCatalogModels import GalaxyBulgeObj
17from lsst.sims.catUtils.baseCatalogModels import GalaxyDiskObj
18from lsst.sims.catUtils.baseCatalogModels import GalaxyAgnObj
20__all__ = ["Tile", "UWGalaxyTiles", "UWGalaxyTileObj",
21 "UWGalaxyDiskObj", "UWGalaxyBulgeObj",
22 "UWGalaxyAgnObj"]
25class Tile(object):
27 def __init__(self, box_corners):
28 self._trixel_bounds = None
29 self._trixel_bound_level = None
30 self._hs_list = []
31 if len(box_corners) == 0:
32 return
33 self._init_from_corners(box_corners)
35 def _init_from_corners(self, box_corners):
36 ra_range = [c[0] for c in box_corners]
37 ra_min = min(ra_range)
38 ra_max = max(ra_range)
39 dec_range = [c[1] for c in box_corners]
40 dec_min = min(dec_range)
41 dec_max = max(dec_range)
42 tol = 1.0e-10
43 for i_c1 in range(len(box_corners)):
44 c1 = box_corners[i_c1]
45 pt1 = cartesianFromSpherical(np.degrees(c1[0]), np.degrees(c1[1]))
46 for i_c2 in range(i_c1+1, len(box_corners), 1):
47 hs = None
48 c2 = box_corners[i_c2]
49 pt2 = cartesianFromSpherical(np.degrees(c2[0]), np.degrees(c2[1]))
50 if np.abs(1.0-np.dot(pt1, pt2))<tol:
51 continue
53 dra = np.abs(c1[0]-c2[0])
54 ddec = np.abs(c1[1]-c2[1])
56 if dra<tol and ddec>tol:
57 # The RAs of the two corners is identical, but the Decs are
58 # different; this Half Space is defined by a Great Circle
59 if np.abs(c1[0]-ra_min)<tol:
60 inner_pt = (ra_min+0.001, dec_min+0.001)
61 else:
62 inner_pt = (ra_max-0.001, dec_min+0.001)
63 hs = halfSpaceFromPoints(c1, c2, inner_pt)
64 elif ddec<tol and dra>tol:
65 # The Decs of the two corners is identical, bu the RAs are
66 # different; this Half Space is defined by a line of constant
67 # Dec and should be centered at one of the poles
68 if np.abs(c1[1]-dec_min)<tol:
69 hs = halfSpaceFromRaDec(0.0, 90.0, 90.0-dec_min)
70 else:
71 hs = halfSpaceFromRaDec(0.0, -90.0, 90.0+dec_max)
72 else:
73 continue
75 if hs is None:
76 raise RuntimeError("Somehow Half Space == None")
77 self._hs_list.append(hs)
79 def contains_many_pts(self, pts):
80 result = None
81 for hs in self.half_space_list:
82 valid = hs.contains_many_pts(pts)
83 if result is None:
84 result = valid
85 else:
86 result &= valid
87 return result
89 @property
90 def half_space_list(self):
91 return self._hs_list
93 def rotate(self, matrix):
94 new_tile = Tile([])
95 for hs in self.half_space_list:
96 vv = np.dot(matrix, hs.vector)
97 new_hs = HalfSpace(vv, hs.dd)
98 new_tile._hs_list.append(new_hs)
99 return new_tile
101 def intersects_circle(self, center_pt, radius_rad):
102 gross_is_contained = True
103 for hs in self.half_space_list:
104 if not hs.intersects_circle(center_pt, radius_rad):
105 gross_is_contained = False
106 break
107 if not gross_is_contained:
108 return False
110 hs_interest = HalfSpace(center_pt, np.cos(radius_rad))
111 for i_h1 in range(len(self.half_space_list)):
112 hs1 = self.half_space_list[i_h1]
113 roots = intersectHalfSpaces(hs1, hs_interest)
114 if len(roots) == 0:
115 continue
117 if len(roots.shape)==1:
118 roots = [roots]
120 for i_h2 in range(len(self.half_space_list)):
121 if i_h1 == i_h2:
122 continue
123 hs2 = self.half_space_list[i_h2]
124 local_contained = False
125 for rr in roots:
126 if hs2.contains_pt(rr, tol=1.0e-10):
127 local_contained = True
128 break
129 if not local_contained:
130 return False
131 return True
133 def _generate_all_trixels(self, level):
134 output = None
135 for hs in self.half_space_list:
136 local_limits = hs.findAllTrixels(level)
137 if output is None:
138 output = local_limits
139 else:
140 output = HalfSpace.join_trixel_bound_sets(output, local_limits)
141 self._trixel_bounds = output
142 self._trixel_bound_level = level
143 return None
145 @property
146 def trixel_bound_level(self):
147 return self._trixel_bound_level
149 @property
150 def trixel_bounds(self):
151 return self._trixel_bounds
153 def find_all_trixels(self, level):
154 if self._trixel_bounds is None or self.trixel_bound_level != level:
155 self._generate_all_trixels(level)
156 return self._trixel_bounds
159class UWGalaxyTiles(object):
160 """
161 A class to store the UW CatSim server galaxy tiles as a series of
162 Half Spaces
163 """
165 def __init__(self):
166 data_dir = os.path.join(getPackageDir('sims_catUtils'), 'data')
167 data_file = os.path.join(data_dir, 'tile_data.txt')
168 dtype = np.dtype([('id', int), ('ra', float), ('dec', float),
169 ('box', str, 500)])
170 tile_data = np.genfromtxt(data_file, dtype=dtype, delimiter=';')
172 self._tile_id = tile_data['id']
173 self._tile_ra = {}
174 self._tile_dec = {}
175 self._rotation_matrix_dict = {}
176 for ii, rr, dd, in zip(tile_data['id'], tile_data['ra'], tile_data['dec']):
177 self._tile_ra[ii] = rr
178 self._tile_dec[ii] = dd
179 ra_rad = np.radians(rr)
180 dec_rad = np.radians(dd)
182 ra_mat = np.array([[np.cos(ra_rad), np.sin(ra_rad), 0.0],
183 [-np.sin(ra_rad), np.cos(ra_rad), 0.0],
184 [0.0, 0.0, 1.0]])
186 dec_mat = np.array([[np.cos(dec_rad), 0.0, np.sin(dec_rad)],
187 [0.0, 1.0, 0.0],
188 [-np.sin(dec_rad), 0.0, np.cos(dec_rad)]])
190 full_mat = np.dot(dec_mat, ra_mat)
191 self._rotation_matrix_dict[ii] = full_mat
193 self._tile_dict = {}
194 for tile_id, box in zip(tile_data['id'], tile_data['box']):
195 box_corners = json.loads(box)
196 self._tile_dict[tile_id] = Tile(box_corners)
199 def tile_ra(self, tile_idx):
200 return self._tile_ra[tile_idx]
202 def tile_dec(self, tile_idx):
203 return self._tile_dec[tile_idx]
205 def rotation_matrix(self, tile_idx):
206 return self._rotation_matrix_dict[tile_idx]
208 def tile(self, tile_idx):
209 return self._tile_dict[tile_idx]
211 def find_all_tiles(self, ra, dec, radius):
212 """
213 ra, dec, radius are all in degrees
215 returns a numpy array of tile IDs that intersect the circle
216 """
217 valid_id = []
218 radius_rad = np.radians(radius)
219 center_pt = cartesianFromSpherical(np.radians(ra), np.radians(dec))
220 for tile_id in self._tile_dict:
221 tile = self._tile_dict[tile_id]
222 is_contained = tile.intersects_circle(center_pt, radius_rad)
223 if is_contained:
224 valid_id.append(tile_id)
226 return np.array(valid_id)
229class UWGalaxyChunkIterator(ChunkIterator):
231 def __init__(self, dbobj, colnames, obs_metadata, chunk_size, constraint):
232 """
233 Parameters
234 ----------
235 dbobj -- a CatalogDBObject connected to the 'galaxies' table on
236 the UW CatSim server
238 colnames -- a list of the columns to query
240 chunk_size -- size of chunks to return
242 constraint -- a string specifying a SQL 'WHERE' clause
243 """
244 self.arbitrarySQL = False
245 self.dbobj = dbobj
246 if 'ra' not in colnames:
247 query_colnames = ['htmid', 'galid', 'ra', 'dec'] + colnames
248 else:
249 query_colnames = ['htmid', 'galid'] + colnames
250 self._column_query = dbobj._get_column_query(query_colnames)
251 self.chunk_size = chunk_size
252 tile_idx_list = np.sort(self._find_tiles(obs_metadata))
253 self._trixel_search_level = 9
254 self.obs_metadata = obs_metadata
255 total_trixel_bounds = []
256 self._00_bounds = []
257 self._rotate_to_sky = []
258 self._sky_tile = []
259 self._tile_idx = []
261 # construct a HalfSpace based on obs_metadata
262 self.obs_hs = halfSpaceFromRaDec(obs_metadata.pointingRA,
263 obs_metadata.pointingDec,
264 obs_metadata.boundLength)
266 obs_where_clause = "("
267 for tile_idx in tile_idx_list:
268 rotate_to_00 = self.uwgalaxy_tiles.rotation_matrix(tile_idx)
270 # find the bounds for trixels contained by the field of view
271 # when rotated from the current tile to RA=Dec=0
272 new_vv = np.dot(rotate_to_00, self.obs_hs.vector)
273 new_ra, new_dec = sphericalFromCartesian(new_vv)
274 new_obs = ObservationMetaData(pointingRA=np.degrees(new_ra),
275 pointingDec=np.degrees(new_dec),
276 boundType='circle',
277 boundLength=self.obs_metadata.boundLength)
279 if obs_where_clause != "(":
280 obs_where_clause += " OR ("
281 else:
282 obs_where_clause += "("
283 obs_where_clause += new_obs.bounds.to_SQL('ra', 'dec')
284 obs_where_clause += ")"
286 obs_hs_00 = HalfSpace(new_vv, self.obs_hs.dd)
287 obs_hs_00_trixels = obs_hs_00.findAllTrixels(self._trixel_search_level)
289 # find the trixels in the current tile when it is rotated
290 # to RA=Dec=0
291 sky_tile = self.uwgalaxy_tiles.tile(tile_idx)
292 single_tile = sky_tile.rotate(rotate_to_00)
293 local_bounds = single_tile.find_all_trixels(self._trixel_search_level)
294 local_bounds = HalfSpace.join_trixel_bound_sets(local_bounds, obs_hs_00_trixels)
296 total_trixel_bounds += local_bounds
298 self._sky_tile.append(sky_tile)
299 self._00_bounds.append(local_bounds)
300 self._rotate_to_sky.append(np.linalg.inv(rotate_to_00))
301 self._tile_idx.append(tile_idx)
302 obs_where_clause += ")"
304 total_trixel_bounds = HalfSpace.merge_trixel_bounds(total_trixel_bounds)
306 where_clause = "("
307 for i_bound, bound in enumerate(total_trixel_bounds):
308 if i_bound>0:
309 where_clause += " OR "
310 htmid_min = bound[0] << 2*(21-self._trixel_search_level)
311 htmid_max = (bound[1]+1) << 2*(21-self._trixel_search_level)
312 assert levelFromHtmid(htmid_min) == 21
313 assert levelFromHtmid(htmid_max) == 21
314 assert htmid_min<htmid_max
315 where_clause += "(htmid>=%d AND htmid<=%d)" % (htmid_min, htmid_max)
316 where_clause += ")"
317 where_clause += " AND "
318 where_clause += obs_where_clause
320 if constraint is not None:
321 where_clause += " AND (%s)" % text(constraint)
323 query = self._column_query
324 query = query.filter(text(where_clause))
325 query = query.order_by('redshift')
327 self._galaxy_query = dbobj.connection.session.execute(query)
328 self._tile_to_do = 0
330 self._has_J2000 = False
331 if 'raJ2000' in colnames:
332 self._has_J2000 = True
333 self._valid_tiles = 0
334 self._n_chunks = 0
335 self._n_rows = 0
336 self._rows_kept = 0
339 def __next__(self):
340 if self._tile_to_do == 0:
341 self._valid_tiles = 0
342 self._n_chunks += 1
343 if self.chunk_size is None and not self._galaxy_query.closed:
344 results = self._galaxy_query.fetchall()
345 elif self.chunk_size is not None:
346 results = self._galaxy_query.fetchmany(self.chunk_size)
347 else:
348 raise StopIteration
349 self._galaxy_cache = self.dbobj._convert_results_to_numpy_recarray_catalogDBObj(results)
350 self._n_rows += len(self._galaxy_cache)
351 if len(self._galaxy_cache) == 0:
352 raise StopIteration
354 current_chunk = copy.deepcopy(self._galaxy_cache)
355 rot_mat = self._rotate_to_sky[self._tile_to_do]
356 bounds = self._00_bounds[self._tile_to_do]
357 sky_tile = self._sky_tile[self._tile_to_do]
358 tile_idx = self._tile_idx[self._tile_to_do]
360 make_the_cut = None
361 for bb in bounds:
362 htmid_min = bb[0] << 2*(21-self._trixel_search_level)
363 htmid_max = (bb[1]+1) << 2*(21-self._trixel_search_level)
364 valid = ((current_chunk['htmid']>=htmid_min) & (current_chunk['htmid']<=htmid_max))
365 if make_the_cut is None:
366 make_the_cut = valid
367 else:
368 make_the_cut |= valid
370 good_dexes = np.where(make_the_cut)[0]
371 if len(good_dexes) < len(current_chunk):
372 current_chunk = current_chunk[good_dexes]
374 self._tile_to_do += 1
375 if self._tile_to_do >= len(self._rotate_to_sky):
376 self._tile_to_do = 0
378 if len(current_chunk) == 0:
379 return self.__next__()
381 xyz = cartesianFromSpherical(np.radians(current_chunk['ra']),
382 np.radians(current_chunk['dec']))
384 xyz_sky = np.dot(rot_mat, xyz.transpose()).transpose()
386 final_cut = sky_tile.contains_many_pts(xyz_sky)
387 final_cut &= self.obs_hs.contains_many_pts(xyz_sky)
388 final_cut = np.where(final_cut)
390 xyz_sky = xyz_sky[final_cut]
391 current_chunk = current_chunk[final_cut]
392 if len(current_chunk) == 0:
393 return self.__next__()
395 ra_dec_sky = sphericalFromCartesian(xyz_sky)
396 current_chunk['ra'] = np.degrees(ra_dec_sky[0]) % 360.0
397 current_chunk['dec'] = np.degrees(ra_dec_sky[1]) % 360.0
398 current_chunk['dec'] = np.where(current_chunk['dec']<270.0,
399 current_chunk['dec'],
400 current_chunk['dec']-360.0)
401 current_chunk['dec'] = np.where(np.abs(current_chunk['dec'])<=90.0,
402 current_chunk['dec'],
403 180.0-current_chunk['dec'])
404 if self._has_J2000:
405 current_chunk['raJ2000'] = ra_dec_sky[0] % (2.0*np.pi)
406 _dec = ra_dec_sky[1] % (2.0*np.pi)
407 current_chunk['decJ2000'] = np.where(_dec<1.5*np.pi,
408 _dec,
409 _dec-2.0*np.pi)
410 current_chunk['decJ2000'] = np.where(np.abs(current_chunk['decJ2000'])<=0.5*np.pi,
411 current_chunk['decJ2000'],
412 np.pi-current_chunk['decJ2000'])
415 #>>> r2 = recfunc.append_fields(r,['d','e'],d,dtypes=[float, int], usemask=False, asrecarray=True)
417 galtileid = tile_idx*100000000+current_chunk['id']
418 current_chunk = np_recfn.append_fields(current_chunk, ['galtileid'], [galtileid],
419 dtypes=[int], usemask=False, asrecarray=True)
421 self._valid_tiles += 1
422 self._rows_kept += len(current_chunk)
423 return self._postprocess_results(current_chunk)
425 @property
426 def uwgalaxy_tiles(self):
427 if not hasattr(self, '_uwgalaxy_tiles'):
428 self._uwgalaxy_tiles = UWGalaxyTiles()
429 return self._uwgalaxy_tiles
431 def _find_tiles(self, obs_metadata):
433 if obs_metadata.boundType != 'circle':
434 raise RuntimeError("Cannot use ObservationMetaData with "
435 "boundType == %s in UWGalaxyTileObj" % obs_metadata.boundType)
437 return self.uwgalaxy_tiles.find_all_tiles(obs_metadata.pointingRA,
438 obs_metadata.pointingDec,
439 obs_metadata.boundLength)
443class UWGalaxyTileObj(GalaxyObj):
444 _class_constraint = None
446 def query_columns(self, colnames=None, chunk_size=None, obs_metadata=None, constraint=None,
447 limit=None):
448 """Execute a query
450 **Parameters**
452 * colnames : list or None
453 a list of valid column names, corresponding to entries in the
454 `columns` class attribute. If not specified, all columns are
455 queried.
456 * chunksize : int (optional)
457 if specified, then return an iterator object to query the database,
458 each time returning the next `chunksize` elements. If not
459 specified, all matching results will be returned.
460 * obs_metadata : object (optional)
461 object containing information on the observation including the region of the sky
462 to query and time of the observation.
463 * constraint : string (optional)
464 if specified, the predicate is added to the query verbatim using AND
465 * limit:
466 This kwarg is not actually used. It exists to preserve the same interface
467 as other definitions of query_columns elsewhere in CatSim. If not None,
468 a warning will be emitted, pointing out to the user that 'limit' is not used.
470 **Returns**
472 * result : structured array or iterator
473 If chunksize is not specified, then result is a structured array of all
474 items which match the specified query with columns named by the column
475 names in the columns class attribute. If chunksize is specified,
476 then result is an iterator over structured arrays of the given size.
478 """
480 if colnames is None:
481 colnames = [k for k in self.columnMap.keys()]
483 # We know that galtileid comes back with the query, but we don't want
484 # to add it to the query since it's generated on the fly.
485 #
486 # 25 August 2015
487 # The code below has been modified to remove all column names
488 # that contain 'galtileid.' This is to accommodate the
489 # CompoundInstanceCatalog and CompoundDBObject classes, which
490 # mangle column names such that they include the objid of the
491 # specific CatalogDBObject that is asking for them.
492 query_colnames = copy.deepcopy(colnames)
494 for name in query_colnames:
495 if 'galtileid' in name:
496 query_colnames.remove(name)
498 if limit is not None:
499 warnings.warn("You specified a row number limit in your query of a UWGalaxyTileObj "
500 "daughter class. Because of the way UWGalaxyTileObj is searched, row "
501 "number limits are not possible. If you really want to limit the number "
502 "of rows returned by you query, consider using GalaxyObj (note that you "
503 "will have to you limit your search to -2.5<RA<2.5 -2.25<Dec<2.25 -- both in "
504 "degrees -- as this is the only region where galaxies exist in GalaxyObj).")
506 # should probably write a new ChunkIterator that will do the query once
507 # and then selectively munge the outputs per relevant tile
509 if constraint is not None and self._class_constraint is not None:
510 constraint = '(%s AND %s)' % (constraint, text(self._class_constraint))
511 elif constraint is None and self._class_constraint is not None:
512 constraint = '(%s)' % text(self._class_constraint)
514 return UWGalaxyChunkIterator(self, query_colnames, obs_metadata,
515 chunk_size, constraint)
518class UWGalaxyBulgeObj(UWGalaxyTileObj):
519 columns = GalaxyBulgeObj.columns
520 tableid = 'galaxy_disk'
523class UWGalaxyDiskObj(UWGalaxyTileObj):
524 columns = GalaxyDiskObj.columns
525 tableid = 'galaxy_disk'
528class UWGalaxyAgnObj(UWGalaxyTileObj):
529 columns = GalaxyAgnObj.columns
530 tableid = 'galaxy_agn'