Coverage for tests/testAllObjects.py : 11%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import print_function
2from builtins import zip
3from builtins import next
4import os
5import inspect
6import numpy as np
7import sys
8import traceback
9import unittest
10import tempfile
11import shutil
12import lsst.utils.tests
14from lsst.utils import getPackageDir
15from lsst.sims.catalogs.db import CatalogDBObject
16from lsst.sims.catalogs.definitions import InstanceCatalog
17from lsst.sims.catUtils.exampleCatalogDefinitions import ObsStarCatalogBase
18from lsst.sims.catUtils.utils import failedOnFatboy
19# The following is to get the object ids in the registry
20import lsst.sims.catUtils.baseCatalogModels as bcm
23def setup_module(module):
24 lsst.utils.tests.init()
27class DummyCat(InstanceCatalog):
28 catalog_type = __file__ + 'unit_test_catalog'
29 column_outputs = ['raJ2000', 'decJ2000']
32class basicAccessTest(unittest.TestCase):
34 longMessage = True
36 def testObjects(self):
37 catDir = tempfile.mkdtemp('basicAccessTest_testObjects')
38 if not os.path.exists(catDir):
39 os.mkdir(catDir)
40 catName = tempfile.mktemp(prefix='basicAccessTest_testObjects',
41 dir=catDir, suffix='.txt')
42 ct_connected = 0
43 ct_failed_connection = 0
44 list_of_failures = []
46 for objname, objcls in CatalogDBObject.registry.items():
47 if not objcls.doRunTest or (objcls.testObservationMetaData is None):
48 continue
50 print("Running tests for", objname)
51 try:
52 dbobj = objcls(verbose=False)
53 except:
54 trace = traceback.extract_tb(sys.exc_info()[2], limit=20)
55 msg = sys.exc_info()[1].args[0]
56 if 'Failed to connect' in str(msg) or failedOnFatboy(trace):
58 # if the exception was due to a failed connection
59 # to fatboy, ignore it
61 ct_failed_connection += 1
62 list_of_failures.append(objname)
63 continue
64 else:
65 raise
67 obs_metadata = dbobj.testObservationMetaData
69 # Get results all at once
70 try:
71 result = dbobj.query_columns(obs_metadata=obs_metadata)
72 except:
74 # This is because the solar system object 'tables'
75 # don't actually connect to tables on fatboy; they just
76 # call methods stored on fatboy. Therefore, the connection
77 # failure will not be noticed until this part of the test
79 ct_failed_connection += 1
80 list_of_failures.append(objname)
81 msg = sys.exc_info()[1].args[0]
82 if 'DB-Lib error' in msg:
83 continue
84 else:
85 raise
87 ct_connected += 1
89 # Since there is only one chunk,
90 try:
91 result = next(result)
92 except StopIteration:
93 raise RuntimeError("No results for %s defined in %s"%(objname,
94 inspect.getsourcefile(dbobj.__class__)))
95 if objname.startswith('galaxy'):
96 DummyCat.column_outputs = ['galid', 'raJ2000', 'decJ2000']
97 else:
98 DummyCat.column_outputs = ['raJ2000', 'decJ2000']
99 cat = dbobj.getCatalog(__file__+'unit_test_catalog', obs_metadata)
100 if os.path.exists(catName):
101 os.unlink(catName)
102 try:
103 cat.write_catalog(catName)
104 dtypeList = [(name, np.float) for name in cat._column_outputs]
105 testData = np.genfromtxt(catName, delimiter = ', ',
106 dtype=np.dtype(dtypeList))
107 self.assertGreater(len(testData), 0)
108 finally:
109 if os.path.exists(catName):
110 os.unlink(catName)
112 if os.path.exists(catDir):
113 shutil.rmtree(catDir)
115 self.assertEqual(len(list_of_failures), ct_failed_connection)
117 print('\n================')
118 print('Do not worry about this message')
119 print('sometimes, connections to the UW database fail.')
120 print('It is expected.')
121 print('This is just a tally so that you know how often that happened.')
122 print('successful connections: ', ct_connected)
123 print('failed connections: ', ct_failed_connection)
124 if len(list_of_failures) > 0:
125 print('objects that failed to connect: ', list_of_failures)
127 def testObsCat(self):
128 objname = 'wdstars'
129 catDir = tempfile.mkdtemp('basicAccessTest_testObsCat')
130 if not os.path.exists(catDir):
131 os.mkdir(catDir)
132 catName = tempfile.mktemp(prefix='basicAccessTest_testObsCat',
133 dir=catDir, suffix='.txt')
135 try:
136 dbobj = CatalogDBObject.from_objid(objname)
137 obs_metadata = dbobj.testObservationMetaData
138 # To cover the central ~raft
139 obs_metadata.boundLength = 0.4
140 obs_metadata.rotSkyPos = 0.0
141 cat = dbobj.getCatalog('obs_star_cat', obs_metadata)
142 if os.path.exists(catName):
143 os.unlink(catName)
144 try:
145 cat.write_catalog(catName)
146 dtypeList = [(name, np.float) for name in cat._column_outputs]
147 testData = np.genfromtxt(catName, delimiter = ', ',
148 dtype=np.dtype(dtypeList))
149 self.assertGreater(len(testData), 0)
150 finally:
151 if os.path.exists(catName):
152 os.unlink(catName)
153 if os.path.exists(catDir):
154 shutil.rmtree(catDir)
156 print('\ntestObsCat successfully connected to fatboy')
158 except:
159 trace = traceback.extract_tb(sys.exc_info()[2], limit=20)
160 msg = sys.exc_info()[1].args[0]
161 if 'Failed to connect' in str(msg) or failedOnFatboy(trace):
163 # if the exception was because of a failed connection
164 # to fatboy, ignore it.
166 print('\ntestObsCat failed to connect to fatboy')
167 print('Sometimes that happens. Do not worry.')
169 if os.path.exists(catDir):
170 shutil.rmtree(catDir)
172 pass
173 else:
174 raise
176 def test_limit(self):
177 """
178 Test that the limit kwarg in query_columns behaves correctly.
180 Will test on one star table and one galaxy table.
181 """
182 list_of_failures = []
183 for objcls, clsname in zip((bcm.StarObj, bcm.GalaxyObj), ('StarObj', 'GalaxyObj')):
184 msg = "failed the limit test\noffending class is %s" % clsname
185 try:
186 dbobj = objcls(verbose=False)
187 except:
188 trace = traceback.extract_tb(sys.exc_info()[2], limit=20)
189 msg = sys.exc_info()[1].args[0]
190 if 'Failed to connect' in str(msg) or failedOnFatboy(trace):
192 # if the exception was due to a failed connection
193 # to fatboy, ignore it
194 list_of_failures.append(clsname)
195 continue
196 else:
197 raise
199 obs_metadata = dbobj.testObservationMetaData
201 results = dbobj.query_columns(obs_metadata=obs_metadata)
203 ct_res = 0
204 for chunk in results:
205 for line in chunk:
206 ct_res += 1
208 self.assertGreater(ct_res, 10, msg=msg)
210 limited_results = dbobj.query_columns(obs_metadata=obs_metadata, limit=10)
212 ct_limit = 0
213 for chunk in limited_results:
214 for line in chunk:
215 ct_limit += 1
217 self.assertEqual(ct_limit, 10, msg=msg)
219 if len(list_of_failures) > 0:
220 print("\nList of DBObjects that could not connect to fatboy " \
221 "for the test on the limit kwarg")
222 for nn in list_of_failures:
223 print(nn)
225 def test_constraint(self):
226 """
227 Test that passing a constraint into query_columns works (i.e. if I only want
228 to select galaxies whose varParamStr is not NULL).
229 """
230 list_of_failures = []
231 constraint = "varParamStr IS NOT NULL"
232 for objcls, clsname in zip((bcm.GalaxyObj, bcm.GalaxyTileObj), ('GalaxyObj', 'GalaxyTileObj')):
233 msg = "failed the constraint test\noffending class is %s" % clsname
234 try:
235 dbobj = objcls(verbose=False)
236 except:
237 trace = traceback.extract_tb(sys.exc_info()[2], limit=20)
238 msg = sys.exc_info()[1].args[0]
239 if 'Failed to connect' in str(msg) or failedOnFatboy(trace):
241 # if the exception was due to a failed connection
242 # to fatboy, ignore it
243 list_of_failures.append(clsname)
244 continue
245 else:
246 raise
248 obs_metadata = dbobj.testObservationMetaData
250 # query witout a constraint on varParamStr
251 results = dbobj.query_columns(colnames=['raJ2000', 'decJ2000', 'varParamStr'],
252 obs_metadata=obs_metadata)
254 # count total number of rows (ct_res) and number of rows with a null
255 # varParamStr (ct_no_varparamstr). Note that varParamStr will be the
256 # index=3 entry in result rows because the id of the galaxy gets
257 # automatically added to query results.
258 ct_res = 0
259 ct_no_varparamstr = 0
260 for chunk in results:
261 for line in chunk:
262 if line[3] == 'None':
263 ct_no_varparamstr += 1
264 ct_res += 1
266 # run the same query, but demanding that varParamStr is not NULL
267 constrained_results = dbobj.query_columns(colnames=['raJ2000', 'decJ2000', 'varParamStr'],
268 obs_metadata=obs_metadata,
269 constraint=constraint)
271 # count the number of rows with non-NULL varParamStr
272 ct_con = 0
273 for chunk in constrained_results:
274 for line in chunk:
275 ct_con += 1
276 self.assertNotEqual(line[3], 'None')
278 # check that the number of non-NULL varParamStr and NULL varParamStr rows
279 # compare the way that they should
280 self.assertGreater(ct_res, ct_con)
281 self.assertGreater(ct_no_varparamstr, 0)
282 self.assertEqual(ct_res-ct_con, ct_no_varparamstr)
284 if len(list_of_failures) > 0:
285 print("\nList of DBObjects that could not connect to fatboy " \
286 "for the test on the constraint kwarg")
287 for nn in list_of_failures:
288 print(nn)
290 def test_limit_and_constraint(self):
291 """
292 Test that limit and constraint work together
293 """
294 list_of_failures = []
295 constraint = "varParamStr IS NOT NULL"
296 could_connect = True
297 try:
298 dbobj = bcm.GalaxyObj(verbose=False)
299 except:
300 trace = traceback.extract_tb(sys.exc_info()[2], limit=20)
301 msg = sys.exc_info()[1].args[0]
302 if 'Failed to connect' in str(msg) or failedOnFatboy(trace):
304 # if the exception was due to a failed connection
305 # to fatboy, ignore it
306 list_of_failures.append('GalaxyObj')
307 could_connect = False
308 else:
309 raise
311 if could_connect:
312 obs_metadata = dbobj.testObservationMetaData
314 # query with a constraint on varParamStr but no limit
315 results_no_limit = dbobj.query_columns(colnames=['raJ2000', 'decJ2000', 'varParamStr'],
316 obs_metadata=obs_metadata,
317 constraint=constraint)
319 ct_res = 0
320 for chunk in results_no_limit:
321 for line in chunk:
322 self.assertNotEqual(line[3], 'None')
323 ct_res += 1
325 self.assertGreater(ct_res, 1)
327 # run the same query, but limiting the results
328 limited_results = dbobj.query_columns(colnames=['raJ2000', 'decJ2000', 'varParamStr'],
329 obs_metadata=obs_metadata,
330 constraint=constraint,
331 limit=ct_res-1)
332 ct_lim = 0
333 for chunk in limited_results:
334 for line in chunk:
335 ct_lim += 1
336 self.assertNotEqual(line[3], 'None')
338 self.assertEqual(ct_lim, ct_res-1)
340 if len(list_of_failures) > 0:
341 print("\nList of DBObjects that could not connect to fatboy " \
342 "for the test on the constraint and limit kwargs")
343 for nn in list_of_failures:
344 print(nn)
347class MemoryTestClass(lsst.utils.tests.MemoryTestCase):
348 pass
350if __name__ == "__main__": 350 ↛ 351line 350 didn't jump to line 351, because the condition on line 350 was never true
351 lsst.utils.tests.init()
352 unittest.main()