Coverage for tests/testAllObjects.py : 10%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import print_function
2from builtins import zip
3from builtins import next
4import os
5import inspect
6import numpy as np
7import sys
8import traceback
9import unittest
10import tempfile
11import shutil
12import lsst.utils.tests
14from lsst.sims.catalogs.db import CatalogDBObject
15from lsst.sims.catalogs.definitions import InstanceCatalog
16from lsst.sims.catUtils.utils import failedOnFatboy
17# The following is to get the object ids in the registry
18import lsst.sims.catUtils.baseCatalogModels as bcm
21def setup_module(module):
22 lsst.utils.tests.init()
25class DummyCat(InstanceCatalog):
26 catalog_type = __file__ + 'unit_test_catalog'
27 column_outputs = ['raJ2000', 'decJ2000']
30class basicAccessTest(unittest.TestCase):
32 longMessage = True
34 def testObjects(self):
35 catDir = tempfile.mkdtemp('basicAccessTest_testObjects')
36 if not os.path.exists(catDir):
37 os.mkdir(catDir)
38 catName = tempfile.mktemp(prefix='basicAccessTest_testObjects',
39 dir=catDir, suffix='.txt')
40 ct_connected = 0
41 ct_failed_connection = 0
42 list_of_failures = []
44 for objname, objcls in CatalogDBObject.registry.items():
45 if not objcls.doRunTest or (objcls.testObservationMetaData is None):
46 continue
48 print("Running tests for", objname)
49 try:
50 dbobj = objcls(verbose=False)
51 except:
52 trace = traceback.extract_tb(sys.exc_info()[2], limit=20)
53 msg = sys.exc_info()[1].args[0]
54 if 'Failed to connect' in str(msg) or failedOnFatboy(trace):
56 # if the exception was due to a failed connection
57 # to fatboy, ignore it
59 ct_failed_connection += 1
60 list_of_failures.append(objname)
61 continue
62 else:
63 raise
65 obs_metadata = dbobj.testObservationMetaData
67 # Get results all at once
68 try:
69 result = dbobj.query_columns(obs_metadata=obs_metadata)
70 except:
72 # This is because the solar system object 'tables'
73 # don't actually connect to tables on fatboy; they just
74 # call methods stored on fatboy. Therefore, the connection
75 # failure will not be noticed until this part of the test
77 ct_failed_connection += 1
78 list_of_failures.append(objname)
79 msg = sys.exc_info()[1].args[0]
80 if 'DB-Lib error' in msg:
81 continue
82 else:
83 raise
85 ct_connected += 1
87 # Since there is only one chunk,
88 try:
89 result = next(result)
90 except StopIteration:
91 raise RuntimeError("No results for %s defined in %s"%(objname,
92 inspect.getsourcefile(dbobj.__class__)))
93 if objname.startswith('galaxy'):
94 DummyCat.column_outputs = ['galid', 'raJ2000', 'decJ2000']
95 else:
96 DummyCat.column_outputs = ['raJ2000', 'decJ2000']
97 cat = dbobj.getCatalog(__file__+'unit_test_catalog', obs_metadata)
98 if os.path.exists(catName):
99 os.unlink(catName)
100 try:
101 cat.write_catalog(catName)
102 dtypeList = [(name, np.float) for name in cat._column_outputs]
103 testData = np.genfromtxt(catName, delimiter = ', ',
104 dtype=np.dtype(dtypeList))
105 self.assertGreater(len(testData), 0)
106 finally:
107 if os.path.exists(catName):
108 os.unlink(catName)
110 if os.path.exists(catDir):
111 shutil.rmtree(catDir, ignore_errors=True)
113 self.assertEqual(len(list_of_failures), ct_failed_connection)
115 print('\n================')
116 print('Do not worry about this message')
117 print('sometimes, connections to the UW database fail.')
118 print('It is expected.')
119 print('This is just a tally so that you know how often that happened.')
120 print('successful connections: ', ct_connected)
121 print('failed connections: ', ct_failed_connection)
122 if len(list_of_failures) > 0:
123 print('objects that failed to connect: ', list_of_failures)
125 def testObsCat(self):
126 objname = 'wdstars'
127 catDir = tempfile.mkdtemp('basicAccessTest_testObsCat')
128 if not os.path.exists(catDir):
129 os.mkdir(catDir)
130 catName = tempfile.mktemp(prefix='basicAccessTest_testObsCat',
131 dir=catDir, suffix='.txt')
133 try:
134 dbobj = CatalogDBObject.from_objid(objname)
135 obs_metadata = dbobj.testObservationMetaData
136 # To cover the central ~raft
137 obs_metadata.boundLength = 0.4
138 obs_metadata.rotSkyPos = 0.0
139 cat = dbobj.getCatalog('obs_star_cat', obs_metadata)
140 if os.path.exists(catName):
141 os.unlink(catName)
142 try:
143 cat.write_catalog(catName)
144 dtypeList = [(name, np.float) for name in cat._column_outputs]
145 testData = np.genfromtxt(catName, delimiter = ', ',
146 dtype=np.dtype(dtypeList))
147 self.assertGreater(len(testData), 0)
148 finally:
149 if os.path.exists(catName):
150 os.unlink(catName)
151 if os.path.exists(catDir):
152 shutil.rmtree(catDir, ignore_errors=True)
154 print('\ntestObsCat successfully connected to fatboy')
156 except:
157 trace = traceback.extract_tb(sys.exc_info()[2], limit=20)
158 msg = sys.exc_info()[1].args[0]
159 if 'Failed to connect' in str(msg) or failedOnFatboy(trace):
161 # if the exception was because of a failed connection
162 # to fatboy, ignore it.
164 print('\ntestObsCat failed to connect to fatboy')
165 print('Sometimes that happens. Do not worry.')
167 if os.path.exists(catDir):
168 shutil.rmtree(catDir, ignore_errors=True)
170 pass
171 else:
172 raise
174 def test_limit(self):
175 """
176 Test that the limit kwarg in query_columns behaves correctly.
178 Will test on one star table and one galaxy table.
179 """
180 list_of_failures = []
181 for objcls, clsname in zip((bcm.StarObj, bcm.GalaxyObj), ('StarObj', 'GalaxyObj')):
182 msg = "failed the limit test\noffending class is %s" % clsname
183 try:
184 dbobj = objcls(verbose=False)
185 except:
186 trace = traceback.extract_tb(sys.exc_info()[2], limit=20)
187 msg = sys.exc_info()[1].args[0]
188 if 'Failed to connect' in str(msg) or failedOnFatboy(trace):
190 # if the exception was due to a failed connection
191 # to fatboy, ignore it
192 list_of_failures.append(clsname)
193 continue
194 else:
195 raise
197 obs_metadata = dbobj.testObservationMetaData
199 results = dbobj.query_columns(obs_metadata=obs_metadata)
201 ct_res = 0
202 for chunk in results:
203 for line in chunk:
204 ct_res += 1
206 self.assertGreater(ct_res, 10, msg=msg)
208 limited_results = dbobj.query_columns(obs_metadata=obs_metadata, limit=10)
210 ct_limit = 0
211 for chunk in limited_results:
212 for line in chunk:
213 ct_limit += 1
215 self.assertEqual(ct_limit, 10, msg=msg)
217 if len(list_of_failures) > 0:
218 print("\nList of DBObjects that could not connect to fatboy " \
219 "for the test on the limit kwarg")
220 for nn in list_of_failures:
221 print(nn)
223 def test_constraint(self):
224 """
225 Test that passing a constraint into query_columns works (i.e. if I only want
226 to select galaxies whose varParamStr is not NULL).
227 """
228 list_of_failures = []
229 constraint = "varParamStr IS NOT NULL"
230 for objcls, clsname in zip((bcm.GalaxyObj, bcm.GalaxyTileObj), ('GalaxyObj', 'GalaxyTileObj')):
231 msg = "failed the constraint test\noffending class is %s" % clsname
232 try:
233 dbobj = objcls(verbose=False)
234 except:
235 trace = traceback.extract_tb(sys.exc_info()[2], limit=20)
236 msg = sys.exc_info()[1].args[0]
237 if 'Failed to connect' in str(msg) or failedOnFatboy(trace):
239 # if the exception was due to a failed connection
240 # to fatboy, ignore it
241 list_of_failures.append(clsname)
242 continue
243 else:
244 raise
246 obs_metadata = dbobj.testObservationMetaData
248 # query witout a constraint on varParamStr
249 results = dbobj.query_columns(colnames=['raJ2000', 'decJ2000', 'varParamStr'],
250 obs_metadata=obs_metadata)
252 # count total number of rows (ct_res) and number of rows with a null
253 # varParamStr (ct_no_varparamstr). Note that varParamStr will be the
254 # index=3 entry in result rows because the id of the galaxy gets
255 # automatically added to query results.
256 ct_res = 0
257 ct_no_varparamstr = 0
258 for chunk in results:
259 for line in chunk:
260 if line[3] == 'None':
261 ct_no_varparamstr += 1
262 ct_res += 1
264 # run the same query, but demanding that varParamStr is not NULL
265 constrained_results = dbobj.query_columns(colnames=['raJ2000', 'decJ2000', 'varParamStr'],
266 obs_metadata=obs_metadata,
267 constraint=constraint)
269 # count the number of rows with non-NULL varParamStr
270 ct_con = 0
271 for chunk in constrained_results:
272 for line in chunk:
273 ct_con += 1
274 self.assertNotEqual(line[3], 'None')
276 # check that the number of non-NULL varParamStr and NULL varParamStr rows
277 # compare the way that they should
278 self.assertGreater(ct_res, ct_con)
279 self.assertGreater(ct_no_varparamstr, 0)
280 self.assertEqual(ct_res-ct_con, ct_no_varparamstr)
282 if len(list_of_failures) > 0:
283 print("\nList of DBObjects that could not connect to fatboy " \
284 "for the test on the constraint kwarg")
285 for nn in list_of_failures:
286 print(nn)
288 def test_limit_and_constraint(self):
289 """
290 Test that limit and constraint work together
291 """
292 list_of_failures = []
293 constraint = "varParamStr IS NOT NULL"
294 could_connect = True
295 try:
296 dbobj = bcm.GalaxyObj(verbose=False)
297 except:
298 trace = traceback.extract_tb(sys.exc_info()[2], limit=20)
299 msg = sys.exc_info()[1].args[0]
300 if 'Failed to connect' in str(msg) or failedOnFatboy(trace):
302 # if the exception was due to a failed connection
303 # to fatboy, ignore it
304 list_of_failures.append('GalaxyObj')
305 could_connect = False
306 else:
307 raise
309 if could_connect:
310 obs_metadata = dbobj.testObservationMetaData
312 # query with a constraint on varParamStr but no limit
313 results_no_limit = dbobj.query_columns(colnames=['raJ2000', 'decJ2000', 'varParamStr'],
314 obs_metadata=obs_metadata,
315 constraint=constraint)
317 ct_res = 0
318 for chunk in results_no_limit:
319 for line in chunk:
320 self.assertNotEqual(line[3], 'None')
321 ct_res += 1
323 self.assertGreater(ct_res, 1)
325 # run the same query, but limiting the results
326 limited_results = dbobj.query_columns(colnames=['raJ2000', 'decJ2000', 'varParamStr'],
327 obs_metadata=obs_metadata,
328 constraint=constraint,
329 limit=ct_res-1)
330 ct_lim = 0
331 for chunk in limited_results:
332 for line in chunk:
333 ct_lim += 1
334 self.assertNotEqual(line[3], 'None')
336 self.assertEqual(ct_lim, ct_res-1)
338 if len(list_of_failures) > 0:
339 print("\nList of DBObjects that could not connect to fatboy " \
340 "for the test on the constraint and limit kwargs")
341 for nn in list_of_failures:
342 print(nn)
345class MemoryTestClass(lsst.utils.tests.MemoryTestCase):
346 pass
348if __name__ == "__main__": 348 ↛ 349line 348 didn't jump to line 349, because the condition on line 348 was never true
349 lsst.utils.tests.init()
350 unittest.main()