Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from __future__ import print_function 

2from builtins import zip 

3from builtins import next 

4import os 

5import inspect 

6import numpy as np 

7import sys 

8import traceback 

9import unittest 

10import tempfile 

11import shutil 

12import lsst.utils.tests 

13 

14from lsst.sims.catalogs.db import CatalogDBObject 

15from lsst.sims.catalogs.definitions import InstanceCatalog 

16from lsst.sims.catUtils.utils import failedOnFatboy 

17# The following is to get the object ids in the registry 

18import lsst.sims.catUtils.baseCatalogModels as bcm 

19 

20 

21def setup_module(module): 

22 lsst.utils.tests.init() 

23 

24 

25class DummyCat(InstanceCatalog): 

26 catalog_type = __file__ + 'unit_test_catalog' 

27 column_outputs = ['raJ2000', 'decJ2000'] 

28 

29 

30class basicAccessTest(unittest.TestCase): 

31 

32 longMessage = True 

33 

34 def testObjects(self): 

35 catDir = tempfile.mkdtemp('basicAccessTest_testObjects') 

36 if not os.path.exists(catDir): 

37 os.mkdir(catDir) 

38 catName = tempfile.mktemp(prefix='basicAccessTest_testObjects', 

39 dir=catDir, suffix='.txt') 

40 ct_connected = 0 

41 ct_failed_connection = 0 

42 list_of_failures = [] 

43 

44 for objname, objcls in CatalogDBObject.registry.items(): 

45 if not objcls.doRunTest or (objcls.testObservationMetaData is None): 

46 continue 

47 

48 print("Running tests for", objname) 

49 try: 

50 dbobj = objcls(verbose=False) 

51 except: 

52 trace = traceback.extract_tb(sys.exc_info()[2], limit=20) 

53 msg = sys.exc_info()[1].args[0] 

54 if 'Failed to connect' in str(msg) or failedOnFatboy(trace): 

55 

56 # if the exception was due to a failed connection 

57 # to fatboy, ignore it 

58 

59 ct_failed_connection += 1 

60 list_of_failures.append(objname) 

61 continue 

62 else: 

63 raise 

64 

65 obs_metadata = dbobj.testObservationMetaData 

66 

67 # Get results all at once 

68 try: 

69 result = dbobj.query_columns(obs_metadata=obs_metadata) 

70 except: 

71 

72 # This is because the solar system object 'tables' 

73 # don't actually connect to tables on fatboy; they just 

74 # call methods stored on fatboy. Therefore, the connection 

75 # failure will not be noticed until this part of the test 

76 

77 ct_failed_connection += 1 

78 list_of_failures.append(objname) 

79 msg = sys.exc_info()[1].args[0] 

80 if 'DB-Lib error' in msg: 

81 continue 

82 else: 

83 raise 

84 

85 ct_connected += 1 

86 

87 # Since there is only one chunk, 

88 try: 

89 result = next(result) 

90 except StopIteration: 

91 raise RuntimeError("No results for %s defined in %s"%(objname, 

92 inspect.getsourcefile(dbobj.__class__))) 

93 if objname.startswith('galaxy'): 

94 DummyCat.column_outputs = ['galid', 'raJ2000', 'decJ2000'] 

95 else: 

96 DummyCat.column_outputs = ['raJ2000', 'decJ2000'] 

97 cat = dbobj.getCatalog(__file__+'unit_test_catalog', obs_metadata) 

98 if os.path.exists(catName): 

99 os.unlink(catName) 

100 try: 

101 cat.write_catalog(catName) 

102 dtypeList = [(name, np.float) for name in cat._column_outputs] 

103 testData = np.genfromtxt(catName, delimiter = ', ', 

104 dtype=np.dtype(dtypeList)) 

105 self.assertGreater(len(testData), 0) 

106 finally: 

107 if os.path.exists(catName): 

108 os.unlink(catName) 

109 

110 if os.path.exists(catDir): 

111 shutil.rmtree(catDir, ignore_errors=True) 

112 

113 self.assertEqual(len(list_of_failures), ct_failed_connection) 

114 

115 print('\n================') 

116 print('Do not worry about this message') 

117 print('sometimes, connections to the UW database fail.') 

118 print('It is expected.') 

119 print('This is just a tally so that you know how often that happened.') 

120 print('successful connections: ', ct_connected) 

121 print('failed connections: ', ct_failed_connection) 

122 if len(list_of_failures) > 0: 

123 print('objects that failed to connect: ', list_of_failures) 

124 

125 def testObsCat(self): 

126 objname = 'wdstars' 

127 catDir = tempfile.mkdtemp('basicAccessTest_testObsCat') 

128 if not os.path.exists(catDir): 

129 os.mkdir(catDir) 

130 catName = tempfile.mktemp(prefix='basicAccessTest_testObsCat', 

131 dir=catDir, suffix='.txt') 

132 

133 try: 

134 dbobj = CatalogDBObject.from_objid(objname) 

135 obs_metadata = dbobj.testObservationMetaData 

136 # To cover the central ~raft 

137 obs_metadata.boundLength = 0.4 

138 obs_metadata.rotSkyPos = 0.0 

139 cat = dbobj.getCatalog('obs_star_cat', obs_metadata) 

140 if os.path.exists(catName): 

141 os.unlink(catName) 

142 try: 

143 cat.write_catalog(catName) 

144 dtypeList = [(name, np.float) for name in cat._column_outputs] 

145 testData = np.genfromtxt(catName, delimiter = ', ', 

146 dtype=np.dtype(dtypeList)) 

147 self.assertGreater(len(testData), 0) 

148 finally: 

149 if os.path.exists(catName): 

150 os.unlink(catName) 

151 if os.path.exists(catDir): 

152 shutil.rmtree(catDir, ignore_errors=True) 

153 

154 print('\ntestObsCat successfully connected to fatboy') 

155 

156 except: 

157 trace = traceback.extract_tb(sys.exc_info()[2], limit=20) 

158 msg = sys.exc_info()[1].args[0] 

159 if 'Failed to connect' in str(msg) or failedOnFatboy(trace): 

160 

161 # if the exception was because of a failed connection 

162 # to fatboy, ignore it. 

163 

164 print('\ntestObsCat failed to connect to fatboy') 

165 print('Sometimes that happens. Do not worry.') 

166 

167 if os.path.exists(catDir): 

168 shutil.rmtree(catDir, ignore_errors=True) 

169 

170 pass 

171 else: 

172 raise 

173 

174 def test_limit(self): 

175 """ 

176 Test that the limit kwarg in query_columns behaves correctly. 

177 

178 Will test on one star table and one galaxy table. 

179 """ 

180 list_of_failures = [] 

181 for objcls, clsname in zip((bcm.StarObj, bcm.GalaxyObj), ('StarObj', 'GalaxyObj')): 

182 msg = "failed the limit test\noffending class is %s" % clsname 

183 try: 

184 dbobj = objcls(verbose=False) 

185 except: 

186 trace = traceback.extract_tb(sys.exc_info()[2], limit=20) 

187 msg = sys.exc_info()[1].args[0] 

188 if 'Failed to connect' in str(msg) or failedOnFatboy(trace): 

189 

190 # if the exception was due to a failed connection 

191 # to fatboy, ignore it 

192 list_of_failures.append(clsname) 

193 continue 

194 else: 

195 raise 

196 

197 obs_metadata = dbobj.testObservationMetaData 

198 

199 results = dbobj.query_columns(obs_metadata=obs_metadata) 

200 

201 ct_res = 0 

202 for chunk in results: 

203 for line in chunk: 

204 ct_res += 1 

205 

206 self.assertGreater(ct_res, 10, msg=msg) 

207 

208 limited_results = dbobj.query_columns(obs_metadata=obs_metadata, limit=10) 

209 

210 ct_limit = 0 

211 for chunk in limited_results: 

212 for line in chunk: 

213 ct_limit += 1 

214 

215 self.assertEqual(ct_limit, 10, msg=msg) 

216 

217 if len(list_of_failures) > 0: 

218 print("\nList of DBObjects that could not connect to fatboy " \ 

219 "for the test on the limit kwarg") 

220 for nn in list_of_failures: 

221 print(nn) 

222 

223 def test_constraint(self): 

224 """ 

225 Test that passing a constraint into query_columns works (i.e. if I only want 

226 to select galaxies whose varParamStr is not NULL). 

227 """ 

228 list_of_failures = [] 

229 constraint = "varParamStr IS NOT NULL" 

230 for objcls, clsname in zip((bcm.GalaxyObj, bcm.GalaxyTileObj), ('GalaxyObj', 'GalaxyTileObj')): 

231 msg = "failed the constraint test\noffending class is %s" % clsname 

232 try: 

233 dbobj = objcls(verbose=False) 

234 except: 

235 trace = traceback.extract_tb(sys.exc_info()[2], limit=20) 

236 msg = sys.exc_info()[1].args[0] 

237 if 'Failed to connect' in str(msg) or failedOnFatboy(trace): 

238 

239 # if the exception was due to a failed connection 

240 # to fatboy, ignore it 

241 list_of_failures.append(clsname) 

242 continue 

243 else: 

244 raise 

245 

246 obs_metadata = dbobj.testObservationMetaData 

247 

248 # query witout a constraint on varParamStr 

249 results = dbobj.query_columns(colnames=['raJ2000', 'decJ2000', 'varParamStr'], 

250 obs_metadata=obs_metadata) 

251 

252 # count total number of rows (ct_res) and number of rows with a null 

253 # varParamStr (ct_no_varparamstr). Note that varParamStr will be the 

254 # index=3 entry in result rows because the id of the galaxy gets 

255 # automatically added to query results. 

256 ct_res = 0 

257 ct_no_varparamstr = 0 

258 for chunk in results: 

259 for line in chunk: 

260 if line[3] == 'None': 

261 ct_no_varparamstr += 1 

262 ct_res += 1 

263 

264 # run the same query, but demanding that varParamStr is not NULL 

265 constrained_results = dbobj.query_columns(colnames=['raJ2000', 'decJ2000', 'varParamStr'], 

266 obs_metadata=obs_metadata, 

267 constraint=constraint) 

268 

269 # count the number of rows with non-NULL varParamStr 

270 ct_con = 0 

271 for chunk in constrained_results: 

272 for line in chunk: 

273 ct_con += 1 

274 self.assertNotEqual(line[3], 'None') 

275 

276 # check that the number of non-NULL varParamStr and NULL varParamStr rows 

277 # compare the way that they should 

278 self.assertGreater(ct_res, ct_con) 

279 self.assertGreater(ct_no_varparamstr, 0) 

280 self.assertEqual(ct_res-ct_con, ct_no_varparamstr) 

281 

282 if len(list_of_failures) > 0: 

283 print("\nList of DBObjects that could not connect to fatboy " \ 

284 "for the test on the constraint kwarg") 

285 for nn in list_of_failures: 

286 print(nn) 

287 

288 def test_limit_and_constraint(self): 

289 """ 

290 Test that limit and constraint work together 

291 """ 

292 list_of_failures = [] 

293 constraint = "varParamStr IS NOT NULL" 

294 could_connect = True 

295 try: 

296 dbobj = bcm.GalaxyObj(verbose=False) 

297 except: 

298 trace = traceback.extract_tb(sys.exc_info()[2], limit=20) 

299 msg = sys.exc_info()[1].args[0] 

300 if 'Failed to connect' in str(msg) or failedOnFatboy(trace): 

301 

302 # if the exception was due to a failed connection 

303 # to fatboy, ignore it 

304 list_of_failures.append('GalaxyObj') 

305 could_connect = False 

306 else: 

307 raise 

308 

309 if could_connect: 

310 obs_metadata = dbobj.testObservationMetaData 

311 

312 # query with a constraint on varParamStr but no limit 

313 results_no_limit = dbobj.query_columns(colnames=['raJ2000', 'decJ2000', 'varParamStr'], 

314 obs_metadata=obs_metadata, 

315 constraint=constraint) 

316 

317 ct_res = 0 

318 for chunk in results_no_limit: 

319 for line in chunk: 

320 self.assertNotEqual(line[3], 'None') 

321 ct_res += 1 

322 

323 self.assertGreater(ct_res, 1) 

324 

325 # run the same query, but limiting the results 

326 limited_results = dbobj.query_columns(colnames=['raJ2000', 'decJ2000', 'varParamStr'], 

327 obs_metadata=obs_metadata, 

328 constraint=constraint, 

329 limit=ct_res-1) 

330 ct_lim = 0 

331 for chunk in limited_results: 

332 for line in chunk: 

333 ct_lim += 1 

334 self.assertNotEqual(line[3], 'None') 

335 

336 self.assertEqual(ct_lim, ct_res-1) 

337 

338 if len(list_of_failures) > 0: 

339 print("\nList of DBObjects that could not connect to fatboy " \ 

340 "for the test on the constraint and limit kwargs") 

341 for nn in list_of_failures: 

342 print(nn) 

343 

344 

345class MemoryTestClass(lsst.utils.tests.MemoryTestCase): 

346 pass 

347 

348if __name__ == "__main__": 348 ↛ 349line 348 didn't jump to line 349, because the condition on line 348 was never true

349 lsst.utils.tests.init() 

350 unittest.main()