Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from __future__ import print_function 

2from builtins import zip 

3from builtins import next 

4import os 

5import inspect 

6import numpy as np 

7import sys 

8import traceback 

9import unittest 

10import tempfile 

11import shutil 

12import lsst.utils.tests 

13 

14from lsst.utils import getPackageDir 

15from lsst.sims.catalogs.db import CatalogDBObject 

16from lsst.sims.catalogs.definitions import InstanceCatalog 

17from lsst.sims.catUtils.exampleCatalogDefinitions import ObsStarCatalogBase 

18from lsst.sims.catUtils.utils import failedOnFatboy 

19# The following is to get the object ids in the registry 

20import lsst.sims.catUtils.baseCatalogModels as bcm 

21 

22 

23def setup_module(module): 

24 lsst.utils.tests.init() 

25 

26 

27class DummyCat(InstanceCatalog): 

28 catalog_type = __file__ + 'unit_test_catalog' 

29 column_outputs = ['raJ2000', 'decJ2000'] 

30 

31 

32class basicAccessTest(unittest.TestCase): 

33 

34 longMessage = True 

35 

36 def testObjects(self): 

37 catDir = tempfile.mkdtemp('basicAccessTest_testObjects') 

38 if not os.path.exists(catDir): 

39 os.mkdir(catDir) 

40 catName = tempfile.mktemp(prefix='basicAccessTest_testObjects', 

41 dir=catDir, suffix='.txt') 

42 ct_connected = 0 

43 ct_failed_connection = 0 

44 list_of_failures = [] 

45 

46 for objname, objcls in CatalogDBObject.registry.items(): 

47 if not objcls.doRunTest or (objcls.testObservationMetaData is None): 

48 continue 

49 

50 print("Running tests for", objname) 

51 try: 

52 dbobj = objcls(verbose=False) 

53 except: 

54 trace = traceback.extract_tb(sys.exc_info()[2], limit=20) 

55 msg = sys.exc_info()[1].args[0] 

56 if 'Failed to connect' in str(msg) or failedOnFatboy(trace): 

57 

58 # if the exception was due to a failed connection 

59 # to fatboy, ignore it 

60 

61 ct_failed_connection += 1 

62 list_of_failures.append(objname) 

63 continue 

64 else: 

65 raise 

66 

67 obs_metadata = dbobj.testObservationMetaData 

68 

69 # Get results all at once 

70 try: 

71 result = dbobj.query_columns(obs_metadata=obs_metadata) 

72 except: 

73 

74 # This is because the solar system object 'tables' 

75 # don't actually connect to tables on fatboy; they just 

76 # call methods stored on fatboy. Therefore, the connection 

77 # failure will not be noticed until this part of the test 

78 

79 ct_failed_connection += 1 

80 list_of_failures.append(objname) 

81 msg = sys.exc_info()[1].args[0] 

82 if 'DB-Lib error' in msg: 

83 continue 

84 else: 

85 raise 

86 

87 ct_connected += 1 

88 

89 # Since there is only one chunk, 

90 try: 

91 result = next(result) 

92 except StopIteration: 

93 raise RuntimeError("No results for %s defined in %s"%(objname, 

94 inspect.getsourcefile(dbobj.__class__))) 

95 if objname.startswith('galaxy'): 

96 DummyCat.column_outputs = ['galid', 'raJ2000', 'decJ2000'] 

97 else: 

98 DummyCat.column_outputs = ['raJ2000', 'decJ2000'] 

99 cat = dbobj.getCatalog(__file__+'unit_test_catalog', obs_metadata) 

100 if os.path.exists(catName): 

101 os.unlink(catName) 

102 try: 

103 cat.write_catalog(catName) 

104 dtypeList = [(name, np.float) for name in cat._column_outputs] 

105 testData = np.genfromtxt(catName, delimiter = ', ', 

106 dtype=np.dtype(dtypeList)) 

107 self.assertGreater(len(testData), 0) 

108 finally: 

109 if os.path.exists(catName): 

110 os.unlink(catName) 

111 

112 if os.path.exists(catDir): 

113 shutil.rmtree(catDir) 

114 

115 self.assertEqual(len(list_of_failures), ct_failed_connection) 

116 

117 print('\n================') 

118 print('Do not worry about this message') 

119 print('sometimes, connections to the UW database fail.') 

120 print('It is expected.') 

121 print('This is just a tally so that you know how often that happened.') 

122 print('successful connections: ', ct_connected) 

123 print('failed connections: ', ct_failed_connection) 

124 if len(list_of_failures) > 0: 

125 print('objects that failed to connect: ', list_of_failures) 

126 

127 def testObsCat(self): 

128 objname = 'wdstars' 

129 catDir = tempfile.mkdtemp('basicAccessTest_testObsCat') 

130 if not os.path.exists(catDir): 

131 os.mkdir(catDir) 

132 catName = tempfile.mktemp(prefix='basicAccessTest_testObsCat', 

133 dir=catDir, suffix='.txt') 

134 

135 try: 

136 dbobj = CatalogDBObject.from_objid(objname) 

137 obs_metadata = dbobj.testObservationMetaData 

138 # To cover the central ~raft 

139 obs_metadata.boundLength = 0.4 

140 obs_metadata.rotSkyPos = 0.0 

141 cat = dbobj.getCatalog('obs_star_cat', obs_metadata) 

142 if os.path.exists(catName): 

143 os.unlink(catName) 

144 try: 

145 cat.write_catalog(catName) 

146 dtypeList = [(name, np.float) for name in cat._column_outputs] 

147 testData = np.genfromtxt(catName, delimiter = ', ', 

148 dtype=np.dtype(dtypeList)) 

149 self.assertGreater(len(testData), 0) 

150 finally: 

151 if os.path.exists(catName): 

152 os.unlink(catName) 

153 if os.path.exists(catDir): 

154 shutil.rmtree(catDir) 

155 

156 print('\ntestObsCat successfully connected to fatboy') 

157 

158 except: 

159 trace = traceback.extract_tb(sys.exc_info()[2], limit=20) 

160 msg = sys.exc_info()[1].args[0] 

161 if 'Failed to connect' in str(msg) or failedOnFatboy(trace): 

162 

163 # if the exception was because of a failed connection 

164 # to fatboy, ignore it. 

165 

166 print('\ntestObsCat failed to connect to fatboy') 

167 print('Sometimes that happens. Do not worry.') 

168 

169 if os.path.exists(catDir): 

170 shutil.rmtree(catDir) 

171 

172 pass 

173 else: 

174 raise 

175 

176 def test_limit(self): 

177 """ 

178 Test that the limit kwarg in query_columns behaves correctly. 

179 

180 Will test on one star table and one galaxy table. 

181 """ 

182 list_of_failures = [] 

183 for objcls, clsname in zip((bcm.StarObj, bcm.GalaxyObj), ('StarObj', 'GalaxyObj')): 

184 msg = "failed the limit test\noffending class is %s" % clsname 

185 try: 

186 dbobj = objcls(verbose=False) 

187 except: 

188 trace = traceback.extract_tb(sys.exc_info()[2], limit=20) 

189 msg = sys.exc_info()[1].args[0] 

190 if 'Failed to connect' in str(msg) or failedOnFatboy(trace): 

191 

192 # if the exception was due to a failed connection 

193 # to fatboy, ignore it 

194 list_of_failures.append(clsname) 

195 continue 

196 else: 

197 raise 

198 

199 obs_metadata = dbobj.testObservationMetaData 

200 

201 results = dbobj.query_columns(obs_metadata=obs_metadata) 

202 

203 ct_res = 0 

204 for chunk in results: 

205 for line in chunk: 

206 ct_res += 1 

207 

208 self.assertGreater(ct_res, 10, msg=msg) 

209 

210 limited_results = dbobj.query_columns(obs_metadata=obs_metadata, limit=10) 

211 

212 ct_limit = 0 

213 for chunk in limited_results: 

214 for line in chunk: 

215 ct_limit += 1 

216 

217 self.assertEqual(ct_limit, 10, msg=msg) 

218 

219 if len(list_of_failures) > 0: 

220 print("\nList of DBObjects that could not connect to fatboy " \ 

221 "for the test on the limit kwarg") 

222 for nn in list_of_failures: 

223 print(nn) 

224 

225 def test_constraint(self): 

226 """ 

227 Test that passing a constraint into query_columns works (i.e. if I only want 

228 to select galaxies whose varParamStr is not NULL). 

229 """ 

230 list_of_failures = [] 

231 constraint = "varParamStr IS NOT NULL" 

232 for objcls, clsname in zip((bcm.GalaxyObj, bcm.GalaxyTileObj), ('GalaxyObj', 'GalaxyTileObj')): 

233 msg = "failed the constraint test\noffending class is %s" % clsname 

234 try: 

235 dbobj = objcls(verbose=False) 

236 except: 

237 trace = traceback.extract_tb(sys.exc_info()[2], limit=20) 

238 msg = sys.exc_info()[1].args[0] 

239 if 'Failed to connect' in str(msg) or failedOnFatboy(trace): 

240 

241 # if the exception was due to a failed connection 

242 # to fatboy, ignore it 

243 list_of_failures.append(clsname) 

244 continue 

245 else: 

246 raise 

247 

248 obs_metadata = dbobj.testObservationMetaData 

249 

250 # query witout a constraint on varParamStr 

251 results = dbobj.query_columns(colnames=['raJ2000', 'decJ2000', 'varParamStr'], 

252 obs_metadata=obs_metadata) 

253 

254 # count total number of rows (ct_res) and number of rows with a null 

255 # varParamStr (ct_no_varparamstr). Note that varParamStr will be the 

256 # index=3 entry in result rows because the id of the galaxy gets 

257 # automatically added to query results. 

258 ct_res = 0 

259 ct_no_varparamstr = 0 

260 for chunk in results: 

261 for line in chunk: 

262 if line[3] == 'None': 

263 ct_no_varparamstr += 1 

264 ct_res += 1 

265 

266 # run the same query, but demanding that varParamStr is not NULL 

267 constrained_results = dbobj.query_columns(colnames=['raJ2000', 'decJ2000', 'varParamStr'], 

268 obs_metadata=obs_metadata, 

269 constraint=constraint) 

270 

271 # count the number of rows with non-NULL varParamStr 

272 ct_con = 0 

273 for chunk in constrained_results: 

274 for line in chunk: 

275 ct_con += 1 

276 self.assertNotEqual(line[3], 'None') 

277 

278 # check that the number of non-NULL varParamStr and NULL varParamStr rows 

279 # compare the way that they should 

280 self.assertGreater(ct_res, ct_con) 

281 self.assertGreater(ct_no_varparamstr, 0) 

282 self.assertEqual(ct_res-ct_con, ct_no_varparamstr) 

283 

284 if len(list_of_failures) > 0: 

285 print("\nList of DBObjects that could not connect to fatboy " \ 

286 "for the test on the constraint kwarg") 

287 for nn in list_of_failures: 

288 print(nn) 

289 

290 def test_limit_and_constraint(self): 

291 """ 

292 Test that limit and constraint work together 

293 """ 

294 list_of_failures = [] 

295 constraint = "varParamStr IS NOT NULL" 

296 could_connect = True 

297 try: 

298 dbobj = bcm.GalaxyObj(verbose=False) 

299 except: 

300 trace = traceback.extract_tb(sys.exc_info()[2], limit=20) 

301 msg = sys.exc_info()[1].args[0] 

302 if 'Failed to connect' in str(msg) or failedOnFatboy(trace): 

303 

304 # if the exception was due to a failed connection 

305 # to fatboy, ignore it 

306 list_of_failures.append('GalaxyObj') 

307 could_connect = False 

308 else: 

309 raise 

310 

311 if could_connect: 

312 obs_metadata = dbobj.testObservationMetaData 

313 

314 # query with a constraint on varParamStr but no limit 

315 results_no_limit = dbobj.query_columns(colnames=['raJ2000', 'decJ2000', 'varParamStr'], 

316 obs_metadata=obs_metadata, 

317 constraint=constraint) 

318 

319 ct_res = 0 

320 for chunk in results_no_limit: 

321 for line in chunk: 

322 self.assertNotEqual(line[3], 'None') 

323 ct_res += 1 

324 

325 self.assertGreater(ct_res, 1) 

326 

327 # run the same query, but limiting the results 

328 limited_results = dbobj.query_columns(colnames=['raJ2000', 'decJ2000', 'varParamStr'], 

329 obs_metadata=obs_metadata, 

330 constraint=constraint, 

331 limit=ct_res-1) 

332 ct_lim = 0 

333 for chunk in limited_results: 

334 for line in chunk: 

335 ct_lim += 1 

336 self.assertNotEqual(line[3], 'None') 

337 

338 self.assertEqual(ct_lim, ct_res-1) 

339 

340 if len(list_of_failures) > 0: 

341 print("\nList of DBObjects that could not connect to fatboy " \ 

342 "for the test on the constraint and limit kwargs") 

343 for nn in list_of_failures: 

344 print(nn) 

345 

346 

347class MemoryTestClass(lsst.utils.tests.MemoryTestCase): 

348 pass 

349 

350if __name__ == "__main__": 350 ↛ 351line 350 didn't jump to line 351, because the condition on line 350 was never true

351 lsst.utils.tests.init() 

352 unittest.main()