Coverage for tests/testCatalogDBObject.py : 9%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import with_statement
2from __future__ import print_function
3from builtins import zip
4from builtins import str
5from builtins import super
6import os
7import sqlite3
8import sys
9import json
11import unittest
12import numpy as np
13import tempfile
14import shutil
15import lsst.utils.tests
16from lsst.sims.utils.CodeUtilities import sims_clean_up
17from lsst.sims.utils import ObservationMetaData
18from lsst.sims.catalogs.db import CatalogDBObject, fileDBObject
19import lsst.sims.catalogs.utils.testUtils as tu
20from lsst.sims.catalogs.utils.testUtils import myTestStars, myTestGals
21from lsst.sims.utils import haversine
23ROOT = os.path.abspath(os.path.dirname(__file__))
26def setup_module(module):
27 lsst.utils.tests.init()
30class dbForQueryColumnsTest(CatalogDBObject):
31 objid = 'queryColumnsNonsense'
32 tableid = 'queryColumnsTest'
33 idColKey = 'i1'
34 dbDefaultValues = {'i2': -1, 'i3': -2}
37class myNonsenseDB(CatalogDBObject):
38 objid = 'Nonsense'
39 tableid = 'test'
40 idColKey = 'NonsenseId'
41 driver = 'sqlite'
42 raColName = 'ra'
43 decColName = 'dec'
44 columns = [('NonsenseId', 'id', int),
45 ('NonsenseRaJ2000', 'ra*%f'%(np.pi/180.)),
46 ('NonsenseDecJ2000', 'dec*%f'%(np.pi/180.)),
47 ('NonsenseMag', 'mag', float)]
50class myNonsenseDB_noConnection(CatalogDBObject):
51 """
52 In order to test that we can pass a DBConnection in
53 through the constructor
54 """
55 objid = 'Nonsense_noConnection'
56 tableid = 'test'
57 idColKey = 'NonsenseId'
58 raColName = 'ra'
59 decColName = 'dec'
60 columns = [('NonsenseId', 'id', int),
61 ('NonsenseRaJ2000', 'ra*%f'%(np.pi/180.)),
62 ('NonsenseDecJ2000', 'dec*%f'%(np.pi/180.)),
63 ('NonsenseMag', 'mag', float)]
66class myNonsenseFileDB(fileDBObject):
67 objid = 'fileNonsense'
68 tableid = 'test'
69 idColKey = 'NonsenseId'
70 raColName = 'ra'
71 decColName = 'dec'
72 columns = [('NonsenseId', 'id', int),
73 ('NonsenseRaJ2000', 'ra*%f'%(np.pi/180.)),
74 ('NonsenseDecJ2000', 'dec*%f'%(np.pi/180.)),
75 ('NonsenseMag', 'mag', float)]
78class testCatalogDBObjectTestStars(myTestStars):
79 objid = 'testCatalogDBObjectTeststars'
80 driver = 'sqlite'
83class testCatalogDBObjectTestGalaxies(myTestGals):
84 objid = 'testCatalogDBObjectTestgals'
85 driver = 'sqlite'
88class CatalogDBObjectTestCase(unittest.TestCase):
90 @classmethod
91 def setUpClass(cls):
92 cls.scratch_dir = tempfile.mkdtemp(dir=ROOT,
93 prefix='CatalogDBObjectTestCase')
94 # Delete the test database if it exists and start fresh.
95 cls.dbo_db_name = os.path.join(cls.scratch_dir, 'testCatalogDBObjectDatabase.db')
96 if os.path.exists(cls.dbo_db_name):
97 print("deleting database")
98 os.unlink(cls.dbo_db_name)
99 tu.makeStarTestDB(filename=cls.dbo_db_name, size=5000, seedVal=1)
100 tu.makeGalTestDB(filename=cls.dbo_db_name, size=5000, seedVal=1)
102 #Create a database from generic data stored in testData/CatalogsGenerationTestData.txt
103 #This will be used to make sure that circle and box spatial bounds yield the points
104 #they are supposed to.
105 dataDir = os.path.join(ROOT, 'testData')
106 cls.nonsense_db_name = os.path.join(cls.scratch_dir, 'testCatalogDBObjectNonsenseDB.db')
107 if os.path.exists(cls.nonsense_db_name):
108 os.unlink(cls.nonsense_db_name)
110 conn = sqlite3.connect(cls.nonsense_db_name)
111 c = conn.cursor()
112 try:
113 c.execute('''CREATE TABLE test (id int, ra real, dec real, mag real)''')
114 conn.commit()
115 except:
116 raise RuntimeError("Error creating database table test.")
118 try:
119 c.execute('''CREATE TABLE test2 (id int, mag real)''')
120 conn.commit()
121 except:
122 raise RuntimeError("Error creating database table test2.")
124 with open(os.path.join(dataDir, 'CatalogsGenerationTestData.txt'), 'r') as inFile:
125 for line in inFile:
126 values = line.split()
127 cmd = '''INSERT INTO test VALUES (%s, %s, %s, %s)''' % \
128 (values[0], values[1], values[2], values[3])
129 c.execute(cmd)
130 if int(values[0])%2 == 0:
131 cmd = '''INSERT INTO test2 VALUES (%s, %s)''' % (values[0], str(2.0*float(values[3])))
132 c.execute(cmd)
134 conn.commit()
136 try:
137 c.execute('''CREATE TABLE queryColumnsTest (i1 int, i2 int, i3 int)''')
138 conn.commit()
139 except:
140 raise RuntimeError("Error creating database table queryColumnsTest.")
142 with open(os.path.join(dataDir, 'QueryColumnsTestData.txt'), 'r') as inputFile:
143 for line in inputFile:
144 vv = line.split()
145 cmd = '''INSERT INTO queryColumnsTest VALUES (%s, %s, %s)''' % (vv[0], vv[1], vv[2])
146 c.execute(cmd)
148 conn.commit()
149 conn.close()
151 @classmethod
152 def tearDownClass(cls):
153 sims_clean_up()
154 if os.path.exists(cls.dbo_db_name):
155 os.unlink(cls.dbo_db_name)
156 if os.path.exists(cls.nonsense_db_name):
157 os.unlink(cls.nonsense_db_name)
158 if os.path.exists(cls.scratch_dir):
159 shutil.rmtree(cls.scratch_dir)
161 def setUp(self):
162 self.obsMd = ObservationMetaData(pointingRA=210.0, pointingDec=-60.0, boundLength=1.75,
163 boundType='circle', mjd=52000., bandpassName='r')
165 self.filepath = os.path.join(ROOT,
166 'testData', 'CatalogsGenerationTestData.txt')
168 """
169 baselineData will store another copy of the data that should be stored in
170 testCatalogDBObjectNonsenseDB.db. This will give us something to test database queries
171 against when we ask for all of the objects within a certain box or circle.
172 """
174 self.dtype = [('id', int), ('ra', float), ('dec', float), ('mag', float)]
175 self.baselineData = np.loadtxt(self.filepath, dtype=self.dtype)
177 def tearDown(self):
178 del self.obsMd
179 del self.filepath
180 del self.dtype
181 del self.baselineData
183 def testObsMD(self):
184 self.assertEqual(self.obsMd.bandpass, 'r')
185 self.assertAlmostEqual(self.obsMd.mjd.TAI, 52000., 6)
187 def testDbObj(self):
188 db_name = os.path.join(self.scratch_dir, 'testCatalogDBObjectDatabase.db')
189 mystars = testCatalogDBObjectTestStars(database=db_name)
190 mygals = testCatalogDBObjectTestGalaxies(database=db_name)
191 result = mystars.query_columns(obs_metadata=self.obsMd)
192 tu.writeResult(result, "/dev/null")
193 result = mygals.query_columns(obs_metadata=self.obsMd)
194 tu.writeResult(result, "/dev/null")
196 def testRealQueryConstraints(self):
197 db_name = os.path.join(self.scratch_dir, 'testCatalogDBObjectDatabase.db')
198 mystars = testCatalogDBObjectTestStars(database=db_name)
199 mycolumns = ['id', 'raJ2000', 'decJ2000', 'umag', 'gmag', 'rmag', 'imag', 'zmag', 'ymag']
201 # recall that ra and dec are stored in degrees in the data base
202 myquery = mystars.query_columns(colnames = mycolumns,
203 constraint = 'ra < 90. and ra > 45.')
205 tol = 1.0e-3
206 ct = 0
207 for chunk in myquery:
208 for star in chunk:
209 ct += 1
210 self.assertLess(np.degrees(star[1]), 90.0+tol)
211 self.assertGreater(np.degrees(star[1]), 45.0-tol)
212 self.assertGreater(ct, 0)
214 def testNonsenseCircularConstraints(self):
215 """
216 Test that a query performed on a circle bound gets all of the objects (and only all
217 of the objects) within that circle
218 """
219 db_name = os.path.join(self.scratch_dir, 'testCatalogDBObjectNonsenseDB.db')
220 myNonsense = myNonsenseDB(database=db_name)
222 radius = 20.0
223 raCenter = 210.0
224 decCenter = -60.0
226 mycolumns = ['NonsenseId', 'NonsenseRaJ2000', 'NonsenseDecJ2000', 'NonsenseMag']
228 circObsMd = ObservationMetaData(boundType='circle', pointingRA=raCenter, pointingDec=decCenter,
229 boundLength=radius, mjd=52000., bandpassName='r')
231 circQuery = myNonsense.query_columns(colnames = mycolumns, obs_metadata=circObsMd, chunk_size=100)
233 raCenter = np.radians(raCenter)
234 decCenter = np.radians(decCenter)
235 radius = np.radians(radius)
237 goodPoints = []
239 ct = 0
240 for chunk in circQuery:
241 for row in chunk:
242 ct += 1
243 distance = haversine(raCenter, decCenter, row[1], row[2])
245 self.assertLess(distance, radius)
247 dex = np.where(self.baselineData['id'] == row[0])[0][0]
249 # store a list of which objects fell within our circle bound
250 goodPoints.append(row[0])
252 self.assertAlmostEqual(np.radians(self.baselineData['ra'][dex]), row[1], 3)
253 self.assertAlmostEqual(np.radians(self.baselineData['dec'][dex]), row[2], 3)
254 self.assertAlmostEqual(self.baselineData['mag'][dex], row[3], 3)
255 self.assertGreater(ct, 0)
257 ct = 0
258 for entry in [xx for xx in self.baselineData if xx[0] not in goodPoints]:
259 # make sure that all of the points not returned by the query were, in fact, outside of
260 # the circle bound
261 ct += 1
262 distance = haversine(raCenter, decCenter, np.radians(entry[1]), np.radians(entry[2]))
263 self.assertGreater(distance, radius)
264 self.assertGreater(ct, 0)
266 def testNonsenseSelectOnlySomeColumns(self):
267 """
268 Test a query performed only a subset of the available columns
269 """
270 db_name = os.path.join(self.scratch_dir, 'testCatalogDBObjectNonsenseDB.db')
271 myNonsense = myNonsenseDB(database=db_name)
273 mycolumns = ['NonsenseId', 'NonsenseRaJ2000', 'NonsenseMag']
275 query = myNonsense.query_columns(colnames=mycolumns, constraint = 'ra < 45.', chunk_size=100)
277 goodPoints = []
279 ct = 0
280 for chunk in query:
281 for row in chunk:
282 ct += 1
283 self.assertLess(row[1], 45.0)
285 dex = np.where(self.baselineData['id'] == row[0])[0][0]
287 goodPoints.append(row[0])
289 self.assertAlmostEqual(np.radians(self.baselineData['ra'][dex]), row[1], 3)
290 self.assertAlmostEqual(self.baselineData['mag'][dex], row[2], 3)
292 self.assertGreater(ct, 0)
294 ct = 0
295 for entry in [xx for xx in self.baselineData if xx[0] not in goodPoints]:
296 self.assertGreater(entry[1], 45.0)
297 ct += 1
298 self.assertGreater(ct, 0)
300 def testNonsenseBoxConstraints(self):
301 """
302 Test that a query performed on a box bound gets all of the points (and only all of the
303 points) inside that box bound.
304 """
306 db_name = os.path.join(self.scratch_dir, 'testCatalogDBObjectNonsenseDB.db')
307 myNonsense = myNonsenseDB(database=db_name)
309 raMin = 50.0
310 raMax = 150.0
311 decMax = 30.0
312 decMin = -20.0
314 raCenter = 0.5*(raMin+raMax)
315 decCenter = 0.5*(decMin+decMax)
317 mycolumns = ['NonsenseId', 'NonsenseRaJ2000', 'NonsenseDecJ2000', 'NonsenseMag']
319 boxObsMd = ObservationMetaData(boundType='box', pointingDec=decCenter, pointingRA=raCenter,
320 boundLength=np.array([0.5*(raMax-raMin), 0.5*(decMax-decMin)]),
321 mjd=52000., bandpassName='r')
323 boxQuery = myNonsense.query_columns(obs_metadata=boxObsMd, chunk_size=100, colnames=mycolumns)
325 raMin = np.radians(raMin)
326 raMax = np.radians(raMax)
327 decMin = np.radians(decMin)
328 decMax = np.radians(decMax)
330 goodPoints = []
332 ct = 0
333 for chunk in boxQuery:
334 for row in chunk:
335 ct += 1
336 self.assertLess(row[1], raMax)
337 self.assertGreater(row[1], raMin)
338 self.assertLess(row[2], decMax)
339 self.assertGreater(row[2], decMin)
341 dex = np.where(self.baselineData['id'] == row[0])[0][0]
343 # keep a list of which points were returned by teh query
344 goodPoints.append(row[0])
346 self.assertAlmostEqual(np.radians(self.baselineData['ra'][dex]), row[1], 3)
347 self.assertAlmostEqual(np.radians(self.baselineData['dec'][dex]), row[2], 3)
348 self.assertAlmostEqual(self.baselineData['mag'][dex], row[3], 3)
350 self.assertGreater(ct, 0)
352 ct = 0
353 for entry in [xx for xx in self.baselineData if xx[0] not in goodPoints]:
354 # make sure that the points not returned by the query are, in fact, outside of the
355 # box bound
357 switch = (entry[1] > raMax or entry[1] < raMin or entry[2] > decMax or entry[2] < decMin)
358 self.assertTrue(switch, msg='query failed to find a star that was within bounds')
359 ct += 1
360 self.assertGreater(ct, 0)
362 def testNonsenseArbitraryConstraints(self):
363 """
364 Test a query with a user-specified constraint on the magnitude column
365 """
366 db_name = os.path.join(self.scratch_dir, 'testCatalogDBObjectNonsenseDB.db')
367 myNonsense = myNonsenseDB(database=db_name)
369 raMin = 50.0
370 raMax = 150.0
371 decMax = 30.0
372 decMin = -20.0
373 raCenter = 0.5*(raMin + raMax)
374 decCenter = 0.5*(decMin + decMax)
376 mycolumns = ['NonsenseId', 'NonsenseRaJ2000', 'NonsenseDecJ2000', 'NonsenseMag']
378 boxObsMd = ObservationMetaData(boundType='box', pointingRA=raCenter, pointingDec=decCenter,
379 boundLength=np.array([0.5*(raMax-raMin), 0.5*(decMax-decMin)]),
380 mjd=52000., bandpassName='r')
382 boxQuery = myNonsense.query_columns(colnames = mycolumns,
383 obs_metadata=boxObsMd, chunk_size=100,
384 constraint = 'mag > 11.0')
386 raMin = np.radians(raMin)
387 raMax = np.radians(raMax)
388 decMin = np.radians(decMin)
389 decMax = np.radians(decMax)
391 goodPoints = []
393 ct = 0
394 for chunk in boxQuery:
395 for row in chunk:
396 ct += 1
398 self.assertLess(row[1], raMax)
399 self.assertGreater(row[1], raMin)
400 self.assertLess(row[2], decMax)
401 self.assertGreater(row[2], decMin)
402 self.assertGreater(row[3], 11.0)
404 dex = np.where(self.baselineData['id'] == row[0])[0][0]
406 # keep a list of the points returned by the query
407 goodPoints.append(row[0])
409 self.assertAlmostEqual(np.radians(self.baselineData['ra'][dex]), row[1], 3)
410 self.assertAlmostEqual(np.radians(self.baselineData['dec'][dex]), row[2], 3)
411 self.assertAlmostEqual(self.baselineData['mag'][dex], row[3], 3)
413 self.assertGreater(ct, 0)
415 ct = 0
416 for entry in [xx for xx in self.baselineData if xx[0] not in goodPoints]:
417 # make sure that the points not returned by the query did, in fact, violate one of the
418 # constraints of the query (either the box bound or the magnitude cut off)
419 switch = (entry[1] > raMax or entry[1] < raMin or
420 entry[2] > decMax or entry[2] < decMin or entry[3] < 11.0)
422 self.assertTrue(switch, msg='query failed to find a star that was within bounds')
423 ct += 1
424 self.assertGreater(ct, 0)
426 def testArbitraryQuery(self):
427 """
428 Test method to directly execute an arbitrary SQL query (inherited from DBObject class)
429 """
430 db_name = os.path.join(self.scratch_dir, 'testCatalogDBObjectNonsenseDB.db')
431 myNonsense = myNonsenseDB(database=db_name)
432 query = 'SELECT test.id, test.mag, test2.id, test2.mag FROM test, test2 WHERE test.id=test2.id'
433 results = myNonsense.execute_arbitrary(query)
434 self.assertEqual(len(results), 1250)
435 for row in results:
436 self.assertEqual(row[0], row[2])
437 self.assertAlmostEqual(row[1], 0.5*row[3], 6)
438 self.assertGreater(len(results), 0)
440 def testArbitraryChunkIterator(self):
441 """
442 Test method to create a ChunkIterator from an arbitrary SQL query (inherited from DBObject class)
443 """
444 db_name = os.path.join(self.scratch_dir, 'testCatalogDBObjectNonsenseDB.db')
445 myNonsense = myNonsenseDB(database=db_name)
446 query = 'SELECT test.id, test.mag, test2.id, test2.mag FROM test, test2 WHERE test.id=test2.id'
447 dtype = np.dtype([('id1', int), ('mag1', float), ('id2', int), ('mag2', float)])
448 results = myNonsense.get_chunk_iterator(query, chunk_size=100, dtype=dtype)
449 i = 0
450 for chunk in results:
451 for row in chunk:
452 self.assertEqual(row[0], row[2])
453 self.assertAlmostEqual(row[1], 0.5*row[3], 6)
454 i += 1
455 self.assertEqual(i, 1250)
457 def testChunking(self):
458 """
459 Test that a query with a specified chunk_size does, in fact, return chunks of that size
460 """
461 db_name = os.path.join(self.scratch_dir, 'testCatalogDBObjectDatabase.db')
462 mystars = testCatalogDBObjectTestStars(database=db_name)
463 mycolumns = ['id', 'raJ2000', 'decJ2000', 'umag', 'gmag']
464 myquery = mystars.query_columns(colnames = mycolumns, chunk_size = 1000)
466 ct = 0
467 for chunk in myquery:
468 self.assertEqual(chunk.size, 1000)
469 for row in chunk:
470 ct += 1
471 self.assertEqual(len(row), 5)
472 self.assertGreater(ct, 0)
474 def testClassVariables(self):
475 """
476 Make sure that the daughter classes of CatalogDBObject properly overwrite the member
477 variables of CatalogDBObject
478 """
479 db_name = os.path.join(self.scratch_dir, 'testCatalogDBObjectDatabase.db')
480 mystars = testCatalogDBObjectTestStars(database=db_name)
481 mygalaxies = testCatalogDBObjectTestGalaxies(database=db_name)
483 db_name = os.path.join(self.scratch_dir, 'testCatalogDBObjectNonsenseDB.db')
484 myNonsense = myNonsenseDB(database=db_name)
486 self.assertEqual(mystars.raColName, 'ra')
487 self.assertEqual(mystars.decColName, 'decl')
488 self.assertEqual(mystars.idColKey, 'id')
489 self.assertEqual(mystars.driver, 'sqlite')
490 self.assertEqual(mystars.database, self.dbo_db_name)
491 self.assertEqual(mystars.appendint, 1023)
492 self.assertEqual(mystars.tableid, 'stars')
493 self.assertFalse(hasattr(mystars, 'spatialModel'),
494 msg="mystars has attr 'spatialModel', which it should not")
495 self.assertEqual(mystars.objid, 'testCatalogDBObjectTeststars')
497 self.assertEqual(mygalaxies.raColName, 'ra')
498 self.assertEqual(mygalaxies.decColName, 'decl')
499 self.assertEqual(mygalaxies.idColKey, 'id')
500 self.assertEqual(mygalaxies.driver, 'sqlite')
501 self.assertEqual(mygalaxies.database, self.dbo_db_name)
502 self.assertEqual(mygalaxies.appendint, 1022)
503 self.assertEqual(mygalaxies.tableid, 'galaxies')
504 self.assertTrue(hasattr(mygalaxies, 'spatialModel'),
505 msg="mygalaxies does not have attr 'spatialModel', which it should")
506 self.assertEqual(mygalaxies.spatialModel, 'SERSIC2D')
507 self.assertEqual(mygalaxies.objid, 'testCatalogDBObjectTestgals')
509 self.assertEqual(myNonsense.raColName, 'ra')
510 self.assertEqual(myNonsense.decColName, 'dec')
511 self.assertEqual(myNonsense.idColKey, 'NonsenseId')
512 self.assertEqual(myNonsense.driver, 'sqlite')
513 self.assertEqual(myNonsense.database, self.nonsense_db_name)
514 self.assertFalse(hasattr(myNonsense, 'appendint'),
515 msg="myNonsense has attr 'appendint', which it should not")
516 self.assertEqual(myNonsense.tableid, 'test')
517 self.assertFalse(hasattr(myNonsense, 'spatialModel'),
518 msg="myNonsense has attr 'spatialModel', which it should not")
519 self.assertEqual(myNonsense.objid, 'Nonsense')
521 self.assertIn('teststars', CatalogDBObject.registry)
522 self.assertIn('testgals', CatalogDBObject.registry)
523 self.assertIn('testCatalogDBObjectTeststars', CatalogDBObject.registry)
524 self.assertIn('testCatalogDBObjectTestgals', CatalogDBObject.registry)
525 self.assertIn('Nonsense', CatalogDBObject.registry)
527 colsShouldBe = [('id', None, int), ('raJ2000', 'ra*%f'%(np.pi/180.)),
528 ('decJ2000', 'decl*%f'%(np.pi/180.)),
529 ('parallax', 'parallax*%.15f'%(np.pi/(648000000.0))),
530 ('properMotionRa', 'properMotionRa*%.15f'%(np.pi/180.)),
531 ('properMotionDec', 'properMotionDec*%.15f'%(np.pi/180.)),
532 ('umag', None), ('gmag', None), ('rmag', None), ('imag', None),
533 ('zmag', None), ('ymag', None),
534 ('magNorm', 'mag_norm', float)]
536 for (col, coltest) in zip(mystars.columns, colsShouldBe):
537 self.assertEqual(col, coltest)
539 colsShouldBe = [('NonsenseId', 'id', int),
540 ('NonsenseRaJ2000', 'ra*%f'%(np.pi/180.)),
541 ('NonsenseDecJ2000', 'dec*%f'%(np.pi/180.)),
542 ('NonsenseMag', 'mag', float)]
544 for (col, coltest) in zip(myNonsense.columns, colsShouldBe):
545 self.assertEqual(col, coltest)
547 colsShouldBe = [('id', None, int),
548 ('raJ2000', 'ra*%f' % (np.pi/180.)),
549 ('decJ2000', 'decl*%f'%(np.pi/180.)),
550 ('umag', None),
551 ('gmag', None),
552 ('rmag', None),
553 ('imag', None),
554 ('zmag', None),
555 ('ymag', None),
556 ('magNormAgn', 'mag_norm_agn', None),
557 ('magNormDisk', 'mag_norm_disk', None),
558 ('magNormBulge', 'mag_norm_bulge', None),
559 ('redshift', None),
560 ('a_disk', None),
561 ('b_disk', None),
562 ('a_bulge', None),
563 ('b_bulge', None)]
565 for (col, coltest) in zip(mygalaxies.columns, colsShouldBe):
566 self.assertEqual(col, coltest)
568 def testQueryColumnsDefaults(self):
569 """
570 Test that dbDefaultValues get properly applied when query_columns is called
571 """
572 db_name = os.path.join(self.scratch_dir, 'testCatalogDBObjectNonsenseDB.db')
574 db = dbForQueryColumnsTest(database=db_name,driver='sqlite')
575 colnames = ['i1', 'i2', 'i3']
576 results = db.query_columns(colnames)
577 controlArr = [(1, -1, 2), (3, 4, -2), (5, 6, 7)]
579 ct = 0
580 for chunk in results:
581 for ix, line in enumerate(chunk):
582 ct += 1
583 self.assertEqual(line[0], controlArr[ix][0])
584 self.assertEqual(line[1], controlArr[ix][1])
585 self.assertEqual(line[2], controlArr[ix][2])
587 self.assertGreater(ct, 0)
589 # The tests below all replicate tests above, except with CatalogDBObjects whose
590 # connection was passed directly in from the constructor, in order to make sure
591 # that passing a connection in works.
593 def testNonsenseCircularConstraints_passConnection(self):
594 """
595 Test that a query performed on a circle bound gets all of the objects (and only all
596 of the objects) within that circle.
598 Pass connection directly in to the constructor.
599 """
601 db_name = os.path.join(self.scratch_dir, 'testCatalogDBObjectNonsenseDB.db')
602 myNonsense_base = myNonsenseDB(database=db_name)
604 myNonsense = myNonsenseDB_noConnection(connection=myNonsense_base.connection)
606 radius = 20.0
607 raCenter = 210.0
608 decCenter = -60.0
610 mycolumns = ['NonsenseId', 'NonsenseRaJ2000', 'NonsenseDecJ2000', 'NonsenseMag']
612 circObsMd = ObservationMetaData(boundType='circle', pointingRA=raCenter, pointingDec=decCenter,
613 boundLength=radius, mjd=52000., bandpassName='r')
615 circQuery = myNonsense.query_columns(colnames = mycolumns, obs_metadata=circObsMd, chunk_size=100)
617 raCenter = np.radians(raCenter)
618 decCenter = np.radians(decCenter)
619 radius = np.radians(radius)
621 goodPoints = []
623 ct = 0
624 for chunk in circQuery:
625 for row in chunk:
626 ct += 1
627 distance = haversine(raCenter, decCenter, row[1], row[2])
629 self.assertLess(distance, radius)
631 dex = np.where(self.baselineData['id'] == row[0])[0][0]
633 # store a list of which objects fell within our circle bound
634 goodPoints.append(row[0])
636 self.assertAlmostEqual(np.radians(self.baselineData['ra'][dex]), row[1], 3)
637 self.assertAlmostEqual(np.radians(self.baselineData['dec'][dex]), row[2], 3)
638 self.assertAlmostEqual(self.baselineData['mag'][dex], row[3], 3)
639 self.assertGreater(ct, 0)
641 ct = 0
642 for entry in [xx for xx in self.baselineData if xx[0] not in goodPoints]:
643 # make sure that all of the points not returned by the query were, in fact, outside of
644 # the circle bound
645 distance = haversine(raCenter, decCenter, np.radians(entry[1]), np.radians(entry[2]))
646 self.assertGreater(distance, radius)
647 ct += 1
648 self.assertGreater(ct, 0)
650 def testNonsenseSelectOnlySomeColumns_passConnection(self):
651 """
652 Test a query performed only a subset of the available columns
654 Pass connection directly in to the constructor.
655 """
656 db_name = os.path.join(self.scratch_dir, 'testCatalogDBObjectNonsenseDB.db')
657 myNonsense_base = myNonsenseDB(database=db_name)
659 myNonsense = myNonsenseDB_noConnection(connection=myNonsense_base.connection)
661 mycolumns = ['NonsenseId', 'NonsenseRaJ2000', 'NonsenseMag']
663 query = myNonsense.query_columns(colnames=mycolumns, constraint = 'ra < 45.', chunk_size=100)
665 goodPoints = []
667 ct = 0
668 for chunk in query:
669 for row in chunk:
670 ct += 1
671 self.assertLess(row[1], 45.0)
673 dex = np.where(self.baselineData['id'] == row[0])[0][0]
675 goodPoints.append(row[0])
677 self.assertAlmostEqual(np.radians(self.baselineData['ra'][dex]), row[1], 3)
678 self.assertAlmostEqual(self.baselineData['mag'][dex], row[2], 3)
680 self.assertGreater(ct, 0)
682 ct = 0
683 for entry in [xx for xx in self.baselineData if xx[0] not in goodPoints]:
684 self.assertGreater(entry[1], 45.0)
685 ct += 1
686 self.assertGreater(ct, 0)
688 def testNonsenseBoxConstraints_passConnection(self):
689 """
690 Test that a query performed on a box bound gets all of the points (and only all of the
691 points) inside that box bound.
693 Pass connection directly in to the constructor.
694 """
696 db_name = os.path.join(self.scratch_dir, 'testCatalogDBObjectNonsenseDB.db')
697 myNonsense_base = myNonsenseDB(database=db_name)
698 myNonsense = myNonsenseDB_noConnection(connection=myNonsense_base.connection)
700 raMin = 50.0
701 raMax = 150.0
702 decMax = 30.0
703 decMin = -20.0
705 raCenter = 0.5*(raMin+raMax)
706 decCenter = 0.5*(decMin+decMax)
708 mycolumns = ['NonsenseId', 'NonsenseRaJ2000', 'NonsenseDecJ2000', 'NonsenseMag']
710 boxObsMd = ObservationMetaData(boundType='box', pointingDec=decCenter, pointingRA=raCenter,
711 boundLength=np.array([0.5*(raMax-raMin), 0.5*(decMax-decMin)]),
712 mjd=52000., bandpassName='r')
714 boxQuery = myNonsense.query_columns(obs_metadata=boxObsMd, chunk_size=100, colnames=mycolumns)
716 raMin = np.radians(raMin)
717 raMax = np.radians(raMax)
718 decMin = np.radians(decMin)
719 decMax = np.radians(decMax)
721 goodPoints = []
723 ct = 0
724 for chunk in boxQuery:
725 for row in chunk:
726 ct += 1
727 self.assertLess(row[1], raMax)
728 self.assertGreater(row[1], raMin)
729 self.assertLess(row[2], decMax)
730 self.assertGreater(row[2], decMin)
732 dex = np.where(self.baselineData['id'] == row[0])[0][0]
734 # keep a list of which points were returned by teh query
735 goodPoints.append(row[0])
737 self.assertAlmostEqual(np.radians(self.baselineData['ra'][dex]), row[1], 3)
738 self.assertAlmostEqual(np.radians(self.baselineData['dec'][dex]), row[2], 3)
739 self.assertAlmostEqual(self.baselineData['mag'][dex], row[3], 3)
740 self.assertGreater(ct, 0)
742 ct = 0
743 for entry in [xx for xx in self.baselineData if xx[0] not in goodPoints]:
744 # make sure that the points not returned by the query are, in fact, outside of the
745 # box bound
747 switch = (entry[1] > raMax or entry[1] < raMin or entry[2] > decMax or entry[2] < decMin)
748 self.assertTrue(switch, msg='query failed to find a star that was within bounds')
749 ct += 1
750 self.assertGreater(ct, 0)
752 def testNonsenseArbitraryConstraints_passConnection(self):
753 """
754 Test a query with a user-specified constraint on the magnitude column
756 Pass connection directly in to the constructor.
757 """
758 db_name = os.path.join(self.scratch_dir, 'testCatalogDBObjectNonsenseDB.db')
759 myNonsense_base = myNonsenseDB(database=db_name)
760 myNonsense = myNonsenseDB_noConnection(connection=myNonsense_base.connection)
762 raMin = 50.0
763 raMax = 150.0
764 decMax = 30.0
765 decMin = -20.0
766 raCenter = 0.5*(raMin + raMax)
767 decCenter = 0.5*(decMin + decMax)
769 mycolumns = ['NonsenseId', 'NonsenseRaJ2000', 'NonsenseDecJ2000', 'NonsenseMag']
771 boxObsMd = ObservationMetaData(boundType='box', pointingRA=raCenter, pointingDec=decCenter,
772 boundLength=np.array([0.5*(raMax-raMin), 0.5*(decMax-decMin)]),
773 mjd=52000., bandpassName='r')
775 boxQuery = myNonsense.query_columns(colnames = mycolumns,
776 obs_metadata=boxObsMd, chunk_size=100,
777 constraint = 'mag > 11.0')
779 raMin = np.radians(raMin)
780 raMax = np.radians(raMax)
781 decMin = np.radians(decMin)
782 decMax = np.radians(decMax)
784 goodPoints = []
786 ct = 0
787 for chunk in boxQuery:
788 for row in chunk:
789 ct += 1
791 self.assertLess(row[1], raMax)
792 self.assertGreater(row[1], raMin)
793 self.assertLess(row[2], decMax)
794 self.assertGreater(row[2], decMin)
795 self.assertGreater(row[3], 11.0)
797 dex = np.where(self.baselineData['id'] == row[0])[0][0]
799 # keep a list of the points returned by the query
800 goodPoints.append(row[0])
802 self.assertAlmostEqual(np.radians(self.baselineData['ra'][dex]), row[1], 3)
803 self.assertAlmostEqual(np.radians(self.baselineData['dec'][dex]), row[2], 3)
804 self.assertAlmostEqual(self.baselineData['mag'][dex], row[3], 3)
805 self.assertGreater(ct, 0)
807 ct = 0
808 for entry in [xx for xx in self.baselineData if xx[0] not in goodPoints]:
809 # make sure that the points not returned by the query did, in fact, violate one of the
810 # constraints of the query (either the box bound or the magnitude cut off)
811 switch = (entry[1] > raMax or entry[1] < raMin or
812 entry[2] > decMax or entry[2] < decMin or entry[3] < 11.0)
814 self.assertTrue(switch, msg='query failed to find a star that was within bounds')
815 ct += 1
816 self.assertGreater(ct, 0)
818 def testArbitraryQuery_passConnection(self):
819 """
820 Test method to directly execute an arbitrary SQL query (inherited from DBObject class)
822 Pass connection directly in to the constructor.
823 """
824 db_name = os.path.join(self.scratch_dir, 'testCatalogDBObjectNonsenseDB.db')
825 myNonsense_base = myNonsenseDB(db_name)
826 myNonsense = myNonsenseDB_noConnection(connection=myNonsense_base.connection)
827 query = 'SELECT test.id, test.mag, test2.id, test2.mag FROM test, test2 WHERE test.id=test2.id'
828 results = myNonsense.execute_arbitrary(query)
829 self.assertEqual(len(results), 1250)
830 ct = 0
831 for row in results:
832 ct += 1
833 self.assertEqual(row[0], row[2])
834 self.assertAlmostEqual(row[1], 0.5*row[3], 6)
835 self.assertGreater(ct, 0)
837 def testArbitraryChunkIterator_passConnection(self):
838 """
839 Test method to create a ChunkIterator from an arbitrary SQL query (inherited from DBObject class)
841 Pass connection directly in to the constructor.
842 """
843 db_name = os.path.join(self.scratch_dir, 'testCatalogDBObjectNonsenseDB.db')
844 myNonsense_base = myNonsenseDB(database=db_name)
845 myNonsense = myNonsenseDB_noConnection(connection=myNonsense_base.connection)
846 query = 'SELECT test.id, test.mag, test2.id, test2.mag FROM test, test2 WHERE test.id=test2.id'
847 dtype = np.dtype([('id1', int), ('mag1', float), ('id2', int), ('mag2', float)])
848 results = myNonsense.get_chunk_iterator(query, chunk_size=100, dtype=dtype)
849 i = 0
850 for chunk in results:
851 for row in chunk:
852 self.assertEqual(row[0], row[2])
853 self.assertAlmostEqual(row[1], 0.5*row[3], 6)
854 i += 1
855 self.assertEqual(i, 1250)
857 def testPassingConnectionDifferentTables(self):
858 """
859 Test that we can pass a DBConnection between DBObjects that connect to different
860 tables on the same database
861 """
862 db_name = os.path.join(self.scratch_dir, 'testCatalogDBObjectDatabase.db')
863 dbo1 = testCatalogDBObjectTestStars(database=db_name)
864 cols = ['raJ2000', 'decJ2000', 'umag']
865 results = dbo1.query_columns(cols)
866 ct = 0
867 for chunk in results:
868 for line in chunk:
869 ct += 1
870 self.assertGreater(ct, 0)
872 ct = 0
873 db02 = testCatalogDBObjectTestGalaxies(connection=dbo1.connection)
874 cols = ['raJ2000', 'decJ2000', 'redshift']
875 results = db02.query_columns(cols)
876 for chunk in results:
877 for line in chunk:
878 ct += 1
879 self.assertGreater(ct, 0)
881 def test_dtype_detection(self):
882 """
883 Test that, if we execute different queries on the same CatalogDBObject,
884 the dtype is correctly detected
885 """
886 db_name = os.path.join(self.scratch_dir, 'testCatalogDBObjectDatabase.db')
887 db = testCatalogDBObjectTestStars(database=db_name)
888 results = db.query_columns(colnames=['ra', 'id', 'varParamStr'], chunk_size=1000)
889 n_chunks = 0
890 for chunk in results:
891 n_chunks += 1
892 self.assertGreater(len(chunk), 0)
893 self.assertEqual(str(chunk.dtype['ra']), 'float64')
894 self.assertEqual(str(chunk.dtype['id']), 'int64')
895 if sys.version_info.major == 2:
896 self.assertEqual(str(chunk.dtype['varParamStr']), '|S256')
897 else:
898 self.assertEqual(str(chunk.dtype['varParamStr']), '<U256')
899 self.assertEqual(len(chunk.dtype.names), 3)
900 self.assertGreater(n_chunks, 0)
902 results = db.query_columns(colnames=['ra', 'id', 'ebv'], chunk_size=1000)
903 n_chunks = 0
904 for chunk in results:
905 n_chunks += 1
906 self.assertGreater(len(chunk), 0)
907 self.assertEqual(str(chunk.dtype['ra']), 'float64')
908 self.assertEqual(str(chunk.dtype['id']), 'int64')
909 self.assertEqual(str(chunk.dtype['ebv']), 'float64')
910 self.assertEqual(len(chunk.dtype.names), 3)
911 self.assertGreater(n_chunks, 0)
913 # test that running query_columns() after execute_arbitrary()
914 # still gives the correct dtype
915 cmd = 'SELECT id, ra, varParamStr, umag FROM stars'
916 results = db.execute_arbitrary(cmd)
917 self.assertGreater(len(results), 0)
918 self.assertEqual(str(results.dtype['ra']), 'float64')
919 self.assertEqual(str(results.dtype['id']), 'int64')
921 # The specific dtype for varParamStr is different from above
922 # because, with execute_arbitrary(), the dtype detects the
923 # exact length of the string. With query_columns() it uses
924 # a value that is encoded in CatalogDBObject
925 if sys.version_info.major == 2:
926 self.assertEqual(str(results.dtype['varParamStr']), '|S89')
927 else:
928 self.assertEqual(str(results.dtype['varParamStr']), '<U89')
930 # verify that json can load varParamStr as a dict (indicating that
931 # the whole string was loaded properly
932 for val in results['varParamStr']:
933 test_dict = json.loads(val)
934 self.assertIsInstance(test_dict, dict)
936 self.assertEqual(str(results.dtype['umag']), 'float64')
937 self.assertEqual(len(results.dtype.names), 4)
939 results = db.query_columns(colnames=['zmag', 'id', 'rmag'], chunk_size=1000)
940 n_chunks = 0
941 for chunk in results:
942 n_chunks += 1
943 self.assertGreater(len(chunk), 0)
944 self.assertEqual(str(chunk.dtype['zmag']), 'float64')
945 self.assertEqual(str(chunk.dtype['id']), 'int64')
946 self.assertEqual(str(chunk.dtype['rmag']), 'float64')
947 self.assertEqual(len(chunk.dtype.names), 3)
948 self.assertGreater(n_chunks, 0)
950 # now try it specifying the dtype
951 dtype = np.dtype([('id', int), ('ra', float), ('varParamStr', 'S102'), ('umag', float)])
952 cmd = 'SELECT id, ra, varParamStr, umag FROM stars'
953 results = db.execute_arbitrary(cmd, dtype=dtype)
954 self.assertGreater(len(results), 0)
955 self.assertEqual(str(results.dtype['ra']), 'float64')
956 self.assertEqual(str(results.dtype['id']), 'int64')
958 # The specific dtype for varParamStr is different from above
959 # because, with execute_arbitrary(), the dtype detects the
960 # exact length of the string. With query_columns() it uses
961 # a value that is encoded in CatalogDBObject
962 self.assertEqual(str(results.dtype['varParamStr']), '|S102')
963 self.assertEqual(str(results.dtype['umag']), 'float64')
964 self.assertEqual(len(results.dtype.names), 4)
966 results = db.query_columns(colnames=['zmag', 'id', 'rmag'], chunk_size=1000)
967 n_chunks = 0
968 for chunk in results:
969 n_chunks += 1
970 self.assertGreater(len(chunk), 0)
971 self.assertEqual(str(chunk.dtype['zmag']), 'float64')
972 self.assertEqual(str(chunk.dtype['id']), 'int64')
973 self.assertEqual(str(chunk.dtype['rmag']), 'float64')
974 self.assertEqual(len(chunk.dtype.names), 3)
975 self.assertGreater(n_chunks, 0)
978class fileDBObjectTestCase(unittest.TestCase):
979 """
980 This class will re-implement the tests from CatalogDBObjectTestCase,
981 except that it will use a Nonsense CatalogDBObject loaded with fileDBObject
982 to make sure that fileDBObject properly loads the file into a
983 database.
984 """
986 @classmethod
987 def tearDownClass(self):
988 sims_clean_up()
990 def setUp(self):
991 self.testDataFile = os.path.join(
992 ROOT, 'testData', 'CatalogsGenerationTestData.txt')
993 self.testHeaderFile = os.path.join(
994 ROOT, 'testData', 'CatalogsGenerationTestDataHeader.txt')
996 self.myNonsense = fileDBObject.from_objid('fileNonsense', self.testDataFile,
997 dtype = np.dtype([('id', int), ('ra', float),
998 ('dec', float), ('mag', float)]),
999 skipLines = 0)
1000 # note that skipLines defaults to 1 so, if you do not include this, you will
1001 # lose the first line of your input file (which maybe you want to do if that
1002 # is a header)
1004 self.myNonsenseHeader = fileDBObject.from_objid('fileNonsense', self.testHeaderFile)
1005 # this time, make fileDBObject learn the dtype from a header
1007 """
1008 baselineData will store another copy of the data that should be stored in
1009 testCatalogDBObjectNonsenseDB.db. This will give us something to test database queries
1010 against when we ask for all of the objects within a certain box or circle bound
1011 """
1012 self.dtype = [('id', int), ('ra', float), ('dec', float), ('mag', float)]
1013 self.baselineData = np.loadtxt(self.testDataFile, dtype=self.dtype)
1015 def tearDown(self):
1016 del self.testDataFile
1017 del self.testHeaderFile
1018 del self.myNonsense
1019 del self.myNonsenseHeader
1020 del self.dtype
1021 del self.baselineData
1023 def testDatabaseName(self):
1024 self.assertEqual(self.myNonsense.database, ':memory:')
1026 def testNonsenseCircularConstraints(self):
1027 """
1028 Test that a query performed on a circle bound gets all of the objects (and only all
1029 of the objects) within that circle
1030 """
1032 radius = 20.0
1033 raCenter = 210.0
1034 decCenter = -60.0
1036 mycolumns = ['NonsenseId', 'NonsenseRaJ2000', 'NonsenseDecJ2000', 'NonsenseMag']
1038 circObsMd = ObservationMetaData(boundType='circle', pointingRA=raCenter, pointingDec=decCenter,
1039 boundLength=radius, mjd=52000., bandpassName='r')
1041 circQuery = self.myNonsense.query_columns(colnames = mycolumns, obs_metadata=circObsMd,
1042 chunk_size=100)
1044 raCenter = np.radians(raCenter)
1045 decCenter = np.radians(decCenter)
1046 radius = np.radians(radius)
1048 goodPoints = []
1050 ct = 0
1051 for chunk in circQuery:
1052 for row in chunk:
1053 ct += 1
1054 distance = haversine(raCenter, decCenter, row[1], row[2])
1056 self.assertLess(distance, radius)
1058 dex = np.where(self.baselineData['id'] == row[0])[0][0]
1060 # store a list of which objects fell within our circle bound
1061 goodPoints.append(row[0])
1063 self.assertAlmostEqual(np.radians(self.baselineData['ra'][dex]), row[1], 3)
1064 self.assertAlmostEqual(np.radians(self.baselineData['dec'][dex]), row[2], 3)
1065 self.assertAlmostEqual(self.baselineData['mag'][dex], row[3], 3)
1066 self.assertGreater(ct, 0)
1068 ct = 0
1069 for entry in [xx for xx in self.baselineData if xx[0] not in goodPoints]:
1070 # make sure that all of the points not returned by the query were, in fact, outside of
1071 # the circle bound
1072 distance = haversine(raCenter, decCenter, np.radians(entry[1]), np.radians(entry[2]))
1073 self.assertGreater(distance, radius)
1074 ct += 1
1075 self.assertGreater(ct, 0)
1077 # make sure that the CatalogDBObject which used a header gets the same result
1078 headerQuery = self.myNonsenseHeader.query_columns(colnames = mycolumns,
1079 obs_metadata=circObsMd,
1080 chunk_size=100)
1081 goodPointsHeader = []
1082 for chunk in headerQuery:
1083 for row in chunk:
1084 distance = haversine(raCenter, decCenter, row[1], row[2])
1085 dex = np.where(self.baselineData['id'] == row[0])[0][0]
1086 goodPointsHeader.append(row[0])
1087 self.assertAlmostEqual(np.radians(self.baselineData['ra'][dex]), row[1], 3)
1088 self.assertAlmostEqual(np.radians(self.baselineData['dec'][dex]), row[2], 3)
1089 self.assertAlmostEqual(self.baselineData['mag'][dex], row[3], 3)
1091 self.assertEqual(len(goodPoints), len(goodPointsHeader))
1092 for xx in goodPoints:
1093 self.assertIn(xx, goodPointsHeader)
1095 def testNonsenseSelectOnlySomeColumns(self):
1096 """
1097 Test a query performed only a subset of the available columns
1098 """
1100 mycolumns = ['NonsenseId', 'NonsenseRaJ2000', 'NonsenseMag']
1102 query = self.myNonsense.query_columns(colnames=mycolumns, constraint = 'ra < 45.', chunk_size=100)
1104 goodPoints = []
1106 ct = 0
1107 for chunk in query:
1108 for row in chunk:
1109 ct += 1
1110 self.assertLess(row[1], 45.0)
1112 dex = np.where(self.baselineData['id'] == row[0])[0][0]
1114 goodPoints.append(row[0])
1116 self.assertAlmostEqual(np.radians(self.baselineData['ra'][dex]), row[1], 3)
1117 self.assertAlmostEqual(self.baselineData['mag'][dex], row[2], 3)
1119 self.assertGreater(ct, 0)
1121 ct = 0
1122 for entry in [xx for xx in self.baselineData if xx[0] not in goodPoints]:
1123 self.assertGreater(entry[1], 45.0)
1124 ct += 1
1125 self.assertGreater(ct, 0)
1127 headerQuery = self.myNonsenseHeader.query_columns(colnames=mycolumns,
1128 constraint = 'ra < 45.',
1129 chunk_size=100)
1130 goodPointsHeader = []
1131 for chunk in headerQuery:
1132 for row in chunk:
1133 dex = np.where(self.baselineData['id'] == row[0])[0][0]
1134 goodPointsHeader.append(row[0])
1135 self.assertAlmostEqual(np.radians(self.baselineData['ra'][dex]), row[1], 3)
1136 self.assertAlmostEqual(self.baselineData['mag'][dex], row[2], 3)
1138 self.assertEqual(len(goodPoints), len(goodPointsHeader))
1139 for xx in goodPoints:
1140 self.assertIn(xx, goodPointsHeader)
1142 def testNonsenseBoxConstraints(self):
1143 """
1144 Test that a query performed on a box bound gets all of the points (and only all of the
1145 points) inside that box bound.
1146 """
1148 raMin = 50.0
1149 raMax = 150.0
1150 decMax = 30.0
1151 decMin = -20.0
1152 raCenter = 0.5*(raMin + raMax)
1153 decCenter = 0.5*(decMin + decMax)
1155 mycolumns = ['NonsenseId', 'NonsenseRaJ2000', 'NonsenseDecJ2000', 'NonsenseMag']
1157 boxObsMd = ObservationMetaData(boundType='box', pointingRA=raCenter, pointingDec=decCenter,
1158 boundLength=np.array([0.5*(raMax-raMin), 0.5*(decMax-decMin)]),
1159 mjd=52000., bandpassName='r')
1161 boxQuery = self.myNonsense.query_columns(obs_metadata=boxObsMd, chunk_size=100, colnames=mycolumns)
1163 raMin = np.radians(raMin)
1164 raMax = np.radians(raMax)
1165 decMin = np.radians(decMin)
1166 decMax = np.radians(decMax)
1168 goodPoints = []
1170 ct = 0
1171 for chunk in boxQuery:
1172 for row in chunk:
1173 ct += 1
1174 self.assertLess(row[1], raMax)
1175 self.assertGreater(row[1], raMin)
1176 self.assertLess(row[2], decMax)
1177 self.assertGreater(row[2], decMin)
1179 dex = np.where(self.baselineData['id'] == row[0])[0][0]
1181 # keep a list of which points were returned by teh query
1182 goodPoints.append(row[0])
1184 self.assertAlmostEqual(np.radians(self.baselineData['ra'][dex]), row[1], 3)
1185 self.assertAlmostEqual(np.radians(self.baselineData['dec'][dex]), row[2], 3)
1186 self.assertAlmostEqual(self.baselineData['mag'][dex], row[3], 3)
1187 self.assertGreater(ct, 0)
1189 ct = 0
1190 for entry in [xx for xx in self.baselineData if xx[0] not in goodPoints]:
1191 # make sure that the points not returned by the query are, in fact, outside of the
1192 # box bound
1194 switch = (entry[1] > raMax or entry[1] < raMin or entry[2] > decMax or entry[2] < decMin)
1195 self.assertTrue(switch, msg='query failed to find a star that was within bounds')
1196 ct += 1
1197 self.assertGreater(ct, 0)
1199 headerQuery = self.myNonsenseHeader.query_columns(obs_metadata=boxObsMd,
1200 chunk_size=100, colnames=mycolumns)
1201 goodPointsHeader = []
1202 for chunk in headerQuery:
1203 for row in chunk:
1204 dex = np.where(self.baselineData['id'] == row[0])[0][0]
1205 goodPointsHeader.append(row[0])
1206 self.assertAlmostEqual(np.radians(self.baselineData['ra'][dex]), row[1], 3)
1207 self.assertAlmostEqual(np.radians(self.baselineData['dec'][dex]), row[2], 3)
1208 self.assertAlmostEqual(self.baselineData['mag'][dex], row[3], 3)
1210 self.assertEqual(len(goodPoints), len(goodPointsHeader))
1211 for xx in goodPoints:
1212 self.assertIn(xx, goodPointsHeader)
1214 def testNonsenseArbitraryConstraints(self):
1215 """
1216 Test a query with a user-specified constraint on the magnitude column
1217 """
1219 raMin = 50.0
1220 raMax = 150.0
1221 decMax = 30.0
1222 decMin = -20.0
1223 raCenter = 0.5*(raMin + raMax)
1224 decCenter = 0.5*(decMin + decMax)
1226 mycolumns = ['NonsenseId', 'NonsenseRaJ2000', 'NonsenseDecJ2000', 'NonsenseMag']
1228 boxObsMd = ObservationMetaData(boundType='box', pointingRA=raCenter, pointingDec=decCenter,
1229 boundLength=np.array([0.5*(raMax-raMin), 0.5*(decMax-decMin)]),
1230 mjd=52000., bandpassName='r')
1232 boxQuery = self.myNonsense.query_columns(colnames = mycolumns,
1233 obs_metadata=boxObsMd, chunk_size=100,
1234 constraint = 'mag > 11.0')
1236 raMin = np.radians(raMin)
1237 raMax = np.radians(raMax)
1238 decMin = np.radians(decMin)
1239 decMax = np.radians(decMax)
1241 goodPoints = []
1243 ct = 0
1244 for chunk in boxQuery:
1245 for row in chunk:
1246 ct += 1
1248 self.assertLess(row[1], raMax)
1249 self.assertGreater(row[1], raMin)
1250 self.assertLess(row[2], decMax)
1251 self.assertGreater(row[2], decMin)
1252 self.assertGreater(row[3], 11.0)
1254 dex = np.where(self.baselineData['id'] == row[0])[0][0]
1256 # keep a list of the points returned by the query
1257 goodPoints.append(row[0])
1259 self.assertAlmostEqual(np.radians(self.baselineData['ra'][dex]), row[1], 3)
1260 self.assertAlmostEqual(np.radians(self.baselineData['dec'][dex]), row[2], 3)
1261 self.assertAlmostEqual(self.baselineData['mag'][dex], row[3], 3)
1262 self.assertGreater(ct, 0)
1264 ct = 0
1265 for entry in [xx for xx in self.baselineData if xx[0] not in goodPoints]:
1266 # make sure that the points not returned by the query did, in fact, violate one of the
1267 # constraints of the query (either the box bound or the magnitude cut off)
1268 switch = (entry[1] > raMax or entry[1] < raMin or
1269 entry[2] > decMax or entry[2] < decMin or
1270 entry[3] < 11.0)
1272 self.assertTrue(switch, msg='query failed to find a star that was within bounds')
1273 ct += 1
1274 self.assertGreater(ct, 0)
1276 headerQuery = self.myNonsenseHeader.query_columns(colnames = mycolumns,
1277 obs_metadata=boxObsMd, chunk_size=100,
1278 constraint='mag > 11.0')
1279 goodPointsHeader = []
1280 for chunk in headerQuery:
1281 for row in chunk:
1282 dex = np.where(self.baselineData['id'] == row[0])[0][0]
1283 goodPointsHeader.append(row[0])
1284 self.assertAlmostEqual(np.radians(self.baselineData['ra'][dex]), row[1], 3)
1285 self.assertAlmostEqual(np.radians(self.baselineData['dec'][dex]), row[2], 3)
1286 self.assertAlmostEqual(self.baselineData['mag'][dex], row[3], 3)
1288 self.assertEqual(len(goodPoints), len(goodPointsHeader))
1289 for xx in goodPoints:
1290 self.assertIn(xx, goodPointsHeader)
1292 def testChunking(self):
1293 """
1294 Test that a query with a specified chunk_size does, in fact, return chunks of that size
1295 """
1297 mycolumns = ['NonsenseId', 'NonsenseRaJ2000', 'NonsenseDecJ2000', 'NonsenseMag']
1298 myquery = self.myNonsense.query_columns(colnames = mycolumns, chunk_size = 100)
1300 ct = 0
1301 for chunk in myquery:
1302 self.assertEqual(chunk.size, 100)
1303 for row in chunk:
1304 ct += 1
1305 self.assertEqual(len(row), 4)
1306 self.assertGreater(ct, 0)
1309class MemoryTestClass(lsst.utils.tests.MemoryTestCase):
1310 pass
1313if __name__ == "__main__": 1313 ↛ 1314line 1313 didn't jump to line 1314, because the condition on line 1313 was never true
1314 setup_module(None)
1315 unittest.main()