Coverage for tests/testLightCurveGenerator.py : 8%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import with_statement
2from builtins import zip
3from builtins import range
4import unittest
5import os
6import json
7import sqlite3
8import numpy as np
9import tempfile
10import shutil
11import lsst
12# import lsst.utils.tests as utilsTests
13from lsst.utils import getPackageDir
14from lsst.sims.utils.CodeUtilities import sims_clean_up
16from lsst.sims.catalogs.db import fileDBObject
18from lsst.sims.catalogs.definitions import InstanceCatalog
19from lsst.sims.catalogs.decorators import compound
20from lsst.sims.catUtils.utils import ObservationMetaDataGenerator
22from lsst.sims.catUtils.mixins import PhotometryStars, VariabilityStars
23from lsst.sims.catUtils.utils import StellarLightCurveGenerator
25from lsst.sims.catalogs.db import CatalogDBObject
27from lsst.sims.catUtils.mixins import PhotometryGalaxies, VariabilityGalaxies
28from lsst.sims.catUtils.utils import AgnLightCurveGenerator
29from lsst.sims.utils import ModifiedJulianDate
31ROOT = os.path.abspath(os.path.dirname(__file__))
34def setup_module(module):
35 lsst.utils.tests.init()
38class stellarControlCatalog(InstanceCatalog,
39 PhotometryStars, VariabilityStars):
40 catalog_type = __file__ + 'stellar_control_catalog'
42 column_outputs = ["uniqueId", "raJ2000", "decJ2000", "mag", "sigma_mag"]
44 @compound("mag", "sigma_mag")
45 def get_phot(self):
46 mm = self.column_by_name("lsst_%s" % self.obs_metadata.bandpass)
47 sig = self.column_by_name("sigma_lsst_%s" % self.obs_metadata.bandpass)
48 return np.array([mm, sig])
51class agnControlCatalog(InstanceCatalog,
52 PhotometryGalaxies, VariabilityGalaxies):
54 catalog_type = __file__ + 'agn_control_catalog'
55 column_outputs = ["uniqueId", "raJ2000", "decJ2000", "mag", "sigma_mag"]
57 @compound("mag", "sigma_mag")
58 def get_phot(self):
59 return np.array([self.column_by_name("%sAgn" % self.obs_metadata.bandpass),
60 self.column_by_name("sigma_%sAgn" % self.obs_metadata.bandpass)])
63class StellarLightCurveTest(unittest.TestCase):
65 @classmethod
66 def setUpClass(cls):
67 """
68 Create a fake catalog of RR Lyrae stars. Store it in cls.stellar_db
69 """
70 cls.scratchDir = tempfile.mkdtemp(dir=ROOT, prefix='StellarLigghtCurveTest-')
72 rng = np.random.RandomState(88)
73 n_stars = 10000
74 sed_dir = os.path.join(getPackageDir("sims_sed_library"))
75 sed_dir = os.path.join(sed_dir, "starSED", "kurucz")
76 list_of_seds = os.listdir(sed_dir)
78 lc_dir = os.path.join(getPackageDir("sims_sed_library"), "rrly_lc")
79 lc_dir = os.path.join(lc_dir, "RRab")
80 list_of_lc = ['rrly_lc/RRab/%s' % ww for ww in os.listdir(lc_dir) if "per.txt" in ww]
82 cls.dtype = np.dtype([('id', np.int),
83 ('raDeg', np.float),
84 ('decDeg', np.float),
85 ('raJ2000', np.float),
86 ('decJ2000', np.float),
87 ('magNorm', np.float),
88 ('galacticAv', np.float),
89 ('sedFilename', str, 300),
90 ('varParamStr', str, 300),
91 ('parallax', np.float),
92 ('ebv', np.float)])
94 # write the catalog as a text file to be ingested with fileDBObject
95 cls.txt_name = os.path.join(cls.scratchDir, "stellar_lc_catalog.txt")
96 with open(cls.txt_name, "w") as output_file:
97 sed_dex = rng.randint(0, len(list_of_seds), size=n_stars)
98 lc_dex = rng.randint(0, len(list_of_lc), size=n_stars)
99 mjd0 = rng.random_sample(n_stars)*10000.0+40000.0
100 raList = rng.random_sample(n_stars)*360.0
101 decList = -90.0 + rng.random_sample(n_stars)*120.0
102 magNormList = rng.random_sample(n_stars)*3.0+14.0
103 AvList = rng.random_sample(n_stars)*0.2+0.1
104 pxList = rng.random_sample(n_stars)*0.1
105 for ix in range(n_stars):
106 varparams = {'varMethodName': 'applyRRly',
107 'pars': {'tStartMjd': mjd0[ix],
108 'filename': list_of_lc[lc_dex[ix]]}}
109 varparamstr = json.dumps(varparams)
110 output_file.write("%d;%lf;%lf;%lf;%lf;%lf;%lf;%s;%s;%lf;%lf\n"
111 % (ix, raList[ix], decList[ix],
112 np.radians(raList[ix]),
113 np.radians(decList[ix]),
114 magNormList[ix], AvList[ix],
115 list_of_seds[sed_dex[ix]],
116 varparamstr,pxList[ix],
117 AvList[ix]/3.1))
119 cls.stellar_db = fileDBObject(cls.txt_name, delimiter=';',
120 runtable='test', dtype=cls.dtype,
121 idColKey='id')
123 cls.stellar_db.raColName = 'raDeg'
124 cls.stellar_db.decColName = 'decDeg'
125 cls.stellar_db.objectTypeId = 32
127 cls.opsimDb = os.path.join(getPackageDir("sims_data"), "OpSimData")
128 cls.opsimDb = os.path.join(cls.opsimDb, "opsimblitz1_1133_sqlite.db")
130 @classmethod
131 def tearDownClass(cls):
132 sims_clean_up()
133 if os.path.exists(cls.txt_name):
134 os.unlink(cls.txt_name)
135 if os.path.exists(cls.scratchDir):
136 shutil.rmtree(cls.scratchDir)
138 def test_get_pointings(self):
139 """
140 Test that the get_pointings method does, in fact, return ObservationMetaData
141 that are grouped appropriately.
142 """
144 raRange = (78.0, 89.0)
145 decRange = (-74.0, -60.0)
146 bandpass = 'g'
148 lc_gen = StellarLightCurveGenerator(self.stellar_db, self.opsimDb)
150 pointings = lc_gen.get_pointings(raRange, decRange, bandpass=bandpass)
152 self.assertGreater(len(pointings), 1)
154 for group in pointings:
155 for ix, obs in enumerate(group):
156 self.assertAlmostEqual(obs.pointingRA, group[0].pointingRA, 12)
157 self.assertAlmostEqual(obs.pointingDec, group[0].pointingDec, 12)
158 self.assertEqual(obs.bandpass, bandpass)
159 if ix > 0:
160 self.assertGreater(obs.mjd.TAI, group[ix-1].mjd.TAI)
162 def test_get_pointings_v4(self):
163 """
164 Test that the get_pointings method runs on an OpSim v4 database
165 """
167 v4_db = os.path.join(getPackageDir('sims_data'), 'OpSimData',
168 'astro-lsst-01_2014.db')
170 raRange = (78.0, 89.0)
171 decRange = (-74.0, -60.0)
172 bandpass = 'y'
174 lc_gen = StellarLightCurveGenerator(self.stellar_db, v4_db)
176 pointings = lc_gen.get_pointings(raRange, decRange, bandpass=bandpass)
178 self.assertGreater(len(pointings), 1)
180 for group in pointings:
181 for ix, obs in enumerate(group):
182 self.assertAlmostEqual(obs.pointingRA, group[0].pointingRA, 12)
183 self.assertAlmostEqual(obs.pointingDec, group[0].pointingDec, 12)
184 self.assertEqual(obs.bandpass, bandpass)
185 if ix > 0:
186 self.assertGreater(obs.mjd.TAI, group[ix-1].mjd.TAI)
188 def test_get_pointings_multiband(self):
189 """
190 Test that the get_pointings method does, in fact, return ObservationMetaData
191 that are grouped appropriately. Test on more than one filter.
192 """
194 raRange = (78.0, 89.0)
195 decRange = (-74.0, -60.0)
196 bandpass = ('g', 'z')
198 lc_gen = StellarLightCurveGenerator(self.stellar_db, self.opsimDb)
200 pointings = lc_gen.get_pointings(raRange, decRange, bandpass=bandpass)
202 self.assertGreater(len(pointings), 1)
204 ct_g = 0
205 ct_z = 0
207 for group in pointings:
208 for ix, obs in enumerate(group):
209 self.assertAlmostEqual(obs.pointingRA, group[0].pointingRA, 12)
210 self.assertAlmostEqual(obs.pointingDec, group[0].pointingDec, 12)
212 if obs.bandpass == 'g':
213 ct_g += 1
214 elif obs.bandpass == 'z':
215 ct_z += 1
216 else:
217 raise RuntimeError("Asked for filters (g,z) but got %s" % obs.bandpass)
219 if ix > 0:
220 self.assertGreater(obs.mjd.TAI, group[ix-1].mjd.TAI)
222 self.assertGreater(ct_g, 0)
223 self.assertGreater(ct_z, 0)
225 def test_stellar_light_curves(self):
226 """
227 Test the StellarLightCurveGenerator by generating some RR Lyrae light
228 curves and comparing them to the results obtained by generating a
229 series of InstanceCatalogs containing the same objects at the same
230 MJDs
231 """
233 raRange = (78.0, 85.0)
234 decRange = (-69.0, -65.0)
235 bandpass = 'r'
237 lc_gen = StellarLightCurveGenerator(self.stellar_db, self.opsimDb)
238 pointings = lc_gen.get_pointings(raRange, decRange, bandpass=bandpass)
239 test_light_curves, truth_info = lc_gen.light_curves_from_pointings(pointings)
241 self.assertGreater(len(test_light_curves), 2) # make sure we got some light curves
243 for unique_id in test_light_curves:
244 # verify that the sources returned all do vary by making sure that the
245 # np.diff run on the magnitudes reutrns something non-zero
246 self.assertGreater(np.abs(np.diff(test_light_curves[unique_id][bandpass]['mag'])).max(), 0.0)
247 self.assertGreater(len(test_light_curves[unique_id][bandpass]['mjd']), 0)
249 # Now test that specifying a small chunk_size does not change the output
250 # light curves
251 chunk_light_curves, truth_info = lc_gen.light_curves_from_pointings(pointings, chunk_size=1)
252 self.assertGreater(len(chunk_light_curves), 2)
254 for unique_id in test_light_curves:
255 self.assertEqual(len(test_light_curves[unique_id][bandpass]['mjd']),
256 len(chunk_light_curves[unique_id][bandpass]['mjd']))
257 np.testing.assert_array_equal(test_light_curves[unique_id][bandpass]['mjd'],
258 chunk_light_curves[unique_id][bandpass]['mjd'])
259 np.testing.assert_array_equal(test_light_curves[unique_id][bandpass]['mag'],
260 chunk_light_curves[unique_id][bandpass]['mag'])
261 np.testing.assert_array_equal(test_light_curves[unique_id][bandpass]['error'],
262 chunk_light_curves[unique_id][bandpass]['error'])
264 # Now find all of the ObservationMetaData that were included in our
265 # light curves, generate InstanceCatalogs from them separately,
266 # and verify that the contents of the InstanceCatalogs agree with
267 # the contents of the light curves.
269 gen = ObservationMetaDataGenerator(database=self.opsimDb,
270 driver='sqlite')
272 obs_list = gen.getObservationMetaData(fieldRA=raRange,
273 fieldDec=decRange,
274 telescopeFilter=bandpass,
275 boundLength=1.75)
277 ct = 0
278 for obs in obs_list:
279 cat = stellarControlCatalog(self.stellar_db,
280 obs_metadata=obs)
282 for star_obj in cat.iter_catalog():
283 ct += 1
284 lc = test_light_curves[star_obj[0]][bandpass]
285 dex = np.argmin(np.abs(lc['mjd']-obs.mjd.TAI))
286 self.assertLess(np.abs(lc['mjd'][dex]-obs.mjd.TAI), 1.0e-7)
287 self.assertLess(np.abs(lc['mag'][dex]-star_obj[3]), 1.0e-7)
288 self.assertLess(np.abs(lc['error'][dex]-star_obj[4]), 1.0e-7)
290 # Verify that the same number of objects and observations were found in the
291 # catalogs and the LightCurveGenerator output
292 total_ct = 0
293 for obj_name in test_light_curves:
294 for bandpass in test_light_curves[obj_name]:
295 total_ct += len(test_light_curves[obj_name][bandpass]['mjd'])
296 self.assertEqual(ct, total_ct)
299 def test_limited_stellar_light_curves(self):
300 """
301 Test that we can ask for a limited number of light curves per field of view
302 """
304 lc_limit = 2
305 raRange = (78.0, 82.0)
306 decRange = (-69.0, -65.0)
307 bandpass = 'r'
309 lc_gen = StellarLightCurveGenerator(self.stellar_db, self.opsimDb)
310 pointings = lc_gen.get_pointings(raRange, decRange, bandpass=bandpass)
312 self.assertEqual(len(pointings), 1)
314 control_light_curves, truth_info = lc_gen.light_curves_from_pointings(pointings)
315 self.assertGreater(len(control_light_curves), 2)
317 test_light_curves, truth_info = lc_gen.light_curves_from_pointings(pointings,
318 lc_per_field=lc_limit)
320 self.assertGreater(len(control_light_curves), len(test_light_curves))
321 self.assertEqual(len(test_light_curves), lc_limit)
323 def test_date_range(self):
324 """
325 Run test_stellar_light_curves, this time specifying a range in MJD.
326 """
328 raRange = (0.0, 110.0)
329 decRange = (-90.0, -50.0)
330 bandpass = 'g'
331 mjdRange = (49356.0, 49357.0)
333 lc_gen = StellarLightCurveGenerator(self.stellar_db, self.opsimDb)
334 pointings = lc_gen.get_pointings(raRange, decRange, bandpass=bandpass, expMJD=mjdRange)
335 test_light_curves, truth_info = lc_gen.light_curves_from_pointings(pointings)
337 self.assertGreater(len(test_light_curves), 2)
339 for unique_id in test_light_curves:
340 # verify that the sources returned all do vary by making sure that the
341 # np.diff run on the magnitudes reutrns something non-zero
342 self.assertGreater(np.abs(np.diff(test_light_curves[unique_id][bandpass]['mag'])).max(), 0.0)
343 self.assertGreater(len(test_light_curves[unique_id][bandpass]['mjd']), 0)
344 self.assertGreater(test_light_curves[unique_id][bandpass]['mjd'].min(), mjdRange[0]-1.0e-12)
345 self.assertLess(test_light_curves[unique_id][bandpass]['mjd'].max(), mjdRange[1]+1.0e-12)
347 # Now test that specifying a small chunk_size does not change the output
348 # light curves
349 chunk_light_curves, truth_info = lc_gen.light_curves_from_pointings(pointings, chunk_size=1)
350 self.assertGreater(len(chunk_light_curves), 2)
352 for unique_id in test_light_curves:
353 self.assertEqual(len(test_light_curves[unique_id][bandpass]['mjd']),
354 len(chunk_light_curves[unique_id][bandpass]['mjd']))
355 np.testing.assert_array_equal(test_light_curves[unique_id][bandpass]['mjd'],
356 chunk_light_curves[unique_id][bandpass]['mjd'])
357 np.testing.assert_array_equal(test_light_curves[unique_id][bandpass]['mag'],
358 chunk_light_curves[unique_id][bandpass]['mag'])
359 np.testing.assert_array_equal(test_light_curves[unique_id][bandpass]['error'],
360 chunk_light_curves[unique_id][bandpass]['error'])
362 # Now find all of the ObservationMetaData that were included in our
363 # light curves, generate InstanceCatalogs from them separately,
364 # and verify that the contents of the InstanceCatalogs agree with
365 # the contents of the light curves.
367 gen = ObservationMetaDataGenerator(database=self.opsimDb,
368 driver='sqlite')
370 obs_list = gen.getObservationMetaData(fieldRA=raRange,
371 fieldDec=decRange,
372 telescopeFilter=bandpass,
373 expMJD=mjdRange,
374 boundLength=1.75)
376 ct = 0
377 for obs in obs_list:
378 cat = stellarControlCatalog(self.stellar_db,
379 obs_metadata=obs)
381 for star_obj in cat.iter_catalog():
382 ct += 1
383 lc = test_light_curves[star_obj[0]][bandpass]
384 dex = np.argmin(np.abs(lc['mjd']-obs.mjd.TAI))
385 self.assertLess(np.abs(lc['mjd'][dex]-obs.mjd.TAI), 1.0e-7)
386 self.assertLess(np.abs(lc['mag'][dex]-star_obj[3]), 1.0e-7)
387 self.assertLess(np.abs(lc['error'][dex]-star_obj[4]), 1.0e-7)
389 # Verify that the same number of objects and observations were found in the
390 # catalogs and the LightCurveGenerator output
391 total_ct = 0
392 for obj_name in test_light_curves:
393 for bandpass in test_light_curves[obj_name]:
394 total_ct += len(test_light_curves[obj_name][bandpass]['mjd'])
395 self.assertEqual(ct, total_ct)
397 def test_multiband_light_curves(self):
398 """
399 Check that multi-band light curves are returned correctly.
400 """
402 raRange = (78.0, 82.0)
403 decRange = (-69.0, -65.0)
404 bandpass = ('r', 'g')
406 gen = StellarLightCurveGenerator(self.stellar_db, self.opsimDb)
407 pointings = gen.get_pointings(raRange, decRange, bandpass=bandpass)
408 lc_dict, truth_info = gen.light_curves_from_pointings(pointings)
409 self.assertGreater(len(lc_dict), 2)
411 obs_gen = ObservationMetaDataGenerator(database=self.opsimDb, driver='sqlite')
412 control_pointings_r = obs_gen.getObservationMetaData(fieldRA=raRange, fieldDec=decRange,
413 telescopeFilter='r', boundLength=1.75)
415 control_pointings_g = obs_gen.getObservationMetaData(fieldRA=raRange, fieldDec=decRange,
416 telescopeFilter='g', boundLength=1.75)
418 self.assertGreater(len(control_pointings_g), 0)
419 self.assertGreater(len(control_pointings_r), 0)
421 ct = 0
422 for obs in control_pointings_r:
423 cat = stellarControlCatalog(self.stellar_db,
424 obs_metadata=obs)
426 for star_obj in cat.iter_catalog():
427 ct += 1
428 lc = lc_dict[star_obj[0]]['r']
429 dex = np.argmin(np.abs(lc['mjd']-obs.mjd.TAI))
430 self.assertLess(np.abs(lc['mjd'][dex]-obs.mjd.TAI), 1.0e-7)
431 self.assertLess(np.abs(lc['mag'][dex]-star_obj[3]), 1.0e-7)
432 self.assertLess(np.abs(lc['error'][dex]-star_obj[4]), 1.0e-7)
434 for obs in control_pointings_g:
435 cat = stellarControlCatalog(self.stellar_db,
436 obs_metadata=obs)
438 for star_obj in cat.iter_catalog():
439 ct += 1
440 lc = lc_dict[star_obj[0]]['g']
441 dex = np.argmin(np.abs(lc['mjd']-obs.mjd.TAI))
442 self.assertLess(np.abs(lc['mjd'][dex]-obs.mjd.TAI), 1.0e-7)
443 self.assertLess(np.abs(lc['mag'][dex]-star_obj[3]), 1.0e-7)
444 self.assertLess(np.abs(lc['error'][dex]-star_obj[4]), 1.0e-7)
446 # Verify that the same number of objects and observations were found in the
447 # catalogs and the LightCurveGenerator output
448 total_ct = 0
449 for obj_name in lc_dict:
450 for bandpass in lc_dict[obj_name]:
451 total_ct += len(lc_dict[obj_name][bandpass]['mjd'])
452 self.assertEqual(ct, total_ct)
454 def test_constraint(self):
455 """
456 Test that the light curve generator correctly ignores objects
457 with varParamStr == None
459 We do this by generating a database of two stars with RRLyrae-like
460 varParamStr and two stars with no varParamStr. We run this database
461 through a LightCurveGenerator and an InstanceCatalog. We verify that
462 the LightCurveGenerator finds 2 stars while the InstanceCatalog finds
463 4 stars.
464 """
466 rng = np.random.RandomState(83)
468 raRange = (80.8, 83.8)
469 decRange = (-71.0, -69.0)
470 bandpass = 'g'
472 sed_name = "kp01_7500.fits_g40_7600.gz"
474 varparams = {'varMethodName': 'applyRRly',
475 'pars': {'tStartMjd': 30000.0,
476 'filename': 'rrly_lc/RRab/1096833_per.txt'}}
478 varParamStr = json.dumps(varparams)
480 # create the dummy database
481 db_name = tempfile.mktemp(prefix='stellar_constraint_cata_sqlite-', suffix='.db', dir=ROOT)
483 conn = sqlite3.connect(db_name)
484 c = conn.cursor()
485 c.execute('''CREATE TABLE rrly
486 (id int, ra real, dec real, sedFilename text, magNorm real,
487 varParamStr text, galacticAv real, parallax real, ebv real)''')
488 conn.commit()
490 for ix, (rr, dd, mn, px) in \
491 enumerate(zip(rng.random_sample(4)*(raRange[1]-raRange[0])+raRange[0],
492 rng.random_sample(4)*(decRange[1]-decRange[0])+decRange[0],
493 rng.random_sample(4)*5.0+16.0, rng.random_sample(4)*0.01)):
495 if ix < 2:
496 cmd = '''INSERT INTO rrly VALUES(%d, %e, %e, '%s', %e, '%s', 0.1, 0.032, %e)''' % \
497 (ix, rr, dd, sed_name, mn, varParamStr, px)
498 else:
499 cmd = '''INSERT INTO rrly VALUES(%d, %e, %e, '%s', %e, NULL, 0.1, 0.032, %e)''' % \
500 (ix, rr, dd, sed_name, mn, px)
502 c.execute(cmd)
504 conn.commit()
506 # create a CatalogDBObject class to interface with the dummy database
507 class dummyRRLyDBObject(CatalogDBObject):
508 database = db_name
509 host = None
510 driver = 'sqlite'
511 tableid = 'rrly'
512 raColName = 'ra'
513 decColName = 'dec'
514 idColKey = 'id'
515 objid = 'dummyRRLy'
516 skipRegistration = True
517 objectTypeId = 99
519 columns = [('raJ2000', 'ra*PI()/180.0', float),
520 ('decJ2000', 'dec*PI()/180.0', float)]
522 star_db = dummyRRLyDBObject()
524 # verify that the LightCurveGenerator finds the two variable stars
525 lc_gen = StellarLightCurveGenerator(star_db, self.opsimDb)
526 ptngs = lc_gen.get_pointings(raRange, decRange, bandpass=bandpass)
527 lc_dict, truth_dict = lc_gen.light_curves_from_pointings(ptngs)
528 self.assertEqual(len(lc_dict), 2)
530 if os.path.exists(db_name):
531 os.unlink(db_name)
533 # verify that an InstanceCatalog finds all 4 stars
534 obs = ptngs[0][0]
535 cat = stellarControlCatalog(star_db, obs_metadata=obs)
536 with lsst.utils.tests.getTempFilePath('.txt') as dummy_cat_name:
537 cat.write_catalog(dummy_cat_name)
538 with open(dummy_cat_name, 'r') as input_file:
539 lines = input_file.readlines()
540 self.assertEqual(len(lines), 5)
542 def test_manual_constraint(self):
543 """
544 Test that a constraint put in by hand is properly applied
545 """
547 raRange = (78.0, 89.0)
548 decRange = (-74.0, -60.0)
549 bandpass = 'g'
551 lc_gen = StellarLightCurveGenerator(self.stellar_db, self.opsimDb)
553 pointings = lc_gen.get_pointings(raRange, decRange, bandpass=bandpass)
555 (lc_unconstrained,
556 truth_unconstrianed) = lc_gen.light_curves_from_pointings(pointings)
558 (lc_constrained,
559 truth_constrained) = lc_gen.light_curves_from_pointings(pointings,
560 constraint = 'ebv>0.05')
562 self.assertGreater(len(lc_constrained), 0)
563 self.assertLess(len(lc_constrained), len(lc_unconstrained))
565 # create catalogs based on all of the pointings in 'pointings';
566 # verify that the objects in those catalogs appear correctly
567 # in the constrained and unconstrained light curves.
569 class ConstraintCatalogClass(InstanceCatalog):
570 column_outputs= ['uniqueId', 'ebv']
572 ct_unconstrained = 0
573 ct_constrained = 0
574 for field in pointings:
575 for obs in field:
576 cat = ConstraintCatalogClass(self.stellar_db, obs_metadata=obs)
577 for star_obj in cat.iter_catalog():
578 if star_obj[1]>0.05:
579 self.assertIn(star_obj[0], lc_constrained)
580 ct_constrained += 1
581 self.assertIn(star_obj[0], lc_unconstrained)
582 ct_unconstrained += 1
584 total_ct = 0
585 for obj_name in lc_unconstrained:
586 for band in lc_unconstrained[obj_name]:
587 total_ct += len(lc_unconstrained[obj_name][band]['mjd'])
588 self.assertEqual(ct_unconstrained, total_ct)
590 total_ct = 0
591 for obj_name in lc_constrained:
592 for band in lc_constrained[obj_name]:
593 total_ct += len(lc_constrained[obj_name][band]['mjd'])
594 self.assertEqual(ct_constrained, total_ct)
597class AgnLightCurveTest(unittest.TestCase):
599 @classmethod
600 def setUpClass(cls):
601 rng = np.random.RandomState(119)
603 cls.txt_cat_name = tempfile.mktemp(prefix='agn_lc_cat', suffix='.txt', dir=ROOT)
605 n_galaxies = 20
607 sed_dir = os.path.join(getPackageDir("sims_sed_library"), "galaxySED")
608 list_of_seds = os.listdir(sed_dir)
609 disk_sed_dexes = rng.randint(0, len(list_of_seds), size=n_galaxies)
610 bulge_sed_dexes = rng.randint(0, len(list_of_seds), size=n_galaxies)
612 avBulge = rng.random_sample(n_galaxies)*0.3+0.1
613 avDisk = rng.random_sample(n_galaxies)*0.3+0.1
615 mjdList = rng.random_sample(n_galaxies)*10.0+49330.0
616 redshiftList = rng.random_sample(n_galaxies)*1.5+0.01
618 tauList = rng.random_sample(n_galaxies)*1.0+1.0
619 sfuList = rng.random_sample(n_galaxies)*2.0+1.0
620 sfgList = rng.random_sample(n_galaxies)*2.0+1.0
621 sfrList = rng.random_sample(n_galaxies)*2.0+1.0
622 sfiList = rng.random_sample(n_galaxies)*2.0+1.0
623 sfzList = rng.random_sample(n_galaxies)*2.0+1.0
624 sfyList = rng.random_sample(n_galaxies)*2.0+1.0
626 raList = rng.random_sample(n_galaxies)*7.0+78.0
627 decList = rng.random_sample(n_galaxies)*4.0-69.0
629 normDisk = rng.random_sample(n_galaxies)*5.0+20.0
630 normBulge = rng.random_sample(n_galaxies)*5.0+20.0
631 normAgn = rng.random_sample(n_galaxies)*5.0+20.0
633 with open(cls.txt_cat_name, "w") as output_file:
634 for ix in range(n_galaxies):
635 varParam = {'varMethodName': 'applyAgn',
636 'pars': {'agn_tau': tauList[ix], 'agn_sfu': sfuList[ix],
637 'agn_sfg': sfgList[ix], 'agn_sfr': sfrList[ix],
638 'agn_sfi': sfiList[ix], 'agn_sfz': sfzList[ix],
639 'agn_sfy': sfyList[ix], 't0_mjd': mjdList[ix],
640 'seed': rng.randint(0, 200000)}}
642 paramStr = json.dumps(varParam)
644 output_file.write("%d;%f;%f;" % (ix, raList[ix], decList[ix])
645 + "%f;%f;" % (np.radians(raList[ix]), np.radians(decList[ix]))
646 + "%f;" % (redshiftList[ix])
647 + "%s;%f;%f;" % (list_of_seds[disk_sed_dexes[ix]],
648 avDisk[ix], normDisk[ix])
649 + "%s;%f;%f;" % (list_of_seds[bulge_sed_dexes[ix]],
650 avBulge[ix], normBulge[ix])
651 + "agn.spec;%s;%f\n" % (paramStr, normAgn[ix]))
653 dtype = np.dtype([
654 ('galid', np.int),
655 ('raDeg', np.float), ('decDeg', np.float),
656 ('raJ2000', np.float), ('decJ2000', np.float),
657 ('redshift', np.float),
658 ('sedFilenameDisk', str, 300), ('internalAvDisk', np.float),
659 ('magNormDisk', np.float),
660 ('sedFilenameBulge', str, 300), ('internalAvBulge', np.float),
661 ('magNormBulge', np.float),
662 ('sedFilenameAgn', str, 300), ('varParamStr', str, 600),
663 ('magNormAgn', np.float)
664 ])
666 cls.agn_db = fileDBObject(cls.txt_cat_name, delimiter=';',
667 runtable='test', dtype=dtype,
668 idColKey='galid')
670 cls.agn_db.raColName = 'raDeg'
671 cls.agn_db.decColName = 'decDeg'
672 cls.agn_db.objectTypeId = 112
674 # what follows is a hack to deal with the fact thar
675 # our varParamStr values are longer than 256 characters
676 # which is the default maximum length that a
677 # CatalogDBObject expects a string to be
678 #
679 cls.agn_db.dbTypeMap['STRING'] = (str, 600)
680 cls.agn_db.columns = None
681 cls.agn_db._make_default_columns()
682 cls.agn_db._make_column_map()
683 cls.agn_db._make_type_map()
685 cls.opsimDb = os.path.join(getPackageDir("sims_data"), "OpSimData")
686 cls.opsimDb = os.path.join(cls.opsimDb, "opsimblitz1_1133_sqlite.db")
688 @classmethod
689 def tearDownClass(cls):
690 sims_clean_up()
691 if os.path.exists(cls.txt_cat_name):
692 os.unlink(cls.txt_cat_name)
694 def test_agn_light_curves(self):
695 """
696 Test the AgnLightCurveGenerator by generating some AGN light
697 curves and comparing them to the results obtained by generating a
698 series of InstanceCatalogs containing the same objects at the same
699 MJDs
700 """
702 raRange = (78.0, 85.0)
703 decRange = (-69.0, -65.0)
704 bandpass = 'g'
706 lc_gen = AgnLightCurveGenerator(self.agn_db, self.opsimDb)
707 pointings = lc_gen.get_pointings(raRange, decRange, bandpass=bandpass)
708 for row in pointings:
709 for obs in row:
710 mjd = ModifiedJulianDate(TAI=obs.mjd.TAI-49000.0+59580.0)
711 obs.mjd = mjd
713 test_light_curves, truth_info = lc_gen.light_curves_from_pointings(pointings)
715 self.assertGreater(len(test_light_curves), 2) # make sure we got some light curves
717 for unique_id in test_light_curves:
718 # verify that the sources returned all do vary by making sure that the
719 # np.diff run on the magnitudes reutrns something non-zero
720 self.assertGreater(np.abs(np.diff(test_light_curves[unique_id][bandpass]['mag'])).max(), 0.0)
721 self.assertGreater(len(test_light_curves[unique_id][bandpass]['mjd']), 0)
723 # Now test that specifying a small chunk_size does not change the output
724 # light curves
725 chunk_light_curves, truth_info = lc_gen.light_curves_from_pointings(pointings, chunk_size=1)
726 self.assertGreater(len(chunk_light_curves), 2)
728 for unique_id in test_light_curves:
729 self.assertEqual(len(test_light_curves[unique_id][bandpass]['mjd']),
730 len(chunk_light_curves[unique_id][bandpass]['mjd']))
731 np.testing.assert_array_equal(test_light_curves[unique_id][bandpass]['mjd'],
732 chunk_light_curves[unique_id][bandpass]['mjd'])
733 np.testing.assert_array_equal(test_light_curves[unique_id][bandpass]['mag'],
734 chunk_light_curves[unique_id][bandpass]['mag'])
735 np.testing.assert_array_equal(test_light_curves[unique_id][bandpass]['error'],
736 chunk_light_curves[unique_id][bandpass]['error'])
738 # Now find all of the ObservationMetaData that were included in our
739 # light curves, generate InstanceCatalogs from them separately,
740 # and verify that the contents of the InstanceCatalogs agree with
741 # the contents of the light curves.
743 gen = ObservationMetaDataGenerator(database=self.opsimDb,
744 driver='sqlite')
746 obs_list = gen.getObservationMetaData(fieldRA=raRange,
747 fieldDec=decRange,
748 telescopeFilter=bandpass,
749 boundLength=1.75)
750 for obs in obs_list:
751 mjd = ModifiedJulianDate(TAI=obs.mjd.TAI-49000.0+59580.0)
752 obs.mjd = mjd
754 ct = 0
755 for obs in obs_list:
756 cat = agnControlCatalog(self.agn_db,
757 obs_metadata=obs)
759 for agn_obj in cat.iter_catalog():
760 ct += 1
761 lc = test_light_curves[agn_obj[0]][bandpass]
762 dex = np.argmin(np.abs(lc['mjd']-obs.mjd.TAI))
763 self.assertLess(np.abs(lc['mjd'][dex]-obs.mjd.TAI), 1.0e-7)
764 self.assertLess(np.abs(lc['mag'][dex]-agn_obj[3]), 1.0e-7)
765 self.assertLess(np.abs(lc['error'][dex]-agn_obj[4]), 1.0e-7)
767 # Verify that the catalogs and LightCurveGenerator returned the
768 # same number of observations
769 total_ct = 0
770 for obj_name in test_light_curves:
771 for band in test_light_curves[obj_name]:
772 total_ct += len(test_light_curves[obj_name][band]['mjd'])
773 self.assertEqual(ct, total_ct)
775 def test_limited_agn_light_curves(self):
776 """
777 Test that we can select only a limited number of agn light curves
778 per field of view
779 """
781 lc_limit = 2
782 raRange = (78.0, 82.0)
783 decRange = (-69.0, -65.0)
784 bandpass = 'g'
786 lc_gen = AgnLightCurveGenerator(self.agn_db, self.opsimDb)
787 pointings = lc_gen.get_pointings(raRange, decRange, bandpass=bandpass)
788 for row in pointings:
789 for obs in row:
790 mjd = ModifiedJulianDate(TAI=obs.mjd.TAI-49000.0+59580.0)
791 obs.mjd = mjd
792 self.assertEqual(len(pointings), 1)
794 control_lc, truth = lc_gen.light_curves_from_pointings(pointings)
795 self.assertGreater(len(control_lc), 2)
796 test_lc, truth = lc_gen.light_curves_from_pointings(pointings, lc_per_field=lc_limit)
797 self.assertGreater(len(control_lc), len(test_lc))
798 self.assertEqual(len(test_lc), lc_limit)
800 def test_multiband_light_curves(self):
801 """
802 Check that multi-band light curves are returned correctly.
803 """
805 raRange = (78.0, 82.0)
806 decRange = (-69.0, -65.0)
807 bandpass = ('r', 'g')
809 gen = AgnLightCurveGenerator(self.agn_db, self.opsimDb)
810 pointings = gen.get_pointings(raRange, decRange, bandpass=bandpass)
811 for row in pointings:
812 for obs in row:
813 mjd = ModifiedJulianDate(TAI=obs.mjd.TAI-49000.0+59580.0)
814 obs.mjd = mjd
816 lc_dict, truth_info = gen.light_curves_from_pointings(pointings)
817 self.assertGreater(len(lc_dict), 2)
819 obs_gen = ObservationMetaDataGenerator(database=self.opsimDb, driver='sqlite')
820 control_pointings_r = obs_gen.getObservationMetaData(fieldRA=raRange, fieldDec=decRange,
821 telescopeFilter='r', boundLength=1.75)
823 for obs in control_pointings_r:
824 mjd = ModifiedJulianDate(TAI=obs.mjd.TAI-49000.0+59580.0)
825 obs.mjd = mjd
827 control_pointings_g = obs_gen.getObservationMetaData(fieldRA=raRange, fieldDec=decRange,
828 telescopeFilter='g', boundLength=1.75)
829 for obs in control_pointings_g:
830 mjd = ModifiedJulianDate(TAI=obs.mjd.TAI-49000.0+59580.0)
831 obs.mjd = mjd
833 self.assertGreater(len(control_pointings_g), 0)
834 self.assertGreater(len(control_pointings_r), 0)
836 ct = 0
837 for obs in control_pointings_r:
838 cat = agnControlCatalog(self.agn_db,
839 obs_metadata=obs)
841 for star_obj in cat.iter_catalog():
842 ct += 1
843 lc = lc_dict[star_obj[0]]['r']
844 dex = np.argmin(np.abs(lc['mjd']-obs.mjd.TAI))
845 self.assertLess(np.abs(lc['mjd'][dex]-obs.mjd.TAI), 1.0e-7)
846 self.assertLess(np.abs(lc['mag'][dex]-star_obj[3]), 1.0e-7)
847 self.assertLess(np.abs(lc['error'][dex]-star_obj[4]), 1.0e-7)
849 for obs in control_pointings_g:
850 cat = agnControlCatalog(self.agn_db,
851 obs_metadata=obs)
853 for star_obj in cat.iter_catalog():
854 ct += 1
855 lc = lc_dict[star_obj[0]]['g']
856 dex = np.argmin(np.abs(lc['mjd']-obs.mjd.TAI))
857 self.assertLess(np.abs(lc['mjd'][dex]-obs.mjd.TAI), 1.0e-7)
858 self.assertLess(np.abs(lc['mag'][dex]-star_obj[3]), 1.0e-7)
859 self.assertLess(np.abs(lc['error'][dex]-star_obj[4]), 1.0e-7)
861 # Verify that the catalogs and LightCurveGenerator returned the
862 # same number of observations
863 total_ct = 0
864 for obj_name in lc_dict:
865 for band in lc_dict[obj_name]:
866 total_ct += len(lc_dict[obj_name][band]['mjd'])
867 self.assertEqual(ct, total_ct)
869 def test_agn_constraint(self):
870 """
871 Test that the light curve generator correctly ignores objects
872 with varParamStr == None
874 We do this by generating a database of two galaxies with AGN-like
875 varParamStr and two stars with no varParamStr. We run this database
876 through a LightCurveGenerator and an InstanceCatalog. We verify that
877 the LightCurveGenerator finds 2 AGN while the InstanceCatalog finds
878 4 stars.
879 """
881 rng = np.random.RandomState(83)
883 raRange = (80.8, 83.8)
884 decRange = (-71.0, -69.0)
885 bandpass = 'g'
887 agn_sed_name = "agn.spec"
888 sed_name = "Burst.10E10.1Z.spec.gz"
890 varparams = {'varMethodName': 'applyAgn',
891 'pars': {'agn_tau': 20.0, 'agn_sfu': 11.0,
892 'agn_sfg': 12.0, 'agn_sfr': 13.0,
893 'agn_sfi': 14.0, 'agn_sfz': 15.0,
894 'agn_sfy': 16.0, 't0_mjd': 49330.0,
895 'seed': rng.randint(0, 200000)}}
897 varParamStr = json.dumps(varparams)
899 # create the dummy database
900 db_name = tempfile.mktemp(prefix="agn_constraint_cat_sqlite", suffix=".db", dir=ROOT)
901 conn = sqlite3.connect(db_name)
902 c = conn.cursor()
903 c.execute('''CREATE TABLE agn
904 (id int, ra real, dec real, redshift real,
905 sedFilenameDisk text, internalAvDisk real,
906 magNormDisk real,
907 sedFilenameBulge text, internalAvBulge real,
908 magNormBulge real,
909 sedFilenameAgn text, varParamStr text,
910 magNormAgn real)''')
911 conn.commit()
913 for ix, (rr, dd, zz, avb, mnb, avd, mnd, mnagn) in \
914 enumerate(zip(rng.random_sample(4)*(raRange[1]-raRange[0])+raRange[0],
915 rng.random_sample(4)*(decRange[1]-decRange[0])+decRange[0],
916 rng.random_sample(4)*0.5+0.1,
917 rng.random_sample(4)*0.5+0.1, rng.random_sample(4)*3.0+17.0,
918 rng.random_sample(4)*0.5+0.1, rng.random_sample(4)*3.0+17.0,
919 rng.random_sample(4)*3.0+17.0)):
921 if ix < 2:
922 cmd = '''INSERT INTO agn VALUES(%d, %e, %e, %e, '%s', %e,
923 %e, '%s', %e, %e, '%s', '%s', %e)''' % \
924 (ix, rr, dd, zz, sed_name, avb, mnb, sed_name, avd, mnd,
925 agn_sed_name, varParamStr, mnagn)
926 else:
927 cmd = '''INSERT INTO agn VALUES(%d, %e, %e, %e, '%s', %e,
928 %e, '%s', %e, %e, '%s', NULL, %e)''' % \
929 (ix, rr, dd, zz, sed_name, avb, mnb, sed_name, avd, mnd,
930 agn_sed_name, mnagn)
932 c.execute(cmd)
934 conn.commit()
936 # create a CatalogDBObject class to interface with the dummy database
937 class dummyAgnDBObject(CatalogDBObject):
938 database = db_name
939 host = None
940 driver = 'sqlite'
941 tableid = 'agn'
942 raColName = 'ra'
943 decColName = 'dec'
944 idColKey = 'id'
945 objid = 'dummyAgn'
946 skipRegistration = True
947 objectTypeId = 99
949 columns = [('raJ2000', 'ra*PI()/180.0', float),
950 ('decJ2000', 'dec*PI()/180.0', float)]
952 agn_db = dummyAgnDBObject()
954 # verify that the LightCurveGenerator finds the two variable stars
955 lc_gen = AgnLightCurveGenerator(agn_db, self.opsimDb)
956 ptngs = lc_gen.get_pointings(raRange, decRange, bandpass=bandpass)
957 for row in ptngs:
958 for obs in row:
959 mjd = ModifiedJulianDate(TAI=obs.mjd.TAI-49000.0+59580.0)
960 obs.mjd = mjd
962 lc_dict, truth_dict = lc_gen.light_curves_from_pointings(ptngs)
963 self.assertEqual(len(lc_dict), 2)
965 if os.path.exists(db_name):
966 os.unlink(db_name)
968 # verify that an InstanceCatalog finds all 4 stars
969 obs = ptngs[0][0]
970 cat = agnControlCatalog(agn_db, obs_metadata=obs)
971 with lsst.utils.tests.getTempFilePath('.txt') as dummy_cat_name:
972 cat.write_catalog(dummy_cat_name)
973 with open(dummy_cat_name, 'r') as input_file:
974 lines = input_file.readlines()
975 self.assertEqual(len(lines), 5)
978class MemoryTestClass(lsst.utils.tests.MemoryTestCase):
979 pass
982if __name__ == "__main__": 982 ↛ 983line 982 didn't jump to line 983, because the condition on line 982 was never true
983 lsst.utils.tests.init()
984 unittest.main()