Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from __future__ import with_statement 

2from builtins import zip 

3from builtins import range 

4import unittest 

5import os 

6import json 

7import sqlite3 

8import numpy as np 

9import tempfile 

10import shutil 

11import lsst 

12# import lsst.utils.tests as utilsTests 

13from lsst.utils import getPackageDir 

14from lsst.sims.utils.CodeUtilities import sims_clean_up 

15 

16from lsst.sims.catalogs.db import fileDBObject 

17 

18from lsst.sims.catalogs.definitions import InstanceCatalog 

19from lsst.sims.catalogs.decorators import compound 

20from lsst.sims.catUtils.utils import ObservationMetaDataGenerator 

21 

22from lsst.sims.catUtils.mixins import PhotometryStars, VariabilityStars 

23from lsst.sims.catUtils.utils import StellarLightCurveGenerator 

24 

25from lsst.sims.catalogs.db import CatalogDBObject 

26 

27from lsst.sims.catUtils.mixins import PhotometryGalaxies, VariabilityGalaxies 

28from lsst.sims.catUtils.utils import AgnLightCurveGenerator 

29from lsst.sims.utils import ModifiedJulianDate 

30 

31ROOT = os.path.abspath(os.path.dirname(__file__)) 

32 

33 

34def setup_module(module): 

35 lsst.utils.tests.init() 

36 

37 

38class stellarControlCatalog(InstanceCatalog, 

39 PhotometryStars, VariabilityStars): 

40 catalog_type = __file__ + 'stellar_control_catalog' 

41 

42 column_outputs = ["uniqueId", "raJ2000", "decJ2000", "mag", "sigma_mag"] 

43 

44 @compound("mag", "sigma_mag") 

45 def get_phot(self): 

46 mm = self.column_by_name("lsst_%s" % self.obs_metadata.bandpass) 

47 sig = self.column_by_name("sigma_lsst_%s" % self.obs_metadata.bandpass) 

48 return np.array([mm, sig]) 

49 

50 

51class agnControlCatalog(InstanceCatalog, 

52 PhotometryGalaxies, VariabilityGalaxies): 

53 

54 catalog_type = __file__ + 'agn_control_catalog' 

55 column_outputs = ["uniqueId", "raJ2000", "decJ2000", "mag", "sigma_mag"] 

56 

57 @compound("mag", "sigma_mag") 

58 def get_phot(self): 

59 return np.array([self.column_by_name("%sAgn" % self.obs_metadata.bandpass), 

60 self.column_by_name("sigma_%sAgn" % self.obs_metadata.bandpass)]) 

61 

62 

63class StellarLightCurveTest(unittest.TestCase): 

64 

65 @classmethod 

66 def setUpClass(cls): 

67 """ 

68 Create a fake catalog of RR Lyrae stars. Store it in cls.stellar_db 

69 """ 

70 cls.scratchDir = tempfile.mkdtemp(dir=ROOT, prefix='StellarLigghtCurveTest-') 

71 

72 rng = np.random.RandomState(88) 

73 n_stars = 10000 

74 sed_dir = os.path.join(getPackageDir("sims_sed_library")) 

75 sed_dir = os.path.join(sed_dir, "starSED", "kurucz") 

76 list_of_seds = os.listdir(sed_dir) 

77 

78 lc_dir = os.path.join(getPackageDir("sims_sed_library"), "rrly_lc") 

79 lc_dir = os.path.join(lc_dir, "RRab") 

80 list_of_lc = ['rrly_lc/RRab/%s' % ww for ww in os.listdir(lc_dir) if "per.txt" in ww] 

81 

82 cls.dtype = np.dtype([('id', np.int), 

83 ('raDeg', np.float), 

84 ('decDeg', np.float), 

85 ('raJ2000', np.float), 

86 ('decJ2000', np.float), 

87 ('magNorm', np.float), 

88 ('galacticAv', np.float), 

89 ('sedFilename', str, 300), 

90 ('varParamStr', str, 300), 

91 ('parallax', np.float), 

92 ('ebv', np.float)]) 

93 

94 # write the catalog as a text file to be ingested with fileDBObject 

95 cls.txt_name = os.path.join(cls.scratchDir, "stellar_lc_catalog.txt") 

96 with open(cls.txt_name, "w") as output_file: 

97 sed_dex = rng.randint(0, len(list_of_seds), size=n_stars) 

98 lc_dex = rng.randint(0, len(list_of_lc), size=n_stars) 

99 mjd0 = rng.random_sample(n_stars)*10000.0+40000.0 

100 raList = rng.random_sample(n_stars)*360.0 

101 decList = -90.0 + rng.random_sample(n_stars)*120.0 

102 magNormList = rng.random_sample(n_stars)*3.0+14.0 

103 AvList = rng.random_sample(n_stars)*0.2+0.1 

104 pxList = rng.random_sample(n_stars)*0.1 

105 for ix in range(n_stars): 

106 varparams = {'varMethodName': 'applyRRly', 

107 'pars': {'tStartMjd': mjd0[ix], 

108 'filename': list_of_lc[lc_dex[ix]]}} 

109 varparamstr = json.dumps(varparams) 

110 output_file.write("%d;%lf;%lf;%lf;%lf;%lf;%lf;%s;%s;%lf;%lf\n" 

111 % (ix, raList[ix], decList[ix], 

112 np.radians(raList[ix]), 

113 np.radians(decList[ix]), 

114 magNormList[ix], AvList[ix], 

115 list_of_seds[sed_dex[ix]], 

116 varparamstr,pxList[ix], 

117 AvList[ix]/3.1)) 

118 

119 cls.stellar_db = fileDBObject(cls.txt_name, delimiter=';', 

120 runtable='test', dtype=cls.dtype, 

121 idColKey='id') 

122 

123 cls.stellar_db.raColName = 'raDeg' 

124 cls.stellar_db.decColName = 'decDeg' 

125 cls.stellar_db.objectTypeId = 32 

126 

127 cls.opsimDb = os.path.join(getPackageDir("sims_data"), "OpSimData") 

128 cls.opsimDb = os.path.join(cls.opsimDb, "opsimblitz1_1133_sqlite.db") 

129 

130 @classmethod 

131 def tearDownClass(cls): 

132 sims_clean_up() 

133 if os.path.exists(cls.txt_name): 

134 os.unlink(cls.txt_name) 

135 if os.path.exists(cls.scratchDir): 

136 shutil.rmtree(cls.scratchDir) 

137 

138 def test_get_pointings(self): 

139 """ 

140 Test that the get_pointings method does, in fact, return ObservationMetaData 

141 that are grouped appropriately. 

142 """ 

143 

144 raRange = (78.0, 89.0) 

145 decRange = (-74.0, -60.0) 

146 bandpass = 'g' 

147 

148 lc_gen = StellarLightCurveGenerator(self.stellar_db, self.opsimDb) 

149 

150 pointings = lc_gen.get_pointings(raRange, decRange, bandpass=bandpass) 

151 

152 self.assertGreater(len(pointings), 1) 

153 

154 for group in pointings: 

155 for ix, obs in enumerate(group): 

156 self.assertAlmostEqual(obs.pointingRA, group[0].pointingRA, 12) 

157 self.assertAlmostEqual(obs.pointingDec, group[0].pointingDec, 12) 

158 self.assertEqual(obs.bandpass, bandpass) 

159 if ix > 0: 

160 self.assertGreater(obs.mjd.TAI, group[ix-1].mjd.TAI) 

161 

162 def test_get_pointings_v4(self): 

163 """ 

164 Test that the get_pointings method runs on an OpSim v4 database 

165 """ 

166 

167 v4_db = os.path.join(getPackageDir('sims_data'), 'OpSimData', 

168 'astro-lsst-01_2014.db') 

169 

170 raRange = (78.0, 89.0) 

171 decRange = (-74.0, -60.0) 

172 bandpass = 'y' 

173 

174 lc_gen = StellarLightCurveGenerator(self.stellar_db, v4_db) 

175 

176 pointings = lc_gen.get_pointings(raRange, decRange, bandpass=bandpass) 

177 

178 self.assertGreater(len(pointings), 1) 

179 

180 for group in pointings: 

181 for ix, obs in enumerate(group): 

182 self.assertAlmostEqual(obs.pointingRA, group[0].pointingRA, 12) 

183 self.assertAlmostEqual(obs.pointingDec, group[0].pointingDec, 12) 

184 self.assertEqual(obs.bandpass, bandpass) 

185 if ix > 0: 

186 self.assertGreater(obs.mjd.TAI, group[ix-1].mjd.TAI) 

187 

188 def test_get_pointings_multiband(self): 

189 """ 

190 Test that the get_pointings method does, in fact, return ObservationMetaData 

191 that are grouped appropriately. Test on more than one filter. 

192 """ 

193 

194 raRange = (78.0, 89.0) 

195 decRange = (-74.0, -60.0) 

196 bandpass = ('g', 'z') 

197 

198 lc_gen = StellarLightCurveGenerator(self.stellar_db, self.opsimDb) 

199 

200 pointings = lc_gen.get_pointings(raRange, decRange, bandpass=bandpass) 

201 

202 self.assertGreater(len(pointings), 1) 

203 

204 ct_g = 0 

205 ct_z = 0 

206 

207 for group in pointings: 

208 for ix, obs in enumerate(group): 

209 self.assertAlmostEqual(obs.pointingRA, group[0].pointingRA, 12) 

210 self.assertAlmostEqual(obs.pointingDec, group[0].pointingDec, 12) 

211 

212 if obs.bandpass == 'g': 

213 ct_g += 1 

214 elif obs.bandpass == 'z': 

215 ct_z += 1 

216 else: 

217 raise RuntimeError("Asked for filters (g,z) but got %s" % obs.bandpass) 

218 

219 if ix > 0: 

220 self.assertGreater(obs.mjd.TAI, group[ix-1].mjd.TAI) 

221 

222 self.assertGreater(ct_g, 0) 

223 self.assertGreater(ct_z, 0) 

224 

225 def test_stellar_light_curves(self): 

226 """ 

227 Test the StellarLightCurveGenerator by generating some RR Lyrae light 

228 curves and comparing them to the results obtained by generating a 

229 series of InstanceCatalogs containing the same objects at the same 

230 MJDs 

231 """ 

232 

233 raRange = (78.0, 85.0) 

234 decRange = (-69.0, -65.0) 

235 bandpass = 'r' 

236 

237 lc_gen = StellarLightCurveGenerator(self.stellar_db, self.opsimDb) 

238 pointings = lc_gen.get_pointings(raRange, decRange, bandpass=bandpass) 

239 test_light_curves, truth_info = lc_gen.light_curves_from_pointings(pointings) 

240 

241 self.assertGreater(len(test_light_curves), 2) # make sure we got some light curves 

242 

243 for unique_id in test_light_curves: 

244 # verify that the sources returned all do vary by making sure that the 

245 # np.diff run on the magnitudes reutrns something non-zero 

246 self.assertGreater(np.abs(np.diff(test_light_curves[unique_id][bandpass]['mag'])).max(), 0.0) 

247 self.assertGreater(len(test_light_curves[unique_id][bandpass]['mjd']), 0) 

248 

249 # Now test that specifying a small chunk_size does not change the output 

250 # light curves 

251 chunk_light_curves, truth_info = lc_gen.light_curves_from_pointings(pointings, chunk_size=1) 

252 self.assertGreater(len(chunk_light_curves), 2) 

253 

254 for unique_id in test_light_curves: 

255 self.assertEqual(len(test_light_curves[unique_id][bandpass]['mjd']), 

256 len(chunk_light_curves[unique_id][bandpass]['mjd'])) 

257 np.testing.assert_array_equal(test_light_curves[unique_id][bandpass]['mjd'], 

258 chunk_light_curves[unique_id][bandpass]['mjd']) 

259 np.testing.assert_array_equal(test_light_curves[unique_id][bandpass]['mag'], 

260 chunk_light_curves[unique_id][bandpass]['mag']) 

261 np.testing.assert_array_equal(test_light_curves[unique_id][bandpass]['error'], 

262 chunk_light_curves[unique_id][bandpass]['error']) 

263 

264 # Now find all of the ObservationMetaData that were included in our 

265 # light curves, generate InstanceCatalogs from them separately, 

266 # and verify that the contents of the InstanceCatalogs agree with 

267 # the contents of the light curves. 

268 

269 gen = ObservationMetaDataGenerator(database=self.opsimDb, 

270 driver='sqlite') 

271 

272 obs_list = gen.getObservationMetaData(fieldRA=raRange, 

273 fieldDec=decRange, 

274 telescopeFilter=bandpass, 

275 boundLength=1.75) 

276 

277 ct = 0 

278 for obs in obs_list: 

279 cat = stellarControlCatalog(self.stellar_db, 

280 obs_metadata=obs) 

281 

282 for star_obj in cat.iter_catalog(): 

283 ct += 1 

284 lc = test_light_curves[star_obj[0]][bandpass] 

285 dex = np.argmin(np.abs(lc['mjd']-obs.mjd.TAI)) 

286 self.assertLess(np.abs(lc['mjd'][dex]-obs.mjd.TAI), 1.0e-7) 

287 self.assertLess(np.abs(lc['mag'][dex]-star_obj[3]), 1.0e-7) 

288 self.assertLess(np.abs(lc['error'][dex]-star_obj[4]), 1.0e-7) 

289 

290 # Verify that the same number of objects and observations were found in the 

291 # catalogs and the LightCurveGenerator output 

292 total_ct = 0 

293 for obj_name in test_light_curves: 

294 for bandpass in test_light_curves[obj_name]: 

295 total_ct += len(test_light_curves[obj_name][bandpass]['mjd']) 

296 self.assertEqual(ct, total_ct) 

297 

298 

299 def test_limited_stellar_light_curves(self): 

300 """ 

301 Test that we can ask for a limited number of light curves per field of view 

302 """ 

303 

304 lc_limit = 2 

305 raRange = (78.0, 82.0) 

306 decRange = (-69.0, -65.0) 

307 bandpass = 'r' 

308 

309 lc_gen = StellarLightCurveGenerator(self.stellar_db, self.opsimDb) 

310 pointings = lc_gen.get_pointings(raRange, decRange, bandpass=bandpass) 

311 

312 self.assertEqual(len(pointings), 1) 

313 

314 control_light_curves, truth_info = lc_gen.light_curves_from_pointings(pointings) 

315 self.assertGreater(len(control_light_curves), 2) 

316 

317 test_light_curves, truth_info = lc_gen.light_curves_from_pointings(pointings, 

318 lc_per_field=lc_limit) 

319 

320 self.assertGreater(len(control_light_curves), len(test_light_curves)) 

321 self.assertEqual(len(test_light_curves), lc_limit) 

322 

323 def test_date_range(self): 

324 """ 

325 Run test_stellar_light_curves, this time specifying a range in MJD. 

326 """ 

327 

328 raRange = (0.0, 110.0) 

329 decRange = (-90.0, -50.0) 

330 bandpass = 'g' 

331 mjdRange = (49356.0, 49357.0) 

332 

333 lc_gen = StellarLightCurveGenerator(self.stellar_db, self.opsimDb) 

334 pointings = lc_gen.get_pointings(raRange, decRange, bandpass=bandpass, expMJD=mjdRange) 

335 test_light_curves, truth_info = lc_gen.light_curves_from_pointings(pointings) 

336 

337 self.assertGreater(len(test_light_curves), 2) 

338 

339 for unique_id in test_light_curves: 

340 # verify that the sources returned all do vary by making sure that the 

341 # np.diff run on the magnitudes reutrns something non-zero 

342 self.assertGreater(np.abs(np.diff(test_light_curves[unique_id][bandpass]['mag'])).max(), 0.0) 

343 self.assertGreater(len(test_light_curves[unique_id][bandpass]['mjd']), 0) 

344 self.assertGreater(test_light_curves[unique_id][bandpass]['mjd'].min(), mjdRange[0]-1.0e-12) 

345 self.assertLess(test_light_curves[unique_id][bandpass]['mjd'].max(), mjdRange[1]+1.0e-12) 

346 

347 # Now test that specifying a small chunk_size does not change the output 

348 # light curves 

349 chunk_light_curves, truth_info = lc_gen.light_curves_from_pointings(pointings, chunk_size=1) 

350 self.assertGreater(len(chunk_light_curves), 2) 

351 

352 for unique_id in test_light_curves: 

353 self.assertEqual(len(test_light_curves[unique_id][bandpass]['mjd']), 

354 len(chunk_light_curves[unique_id][bandpass]['mjd'])) 

355 np.testing.assert_array_equal(test_light_curves[unique_id][bandpass]['mjd'], 

356 chunk_light_curves[unique_id][bandpass]['mjd']) 

357 np.testing.assert_array_equal(test_light_curves[unique_id][bandpass]['mag'], 

358 chunk_light_curves[unique_id][bandpass]['mag']) 

359 np.testing.assert_array_equal(test_light_curves[unique_id][bandpass]['error'], 

360 chunk_light_curves[unique_id][bandpass]['error']) 

361 

362 # Now find all of the ObservationMetaData that were included in our 

363 # light curves, generate InstanceCatalogs from them separately, 

364 # and verify that the contents of the InstanceCatalogs agree with 

365 # the contents of the light curves. 

366 

367 gen = ObservationMetaDataGenerator(database=self.opsimDb, 

368 driver='sqlite') 

369 

370 obs_list = gen.getObservationMetaData(fieldRA=raRange, 

371 fieldDec=decRange, 

372 telescopeFilter=bandpass, 

373 expMJD=mjdRange, 

374 boundLength=1.75) 

375 

376 ct = 0 

377 for obs in obs_list: 

378 cat = stellarControlCatalog(self.stellar_db, 

379 obs_metadata=obs) 

380 

381 for star_obj in cat.iter_catalog(): 

382 ct += 1 

383 lc = test_light_curves[star_obj[0]][bandpass] 

384 dex = np.argmin(np.abs(lc['mjd']-obs.mjd.TAI)) 

385 self.assertLess(np.abs(lc['mjd'][dex]-obs.mjd.TAI), 1.0e-7) 

386 self.assertLess(np.abs(lc['mag'][dex]-star_obj[3]), 1.0e-7) 

387 self.assertLess(np.abs(lc['error'][dex]-star_obj[4]), 1.0e-7) 

388 

389 # Verify that the same number of objects and observations were found in the 

390 # catalogs and the LightCurveGenerator output 

391 total_ct = 0 

392 for obj_name in test_light_curves: 

393 for bandpass in test_light_curves[obj_name]: 

394 total_ct += len(test_light_curves[obj_name][bandpass]['mjd']) 

395 self.assertEqual(ct, total_ct) 

396 

397 def test_multiband_light_curves(self): 

398 """ 

399 Check that multi-band light curves are returned correctly. 

400 """ 

401 

402 raRange = (78.0, 82.0) 

403 decRange = (-69.0, -65.0) 

404 bandpass = ('r', 'g') 

405 

406 gen = StellarLightCurveGenerator(self.stellar_db, self.opsimDb) 

407 pointings = gen.get_pointings(raRange, decRange, bandpass=bandpass) 

408 lc_dict, truth_info = gen.light_curves_from_pointings(pointings) 

409 self.assertGreater(len(lc_dict), 2) 

410 

411 obs_gen = ObservationMetaDataGenerator(database=self.opsimDb, driver='sqlite') 

412 control_pointings_r = obs_gen.getObservationMetaData(fieldRA=raRange, fieldDec=decRange, 

413 telescopeFilter='r', boundLength=1.75) 

414 

415 control_pointings_g = obs_gen.getObservationMetaData(fieldRA=raRange, fieldDec=decRange, 

416 telescopeFilter='g', boundLength=1.75) 

417 

418 self.assertGreater(len(control_pointings_g), 0) 

419 self.assertGreater(len(control_pointings_r), 0) 

420 

421 ct = 0 

422 for obs in control_pointings_r: 

423 cat = stellarControlCatalog(self.stellar_db, 

424 obs_metadata=obs) 

425 

426 for star_obj in cat.iter_catalog(): 

427 ct += 1 

428 lc = lc_dict[star_obj[0]]['r'] 

429 dex = np.argmin(np.abs(lc['mjd']-obs.mjd.TAI)) 

430 self.assertLess(np.abs(lc['mjd'][dex]-obs.mjd.TAI), 1.0e-7) 

431 self.assertLess(np.abs(lc['mag'][dex]-star_obj[3]), 1.0e-7) 

432 self.assertLess(np.abs(lc['error'][dex]-star_obj[4]), 1.0e-7) 

433 

434 for obs in control_pointings_g: 

435 cat = stellarControlCatalog(self.stellar_db, 

436 obs_metadata=obs) 

437 

438 for star_obj in cat.iter_catalog(): 

439 ct += 1 

440 lc = lc_dict[star_obj[0]]['g'] 

441 dex = np.argmin(np.abs(lc['mjd']-obs.mjd.TAI)) 

442 self.assertLess(np.abs(lc['mjd'][dex]-obs.mjd.TAI), 1.0e-7) 

443 self.assertLess(np.abs(lc['mag'][dex]-star_obj[3]), 1.0e-7) 

444 self.assertLess(np.abs(lc['error'][dex]-star_obj[4]), 1.0e-7) 

445 

446 # Verify that the same number of objects and observations were found in the 

447 # catalogs and the LightCurveGenerator output 

448 total_ct = 0 

449 for obj_name in lc_dict: 

450 for bandpass in lc_dict[obj_name]: 

451 total_ct += len(lc_dict[obj_name][bandpass]['mjd']) 

452 self.assertEqual(ct, total_ct) 

453 

454 def test_constraint(self): 

455 """ 

456 Test that the light curve generator correctly ignores objects 

457 with varParamStr == None 

458 

459 We do this by generating a database of two stars with RRLyrae-like 

460 varParamStr and two stars with no varParamStr. We run this database 

461 through a LightCurveGenerator and an InstanceCatalog. We verify that 

462 the LightCurveGenerator finds 2 stars while the InstanceCatalog finds 

463 4 stars. 

464 """ 

465 

466 rng = np.random.RandomState(83) 

467 

468 raRange = (80.8, 83.8) 

469 decRange = (-71.0, -69.0) 

470 bandpass = 'g' 

471 

472 sed_name = "kp01_7500.fits_g40_7600.gz" 

473 

474 varparams = {'varMethodName': 'applyRRly', 

475 'pars': {'tStartMjd': 30000.0, 

476 'filename': 'rrly_lc/RRab/1096833_per.txt'}} 

477 

478 varParamStr = json.dumps(varparams) 

479 

480 # create the dummy database 

481 db_name = tempfile.mktemp(prefix='stellar_constraint_cata_sqlite-', suffix='.db', dir=ROOT) 

482 

483 conn = sqlite3.connect(db_name) 

484 c = conn.cursor() 

485 c.execute('''CREATE TABLE rrly 

486 (id int, ra real, dec real, sedFilename text, magNorm real, 

487 varParamStr text, galacticAv real, parallax real, ebv real)''') 

488 conn.commit() 

489 

490 for ix, (rr, dd, mn, px) in \ 

491 enumerate(zip(rng.random_sample(4)*(raRange[1]-raRange[0])+raRange[0], 

492 rng.random_sample(4)*(decRange[1]-decRange[0])+decRange[0], 

493 rng.random_sample(4)*5.0+16.0, rng.random_sample(4)*0.01)): 

494 

495 if ix < 2: 

496 cmd = '''INSERT INTO rrly VALUES(%d, %e, %e, '%s', %e, '%s', 0.1, 0.032, %e)''' % \ 

497 (ix, rr, dd, sed_name, mn, varParamStr, px) 

498 else: 

499 cmd = '''INSERT INTO rrly VALUES(%d, %e, %e, '%s', %e, NULL, 0.1, 0.032, %e)''' % \ 

500 (ix, rr, dd, sed_name, mn, px) 

501 

502 c.execute(cmd) 

503 

504 conn.commit() 

505 

506 # create a CatalogDBObject class to interface with the dummy database 

507 class dummyRRLyDBObject(CatalogDBObject): 

508 database = db_name 

509 host = None 

510 driver = 'sqlite' 

511 tableid = 'rrly' 

512 raColName = 'ra' 

513 decColName = 'dec' 

514 idColKey = 'id' 

515 objid = 'dummyRRLy' 

516 skipRegistration = True 

517 objectTypeId = 99 

518 

519 columns = [('raJ2000', 'ra*PI()/180.0', float), 

520 ('decJ2000', 'dec*PI()/180.0', float)] 

521 

522 star_db = dummyRRLyDBObject() 

523 

524 # verify that the LightCurveGenerator finds the two variable stars 

525 lc_gen = StellarLightCurveGenerator(star_db, self.opsimDb) 

526 ptngs = lc_gen.get_pointings(raRange, decRange, bandpass=bandpass) 

527 lc_dict, truth_dict = lc_gen.light_curves_from_pointings(ptngs) 

528 self.assertEqual(len(lc_dict), 2) 

529 

530 if os.path.exists(db_name): 

531 os.unlink(db_name) 

532 

533 # verify that an InstanceCatalog finds all 4 stars 

534 obs = ptngs[0][0] 

535 cat = stellarControlCatalog(star_db, obs_metadata=obs) 

536 with lsst.utils.tests.getTempFilePath('.txt') as dummy_cat_name: 

537 cat.write_catalog(dummy_cat_name) 

538 with open(dummy_cat_name, 'r') as input_file: 

539 lines = input_file.readlines() 

540 self.assertEqual(len(lines), 5) 

541 

542 def test_manual_constraint(self): 

543 """ 

544 Test that a constraint put in by hand is properly applied 

545 """ 

546 

547 raRange = (78.0, 89.0) 

548 decRange = (-74.0, -60.0) 

549 bandpass = 'g' 

550 

551 lc_gen = StellarLightCurveGenerator(self.stellar_db, self.opsimDb) 

552 

553 pointings = lc_gen.get_pointings(raRange, decRange, bandpass=bandpass) 

554 

555 (lc_unconstrained, 

556 truth_unconstrianed) = lc_gen.light_curves_from_pointings(pointings) 

557 

558 (lc_constrained, 

559 truth_constrained) = lc_gen.light_curves_from_pointings(pointings, 

560 constraint = 'ebv>0.05') 

561 

562 self.assertGreater(len(lc_constrained), 0) 

563 self.assertLess(len(lc_constrained), len(lc_unconstrained)) 

564 

565 # create catalogs based on all of the pointings in 'pointings'; 

566 # verify that the objects in those catalogs appear correctly 

567 # in the constrained and unconstrained light curves. 

568 

569 class ConstraintCatalogClass(InstanceCatalog): 

570 column_outputs= ['uniqueId', 'ebv'] 

571 

572 ct_unconstrained = 0 

573 ct_constrained = 0 

574 for field in pointings: 

575 for obs in field: 

576 cat = ConstraintCatalogClass(self.stellar_db, obs_metadata=obs) 

577 for star_obj in cat.iter_catalog(): 

578 if star_obj[1]>0.05: 

579 self.assertIn(star_obj[0], lc_constrained) 

580 ct_constrained += 1 

581 self.assertIn(star_obj[0], lc_unconstrained) 

582 ct_unconstrained += 1 

583 

584 total_ct = 0 

585 for obj_name in lc_unconstrained: 

586 for band in lc_unconstrained[obj_name]: 

587 total_ct += len(lc_unconstrained[obj_name][band]['mjd']) 

588 self.assertEqual(ct_unconstrained, total_ct) 

589 

590 total_ct = 0 

591 for obj_name in lc_constrained: 

592 for band in lc_constrained[obj_name]: 

593 total_ct += len(lc_constrained[obj_name][band]['mjd']) 

594 self.assertEqual(ct_constrained, total_ct) 

595 

596 

597class AgnLightCurveTest(unittest.TestCase): 

598 

599 @classmethod 

600 def setUpClass(cls): 

601 rng = np.random.RandomState(119) 

602 

603 cls.txt_cat_name = tempfile.mktemp(prefix='agn_lc_cat', suffix='.txt', dir=ROOT) 

604 

605 n_galaxies = 20 

606 

607 sed_dir = os.path.join(getPackageDir("sims_sed_library"), "galaxySED") 

608 list_of_seds = os.listdir(sed_dir) 

609 disk_sed_dexes = rng.randint(0, len(list_of_seds), size=n_galaxies) 

610 bulge_sed_dexes = rng.randint(0, len(list_of_seds), size=n_galaxies) 

611 

612 avBulge = rng.random_sample(n_galaxies)*0.3+0.1 

613 avDisk = rng.random_sample(n_galaxies)*0.3+0.1 

614 

615 mjdList = rng.random_sample(n_galaxies)*10.0+49330.0 

616 redshiftList = rng.random_sample(n_galaxies)*1.5+0.01 

617 

618 tauList = rng.random_sample(n_galaxies)*1.0+1.0 

619 sfuList = rng.random_sample(n_galaxies)*2.0+1.0 

620 sfgList = rng.random_sample(n_galaxies)*2.0+1.0 

621 sfrList = rng.random_sample(n_galaxies)*2.0+1.0 

622 sfiList = rng.random_sample(n_galaxies)*2.0+1.0 

623 sfzList = rng.random_sample(n_galaxies)*2.0+1.0 

624 sfyList = rng.random_sample(n_galaxies)*2.0+1.0 

625 

626 raList = rng.random_sample(n_galaxies)*7.0+78.0 

627 decList = rng.random_sample(n_galaxies)*4.0-69.0 

628 

629 normDisk = rng.random_sample(n_galaxies)*5.0+20.0 

630 normBulge = rng.random_sample(n_galaxies)*5.0+20.0 

631 normAgn = rng.random_sample(n_galaxies)*5.0+20.0 

632 

633 with open(cls.txt_cat_name, "w") as output_file: 

634 for ix in range(n_galaxies): 

635 varParam = {'varMethodName': 'applyAgn', 

636 'pars': {'agn_tau': tauList[ix], 'agn_sfu': sfuList[ix], 

637 'agn_sfg': sfgList[ix], 'agn_sfr': sfrList[ix], 

638 'agn_sfi': sfiList[ix], 'agn_sfz': sfzList[ix], 

639 'agn_sfy': sfyList[ix], 't0_mjd': mjdList[ix], 

640 'seed': rng.randint(0, 200000)}} 

641 

642 paramStr = json.dumps(varParam) 

643 

644 output_file.write("%d;%f;%f;" % (ix, raList[ix], decList[ix]) 

645 + "%f;%f;" % (np.radians(raList[ix]), np.radians(decList[ix])) 

646 + "%f;" % (redshiftList[ix]) 

647 + "%s;%f;%f;" % (list_of_seds[disk_sed_dexes[ix]], 

648 avDisk[ix], normDisk[ix]) 

649 + "%s;%f;%f;" % (list_of_seds[bulge_sed_dexes[ix]], 

650 avBulge[ix], normBulge[ix]) 

651 + "agn.spec;%s;%f\n" % (paramStr, normAgn[ix])) 

652 

653 dtype = np.dtype([ 

654 ('galid', np.int), 

655 ('raDeg', np.float), ('decDeg', np.float), 

656 ('raJ2000', np.float), ('decJ2000', np.float), 

657 ('redshift', np.float), 

658 ('sedFilenameDisk', str, 300), ('internalAvDisk', np.float), 

659 ('magNormDisk', np.float), 

660 ('sedFilenameBulge', str, 300), ('internalAvBulge', np.float), 

661 ('magNormBulge', np.float), 

662 ('sedFilenameAgn', str, 300), ('varParamStr', str, 600), 

663 ('magNormAgn', np.float) 

664 ]) 

665 

666 cls.agn_db = fileDBObject(cls.txt_cat_name, delimiter=';', 

667 runtable='test', dtype=dtype, 

668 idColKey='galid') 

669 

670 cls.agn_db.raColName = 'raDeg' 

671 cls.agn_db.decColName = 'decDeg' 

672 cls.agn_db.objectTypeId = 112 

673 

674 # what follows is a hack to deal with the fact thar 

675 # our varParamStr values are longer than 256 characters 

676 # which is the default maximum length that a 

677 # CatalogDBObject expects a string to be 

678 # 

679 cls.agn_db.dbTypeMap['STRING'] = (str, 600) 

680 cls.agn_db.columns = None 

681 cls.agn_db._make_default_columns() 

682 cls.agn_db._make_column_map() 

683 cls.agn_db._make_type_map() 

684 

685 cls.opsimDb = os.path.join(getPackageDir("sims_data"), "OpSimData") 

686 cls.opsimDb = os.path.join(cls.opsimDb, "opsimblitz1_1133_sqlite.db") 

687 

688 @classmethod 

689 def tearDownClass(cls): 

690 sims_clean_up() 

691 if os.path.exists(cls.txt_cat_name): 

692 os.unlink(cls.txt_cat_name) 

693 

694 def test_agn_light_curves(self): 

695 """ 

696 Test the AgnLightCurveGenerator by generating some AGN light 

697 curves and comparing them to the results obtained by generating a 

698 series of InstanceCatalogs containing the same objects at the same 

699 MJDs 

700 """ 

701 

702 raRange = (78.0, 85.0) 

703 decRange = (-69.0, -65.0) 

704 bandpass = 'g' 

705 

706 lc_gen = AgnLightCurveGenerator(self.agn_db, self.opsimDb) 

707 pointings = lc_gen.get_pointings(raRange, decRange, bandpass=bandpass) 

708 for row in pointings: 

709 for obs in row: 

710 mjd = ModifiedJulianDate(TAI=obs.mjd.TAI-49000.0+59580.0) 

711 obs.mjd = mjd 

712 

713 test_light_curves, truth_info = lc_gen.light_curves_from_pointings(pointings) 

714 

715 self.assertGreater(len(test_light_curves), 2) # make sure we got some light curves 

716 

717 for unique_id in test_light_curves: 

718 # verify that the sources returned all do vary by making sure that the 

719 # np.diff run on the magnitudes reutrns something non-zero 

720 self.assertGreater(np.abs(np.diff(test_light_curves[unique_id][bandpass]['mag'])).max(), 0.0) 

721 self.assertGreater(len(test_light_curves[unique_id][bandpass]['mjd']), 0) 

722 

723 # Now test that specifying a small chunk_size does not change the output 

724 # light curves 

725 chunk_light_curves, truth_info = lc_gen.light_curves_from_pointings(pointings, chunk_size=1) 

726 self.assertGreater(len(chunk_light_curves), 2) 

727 

728 for unique_id in test_light_curves: 

729 self.assertEqual(len(test_light_curves[unique_id][bandpass]['mjd']), 

730 len(chunk_light_curves[unique_id][bandpass]['mjd'])) 

731 np.testing.assert_array_equal(test_light_curves[unique_id][bandpass]['mjd'], 

732 chunk_light_curves[unique_id][bandpass]['mjd']) 

733 np.testing.assert_array_equal(test_light_curves[unique_id][bandpass]['mag'], 

734 chunk_light_curves[unique_id][bandpass]['mag']) 

735 np.testing.assert_array_equal(test_light_curves[unique_id][bandpass]['error'], 

736 chunk_light_curves[unique_id][bandpass]['error']) 

737 

738 # Now find all of the ObservationMetaData that were included in our 

739 # light curves, generate InstanceCatalogs from them separately, 

740 # and verify that the contents of the InstanceCatalogs agree with 

741 # the contents of the light curves. 

742 

743 gen = ObservationMetaDataGenerator(database=self.opsimDb, 

744 driver='sqlite') 

745 

746 obs_list = gen.getObservationMetaData(fieldRA=raRange, 

747 fieldDec=decRange, 

748 telescopeFilter=bandpass, 

749 boundLength=1.75) 

750 for obs in obs_list: 

751 mjd = ModifiedJulianDate(TAI=obs.mjd.TAI-49000.0+59580.0) 

752 obs.mjd = mjd 

753 

754 ct = 0 

755 for obs in obs_list: 

756 cat = agnControlCatalog(self.agn_db, 

757 obs_metadata=obs) 

758 

759 for agn_obj in cat.iter_catalog(): 

760 ct += 1 

761 lc = test_light_curves[agn_obj[0]][bandpass] 

762 dex = np.argmin(np.abs(lc['mjd']-obs.mjd.TAI)) 

763 self.assertLess(np.abs(lc['mjd'][dex]-obs.mjd.TAI), 1.0e-7) 

764 self.assertLess(np.abs(lc['mag'][dex]-agn_obj[3]), 1.0e-7) 

765 self.assertLess(np.abs(lc['error'][dex]-agn_obj[4]), 1.0e-7) 

766 

767 # Verify that the catalogs and LightCurveGenerator returned the 

768 # same number of observations 

769 total_ct = 0 

770 for obj_name in test_light_curves: 

771 for band in test_light_curves[obj_name]: 

772 total_ct += len(test_light_curves[obj_name][band]['mjd']) 

773 self.assertEqual(ct, total_ct) 

774 

775 def test_limited_agn_light_curves(self): 

776 """ 

777 Test that we can select only a limited number of agn light curves 

778 per field of view 

779 """ 

780 

781 lc_limit = 2 

782 raRange = (78.0, 82.0) 

783 decRange = (-69.0, -65.0) 

784 bandpass = 'g' 

785 

786 lc_gen = AgnLightCurveGenerator(self.agn_db, self.opsimDb) 

787 pointings = lc_gen.get_pointings(raRange, decRange, bandpass=bandpass) 

788 for row in pointings: 

789 for obs in row: 

790 mjd = ModifiedJulianDate(TAI=obs.mjd.TAI-49000.0+59580.0) 

791 obs.mjd = mjd 

792 self.assertEqual(len(pointings), 1) 

793 

794 control_lc, truth = lc_gen.light_curves_from_pointings(pointings) 

795 self.assertGreater(len(control_lc), 2) 

796 test_lc, truth = lc_gen.light_curves_from_pointings(pointings, lc_per_field=lc_limit) 

797 self.assertGreater(len(control_lc), len(test_lc)) 

798 self.assertEqual(len(test_lc), lc_limit) 

799 

800 def test_multiband_light_curves(self): 

801 """ 

802 Check that multi-band light curves are returned correctly. 

803 """ 

804 

805 raRange = (78.0, 82.0) 

806 decRange = (-69.0, -65.0) 

807 bandpass = ('r', 'g') 

808 

809 gen = AgnLightCurveGenerator(self.agn_db, self.opsimDb) 

810 pointings = gen.get_pointings(raRange, decRange, bandpass=bandpass) 

811 for row in pointings: 

812 for obs in row: 

813 mjd = ModifiedJulianDate(TAI=obs.mjd.TAI-49000.0+59580.0) 

814 obs.mjd = mjd 

815 

816 lc_dict, truth_info = gen.light_curves_from_pointings(pointings) 

817 self.assertGreater(len(lc_dict), 2) 

818 

819 obs_gen = ObservationMetaDataGenerator(database=self.opsimDb, driver='sqlite') 

820 control_pointings_r = obs_gen.getObservationMetaData(fieldRA=raRange, fieldDec=decRange, 

821 telescopeFilter='r', boundLength=1.75) 

822 

823 for obs in control_pointings_r: 

824 mjd = ModifiedJulianDate(TAI=obs.mjd.TAI-49000.0+59580.0) 

825 obs.mjd = mjd 

826 

827 control_pointings_g = obs_gen.getObservationMetaData(fieldRA=raRange, fieldDec=decRange, 

828 telescopeFilter='g', boundLength=1.75) 

829 for obs in control_pointings_g: 

830 mjd = ModifiedJulianDate(TAI=obs.mjd.TAI-49000.0+59580.0) 

831 obs.mjd = mjd 

832 

833 self.assertGreater(len(control_pointings_g), 0) 

834 self.assertGreater(len(control_pointings_r), 0) 

835 

836 ct = 0 

837 for obs in control_pointings_r: 

838 cat = agnControlCatalog(self.agn_db, 

839 obs_metadata=obs) 

840 

841 for star_obj in cat.iter_catalog(): 

842 ct += 1 

843 lc = lc_dict[star_obj[0]]['r'] 

844 dex = np.argmin(np.abs(lc['mjd']-obs.mjd.TAI)) 

845 self.assertLess(np.abs(lc['mjd'][dex]-obs.mjd.TAI), 1.0e-7) 

846 self.assertLess(np.abs(lc['mag'][dex]-star_obj[3]), 1.0e-7) 

847 self.assertLess(np.abs(lc['error'][dex]-star_obj[4]), 1.0e-7) 

848 

849 for obs in control_pointings_g: 

850 cat = agnControlCatalog(self.agn_db, 

851 obs_metadata=obs) 

852 

853 for star_obj in cat.iter_catalog(): 

854 ct += 1 

855 lc = lc_dict[star_obj[0]]['g'] 

856 dex = np.argmin(np.abs(lc['mjd']-obs.mjd.TAI)) 

857 self.assertLess(np.abs(lc['mjd'][dex]-obs.mjd.TAI), 1.0e-7) 

858 self.assertLess(np.abs(lc['mag'][dex]-star_obj[3]), 1.0e-7) 

859 self.assertLess(np.abs(lc['error'][dex]-star_obj[4]), 1.0e-7) 

860 

861 # Verify that the catalogs and LightCurveGenerator returned the 

862 # same number of observations 

863 total_ct = 0 

864 for obj_name in lc_dict: 

865 for band in lc_dict[obj_name]: 

866 total_ct += len(lc_dict[obj_name][band]['mjd']) 

867 self.assertEqual(ct, total_ct) 

868 

869 def test_agn_constraint(self): 

870 """ 

871 Test that the light curve generator correctly ignores objects 

872 with varParamStr == None 

873 

874 We do this by generating a database of two galaxies with AGN-like 

875 varParamStr and two stars with no varParamStr. We run this database 

876 through a LightCurveGenerator and an InstanceCatalog. We verify that 

877 the LightCurveGenerator finds 2 AGN while the InstanceCatalog finds 

878 4 stars. 

879 """ 

880 

881 rng = np.random.RandomState(83) 

882 

883 raRange = (80.8, 83.8) 

884 decRange = (-71.0, -69.0) 

885 bandpass = 'g' 

886 

887 agn_sed_name = "agn.spec" 

888 sed_name = "Burst.10E10.1Z.spec.gz" 

889 

890 varparams = {'varMethodName': 'applyAgn', 

891 'pars': {'agn_tau': 20.0, 'agn_sfu': 11.0, 

892 'agn_sfg': 12.0, 'agn_sfr': 13.0, 

893 'agn_sfi': 14.0, 'agn_sfz': 15.0, 

894 'agn_sfy': 16.0, 't0_mjd': 49330.0, 

895 'seed': rng.randint(0, 200000)}} 

896 

897 varParamStr = json.dumps(varparams) 

898 

899 # create the dummy database 

900 db_name = tempfile.mktemp(prefix="agn_constraint_cat_sqlite", suffix=".db", dir=ROOT) 

901 conn = sqlite3.connect(db_name) 

902 c = conn.cursor() 

903 c.execute('''CREATE TABLE agn 

904 (id int, ra real, dec real, redshift real, 

905 sedFilenameDisk text, internalAvDisk real, 

906 magNormDisk real, 

907 sedFilenameBulge text, internalAvBulge real, 

908 magNormBulge real, 

909 sedFilenameAgn text, varParamStr text, 

910 magNormAgn real)''') 

911 conn.commit() 

912 

913 for ix, (rr, dd, zz, avb, mnb, avd, mnd, mnagn) in \ 

914 enumerate(zip(rng.random_sample(4)*(raRange[1]-raRange[0])+raRange[0], 

915 rng.random_sample(4)*(decRange[1]-decRange[0])+decRange[0], 

916 rng.random_sample(4)*0.5+0.1, 

917 rng.random_sample(4)*0.5+0.1, rng.random_sample(4)*3.0+17.0, 

918 rng.random_sample(4)*0.5+0.1, rng.random_sample(4)*3.0+17.0, 

919 rng.random_sample(4)*3.0+17.0)): 

920 

921 if ix < 2: 

922 cmd = '''INSERT INTO agn VALUES(%d, %e, %e, %e, '%s', %e, 

923 %e, '%s', %e, %e, '%s', '%s', %e)''' % \ 

924 (ix, rr, dd, zz, sed_name, avb, mnb, sed_name, avd, mnd, 

925 agn_sed_name, varParamStr, mnagn) 

926 else: 

927 cmd = '''INSERT INTO agn VALUES(%d, %e, %e, %e, '%s', %e, 

928 %e, '%s', %e, %e, '%s', NULL, %e)''' % \ 

929 (ix, rr, dd, zz, sed_name, avb, mnb, sed_name, avd, mnd, 

930 agn_sed_name, mnagn) 

931 

932 c.execute(cmd) 

933 

934 conn.commit() 

935 

936 # create a CatalogDBObject class to interface with the dummy database 

937 class dummyAgnDBObject(CatalogDBObject): 

938 database = db_name 

939 host = None 

940 driver = 'sqlite' 

941 tableid = 'agn' 

942 raColName = 'ra' 

943 decColName = 'dec' 

944 idColKey = 'id' 

945 objid = 'dummyAgn' 

946 skipRegistration = True 

947 objectTypeId = 99 

948 

949 columns = [('raJ2000', 'ra*PI()/180.0', float), 

950 ('decJ2000', 'dec*PI()/180.0', float)] 

951 

952 agn_db = dummyAgnDBObject() 

953 

954 # verify that the LightCurveGenerator finds the two variable stars 

955 lc_gen = AgnLightCurveGenerator(agn_db, self.opsimDb) 

956 ptngs = lc_gen.get_pointings(raRange, decRange, bandpass=bandpass) 

957 for row in ptngs: 

958 for obs in row: 

959 mjd = ModifiedJulianDate(TAI=obs.mjd.TAI-49000.0+59580.0) 

960 obs.mjd = mjd 

961 

962 lc_dict, truth_dict = lc_gen.light_curves_from_pointings(ptngs) 

963 self.assertEqual(len(lc_dict), 2) 

964 

965 if os.path.exists(db_name): 

966 os.unlink(db_name) 

967 

968 # verify that an InstanceCatalog finds all 4 stars 

969 obs = ptngs[0][0] 

970 cat = agnControlCatalog(agn_db, obs_metadata=obs) 

971 with lsst.utils.tests.getTempFilePath('.txt') as dummy_cat_name: 

972 cat.write_catalog(dummy_cat_name) 

973 with open(dummy_cat_name, 'r') as input_file: 

974 lines = input_file.readlines() 

975 self.assertEqual(len(lines), 5) 

976 

977 

978class MemoryTestClass(lsst.utils.tests.MemoryTestCase): 

979 pass 

980 

981 

982if __name__ == "__main__": 982 ↛ 983line 982 didn't jump to line 983, because the condition on line 982 was never true

983 lsst.utils.tests.init() 

984 unittest.main()