Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import unittest 

2import os 

3import tempfile 

4import shutil 

5import gc 

6import numpy as np 

7import sqlite3 

8import numbers 

9import lsst.utils.tests 

10 

11from lsst.utils import getPackageDir 

12import lsst.obs.lsst.phosim as obs_lsst_phosim 

13from lsst.sims.utils.CodeUtilities import sims_clean_up 

14from lsst.sims.catalogs.decorators import register_method 

15from lsst.sims.catalogs.db import CatalogDBObject 

16from lsst.sims.catUtils.utils import ObservationMetaDataGenerator 

17from lsst.sims.catUtils.utils import AlertStellarVariabilityCatalog 

18from lsst.sims.catUtils.utils import StellarAlertDBObjMixin 

19from lsst.sims.utils import findHtmid 

20from lsst.sims.utils import applyProperMotion, ModifiedJulianDate 

21from lsst.sims.photUtils import Sed, calcSNR_m5, BandpassDict 

22from lsst.sims.photUtils import PhotometricParameters 

23from lsst.sims.catUtils.mixins import CameraCoords 

24from lsst.sims.catUtils.mixins import AstrometryStars 

25from lsst.sims.catUtils.mixins import Variability 

26from lsst.sims.catalogs.definitions import InstanceCatalog 

27from lsst.sims.catalogs.decorators import compound, cached 

28 

29from lsst.sims.catUtils.utils import AlertDataGenerator 

30from lsst.sims.catUtils.utils import AvroAlertGenerator 

31 

32 

33_avro_is_installed = True 

34try: 

35 from avro.io import DatumReader 

36 from avro.datafile import DataFileReader 

37except ImportError: 

38 _avro_is_installed = False 

39 pass 

40 

41 

42ROOT = os.path.abspath(os.path.dirname(__file__)) 

43 

44 

45def setup_module(module): 

46 lsst.utils.tests.init() 

47 

48 

49class StarAlertTestDBObj_avro(StellarAlertDBObjMixin, CatalogDBObject): 

50 objid = 'star_alert' 

51 tableid = 'stars' 

52 idColKey = 'simobjid' 

53 raColName = 'ra' 

54 decColName = 'dec' 

55 objectTypeId = 0 

56 columns = [('raJ2000', 'ra*0.01745329252'), 

57 ('decJ2000', 'dec*0.01745329252'), 

58 ('parallax', 'px*0.01745329252/3600.0'), 

59 ('properMotionRa', 'pmra*0.01745329252/3600.0'), 

60 ('properMotionDec', 'pmdec*0.01745329252/3600.0'), 

61 ('radialVelocity', 'vrad'), 

62 ('variabilityParameters', 'varParamStr', str, 500)] 

63 

64 

65class TestAlertsVarCatMixin_avro(object): 

66 

67 @register_method('avro_test') 

68 def applyAlertTest(self, valid_dexes, params, expmjd, variability_cache=None): 

69 if len(params) == 0: 

70 return np.array([[], [], [], [], [], []]) 

71 

72 if isinstance(expmjd, numbers.Number): 

73 dMags_out = np.zeros((6, self.num_variable_obj(params))) 

74 else: 

75 dMags_out = np.zeros((6, self.num_variable_obj(params), len(expmjd))) 

76 

77 for i_star in range(self.num_variable_obj(params)): 

78 if params['amp'][i_star] is not None: 

79 dmags = params['amp'][i_star]*np.cos(params['per'][i_star]*expmjd) 

80 for i_filter in range(6): 

81 dMags_out[i_filter][i_star] = dmags 

82 

83 return dMags_out 

84 

85 

86class TestAlertsVarCat_avro(TestAlertsVarCatMixin_avro, AlertStellarVariabilityCatalog): 

87 pass 

88 

89 

90class TestAlertsTruthCat_avro(TestAlertsVarCatMixin_avro, CameraCoords, AstrometryStars, 

91 Variability, InstanceCatalog): 

92 column_outputs = ['uniqueId', 'chipName', 'dmagAlert', 'magAlert', 

93 'raICRS', 'decICRS', 'xPix', 'yPix'] 

94 

95 camera = obs_lsst_phosim.PhosimMapper().camera 

96 

97 @compound('delta_umag', 'delta_gmag', 'delta_rmag', 

98 'delta_imag', 'delta_zmag', 'delta_ymag') 

99 def get_TruthVariability(self): 

100 return self.applyVariability(self.column_by_name('varParamStr')) 

101 

102 @cached 

103 def get_dmagAlert(self): 

104 return self.column_by_name('delta_%smag' % self.obs_metadata.bandpass) 

105 

106 @cached 

107 def get_magAlert(self): 

108 return self.column_by_name('%smag' % self.obs_metadata.bandpass) + \ 

109 self.column_by_name('dmagAlert') 

110 

111 

112@unittest.skipIf(not _avro_is_installed, 'avro is not installed on this system') 

113class AvroAlertTestCase(unittest.TestCase): 

114 

115 longMessage = True 

116 

117 @classmethod 

118 def setUpClass(cls): 

119 print('setting up %s' % sims_clean_up.targets) 

120 

121 # These represent the dimmest magnitudes at which objects 

122 # are considered visible in each of the LSST filters 

123 # (taken from Table 2 of the overview paper) 

124 cls.obs_mag_cutoff = (23.68, 24.89, 24.43, 24.0, 24.45, 22.60) 

125 

126 cls.opsim_db = os.path.join(getPackageDir('sims_data'), 

127 'OpSimData', 

128 'opsimblitz1_1133_sqlite.db') 

129 

130 rng = np.random.RandomState(8123) 

131 

132 obs_gen = ObservationMetaDataGenerator(database=cls.opsim_db) 

133 cls.obs_list = obs_gen.getObservationMetaData(night=(0, 2)) 

134 cls.obs_list = rng.choice(cls.obs_list, 10, replace=False) 

135 fieldid_list = [] 

136 for obs in cls.obs_list: 

137 fieldid_list.append(obs.OpsimMetaData['fieldID']) 

138 

139 # make sure we have selected observations such that the 

140 # same field is revisited more than once 

141 assert len(np.unique(fieldid_list)) < len(fieldid_list) 

142 

143 cls.input_dir = tempfile.mkdtemp(prefix='avroAlertGen', 

144 dir=ROOT) 

145 

146 cls.star_db_name = tempfile.mktemp(prefix='avroAlertGen_star_db', 

147 dir=cls.input_dir, 

148 suffix='.db') 

149 

150 conn = sqlite3.connect(cls.star_db_name) 

151 cursor = conn.cursor() 

152 cursor.execute('''CREATE TABLE stars 

153 (simobjid int, htmid int, ra real, dec real, 

154 umag real, gmag real, rmag real, 

155 imag real, zmag real, ymag real, 

156 px real, pmra real, pmdec real, 

157 vrad real, varParamStr text)''') 

158 conn.commit() 

159 

160 n_stars = 10 

161 

162 cls.ra_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

163 cls.dec_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

164 u_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

165 g_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

166 r_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

167 i_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

168 z_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

169 y_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

170 cls.px_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

171 cls.pmra_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

172 cls.pmdec_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

173 cls.vrad_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

174 cls.amp_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

175 cls.period_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

176 

177 id_offset = -n_stars 

178 for obs in cls.obs_list: 

179 id_offset += n_stars 

180 ra_0 = obs.pointingRA 

181 dec_0 = obs.pointingDec 

182 rr = rng.random_sample(n_stars) 

183 theta = rng.random_sample(n_stars)*2.0*np.pi 

184 ra = ra_0 + rr*np.cos(theta) 

185 dec = dec_0 + rr*np.sin(theta) 

186 var_period = rng.random_sample(n_stars)*0.25 

187 var_amp = rng.random_sample(n_stars)*1.0 + 0.01 

188 

189 subset = rng.randint(0, high=len(var_amp)-1, size=3) 

190 var_amp[subset[:2]] = 0.0 

191 var_amp[subset[-1]] = -1.0 

192 

193 umag = rng.random_sample(n_stars)*5.0 + 15.0 

194 gmag = rng.random_sample(n_stars)*5.0 + 15.0 

195 rmag = rng.random_sample(n_stars)*5.0 + 15.0 

196 imag = rng.random_sample(n_stars)*5.0 + 15.0 

197 zmag = rng.random_sample(n_stars)*5.0 + 15.0 

198 ymag = rng.random_sample(n_stars)*5.0 + 15.0 

199 px = rng.random_sample(n_stars)*0.1 # say it is arcsec 

200 pmra = rng.random_sample(n_stars)*50.0+100.0 # say it is arcsec/yr 

201 pmdec = rng.random_sample(n_stars)*50.0+100.0 # say it is arcsec/yr 

202 vrad = rng.random_sample(n_stars)*600.0 - 300.0 

203 

204 subset = rng.randint(0, high=n_stars-1, size=3) 

205 umag[subset] = 40.0 

206 gmag[subset] = 40.0 

207 rmag[subset] = 40.0 

208 imag[subset] = 40.0 

209 zmag[subset] = 40.0 

210 ymag[subset] = 40.0 

211 

212 cls.ra_truth[id_offset:id_offset+n_stars] = np.round(ra, decimals=6) 

213 cls.dec_truth[id_offset:id_offset+n_stars] = np.round(dec, decimals=6) 

214 u_truth[id_offset:id_offset+n_stars] = np.round(umag, decimals=4) 

215 g_truth[id_offset:id_offset+n_stars] = np.round(gmag, decimals=4) 

216 r_truth[id_offset:id_offset+n_stars] = np.round(rmag, decimals=4) 

217 i_truth[id_offset:id_offset+n_stars] = np.round(imag, decimals=4) 

218 z_truth[id_offset:id_offset+n_stars] = np.round(zmag, decimals=4) 

219 y_truth[id_offset:id_offset+n_stars] = np.round(ymag, decimals=4) 

220 cls.px_truth[id_offset:id_offset+n_stars] = np.round(px, decimals=4) 

221 cls.pmra_truth[id_offset:id_offset+n_stars] = np.round(pmra, decimals=4) 

222 cls.pmdec_truth[id_offset:id_offset+n_stars] = np.round(pmdec, decimals=4) 

223 cls.vrad_truth[id_offset:id_offset+n_stars] = np.round(vrad, decimals=4) 

224 cls.amp_truth[id_offset:id_offset+n_stars] = np.round(var_amp, decimals=4) 

225 cls.period_truth[id_offset:id_offset+n_stars] = np.round(var_period, decimals=4) 

226 

227 max_str_len = -1 

228 

229 for i_star in range(n_stars): 

230 if var_amp[i_star] >= -0.1: 

231 varParamStr = ('{"m":"avro_test", "p":{"amp":%.4f, "per": %.4f}}' 

232 % (var_amp[i_star], var_period[i_star])) 

233 else: 

234 varParamStr = 'None' 

235 

236 if len(varParamStr) > max_str_len: 

237 max_str_len = len(varParamStr) 

238 

239 htmid = findHtmid(ra[i_star], dec[i_star], 21) 

240 

241 query = ('''INSERT INTO stars VALUES(%d, %d, %.6f, %.6f, 

242 %.4f, %.4f, %.4f, %.4f, %.4f, %.4f, 

243 %.4f, %.4f, %.4f, %.4f, '%s')''' 

244 % (i_star+id_offset+1, htmid, ra[i_star], dec[i_star], 

245 umag[i_star], gmag[i_star], rmag[i_star], 

246 imag[i_star], zmag[i_star], ymag[i_star], 

247 px[i_star], pmra[i_star], pmdec[i_star], 

248 vrad[i_star], varParamStr)) 

249 

250 cursor.execute(query) 

251 conn.commit() 

252 conn.close() 

253 

254 cls.mag0_truth_dict = {} 

255 cls.mag0_truth_dict[0] = u_truth 

256 cls.mag0_truth_dict[1] = g_truth 

257 cls.mag0_truth_dict[2] = r_truth 

258 cls.mag0_truth_dict[3] = i_truth 

259 cls.mag0_truth_dict[4] = z_truth 

260 cls.mag0_truth_dict[5] = y_truth 

261 assert max_str_len < 500 # make sure varParamStr fits in the space alotted to it 

262 # in StarAlertTestDBObj_avro 

263 

264 @classmethod 

265 def tearDownClass(cls): 

266 sims_clean_up() 

267 if os.path.exists(cls.star_db_name): 

268 os.unlink(cls.star_db_name) 

269 if os.path.exists(cls.input_dir): 

270 shutil.rmtree(cls.input_dir) 

271 

272 def setUp(self): 

273 self.alert_data_output_dir = tempfile.mkdtemp(dir=ROOT, prefix='avro_gen_output') 

274 self.avro_out_dir = tempfile.mkdtemp(dir=ROOT, prefix='avroTestOut') 

275 

276 def tearDown(self): 

277 for file_name in os.listdir(self.alert_data_output_dir): 

278 os.unlink(os.path.join(self.alert_data_output_dir, file_name)) 

279 shutil.rmtree(self.alert_data_output_dir) 

280 

281 for file_name in os.listdir(self.avro_out_dir): 

282 os.unlink(os.path.join(self.avro_out_dir, file_name)) 

283 shutil.rmtree(self.avro_out_dir) 

284 

285 def test_avro_alert_generation(self): 

286 dmag_cutoff = 0.005 

287 mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5} 

288 

289 star_db = StarAlertTestDBObj_avro(database=self.star_db_name, driver='sqlite') 

290 

291 # assemble a dict of all of the alerts that need to be generated 

292 

293 obshistid_list = [] 

294 for obs in self.obs_list: 

295 obshistid_list.append(obs.OpsimMetaData['obsHistID']) 

296 obshistid_max = max(obshistid_list) 

297 obshistid_bits = int(np.ceil(np.log(obshistid_max)/np.log(2.0))) 

298 

299 true_alert_dict = {} 

300 obs_dict = {} 

301 for obs in self.obs_list: 

302 obs_dict[obs.OpsimMetaData['obsHistID']] = obs 

303 obshistid = obs.OpsimMetaData['obsHistID'] 

304 cat = TestAlertsTruthCat_avro(star_db, obs_metadata=obs) 

305 cat.camera = obs_lsst_phosim.PhosimMapper().camera 

306 

307 for line in cat.iter_catalog(): 

308 if line[1] is None: 

309 continue 

310 

311 dmag = line[2] 

312 mag = line[3] 

313 if (np.abs(dmag) > dmag_cutoff and 

314 mag <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]): 

315 

316 alertId = (line[0] << obshistid_bits) + obshistid 

317 self.assertNotIn(alertId, true_alert_dict) 

318 true_alert_dict[alertId] = {} 

319 true_alert_dict[alertId]['chipName'] = line[1] 

320 true_alert_dict[alertId]['dmag'] = dmag 

321 true_alert_dict[alertId]['mag'] = mag 

322 true_alert_dict[alertId]['ra'] = np.degrees(line[4]) 

323 true_alert_dict[alertId]['decl'] = np.degrees(line[5]) 

324 true_alert_dict[alertId]['xPix'] = line[6] 

325 true_alert_dict[alertId]['yPix'] = line[7] 

326 

327 self.assertGreater(len(true_alert_dict), 10) 

328 

329 log_file_name = tempfile.mktemp(dir=self.alert_data_output_dir, suffix='log.txt') 

330 alert_gen = AlertDataGenerator(testing=True) 

331 

332 alert_gen.subdivide_obs(self.obs_list, htmid_level=6) 

333 

334 for htmid in alert_gen.htmid_list: 

335 alert_gen.alert_data_from_htmid(htmid, star_db, 

336 photometry_class=TestAlertsVarCat_avro, 

337 output_prefix='alert_test', 

338 output_dir=self.alert_data_output_dir, 

339 dmag_cutoff=dmag_cutoff, 

340 log_file_name=log_file_name) 

341 

342 obshistid_to_htmid = {} 

343 for htmid in alert_gen.htmid_list: 

344 for obs in alert_gen.obs_from_htmid(htmid): 

345 obshistid = obs.OpsimMetaData['obsHistID'] 

346 if obshistid not in obshistid_to_htmid: 

347 obshistid_to_htmid[obshistid] = [] 

348 obshistid_to_htmid[obshistid].append(htmid) 

349 

350 avro_gen = AvroAlertGenerator() 

351 avro_gen.load_schema(os.path.join(getPackageDir('sims_catUtils'), 'tests', 'testData', 'avroSchema')) 

352 sql_prefix_list = ['alert_test'] 

353 out_prefix = 'test_avro' 

354 log_file_name = tempfile.mktemp(dir=self.avro_out_dir, 

355 prefix='test_avro', 

356 suffix='log.txt') 

357 for obshistid in obshistid_list: 

358 avro_gen.write_alerts(obshistid, self.alert_data_output_dir, 

359 sql_prefix_list, 

360 obshistid_to_htmid[obshistid], 

361 self.avro_out_dir, out_prefix, 

362 dmag_cutoff, lock=None, 

363 log_file_name=log_file_name) 

364 

365 list_of_avro_files = os.listdir(self.avro_out_dir) 

366 self.assertGreater(len(list_of_avro_files), 2) 

367 alert_ct = 0 

368 dummy_sed = Sed() 

369 bp_dict = BandpassDict.loadTotalBandpassesFromFiles() 

370 photParams = PhotometricParameters() 

371 diasourceId_set = set() 

372 for avro_file_name in list_of_avro_files: 

373 if avro_file_name.endswith('log.txt'): 

374 continue 

375 full_name = os.path.join(self.avro_out_dir, avro_file_name) 

376 with DataFileReader(open(full_name, 'rb'), DatumReader()) as data_reader: 

377 for alert in data_reader: 

378 alert_ct += 1 

379 obshistid = alert['alertId'] >> 20 

380 obs = obs_dict[obshistid] 

381 uniqueId = alert['diaObject']['diaObjectId'] 

382 true_alert_id = (uniqueId << obshistid_bits) + obshistid 

383 self.assertIn(true_alert_id, true_alert_dict) 

384 self.assertEqual(alert['l1dbId'], uniqueId) 

385 

386 true_alert = true_alert_dict[true_alert_id] 

387 

388 diaSource = alert['diaSource'] 

389 self.assertAlmostEqual(diaSource['ra'], true_alert['ra'], 10) 

390 self.assertAlmostEqual(diaSource['decl'], true_alert['decl'], 10) 

391 self.assertAlmostEqual(diaSource['x'], true_alert['xPix'], 3) 

392 self.assertAlmostEqual(diaSource['y'], true_alert['yPix'], 3) 

393 self.assertAlmostEqual(diaSource['midPointTai'], obs.mjd.TAI, 4) 

394 

395 true_tot_flux = dummy_sed.fluxFromMag(true_alert['mag']) 

396 true_q_mag = true_alert['mag'] - true_alert['dmag'] 

397 true_q_flux = dummy_sed.fluxFromMag(true_q_mag) 

398 true_dflux = true_tot_flux - true_q_flux 

399 self.assertAlmostEqual(diaSource['psFlux']/true_dflux, 1.0, 6) 

400 self.assertAlmostEqual(diaSource['totFlux']/true_tot_flux, 1.0, 6) 

401 self.assertAlmostEqual(diaSource['diffFlux']/true_dflux, 1.0, 6) 

402 

403 true_tot_snr, gamma = calcSNR_m5(true_alert['mag'], bp_dict[obs.bandpass], 

404 obs.m5[obs.bandpass], photParams) 

405 

406 true_q_snr, gamma = calcSNR_m5(true_q_mag, bp_dict[obs.bandpass], 

407 self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]], 

408 photParams) 

409 

410 true_tot_err = true_tot_flux/true_tot_snr 

411 true_q_err = true_q_flux/true_q_snr 

412 true_diff_err = np.sqrt(true_tot_err**2 + true_q_err**2) 

413 

414 self.assertAlmostEqual(diaSource['snr']/np.abs(true_dflux/true_diff_err), 

415 1.0, 6) 

416 

417 self.assertAlmostEqual(diaSource['totFluxErr']/true_tot_err, 1.0, 6) 

418 self.assertAlmostEqual(diaSource['diffFluxErr']/true_diff_err, 1.0, 6) 

419 

420 chipnum = int(true_alert['chipName'].replace('R', '').replace('S', ''). 

421 replace(',', '').replace(':', '').replace(' ', '')) 

422 

423 true_ccdid = (chipnum*10**7)+obshistid 

424 self.assertEqual(true_ccdid, diaSource['ccdVisitId']) 

425 self.assertEqual(uniqueId, diaSource['diaObjectId']) 

426 

427 self.assertNotIn(diaSource['diaSourceId'], diasourceId_set) 

428 diasourceId_set.add(diaSource['diaSourceId']) 

429 

430 diaObject = alert['diaObject'] 

431 obj_dex = (uniqueId//1024) - 1 

432 self.assertAlmostEqual(0.001*diaObject['pmRa']/self.pmra_truth[obj_dex], 1.0, 5) 

433 self.assertAlmostEqual(0.001*diaObject['pmDecl']/self.pmdec_truth[obj_dex], 1.0, 5) 

434 self.assertAlmostEqual(0.001*diaObject['parallax']/self.px_truth[obj_dex], 1.0, 5) 

435 

436 (true_ra_base, 

437 true_dec_base) = applyProperMotion(self.ra_truth[obj_dex], 

438 self.dec_truth[obj_dex], 

439 self.pmra_truth[obj_dex], 

440 self.pmdec_truth[obj_dex], 

441 self.px_truth[obj_dex], 

442 self.vrad_truth[obj_dex], 

443 mjd=ModifiedJulianDate(TAI=diaObject['radecTai'])) 

444 

445 self.assertAlmostEqual(true_ra_base, diaObject['ra'], 7) 

446 self.assertAlmostEqual(true_dec_base, diaObject['decl'], 7) 

447 

448 self.assertEqual(alert_ct, len(true_alert_dict)) 

449 

450 def test_avro_alert_generation_diff_dmag(self): 

451 """ 

452 Make sure everything works properly when the AlertDataGenerator 

453 and the AvroAlertGenerator have different dmag thresholds 

454 """ 

455 dmag_cutoff_sqlite = 0.005 

456 dmag_cutoff_avro = 0.2 

457 mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5} 

458 

459 star_db = StarAlertTestDBObj_avro(database=self.star_db_name, driver='sqlite') 

460 

461 # assemble a dict of all of the alerts that need to be generated 

462 

463 obshistid_list = [] 

464 for obs in self.obs_list: 

465 obshistid_list.append(obs.OpsimMetaData['obsHistID']) 

466 obshistid_max = max(obshistid_list) 

467 obshistid_bits = int(np.ceil(np.log(obshistid_max)/np.log(2.0))) 

468 

469 true_alert_dict = {} 

470 obs_dict = {} 

471 ignored_sqlite = 0 # count number of alerts written to sqlite, but not avro 

472 for obs in self.obs_list: 

473 obs_dict[obs.OpsimMetaData['obsHistID']] = obs 

474 obshistid = obs.OpsimMetaData['obsHistID'] 

475 cat = TestAlertsTruthCat_avro(star_db, obs_metadata=obs) 

476 cat.camera = obs_lsst_phosim.PhosimMapper().camera 

477 

478 for line in cat.iter_catalog(): 

479 if line[1] is None: 

480 continue 

481 

482 dmag = line[2] 

483 mag = line[3] 

484 if (np.abs(dmag) > dmag_cutoff_avro and 

485 mag <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]): 

486 

487 alertId = (line[0] << obshistid_bits) + obshistid 

488 self.assertNotIn(alertId, true_alert_dict) 

489 true_alert_dict[alertId] = {} 

490 true_alert_dict[alertId]['chipName'] = line[1] 

491 true_alert_dict[alertId]['dmag'] = dmag 

492 true_alert_dict[alertId]['mag'] = mag 

493 true_alert_dict[alertId]['ra'] = np.degrees(line[4]) 

494 true_alert_dict[alertId]['decl'] = np.degrees(line[5]) 

495 true_alert_dict[alertId]['xPix'] = line[6] 

496 true_alert_dict[alertId]['yPix'] = line[7] 

497 elif np.abs(dmag) > dmag_cutoff_sqlite: 

498 ignored_sqlite += 1 

499 

500 self.assertGreater(len(true_alert_dict), 10) 

501 

502 self.assertGreater(ignored_sqlite, 50) # just make sure that some sqlite 

503 # alerts were ignored by the more 

504 # stringent avro cut 

505 

506 log_file_name = tempfile.mktemp(dir=self.alert_data_output_dir, suffix='log.txt') 

507 alert_gen = AlertDataGenerator(testing=True) 

508 

509 alert_gen.subdivide_obs(self.obs_list, htmid_level=6) 

510 

511 for htmid in alert_gen.htmid_list: 

512 alert_gen.alert_data_from_htmid(htmid, star_db, 

513 photometry_class=TestAlertsVarCat_avro, 

514 output_prefix='alert_test', 

515 output_dir=self.alert_data_output_dir, 

516 dmag_cutoff=dmag_cutoff_sqlite, 

517 log_file_name=log_file_name) 

518 

519 obshistid_to_htmid = {} 

520 for htmid in alert_gen.htmid_list: 

521 for obs in alert_gen.obs_from_htmid(htmid): 

522 obshistid = obs.OpsimMetaData['obsHistID'] 

523 if obshistid not in obshistid_to_htmid: 

524 obshistid_to_htmid[obshistid] = [] 

525 obshistid_to_htmid[obshistid].append(htmid) 

526 

527 avro_gen = AvroAlertGenerator() 

528 avro_gen.load_schema(os.path.join(getPackageDir('sims_catUtils'), 'tests', 'testData', 'avroSchema')) 

529 sql_prefix_list = ['alert_test'] 

530 out_prefix = 'test_avro' 

531 log_file_name = tempfile.mktemp(dir=self.avro_out_dir, 

532 prefix='test_avro', 

533 suffix='log.txt') 

534 for obshistid in obshistid_list: 

535 avro_gen.write_alerts(obshistid, self.alert_data_output_dir, 

536 sql_prefix_list, 

537 obshistid_to_htmid[obshistid], 

538 self.avro_out_dir, out_prefix, 

539 dmag_cutoff_avro, lock=None, 

540 log_file_name=log_file_name) 

541 

542 list_of_avro_files = os.listdir(self.avro_out_dir) 

543 self.assertGreater(len(list_of_avro_files), 2) 

544 alert_ct = 0 

545 dummy_sed = Sed() 

546 bp_dict = BandpassDict.loadTotalBandpassesFromFiles() 

547 photParams = PhotometricParameters() 

548 diasourceId_set = set() 

549 for avro_file_name in list_of_avro_files: 

550 if avro_file_name.endswith('log.txt'): 

551 continue 

552 full_name = os.path.join(self.avro_out_dir, avro_file_name) 

553 with DataFileReader(open(full_name, 'rb'), DatumReader()) as data_reader: 

554 for alert in data_reader: 

555 alert_ct += 1 

556 obshistid = alert['alertId'] >> 20 

557 obs = obs_dict[obshistid] 

558 uniqueId = alert['diaObject']['diaObjectId'] 

559 true_alert_id = (uniqueId << obshistid_bits) + obshistid 

560 self.assertIn(true_alert_id, true_alert_dict) 

561 self.assertEqual(alert['l1dbId'], uniqueId) 

562 

563 true_alert = true_alert_dict[true_alert_id] 

564 

565 diaSource = alert['diaSource'] 

566 self.assertAlmostEqual(diaSource['ra'], true_alert['ra'], 10) 

567 self.assertAlmostEqual(diaSource['decl'], true_alert['decl'], 10) 

568 self.assertAlmostEqual(diaSource['x'], true_alert['xPix'], 3) 

569 self.assertAlmostEqual(diaSource['y'], true_alert['yPix'], 3) 

570 self.assertAlmostEqual(diaSource['midPointTai'], obs.mjd.TAI, 4) 

571 

572 true_tot_flux = dummy_sed.fluxFromMag(true_alert['mag']) 

573 true_q_mag = true_alert['mag'] - true_alert['dmag'] 

574 true_q_flux = dummy_sed.fluxFromMag(true_q_mag) 

575 true_dflux = true_tot_flux - true_q_flux 

576 self.assertAlmostEqual(diaSource['psFlux']/true_dflux, 1.0, 6) 

577 self.assertAlmostEqual(diaSource['totFlux']/true_tot_flux, 1.0, 6) 

578 self.assertAlmostEqual(diaSource['diffFlux']/true_dflux, 1.0, 6) 

579 

580 true_tot_snr, gamma = calcSNR_m5(true_alert['mag'], bp_dict[obs.bandpass], 

581 obs.m5[obs.bandpass], photParams) 

582 

583 true_q_snr, gamma = calcSNR_m5(true_q_mag, bp_dict[obs.bandpass], 

584 self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]], 

585 photParams) 

586 

587 true_tot_err = true_tot_flux/true_tot_snr 

588 true_q_err = true_q_flux/true_q_snr 

589 true_diff_err = np.sqrt(true_tot_err**2 + true_q_err**2) 

590 

591 self.assertAlmostEqual(diaSource['snr']/np.abs(true_dflux/true_diff_err), 

592 1.0, 6) 

593 

594 self.assertAlmostEqual(diaSource['totFluxErr']/true_tot_err, 1.0, 6) 

595 self.assertAlmostEqual(diaSource['diffFluxErr']/true_diff_err, 1.0, 6) 

596 

597 chipnum = int(true_alert['chipName'].replace('R', '').replace('S', ''). 

598 replace(',', '').replace(':', '').replace(' ', '')) 

599 

600 true_ccdid = (chipnum*10**7)+obshistid 

601 self.assertEqual(true_ccdid, diaSource['ccdVisitId']) 

602 self.assertEqual(uniqueId, diaSource['diaObjectId']) 

603 

604 self.assertNotIn(diaSource['diaSourceId'], diasourceId_set) 

605 diasourceId_set.add(diaSource['diaSourceId']) 

606 

607 diaObject = alert['diaObject'] 

608 obj_dex = (uniqueId//1024) - 1 

609 self.assertAlmostEqual(0.001*diaObject['pmRa']/self.pmra_truth[obj_dex], 1.0, 5) 

610 self.assertAlmostEqual(0.001*diaObject['pmDecl']/self.pmdec_truth[obj_dex], 1.0, 5) 

611 self.assertAlmostEqual(0.001*diaObject['parallax']/self.px_truth[obj_dex], 1.0, 5) 

612 

613 (true_ra_base, 

614 true_dec_base) = applyProperMotion(self.ra_truth[obj_dex], 

615 self.dec_truth[obj_dex], 

616 self.pmra_truth[obj_dex], 

617 self.pmdec_truth[obj_dex], 

618 self.px_truth[obj_dex], 

619 self.vrad_truth[obj_dex], 

620 mjd=ModifiedJulianDate(TAI=diaObject['radecTai'])) 

621 

622 self.assertAlmostEqual(true_ra_base, diaObject['ra'], 7) 

623 self.assertAlmostEqual(true_dec_base, diaObject['decl'], 7) 

624 

625 self.assertEqual(alert_ct, len(true_alert_dict)) 

626 

627 

628class MemoryTestClass(lsst.utils.tests.MemoryTestCase): 

629 pass 

630 

631 

632if __name__ == "__main__": 632 ↛ 633line 632 didn't jump to line 633, because the condition on line 632 was never true

633 lsst.utils.tests.init() 

634 unittest.main()