Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import unittest 

2import os 

3import tempfile 

4import shutil 

5import gc 

6import numpy as np 

7import sqlite3 

8import numbers 

9import lsst.utils.tests 

10 

11from lsst.utils import getPackageDir 

12from lsst.sims.utils.CodeUtilities import sims_clean_up 

13from lsst.sims.catalogs.decorators import register_method 

14from lsst.sims.catalogs.db import CatalogDBObject 

15from lsst.sims.catUtils.utils import ObservationMetaDataGenerator 

16from lsst.sims.catUtils.utils import AlertStellarVariabilityCatalog 

17from lsst.sims.catUtils.utils import StellarAlertDBObjMixin 

18from lsst.sims.utils import findHtmid 

19from lsst.sims.utils import applyProperMotion, ModifiedJulianDate 

20from lsst.sims.photUtils import Sed, calcSNR_m5, BandpassDict 

21from lsst.sims.photUtils import PhotometricParameters 

22from lsst.sims.coordUtils import lsst_camera 

23from lsst.sims.coordUtils import chipNameFromPupilCoordsLSST 

24from lsst.sims.catUtils.mixins import CameraCoordsLSST 

25from lsst.sims.catUtils.mixins import AstrometryStars 

26from lsst.sims.catUtils.mixins import Variability 

27from lsst.sims.catalogs.definitions import InstanceCatalog 

28from lsst.sims.catalogs.decorators import compound, cached 

29 

30from lsst.sims.catUtils.utils import AlertDataGenerator 

31from lsst.sims.catUtils.utils import AvroAlertGenerator 

32 

33from lsst.sims.coordUtils import clean_up_lsst_camera 

34 

35_avro_is_installed = True 

36try: 

37 from avro.io import DatumReader 

38 from avro.datafile import DataFileReader 

39except ImportError: 

40 _avro_is_installed = False 

41 pass 

42 

43 

44ROOT = os.path.abspath(os.path.dirname(__file__)) 

45 

46 

47def setup_module(module): 

48 lsst.utils.tests.init() 

49 

50 

51class StarAlertTestDBObj_avro(StellarAlertDBObjMixin, CatalogDBObject): 

52 objid = 'star_alert' 

53 tableid = 'stars' 

54 idColKey = 'simobjid' 

55 raColName = 'ra' 

56 decColName = 'dec' 

57 objectTypeId = 0 

58 columns = [('raJ2000', 'ra*0.01745329252'), 

59 ('decJ2000', 'dec*0.01745329252'), 

60 ('parallax', 'px*0.01745329252/3600.0'), 

61 ('properMotionRa', 'pmra*0.01745329252/3600.0'), 

62 ('properMotionDec', 'pmdec*0.01745329252/3600.0'), 

63 ('radialVelocity', 'vrad'), 

64 ('variabilityParameters', 'varParamStr', str, 500)] 

65 

66 

67class TestAlertsVarCatMixin_avro(object): 

68 

69 @register_method('avro_test') 

70 def applyAlertTest(self, valid_dexes, params, expmjd, variability_cache=None): 

71 if len(params) == 0: 

72 return np.array([[], [], [], [], [], []]) 

73 

74 if isinstance(expmjd, numbers.Number): 

75 dMags_out = np.zeros((6, self.num_variable_obj(params))) 

76 else: 

77 dMags_out = np.zeros((6, self.num_variable_obj(params), len(expmjd))) 

78 

79 for i_star in range(self.num_variable_obj(params)): 

80 if params['amp'][i_star] is not None: 

81 dmags = params['amp'][i_star]*np.cos(params['per'][i_star]*expmjd) 

82 for i_filter in range(6): 

83 dMags_out[i_filter][i_star] = dmags 

84 

85 return dMags_out 

86 

87 

88class TestAlertsVarCat_avro(TestAlertsVarCatMixin_avro, AlertStellarVariabilityCatalog): 

89 pass 

90 

91 

92class TestAlertsTruthCat_avro(TestAlertsVarCatMixin_avro, CameraCoordsLSST, AstrometryStars, 

93 Variability, InstanceCatalog): 

94 column_outputs = ['uniqueId', 'chipName', 'dmagAlert', 'magAlert', 

95 'raICRS', 'decICRS', 'xPix', 'yPix'] 

96 

97 @compound('delta_umag', 'delta_gmag', 'delta_rmag', 

98 'delta_imag', 'delta_zmag', 'delta_ymag') 

99 def get_TruthVariability(self): 

100 return self.applyVariability(self.column_by_name('varParamStr')) 

101 

102 @cached 

103 def get_dmagAlert(self): 

104 return self.column_by_name('delta_%smag' % self.obs_metadata.bandpass) 

105 

106 @cached 

107 def get_magAlert(self): 

108 return self.column_by_name('%smag' % self.obs_metadata.bandpass) + \ 

109 self.column_by_name('dmagAlert') 

110 

111 

112@unittest.skipIf(not _avro_is_installed, 'avro is not installed on this system') 

113class AvroAlertTestCase(unittest.TestCase): 

114 

115 longMessage = True 

116 

117 @classmethod 

118 def setUpClass(cls): 

119 print('setting up %s' % sims_clean_up.targets) 

120 

121 # These represent the dimmest magnitudes at which objects 

122 # are considered visible in each of the LSST filters 

123 # (taken from Table 2 of the overview paper) 

124 cls.obs_mag_cutoff = (23.68, 24.89, 24.43, 24.0, 24.45, 22.60) 

125 

126 cls.opsim_db = os.path.join(getPackageDir('sims_data'), 

127 'OpSimData', 

128 'opsimblitz1_1133_sqlite.db') 

129 

130 rng = np.random.RandomState(8123) 

131 

132 obs_gen = ObservationMetaDataGenerator(database=cls.opsim_db) 

133 cls.obs_list = obs_gen.getObservationMetaData(night=(0, 2)) 

134 cls.obs_list = rng.choice(cls.obs_list, 10, replace=False) 

135 fieldid_list = [] 

136 for obs in cls.obs_list: 

137 fieldid_list.append(obs.OpsimMetaData['fieldID']) 

138 

139 # make sure we have selected observations such that the 

140 # same field is revisited more than once 

141 assert len(np.unique(fieldid_list)) < len(fieldid_list) 

142 

143 cls.input_dir = tempfile.mkdtemp(prefix='avroAlertGen', 

144 dir=ROOT) 

145 

146 cls.star_db_name = tempfile.mktemp(prefix='avroAlertGen_star_db', 

147 dir=cls.input_dir, 

148 suffix='.db') 

149 

150 conn = sqlite3.connect(cls.star_db_name) 

151 cursor = conn.cursor() 

152 cursor.execute('''CREATE TABLE stars 

153 (simobjid int, htmid int, ra real, dec real, 

154 umag real, gmag real, rmag real, 

155 imag real, zmag real, ymag real, 

156 px real, pmra real, pmdec real, 

157 vrad real, varParamStr text)''') 

158 conn.commit() 

159 

160 n_stars = 10 

161 

162 cls.ra_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

163 cls.dec_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

164 u_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

165 g_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

166 r_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

167 i_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

168 z_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

169 y_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

170 cls.px_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

171 cls.pmra_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

172 cls.pmdec_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

173 cls.vrad_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

174 cls.amp_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

175 cls.period_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) 

176 

177 id_offset = -n_stars 

178 for obs in cls.obs_list: 

179 id_offset += n_stars 

180 ra_0 = obs.pointingRA 

181 dec_0 = obs.pointingDec 

182 rr = rng.random_sample(n_stars) 

183 theta = rng.random_sample(n_stars)*2.0*np.pi 

184 ra = ra_0 + rr*np.cos(theta) 

185 dec = dec_0 + rr*np.sin(theta) 

186 var_period = rng.random_sample(n_stars)*0.25 

187 var_amp = rng.random_sample(n_stars)*1.0 + 0.01 

188 

189 subset = rng.randint(0, high=len(var_amp)-1, size=3) 

190 var_amp[subset[:2]] = 0.0 

191 var_amp[subset[-1]] = -1.0 

192 

193 umag = rng.random_sample(n_stars)*5.0 + 15.0 

194 gmag = rng.random_sample(n_stars)*5.0 + 15.0 

195 rmag = rng.random_sample(n_stars)*5.0 + 15.0 

196 imag = rng.random_sample(n_stars)*5.0 + 15.0 

197 zmag = rng.random_sample(n_stars)*5.0 + 15.0 

198 ymag = rng.random_sample(n_stars)*5.0 + 15.0 

199 px = rng.random_sample(n_stars)*0.1 # say it is arcsec 

200 pmra = rng.random_sample(n_stars)*50.0+100.0 # say it is arcsec/yr 

201 pmdec = rng.random_sample(n_stars)*50.0+100.0 # say it is arcsec/yr 

202 vrad = rng.random_sample(n_stars)*600.0 - 300.0 

203 

204 subset = rng.randint(0, high=n_stars-1, size=3) 

205 umag[subset] = 40.0 

206 gmag[subset] = 40.0 

207 rmag[subset] = 40.0 

208 imag[subset] = 40.0 

209 zmag[subset] = 40.0 

210 ymag[subset] = 40.0 

211 

212 cls.ra_truth[id_offset:id_offset+n_stars] = np.round(ra, decimals=6) 

213 cls.dec_truth[id_offset:id_offset+n_stars] = np.round(dec, decimals=6) 

214 u_truth[id_offset:id_offset+n_stars] = np.round(umag, decimals=4) 

215 g_truth[id_offset:id_offset+n_stars] = np.round(gmag, decimals=4) 

216 r_truth[id_offset:id_offset+n_stars] = np.round(rmag, decimals=4) 

217 i_truth[id_offset:id_offset+n_stars] = np.round(imag, decimals=4) 

218 z_truth[id_offset:id_offset+n_stars] = np.round(zmag, decimals=4) 

219 y_truth[id_offset:id_offset+n_stars] = np.round(ymag, decimals=4) 

220 cls.px_truth[id_offset:id_offset+n_stars] = np.round(px, decimals=4) 

221 cls.pmra_truth[id_offset:id_offset+n_stars] = np.round(pmra, decimals=4) 

222 cls.pmdec_truth[id_offset:id_offset+n_stars] = np.round(pmdec, decimals=4) 

223 cls.vrad_truth[id_offset:id_offset+n_stars] = np.round(vrad, decimals=4) 

224 cls.amp_truth[id_offset:id_offset+n_stars] = np.round(var_amp, decimals=4) 

225 cls.period_truth[id_offset:id_offset+n_stars] = np.round(var_period, decimals=4) 

226 

227 max_str_len = -1 

228 

229 for i_star in range(n_stars): 

230 if var_amp[i_star] >= -0.1: 

231 varParamStr = ('{"m":"avro_test", "p":{"amp":%.4f, "per": %.4f}}' 

232 % (var_amp[i_star], var_period[i_star])) 

233 else: 

234 varParamStr = 'None' 

235 

236 if len(varParamStr) > max_str_len: 

237 max_str_len = len(varParamStr) 

238 

239 htmid = findHtmid(ra[i_star], dec[i_star], 21) 

240 

241 query = ('''INSERT INTO stars VALUES(%d, %d, %.6f, %.6f, 

242 %.4f, %.4f, %.4f, %.4f, %.4f, %.4f, 

243 %.4f, %.4f, %.4f, %.4f, '%s')''' 

244 % (i_star+id_offset+1, htmid, ra[i_star], dec[i_star], 

245 umag[i_star], gmag[i_star], rmag[i_star], 

246 imag[i_star], zmag[i_star], ymag[i_star], 

247 px[i_star], pmra[i_star], pmdec[i_star], 

248 vrad[i_star], varParamStr)) 

249 

250 cursor.execute(query) 

251 conn.commit() 

252 conn.close() 

253 

254 cls.mag0_truth_dict = {} 

255 cls.mag0_truth_dict[0] = u_truth 

256 cls.mag0_truth_dict[1] = g_truth 

257 cls.mag0_truth_dict[2] = r_truth 

258 cls.mag0_truth_dict[3] = i_truth 

259 cls.mag0_truth_dict[4] = z_truth 

260 cls.mag0_truth_dict[5] = y_truth 

261 assert max_str_len < 500 # make sure varParamStr fits in the space alotted to it 

262 # in StarAlertTestDBObj_avro 

263 

264 @classmethod 

265 def tearDownClass(cls): 

266 sims_clean_up() 

267 if os.path.exists(cls.star_db_name): 

268 os.unlink(cls.star_db_name) 

269 if os.path.exists(cls.input_dir): 

270 shutil.rmtree(cls.input_dir) 

271 

272 clean_up_lsst_camera() 

273 

274 def setUp(self): 

275 self.alert_data_output_dir = tempfile.mkdtemp(dir=ROOT, prefix='avro_gen_output') 

276 self.avro_out_dir = tempfile.mkdtemp(dir=ROOT, prefix='avroTestOut') 

277 

278 def tearDown(self): 

279 for file_name in os.listdir(self.alert_data_output_dir): 

280 os.unlink(os.path.join(self.alert_data_output_dir, file_name)) 

281 shutil.rmtree(self.alert_data_output_dir) 

282 

283 for file_name in os.listdir(self.avro_out_dir): 

284 os.unlink(os.path.join(self.avro_out_dir, file_name)) 

285 shutil.rmtree(self.avro_out_dir) 

286 

287 def test_avro_alert_generation(self): 

288 dmag_cutoff = 0.005 

289 mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5} 

290 

291 star_db = StarAlertTestDBObj_avro(database=self.star_db_name, driver='sqlite') 

292 

293 # assemble a dict of all of the alerts that need to be generated 

294 

295 obshistid_list = [] 

296 for obs in self.obs_list: 

297 obshistid_list.append(obs.OpsimMetaData['obsHistID']) 

298 obshistid_max = max(obshistid_list) 

299 obshistid_bits = int(np.ceil(np.log(obshistid_max)/np.log(2.0))) 

300 

301 true_alert_dict = {} 

302 obs_dict = {} 

303 for obs in self.obs_list: 

304 obs_dict[obs.OpsimMetaData['obsHistID']] = obs 

305 obshistid = obs.OpsimMetaData['obsHistID'] 

306 cat = TestAlertsTruthCat_avro(star_db, obs_metadata=obs) 

307 cat.camera = lsst_camera() 

308 

309 for line in cat.iter_catalog(): 

310 if line[1] is None: 

311 continue 

312 

313 dmag = line[2] 

314 mag = line[3] 

315 if (np.abs(dmag) > dmag_cutoff and 

316 mag <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]): 

317 

318 alertId = (line[0] << obshistid_bits) + obshistid 

319 self.assertNotIn(alertId, true_alert_dict) 

320 true_alert_dict[alertId] = {} 

321 true_alert_dict[alertId]['chipName'] = line[1] 

322 true_alert_dict[alertId]['dmag'] = dmag 

323 true_alert_dict[alertId]['mag'] = mag 

324 true_alert_dict[alertId]['ra'] = np.degrees(line[4]) 

325 true_alert_dict[alertId]['decl'] = np.degrees(line[5]) 

326 true_alert_dict[alertId]['xPix'] = line[6] 

327 true_alert_dict[alertId]['yPix'] = line[7] 

328 

329 self.assertGreater(len(true_alert_dict), 10) 

330 

331 log_file_name = tempfile.mktemp(dir=self.alert_data_output_dir, suffix='log.txt') 

332 alert_gen = AlertDataGenerator(testing=True) 

333 

334 alert_gen.subdivide_obs(self.obs_list, htmid_level=6) 

335 

336 for htmid in alert_gen.htmid_list: 

337 alert_gen.alert_data_from_htmid(htmid, star_db, 

338 photometry_class=TestAlertsVarCat_avro, 

339 output_prefix='alert_test', 

340 output_dir=self.alert_data_output_dir, 

341 dmag_cutoff=dmag_cutoff, 

342 log_file_name=log_file_name) 

343 

344 obshistid_to_htmid = {} 

345 for htmid in alert_gen.htmid_list: 

346 for obs in alert_gen.obs_from_htmid(htmid): 

347 obshistid = obs.OpsimMetaData['obsHistID'] 

348 if obshistid not in obshistid_to_htmid: 

349 obshistid_to_htmid[obshistid] = [] 

350 obshistid_to_htmid[obshistid].append(htmid) 

351 

352 avro_gen = AvroAlertGenerator() 

353 avro_gen.load_schema(os.path.join(getPackageDir('sims_catUtils'), 'tests', 'testData', 'avroSchema')) 

354 sql_prefix_list = ['alert_test'] 

355 out_prefix = 'test_avro' 

356 log_file_name = tempfile.mktemp(dir=self.avro_out_dir, 

357 prefix='test_avro', 

358 suffix='log.txt') 

359 for obshistid in obshistid_list: 

360 avro_gen.write_alerts(obshistid, self.alert_data_output_dir, 

361 sql_prefix_list, 

362 obshistid_to_htmid[obshistid], 

363 self.avro_out_dir, out_prefix, 

364 dmag_cutoff, lock=None, 

365 log_file_name=log_file_name) 

366 

367 list_of_avro_files = os.listdir(self.avro_out_dir) 

368 self.assertGreater(len(list_of_avro_files), 2) 

369 alert_ct = 0 

370 dummy_sed = Sed() 

371 bp_dict = BandpassDict.loadTotalBandpassesFromFiles() 

372 photParams = PhotometricParameters() 

373 diasourceId_set = set() 

374 for avro_file_name in list_of_avro_files: 

375 if avro_file_name.endswith('log.txt'): 

376 continue 

377 full_name = os.path.join(self.avro_out_dir, avro_file_name) 

378 with DataFileReader(open(full_name, 'rb'), DatumReader()) as data_reader: 

379 for alert in data_reader: 

380 alert_ct += 1 

381 obshistid = alert['alertId'] >> 20 

382 obs = obs_dict[obshistid] 

383 uniqueId = alert['diaObject']['diaObjectId'] 

384 true_alert_id = (uniqueId << obshistid_bits) + obshistid 

385 self.assertIn(true_alert_id, true_alert_dict) 

386 self.assertEqual(alert['l1dbId'], uniqueId) 

387 

388 true_alert = true_alert_dict[true_alert_id] 

389 

390 diaSource = alert['diaSource'] 

391 self.assertAlmostEqual(diaSource['ra'], true_alert['ra'], 10) 

392 self.assertAlmostEqual(diaSource['decl'], true_alert['decl'], 10) 

393 self.assertAlmostEqual(diaSource['x'], true_alert['xPix'], 3) 

394 self.assertAlmostEqual(diaSource['y'], true_alert['yPix'], 3) 

395 self.assertAlmostEqual(diaSource['midPointTai'], obs.mjd.TAI, 4) 

396 

397 true_tot_flux = dummy_sed.fluxFromMag(true_alert['mag']) 

398 true_q_mag = true_alert['mag'] - true_alert['dmag'] 

399 true_q_flux = dummy_sed.fluxFromMag(true_q_mag) 

400 true_dflux = true_tot_flux - true_q_flux 

401 self.assertAlmostEqual(diaSource['psFlux']/true_dflux, 1.0, 6) 

402 self.assertAlmostEqual(diaSource['totFlux']/true_tot_flux, 1.0, 6) 

403 self.assertAlmostEqual(diaSource['diffFlux']/true_dflux, 1.0, 6) 

404 

405 true_tot_snr, gamma = calcSNR_m5(true_alert['mag'], bp_dict[obs.bandpass], 

406 obs.m5[obs.bandpass], photParams) 

407 

408 true_q_snr, gamma = calcSNR_m5(true_q_mag, bp_dict[obs.bandpass], 

409 self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]], 

410 photParams) 

411 

412 true_tot_err = true_tot_flux/true_tot_snr 

413 true_q_err = true_q_flux/true_q_snr 

414 true_diff_err = np.sqrt(true_tot_err**2 + true_q_err**2) 

415 

416 self.assertAlmostEqual(diaSource['snr']/np.abs(true_dflux/true_diff_err), 

417 1.0, 6) 

418 

419 self.assertAlmostEqual(diaSource['totFluxErr']/true_tot_err, 1.0, 6) 

420 self.assertAlmostEqual(diaSource['diffFluxErr']/true_diff_err, 1.0, 6) 

421 

422 chipnum = int(true_alert['chipName'].replace('R', '').replace('S', ''). 

423 replace(',', '').replace(':', '').replace(' ', '')) 

424 

425 true_ccdid = (chipnum*10**7)+obshistid 

426 self.assertEqual(true_ccdid, diaSource['ccdVisitId']) 

427 self.assertEqual(uniqueId, diaSource['diaObjectId']) 

428 

429 self.assertNotIn(diaSource['diaSourceId'], diasourceId_set) 

430 diasourceId_set.add(diaSource['diaSourceId']) 

431 

432 diaObject = alert['diaObject'] 

433 obj_dex = (uniqueId//1024) - 1 

434 self.assertAlmostEqual(0.001*diaObject['pmRa']/self.pmra_truth[obj_dex], 1.0, 5) 

435 self.assertAlmostEqual(0.001*diaObject['pmDecl']/self.pmdec_truth[obj_dex], 1.0, 5) 

436 self.assertAlmostEqual(0.001*diaObject['parallax']/self.px_truth[obj_dex], 1.0, 5) 

437 

438 (true_ra_base, 

439 true_dec_base) = applyProperMotion(self.ra_truth[obj_dex], 

440 self.dec_truth[obj_dex], 

441 self.pmra_truth[obj_dex], 

442 self.pmdec_truth[obj_dex], 

443 self.px_truth[obj_dex], 

444 self.vrad_truth[obj_dex], 

445 mjd=ModifiedJulianDate(TAI=diaObject['radecTai'])) 

446 

447 self.assertAlmostEqual(true_ra_base, diaObject['ra'], 7) 

448 self.assertAlmostEqual(true_dec_base, diaObject['decl'], 7) 

449 

450 self.assertEqual(alert_ct, len(true_alert_dict)) 

451 

452 def test_avro_alert_generation_diff_dmag(self): 

453 """ 

454 Make sure everything works properly when the AlertDataGenerator 

455 and the AvroAlertGenerator have different dmag thresholds 

456 """ 

457 dmag_cutoff_sqlite = 0.005 

458 dmag_cutoff_avro = 0.2 

459 mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5} 

460 

461 star_db = StarAlertTestDBObj_avro(database=self.star_db_name, driver='sqlite') 

462 

463 # assemble a dict of all of the alerts that need to be generated 

464 

465 obshistid_list = [] 

466 for obs in self.obs_list: 

467 obshistid_list.append(obs.OpsimMetaData['obsHistID']) 

468 obshistid_max = max(obshistid_list) 

469 obshistid_bits = int(np.ceil(np.log(obshistid_max)/np.log(2.0))) 

470 

471 true_alert_dict = {} 

472 obs_dict = {} 

473 ignored_sqlite = 0 # count number of alerts written to sqlite, but not avro 

474 for obs in self.obs_list: 

475 obs_dict[obs.OpsimMetaData['obsHistID']] = obs 

476 obshistid = obs.OpsimMetaData['obsHistID'] 

477 cat = TestAlertsTruthCat_avro(star_db, obs_metadata=obs) 

478 cat.camera = lsst_camera() 

479 

480 for line in cat.iter_catalog(): 

481 if line[1] is None: 

482 continue 

483 

484 dmag = line[2] 

485 mag = line[3] 

486 if (np.abs(dmag) > dmag_cutoff_avro and 

487 mag <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]): 

488 

489 alertId = (line[0] << obshistid_bits) + obshistid 

490 self.assertNotIn(alertId, true_alert_dict) 

491 true_alert_dict[alertId] = {} 

492 true_alert_dict[alertId]['chipName'] = line[1] 

493 true_alert_dict[alertId]['dmag'] = dmag 

494 true_alert_dict[alertId]['mag'] = mag 

495 true_alert_dict[alertId]['ra'] = np.degrees(line[4]) 

496 true_alert_dict[alertId]['decl'] = np.degrees(line[5]) 

497 true_alert_dict[alertId]['xPix'] = line[6] 

498 true_alert_dict[alertId]['yPix'] = line[7] 

499 elif np.abs(dmag) > dmag_cutoff_sqlite: 

500 ignored_sqlite += 1 

501 

502 self.assertGreater(len(true_alert_dict), 10) 

503 

504 self.assertGreater(ignored_sqlite, 50) # just make sure that some sqlite 

505 # alerts were ignored by the more 

506 # stringent avro cut 

507 

508 log_file_name = tempfile.mktemp(dir=self.alert_data_output_dir, suffix='log.txt') 

509 alert_gen = AlertDataGenerator(testing=True) 

510 

511 alert_gen.subdivide_obs(self.obs_list, htmid_level=6) 

512 

513 for htmid in alert_gen.htmid_list: 

514 alert_gen.alert_data_from_htmid(htmid, star_db, 

515 photometry_class=TestAlertsVarCat_avro, 

516 output_prefix='alert_test', 

517 output_dir=self.alert_data_output_dir, 

518 dmag_cutoff=dmag_cutoff_sqlite, 

519 log_file_name=log_file_name) 

520 

521 obshistid_to_htmid = {} 

522 for htmid in alert_gen.htmid_list: 

523 for obs in alert_gen.obs_from_htmid(htmid): 

524 obshistid = obs.OpsimMetaData['obsHistID'] 

525 if obshistid not in obshistid_to_htmid: 

526 obshistid_to_htmid[obshistid] = [] 

527 obshistid_to_htmid[obshistid].append(htmid) 

528 

529 avro_gen = AvroAlertGenerator() 

530 avro_gen.load_schema(os.path.join(getPackageDir('sims_catUtils'), 'tests', 'testData', 'avroSchema')) 

531 sql_prefix_list = ['alert_test'] 

532 out_prefix = 'test_avro' 

533 log_file_name = tempfile.mktemp(dir=self.avro_out_dir, 

534 prefix='test_avro', 

535 suffix='log.txt') 

536 for obshistid in obshistid_list: 

537 avro_gen.write_alerts(obshistid, self.alert_data_output_dir, 

538 sql_prefix_list, 

539 obshistid_to_htmid[obshistid], 

540 self.avro_out_dir, out_prefix, 

541 dmag_cutoff_avro, lock=None, 

542 log_file_name=log_file_name) 

543 

544 list_of_avro_files = os.listdir(self.avro_out_dir) 

545 self.assertGreater(len(list_of_avro_files), 2) 

546 alert_ct = 0 

547 dummy_sed = Sed() 

548 bp_dict = BandpassDict.loadTotalBandpassesFromFiles() 

549 photParams = PhotometricParameters() 

550 diasourceId_set = set() 

551 for avro_file_name in list_of_avro_files: 

552 if avro_file_name.endswith('log.txt'): 

553 continue 

554 full_name = os.path.join(self.avro_out_dir, avro_file_name) 

555 with DataFileReader(open(full_name, 'rb'), DatumReader()) as data_reader: 

556 for alert in data_reader: 

557 alert_ct += 1 

558 obshistid = alert['alertId'] >> 20 

559 obs = obs_dict[obshistid] 

560 uniqueId = alert['diaObject']['diaObjectId'] 

561 true_alert_id = (uniqueId << obshistid_bits) + obshistid 

562 self.assertIn(true_alert_id, true_alert_dict) 

563 self.assertEqual(alert['l1dbId'], uniqueId) 

564 

565 true_alert = true_alert_dict[true_alert_id] 

566 

567 diaSource = alert['diaSource'] 

568 self.assertAlmostEqual(diaSource['ra'], true_alert['ra'], 10) 

569 self.assertAlmostEqual(diaSource['decl'], true_alert['decl'], 10) 

570 self.assertAlmostEqual(diaSource['x'], true_alert['xPix'], 3) 

571 self.assertAlmostEqual(diaSource['y'], true_alert['yPix'], 3) 

572 self.assertAlmostEqual(diaSource['midPointTai'], obs.mjd.TAI, 4) 

573 

574 true_tot_flux = dummy_sed.fluxFromMag(true_alert['mag']) 

575 true_q_mag = true_alert['mag'] - true_alert['dmag'] 

576 true_q_flux = dummy_sed.fluxFromMag(true_q_mag) 

577 true_dflux = true_tot_flux - true_q_flux 

578 self.assertAlmostEqual(diaSource['psFlux']/true_dflux, 1.0, 6) 

579 self.assertAlmostEqual(diaSource['totFlux']/true_tot_flux, 1.0, 6) 

580 self.assertAlmostEqual(diaSource['diffFlux']/true_dflux, 1.0, 6) 

581 

582 true_tot_snr, gamma = calcSNR_m5(true_alert['mag'], bp_dict[obs.bandpass], 

583 obs.m5[obs.bandpass], photParams) 

584 

585 true_q_snr, gamma = calcSNR_m5(true_q_mag, bp_dict[obs.bandpass], 

586 self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]], 

587 photParams) 

588 

589 true_tot_err = true_tot_flux/true_tot_snr 

590 true_q_err = true_q_flux/true_q_snr 

591 true_diff_err = np.sqrt(true_tot_err**2 + true_q_err**2) 

592 

593 self.assertAlmostEqual(diaSource['snr']/np.abs(true_dflux/true_diff_err), 

594 1.0, 6) 

595 

596 self.assertAlmostEqual(diaSource['totFluxErr']/true_tot_err, 1.0, 6) 

597 self.assertAlmostEqual(diaSource['diffFluxErr']/true_diff_err, 1.0, 6) 

598 

599 chipnum = int(true_alert['chipName'].replace('R', '').replace('S', ''). 

600 replace(',', '').replace(':', '').replace(' ', '')) 

601 

602 true_ccdid = (chipnum*10**7)+obshistid 

603 self.assertEqual(true_ccdid, diaSource['ccdVisitId']) 

604 self.assertEqual(uniqueId, diaSource['diaObjectId']) 

605 

606 self.assertNotIn(diaSource['diaSourceId'], diasourceId_set) 

607 diasourceId_set.add(diaSource['diaSourceId']) 

608 

609 diaObject = alert['diaObject'] 

610 obj_dex = (uniqueId//1024) - 1 

611 self.assertAlmostEqual(0.001*diaObject['pmRa']/self.pmra_truth[obj_dex], 1.0, 5) 

612 self.assertAlmostEqual(0.001*diaObject['pmDecl']/self.pmdec_truth[obj_dex], 1.0, 5) 

613 self.assertAlmostEqual(0.001*diaObject['parallax']/self.px_truth[obj_dex], 1.0, 5) 

614 

615 (true_ra_base, 

616 true_dec_base) = applyProperMotion(self.ra_truth[obj_dex], 

617 self.dec_truth[obj_dex], 

618 self.pmra_truth[obj_dex], 

619 self.pmdec_truth[obj_dex], 

620 self.px_truth[obj_dex], 

621 self.vrad_truth[obj_dex], 

622 mjd=ModifiedJulianDate(TAI=diaObject['radecTai'])) 

623 

624 self.assertAlmostEqual(true_ra_base, diaObject['ra'], 7) 

625 self.assertAlmostEqual(true_dec_base, diaObject['decl'], 7) 

626 

627 self.assertEqual(alert_ct, len(true_alert_dict)) 

628 

629 

630class MemoryTestClass(lsst.utils.tests.MemoryTestCase): 

631 pass 

632 

633 

634if __name__ == "__main__": 634 ↛ 635line 634 didn't jump to line 635, because the condition on line 634 was never true

635 lsst.utils.tests.init() 

636 unittest.main()