Coverage for tests/test_avroAlertGenerator.py : 14%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import unittest
2import os
3import tempfile
4import shutil
5import gc
6import numpy as np
7import sqlite3
8import numbers
9import lsst.utils.tests
11from lsst.utils import getPackageDir
12import lsst.obs.lsst.phosim as obs_lsst_phosim
13from lsst.sims.utils.CodeUtilities import sims_clean_up
14from lsst.sims.catalogs.decorators import register_method
15from lsst.sims.catalogs.db import CatalogDBObject
16from lsst.sims.catUtils.utils import ObservationMetaDataGenerator
17from lsst.sims.catUtils.utils import AlertStellarVariabilityCatalog
18from lsst.sims.catUtils.utils import StellarAlertDBObjMixin
19from lsst.sims.utils import findHtmid
20from lsst.sims.utils import applyProperMotion, ModifiedJulianDate
21from lsst.sims.photUtils import Sed, calcSNR_m5, BandpassDict
22from lsst.sims.photUtils import PhotometricParameters
23from lsst.sims.catUtils.mixins import CameraCoords
24from lsst.sims.catUtils.mixins import AstrometryStars
25from lsst.sims.catUtils.mixins import Variability
26from lsst.sims.catalogs.definitions import InstanceCatalog
27from lsst.sims.catalogs.decorators import compound, cached
29from lsst.sims.catUtils.utils import AlertDataGenerator
30from lsst.sims.catUtils.utils import AvroAlertGenerator
33_avro_is_installed = True
34try:
35 from avro.io import DatumReader
36 from avro.datafile import DataFileReader
37except ImportError:
38 _avro_is_installed = False
39 pass
42ROOT = os.path.abspath(os.path.dirname(__file__))
45def setup_module(module):
46 lsst.utils.tests.init()
49class StarAlertTestDBObj_avro(StellarAlertDBObjMixin, CatalogDBObject):
50 objid = 'star_alert'
51 tableid = 'stars'
52 idColKey = 'simobjid'
53 raColName = 'ra'
54 decColName = 'dec'
55 objectTypeId = 0
56 columns = [('raJ2000', 'ra*0.01745329252'),
57 ('decJ2000', 'dec*0.01745329252'),
58 ('parallax', 'px*0.01745329252/3600.0'),
59 ('properMotionRa', 'pmra*0.01745329252/3600.0'),
60 ('properMotionDec', 'pmdec*0.01745329252/3600.0'),
61 ('radialVelocity', 'vrad'),
62 ('variabilityParameters', 'varParamStr', str, 500)]
65class TestAlertsVarCatMixin_avro(object):
67 @register_method('avro_test')
68 def applyAlertTest(self, valid_dexes, params, expmjd, variability_cache=None):
69 if len(params) == 0:
70 return np.array([[], [], [], [], [], []])
72 if isinstance(expmjd, numbers.Number):
73 dMags_out = np.zeros((6, self.num_variable_obj(params)))
74 else:
75 dMags_out = np.zeros((6, self.num_variable_obj(params), len(expmjd)))
77 for i_star in range(self.num_variable_obj(params)):
78 if params['amp'][i_star] is not None:
79 dmags = params['amp'][i_star]*np.cos(params['per'][i_star]*expmjd)
80 for i_filter in range(6):
81 dMags_out[i_filter][i_star] = dmags
83 return dMags_out
86class TestAlertsVarCat_avro(TestAlertsVarCatMixin_avro, AlertStellarVariabilityCatalog):
87 pass
90class TestAlertsTruthCat_avro(TestAlertsVarCatMixin_avro, CameraCoords, AstrometryStars,
91 Variability, InstanceCatalog):
92 column_outputs = ['uniqueId', 'chipName', 'dmagAlert', 'magAlert',
93 'raICRS', 'decICRS', 'xPix', 'yPix']
95 camera = obs_lsst_phosim.PhosimMapper().camera
97 @compound('delta_umag', 'delta_gmag', 'delta_rmag',
98 'delta_imag', 'delta_zmag', 'delta_ymag')
99 def get_TruthVariability(self):
100 return self.applyVariability(self.column_by_name('varParamStr'))
102 @cached
103 def get_dmagAlert(self):
104 return self.column_by_name('delta_%smag' % self.obs_metadata.bandpass)
106 @cached
107 def get_magAlert(self):
108 return self.column_by_name('%smag' % self.obs_metadata.bandpass) + \
109 self.column_by_name('dmagAlert')
112@unittest.skipIf(not _avro_is_installed, 'avro is not installed on this system')
113class AvroAlertTestCase(unittest.TestCase):
115 longMessage = True
117 @classmethod
118 def setUpClass(cls):
119 print('setting up %s' % sims_clean_up.targets)
121 # These represent the dimmest magnitudes at which objects
122 # are considered visible in each of the LSST filters
123 # (taken from Table 2 of the overview paper)
124 cls.obs_mag_cutoff = (23.68, 24.89, 24.43, 24.0, 24.45, 22.60)
126 cls.opsim_db = os.path.join(getPackageDir('sims_data'),
127 'OpSimData',
128 'opsimblitz1_1133_sqlite.db')
130 rng = np.random.RandomState(8123)
132 obs_gen = ObservationMetaDataGenerator(database=cls.opsim_db)
133 cls.obs_list = obs_gen.getObservationMetaData(night=(0, 2))
134 cls.obs_list = rng.choice(cls.obs_list, 10, replace=False)
135 fieldid_list = []
136 for obs in cls.obs_list:
137 fieldid_list.append(obs.OpsimMetaData['fieldID'])
139 # make sure we have selected observations such that the
140 # same field is revisited more than once
141 assert len(np.unique(fieldid_list)) < len(fieldid_list)
143 cls.input_dir = tempfile.mkdtemp(prefix='avroAlertGen',
144 dir=ROOT)
146 cls.star_db_name = tempfile.mktemp(prefix='avroAlertGen_star_db',
147 dir=cls.input_dir,
148 suffix='.db')
150 conn = sqlite3.connect(cls.star_db_name)
151 cursor = conn.cursor()
152 cursor.execute('''CREATE TABLE stars
153 (simobjid int, htmid int, ra real, dec real,
154 umag real, gmag real, rmag real,
155 imag real, zmag real, ymag real,
156 px real, pmra real, pmdec real,
157 vrad real, varParamStr text)''')
158 conn.commit()
160 n_stars = 10
162 cls.ra_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
163 cls.dec_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
164 u_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
165 g_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
166 r_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
167 i_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
168 z_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
169 y_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
170 cls.px_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
171 cls.pmra_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
172 cls.pmdec_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
173 cls.vrad_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
174 cls.amp_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
175 cls.period_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
177 id_offset = -n_stars
178 for obs in cls.obs_list:
179 id_offset += n_stars
180 ra_0 = obs.pointingRA
181 dec_0 = obs.pointingDec
182 rr = rng.random_sample(n_stars)
183 theta = rng.random_sample(n_stars)*2.0*np.pi
184 ra = ra_0 + rr*np.cos(theta)
185 dec = dec_0 + rr*np.sin(theta)
186 var_period = rng.random_sample(n_stars)*0.25
187 var_amp = rng.random_sample(n_stars)*1.0 + 0.01
189 subset = rng.randint(0, high=len(var_amp)-1, size=3)
190 var_amp[subset[:2]] = 0.0
191 var_amp[subset[-1]] = -1.0
193 umag = rng.random_sample(n_stars)*5.0 + 15.0
194 gmag = rng.random_sample(n_stars)*5.0 + 15.0
195 rmag = rng.random_sample(n_stars)*5.0 + 15.0
196 imag = rng.random_sample(n_stars)*5.0 + 15.0
197 zmag = rng.random_sample(n_stars)*5.0 + 15.0
198 ymag = rng.random_sample(n_stars)*5.0 + 15.0
199 px = rng.random_sample(n_stars)*0.1 # say it is arcsec
200 pmra = rng.random_sample(n_stars)*50.0+100.0 # say it is arcsec/yr
201 pmdec = rng.random_sample(n_stars)*50.0+100.0 # say it is arcsec/yr
202 vrad = rng.random_sample(n_stars)*600.0 - 300.0
204 subset = rng.randint(0, high=n_stars-1, size=3)
205 umag[subset] = 40.0
206 gmag[subset] = 40.0
207 rmag[subset] = 40.0
208 imag[subset] = 40.0
209 zmag[subset] = 40.0
210 ymag[subset] = 40.0
212 cls.ra_truth[id_offset:id_offset+n_stars] = np.round(ra, decimals=6)
213 cls.dec_truth[id_offset:id_offset+n_stars] = np.round(dec, decimals=6)
214 u_truth[id_offset:id_offset+n_stars] = np.round(umag, decimals=4)
215 g_truth[id_offset:id_offset+n_stars] = np.round(gmag, decimals=4)
216 r_truth[id_offset:id_offset+n_stars] = np.round(rmag, decimals=4)
217 i_truth[id_offset:id_offset+n_stars] = np.round(imag, decimals=4)
218 z_truth[id_offset:id_offset+n_stars] = np.round(zmag, decimals=4)
219 y_truth[id_offset:id_offset+n_stars] = np.round(ymag, decimals=4)
220 cls.px_truth[id_offset:id_offset+n_stars] = np.round(px, decimals=4)
221 cls.pmra_truth[id_offset:id_offset+n_stars] = np.round(pmra, decimals=4)
222 cls.pmdec_truth[id_offset:id_offset+n_stars] = np.round(pmdec, decimals=4)
223 cls.vrad_truth[id_offset:id_offset+n_stars] = np.round(vrad, decimals=4)
224 cls.amp_truth[id_offset:id_offset+n_stars] = np.round(var_amp, decimals=4)
225 cls.period_truth[id_offset:id_offset+n_stars] = np.round(var_period, decimals=4)
227 max_str_len = -1
229 for i_star in range(n_stars):
230 if var_amp[i_star] >= -0.1:
231 varParamStr = ('{"m":"avro_test", "p":{"amp":%.4f, "per": %.4f}}'
232 % (var_amp[i_star], var_period[i_star]))
233 else:
234 varParamStr = 'None'
236 if len(varParamStr) > max_str_len:
237 max_str_len = len(varParamStr)
239 htmid = findHtmid(ra[i_star], dec[i_star], 21)
241 query = ('''INSERT INTO stars VALUES(%d, %d, %.6f, %.6f,
242 %.4f, %.4f, %.4f, %.4f, %.4f, %.4f,
243 %.4f, %.4f, %.4f, %.4f, '%s')'''
244 % (i_star+id_offset+1, htmid, ra[i_star], dec[i_star],
245 umag[i_star], gmag[i_star], rmag[i_star],
246 imag[i_star], zmag[i_star], ymag[i_star],
247 px[i_star], pmra[i_star], pmdec[i_star],
248 vrad[i_star], varParamStr))
250 cursor.execute(query)
251 conn.commit()
252 conn.close()
254 cls.mag0_truth_dict = {}
255 cls.mag0_truth_dict[0] = u_truth
256 cls.mag0_truth_dict[1] = g_truth
257 cls.mag0_truth_dict[2] = r_truth
258 cls.mag0_truth_dict[3] = i_truth
259 cls.mag0_truth_dict[4] = z_truth
260 cls.mag0_truth_dict[5] = y_truth
261 assert max_str_len < 500 # make sure varParamStr fits in the space alotted to it
262 # in StarAlertTestDBObj_avro
264 @classmethod
265 def tearDownClass(cls):
266 sims_clean_up()
267 if os.path.exists(cls.star_db_name):
268 os.unlink(cls.star_db_name)
269 if os.path.exists(cls.input_dir):
270 shutil.rmtree(cls.input_dir)
272 def setUp(self):
273 self.alert_data_output_dir = tempfile.mkdtemp(dir=ROOT, prefix='avro_gen_output')
274 self.avro_out_dir = tempfile.mkdtemp(dir=ROOT, prefix='avroTestOut')
276 def tearDown(self):
277 for file_name in os.listdir(self.alert_data_output_dir):
278 os.unlink(os.path.join(self.alert_data_output_dir, file_name))
279 shutil.rmtree(self.alert_data_output_dir)
281 for file_name in os.listdir(self.avro_out_dir):
282 os.unlink(os.path.join(self.avro_out_dir, file_name))
283 shutil.rmtree(self.avro_out_dir)
285 def test_avro_alert_generation(self):
286 dmag_cutoff = 0.005
287 mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5}
289 star_db = StarAlertTestDBObj_avro(database=self.star_db_name, driver='sqlite')
291 # assemble a dict of all of the alerts that need to be generated
293 obshistid_list = []
294 for obs in self.obs_list:
295 obshistid_list.append(obs.OpsimMetaData['obsHistID'])
296 obshistid_max = max(obshistid_list)
297 obshistid_bits = int(np.ceil(np.log(obshistid_max)/np.log(2.0)))
299 true_alert_dict = {}
300 obs_dict = {}
301 for obs in self.obs_list:
302 obs_dict[obs.OpsimMetaData['obsHistID']] = obs
303 obshistid = obs.OpsimMetaData['obsHistID']
304 cat = TestAlertsTruthCat_avro(star_db, obs_metadata=obs)
305 cat.camera = obs_lsst_phosim.PhosimMapper().camera
307 for line in cat.iter_catalog():
308 if line[1] is None:
309 continue
311 dmag = line[2]
312 mag = line[3]
313 if (np.abs(dmag) > dmag_cutoff and
314 mag <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]):
316 alertId = (line[0] << obshistid_bits) + obshistid
317 self.assertNotIn(alertId, true_alert_dict)
318 true_alert_dict[alertId] = {}
319 true_alert_dict[alertId]['chipName'] = line[1]
320 true_alert_dict[alertId]['dmag'] = dmag
321 true_alert_dict[alertId]['mag'] = mag
322 true_alert_dict[alertId]['ra'] = np.degrees(line[4])
323 true_alert_dict[alertId]['decl'] = np.degrees(line[5])
324 true_alert_dict[alertId]['xPix'] = line[6]
325 true_alert_dict[alertId]['yPix'] = line[7]
327 self.assertGreater(len(true_alert_dict), 10)
329 log_file_name = tempfile.mktemp(dir=self.alert_data_output_dir, suffix='log.txt')
330 alert_gen = AlertDataGenerator(testing=True)
332 alert_gen.subdivide_obs(self.obs_list, htmid_level=6)
334 for htmid in alert_gen.htmid_list:
335 alert_gen.alert_data_from_htmid(htmid, star_db,
336 photometry_class=TestAlertsVarCat_avro,
337 output_prefix='alert_test',
338 output_dir=self.alert_data_output_dir,
339 dmag_cutoff=dmag_cutoff,
340 log_file_name=log_file_name)
342 obshistid_to_htmid = {}
343 for htmid in alert_gen.htmid_list:
344 for obs in alert_gen.obs_from_htmid(htmid):
345 obshistid = obs.OpsimMetaData['obsHistID']
346 if obshistid not in obshistid_to_htmid:
347 obshistid_to_htmid[obshistid] = []
348 obshistid_to_htmid[obshistid].append(htmid)
350 avro_gen = AvroAlertGenerator()
351 avro_gen.load_schema(os.path.join(getPackageDir('sims_catUtils'), 'tests', 'testData', 'avroSchema'))
352 sql_prefix_list = ['alert_test']
353 out_prefix = 'test_avro'
354 log_file_name = tempfile.mktemp(dir=self.avro_out_dir,
355 prefix='test_avro',
356 suffix='log.txt')
357 for obshistid in obshistid_list:
358 avro_gen.write_alerts(obshistid, self.alert_data_output_dir,
359 sql_prefix_list,
360 obshistid_to_htmid[obshistid],
361 self.avro_out_dir, out_prefix,
362 dmag_cutoff, lock=None,
363 log_file_name=log_file_name)
365 list_of_avro_files = os.listdir(self.avro_out_dir)
366 self.assertGreater(len(list_of_avro_files), 2)
367 alert_ct = 0
368 dummy_sed = Sed()
369 bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
370 photParams = PhotometricParameters()
371 diasourceId_set = set()
372 for avro_file_name in list_of_avro_files:
373 if avro_file_name.endswith('log.txt'):
374 continue
375 full_name = os.path.join(self.avro_out_dir, avro_file_name)
376 with DataFileReader(open(full_name, 'rb'), DatumReader()) as data_reader:
377 for alert in data_reader:
378 alert_ct += 1
379 obshistid = alert['alertId'] >> 20
380 obs = obs_dict[obshistid]
381 uniqueId = alert['diaObject']['diaObjectId']
382 true_alert_id = (uniqueId << obshistid_bits) + obshistid
383 self.assertIn(true_alert_id, true_alert_dict)
384 self.assertEqual(alert['l1dbId'], uniqueId)
386 true_alert = true_alert_dict[true_alert_id]
388 diaSource = alert['diaSource']
389 self.assertAlmostEqual(diaSource['ra'], true_alert['ra'], 10)
390 self.assertAlmostEqual(diaSource['decl'], true_alert['decl'], 10)
391 self.assertAlmostEqual(diaSource['x'], true_alert['xPix'], 3)
392 self.assertAlmostEqual(diaSource['y'], true_alert['yPix'], 3)
393 self.assertAlmostEqual(diaSource['midPointTai'], obs.mjd.TAI, 4)
395 true_tot_flux = dummy_sed.fluxFromMag(true_alert['mag'])
396 true_q_mag = true_alert['mag'] - true_alert['dmag']
397 true_q_flux = dummy_sed.fluxFromMag(true_q_mag)
398 true_dflux = true_tot_flux - true_q_flux
399 self.assertAlmostEqual(diaSource['psFlux']/true_dflux, 1.0, 6)
400 self.assertAlmostEqual(diaSource['totFlux']/true_tot_flux, 1.0, 6)
401 self.assertAlmostEqual(diaSource['diffFlux']/true_dflux, 1.0, 6)
403 true_tot_snr, gamma = calcSNR_m5(true_alert['mag'], bp_dict[obs.bandpass],
404 obs.m5[obs.bandpass], photParams)
406 true_q_snr, gamma = calcSNR_m5(true_q_mag, bp_dict[obs.bandpass],
407 self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]],
408 photParams)
410 true_tot_err = true_tot_flux/true_tot_snr
411 true_q_err = true_q_flux/true_q_snr
412 true_diff_err = np.sqrt(true_tot_err**2 + true_q_err**2)
414 self.assertAlmostEqual(diaSource['snr']/np.abs(true_dflux/true_diff_err),
415 1.0, 6)
417 self.assertAlmostEqual(diaSource['totFluxErr']/true_tot_err, 1.0, 6)
418 self.assertAlmostEqual(diaSource['diffFluxErr']/true_diff_err, 1.0, 6)
420 chipnum = int(true_alert['chipName'].replace('R', '').replace('S', '').
421 replace(',', '').replace(':', '').replace(' ', ''))
423 true_ccdid = (chipnum*10**7)+obshistid
424 self.assertEqual(true_ccdid, diaSource['ccdVisitId'])
425 self.assertEqual(uniqueId, diaSource['diaObjectId'])
427 self.assertNotIn(diaSource['diaSourceId'], diasourceId_set)
428 diasourceId_set.add(diaSource['diaSourceId'])
430 diaObject = alert['diaObject']
431 obj_dex = (uniqueId//1024) - 1
432 self.assertAlmostEqual(0.001*diaObject['pmRa']/self.pmra_truth[obj_dex], 1.0, 5)
433 self.assertAlmostEqual(0.001*diaObject['pmDecl']/self.pmdec_truth[obj_dex], 1.0, 5)
434 self.assertAlmostEqual(0.001*diaObject['parallax']/self.px_truth[obj_dex], 1.0, 5)
436 (true_ra_base,
437 true_dec_base) = applyProperMotion(self.ra_truth[obj_dex],
438 self.dec_truth[obj_dex],
439 self.pmra_truth[obj_dex],
440 self.pmdec_truth[obj_dex],
441 self.px_truth[obj_dex],
442 self.vrad_truth[obj_dex],
443 mjd=ModifiedJulianDate(TAI=diaObject['radecTai']))
445 self.assertAlmostEqual(true_ra_base, diaObject['ra'], 7)
446 self.assertAlmostEqual(true_dec_base, diaObject['decl'], 7)
448 self.assertEqual(alert_ct, len(true_alert_dict))
450 def test_avro_alert_generation_diff_dmag(self):
451 """
452 Make sure everything works properly when the AlertDataGenerator
453 and the AvroAlertGenerator have different dmag thresholds
454 """
455 dmag_cutoff_sqlite = 0.005
456 dmag_cutoff_avro = 0.2
457 mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5}
459 star_db = StarAlertTestDBObj_avro(database=self.star_db_name, driver='sqlite')
461 # assemble a dict of all of the alerts that need to be generated
463 obshistid_list = []
464 for obs in self.obs_list:
465 obshistid_list.append(obs.OpsimMetaData['obsHistID'])
466 obshistid_max = max(obshistid_list)
467 obshistid_bits = int(np.ceil(np.log(obshistid_max)/np.log(2.0)))
469 true_alert_dict = {}
470 obs_dict = {}
471 ignored_sqlite = 0 # count number of alerts written to sqlite, but not avro
472 for obs in self.obs_list:
473 obs_dict[obs.OpsimMetaData['obsHistID']] = obs
474 obshistid = obs.OpsimMetaData['obsHistID']
475 cat = TestAlertsTruthCat_avro(star_db, obs_metadata=obs)
476 cat.camera = obs_lsst_phosim.PhosimMapper().camera
478 for line in cat.iter_catalog():
479 if line[1] is None:
480 continue
482 dmag = line[2]
483 mag = line[3]
484 if (np.abs(dmag) > dmag_cutoff_avro and
485 mag <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]):
487 alertId = (line[0] << obshistid_bits) + obshistid
488 self.assertNotIn(alertId, true_alert_dict)
489 true_alert_dict[alertId] = {}
490 true_alert_dict[alertId]['chipName'] = line[1]
491 true_alert_dict[alertId]['dmag'] = dmag
492 true_alert_dict[alertId]['mag'] = mag
493 true_alert_dict[alertId]['ra'] = np.degrees(line[4])
494 true_alert_dict[alertId]['decl'] = np.degrees(line[5])
495 true_alert_dict[alertId]['xPix'] = line[6]
496 true_alert_dict[alertId]['yPix'] = line[7]
497 elif np.abs(dmag) > dmag_cutoff_sqlite:
498 ignored_sqlite += 1
500 self.assertGreater(len(true_alert_dict), 10)
502 self.assertGreater(ignored_sqlite, 50) # just make sure that some sqlite
503 # alerts were ignored by the more
504 # stringent avro cut
506 log_file_name = tempfile.mktemp(dir=self.alert_data_output_dir, suffix='log.txt')
507 alert_gen = AlertDataGenerator(testing=True)
509 alert_gen.subdivide_obs(self.obs_list, htmid_level=6)
511 for htmid in alert_gen.htmid_list:
512 alert_gen.alert_data_from_htmid(htmid, star_db,
513 photometry_class=TestAlertsVarCat_avro,
514 output_prefix='alert_test',
515 output_dir=self.alert_data_output_dir,
516 dmag_cutoff=dmag_cutoff_sqlite,
517 log_file_name=log_file_name)
519 obshistid_to_htmid = {}
520 for htmid in alert_gen.htmid_list:
521 for obs in alert_gen.obs_from_htmid(htmid):
522 obshistid = obs.OpsimMetaData['obsHistID']
523 if obshistid not in obshistid_to_htmid:
524 obshistid_to_htmid[obshistid] = []
525 obshistid_to_htmid[obshistid].append(htmid)
527 avro_gen = AvroAlertGenerator()
528 avro_gen.load_schema(os.path.join(getPackageDir('sims_catUtils'), 'tests', 'testData', 'avroSchema'))
529 sql_prefix_list = ['alert_test']
530 out_prefix = 'test_avro'
531 log_file_name = tempfile.mktemp(dir=self.avro_out_dir,
532 prefix='test_avro',
533 suffix='log.txt')
534 for obshistid in obshistid_list:
535 avro_gen.write_alerts(obshistid, self.alert_data_output_dir,
536 sql_prefix_list,
537 obshistid_to_htmid[obshistid],
538 self.avro_out_dir, out_prefix,
539 dmag_cutoff_avro, lock=None,
540 log_file_name=log_file_name)
542 list_of_avro_files = os.listdir(self.avro_out_dir)
543 self.assertGreater(len(list_of_avro_files), 2)
544 alert_ct = 0
545 dummy_sed = Sed()
546 bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
547 photParams = PhotometricParameters()
548 diasourceId_set = set()
549 for avro_file_name in list_of_avro_files:
550 if avro_file_name.endswith('log.txt'):
551 continue
552 full_name = os.path.join(self.avro_out_dir, avro_file_name)
553 with DataFileReader(open(full_name, 'rb'), DatumReader()) as data_reader:
554 for alert in data_reader:
555 alert_ct += 1
556 obshistid = alert['alertId'] >> 20
557 obs = obs_dict[obshistid]
558 uniqueId = alert['diaObject']['diaObjectId']
559 true_alert_id = (uniqueId << obshistid_bits) + obshistid
560 self.assertIn(true_alert_id, true_alert_dict)
561 self.assertEqual(alert['l1dbId'], uniqueId)
563 true_alert = true_alert_dict[true_alert_id]
565 diaSource = alert['diaSource']
566 self.assertAlmostEqual(diaSource['ra'], true_alert['ra'], 10)
567 self.assertAlmostEqual(diaSource['decl'], true_alert['decl'], 10)
568 self.assertAlmostEqual(diaSource['x'], true_alert['xPix'], 3)
569 self.assertAlmostEqual(diaSource['y'], true_alert['yPix'], 3)
570 self.assertAlmostEqual(diaSource['midPointTai'], obs.mjd.TAI, 4)
572 true_tot_flux = dummy_sed.fluxFromMag(true_alert['mag'])
573 true_q_mag = true_alert['mag'] - true_alert['dmag']
574 true_q_flux = dummy_sed.fluxFromMag(true_q_mag)
575 true_dflux = true_tot_flux - true_q_flux
576 self.assertAlmostEqual(diaSource['psFlux']/true_dflux, 1.0, 6)
577 self.assertAlmostEqual(diaSource['totFlux']/true_tot_flux, 1.0, 6)
578 self.assertAlmostEqual(diaSource['diffFlux']/true_dflux, 1.0, 6)
580 true_tot_snr, gamma = calcSNR_m5(true_alert['mag'], bp_dict[obs.bandpass],
581 obs.m5[obs.bandpass], photParams)
583 true_q_snr, gamma = calcSNR_m5(true_q_mag, bp_dict[obs.bandpass],
584 self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]],
585 photParams)
587 true_tot_err = true_tot_flux/true_tot_snr
588 true_q_err = true_q_flux/true_q_snr
589 true_diff_err = np.sqrt(true_tot_err**2 + true_q_err**2)
591 self.assertAlmostEqual(diaSource['snr']/np.abs(true_dflux/true_diff_err),
592 1.0, 6)
594 self.assertAlmostEqual(diaSource['totFluxErr']/true_tot_err, 1.0, 6)
595 self.assertAlmostEqual(diaSource['diffFluxErr']/true_diff_err, 1.0, 6)
597 chipnum = int(true_alert['chipName'].replace('R', '').replace('S', '').
598 replace(',', '').replace(':', '').replace(' ', ''))
600 true_ccdid = (chipnum*10**7)+obshistid
601 self.assertEqual(true_ccdid, diaSource['ccdVisitId'])
602 self.assertEqual(uniqueId, diaSource['diaObjectId'])
604 self.assertNotIn(diaSource['diaSourceId'], diasourceId_set)
605 diasourceId_set.add(diaSource['diaSourceId'])
607 diaObject = alert['diaObject']
608 obj_dex = (uniqueId//1024) - 1
609 self.assertAlmostEqual(0.001*diaObject['pmRa']/self.pmra_truth[obj_dex], 1.0, 5)
610 self.assertAlmostEqual(0.001*diaObject['pmDecl']/self.pmdec_truth[obj_dex], 1.0, 5)
611 self.assertAlmostEqual(0.001*diaObject['parallax']/self.px_truth[obj_dex], 1.0, 5)
613 (true_ra_base,
614 true_dec_base) = applyProperMotion(self.ra_truth[obj_dex],
615 self.dec_truth[obj_dex],
616 self.pmra_truth[obj_dex],
617 self.pmdec_truth[obj_dex],
618 self.px_truth[obj_dex],
619 self.vrad_truth[obj_dex],
620 mjd=ModifiedJulianDate(TAI=diaObject['radecTai']))
622 self.assertAlmostEqual(true_ra_base, diaObject['ra'], 7)
623 self.assertAlmostEqual(true_dec_base, diaObject['decl'], 7)
625 self.assertEqual(alert_ct, len(true_alert_dict))
628class MemoryTestClass(lsst.utils.tests.MemoryTestCase):
629 pass
632if __name__ == "__main__": 632 ↛ 633line 632 didn't jump to line 633, because the condition on line 632 was never true
633 lsst.utils.tests.init()
634 unittest.main()