Coverage for tests/test_avroAlertGenerator.py : 14%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import unittest
2import os
3import tempfile
4import shutil
5import gc
6import numpy as np
7import sqlite3
8import numbers
9import lsst.utils.tests
11from lsst.utils import getPackageDir
12from lsst.sims.utils.CodeUtilities import sims_clean_up
13from lsst.sims.catalogs.decorators import register_method
14from lsst.sims.catalogs.db import CatalogDBObject
15from lsst.sims.catUtils.utils import ObservationMetaDataGenerator
16from lsst.sims.catUtils.utils import AlertStellarVariabilityCatalog
17from lsst.sims.catUtils.utils import StellarAlertDBObjMixin
18from lsst.sims.utils import findHtmid
19from lsst.sims.utils import applyProperMotion, ModifiedJulianDate
20from lsst.sims.photUtils import Sed, calcSNR_m5, BandpassDict
21from lsst.sims.photUtils import PhotometricParameters
22from lsst.sims.coordUtils import lsst_camera
23from lsst.sims.coordUtils import chipNameFromPupilCoordsLSST
24from lsst.sims.catUtils.mixins import CameraCoordsLSST
25from lsst.sims.catUtils.mixins import AstrometryStars
26from lsst.sims.catUtils.mixins import Variability
27from lsst.sims.catalogs.definitions import InstanceCatalog
28from lsst.sims.catalogs.decorators import compound, cached
30from lsst.sims.catUtils.utils import AlertDataGenerator
31from lsst.sims.catUtils.utils import AvroAlertGenerator
33from lsst.sims.coordUtils import clean_up_lsst_camera
35_avro_is_installed = True
36try:
37 from avro.io import DatumReader
38 from avro.datafile import DataFileReader
39except ImportError:
40 _avro_is_installed = False
41 pass
44ROOT = os.path.abspath(os.path.dirname(__file__))
47def setup_module(module):
48 lsst.utils.tests.init()
51class StarAlertTestDBObj_avro(StellarAlertDBObjMixin, CatalogDBObject):
52 objid = 'star_alert'
53 tableid = 'stars'
54 idColKey = 'simobjid'
55 raColName = 'ra'
56 decColName = 'dec'
57 objectTypeId = 0
58 columns = [('raJ2000', 'ra*0.01745329252'),
59 ('decJ2000', 'dec*0.01745329252'),
60 ('parallax', 'px*0.01745329252/3600.0'),
61 ('properMotionRa', 'pmra*0.01745329252/3600.0'),
62 ('properMotionDec', 'pmdec*0.01745329252/3600.0'),
63 ('radialVelocity', 'vrad'),
64 ('variabilityParameters', 'varParamStr', str, 500)]
67class TestAlertsVarCatMixin_avro(object):
69 @register_method('avro_test')
70 def applyAlertTest(self, valid_dexes, params, expmjd, variability_cache=None):
71 if len(params) == 0:
72 return np.array([[], [], [], [], [], []])
74 if isinstance(expmjd, numbers.Number):
75 dMags_out = np.zeros((6, self.num_variable_obj(params)))
76 else:
77 dMags_out = np.zeros((6, self.num_variable_obj(params), len(expmjd)))
79 for i_star in range(self.num_variable_obj(params)):
80 if params['amp'][i_star] is not None:
81 dmags = params['amp'][i_star]*np.cos(params['per'][i_star]*expmjd)
82 for i_filter in range(6):
83 dMags_out[i_filter][i_star] = dmags
85 return dMags_out
88class TestAlertsVarCat_avro(TestAlertsVarCatMixin_avro, AlertStellarVariabilityCatalog):
89 pass
92class TestAlertsTruthCat_avro(TestAlertsVarCatMixin_avro, CameraCoordsLSST, AstrometryStars,
93 Variability, InstanceCatalog):
94 column_outputs = ['uniqueId', 'chipName', 'dmagAlert', 'magAlert',
95 'raICRS', 'decICRS', 'xPix', 'yPix']
97 @compound('delta_umag', 'delta_gmag', 'delta_rmag',
98 'delta_imag', 'delta_zmag', 'delta_ymag')
99 def get_TruthVariability(self):
100 return self.applyVariability(self.column_by_name('varParamStr'))
102 @cached
103 def get_dmagAlert(self):
104 return self.column_by_name('delta_%smag' % self.obs_metadata.bandpass)
106 @cached
107 def get_magAlert(self):
108 return self.column_by_name('%smag' % self.obs_metadata.bandpass) + \
109 self.column_by_name('dmagAlert')
112@unittest.skipIf(not _avro_is_installed, 'avro is not installed on this system')
113class AvroAlertTestCase(unittest.TestCase):
115 longMessage = True
117 @classmethod
118 def setUpClass(cls):
119 print('setting up %s' % sims_clean_up.targets)
121 # These represent the dimmest magnitudes at which objects
122 # are considered visible in each of the LSST filters
123 # (taken from Table 2 of the overview paper)
124 cls.obs_mag_cutoff = (23.68, 24.89, 24.43, 24.0, 24.45, 22.60)
126 cls.opsim_db = os.path.join(getPackageDir('sims_data'),
127 'OpSimData',
128 'opsimblitz1_1133_sqlite.db')
130 rng = np.random.RandomState(8123)
132 obs_gen = ObservationMetaDataGenerator(database=cls.opsim_db)
133 cls.obs_list = obs_gen.getObservationMetaData(night=(0, 2))
134 cls.obs_list = rng.choice(cls.obs_list, 10, replace=False)
135 fieldid_list = []
136 for obs in cls.obs_list:
137 fieldid_list.append(obs.OpsimMetaData['fieldID'])
139 # make sure we have selected observations such that the
140 # same field is revisited more than once
141 assert len(np.unique(fieldid_list)) < len(fieldid_list)
143 cls.input_dir = tempfile.mkdtemp(prefix='avroAlertGen',
144 dir=ROOT)
146 cls.star_db_name = tempfile.mktemp(prefix='avroAlertGen_star_db',
147 dir=cls.input_dir,
148 suffix='.db')
150 conn = sqlite3.connect(cls.star_db_name)
151 cursor = conn.cursor()
152 cursor.execute('''CREATE TABLE stars
153 (simobjid int, htmid int, ra real, dec real,
154 umag real, gmag real, rmag real,
155 imag real, zmag real, ymag real,
156 px real, pmra real, pmdec real,
157 vrad real, varParamStr text)''')
158 conn.commit()
160 n_stars = 10
162 cls.ra_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
163 cls.dec_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
164 u_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
165 g_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
166 r_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
167 i_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
168 z_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
169 y_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
170 cls.px_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
171 cls.pmra_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
172 cls.pmdec_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
173 cls.vrad_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
174 cls.amp_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
175 cls.period_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float)
177 id_offset = -n_stars
178 for obs in cls.obs_list:
179 id_offset += n_stars
180 ra_0 = obs.pointingRA
181 dec_0 = obs.pointingDec
182 rr = rng.random_sample(n_stars)
183 theta = rng.random_sample(n_stars)*2.0*np.pi
184 ra = ra_0 + rr*np.cos(theta)
185 dec = dec_0 + rr*np.sin(theta)
186 var_period = rng.random_sample(n_stars)*0.25
187 var_amp = rng.random_sample(n_stars)*1.0 + 0.01
189 subset = rng.randint(0, high=len(var_amp)-1, size=3)
190 var_amp[subset[:2]] = 0.0
191 var_amp[subset[-1]] = -1.0
193 umag = rng.random_sample(n_stars)*5.0 + 15.0
194 gmag = rng.random_sample(n_stars)*5.0 + 15.0
195 rmag = rng.random_sample(n_stars)*5.0 + 15.0
196 imag = rng.random_sample(n_stars)*5.0 + 15.0
197 zmag = rng.random_sample(n_stars)*5.0 + 15.0
198 ymag = rng.random_sample(n_stars)*5.0 + 15.0
199 px = rng.random_sample(n_stars)*0.1 # say it is arcsec
200 pmra = rng.random_sample(n_stars)*50.0+100.0 # say it is arcsec/yr
201 pmdec = rng.random_sample(n_stars)*50.0+100.0 # say it is arcsec/yr
202 vrad = rng.random_sample(n_stars)*600.0 - 300.0
204 subset = rng.randint(0, high=n_stars-1, size=3)
205 umag[subset] = 40.0
206 gmag[subset] = 40.0
207 rmag[subset] = 40.0
208 imag[subset] = 40.0
209 zmag[subset] = 40.0
210 ymag[subset] = 40.0
212 cls.ra_truth[id_offset:id_offset+n_stars] = np.round(ra, decimals=6)
213 cls.dec_truth[id_offset:id_offset+n_stars] = np.round(dec, decimals=6)
214 u_truth[id_offset:id_offset+n_stars] = np.round(umag, decimals=4)
215 g_truth[id_offset:id_offset+n_stars] = np.round(gmag, decimals=4)
216 r_truth[id_offset:id_offset+n_stars] = np.round(rmag, decimals=4)
217 i_truth[id_offset:id_offset+n_stars] = np.round(imag, decimals=4)
218 z_truth[id_offset:id_offset+n_stars] = np.round(zmag, decimals=4)
219 y_truth[id_offset:id_offset+n_stars] = np.round(ymag, decimals=4)
220 cls.px_truth[id_offset:id_offset+n_stars] = np.round(px, decimals=4)
221 cls.pmra_truth[id_offset:id_offset+n_stars] = np.round(pmra, decimals=4)
222 cls.pmdec_truth[id_offset:id_offset+n_stars] = np.round(pmdec, decimals=4)
223 cls.vrad_truth[id_offset:id_offset+n_stars] = np.round(vrad, decimals=4)
224 cls.amp_truth[id_offset:id_offset+n_stars] = np.round(var_amp, decimals=4)
225 cls.period_truth[id_offset:id_offset+n_stars] = np.round(var_period, decimals=4)
227 max_str_len = -1
229 for i_star in range(n_stars):
230 if var_amp[i_star] >= -0.1:
231 varParamStr = ('{"m":"avro_test", "p":{"amp":%.4f, "per": %.4f}}'
232 % (var_amp[i_star], var_period[i_star]))
233 else:
234 varParamStr = 'None'
236 if len(varParamStr) > max_str_len:
237 max_str_len = len(varParamStr)
239 htmid = findHtmid(ra[i_star], dec[i_star], 21)
241 query = ('''INSERT INTO stars VALUES(%d, %d, %.6f, %.6f,
242 %.4f, %.4f, %.4f, %.4f, %.4f, %.4f,
243 %.4f, %.4f, %.4f, %.4f, '%s')'''
244 % (i_star+id_offset+1, htmid, ra[i_star], dec[i_star],
245 umag[i_star], gmag[i_star], rmag[i_star],
246 imag[i_star], zmag[i_star], ymag[i_star],
247 px[i_star], pmra[i_star], pmdec[i_star],
248 vrad[i_star], varParamStr))
250 cursor.execute(query)
251 conn.commit()
252 conn.close()
254 cls.mag0_truth_dict = {}
255 cls.mag0_truth_dict[0] = u_truth
256 cls.mag0_truth_dict[1] = g_truth
257 cls.mag0_truth_dict[2] = r_truth
258 cls.mag0_truth_dict[3] = i_truth
259 cls.mag0_truth_dict[4] = z_truth
260 cls.mag0_truth_dict[5] = y_truth
261 assert max_str_len < 500 # make sure varParamStr fits in the space alotted to it
262 # in StarAlertTestDBObj_avro
264 @classmethod
265 def tearDownClass(cls):
266 sims_clean_up()
267 if os.path.exists(cls.star_db_name):
268 os.unlink(cls.star_db_name)
269 if os.path.exists(cls.input_dir):
270 shutil.rmtree(cls.input_dir)
272 clean_up_lsst_camera()
274 def setUp(self):
275 self.alert_data_output_dir = tempfile.mkdtemp(dir=ROOT, prefix='avro_gen_output')
276 self.avro_out_dir = tempfile.mkdtemp(dir=ROOT, prefix='avroTestOut')
278 def tearDown(self):
279 for file_name in os.listdir(self.alert_data_output_dir):
280 os.unlink(os.path.join(self.alert_data_output_dir, file_name))
281 shutil.rmtree(self.alert_data_output_dir)
283 for file_name in os.listdir(self.avro_out_dir):
284 os.unlink(os.path.join(self.avro_out_dir, file_name))
285 shutil.rmtree(self.avro_out_dir)
287 def test_avro_alert_generation(self):
288 dmag_cutoff = 0.005
289 mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5}
291 star_db = StarAlertTestDBObj_avro(database=self.star_db_name, driver='sqlite')
293 # assemble a dict of all of the alerts that need to be generated
295 obshistid_list = []
296 for obs in self.obs_list:
297 obshistid_list.append(obs.OpsimMetaData['obsHistID'])
298 obshistid_max = max(obshistid_list)
299 obshistid_bits = int(np.ceil(np.log(obshistid_max)/np.log(2.0)))
301 true_alert_dict = {}
302 obs_dict = {}
303 for obs in self.obs_list:
304 obs_dict[obs.OpsimMetaData['obsHistID']] = obs
305 obshistid = obs.OpsimMetaData['obsHistID']
306 cat = TestAlertsTruthCat_avro(star_db, obs_metadata=obs)
307 cat.camera = lsst_camera()
309 for line in cat.iter_catalog():
310 if line[1] is None:
311 continue
313 dmag = line[2]
314 mag = line[3]
315 if (np.abs(dmag) > dmag_cutoff and
316 mag <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]):
318 alertId = (line[0] << obshistid_bits) + obshistid
319 self.assertNotIn(alertId, true_alert_dict)
320 true_alert_dict[alertId] = {}
321 true_alert_dict[alertId]['chipName'] = line[1]
322 true_alert_dict[alertId]['dmag'] = dmag
323 true_alert_dict[alertId]['mag'] = mag
324 true_alert_dict[alertId]['ra'] = np.degrees(line[4])
325 true_alert_dict[alertId]['decl'] = np.degrees(line[5])
326 true_alert_dict[alertId]['xPix'] = line[6]
327 true_alert_dict[alertId]['yPix'] = line[7]
329 self.assertGreater(len(true_alert_dict), 10)
331 log_file_name = tempfile.mktemp(dir=self.alert_data_output_dir, suffix='log.txt')
332 alert_gen = AlertDataGenerator(testing=True)
334 alert_gen.subdivide_obs(self.obs_list, htmid_level=6)
336 for htmid in alert_gen.htmid_list:
337 alert_gen.alert_data_from_htmid(htmid, star_db,
338 photometry_class=TestAlertsVarCat_avro,
339 output_prefix='alert_test',
340 output_dir=self.alert_data_output_dir,
341 dmag_cutoff=dmag_cutoff,
342 log_file_name=log_file_name)
344 obshistid_to_htmid = {}
345 for htmid in alert_gen.htmid_list:
346 for obs in alert_gen.obs_from_htmid(htmid):
347 obshistid = obs.OpsimMetaData['obsHistID']
348 if obshistid not in obshistid_to_htmid:
349 obshistid_to_htmid[obshistid] = []
350 obshistid_to_htmid[obshistid].append(htmid)
352 avro_gen = AvroAlertGenerator()
353 avro_gen.load_schema(os.path.join(getPackageDir('sims_catUtils'), 'tests', 'testData', 'avroSchema'))
354 sql_prefix_list = ['alert_test']
355 out_prefix = 'test_avro'
356 log_file_name = tempfile.mktemp(dir=self.avro_out_dir,
357 prefix='test_avro',
358 suffix='log.txt')
359 for obshistid in obshistid_list:
360 avro_gen.write_alerts(obshistid, self.alert_data_output_dir,
361 sql_prefix_list,
362 obshistid_to_htmid[obshistid],
363 self.avro_out_dir, out_prefix,
364 dmag_cutoff, lock=None,
365 log_file_name=log_file_name)
367 list_of_avro_files = os.listdir(self.avro_out_dir)
368 self.assertGreater(len(list_of_avro_files), 2)
369 alert_ct = 0
370 dummy_sed = Sed()
371 bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
372 photParams = PhotometricParameters()
373 diasourceId_set = set()
374 for avro_file_name in list_of_avro_files:
375 if avro_file_name.endswith('log.txt'):
376 continue
377 full_name = os.path.join(self.avro_out_dir, avro_file_name)
378 with DataFileReader(open(full_name, 'rb'), DatumReader()) as data_reader:
379 for alert in data_reader:
380 alert_ct += 1
381 obshistid = alert['alertId'] >> 20
382 obs = obs_dict[obshistid]
383 uniqueId = alert['diaObject']['diaObjectId']
384 true_alert_id = (uniqueId << obshistid_bits) + obshistid
385 self.assertIn(true_alert_id, true_alert_dict)
386 self.assertEqual(alert['l1dbId'], uniqueId)
388 true_alert = true_alert_dict[true_alert_id]
390 diaSource = alert['diaSource']
391 self.assertAlmostEqual(diaSource['ra'], true_alert['ra'], 10)
392 self.assertAlmostEqual(diaSource['decl'], true_alert['decl'], 10)
393 self.assertAlmostEqual(diaSource['x'], true_alert['xPix'], 3)
394 self.assertAlmostEqual(diaSource['y'], true_alert['yPix'], 3)
395 self.assertAlmostEqual(diaSource['midPointTai'], obs.mjd.TAI, 4)
397 true_tot_flux = dummy_sed.fluxFromMag(true_alert['mag'])
398 true_q_mag = true_alert['mag'] - true_alert['dmag']
399 true_q_flux = dummy_sed.fluxFromMag(true_q_mag)
400 true_dflux = true_tot_flux - true_q_flux
401 self.assertAlmostEqual(diaSource['psFlux']/true_dflux, 1.0, 6)
402 self.assertAlmostEqual(diaSource['totFlux']/true_tot_flux, 1.0, 6)
403 self.assertAlmostEqual(diaSource['diffFlux']/true_dflux, 1.0, 6)
405 true_tot_snr, gamma = calcSNR_m5(true_alert['mag'], bp_dict[obs.bandpass],
406 obs.m5[obs.bandpass], photParams)
408 true_q_snr, gamma = calcSNR_m5(true_q_mag, bp_dict[obs.bandpass],
409 self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]],
410 photParams)
412 true_tot_err = true_tot_flux/true_tot_snr
413 true_q_err = true_q_flux/true_q_snr
414 true_diff_err = np.sqrt(true_tot_err**2 + true_q_err**2)
416 self.assertAlmostEqual(diaSource['snr']/np.abs(true_dflux/true_diff_err),
417 1.0, 6)
419 self.assertAlmostEqual(diaSource['totFluxErr']/true_tot_err, 1.0, 6)
420 self.assertAlmostEqual(diaSource['diffFluxErr']/true_diff_err, 1.0, 6)
422 chipnum = int(true_alert['chipName'].replace('R', '').replace('S', '').
423 replace(',', '').replace(':', '').replace(' ', ''))
425 true_ccdid = (chipnum*10**7)+obshistid
426 self.assertEqual(true_ccdid, diaSource['ccdVisitId'])
427 self.assertEqual(uniqueId, diaSource['diaObjectId'])
429 self.assertNotIn(diaSource['diaSourceId'], diasourceId_set)
430 diasourceId_set.add(diaSource['diaSourceId'])
432 diaObject = alert['diaObject']
433 obj_dex = (uniqueId//1024) - 1
434 self.assertAlmostEqual(0.001*diaObject['pmRa']/self.pmra_truth[obj_dex], 1.0, 5)
435 self.assertAlmostEqual(0.001*diaObject['pmDecl']/self.pmdec_truth[obj_dex], 1.0, 5)
436 self.assertAlmostEqual(0.001*diaObject['parallax']/self.px_truth[obj_dex], 1.0, 5)
438 (true_ra_base,
439 true_dec_base) = applyProperMotion(self.ra_truth[obj_dex],
440 self.dec_truth[obj_dex],
441 self.pmra_truth[obj_dex],
442 self.pmdec_truth[obj_dex],
443 self.px_truth[obj_dex],
444 self.vrad_truth[obj_dex],
445 mjd=ModifiedJulianDate(TAI=diaObject['radecTai']))
447 self.assertAlmostEqual(true_ra_base, diaObject['ra'], 7)
448 self.assertAlmostEqual(true_dec_base, diaObject['decl'], 7)
450 self.assertEqual(alert_ct, len(true_alert_dict))
452 def test_avro_alert_generation_diff_dmag(self):
453 """
454 Make sure everything works properly when the AlertDataGenerator
455 and the AvroAlertGenerator have different dmag thresholds
456 """
457 dmag_cutoff_sqlite = 0.005
458 dmag_cutoff_avro = 0.2
459 mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5}
461 star_db = StarAlertTestDBObj_avro(database=self.star_db_name, driver='sqlite')
463 # assemble a dict of all of the alerts that need to be generated
465 obshistid_list = []
466 for obs in self.obs_list:
467 obshistid_list.append(obs.OpsimMetaData['obsHistID'])
468 obshistid_max = max(obshistid_list)
469 obshistid_bits = int(np.ceil(np.log(obshistid_max)/np.log(2.0)))
471 true_alert_dict = {}
472 obs_dict = {}
473 ignored_sqlite = 0 # count number of alerts written to sqlite, but not avro
474 for obs in self.obs_list:
475 obs_dict[obs.OpsimMetaData['obsHistID']] = obs
476 obshistid = obs.OpsimMetaData['obsHistID']
477 cat = TestAlertsTruthCat_avro(star_db, obs_metadata=obs)
478 cat.camera = lsst_camera()
480 for line in cat.iter_catalog():
481 if line[1] is None:
482 continue
484 dmag = line[2]
485 mag = line[3]
486 if (np.abs(dmag) > dmag_cutoff_avro and
487 mag <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]):
489 alertId = (line[0] << obshistid_bits) + obshistid
490 self.assertNotIn(alertId, true_alert_dict)
491 true_alert_dict[alertId] = {}
492 true_alert_dict[alertId]['chipName'] = line[1]
493 true_alert_dict[alertId]['dmag'] = dmag
494 true_alert_dict[alertId]['mag'] = mag
495 true_alert_dict[alertId]['ra'] = np.degrees(line[4])
496 true_alert_dict[alertId]['decl'] = np.degrees(line[5])
497 true_alert_dict[alertId]['xPix'] = line[6]
498 true_alert_dict[alertId]['yPix'] = line[7]
499 elif np.abs(dmag) > dmag_cutoff_sqlite:
500 ignored_sqlite += 1
502 self.assertGreater(len(true_alert_dict), 10)
504 self.assertGreater(ignored_sqlite, 50) # just make sure that some sqlite
505 # alerts were ignored by the more
506 # stringent avro cut
508 log_file_name = tempfile.mktemp(dir=self.alert_data_output_dir, suffix='log.txt')
509 alert_gen = AlertDataGenerator(testing=True)
511 alert_gen.subdivide_obs(self.obs_list, htmid_level=6)
513 for htmid in alert_gen.htmid_list:
514 alert_gen.alert_data_from_htmid(htmid, star_db,
515 photometry_class=TestAlertsVarCat_avro,
516 output_prefix='alert_test',
517 output_dir=self.alert_data_output_dir,
518 dmag_cutoff=dmag_cutoff_sqlite,
519 log_file_name=log_file_name)
521 obshistid_to_htmid = {}
522 for htmid in alert_gen.htmid_list:
523 for obs in alert_gen.obs_from_htmid(htmid):
524 obshistid = obs.OpsimMetaData['obsHistID']
525 if obshistid not in obshistid_to_htmid:
526 obshistid_to_htmid[obshistid] = []
527 obshistid_to_htmid[obshistid].append(htmid)
529 avro_gen = AvroAlertGenerator()
530 avro_gen.load_schema(os.path.join(getPackageDir('sims_catUtils'), 'tests', 'testData', 'avroSchema'))
531 sql_prefix_list = ['alert_test']
532 out_prefix = 'test_avro'
533 log_file_name = tempfile.mktemp(dir=self.avro_out_dir,
534 prefix='test_avro',
535 suffix='log.txt')
536 for obshistid in obshistid_list:
537 avro_gen.write_alerts(obshistid, self.alert_data_output_dir,
538 sql_prefix_list,
539 obshistid_to_htmid[obshistid],
540 self.avro_out_dir, out_prefix,
541 dmag_cutoff_avro, lock=None,
542 log_file_name=log_file_name)
544 list_of_avro_files = os.listdir(self.avro_out_dir)
545 self.assertGreater(len(list_of_avro_files), 2)
546 alert_ct = 0
547 dummy_sed = Sed()
548 bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
549 photParams = PhotometricParameters()
550 diasourceId_set = set()
551 for avro_file_name in list_of_avro_files:
552 if avro_file_name.endswith('log.txt'):
553 continue
554 full_name = os.path.join(self.avro_out_dir, avro_file_name)
555 with DataFileReader(open(full_name, 'rb'), DatumReader()) as data_reader:
556 for alert in data_reader:
557 alert_ct += 1
558 obshistid = alert['alertId'] >> 20
559 obs = obs_dict[obshistid]
560 uniqueId = alert['diaObject']['diaObjectId']
561 true_alert_id = (uniqueId << obshistid_bits) + obshistid
562 self.assertIn(true_alert_id, true_alert_dict)
563 self.assertEqual(alert['l1dbId'], uniqueId)
565 true_alert = true_alert_dict[true_alert_id]
567 diaSource = alert['diaSource']
568 self.assertAlmostEqual(diaSource['ra'], true_alert['ra'], 10)
569 self.assertAlmostEqual(diaSource['decl'], true_alert['decl'], 10)
570 self.assertAlmostEqual(diaSource['x'], true_alert['xPix'], 3)
571 self.assertAlmostEqual(diaSource['y'], true_alert['yPix'], 3)
572 self.assertAlmostEqual(diaSource['midPointTai'], obs.mjd.TAI, 4)
574 true_tot_flux = dummy_sed.fluxFromMag(true_alert['mag'])
575 true_q_mag = true_alert['mag'] - true_alert['dmag']
576 true_q_flux = dummy_sed.fluxFromMag(true_q_mag)
577 true_dflux = true_tot_flux - true_q_flux
578 self.assertAlmostEqual(diaSource['psFlux']/true_dflux, 1.0, 6)
579 self.assertAlmostEqual(diaSource['totFlux']/true_tot_flux, 1.0, 6)
580 self.assertAlmostEqual(diaSource['diffFlux']/true_dflux, 1.0, 6)
582 true_tot_snr, gamma = calcSNR_m5(true_alert['mag'], bp_dict[obs.bandpass],
583 obs.m5[obs.bandpass], photParams)
585 true_q_snr, gamma = calcSNR_m5(true_q_mag, bp_dict[obs.bandpass],
586 self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]],
587 photParams)
589 true_tot_err = true_tot_flux/true_tot_snr
590 true_q_err = true_q_flux/true_q_snr
591 true_diff_err = np.sqrt(true_tot_err**2 + true_q_err**2)
593 self.assertAlmostEqual(diaSource['snr']/np.abs(true_dflux/true_diff_err),
594 1.0, 6)
596 self.assertAlmostEqual(diaSource['totFluxErr']/true_tot_err, 1.0, 6)
597 self.assertAlmostEqual(diaSource['diffFluxErr']/true_diff_err, 1.0, 6)
599 chipnum = int(true_alert['chipName'].replace('R', '').replace('S', '').
600 replace(',', '').replace(':', '').replace(' ', ''))
602 true_ccdid = (chipnum*10**7)+obshistid
603 self.assertEqual(true_ccdid, diaSource['ccdVisitId'])
604 self.assertEqual(uniqueId, diaSource['diaObjectId'])
606 self.assertNotIn(diaSource['diaSourceId'], diasourceId_set)
607 diasourceId_set.add(diaSource['diaSourceId'])
609 diaObject = alert['diaObject']
610 obj_dex = (uniqueId//1024) - 1
611 self.assertAlmostEqual(0.001*diaObject['pmRa']/self.pmra_truth[obj_dex], 1.0, 5)
612 self.assertAlmostEqual(0.001*diaObject['pmDecl']/self.pmdec_truth[obj_dex], 1.0, 5)
613 self.assertAlmostEqual(0.001*diaObject['parallax']/self.px_truth[obj_dex], 1.0, 5)
615 (true_ra_base,
616 true_dec_base) = applyProperMotion(self.ra_truth[obj_dex],
617 self.dec_truth[obj_dex],
618 self.pmra_truth[obj_dex],
619 self.pmdec_truth[obj_dex],
620 self.px_truth[obj_dex],
621 self.vrad_truth[obj_dex],
622 mjd=ModifiedJulianDate(TAI=diaObject['radecTai']))
624 self.assertAlmostEqual(true_ra_base, diaObject['ra'], 7)
625 self.assertAlmostEqual(true_dec_base, diaObject['decl'], 7)
627 self.assertEqual(alert_ct, len(true_alert_dict))
630class MemoryTestClass(lsst.utils.tests.MemoryTestCase):
631 pass
634if __name__ == "__main__": 634 ↛ 635line 634 didn't jump to line 635, because the condition on line 634 was never true
635 lsst.utils.tests.init()
636 unittest.main()