Coverage for python/lsst/sims/catUtils/utils/alertDataGenerator.py : 9%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import numpy as np
2import os
3import re
4import sqlite3
5from collections import OrderedDict
6import time
7import gc
8from lsst.utils import getPackageDir
9from lsst.sims.catalogs.definitions import InstanceCatalog
10from lsst.sims.utils import trixelFromHtmid, getAllTrixels
11from lsst.sims.utils import levelFromHtmid, halfSpaceFromRaDec
12from lsst.sims.utils import angularSeparation, ObservationMetaData
13from lsst.sims.utils import arcsecFromRadians
14from lsst.sims.catUtils.utils import _baseLightCurveCatalog
15from lsst.sims.utils import _pupilCoordsFromRaDec
16from lsst.sims.coordUtils import chipNameFromPupilCoordsLSST
17from lsst.sims.coordUtils import pixelCoordsFromPupilCoordsLSST
19from lsst.sims.catalogs.decorators import compound, cached
20from lsst.sims.photUtils import BandpassDict, Sed, calcSNR_m5
21from lsst.sims.photUtils import PhotometricParameters
22from lsst.sims.catUtils.mixins import VariabilityStars, AstrometryStars
23from lsst.sims.catUtils.mixins import VariabilityGalaxies, AstrometryGalaxies
24from lsst.sims.catUtils.mixins import CameraCoordsLSST, PhotometryBase
25from lsst.sims.catUtils.mixins import ParametrizedLightCurveMixin
26from lsst.sims.catUtils.mixins import create_variability_cache
28from lsst.sims.catUtils.baseCatalogModels import StarObj, GalaxyAgnObj
29from sqlalchemy.sql import text
30from lsst.sims.catalogs.db import ChunkIterator
32__all__ = ["AlertDataGenerator",
33 "AlertStellarVariabilityCatalog",
34 "AlertAgnVariabilityCatalog",
35 "_baseAlertCatalog",
36 "StellarAlertDBObj",
37 "AgnAlertDBObj",
38 "StellarAlertDBObjMixin"]
41class StellarAlertDBObjMixin(object):
42 """
43 Mimics StarObj class, except it allows you to directly query
44 all objects in a trixel specified by an htmid.
45 """
46 def query_columns_htmid(self, colnames=None, chunk_size=None,
47 constraint=None,
48 limit=None, htmid=None):
49 """Execute a query from the primary catsim database
51 Execute a query, taking advantage of the spherical geometry library and
52 htmid indexes on all catalog tables in the UW catsim database
54 **Parameters**
56 * colnames : list or None
57 a list of valid column names, corresponding to entries in the
58 `columns` class attribute. If not specified, all columns are
59 queried.
60 * chunk_size : int (optional)
61 if specified, then return an iterator object to query the database,
62 each time returning the next `chunk_size` elements. If not
63 specified, all matching results will be returned.
64 * constraint : str (optional)
65 a string which is interpreted as SQL and used as a predicate on the query
66 * limit : int (optional)
67 limits the number of rows returned by the query
68 * htmid is the htmid to be queried
70 **Returns**
72 * result : list or iterator
73 If chunk_size is not specified, then result is a list of all
74 items which match the specified query. If chunk_size is specified,
75 then result is an iterator over lists of the given size.
76 """
78 # find the minimum and maximum htmid
79 # (level=21 since that is what is implemented
80 # on fatboy) that we are asking for
81 #
82 # Note that sqlalchemy does not like np.int64
83 # as a data type
84 current_level = levelFromHtmid(htmid)
85 n_bits_off = 2*(21-current_level)
86 htmid_min = int(htmid << n_bits_off)
87 htmid_max = int((htmid+1) << n_bits_off)
89 query = self._get_column_query(colnames)
91 # add spatial constraints to query.
93 # Hint sql engine to seek on htmid
94 if not self.tableid.endswith('forceseek'):
95 query = query.with_hint(self.table, ' WITH(FORCESEEK)', 'mssql')
97 # SQL is not case sensitive but python is:
98 if 'htmID' in self.columnMap:
99 htmid_name = 'htmID'
100 elif 'htmid' in self.columnMap:
101 htmid_name = 'htmid'
102 else:
103 htmid_name = 'htmId'
105 # Range join on htmid ranges
106 query = query.filter(self.table.c[htmid_name].between(htmid_min, htmid_max))
108 if constraint is not None:
109 query = query.filter(text(constraint))
111 if limit is not None:
112 query = query.limit(limit)
114 return ChunkIterator(self, query, chunk_size)
117class StellarAlertDBObj(StellarAlertDBObjMixin, StarObj):
118 pass
121class AgnAlertDBObj(GalaxyAgnObj):
122 """
123 Mimics GalaxyAgnObj class, except it allows you to directly query
124 all objects in a trixel specified by an htmid.
125 """
127 columns = [('htmid', 0, np.int64),
128 ('galtileid', None, np.int64),
129 ('galid', None, str, 30),
130 ('componentra', 'agnra*PI()/180.'),
131 ('componentdec', 'agndec*PI()/180.'),
132 #: This is actually a problem with the stored procedure.
133 #: We need to be able to map columns other than
134 #: just ra/dec to raJ2000/decJ2000. This gets
135 #: important when we start perturbing the three galaxy components
136 ('raJ2000', 'ra'),
137 ('decJ2000', 'dec'),
138 ('magNorm', 'magnorm_agn'),
139 ('magNormAgn', 'magnorm_agn'),
140 ('sedFilename', 'sedname_agn', str, 40),
141 ('sedFilenameAgn', 'sedname_agn', str, 40),
142 ('variabilityParameters', 'varParamStr', str, 256),
143 ('lsst_u', 'u_ab'),
144 ('lsst_g', 'g_ab'),
145 ('lsst_r', 'r_ab'),
146 ('lsst_i', 'i_ab'),
147 ('lsst_z', 'z_ab'),
148 ('lsst_y', 'y_ab')]
150 def query_columns_htmid(self, colnames=None, chunk_size=None,
151 constraint=None,
152 limit=None, htmid=None):
153 """Execute a query from the primary catsim database
155 Execute a query, taking advantage of the spherical geometry library and
156 htmid indexes on all catalog tables in the UW catsim database
158 **Parameters**
160 * colnames : list or None
161 a list of valid column names, corresponding to entries in the
162 `columns` class attribute. If not specified, all columns are
163 queried.
164 * chunk_size : int (optional)
165 if specified, then return an iterator object to query the database,
166 each time returning the next `chunk_size` elements. If not
167 specified, all matching results will be returned.
168 * constraint : str (optional)
169 a string which is interpreted as SQL and used as a predicate on the query
170 * limit : int (optional)
171 limits the number of rows returned by the query
172 * htmid is the htmid to be queried
174 **Returns**
176 * result : list or iterator
177 If chunk_size is not specified, then result is a list of all
178 items which match the specified query. If chunk_size is specified,
179 then result is an iterator over lists of the given size.
180 """
182 trixel = trixelFromHtmid(htmid)
183 ra_0, dec_0 = trixel.get_center()
184 new_obs = ObservationMetaData(pointingRA=ra_0, pointingDec=dec_0, boundType='circle',
185 boundLength=trixel.get_radius()+0.1)
187 self._queried_trixel = trixel
188 self._queried_htmid_level = levelFromHtmid(htmid)
190 return self.query_columns(colnames=colnames, chunk_size=chunk_size,
191 obs_metadata=new_obs, constraint=constraint,
192 limit=limit)
194 def _final_pass(self, results):
195 """Modify the results of raJ2000 and decJ2000 to be in radians.
197 Also filter the results so that any objects outside of the
198 trixel specified in query_columns_htmid are returned with
199 htmid=0.
201 **Parameters**
203 * results : Structured array of results from query
205 **Returns**
207 * results : Modified structured array
209 """
211 if hasattr(self, '_queried_trixel'):
212 htmid = self._queried_trixel.htmid
213 htmid_21 = htmid << 2*(21-self._queried_htmid_level)
214 assert levelFromHtmid(htmid_21) == 21
215 contains_arr = self._queried_trixel.contains(results['raJ2000'], results['decJ2000'])
216 results['htmid'] = np.where(contains_arr, htmid_21, 0)
218 results['raJ2000'] = np.radians(results['raJ2000'])
219 results['decJ2000'] = np.radians(results['decJ2000'])
220 return results
223class _baseAlertCatalog(PhotometryBase, CameraCoordsLSST, _baseLightCurveCatalog):
225 column_outputs = ['htmid', 'uniqueId', 'raICRS', 'decICRS',
226 'flux', 'SNR', 'dflux',
227 'chipNum', 'xPix', 'yPix']
229 default_formats = {'f': '%.4g'}
231 default_columns = [('properMotionRa', 0.0, float),
232 ('properMotionDec', 0.0, float),
233 ('parallax', 0.0, float)]
235 def iter_catalog_chunks(self, chunk_size=None, query_cache=None, column_cache=None):
236 """
237 Returns an iterator over chunks of the catalog.
239 Parameters
240 ----------
241 chunk_size : int, optional, defaults to None
242 the number of rows to return from the database at a time. If None,
243 returns the entire database query in one chunk.
245 query_cache : iterator over database rows, optional, defaults to None
246 the result of calling db_obj.query_columns(). If query_cache is not
247 None, this method will iterate over the rows in query_cache and produce
248 an appropriate InstanceCatalog. DO NOT set to non-None values
249 unless you know what you are doing. It is an optional
250 input for those who want to repeatedly examine the same patch of sky
251 without actually querying the database over and over again. If it is set
252 to None (default), this method will handle the database query.
254 column_cache : a dict that will be copied over into the catalogs self._column_cache.
255 Should be left as None, unless you know what you are doing.
256 """
258 if query_cache is None:
259 # Call the original version of iter_catalog defined in the
260 # InstanceCatalog class. This version of iter_catalog includes
261 # the call to self.db_obj.query_columns, which the user would have
262 # used to generate query_cache.
263 for line in InstanceCatalog.iter_catalog_chunks(self, chunk_size=chunk_size):
264 yield line
265 else:
266 # Otherwise iterate over the query cache
267 transform_keys = list(self.transformations.keys())
268 for chunk in query_cache:
269 self._set_current_chunk(chunk, column_cache=column_cache)
270 chunk_cols = [self.transformations[col](self.column_by_name(col))
271 if col in transform_keys else
272 self.column_by_name(col)
273 for col in self.iter_column_names()]
275 if not hasattr(self, '_chunkColMap_output'):
277 self._chunkColMap_output = dict([(col, i)
278 for i, col in
279 enumerate(self.iter_column_names())])
281 yield chunk_cols, self._chunkColMap_output
283 self._column_cache = {}
284 self._current_chunk = None
286 @cached
287 def get_chipName(self):
288 if len(self.column_by_name('uniqueId')) == 0:
289 return np.array([])
290 raise RuntimeError("Should not get this far in get_chipName")
292 @compound('x_pupil', 'y_pupil')
293 def get_pupilFromSky(self):
294 if len(self.column_by_name('uniqueId')) == 0:
295 return np.array([[], []])
296 raise RuntimeError("Should not get this far in get_pupilFromSky")
298 @cached
299 def get_chipNum(self):
300 """
301 Concatenate the digits in 'R:i,j S:m,n' to make the chip number ijmn
302 """
303 chip_name = self.column_by_name('chipName')
304 return np.array([int(''.join(re.findall(r'\d+', name))) if name is not None else 0
305 for name in chip_name])
307 @compound('xPix', 'yPix')
308 def get_pixelCoordinates(self):
309 xpup = self.column_by_name('x_pupil')
310 ypup = self.column_by_name('y_pupil')
311 chip_name = self.column_by_name('chipName')
312 xpix, ypix = pixelCoordsFromPupilCoordsLSST(xpup, ypup, chipName=chip_name,
313 band=self.obs_metadata.bandpass,
314 includeDistortion=True)
315 return np.array([xpix, ypix])
317 @compound('delta_umag', 'delta_gmag', 'delta_rmag',
318 'delta_imag', 'delta_zmag', 'delta_ymag')
319 def get_deltaMagAvro(self):
320 ra = self.column_by_name('raJ2000')
321 if len(ra) == 0:
322 return np.array([[], [], [], [], [], []])
324 raise RuntimeError("Should not have gotten this far in delta mag getter")
326 @compound('lsst_u', 'lsst_g', 'lsst_r', 'lsst_i', 'lsst_z', 'lsst_y')
327 def get_lsst_magnitudes(self):
328 """
329 getter for LSST stellar magnitudes
330 """
332 magnitudes = np.array([self.column_by_name('quiescent_lsst_u'),
333 self.column_by_name('quiescent_lsst_g'),
334 self.column_by_name('quiescent_lsst_r'),
335 self.column_by_name('quiescent_lsst_i'),
336 self.column_by_name('quiescent_lsst_z'),
337 self.column_by_name('quiescent_lsst_y')])
339 delta = np.array([self.column_by_name('delta_umag'),
340 self.column_by_name('delta_gmag'),
341 self.column_by_name('delta_rmag'),
342 self.column_by_name('delta_imag'),
343 self.column_by_name('delta_zmag'),
344 self.column_by_name('delta_ymag')])
345 magnitudes += delta
347 return magnitudes
349 @compound('mag', 'dmag', 'quiescent_mag')
350 def get_alertPhotometry(self):
351 mag = self.column_by_name('lsst_%s' % self.obs_metadata.bandpass)
352 quiescent_mag = self.column_by_name('quiescent_lsst_%s' % self.obs_metadata.bandpass)
353 dmag = mag - quiescent_mag
355 return np.array([mag, dmag, quiescent_mag])
357 @compound('flux', 'dflux', 'SNR')
358 def get_alertFlux(self):
359 quiescent_mag = self.column_by_name('quiescent_mag')
360 mag = self.column_by_name('mag')
361 if not hasattr(self, '_dummy_sed'):
362 self._dummy_sed = Sed()
363 if not hasattr(self, 'lsstBandpassDict'):
364 self.lsstBandpassDict = BandpassDict.loadTotalBandpassesFromFiles()
365 if not hasattr(self, 'phot_params'):
366 self.phot_params = PhotometricParameters()
367 if not hasattr(self, '_gamma'):
368 self._gamma = None
370 quiescent_flux = self._dummy_sed.fluxFromMag(quiescent_mag)
371 flux = self._dummy_sed.fluxFromMag(mag)
372 dflux = flux - quiescent_flux
374 snr_tot, gamma = calcSNR_m5(mag, self.lsstBandpassDict[self.obs_metadata.bandpass],
375 self.obs_metadata.m5[self.obs_metadata.bandpass],
376 self.phot_params, gamma=self._gamma)
378 if self._gamma is None:
379 self._gamma = gamma
381 return np.array([flux, dflux, snr_tot])
384class AlertStellarVariabilityCatalog(_baseAlertCatalog,
385 VariabilityStars,
386 AstrometryStars):
388 @compound('quiescent_lsst_u', 'quiescent_lsst_g', 'quiescent_lsst_r',
389 'quiescent_lsst_i', 'quiescent_lsst_z', 'quiescent_lsst_y')
390 def get_quiescent_lsst_magnitudes(self):
391 return np.array([self.column_by_name('umag'), self.column_by_name('gmag'),
392 self.column_by_name('rmag'), self.column_by_name('imag'),
393 self.column_by_name('zmag'), self.column_by_name('ymag')])
396class AlertAgnVariabilityCatalog(_baseAlertCatalog,
397 VariabilityGalaxies,
398 AstrometryGalaxies):
400 @compound('quiescent_lsst_u', 'quiescent_lsst_g', 'quiescent_lsst_r',
401 'quiescent_lsst_i', 'quiescent_lsst_z', 'quiescent_lsst_y')
402 def get_quiescent_lsst_magnitudes(self):
403 return np.array([self.column_by_name('u_ab'), self.column_by_name('g_ab'),
404 self.column_by_name('r_ab'), self.column_by_name('i_ab'),
405 self.column_by_name('z_ab'), self.column_by_name('y_ab')])
408class AlertDataGenerator(object):
409 """
410 This class will read in astrophysical sources and variability
411 models from CatSim, observe them with a simulated OpSim
412 cadence, and write a series of sqlite files containing all
413 of the simulated observations that could trigger an alert.
415 In order to make this calculation as efficient as possible,
416 the class works by partitioning the sky according to the
417 Hierarchical Triangular Mesh (HTM) of
419 Kunszt P., Szalay A., Thakar A. (2006) in "Mining The Sky",
420 Banday A, Zaroubi S, Bartelmann M. eds.
421 ESO Astrophysics Symposia
422 https://www.researchgate.net/publication/226072008_The_Hierarchical_Triangular_Mesh
424 Szalay A. et al. (2007)
425 "Indexing the Sphere with the Hierarchical Triangular Mesh"
426 arXiv:cs/0701164
428 and simulating all of the observations in a given trixel (the
429 elementary unit of the HTM) at once. Accordintly, the outputs
430 produced by this class are files named like
432 prefix_NNNN_sqlite.db
434 where prefix is specified by theuser and NNNN is the htmid, the
435 unique identifying integer, corresponding to each simulated trixel.
437 The proper way to run this class is to instantiate it, run
438 subdivide_obs on a list of ObservationMetaData corresponding
439 to the OpSim pointings to be simulated, and then running
440 alert_data_from_htmid on each of the htmid in the class property
441 htmid_list. This last step can easily be parallelized using python's
442 multiprocessing module, with each process handling a different htmid.
444 The sqlite files produced by alert_data_from_htmid will each contain
445 four tables. They are as follows. Columns are listed below the
446 tables.
448 alert_data
449 ----------
450 uniqueId -- int -- a unique identifier for each astrophysical object
452 obshistId -- int -- a unique identifier for each OpSim pointing
454 xPix -- float -- the x pixel coordinate of the source on the focal
455 plane
457 yPix -- float -- the y pixel coordinate of the source on the focal
458 plane
460 dflux -- float -- the difference in flux between the source's current
461 flux and its quiescent flux (the source's quiescent flux can be found
462 in the quiescent_flux table). This is in units of Janskys.
464 snr -- float -- the signal to noise of the current detection of the
465 source (not the signal to noise of the source's detection in a simulated
466 difference image).
468 ra -- float -- the current RA of the source in degrees
470 dec -- float -- the current Declination of the source in degrees.
472 The alert_data table has a mult-column index on uniqueId and obshistId.
474 metadata
475 --------
476 obshistId -- int --- a unique identifier for each OpSim pointing
478 TAI -- float -- the International Atomic Time of the observation
479 as an MJD (in days)
481 band -- int -- denotes the filter used for the observation
482 (0=u, 1=g, 2=r, etc.)
484 The metadata table is indexed on obshistId
486 quiescent_flux
487 --------------
488 uniqueId -- int -- a unique identifier for each astrophysical source
490 band -- int -- an integer denoting each LSST filter (0=u, 1=g, 2=r, etc.)
492 flux -- float -- the flux of the source through the filter specified by
493 band (in units of Janskys)
495 snr -- float -- the signal to noise ratio of the source in the given
496 band with m5 taken from Table 2 of the overview paper (arXiv:0805.2366)
498 The quiescent_flux table has a multi-column index on uniqueId and band.
500 baseline_astrometry
501 -------------------
502 uniqueId -- int -- a unique identifier for each astrophysical source
504 TAI -- float -- the International Atomic Time of the baseline astrometric
505 measurements below as a MJD (in days)
507 ra -- float -- the RA of the source at TAI in degrees
509 dec -- float -- the Declination of the source at TAI in degrees
511 pmRA -- float -- the RA proper motion of the source in milliarcseconds/year
513 pmDec -- float -- the Declination proper motion of the source in
514 milliarcseconds/year
516 parallax -- float -- the parallax of the source in milliarcseconds
518 The baseline_astrometry table is indexed on uniqueId
520 """
522 def __init__(self,
523 testing=False):
524 """
525 Parameters
526 ----------
527 testing as a boolean that should only be True when running the unit tests.
528 If True, it prevents the AlertDataGenerator from pre-caching variability
529 models, which aids performance, but uses more memory than we want to use
530 in a unit test.
531 """
533 self._variability_cache = create_variability_cache()
534 self._stdout_lock = None
535 if not testing:
536 plm = ParametrizedLightCurveMixin()
537 plm.load_parametrized_light_curves(variability_cache = self._variability_cache)
538 self.bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
540 # This is a file that lists the maximum amplitude of variability
541 # for each of the Kepler-derived light curve models. It will be
542 # used by the stellar variability model to figure out which
543 # stars can be skipped because they will never vary above
544 # the alert-triggering threshold.
545 self._dmag_lookup_file_exists = True
546 self._dmag_lookup_file = os.path.join(getPackageDir('sims_data'),
547 'catUtilsData',
548 'kplr_dmag_171204.txt')
550 if not os.path.exists(self._dmag_lookup_file):
551 if not testing:
552 script_name = os.path.join(getPackageDir('sims_catUtils'), 'support_scripts',
553 'get_kepler_dmag.sh')
554 raise RuntimeError('\n%s does not exist; run the script\n\n%s\n\n' %
555 script_name)
556 else:
557 self._dmag_lookup_file_exists = False
559 def acquire_lock(self):
560 """
561 If running with multiprocessing, acquire
562 the lock.
563 """
564 if self._stdout_lock is not None:
565 self._stdout_lock.acquire()
567 def release_lock(self):
568 """
569 If running with multiprocessing, release
570 the lock.
571 """
572 if self._stdout_lock is not None:
573 self._stdout_lock.release()
575 def subdivide_obs(self, obs_list, htmid_level=6):
576 """
577 Take a list of ObservationMetaData and subdivide
578 them according to which trixels (see htmModule.py
579 in sims_utils) they intersect.
581 Parameters
582 ----------
583 obs_list is a list of ObservationMetaData
585 htmid_level is an int denoting the level of
586 the HTM mesh you want to use to tile the sky
587 (higher htmid_level corresponds to a finer
588 tiling). Default = 6
590 Returns
591 -------
592 Nothing.
594 After running this method, this AlertGenerator
595 will contain the following data.
597 - a list of the htmid of every trixel intersected
598 by the fields of view specified in obs_list. This
599 list is accessible from the property
600 AlertGenerator.htmid_list
602 - a dict mapping each htmid to the ObservationMetaData
603 from obs_list that intersect it. The method
604 AlertGenerator.obs_from_htmid(htmid) will return a
605 list of all of the ObservationMetaData that intersect
606 the trixel specified by htmid.
607 """
608 self._trixel_dict = getAllTrixels(htmid_level)
609 valid_htmid = []
610 for htmid in self._trixel_dict:
611 if levelFromHtmid(htmid) == htmid_level:
612 valid_htmid.append(htmid)
614 obs_list = np.array(obs_list)
615 self._obs_list = obs_list
616 obs_ra_list = []
617 obs_dec_list = []
618 halfspace_list = []
619 for obs in obs_list:
620 obs_ra_list.append(obs.pointingRA)
621 obs_dec_list.append(obs.pointingDec)
622 hs = halfSpaceFromRaDec(obs.pointingRA,
623 obs.pointingDec,
624 obs.boundLength)
625 halfspace_list.append(hs)
627 obs_ra_list = np.array(obs_ra_list)
628 obs_dec_list = np.array(obs_dec_list)
629 halfspace_list = np.array(halfspace_list)
631 self._htmid_dict = {}
632 self._htmid_list = []
633 n_obs_list = []
634 fov_radius = 1.75
635 for i_htmid, htmid in enumerate(valid_htmid):
636 trixel = self._trixel_dict[htmid]
637 ra_c, dec_c = trixel.get_center()
638 radius = trixel.get_radius()
639 obs_distance = angularSeparation(ra_c, dec_c, obs_ra_list, obs_dec_list)
640 valid_obs = np.where(obs_distance < radius + fov_radius)
641 if len(valid_obs[0]) > 0:
642 final_obs_list = []
643 for obs_dex in valid_obs[0]:
644 hs = halfspace_list[obs_dex]
645 obs = obs_list[obs_dex]
646 if hs.contains_trixel(trixel) != 'outside':
647 final_obs_list.append(obs_dex)
649 if len(final_obs_list) == 0:
650 continue
652 self._htmid_dict[htmid] = np.array(final_obs_list)
653 self._htmid_list.append(htmid)
654 n_obs_list.append(len(final_obs_list))
656 n_obs_list = np.array(n_obs_list)
657 self._htmid_list = np.array(self._htmid_list)
658 sorted_dex = np.argsort(-1.0*n_obs_list)
659 self._htmid_list = self._htmid_list[sorted_dex]
660 print('done subdividing obs list -- %d htmid' %
661 len(self._htmid_list))
663 @property
664 def htmid_list(self):
665 """
666 A list of the unique htmids corresponding to the trixels
667 that need to be queried to generate the alert data
668 """
669 return self._htmid_list
671 def n_obs(self, htmid):
672 """
673 Return the number of observations that intersect
674 the trixel specified by htmid.
676 Must run subdivide_obs in order for this method to
677 work.
678 """
679 return len(self._htmid_dict[htmid])
681 def obs_from_htmid(self, htmid):
682 """
683 Return a numpy array containing all of the ObservationMetaData
684 that intersect the trixel specified by htmid.
686 Must run subdivide_obs in order for this method to
687 work.
688 """
689 return self._obs_list[self._htmid_dict[htmid]]
691 def _output_alert_data(self, conn, data_cache):
692 """
693 Write a cache of alert data to the sqlite file currently open.
695 Parameters
696 ----------
697 conn is the connection to the sqlite file (already open)
699 data_cache is a dict containing all of the data to be written.
700 It will keyed on a string like 'i_j' where i is the obshistID
701 of an OpSim pointing and j is an arbitrary integer. That key
702 will lead to another dict keyed on the columns being output to
703 the sqlite file. The values of this second layer of dict are
704 numpy arrays.
706 Returns
707 -------
708 The number of rows written to the sqlite file
709 """
711 cursor = conn.cursor()
712 n_rows_0 = cursor.execute('SELECT COUNT(uniqueId) FROM alert_data').fetchall()
714 chunk_lengths = np.zeros(len(data_cache))
716 for i_cache_tag, cache_tag in enumerate(data_cache):
717 obshistid = int(cache_tag.split('_')[0])
718 n_obj = len(data_cache[cache_tag]['uniqueId'])
719 chunk_lengths[i_cache_tag] = n_obj
721 values = ((int(data_cache[cache_tag]['uniqueId'][i_obj]),
722 obshistid,
723 data_cache[cache_tag]['xPix'][i_obj],
724 data_cache[cache_tag]['yPix'][i_obj],
725 int(data_cache[cache_tag]['chipNum'][i_obj]),
726 data_cache[cache_tag]['dflux'][i_obj],
727 data_cache[cache_tag]['SNR'][i_obj],
728 np.degrees(data_cache[cache_tag]['raICRS'][i_obj]),
729 np.degrees(data_cache[cache_tag]['decICRS'][i_obj]))
730 for i_obj in range(n_obj))
731 cursor.executemany('INSERT INTO alert_data VALUES (?,?,?,?,?,?,?,?,?)', values)
732 conn.commit()
734 n_rows_1 = cursor.execute('SELECT COUNT(uniqueId) FROM alert_data').fetchall()
735 conn.commit()
736 n_written = (n_rows_1[0][0]-n_rows_0[0][0])
738 return n_written
740 def _filter_on_photometry_then_chip_name(self, chunk, column_query,
741 obs_valid_dex, expmjd_list,
742 photometry_catalog,
743 dmag_cutoff):
744 """
745 Determine which simulated observations are actually worth storing
746 by first figuring out which observations of which objects are
747 photometrically detectable and alert-worthy, then determining
748 which of those actually fall on an LSST detector.
750 Parameters
751 ----------
752 chunk is the output yielded from a CatSim ChunkIterator. It is
753 a numpy recarray representing one chunk_size query from the
754 underlying simulations database
756 column_query is a list of the columns that were queried from
757 the database
759 obs_valid_dex is a list of integers corresponding to indexes in
760 self._obs_list of the ObservationMetaData that are actually valid
761 for the trixel currently being simulated
763 expmjd_list is a numpy array of the TAI dates of ObservtionMetaData
764 represented by obs_valid_dex
766 photometry_catalog is an instantiation of the InstanceCatalog class
767 being used to calculate magnitudes for these variable sources.
769 Outputs
770 -------
771 chip_name_dict is a dict keyed on i_obs (which is the index of
772 an ObservationMetaData's position in obs_valid_dex, NOT its
773 position in self._obs_list). The values of chip_name_dict are
774 tuples containing:
775 - a list of the names of the detectors that objects from chunk
776 landed on (including Nones for those objects that did not land
777 on any detector)
779 - a list of the xPupil coords for every object in chunk
781 - a list of the yPupil coords for every object in chunk
783 - a list of the indexes in chunk of those objects which actually
784 landed on a detector
786 dmag_arr is a numpy array of the delta_magnitudes of every object
787 in chunk. dmag_arr[11][3][4] is the delta_magnitude of chunk[4]
788 in the 3rd band (i.e. the i band) at TAI = expmjd[11].
790 dmag_arr_transpose is dmag_arr with the time and object columns
791 transposed so that dmag_arr_transpose[4][3][11] == dmag_arr[11][3][4].
793 time_arr is an array of integers with shape == (len(chunk), len(obs_valid_dex)).
794 A -1 in time_arr means that that combination of object and observation did
795 not yield a valid observation. A +1 means that the object and observation
796 combination are valid.
797 """
799 ######################################################
800 # Calculate the delta_magnitude for all of the sources
801 #
802 photometry_catalog._set_current_chunk(chunk)
803 dmag_arr = photometry_catalog.applyVariability(chunk['varParamStr'],
804 variability_cache=self._variability_cache,
805 expmjd=expmjd_list).transpose((2, 0, 1))
807 dmag_arr_transpose = dmag_arr.transpose(2, 1, 0)
809 n_raw_obj = len(chunk)
810 photometrically_valid = -1*np.ones(n_raw_obj, dtype=int)
811 for i_obj in range(n_raw_obj):
812 keep_it = False
813 for i_filter in range(6):
814 if np.abs(dmag_arr_transpose[i_obj][i_filter]).max() >= dmag_cutoff:
815 keep_it = True
816 break
817 if keep_it:
818 photometrically_valid[i_obj] = 1
820 photometrically_valid = np.where(photometrically_valid >= 0)
822 if 'properMotionRa'in column_query:
823 pmra = chunk['properMotionRa'][photometrically_valid]
824 pmdec = chunk['properMotionDec'][photometrically_valid]
825 px = chunk['parallax'][photometrically_valid]
826 vrad = chunk['radialVelocity'][photometrically_valid]
827 else:
828 pmra = None
829 pmdec = None
830 px = None
831 vrad = None
833 ###################################################################
834 # Figure out which sources actually land on an LSST detector during
835 # the observations in question
836 #
837 chip_name_dict = {}
839 # time_arr will keep track of which objects appear in which observations;
840 # 1 means the object appears; -1 means it does not
841 time_arr_transpose = -1*np.ones((len(obs_valid_dex), len(chunk['raJ2000'])),
842 dtype=int)
844 for i_obs, obs_dex in enumerate(obs_valid_dex):
845 obs = self._obs_list[obs_dex]
846 chip_name_list = np.array([None]*n_raw_obj)
847 xpup_list = np.zeros(n_raw_obj, dtype=float)
848 ypup_list = np.zeros(n_raw_obj, dtype=float)
849 chip_int_arr = -1*np.ones(len(chip_name_list), dtype=int)
851 if len(photometrically_valid[0]) > 0:
852 xpup_list_val, ypup_list_val = _pupilCoordsFromRaDec(chunk['raJ2000'][photometrically_valid],
853 chunk['decJ2000'][photometrically_valid],
854 pm_ra=pmra, pm_dec=pmdec,
855 parallax=px, v_rad=vrad,
856 obs_metadata=obs)
858 xpup_list[photometrically_valid] = xpup_list_val
859 ypup_list[photometrically_valid] = ypup_list_val
861 chip_name_list[photometrically_valid] = chipNameFromPupilCoordsLSST(xpup_list_val,
862 ypup_list_val)
864 for i_chip, name in enumerate(chip_name_list):
865 if name is not None:
866 chip_int_arr[i_chip] = 1
868 valid_obj = np.where(chip_int_arr > 0)
869 time_arr_transpose[i_obs][valid_obj] = 1
871 chip_name_dict[i_obs] = (chip_name_list,
872 xpup_list,
873 ypup_list,
874 valid_obj)
876 time_arr = time_arr_transpose.transpose()
877 assert len(chip_name_dict) == len(obs_valid_dex)
879 return chip_name_dict, dmag_arr, dmag_arr_transpose, time_arr
881 def alert_data_from_htmid(self, htmid, dbobj,
882 dmag_cutoff=0.005,
883 chunk_size=1000, write_every=10000,
884 output_dir='.', output_prefix='',
885 log_file_name=None,
886 photometry_class=None,
887 chunk_cutoff=-1,
888 lock=None):
890 """
891 Generate an sqlite file with all of the alert data for a given
892 trixel.
894 Parameters
895 ----------
896 htmid is an integer denoting the trixel from self.htmid_list that should
897 be simulated
899 dbobj is a CatalogDBObject connecting to the data underlying the simulation
901 dmag_cutoff indicates the minimum change magnitude needed to trigger a
902 simulated alert
904 chunk_size denotes the number of objects to query from the database and
905 process at one time
907 write_every indicates how often to write to the sqlite file (i.e. the
908 code will pause the simulation process and write to the sqlite file
909 when it has accumulated this many valid observations)
911 output_dir is the directory in which to create the sqlite file
913 output_prefix is the prefix of the sqlite file's name
915 log_file_name is the name of a text file where progress will be written
917 photometry_class is a InstanceCatalog class (not an instantiation) that
918 contains the methods for calculating the photometry associated with the
919 simulated alerts (see AlertStellarVariabilityCatalog and
920 AlertAgnVariabilityCatalog in this module)
922 chunk_cutoff is an optional int; stop the simulation after this many
923 chunks have been processed. This is for testing purposes.
924 If chunk_cutoff == -1, the code will process all of the astrophysical
925 objects in the trixel.
927 lock is a multiprocessing.Lock() for use if running multiple
928 instances of alert_data_from_htmid. This will prevent multiple processes
929 from writing to the log file or stdout simultaneously.
930 """
932 htmid_level = levelFromHtmid(htmid)
933 if log_file_name is None:
934 raise RuntimeError('must specify log_file_name')
936 if ('_PARAMETRIZED_LC_DMAG_LOOKUP' not in self._variability_cache and
937 self._dmag_lookup_file_exists):
939 self._variability_cache['_PARAMETRIZED_LC_DMAG_CUTOFF'] = dmag_cutoff
940 self._variability_cache['_PARAMETRIZED_LC_DMAG_LOOKUP'] = {}
942 with open(self._dmag_lookup_file, 'r') as in_file:
943 for line in in_file:
944 if line[0] == '#':
945 continue
946 params = line.split()
947 self._variability_cache['_PARAMETRIZED_LC_DMAG_LOOKUP'][int(params[0])] = float(params[1])
949 self._stdout_lock = lock
950 this_pid = os.getpid()
952 t_start = time.time() # so that we can get a sense of how long the full
953 # simulation will take
955 desired_columns = []
956 desired_columns.append('simobjid')
957 desired_columns.append('variabilityParameters')
958 desired_columns.append('varParamStr')
959 desired_columns.append('raJ2000')
960 desired_columns.append('decJ2000')
961 desired_columns.append('properMotionRa')
962 desired_columns.append('properMotionDec')
963 desired_columns.append('parallax')
964 desired_columns.append('radialVelocity')
965 desired_columns.append('ebv')
966 desired_columns.append('redshift')
967 desired_columns.append('htmid')
969 if 'umag' in dbobj.columnMap:
970 desired_columns.append('umag')
971 desired_columns.append('gmag')
972 desired_columns.append('rmag')
973 desired_columns.append('imag')
974 desired_columns.append('zmag')
975 desired_columns.append('ymag')
976 elif 'u_ab' in dbobj.columnMap:
977 desired_columns.append('u_ab')
978 desired_columns.append('g_ab')
979 desired_columns.append('r_ab')
980 desired_columns.append('i_ab')
981 desired_columns.append('z_ab')
982 desired_columns.append('y_ab')
983 else:
984 raise RuntimeError('Not sure what quiescent '
985 'LSST magnitudes are called '
986 'in this CatalogDBObject')
988 if photometry_class is None:
989 raise RuntimeError('Must specify photometry_class')
991 if os.path.exists(output_dir) and not os.path.isdir(output_dir):
992 raise RuntimeError('%s is not a dir' % output_dir)
993 if not os.path.exists(output_dir):
994 os.mkdir(output_dir)
996 dummy_sed = Sed()
998 # a dummy call to make sure that the initialization
999 # is done before we attempt to parallelize calls
1000 # to chipNameFromRaDecLSST
1001 chipNameFromPupilCoordsLSST(0.0, 0.0)
1003 mag_names = ('u', 'g', 'r', 'i', 'z', 'y')
1005 phot_params = PhotometricParameters()
1007 # from Table 2 of the overview paper
1008 obs_mag_cutoff = (23.68, 24.89, 24.43, 24.0, 24.45, 22.60)
1010 gamma_template = {}
1011 for i_filter in range(6):
1012 gamma_template[i_filter] = None
1014 obs_valid_dex = self._htmid_dict[htmid]
1015 print('n valid obs %d' % len(obs_valid_dex))
1017 cat_list = []
1018 expmjd_list = []
1019 mag_name_to_int = {'u': 0, 'g': 1, 'r': 2,
1020 'i': 3, 'z': 4, 'y': 5}
1021 for obs_dex in obs_valid_dex:
1022 obs = self._obs_list[obs_dex]
1023 cat = photometry_class(dbobj, obs_metadata=obs)
1024 cat.lsstBandpassDict = self.bp_dict
1025 cat_list.append(cat)
1026 expmjd_list.append(obs.mjd.TAI)
1028 expmjd_list = np.array(expmjd_list)
1029 cat_list = np.array(cat_list)
1030 sorted_dex = np.argsort(expmjd_list)
1032 expmjd_list = expmjd_list[sorted_dex]
1033 cat_list = cat_list[sorted_dex]
1034 obs_valid_dex = obs_valid_dex[sorted_dex]
1036 available_columns = list(dbobj.columnMap.keys())
1037 column_query = []
1038 for col in desired_columns:
1039 if col in available_columns:
1040 column_query.append(col)
1042 n_bits_off = 2*(21-htmid_level)
1044 data_iter = dbobj.query_columns_htmid(colnames=column_query,
1045 htmid=htmid,
1046 chunk_size=chunk_size)
1048 photometry_catalog = photometry_class(dbobj, self._obs_list[obs_valid_dex[0]],
1049 column_outputs=['lsst_u',
1050 'lsst_g',
1051 'lsst_r',
1052 'lsst_i',
1053 'lsst_z',
1054 'lsst_y'])
1056 i_chunk = 0
1058 output_data_cache = {}
1059 n_rows_cached = 0
1061 n_obj = 0
1062 n_actual_obj = 0
1063 n_time_last = 0
1064 n_rows = 0
1066 t_before_obj = time.time() # so that we can get a sense of how long the
1067 # "iterating over astrophysical objects" part
1068 # of the simulation will take
1070 db_name = os.path.join(output_dir, '%s_%d_sqlite.db' % (output_prefix, htmid))
1071 with sqlite3.connect(db_name, isolation_level='EXCLUSIVE') as conn:
1072 creation_cmd = '''CREATE TABLE alert_data
1073 (uniqueId int, obshistId int, xPix float, yPix float,
1074 chipNum int, dflux float, snr float, ra float, dec float)'''
1076 cursor = conn.cursor()
1077 cursor.execute('PRAGMA journal_mode=WAL;')
1078 conn.commit()
1079 cursor.execute(creation_cmd)
1080 conn.commit()
1082 creation_cmd = '''CREATE TABLE metadata
1083 (obshistId int, TAI float, band int)'''
1084 cursor.execute(creation_cmd)
1085 conn.commit()
1087 for obs_dex in obs_valid_dex:
1088 obs = self._obs_list[obs_dex]
1089 cmd = '''INSERT INTO metadata
1090 VALUES(%d, %.5f, %d)''' % (obs.OpsimMetaData['obsHistID'],
1091 obs.mjd.TAI,
1092 mag_name_to_int[obs.bandpass])
1094 cursor.execute(cmd)
1095 conn.commit()
1097 creation_cmd = '''CREATE TABLE quiescent_flux
1098 (uniqueId int, band int, flux float, snr float)'''
1100 cursor.execute(creation_cmd)
1101 conn.commit()
1103 creation_cmd = '''CREATE TABLE baseline_astrometry
1104 (uniqueId int, ra real, dec real, pmRA real,
1105 pmDec real, parallax real, TAI real)'''
1107 cursor.execute(creation_cmd)
1108 conn.commit()
1110 for chunk in data_iter:
1111 n_raw_obj = len(chunk)
1112 i_chunk += 1
1114 if chunk_cutoff > 0 and i_chunk >= chunk_cutoff:
1115 break
1117 n_time_last = 0
1118 # filter the chunk so that we are only considering sources that are in
1119 # the trixel being considered
1120 reduced_htmid = chunk['htmid'] >> n_bits_off
1122 valid_htmid = np.where(reduced_htmid == htmid)
1123 if len(valid_htmid[0]) == 0:
1124 continue
1125 n_htmid_trim = n_raw_obj-len(valid_htmid[0])
1126 chunk = chunk[valid_htmid]
1127 n_obj += len(valid_htmid[0])
1129 (chip_name_dict,
1130 dmag_arr,
1131 dmag_arr_transpose,
1132 time_arr) = self._filter_on_photometry_then_chip_name(chunk, column_query,
1133 obs_valid_dex,
1134 expmjd_list,
1135 photometry_catalog,
1136 dmag_cutoff)
1138 q_f_dict = {}
1139 q_m_dict = {}
1141 q_m_dict[0] = photometry_catalog.column_by_name('quiescent_lsst_u')
1142 q_m_dict[1] = photometry_catalog.column_by_name('quiescent_lsst_g')
1143 q_m_dict[2] = photometry_catalog.column_by_name('quiescent_lsst_r')
1144 q_m_dict[3] = photometry_catalog.column_by_name('quiescent_lsst_i')
1145 q_m_dict[4] = photometry_catalog.column_by_name('quiescent_lsst_z')
1146 q_m_dict[5] = photometry_catalog.column_by_name('quiescent_lsst_y')
1148 q_f_dict[0] = dummy_sed.fluxFromMag(q_m_dict[0])
1149 q_f_dict[1] = dummy_sed.fluxFromMag(q_m_dict[1])
1150 q_f_dict[2] = dummy_sed.fluxFromMag(q_m_dict[2])
1151 q_f_dict[3] = dummy_sed.fluxFromMag(q_m_dict[3])
1152 q_f_dict[4] = dummy_sed.fluxFromMag(q_m_dict[4])
1153 q_f_dict[5] = dummy_sed.fluxFromMag(q_m_dict[5])
1155 q_pmra = 1000.0*arcsecFromRadians(photometry_catalog.column_by_name('properMotionRa'))
1156 q_pmdec = 1000.0*arcsecFromRadians(photometry_catalog.column_by_name('properMotionDec'))
1157 q_parallax = 1000.0*arcsecFromRadians(photometry_catalog.column_by_name('parallax'))
1158 q_ra = np.degrees(photometry_catalog.column_by_name('raICRS'))
1159 q_dec = np.degrees(photometry_catalog.column_by_name('decICRS'))
1160 q_tai = photometry_catalog.obs_metadata.mjd.TAI
1162 q_snr_dict = {}
1163 for i_filter in range(6):
1165 snr_template, local_gamma = calcSNR_m5(q_m_dict[i_filter],
1166 self.bp_dict[mag_names[i_filter]],
1167 obs_mag_cutoff[i_filter],
1168 phot_params, gamma=gamma_template[i_filter])
1169 q_snr_dict[i_filter] = snr_template
1170 gamma_template[i_filter] = local_gamma
1172 unq = photometry_catalog.column_by_name('uniqueId')
1174 try:
1175 assert dmag_arr_transpose.shape == (len(chunk), len(mag_names), len(expmjd_list))
1176 except AssertionError:
1177 print('dmag_arr_transpose_shape %s' % str(dmag_arr_transpose.shape))
1178 print('should be (%d, %d, %d)' % (len(chunk), len(mag_names), len(expmjd_list)))
1179 raise
1181 # only include those sources for which np.abs(delta_mag) >= dmag_cutoff
1182 # at some point in their history (note that delta_mag is defined with
1183 # respect to the quiescent magnitude)
1184 #
1185 # also demand that the magnitude at some point is less than obs_mag_cutoff
1186 #
1187 # This is different from the dmag>dmag_cutoff check done in
1188 #
1189 # self._filter_on_photometry_then_chip_name()
1190 #
1191 # Now we have information about which observations actually detected
1192 # each object (in self._filter_on_photometry_then_chip_name(),
1193 # we assumed that every object was detected at every time step).
1195 photometrically_valid_obj = []
1196 for i_obj in range(len(chunk)):
1197 keep_it = False
1198 valid_times = np.where(time_arr[i_obj] > 0)
1199 if len(valid_times[0]) == 0:
1200 continue
1201 for i_filter in range(len(mag_names)):
1202 if np.abs(dmag_arr_transpose[i_obj][i_filter][valid_times]).max() > dmag_cutoff:
1203 dmag_min = dmag_arr_transpose[i_obj][i_filter][valid_times].min()
1204 if q_m_dict[i_filter][i_obj] + dmag_min <= obs_mag_cutoff[i_filter]:
1205 keep_it = True
1206 break
1207 if keep_it:
1208 photometrically_valid_obj.append(i_obj)
1209 photometrically_valid_obj = np.array(photometrically_valid_obj)
1211 del dmag_arr_transpose
1212 gc.collect()
1214 if np.abs(dmag_arr).max() < dmag_cutoff:
1215 continue
1217 completely_valid = np.zeros(len(chunk), dtype=int)
1219 ############################
1220 # Process and output sources
1221 #
1222 for i_obs, obs_dex in enumerate(obs_valid_dex):
1223 obs = self._obs_list[obs_dex]
1224 obshistid = obs.OpsimMetaData['obsHistID']
1226 obs_mag = obs.bandpass
1227 actual_i_mag = mag_name_to_int[obs_mag]
1228 assert mag_names[actual_i_mag] == obs_mag
1230 # only include those sources which fall on a detector for this pointing
1231 valid_chip_name, valid_xpup, valid_ypup, chip_valid_obj = chip_name_dict[i_obs]
1233 actually_valid_obj = np.intersect1d(photometrically_valid_obj, chip_valid_obj)
1234 if len(actually_valid_obj) == 0:
1235 continue
1237 try:
1238 completely_valid[actually_valid_obj] += 1
1239 except:
1240 print('failed')
1241 print(actually_valid_obj)
1242 print(completely_valid)
1243 raise
1245 valid_sources = chunk[actually_valid_obj]
1246 local_column_cache = {}
1247 local_column_cache['deltaMagAvro'] = OrderedDict([('delta_%smag' % mag_names[i_mag],
1248 dmag_arr[i_obs][i_mag][actually_valid_obj])
1249 for i_mag in range(len(mag_names))])
1251 local_column_cache['chipName'] = valid_chip_name[actually_valid_obj]
1252 local_column_cache['pupilFromSky'] = OrderedDict([('x_pupil', valid_xpup[actually_valid_obj]),
1253 ('y_pupil', valid_ypup[actually_valid_obj])])
1255 cat = cat_list[i_obs]
1256 i_valid_chunk = 0
1257 for valid_chunk, chunk_map in cat.iter_catalog_chunks(query_cache=[valid_sources],
1258 column_cache=local_column_cache):
1259 i_valid_chunk += 1
1260 assert i_valid_chunk == 1
1261 n_time_last += len(valid_chunk[0])
1262 length_of_chunk = len(valid_chunk[chunk_map['uniqueId']])
1263 cache_tag = '%d_%d' % (obshistid, i_chunk)
1264 output_data_cache[cache_tag] = {}
1266 for col_name in ('uniqueId', 'raICRS', 'decICRS', 'flux', 'dflux', 'SNR',
1267 'chipNum', 'xPix', 'yPix'):
1269 output_data_cache[cache_tag][col_name] = valid_chunk[chunk_map[col_name]]
1271 n_rows_cached += length_of_chunk
1273 completely_valid = np.where(completely_valid > 0)
1274 for i_filter in range(6):
1275 values = ((int(unq[completely_valid][i_q]),
1276 i_filter,
1277 q_f_dict[i_filter][completely_valid][i_q],
1278 q_snr_dict[i_filter][completely_valid][i_q])
1279 for i_q in range(len(completely_valid[0])))
1280 cursor.executemany('INSERT INTO quiescent_flux VALUES (?,?,?,?)', values)
1281 conn.commit()
1283 values = ((int(unq[completely_valid][i_q]),
1284 q_ra[completely_valid][i_q],
1285 q_dec[completely_valid][i_q],
1286 q_pmra[completely_valid][i_q],
1287 q_pmdec[completely_valid][i_q],
1288 q_parallax[completely_valid][i_q],
1289 q_tai)
1290 for i_q in range(len(completely_valid[0])))
1292 cursor.executemany('INSERT INTO baseline_astrometry VALUES (?,?,?,?,?,?,?)', values)
1294 if n_rows_cached >= write_every:
1295 self.acquire_lock()
1296 with open(log_file_name, 'a') as out_file:
1297 out_file.write('%d is writing \n' % os.getpid())
1299 print('%d is writing' % os.getpid())
1301 self.release_lock()
1303 n_rows += self._output_alert_data(conn, output_data_cache)
1304 output_data_cache = {}
1305 n_rows_cached = 0
1307 if n_rows > 0:
1308 self.acquire_lock()
1309 with open(log_file_name, 'a') as out_file:
1310 elapsed = (time.time()-t_before_obj)/3600.0
1311 elapsed_per = elapsed/n_rows
1312 rows_per_chunk = float(n_rows)/float(i_chunk)
1313 total_projection = 1000.0*rows_per_chunk*elapsed_per
1314 out_file.write('\n %d n_obj %d %d trimmed %d\n' %
1315 (this_pid, n_obj, n_actual_obj, n_htmid_trim))
1316 out_file.write(' elapsed %.2e hrs per row %.2e total %2e\n' %
1317 (elapsed, elapsed_per, total_projection))
1318 out_file.write(' n_time_last %d; rows %d\n' % (n_time_last, n_rows))
1320 out_file.write('%d is done writing\n' % os.getpid())
1322 print('\n %d n_obj %d %d trimmed %d' %
1323 (this_pid, n_obj, n_actual_obj, n_htmid_trim))
1324 print(' elapsed %.2e hrs per row %.2e total %2e' %
1325 (elapsed, elapsed_per, total_projection))
1326 print(' n_time_last %d; rows %d\n' % (n_time_last, n_rows))
1327 print('%d is done writing' % os.getpid())
1329 self.release_lock()
1331 if len(output_data_cache) > 0:
1332 n_rows += self._output_alert_data(conn, output_data_cache)
1333 output_data_cache = {}
1335 print('htmid %d that took %.2e hours; n_obj %d n_rows %d' %
1336 (htmid, (time.time()-t_start)/3600.0, n_obj, n_rows))
1338 self.acquire_lock()
1339 print("INDEXING %d" % htmid)
1340 self.release_lock()
1342 cursor.execute('CREATE INDEX unq_obs ON alert_data (uniqueId, obshistId)')
1343 cursor.execute('CREATE INDEX unq_flux ON quiescent_flux (uniqueId, band)')
1344 cursor.execute('CREATE INDEX obs ON metadata (obshistid)')
1345 cursor.execute('CREATE INDEX unq_ast ON baseline_astrometry (uniqueId)')
1346 conn.commit()
1348 self.acquire_lock()
1349 with open(log_file_name, 'a') as out_file:
1350 out_file.write('done with htmid %d -- %e %d\n' %
1351 (htmid, (time.time()-t_start)/3600.0, n_obj))
1352 self.release_lock()
1354 return n_rows