Coverage for python/lsst/sims/catUtils/utils/alertDataGenerator.py : 10%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import numpy as np
2import os
3import re
4import sqlite3
5from collections import OrderedDict
6import time
7import gc
8from lsst.utils import getPackageDir
9import lsst.obs.lsst.phosim as obs_lsst_phosim
10from lsst.sims.catalogs.definitions import InstanceCatalog
11from lsst.sims.utils import trixelFromHtmid, getAllTrixels
12from lsst.sims.utils import levelFromHtmid, halfSpaceFromRaDec
13from lsst.sims.utils import angularSeparation, ObservationMetaData
14from lsst.sims.utils import arcsecFromRadians
15from lsst.sims.catUtils.utils import _baseLightCurveCatalog
16from lsst.sims.utils import _pupilCoordsFromRaDec
17from lsst.sims.coordUtils import chipNameFromPupilCoords
18from lsst.sims.coordUtils import pixelCoordsFromPupilCoords
20from lsst.sims.catalogs.decorators import compound, cached
21from lsst.sims.photUtils import BandpassDict, Sed, calcSNR_m5
22from lsst.sims.photUtils import PhotometricParameters
23from lsst.sims.catUtils.mixins import VariabilityStars, AstrometryStars
24from lsst.sims.catUtils.mixins import VariabilityGalaxies, AstrometryGalaxies
25from lsst.sims.catUtils.mixins import CameraCoords, PhotometryBase
26from lsst.sims.catUtils.mixins import ParametrizedLightCurveMixin
27from lsst.sims.catUtils.mixins import create_variability_cache
29from lsst.sims.catUtils.baseCatalogModels import StarObj, GalaxyAgnObj
30from sqlalchemy.sql import text
31from lsst.sims.catalogs.db import ChunkIterator
33__all__ = ["AlertDataGenerator",
34 "AlertStellarVariabilityCatalog",
35 "AlertAgnVariabilityCatalog",
36 "_baseAlertCatalog",
37 "StellarAlertDBObj",
38 "AgnAlertDBObj",
39 "StellarAlertDBObjMixin"]
42class StellarAlertDBObjMixin(object):
43 """
44 Mimics StarObj class, except it allows you to directly query
45 all objects in a trixel specified by an htmid.
46 """
47 def query_columns_htmid(self, colnames=None, chunk_size=None,
48 constraint=None,
49 limit=None, htmid=None):
50 """Execute a query from the primary catsim database
52 Execute a query, taking advantage of the spherical geometry library and
53 htmid indexes on all catalog tables in the UW catsim database
55 **Parameters**
57 * colnames : list or None
58 a list of valid column names, corresponding to entries in the
59 `columns` class attribute. If not specified, all columns are
60 queried.
61 * chunk_size : int (optional)
62 if specified, then return an iterator object to query the database,
63 each time returning the next `chunk_size` elements. If not
64 specified, all matching results will be returned.
65 * constraint : str (optional)
66 a string which is interpreted as SQL and used as a predicate on the query
67 * limit : int (optional)
68 limits the number of rows returned by the query
69 * htmid is the htmid to be queried
71 **Returns**
73 * result : list or iterator
74 If chunk_size is not specified, then result is a list of all
75 items which match the specified query. If chunk_size is specified,
76 then result is an iterator over lists of the given size.
77 """
79 # find the minimum and maximum htmid
80 # (level=21 since that is what is implemented
81 # on fatboy) that we are asking for
82 #
83 # Note that sqlalchemy does not like np.int64
84 # as a data type
85 current_level = levelFromHtmid(htmid)
86 n_bits_off = 2*(21-current_level)
87 htmid_min = int(htmid << n_bits_off)
88 htmid_max = int((htmid+1) << n_bits_off)
90 query = self._get_column_query(colnames)
92 # add spatial constraints to query.
94 # Hint sql engine to seek on htmid
95 if not self.tableid.endswith('forceseek'):
96 query = query.with_hint(self.table, ' WITH(FORCESEEK)', 'mssql')
98 # SQL is not case sensitive but python is:
99 if 'htmID' in self.columnMap:
100 htmid_name = 'htmID'
101 elif 'htmid' in self.columnMap:
102 htmid_name = 'htmid'
103 else:
104 htmid_name = 'htmId'
106 # Range join on htmid ranges
107 query = query.filter(self.table.c[htmid_name].between(htmid_min, htmid_max))
109 if constraint is not None:
110 query = query.filter(text(constraint))
112 if limit is not None:
113 query = query.limit(limit)
115 return ChunkIterator(self, query, chunk_size)
118class StellarAlertDBObj(StellarAlertDBObjMixin, StarObj):
119 pass
122class AgnAlertDBObj(GalaxyAgnObj):
123 """
124 Mimics GalaxyAgnObj class, except it allows you to directly query
125 all objects in a trixel specified by an htmid.
126 """
128 columns = [('htmid', 0, np.int64),
129 ('galtileid', None, np.int64),
130 ('galid', None, str, 30),
131 ('componentra', 'agnra*PI()/180.'),
132 ('componentdec', 'agndec*PI()/180.'),
133 #: This is actually a problem with the stored procedure.
134 #: We need to be able to map columns other than
135 #: just ra/dec to raJ2000/decJ2000. This gets
136 #: important when we start perturbing the three galaxy components
137 ('raJ2000', 'ra'),
138 ('decJ2000', 'dec'),
139 ('magNorm', 'magnorm_agn'),
140 ('magNormAgn', 'magnorm_agn'),
141 ('sedFilename', 'sedname_agn', str, 40),
142 ('sedFilenameAgn', 'sedname_agn', str, 40),
143 ('variabilityParameters', 'varParamStr', str, 256),
144 ('lsst_u', 'u_ab'),
145 ('lsst_g', 'g_ab'),
146 ('lsst_r', 'r_ab'),
147 ('lsst_i', 'i_ab'),
148 ('lsst_z', 'z_ab'),
149 ('lsst_y', 'y_ab')]
151 def query_columns_htmid(self, colnames=None, chunk_size=None,
152 constraint=None,
153 limit=None, htmid=None):
154 """Execute a query from the primary catsim database
156 Execute a query, taking advantage of the spherical geometry library and
157 htmid indexes on all catalog tables in the UW catsim database
159 **Parameters**
161 * colnames : list or None
162 a list of valid column names, corresponding to entries in the
163 `columns` class attribute. If not specified, all columns are
164 queried.
165 * chunk_size : int (optional)
166 if specified, then return an iterator object to query the database,
167 each time returning the next `chunk_size` elements. If not
168 specified, all matching results will be returned.
169 * constraint : str (optional)
170 a string which is interpreted as SQL and used as a predicate on the query
171 * limit : int (optional)
172 limits the number of rows returned by the query
173 * htmid is the htmid to be queried
175 **Returns**
177 * result : list or iterator
178 If chunk_size is not specified, then result is a list of all
179 items which match the specified query. If chunk_size is specified,
180 then result is an iterator over lists of the given size.
181 """
183 trixel = trixelFromHtmid(htmid)
184 ra_0, dec_0 = trixel.get_center()
185 new_obs = ObservationMetaData(pointingRA=ra_0, pointingDec=dec_0, boundType='circle',
186 boundLength=trixel.get_radius()+0.1)
188 self._queried_trixel = trixel
189 self._queried_htmid_level = levelFromHtmid(htmid)
191 return self.query_columns(colnames=colnames, chunk_size=chunk_size,
192 obs_metadata=new_obs, constraint=constraint,
193 limit=limit)
195 def _final_pass(self, results):
196 """Modify the results of raJ2000 and decJ2000 to be in radians.
198 Also filter the results so that any objects outside of the
199 trixel specified in query_columns_htmid are returned with
200 htmid=0.
202 **Parameters**
204 * results : Structured array of results from query
206 **Returns**
208 * results : Modified structured array
210 """
212 if hasattr(self, '_queried_trixel'):
213 htmid = self._queried_trixel.htmid
214 htmid_21 = htmid << 2*(21-self._queried_htmid_level)
215 assert levelFromHtmid(htmid_21) == 21
216 contains_arr = self._queried_trixel.contains(results['raJ2000'], results['decJ2000'])
217 results['htmid'] = np.where(contains_arr, htmid_21, 0)
219 results['raJ2000'] = np.radians(results['raJ2000'])
220 results['decJ2000'] = np.radians(results['decJ2000'])
221 return results
224class _baseAlertCatalog(PhotometryBase, CameraCoords, _baseLightCurveCatalog):
226 camera = obs_lsst_phosim.PhosimMapper().camera
228 column_outputs = ['htmid', 'uniqueId', 'raICRS', 'decICRS',
229 'flux', 'SNR', 'dflux',
230 'chipNum', 'xPix', 'yPix']
232 default_formats = {'f': '%.4g'}
234 default_columns = [('properMotionRa', 0.0, float),
235 ('properMotionDec', 0.0, float),
236 ('parallax', 0.0, float)]
238 def iter_catalog_chunks(self, chunk_size=None, query_cache=None, column_cache=None):
239 """
240 Returns an iterator over chunks of the catalog.
242 Parameters
243 ----------
244 chunk_size : int, optional, defaults to None
245 the number of rows to return from the database at a time. If None,
246 returns the entire database query in one chunk.
248 query_cache : iterator over database rows, optional, defaults to None
249 the result of calling db_obj.query_columns(). If query_cache is not
250 None, this method will iterate over the rows in query_cache and produce
251 an appropriate InstanceCatalog. DO NOT set to non-None values
252 unless you know what you are doing. It is an optional
253 input for those who want to repeatedly examine the same patch of sky
254 without actually querying the database over and over again. If it is set
255 to None (default), this method will handle the database query.
257 column_cache : a dict that will be copied over into the catalogs self._column_cache.
258 Should be left as None, unless you know what you are doing.
259 """
261 if query_cache is None:
262 # Call the original version of iter_catalog defined in the
263 # InstanceCatalog class. This version of iter_catalog includes
264 # the call to self.db_obj.query_columns, which the user would have
265 # used to generate query_cache.
266 for line in InstanceCatalog.iter_catalog_chunks(self, chunk_size=chunk_size):
267 yield line
268 else:
269 # Otherwise iterate over the query cache
270 transform_keys = list(self.transformations.keys())
271 for chunk in query_cache:
272 self._set_current_chunk(chunk, column_cache=column_cache)
273 chunk_cols = [self.transformations[col](self.column_by_name(col))
274 if col in transform_keys else
275 self.column_by_name(col)
276 for col in self.iter_column_names()]
278 if not hasattr(self, '_chunkColMap_output'):
280 self._chunkColMap_output = dict([(col, i)
281 for i, col in
282 enumerate(self.iter_column_names())])
284 yield chunk_cols, self._chunkColMap_output
286 self._column_cache = {}
287 self._current_chunk = None
289 @cached
290 def get_chipName(self):
291 if len(self.column_by_name('uniqueId')) == 0:
292 return np.array([])
293 raise RuntimeError("Should not get this far in get_chipName")
295 @compound('x_pupil', 'y_pupil')
296 def get_pupilFromSky(self):
297 if len(self.column_by_name('uniqueId')) == 0:
298 return np.array([[], []])
299 raise RuntimeError("Should not get this far in get_pupilFromSky")
301 @cached
302 def get_chipNum(self):
303 """
304 Concatenate the digits in 'R:i,j S:m,n' to make the chip number ijmn
305 """
306 chip_name = self.column_by_name('chipName')
307 return np.array([int(''.join(re.findall(r'\d+', name))) if name is not None else 0
308 for name in chip_name])
310 @compound('xPix', 'yPix')
311 def get_pixelCoordinates(self):
312 xpup = self.column_by_name('x_pupil')
313 ypup = self.column_by_name('y_pupil')
314 chip_name = self.column_by_name('chipName')
315 xpix, ypix = pixelCoordsFromPupilCoords(xpup, ypup, chipName=chip_name,
316 includeDistortion=True,
317 camera=self.camera)
318 return np.array([xpix, ypix])
320 @compound('delta_umag', 'delta_gmag', 'delta_rmag',
321 'delta_imag', 'delta_zmag', 'delta_ymag')
322 def get_deltaMagAvro(self):
323 ra = self.column_by_name('raJ2000')
324 if len(ra) == 0:
325 return np.array([[], [], [], [], [], []])
327 raise RuntimeError("Should not have gotten this far in delta mag getter")
329 @compound('lsst_u', 'lsst_g', 'lsst_r', 'lsst_i', 'lsst_z', 'lsst_y')
330 def get_lsst_magnitudes(self):
331 """
332 getter for LSST stellar magnitudes
333 """
335 magnitudes = np.array([self.column_by_name('quiescent_lsst_u'),
336 self.column_by_name('quiescent_lsst_g'),
337 self.column_by_name('quiescent_lsst_r'),
338 self.column_by_name('quiescent_lsst_i'),
339 self.column_by_name('quiescent_lsst_z'),
340 self.column_by_name('quiescent_lsst_y')])
342 delta = np.array([self.column_by_name('delta_umag'),
343 self.column_by_name('delta_gmag'),
344 self.column_by_name('delta_rmag'),
345 self.column_by_name('delta_imag'),
346 self.column_by_name('delta_zmag'),
347 self.column_by_name('delta_ymag')])
348 magnitudes += delta
350 return magnitudes
352 @compound('mag', 'dmag', 'quiescent_mag')
353 def get_alertPhotometry(self):
354 mag = self.column_by_name('lsst_%s' % self.obs_metadata.bandpass)
355 quiescent_mag = self.column_by_name('quiescent_lsst_%s' % self.obs_metadata.bandpass)
356 dmag = mag - quiescent_mag
358 return np.array([mag, dmag, quiescent_mag])
360 @compound('flux', 'dflux', 'SNR')
361 def get_alertFlux(self):
362 quiescent_mag = self.column_by_name('quiescent_mag')
363 mag = self.column_by_name('mag')
364 if not hasattr(self, '_dummy_sed'):
365 self._dummy_sed = Sed()
366 if not hasattr(self, 'lsstBandpassDict'):
367 self.lsstBandpassDict = BandpassDict.loadTotalBandpassesFromFiles()
368 if not hasattr(self, 'phot_params'):
369 self.phot_params = PhotometricParameters()
370 if not hasattr(self, '_gamma'):
371 self._gamma = None
373 quiescent_flux = self._dummy_sed.fluxFromMag(quiescent_mag)
374 flux = self._dummy_sed.fluxFromMag(mag)
375 dflux = flux - quiescent_flux
377 snr_tot, gamma = calcSNR_m5(mag, self.lsstBandpassDict[self.obs_metadata.bandpass],
378 self.obs_metadata.m5[self.obs_metadata.bandpass],
379 self.phot_params, gamma=self._gamma)
381 if self._gamma is None:
382 self._gamma = gamma
384 return np.array([flux, dflux, snr_tot])
387class AlertStellarVariabilityCatalog(_baseAlertCatalog,
388 VariabilityStars,
389 AstrometryStars):
391 @compound('quiescent_lsst_u', 'quiescent_lsst_g', 'quiescent_lsst_r',
392 'quiescent_lsst_i', 'quiescent_lsst_z', 'quiescent_lsst_y')
393 def get_quiescent_lsst_magnitudes(self):
394 return np.array([self.column_by_name('umag'), self.column_by_name('gmag'),
395 self.column_by_name('rmag'), self.column_by_name('imag'),
396 self.column_by_name('zmag'), self.column_by_name('ymag')])
399class AlertAgnVariabilityCatalog(_baseAlertCatalog,
400 VariabilityGalaxies,
401 AstrometryGalaxies):
403 @compound('quiescent_lsst_u', 'quiescent_lsst_g', 'quiescent_lsst_r',
404 'quiescent_lsst_i', 'quiescent_lsst_z', 'quiescent_lsst_y')
405 def get_quiescent_lsst_magnitudes(self):
406 return np.array([self.column_by_name('u_ab'), self.column_by_name('g_ab'),
407 self.column_by_name('r_ab'), self.column_by_name('i_ab'),
408 self.column_by_name('z_ab'), self.column_by_name('y_ab')])
411class AlertDataGenerator(object):
412 """
413 This class will read in astrophysical sources and variability
414 models from CatSim, observe them with a simulated OpSim
415 cadence, and write a series of sqlite files containing all
416 of the simulated observations that could trigger an alert.
418 In order to make this calculation as efficient as possible,
419 the class works by partitioning the sky according to the
420 Hierarchical Triangular Mesh (HTM) of
422 Kunszt P., Szalay A., Thakar A. (2006) in "Mining The Sky",
423 Banday A, Zaroubi S, Bartelmann M. eds.
424 ESO Astrophysics Symposia
425 https://www.researchgate.net/publication/226072008_The_Hierarchical_Triangular_Mesh
427 Szalay A. et al. (2007)
428 "Indexing the Sphere with the Hierarchical Triangular Mesh"
429 arXiv:cs/0701164
431 and simulating all of the observations in a given trixel (the
432 elementary unit of the HTM) at once. Accordintly, the outputs
433 produced by this class are files named like
435 prefix_NNNN_sqlite.db
437 where prefix is specified by theuser and NNNN is the htmid, the
438 unique identifying integer, corresponding to each simulated trixel.
440 The proper way to run this class is to instantiate it, run
441 subdivide_obs on a list of ObservationMetaData corresponding
442 to the OpSim pointings to be simulated, and then running
443 alert_data_from_htmid on each of the htmid in the class property
444 htmid_list. This last step can easily be parallelized using python's
445 multiprocessing module, with each process handling a different htmid.
447 The sqlite files produced by alert_data_from_htmid will each contain
448 four tables. They are as follows. Columns are listed below the
449 tables.
451 alert_data
452 ----------
453 uniqueId -- int -- a unique identifier for each astrophysical object
455 obshistId -- int -- a unique identifier for each OpSim pointing
457 xPix -- float -- the x pixel coordinate of the source on the focal
458 plane
460 yPix -- float -- the y pixel coordinate of the source on the focal
461 plane
463 dflux -- float -- the difference in flux between the source's current
464 flux and its quiescent flux (the source's quiescent flux can be found
465 in the quiescent_flux table). This is in units of Janskys.
467 snr -- float -- the signal to noise of the current detection of the
468 source (not the signal to noise of the source's detection in a simulated
469 difference image).
471 ra -- float -- the current RA of the source in degrees
473 dec -- float -- the current Declination of the source in degrees.
475 The alert_data table has a mult-column index on uniqueId and obshistId.
477 metadata
478 --------
479 obshistId -- int --- a unique identifier for each OpSim pointing
481 TAI -- float -- the International Atomic Time of the observation
482 as an MJD (in days)
484 band -- int -- denotes the filter used for the observation
485 (0=u, 1=g, 2=r, etc.)
487 The metadata table is indexed on obshistId
489 quiescent_flux
490 --------------
491 uniqueId -- int -- a unique identifier for each astrophysical source
493 band -- int -- an integer denoting each LSST filter (0=u, 1=g, 2=r, etc.)
495 flux -- float -- the flux of the source through the filter specified by
496 band (in units of Janskys)
498 snr -- float -- the signal to noise ratio of the source in the given
499 band with m5 taken from Table 2 of the overview paper (arXiv:0805.2366)
501 The quiescent_flux table has a multi-column index on uniqueId and band.
503 baseline_astrometry
504 -------------------
505 uniqueId -- int -- a unique identifier for each astrophysical source
507 TAI -- float -- the International Atomic Time of the baseline astrometric
508 measurements below as a MJD (in days)
510 ra -- float -- the RA of the source at TAI in degrees
512 dec -- float -- the Declination of the source at TAI in degrees
514 pmRA -- float -- the RA proper motion of the source in milliarcseconds/year
516 pmDec -- float -- the Declination proper motion of the source in
517 milliarcseconds/year
519 parallax -- float -- the parallax of the source in milliarcseconds
521 The baseline_astrometry table is indexed on uniqueId
523 """
525 def __init__(self,
526 testing=False):
527 """
528 Parameters
529 ----------
530 testing as a boolean that should only be True when running the unit tests.
531 If True, it prevents the AlertDataGenerator from pre-caching variability
532 models, which aids performance, but uses more memory than we want to use
533 in a unit test.
534 """
536 self.lsst_camera = obs_lsst_phosim.PhosimMapper().camera
537 self._variability_cache = create_variability_cache()
538 self._stdout_lock = None
539 if not testing:
540 plm = ParametrizedLightCurveMixin()
541 plm.load_parametrized_light_curves(variability_cache = self._variability_cache)
542 self.bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
544 # This is a file that lists the maximum amplitude of variability
545 # for each of the Kepler-derived light curve models. It will be
546 # used by the stellar variability model to figure out which
547 # stars can be skipped because they will never vary above
548 # the alert-triggering threshold.
549 self._dmag_lookup_file_exists = True
550 self._dmag_lookup_file = os.path.join(getPackageDir('sims_data'),
551 'catUtilsData',
552 'kplr_dmag_171204.txt')
554 if not os.path.exists(self._dmag_lookup_file):
555 if not testing:
556 script_name = os.path.join(getPackageDir('sims_catUtils'), 'support_scripts',
557 'get_kepler_dmag.sh')
558 raise RuntimeError('\n%s does not exist; run the script\n\n%s\n\n' %
559 script_name)
560 else:
561 self._dmag_lookup_file_exists = False
563 def acquire_lock(self):
564 """
565 If running with multiprocessing, acquire
566 the lock.
567 """
568 if self._stdout_lock is not None:
569 self._stdout_lock.acquire()
571 def release_lock(self):
572 """
573 If running with multiprocessing, release
574 the lock.
575 """
576 if self._stdout_lock is not None:
577 self._stdout_lock.release()
579 def subdivide_obs(self, obs_list, htmid_level=6):
580 """
581 Take a list of ObservationMetaData and subdivide
582 them according to which trixels (see htmModule.py
583 in sims_utils) they intersect.
585 Parameters
586 ----------
587 obs_list is a list of ObservationMetaData
589 htmid_level is an int denoting the level of
590 the HTM mesh you want to use to tile the sky
591 (higher htmid_level corresponds to a finer
592 tiling). Default = 6
594 Returns
595 -------
596 Nothing.
598 After running this method, this AlertGenerator
599 will contain the following data.
601 - a list of the htmid of every trixel intersected
602 by the fields of view specified in obs_list. This
603 list is accessible from the property
604 AlertGenerator.htmid_list
606 - a dict mapping each htmid to the ObservationMetaData
607 from obs_list that intersect it. The method
608 AlertGenerator.obs_from_htmid(htmid) will return a
609 list of all of the ObservationMetaData that intersect
610 the trixel specified by htmid.
611 """
612 self._trixel_dict = getAllTrixels(htmid_level)
613 valid_htmid = []
614 for htmid in self._trixel_dict:
615 if levelFromHtmid(htmid) == htmid_level:
616 valid_htmid.append(htmid)
618 obs_list = np.array(obs_list)
619 self._obs_list = obs_list
620 obs_ra_list = []
621 obs_dec_list = []
622 halfspace_list = []
623 for obs in obs_list:
624 obs_ra_list.append(obs.pointingRA)
625 obs_dec_list.append(obs.pointingDec)
626 hs = halfSpaceFromRaDec(obs.pointingRA,
627 obs.pointingDec,
628 obs.boundLength)
629 halfspace_list.append(hs)
631 obs_ra_list = np.array(obs_ra_list)
632 obs_dec_list = np.array(obs_dec_list)
633 halfspace_list = np.array(halfspace_list)
635 self._htmid_dict = {}
636 self._htmid_list = []
637 n_obs_list = []
638 fov_radius = 1.75
639 for i_htmid, htmid in enumerate(valid_htmid):
640 trixel = self._trixel_dict[htmid]
641 ra_c, dec_c = trixel.get_center()
642 radius = trixel.get_radius()
643 obs_distance = angularSeparation(ra_c, dec_c, obs_ra_list, obs_dec_list)
644 valid_obs = np.where(obs_distance < radius + fov_radius)
645 if len(valid_obs[0]) > 0:
646 final_obs_list = []
647 for obs_dex in valid_obs[0]:
648 hs = halfspace_list[obs_dex]
649 obs = obs_list[obs_dex]
650 if hs.contains_trixel(trixel) != 'outside':
651 final_obs_list.append(obs_dex)
653 if len(final_obs_list) == 0:
654 continue
656 self._htmid_dict[htmid] = np.array(final_obs_list)
657 self._htmid_list.append(htmid)
658 n_obs_list.append(len(final_obs_list))
660 n_obs_list = np.array(n_obs_list)
661 self._htmid_list = np.array(self._htmid_list)
662 sorted_dex = np.argsort(-1.0*n_obs_list)
663 self._htmid_list = self._htmid_list[sorted_dex]
664 print('done subdividing obs list -- %d htmid' %
665 len(self._htmid_list))
667 @property
668 def htmid_list(self):
669 """
670 A list of the unique htmids corresponding to the trixels
671 that need to be queried to generate the alert data
672 """
673 return self._htmid_list
675 def n_obs(self, htmid):
676 """
677 Return the number of observations that intersect
678 the trixel specified by htmid.
680 Must run subdivide_obs in order for this method to
681 work.
682 """
683 return len(self._htmid_dict[htmid])
685 def obs_from_htmid(self, htmid):
686 """
687 Return a numpy array containing all of the ObservationMetaData
688 that intersect the trixel specified by htmid.
690 Must run subdivide_obs in order for this method to
691 work.
692 """
693 return self._obs_list[self._htmid_dict[htmid]]
695 def _output_alert_data(self, conn, data_cache):
696 """
697 Write a cache of alert data to the sqlite file currently open.
699 Parameters
700 ----------
701 conn is the connection to the sqlite file (already open)
703 data_cache is a dict containing all of the data to be written.
704 It will keyed on a string like 'i_j' where i is the obshistID
705 of an OpSim pointing and j is an arbitrary integer. That key
706 will lead to another dict keyed on the columns being output to
707 the sqlite file. The values of this second layer of dict are
708 numpy arrays.
710 Returns
711 -------
712 The number of rows written to the sqlite file
713 """
715 cursor = conn.cursor()
716 n_rows_0 = cursor.execute('SELECT COUNT(uniqueId) FROM alert_data').fetchall()
718 chunk_lengths = np.zeros(len(data_cache))
720 for i_cache_tag, cache_tag in enumerate(data_cache):
721 obshistid = int(cache_tag.split('_')[0])
722 n_obj = len(data_cache[cache_tag]['uniqueId'])
723 chunk_lengths[i_cache_tag] = n_obj
725 values = ((int(data_cache[cache_tag]['uniqueId'][i_obj]),
726 obshistid,
727 data_cache[cache_tag]['xPix'][i_obj],
728 data_cache[cache_tag]['yPix'][i_obj],
729 int(data_cache[cache_tag]['chipNum'][i_obj]),
730 data_cache[cache_tag]['dflux'][i_obj],
731 data_cache[cache_tag]['SNR'][i_obj],
732 np.degrees(data_cache[cache_tag]['raICRS'][i_obj]),
733 np.degrees(data_cache[cache_tag]['decICRS'][i_obj]))
734 for i_obj in range(n_obj))
735 cursor.executemany('INSERT INTO alert_data VALUES (?,?,?,?,?,?,?,?,?)', values)
736 conn.commit()
738 n_rows_1 = cursor.execute('SELECT COUNT(uniqueId) FROM alert_data').fetchall()
739 conn.commit()
740 n_written = (n_rows_1[0][0]-n_rows_0[0][0])
742 return n_written
744 def _filter_on_photometry_then_chip_name(self, chunk, column_query,
745 obs_valid_dex, expmjd_list,
746 photometry_catalog,
747 dmag_cutoff):
748 """
749 Determine which simulated observations are actually worth storing
750 by first figuring out which observations of which objects are
751 photometrically detectable and alert-worthy, then determining
752 which of those actually fall on an LSST detector.
754 Parameters
755 ----------
756 chunk is the output yielded from a CatSim ChunkIterator. It is
757 a numpy recarray representing one chunk_size query from the
758 underlying simulations database
760 column_query is a list of the columns that were queried from
761 the database
763 obs_valid_dex is a list of integers corresponding to indexes in
764 self._obs_list of the ObservationMetaData that are actually valid
765 for the trixel currently being simulated
767 expmjd_list is a numpy array of the TAI dates of ObservtionMetaData
768 represented by obs_valid_dex
770 photometry_catalog is an instantiation of the InstanceCatalog class
771 being used to calculate magnitudes for these variable sources.
773 Outputs
774 -------
775 chip_name_dict is a dict keyed on i_obs (which is the index of
776 an ObservationMetaData's position in obs_valid_dex, NOT its
777 position in self._obs_list). The values of chip_name_dict are
778 tuples containing:
779 - a list of the names of the detectors that objects from chunk
780 landed on (including Nones for those objects that did not land
781 on any detector)
783 - a list of the xPupil coords for every object in chunk
785 - a list of the yPupil coords for every object in chunk
787 - a list of the indexes in chunk of those objects which actually
788 landed on a detector
790 dmag_arr is a numpy array of the delta_magnitudes of every object
791 in chunk. dmag_arr[11][3][4] is the delta_magnitude of chunk[4]
792 in the 3rd band (i.e. the i band) at TAI = expmjd[11].
794 dmag_arr_transpose is dmag_arr with the time and object columns
795 transposed so that dmag_arr_transpose[4][3][11] == dmag_arr[11][3][4].
797 time_arr is an array of integers with shape == (len(chunk), len(obs_valid_dex)).
798 A -1 in time_arr means that that combination of object and observation did
799 not yield a valid observation. A +1 means that the object and observation
800 combination are valid.
801 """
803 ######################################################
804 # Calculate the delta_magnitude for all of the sources
805 #
806 photometry_catalog._set_current_chunk(chunk)
807 dmag_arr = photometry_catalog.applyVariability(chunk['varParamStr'],
808 variability_cache=self._variability_cache,
809 expmjd=expmjd_list).transpose((2, 0, 1))
811 dmag_arr_transpose = dmag_arr.transpose(2, 1, 0)
813 n_raw_obj = len(chunk)
814 photometrically_valid = -1*np.ones(n_raw_obj, dtype=int)
815 for i_obj in range(n_raw_obj):
816 keep_it = False
817 for i_filter in range(6):
818 if np.abs(dmag_arr_transpose[i_obj][i_filter]).max() >= dmag_cutoff:
819 keep_it = True
820 break
821 if keep_it:
822 photometrically_valid[i_obj] = 1
824 photometrically_valid = np.where(photometrically_valid >= 0)
826 if 'properMotionRa'in column_query:
827 pmra = chunk['properMotionRa'][photometrically_valid]
828 pmdec = chunk['properMotionDec'][photometrically_valid]
829 px = chunk['parallax'][photometrically_valid]
830 vrad = chunk['radialVelocity'][photometrically_valid]
831 else:
832 pmra = None
833 pmdec = None
834 px = None
835 vrad = None
837 ###################################################################
838 # Figure out which sources actually land on an LSST detector during
839 # the observations in question
840 #
841 chip_name_dict = {}
843 # time_arr will keep track of which objects appear in which observations;
844 # 1 means the object appears; -1 means it does not
845 time_arr_transpose = -1*np.ones((len(obs_valid_dex), len(chunk['raJ2000'])),
846 dtype=int)
848 for i_obs, obs_dex in enumerate(obs_valid_dex):
849 obs = self._obs_list[obs_dex]
850 chip_name_list = np.array([None]*n_raw_obj)
851 xpup_list = np.zeros(n_raw_obj, dtype=float)
852 ypup_list = np.zeros(n_raw_obj, dtype=float)
853 chip_int_arr = -1*np.ones(len(chip_name_list), dtype=int)
855 if len(photometrically_valid[0]) > 0:
856 xpup_list_val, ypup_list_val = _pupilCoordsFromRaDec(chunk['raJ2000'][photometrically_valid],
857 chunk['decJ2000'][photometrically_valid],
858 pm_ra=pmra, pm_dec=pmdec,
859 parallax=px, v_rad=vrad,
860 obs_metadata=obs)
862 xpup_list[photometrically_valid] = xpup_list_val
863 ypup_list[photometrically_valid] = ypup_list_val
865 chip_name_list[photometrically_valid] = chipNameFromPupilCoords(xpup_list_val,
866 ypup_list_val,
867 camera=self.lsst_camera)
869 for i_chip, name in enumerate(chip_name_list):
870 if name is not None:
871 chip_int_arr[i_chip] = 1
873 valid_obj = np.where(chip_int_arr > 0)
874 time_arr_transpose[i_obs][valid_obj] = 1
876 chip_name_dict[i_obs] = (chip_name_list,
877 xpup_list,
878 ypup_list,
879 valid_obj)
881 time_arr = time_arr_transpose.transpose()
882 assert len(chip_name_dict) == len(obs_valid_dex)
884 return chip_name_dict, dmag_arr, dmag_arr_transpose, time_arr
886 def alert_data_from_htmid(self, htmid, dbobj,
887 dmag_cutoff=0.005,
888 chunk_size=1000, write_every=10000,
889 output_dir='.', output_prefix='',
890 log_file_name=None,
891 photometry_class=None,
892 chunk_cutoff=-1,
893 lock=None):
895 """
896 Generate an sqlite file with all of the alert data for a given
897 trixel.
899 Parameters
900 ----------
901 htmid is an integer denoting the trixel from self.htmid_list that should
902 be simulated
904 dbobj is a CatalogDBObject connecting to the data underlying the simulation
906 dmag_cutoff indicates the minimum change magnitude needed to trigger a
907 simulated alert
909 chunk_size denotes the number of objects to query from the database and
910 process at one time
912 write_every indicates how often to write to the sqlite file (i.e. the
913 code will pause the simulation process and write to the sqlite file
914 when it has accumulated this many valid observations)
916 output_dir is the directory in which to create the sqlite file
918 output_prefix is the prefix of the sqlite file's name
920 log_file_name is the name of a text file where progress will be written
922 photometry_class is a InstanceCatalog class (not an instantiation) that
923 contains the methods for calculating the photometry associated with the
924 simulated alerts (see AlertStellarVariabilityCatalog and
925 AlertAgnVariabilityCatalog in this module)
927 chunk_cutoff is an optional int; stop the simulation after this many
928 chunks have been processed. This is for testing purposes.
929 If chunk_cutoff == -1, the code will process all of the astrophysical
930 objects in the trixel.
932 lock is a multiprocessing.Lock() for use if running multiple
933 instances of alert_data_from_htmid. This will prevent multiple processes
934 from writing to the log file or stdout simultaneously.
935 """
937 htmid_level = levelFromHtmid(htmid)
938 if log_file_name is None:
939 raise RuntimeError('must specify log_file_name')
941 if ('_PARAMETRIZED_LC_DMAG_LOOKUP' not in self._variability_cache and
942 self._dmag_lookup_file_exists):
944 self._variability_cache['_PARAMETRIZED_LC_DMAG_CUTOFF'] = dmag_cutoff
945 self._variability_cache['_PARAMETRIZED_LC_DMAG_LOOKUP'] = {}
947 with open(self._dmag_lookup_file, 'r') as in_file:
948 for line in in_file:
949 if line[0] == '#':
950 continue
951 params = line.split()
952 self._variability_cache['_PARAMETRIZED_LC_DMAG_LOOKUP'][int(params[0])] = float(params[1])
954 self._stdout_lock = lock
955 this_pid = os.getpid()
957 t_start = time.time() # so that we can get a sense of how long the full
958 # simulation will take
960 desired_columns = []
961 desired_columns.append('simobjid')
962 desired_columns.append('variabilityParameters')
963 desired_columns.append('varParamStr')
964 desired_columns.append('raJ2000')
965 desired_columns.append('decJ2000')
966 desired_columns.append('properMotionRa')
967 desired_columns.append('properMotionDec')
968 desired_columns.append('parallax')
969 desired_columns.append('radialVelocity')
970 desired_columns.append('ebv')
971 desired_columns.append('redshift')
972 desired_columns.append('htmid')
974 if 'umag' in dbobj.columnMap:
975 desired_columns.append('umag')
976 desired_columns.append('gmag')
977 desired_columns.append('rmag')
978 desired_columns.append('imag')
979 desired_columns.append('zmag')
980 desired_columns.append('ymag')
981 elif 'u_ab' in dbobj.columnMap:
982 desired_columns.append('u_ab')
983 desired_columns.append('g_ab')
984 desired_columns.append('r_ab')
985 desired_columns.append('i_ab')
986 desired_columns.append('z_ab')
987 desired_columns.append('y_ab')
988 else:
989 raise RuntimeError('Not sure what quiescent '
990 'LSST magnitudes are called '
991 'in this CatalogDBObject')
993 if photometry_class is None:
994 raise RuntimeError('Must specify photometry_class')
996 if os.path.exists(output_dir) and not os.path.isdir(output_dir):
997 raise RuntimeError('%s is not a dir' % output_dir)
998 if not os.path.exists(output_dir):
999 os.mkdir(output_dir)
1001 dummy_sed = Sed()
1003 mag_names = ('u', 'g', 'r', 'i', 'z', 'y')
1005 phot_params = PhotometricParameters()
1007 # from Table 2 of the overview paper
1008 obs_mag_cutoff = (23.68, 24.89, 24.43, 24.0, 24.45, 22.60)
1010 gamma_template = {}
1011 for i_filter in range(6):
1012 gamma_template[i_filter] = None
1014 obs_valid_dex = self._htmid_dict[htmid]
1015 print('n valid obs %d' % len(obs_valid_dex))
1017 cat_list = []
1018 expmjd_list = []
1019 mag_name_to_int = {'u': 0, 'g': 1, 'r': 2,
1020 'i': 3, 'z': 4, 'y': 5}
1021 for obs_dex in obs_valid_dex:
1022 obs = self._obs_list[obs_dex]
1023 cat = photometry_class(dbobj, obs_metadata=obs)
1024 cat.lsstBandpassDict = self.bp_dict
1025 cat_list.append(cat)
1026 expmjd_list.append(obs.mjd.TAI)
1028 expmjd_list = np.array(expmjd_list)
1029 cat_list = np.array(cat_list)
1030 sorted_dex = np.argsort(expmjd_list)
1032 expmjd_list = expmjd_list[sorted_dex]
1033 cat_list = cat_list[sorted_dex]
1034 obs_valid_dex = obs_valid_dex[sorted_dex]
1036 available_columns = list(dbobj.columnMap.keys())
1037 column_query = []
1038 for col in desired_columns:
1039 if col in available_columns:
1040 column_query.append(col)
1042 n_bits_off = 2*(21-htmid_level)
1044 data_iter = dbobj.query_columns_htmid(colnames=column_query,
1045 htmid=htmid,
1046 chunk_size=chunk_size)
1048 photometry_catalog = photometry_class(dbobj, self._obs_list[obs_valid_dex[0]],
1049 column_outputs=['lsst_u',
1050 'lsst_g',
1051 'lsst_r',
1052 'lsst_i',
1053 'lsst_z',
1054 'lsst_y'])
1056 i_chunk = 0
1058 output_data_cache = {}
1059 n_rows_cached = 0
1061 n_obj = 0
1062 n_actual_obj = 0
1063 n_time_last = 0
1064 n_rows = 0
1066 t_before_obj = time.time() # so that we can get a sense of how long the
1067 # "iterating over astrophysical objects" part
1068 # of the simulation will take
1070 db_name = os.path.join(output_dir, '%s_%d_sqlite.db' % (output_prefix, htmid))
1071 with sqlite3.connect(db_name, isolation_level='EXCLUSIVE') as conn:
1072 creation_cmd = '''CREATE TABLE alert_data
1073 (uniqueId int, obshistId int, xPix float, yPix float,
1074 chipNum int, dflux float, snr float, ra float, dec float)'''
1076 cursor = conn.cursor()
1077 cursor.execute('PRAGMA journal_mode=WAL;')
1078 conn.commit()
1079 cursor.execute(creation_cmd)
1080 conn.commit()
1082 creation_cmd = '''CREATE TABLE metadata
1083 (obshistId int, TAI float, band int)'''
1084 cursor.execute(creation_cmd)
1085 conn.commit()
1087 for obs_dex in obs_valid_dex:
1088 obs = self._obs_list[obs_dex]
1089 cmd = '''INSERT INTO metadata
1090 VALUES(%d, %.5f, %d)''' % (obs.OpsimMetaData['obsHistID'],
1091 obs.mjd.TAI,
1092 mag_name_to_int[obs.bandpass])
1094 cursor.execute(cmd)
1095 conn.commit()
1097 creation_cmd = '''CREATE TABLE quiescent_flux
1098 (uniqueId int, band int, flux float, snr float)'''
1100 cursor.execute(creation_cmd)
1101 conn.commit()
1103 creation_cmd = '''CREATE TABLE baseline_astrometry
1104 (uniqueId int, ra real, dec real, pmRA real,
1105 pmDec real, parallax real, TAI real)'''
1107 cursor.execute(creation_cmd)
1108 conn.commit()
1110 for chunk in data_iter:
1111 n_raw_obj = len(chunk)
1112 i_chunk += 1
1114 if chunk_cutoff > 0 and i_chunk >= chunk_cutoff:
1115 break
1117 n_time_last = 0
1118 # filter the chunk so that we are only considering sources that are in
1119 # the trixel being considered
1120 reduced_htmid = chunk['htmid'] >> n_bits_off
1122 valid_htmid = np.where(reduced_htmid == htmid)
1123 if len(valid_htmid[0]) == 0:
1124 continue
1125 n_htmid_trim = n_raw_obj-len(valid_htmid[0])
1126 chunk = chunk[valid_htmid]
1127 n_obj += len(valid_htmid[0])
1129 (chip_name_dict,
1130 dmag_arr,
1131 dmag_arr_transpose,
1132 time_arr) = self._filter_on_photometry_then_chip_name(chunk, column_query,
1133 obs_valid_dex,
1134 expmjd_list,
1135 photometry_catalog,
1136 dmag_cutoff)
1138 q_f_dict = {}
1139 q_m_dict = {}
1141 q_m_dict[0] = photometry_catalog.column_by_name('quiescent_lsst_u')
1142 q_m_dict[1] = photometry_catalog.column_by_name('quiescent_lsst_g')
1143 q_m_dict[2] = photometry_catalog.column_by_name('quiescent_lsst_r')
1144 q_m_dict[3] = photometry_catalog.column_by_name('quiescent_lsst_i')
1145 q_m_dict[4] = photometry_catalog.column_by_name('quiescent_lsst_z')
1146 q_m_dict[5] = photometry_catalog.column_by_name('quiescent_lsst_y')
1148 q_f_dict[0] = dummy_sed.fluxFromMag(q_m_dict[0])
1149 q_f_dict[1] = dummy_sed.fluxFromMag(q_m_dict[1])
1150 q_f_dict[2] = dummy_sed.fluxFromMag(q_m_dict[2])
1151 q_f_dict[3] = dummy_sed.fluxFromMag(q_m_dict[3])
1152 q_f_dict[4] = dummy_sed.fluxFromMag(q_m_dict[4])
1153 q_f_dict[5] = dummy_sed.fluxFromMag(q_m_dict[5])
1155 q_pmra = 1000.0*arcsecFromRadians(photometry_catalog.column_by_name('properMotionRa'))
1156 q_pmdec = 1000.0*arcsecFromRadians(photometry_catalog.column_by_name('properMotionDec'))
1157 q_parallax = 1000.0*arcsecFromRadians(photometry_catalog.column_by_name('parallax'))
1158 q_ra = np.degrees(photometry_catalog.column_by_name('raICRS'))
1159 q_dec = np.degrees(photometry_catalog.column_by_name('decICRS'))
1160 q_tai = photometry_catalog.obs_metadata.mjd.TAI
1162 q_snr_dict = {}
1163 for i_filter in range(6):
1165 snr_template, local_gamma = calcSNR_m5(q_m_dict[i_filter],
1166 self.bp_dict[mag_names[i_filter]],
1167 obs_mag_cutoff[i_filter],
1168 phot_params, gamma=gamma_template[i_filter])
1169 q_snr_dict[i_filter] = snr_template
1170 gamma_template[i_filter] = local_gamma
1172 unq = photometry_catalog.column_by_name('uniqueId')
1174 try:
1175 assert dmag_arr_transpose.shape == (len(chunk), len(mag_names), len(expmjd_list))
1176 except AssertionError:
1177 print('dmag_arr_transpose_shape %s' % str(dmag_arr_transpose.shape))
1178 print('should be (%d, %d, %d)' % (len(chunk), len(mag_names), len(expmjd_list)))
1179 raise
1181 # only include those sources for which np.abs(delta_mag) >= dmag_cutoff
1182 # at some point in their history (note that delta_mag is defined with
1183 # respect to the quiescent magnitude)
1184 #
1185 # also demand that the magnitude at some point is less than obs_mag_cutoff
1186 #
1187 # This is different from the dmag>dmag_cutoff check done in
1188 #
1189 # self._filter_on_photometry_then_chip_name()
1190 #
1191 # Now we have information about which observations actually detected
1192 # each object (in self._filter_on_photometry_then_chip_name(),
1193 # we assumed that every object was detected at every time step).
1195 photometrically_valid_obj = []
1196 for i_obj in range(len(chunk)):
1197 keep_it = False
1198 valid_times = np.where(time_arr[i_obj] > 0)
1199 if len(valid_times[0]) == 0:
1200 continue
1201 for i_filter in range(len(mag_names)):
1202 if np.abs(dmag_arr_transpose[i_obj][i_filter][valid_times]).max() > dmag_cutoff:
1203 dmag_min = dmag_arr_transpose[i_obj][i_filter][valid_times].min()
1204 if q_m_dict[i_filter][i_obj] + dmag_min <= obs_mag_cutoff[i_filter]:
1205 keep_it = True
1206 break
1207 if keep_it:
1208 photometrically_valid_obj.append(i_obj)
1209 photometrically_valid_obj = np.array(photometrically_valid_obj)
1211 del dmag_arr_transpose
1212 gc.collect()
1214 if np.abs(dmag_arr).max() < dmag_cutoff:
1215 continue
1217 completely_valid = np.zeros(len(chunk), dtype=int)
1219 ############################
1220 # Process and output sources
1221 #
1222 for i_obs, obs_dex in enumerate(obs_valid_dex):
1223 obs = self._obs_list[obs_dex]
1224 obshistid = obs.OpsimMetaData['obsHistID']
1226 obs_mag = obs.bandpass
1227 actual_i_mag = mag_name_to_int[obs_mag]
1228 assert mag_names[actual_i_mag] == obs_mag
1230 # only include those sources which fall on a detector for this pointing
1231 valid_chip_name, valid_xpup, valid_ypup, chip_valid_obj = chip_name_dict[i_obs]
1233 actually_valid_obj = np.intersect1d(photometrically_valid_obj, chip_valid_obj)
1234 if len(actually_valid_obj) == 0:
1235 continue
1237 try:
1238 completely_valid[actually_valid_obj] += 1
1239 except:
1240 print('failed')
1241 print(actually_valid_obj)
1242 print(completely_valid)
1243 raise
1245 valid_sources = chunk[actually_valid_obj]
1246 local_column_cache = {}
1247 local_column_cache['deltaMagAvro'] = OrderedDict([('delta_%smag' % mag_names[i_mag],
1248 dmag_arr[i_obs][i_mag][actually_valid_obj])
1249 for i_mag in range(len(mag_names))])
1251 local_column_cache['chipName'] = valid_chip_name[actually_valid_obj]
1252 local_column_cache['pupilFromSky'] = OrderedDict([('x_pupil', valid_xpup[actually_valid_obj]),
1253 ('y_pupil', valid_ypup[actually_valid_obj])])
1255 cat = cat_list[i_obs]
1256 i_valid_chunk = 0
1257 for valid_chunk, chunk_map in cat.iter_catalog_chunks(query_cache=[valid_sources],
1258 column_cache=local_column_cache):
1259 i_valid_chunk += 1
1260 assert i_valid_chunk == 1
1261 n_time_last += len(valid_chunk[0])
1262 length_of_chunk = len(valid_chunk[chunk_map['uniqueId']])
1263 cache_tag = '%d_%d' % (obshistid, i_chunk)
1264 output_data_cache[cache_tag] = {}
1266 for col_name in ('uniqueId', 'raICRS', 'decICRS', 'flux', 'dflux', 'SNR',
1267 'chipNum', 'xPix', 'yPix'):
1269 output_data_cache[cache_tag][col_name] = valid_chunk[chunk_map[col_name]]
1271 n_rows_cached += length_of_chunk
1273 completely_valid = np.where(completely_valid > 0)
1274 for i_filter in range(6):
1275 values = ((int(unq[completely_valid][i_q]),
1276 i_filter,
1277 q_f_dict[i_filter][completely_valid][i_q],
1278 q_snr_dict[i_filter][completely_valid][i_q])
1279 for i_q in range(len(completely_valid[0])))
1280 cursor.executemany('INSERT INTO quiescent_flux VALUES (?,?,?,?)', values)
1281 conn.commit()
1283 values = ((int(unq[completely_valid][i_q]),
1284 q_ra[completely_valid][i_q],
1285 q_dec[completely_valid][i_q],
1286 q_pmra[completely_valid][i_q],
1287 q_pmdec[completely_valid][i_q],
1288 q_parallax[completely_valid][i_q],
1289 q_tai)
1290 for i_q in range(len(completely_valid[0])))
1292 cursor.executemany('INSERT INTO baseline_astrometry VALUES (?,?,?,?,?,?,?)', values)
1294 if n_rows_cached >= write_every:
1295 self.acquire_lock()
1296 with open(log_file_name, 'a') as out_file:
1297 out_file.write('%d is writing \n' % os.getpid())
1299 print('%d is writing' % os.getpid())
1301 self.release_lock()
1303 n_rows += self._output_alert_data(conn, output_data_cache)
1304 output_data_cache = {}
1305 n_rows_cached = 0
1307 if n_rows > 0:
1308 self.acquire_lock()
1309 with open(log_file_name, 'a') as out_file:
1310 elapsed = (time.time()-t_before_obj)/3600.0
1311 elapsed_per = elapsed/n_rows
1312 rows_per_chunk = float(n_rows)/float(i_chunk)
1313 total_projection = 1000.0*rows_per_chunk*elapsed_per
1314 out_file.write('\n %d n_obj %d %d trimmed %d\n' %
1315 (this_pid, n_obj, n_actual_obj, n_htmid_trim))
1316 out_file.write(' elapsed %.2e hrs per row %.2e total %2e\n' %
1317 (elapsed, elapsed_per, total_projection))
1318 out_file.write(' n_time_last %d; rows %d\n' % (n_time_last, n_rows))
1320 out_file.write('%d is done writing\n' % os.getpid())
1322 print('\n %d n_obj %d %d trimmed %d' %
1323 (this_pid, n_obj, n_actual_obj, n_htmid_trim))
1324 print(' elapsed %.2e hrs per row %.2e total %2e' %
1325 (elapsed, elapsed_per, total_projection))
1326 print(' n_time_last %d; rows %d\n' % (n_time_last, n_rows))
1327 print('%d is done writing' % os.getpid())
1329 self.release_lock()
1331 if len(output_data_cache) > 0:
1332 n_rows += self._output_alert_data(conn, output_data_cache)
1333 output_data_cache = {}
1335 print('htmid %d that took %.2e hours; n_obj %d n_rows %d' %
1336 (htmid, (time.time()-t_start)/3600.0, n_obj, n_rows))
1338 self.acquire_lock()
1339 print("INDEXING %d" % htmid)
1340 self.release_lock()
1342 cursor.execute('CREATE INDEX unq_obs ON alert_data (uniqueId, obshistId)')
1343 cursor.execute('CREATE INDEX unq_flux ON quiescent_flux (uniqueId, band)')
1344 cursor.execute('CREATE INDEX obs ON metadata (obshistid)')
1345 cursor.execute('CREATE INDEX unq_ast ON baseline_astrometry (uniqueId)')
1346 conn.commit()
1348 self.acquire_lock()
1349 with open(log_file_name, 'a') as out_file:
1350 out_file.write('done with htmid %d -- %e %d\n' %
1351 (htmid, (time.time()-t_start)/3600.0, n_obj))
1352 self.release_lock()
1354 return n_rows