Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import numpy as np 

2import os 

3import re 

4import sqlite3 

5from collections import OrderedDict 

6import time 

7import gc 

8from lsst.utils import getPackageDir 

9from lsst.sims.catalogs.definitions import InstanceCatalog 

10from lsst.sims.utils import trixelFromHtmid, getAllTrixels 

11from lsst.sims.utils import levelFromHtmid, halfSpaceFromRaDec 

12from lsst.sims.utils import angularSeparation, ObservationMetaData 

13from lsst.sims.utils import arcsecFromRadians 

14from lsst.sims.catUtils.utils import _baseLightCurveCatalog 

15from lsst.sims.utils import _pupilCoordsFromRaDec 

16from lsst.sims.coordUtils import chipNameFromPupilCoordsLSST 

17from lsst.sims.coordUtils import pixelCoordsFromPupilCoordsLSST 

18 

19from lsst.sims.catalogs.decorators import compound, cached 

20from lsst.sims.photUtils import BandpassDict, Sed, calcSNR_m5 

21from lsst.sims.photUtils import PhotometricParameters 

22from lsst.sims.catUtils.mixins import VariabilityStars, AstrometryStars 

23from lsst.sims.catUtils.mixins import VariabilityGalaxies, AstrometryGalaxies 

24from lsst.sims.catUtils.mixins import CameraCoordsLSST, PhotometryBase 

25from lsst.sims.catUtils.mixins import ParametrizedLightCurveMixin 

26from lsst.sims.catUtils.mixins import create_variability_cache 

27 

28from lsst.sims.catUtils.baseCatalogModels import StarObj, GalaxyAgnObj 

29from sqlalchemy.sql import text 

30from lsst.sims.catalogs.db import ChunkIterator 

31 

32__all__ = ["AlertDataGenerator", 

33 "AlertStellarVariabilityCatalog", 

34 "AlertAgnVariabilityCatalog", 

35 "_baseAlertCatalog", 

36 "StellarAlertDBObj", 

37 "AgnAlertDBObj", 

38 "StellarAlertDBObjMixin"] 

39 

40 

41class StellarAlertDBObjMixin(object): 

42 """ 

43 Mimics StarObj class, except it allows you to directly query 

44 all objects in a trixel specified by an htmid. 

45 """ 

46 def query_columns_htmid(self, colnames=None, chunk_size=None, 

47 constraint=None, 

48 limit=None, htmid=None): 

49 """Execute a query from the primary catsim database 

50 

51 Execute a query, taking advantage of the spherical geometry library and 

52 htmid indexes on all catalog tables in the UW catsim database 

53 

54 **Parameters** 

55 

56 * colnames : list or None 

57 a list of valid column names, corresponding to entries in the 

58 `columns` class attribute. If not specified, all columns are 

59 queried. 

60 * chunk_size : int (optional) 

61 if specified, then return an iterator object to query the database, 

62 each time returning the next `chunk_size` elements. If not 

63 specified, all matching results will be returned. 

64 * constraint : str (optional) 

65 a string which is interpreted as SQL and used as a predicate on the query 

66 * limit : int (optional) 

67 limits the number of rows returned by the query 

68 * htmid is the htmid to be queried 

69 

70 **Returns** 

71 

72 * result : list or iterator 

73 If chunk_size is not specified, then result is a list of all 

74 items which match the specified query. If chunk_size is specified, 

75 then result is an iterator over lists of the given size. 

76 """ 

77 

78 # find the minimum and maximum htmid 

79 # (level=21 since that is what is implemented 

80 # on fatboy) that we are asking for 

81 # 

82 # Note that sqlalchemy does not like np.int64 

83 # as a data type 

84 current_level = levelFromHtmid(htmid) 

85 n_bits_off = 2*(21-current_level) 

86 htmid_min = int(htmid << n_bits_off) 

87 htmid_max = int((htmid+1) << n_bits_off) 

88 

89 query = self._get_column_query(colnames) 

90 

91 # add spatial constraints to query. 

92 

93 # Hint sql engine to seek on htmid 

94 if not self.tableid.endswith('forceseek'): 

95 query = query.with_hint(self.table, ' WITH(FORCESEEK)', 'mssql') 

96 

97 # SQL is not case sensitive but python is: 

98 if 'htmID' in self.columnMap: 

99 htmid_name = 'htmID' 

100 elif 'htmid' in self.columnMap: 

101 htmid_name = 'htmid' 

102 else: 

103 htmid_name = 'htmId' 

104 

105 # Range join on htmid ranges 

106 query = query.filter(self.table.c[htmid_name].between(htmid_min, htmid_max)) 

107 

108 if constraint is not None: 

109 query = query.filter(text(constraint)) 

110 

111 if limit is not None: 

112 query = query.limit(limit) 

113 

114 return ChunkIterator(self, query, chunk_size) 

115 

116 

117class StellarAlertDBObj(StellarAlertDBObjMixin, StarObj): 

118 pass 

119 

120 

121class AgnAlertDBObj(GalaxyAgnObj): 

122 """ 

123 Mimics GalaxyAgnObj class, except it allows you to directly query 

124 all objects in a trixel specified by an htmid. 

125 """ 

126 

127 columns = [('htmid', 0, np.int64), 

128 ('galtileid', None, np.int64), 

129 ('galid', None, str, 30), 

130 ('componentra', 'agnra*PI()/180.'), 

131 ('componentdec', 'agndec*PI()/180.'), 

132 #: This is actually a problem with the stored procedure. 

133 #: We need to be able to map columns other than 

134 #: just ra/dec to raJ2000/decJ2000. This gets 

135 #: important when we start perturbing the three galaxy components 

136 ('raJ2000', 'ra'), 

137 ('decJ2000', 'dec'), 

138 ('magNorm', 'magnorm_agn'), 

139 ('magNormAgn', 'magnorm_agn'), 

140 ('sedFilename', 'sedname_agn', str, 40), 

141 ('sedFilenameAgn', 'sedname_agn', str, 40), 

142 ('variabilityParameters', 'varParamStr', str, 256), 

143 ('lsst_u', 'u_ab'), 

144 ('lsst_g', 'g_ab'), 

145 ('lsst_r', 'r_ab'), 

146 ('lsst_i', 'i_ab'), 

147 ('lsst_z', 'z_ab'), 

148 ('lsst_y', 'y_ab')] 

149 

150 def query_columns_htmid(self, colnames=None, chunk_size=None, 

151 constraint=None, 

152 limit=None, htmid=None): 

153 """Execute a query from the primary catsim database 

154 

155 Execute a query, taking advantage of the spherical geometry library and 

156 htmid indexes on all catalog tables in the UW catsim database 

157 

158 **Parameters** 

159 

160 * colnames : list or None 

161 a list of valid column names, corresponding to entries in the 

162 `columns` class attribute. If not specified, all columns are 

163 queried. 

164 * chunk_size : int (optional) 

165 if specified, then return an iterator object to query the database, 

166 each time returning the next `chunk_size` elements. If not 

167 specified, all matching results will be returned. 

168 * constraint : str (optional) 

169 a string which is interpreted as SQL and used as a predicate on the query 

170 * limit : int (optional) 

171 limits the number of rows returned by the query 

172 * htmid is the htmid to be queried 

173 

174 **Returns** 

175 

176 * result : list or iterator 

177 If chunk_size is not specified, then result is a list of all 

178 items which match the specified query. If chunk_size is specified, 

179 then result is an iterator over lists of the given size. 

180 """ 

181 

182 trixel = trixelFromHtmid(htmid) 

183 ra_0, dec_0 = trixel.get_center() 

184 new_obs = ObservationMetaData(pointingRA=ra_0, pointingDec=dec_0, boundType='circle', 

185 boundLength=trixel.get_radius()+0.1) 

186 

187 self._queried_trixel = trixel 

188 self._queried_htmid_level = levelFromHtmid(htmid) 

189 

190 return self.query_columns(colnames=colnames, chunk_size=chunk_size, 

191 obs_metadata=new_obs, constraint=constraint, 

192 limit=limit) 

193 

194 def _final_pass(self, results): 

195 """Modify the results of raJ2000 and decJ2000 to be in radians. 

196 

197 Also filter the results so that any objects outside of the 

198 trixel specified in query_columns_htmid are returned with 

199 htmid=0. 

200 

201 **Parameters** 

202 

203 * results : Structured array of results from query 

204 

205 **Returns** 

206 

207 * results : Modified structured array 

208 

209 """ 

210 

211 if hasattr(self, '_queried_trixel'): 

212 htmid = self._queried_trixel.htmid 

213 htmid_21 = htmid << 2*(21-self._queried_htmid_level) 

214 assert levelFromHtmid(htmid_21) == 21 

215 contains_arr = self._queried_trixel.contains(results['raJ2000'], results['decJ2000']) 

216 results['htmid'] = np.where(contains_arr, htmid_21, 0) 

217 

218 results['raJ2000'] = np.radians(results['raJ2000']) 

219 results['decJ2000'] = np.radians(results['decJ2000']) 

220 return results 

221 

222 

223class _baseAlertCatalog(PhotometryBase, CameraCoordsLSST, _baseLightCurveCatalog): 

224 

225 column_outputs = ['htmid', 'uniqueId', 'raICRS', 'decICRS', 

226 'flux', 'SNR', 'dflux', 

227 'chipNum', 'xPix', 'yPix'] 

228 

229 default_formats = {'f': '%.4g'} 

230 

231 default_columns = [('properMotionRa', 0.0, float), 

232 ('properMotionDec', 0.0, float), 

233 ('parallax', 0.0, float)] 

234 

235 def iter_catalog_chunks(self, chunk_size=None, query_cache=None, column_cache=None): 

236 """ 

237 Returns an iterator over chunks of the catalog. 

238 

239 Parameters 

240 ---------- 

241 chunk_size : int, optional, defaults to None 

242 the number of rows to return from the database at a time. If None, 

243 returns the entire database query in one chunk. 

244 

245 query_cache : iterator over database rows, optional, defaults to None 

246 the result of calling db_obj.query_columns(). If query_cache is not 

247 None, this method will iterate over the rows in query_cache and produce 

248 an appropriate InstanceCatalog. DO NOT set to non-None values 

249 unless you know what you are doing. It is an optional 

250 input for those who want to repeatedly examine the same patch of sky 

251 without actually querying the database over and over again. If it is set 

252 to None (default), this method will handle the database query. 

253 

254 column_cache : a dict that will be copied over into the catalogs self._column_cache. 

255 Should be left as None, unless you know what you are doing. 

256 """ 

257 

258 if query_cache is None: 

259 # Call the original version of iter_catalog defined in the 

260 # InstanceCatalog class. This version of iter_catalog includes 

261 # the call to self.db_obj.query_columns, which the user would have 

262 # used to generate query_cache. 

263 for line in InstanceCatalog.iter_catalog_chunks(self, chunk_size=chunk_size): 

264 yield line 

265 else: 

266 # Otherwise iterate over the query cache 

267 transform_keys = list(self.transformations.keys()) 

268 for chunk in query_cache: 

269 self._set_current_chunk(chunk, column_cache=column_cache) 

270 chunk_cols = [self.transformations[col](self.column_by_name(col)) 

271 if col in transform_keys else 

272 self.column_by_name(col) 

273 for col in self.iter_column_names()] 

274 

275 if not hasattr(self, '_chunkColMap_output'): 

276 

277 self._chunkColMap_output = dict([(col, i) 

278 for i, col in 

279 enumerate(self.iter_column_names())]) 

280 

281 yield chunk_cols, self._chunkColMap_output 

282 

283 self._column_cache = {} 

284 self._current_chunk = None 

285 

286 @cached 

287 def get_chipName(self): 

288 if len(self.column_by_name('uniqueId')) == 0: 

289 return np.array([]) 

290 raise RuntimeError("Should not get this far in get_chipName") 

291 

292 @compound('x_pupil', 'y_pupil') 

293 def get_pupilFromSky(self): 

294 if len(self.column_by_name('uniqueId')) == 0: 

295 return np.array([[], []]) 

296 raise RuntimeError("Should not get this far in get_pupilFromSky") 

297 

298 @cached 

299 def get_chipNum(self): 

300 """ 

301 Concatenate the digits in 'R:i,j S:m,n' to make the chip number ijmn 

302 """ 

303 chip_name = self.column_by_name('chipName') 

304 return np.array([int(''.join(re.findall(r'\d+', name))) if name is not None else 0 

305 for name in chip_name]) 

306 

307 @compound('xPix', 'yPix') 

308 def get_pixelCoordinates(self): 

309 xpup = self.column_by_name('x_pupil') 

310 ypup = self.column_by_name('y_pupil') 

311 chip_name = self.column_by_name('chipName') 

312 xpix, ypix = pixelCoordsFromPupilCoordsLSST(xpup, ypup, chipName=chip_name, 

313 band=self.obs_metadata.bandpass, 

314 includeDistortion=True) 

315 return np.array([xpix, ypix]) 

316 

317 @compound('delta_umag', 'delta_gmag', 'delta_rmag', 

318 'delta_imag', 'delta_zmag', 'delta_ymag') 

319 def get_deltaMagAvro(self): 

320 ra = self.column_by_name('raJ2000') 

321 if len(ra) == 0: 

322 return np.array([[], [], [], [], [], []]) 

323 

324 raise RuntimeError("Should not have gotten this far in delta mag getter") 

325 

326 @compound('lsst_u', 'lsst_g', 'lsst_r', 'lsst_i', 'lsst_z', 'lsst_y') 

327 def get_lsst_magnitudes(self): 

328 """ 

329 getter for LSST stellar magnitudes 

330 """ 

331 

332 magnitudes = np.array([self.column_by_name('quiescent_lsst_u'), 

333 self.column_by_name('quiescent_lsst_g'), 

334 self.column_by_name('quiescent_lsst_r'), 

335 self.column_by_name('quiescent_lsst_i'), 

336 self.column_by_name('quiescent_lsst_z'), 

337 self.column_by_name('quiescent_lsst_y')]) 

338 

339 delta = np.array([self.column_by_name('delta_umag'), 

340 self.column_by_name('delta_gmag'), 

341 self.column_by_name('delta_rmag'), 

342 self.column_by_name('delta_imag'), 

343 self.column_by_name('delta_zmag'), 

344 self.column_by_name('delta_ymag')]) 

345 magnitudes += delta 

346 

347 return magnitudes 

348 

349 @compound('mag', 'dmag', 'quiescent_mag') 

350 def get_alertPhotometry(self): 

351 mag = self.column_by_name('lsst_%s' % self.obs_metadata.bandpass) 

352 quiescent_mag = self.column_by_name('quiescent_lsst_%s' % self.obs_metadata.bandpass) 

353 dmag = mag - quiescent_mag 

354 

355 return np.array([mag, dmag, quiescent_mag]) 

356 

357 @compound('flux', 'dflux', 'SNR') 

358 def get_alertFlux(self): 

359 quiescent_mag = self.column_by_name('quiescent_mag') 

360 mag = self.column_by_name('mag') 

361 if not hasattr(self, '_dummy_sed'): 

362 self._dummy_sed = Sed() 

363 if not hasattr(self, 'lsstBandpassDict'): 

364 self.lsstBandpassDict = BandpassDict.loadTotalBandpassesFromFiles() 

365 if not hasattr(self, 'phot_params'): 

366 self.phot_params = PhotometricParameters() 

367 if not hasattr(self, '_gamma'): 

368 self._gamma = None 

369 

370 quiescent_flux = self._dummy_sed.fluxFromMag(quiescent_mag) 

371 flux = self._dummy_sed.fluxFromMag(mag) 

372 dflux = flux - quiescent_flux 

373 

374 snr_tot, gamma = calcSNR_m5(mag, self.lsstBandpassDict[self.obs_metadata.bandpass], 

375 self.obs_metadata.m5[self.obs_metadata.bandpass], 

376 self.phot_params, gamma=self._gamma) 

377 

378 if self._gamma is None: 

379 self._gamma = gamma 

380 

381 return np.array([flux, dflux, snr_tot]) 

382 

383 

384class AlertStellarVariabilityCatalog(_baseAlertCatalog, 

385 VariabilityStars, 

386 AstrometryStars): 

387 

388 @compound('quiescent_lsst_u', 'quiescent_lsst_g', 'quiescent_lsst_r', 

389 'quiescent_lsst_i', 'quiescent_lsst_z', 'quiescent_lsst_y') 

390 def get_quiescent_lsst_magnitudes(self): 

391 return np.array([self.column_by_name('umag'), self.column_by_name('gmag'), 

392 self.column_by_name('rmag'), self.column_by_name('imag'), 

393 self.column_by_name('zmag'), self.column_by_name('ymag')]) 

394 

395 

396class AlertAgnVariabilityCatalog(_baseAlertCatalog, 

397 VariabilityGalaxies, 

398 AstrometryGalaxies): 

399 

400 @compound('quiescent_lsst_u', 'quiescent_lsst_g', 'quiescent_lsst_r', 

401 'quiescent_lsst_i', 'quiescent_lsst_z', 'quiescent_lsst_y') 

402 def get_quiescent_lsst_magnitudes(self): 

403 return np.array([self.column_by_name('u_ab'), self.column_by_name('g_ab'), 

404 self.column_by_name('r_ab'), self.column_by_name('i_ab'), 

405 self.column_by_name('z_ab'), self.column_by_name('y_ab')]) 

406 

407 

408class AlertDataGenerator(object): 

409 """ 

410 This class will read in astrophysical sources and variability 

411 models from CatSim, observe them with a simulated OpSim 

412 cadence, and write a series of sqlite files containing all 

413 of the simulated observations that could trigger an alert. 

414 

415 In order to make this calculation as efficient as possible, 

416 the class works by partitioning the sky according to the 

417 Hierarchical Triangular Mesh (HTM) of 

418 

419 Kunszt P., Szalay A., Thakar A. (2006) in "Mining The Sky", 

420 Banday A, Zaroubi S, Bartelmann M. eds. 

421 ESO Astrophysics Symposia 

422 https://www.researchgate.net/publication/226072008_The_Hierarchical_Triangular_Mesh 

423 

424 Szalay A. et al. (2007) 

425 "Indexing the Sphere with the Hierarchical Triangular Mesh" 

426 arXiv:cs/0701164 

427 

428 and simulating all of the observations in a given trixel (the 

429 elementary unit of the HTM) at once. Accordintly, the outputs 

430 produced by this class are files named like 

431 

432 prefix_NNNN_sqlite.db 

433 

434 where prefix is specified by theuser and NNNN is the htmid, the 

435 unique identifying integer, corresponding to each simulated trixel. 

436 

437 The proper way to run this class is to instantiate it, run 

438 subdivide_obs on a list of ObservationMetaData corresponding 

439 to the OpSim pointings to be simulated, and then running 

440 alert_data_from_htmid on each of the htmid in the class property 

441 htmid_list. This last step can easily be parallelized using python's 

442 multiprocessing module, with each process handling a different htmid. 

443 

444 The sqlite files produced by alert_data_from_htmid will each contain 

445 four tables. They are as follows. Columns are listed below the 

446 tables. 

447 

448 alert_data 

449 ---------- 

450 uniqueId -- int -- a unique identifier for each astrophysical object 

451 

452 obshistId -- int -- a unique identifier for each OpSim pointing 

453 

454 xPix -- float -- the x pixel coordinate of the source on the focal 

455 plane 

456 

457 yPix -- float -- the y pixel coordinate of the source on the focal 

458 plane 

459 

460 dflux -- float -- the difference in flux between the source's current 

461 flux and its quiescent flux (the source's quiescent flux can be found 

462 in the quiescent_flux table). This is in units of Janskys. 

463 

464 snr -- float -- the signal to noise of the current detection of the 

465 source (not the signal to noise of the source's detection in a simulated 

466 difference image). 

467 

468 ra -- float -- the current RA of the source in degrees 

469 

470 dec -- float -- the current Declination of the source in degrees. 

471 

472 The alert_data table has a mult-column index on uniqueId and obshistId. 

473 

474 metadata 

475 -------- 

476 obshistId -- int --- a unique identifier for each OpSim pointing 

477 

478 TAI -- float -- the International Atomic Time of the observation 

479 as an MJD (in days) 

480 

481 band -- int -- denotes the filter used for the observation 

482 (0=u, 1=g, 2=r, etc.) 

483 

484 The metadata table is indexed on obshistId 

485 

486 quiescent_flux 

487 -------------- 

488 uniqueId -- int -- a unique identifier for each astrophysical source 

489 

490 band -- int -- an integer denoting each LSST filter (0=u, 1=g, 2=r, etc.) 

491 

492 flux -- float -- the flux of the source through the filter specified by 

493 band (in units of Janskys) 

494 

495 snr -- float -- the signal to noise ratio of the source in the given 

496 band with m5 taken from Table 2 of the overview paper (arXiv:0805.2366) 

497 

498 The quiescent_flux table has a multi-column index on uniqueId and band. 

499 

500 baseline_astrometry 

501 ------------------- 

502 uniqueId -- int -- a unique identifier for each astrophysical source 

503 

504 TAI -- float -- the International Atomic Time of the baseline astrometric 

505 measurements below as a MJD (in days) 

506 

507 ra -- float -- the RA of the source at TAI in degrees 

508 

509 dec -- float -- the Declination of the source at TAI in degrees 

510 

511 pmRA -- float -- the RA proper motion of the source in milliarcseconds/year 

512 

513 pmDec -- float -- the Declination proper motion of the source in 

514 milliarcseconds/year 

515 

516 parallax -- float -- the parallax of the source in milliarcseconds 

517 

518 The baseline_astrometry table is indexed on uniqueId 

519 

520 """ 

521 

522 def __init__(self, 

523 testing=False): 

524 """ 

525 Parameters 

526 ---------- 

527 testing as a boolean that should only be True when running the unit tests. 

528 If True, it prevents the AlertDataGenerator from pre-caching variability 

529 models, which aids performance, but uses more memory than we want to use 

530 in a unit test. 

531 """ 

532 

533 self._variability_cache = create_variability_cache() 

534 self._stdout_lock = None 

535 if not testing: 

536 plm = ParametrizedLightCurveMixin() 

537 plm.load_parametrized_light_curves(variability_cache = self._variability_cache) 

538 self.bp_dict = BandpassDict.loadTotalBandpassesFromFiles() 

539 

540 # This is a file that lists the maximum amplitude of variability 

541 # for each of the Kepler-derived light curve models. It will be 

542 # used by the stellar variability model to figure out which 

543 # stars can be skipped because they will never vary above 

544 # the alert-triggering threshold. 

545 self._dmag_lookup_file_exists = True 

546 self._dmag_lookup_file = os.path.join(getPackageDir('sims_data'), 

547 'catUtilsData', 

548 'kplr_dmag_171204.txt') 

549 

550 if not os.path.exists(self._dmag_lookup_file): 

551 if not testing: 

552 script_name = os.path.join(getPackageDir('sims_catUtils'), 'support_scripts', 

553 'get_kepler_dmag.sh') 

554 raise RuntimeError('\n%s does not exist; run the script\n\n%s\n\n' % 

555 script_name) 

556 else: 

557 self._dmag_lookup_file_exists = False 

558 

559 def acquire_lock(self): 

560 """ 

561 If running with multiprocessing, acquire 

562 the lock. 

563 """ 

564 if self._stdout_lock is not None: 

565 self._stdout_lock.acquire() 

566 

567 def release_lock(self): 

568 """ 

569 If running with multiprocessing, release 

570 the lock. 

571 """ 

572 if self._stdout_lock is not None: 

573 self._stdout_lock.release() 

574 

575 def subdivide_obs(self, obs_list, htmid_level=6): 

576 """ 

577 Take a list of ObservationMetaData and subdivide 

578 them according to which trixels (see htmModule.py 

579 in sims_utils) they intersect. 

580 

581 Parameters 

582 ---------- 

583 obs_list is a list of ObservationMetaData 

584 

585 htmid_level is an int denoting the level of 

586 the HTM mesh you want to use to tile the sky 

587 (higher htmid_level corresponds to a finer 

588 tiling). Default = 6 

589 

590 Returns 

591 ------- 

592 Nothing. 

593 

594 After running this method, this AlertGenerator 

595 will contain the following data. 

596 

597 - a list of the htmid of every trixel intersected 

598 by the fields of view specified in obs_list. This 

599 list is accessible from the property 

600 AlertGenerator.htmid_list 

601 

602 - a dict mapping each htmid to the ObservationMetaData 

603 from obs_list that intersect it. The method 

604 AlertGenerator.obs_from_htmid(htmid) will return a 

605 list of all of the ObservationMetaData that intersect 

606 the trixel specified by htmid. 

607 """ 

608 self._trixel_dict = getAllTrixels(htmid_level) 

609 valid_htmid = [] 

610 for htmid in self._trixel_dict: 

611 if levelFromHtmid(htmid) == htmid_level: 

612 valid_htmid.append(htmid) 

613 

614 obs_list = np.array(obs_list) 

615 self._obs_list = obs_list 

616 obs_ra_list = [] 

617 obs_dec_list = [] 

618 halfspace_list = [] 

619 for obs in obs_list: 

620 obs_ra_list.append(obs.pointingRA) 

621 obs_dec_list.append(obs.pointingDec) 

622 hs = halfSpaceFromRaDec(obs.pointingRA, 

623 obs.pointingDec, 

624 obs.boundLength) 

625 halfspace_list.append(hs) 

626 

627 obs_ra_list = np.array(obs_ra_list) 

628 obs_dec_list = np.array(obs_dec_list) 

629 halfspace_list = np.array(halfspace_list) 

630 

631 self._htmid_dict = {} 

632 self._htmid_list = [] 

633 n_obs_list = [] 

634 fov_radius = 1.75 

635 for i_htmid, htmid in enumerate(valid_htmid): 

636 trixel = self._trixel_dict[htmid] 

637 ra_c, dec_c = trixel.get_center() 

638 radius = trixel.get_radius() 

639 obs_distance = angularSeparation(ra_c, dec_c, obs_ra_list, obs_dec_list) 

640 valid_obs = np.where(obs_distance < radius + fov_radius) 

641 if len(valid_obs[0]) > 0: 

642 final_obs_list = [] 

643 for obs_dex in valid_obs[0]: 

644 hs = halfspace_list[obs_dex] 

645 obs = obs_list[obs_dex] 

646 if hs.contains_trixel(trixel) != 'outside': 

647 final_obs_list.append(obs_dex) 

648 

649 if len(final_obs_list) == 0: 

650 continue 

651 

652 self._htmid_dict[htmid] = np.array(final_obs_list) 

653 self._htmid_list.append(htmid) 

654 n_obs_list.append(len(final_obs_list)) 

655 

656 n_obs_list = np.array(n_obs_list) 

657 self._htmid_list = np.array(self._htmid_list) 

658 sorted_dex = np.argsort(-1.0*n_obs_list) 

659 self._htmid_list = self._htmid_list[sorted_dex] 

660 print('done subdividing obs list -- %d htmid' % 

661 len(self._htmid_list)) 

662 

663 @property 

664 def htmid_list(self): 

665 """ 

666 A list of the unique htmids corresponding to the trixels 

667 that need to be queried to generate the alert data 

668 """ 

669 return self._htmid_list 

670 

671 def n_obs(self, htmid): 

672 """ 

673 Return the number of observations that intersect 

674 the trixel specified by htmid. 

675 

676 Must run subdivide_obs in order for this method to 

677 work. 

678 """ 

679 return len(self._htmid_dict[htmid]) 

680 

681 def obs_from_htmid(self, htmid): 

682 """ 

683 Return a numpy array containing all of the ObservationMetaData 

684 that intersect the trixel specified by htmid. 

685 

686 Must run subdivide_obs in order for this method to 

687 work. 

688 """ 

689 return self._obs_list[self._htmid_dict[htmid]] 

690 

691 def _output_alert_data(self, conn, data_cache): 

692 """ 

693 Write a cache of alert data to the sqlite file currently open. 

694 

695 Parameters 

696 ---------- 

697 conn is the connection to the sqlite file (already open) 

698 

699 data_cache is a dict containing all of the data to be written. 

700 It will keyed on a string like 'i_j' where i is the obshistID 

701 of an OpSim pointing and j is an arbitrary integer. That key 

702 will lead to another dict keyed on the columns being output to 

703 the sqlite file. The values of this second layer of dict are 

704 numpy arrays. 

705 

706 Returns 

707 ------- 

708 The number of rows written to the sqlite file 

709 """ 

710 

711 cursor = conn.cursor() 

712 n_rows_0 = cursor.execute('SELECT COUNT(uniqueId) FROM alert_data').fetchall() 

713 

714 chunk_lengths = np.zeros(len(data_cache)) 

715 

716 for i_cache_tag, cache_tag in enumerate(data_cache): 

717 obshistid = int(cache_tag.split('_')[0]) 

718 n_obj = len(data_cache[cache_tag]['uniqueId']) 

719 chunk_lengths[i_cache_tag] = n_obj 

720 

721 values = ((int(data_cache[cache_tag]['uniqueId'][i_obj]), 

722 obshistid, 

723 data_cache[cache_tag]['xPix'][i_obj], 

724 data_cache[cache_tag]['yPix'][i_obj], 

725 int(data_cache[cache_tag]['chipNum'][i_obj]), 

726 data_cache[cache_tag]['dflux'][i_obj], 

727 data_cache[cache_tag]['SNR'][i_obj], 

728 np.degrees(data_cache[cache_tag]['raICRS'][i_obj]), 

729 np.degrees(data_cache[cache_tag]['decICRS'][i_obj])) 

730 for i_obj in range(n_obj)) 

731 cursor.executemany('INSERT INTO alert_data VALUES (?,?,?,?,?,?,?,?,?)', values) 

732 conn.commit() 

733 

734 n_rows_1 = cursor.execute('SELECT COUNT(uniqueId) FROM alert_data').fetchall() 

735 conn.commit() 

736 n_written = (n_rows_1[0][0]-n_rows_0[0][0]) 

737 

738 return n_written 

739 

740 def _filter_on_photometry_then_chip_name(self, chunk, column_query, 

741 obs_valid_dex, expmjd_list, 

742 photometry_catalog, 

743 dmag_cutoff): 

744 """ 

745 Determine which simulated observations are actually worth storing 

746 by first figuring out which observations of which objects are 

747 photometrically detectable and alert-worthy, then determining 

748 which of those actually fall on an LSST detector. 

749 

750 Parameters 

751 ---------- 

752 chunk is the output yielded from a CatSim ChunkIterator. It is 

753 a numpy recarray representing one chunk_size query from the 

754 underlying simulations database 

755 

756 column_query is a list of the columns that were queried from 

757 the database 

758 

759 obs_valid_dex is a list of integers corresponding to indexes in 

760 self._obs_list of the ObservationMetaData that are actually valid 

761 for the trixel currently being simulated 

762 

763 expmjd_list is a numpy array of the TAI dates of ObservtionMetaData 

764 represented by obs_valid_dex 

765 

766 photometry_catalog is an instantiation of the InstanceCatalog class 

767 being used to calculate magnitudes for these variable sources. 

768 

769 Outputs 

770 ------- 

771 chip_name_dict is a dict keyed on i_obs (which is the index of 

772 an ObservationMetaData's position in obs_valid_dex, NOT its 

773 position in self._obs_list). The values of chip_name_dict are 

774 tuples containing: 

775 - a list of the names of the detectors that objects from chunk 

776 landed on (including Nones for those objects that did not land 

777 on any detector) 

778 

779 - a list of the xPupil coords for every object in chunk 

780 

781 - a list of the yPupil coords for every object in chunk 

782 

783 - a list of the indexes in chunk of those objects which actually 

784 landed on a detector 

785 

786 dmag_arr is a numpy array of the delta_magnitudes of every object 

787 in chunk. dmag_arr[11][3][4] is the delta_magnitude of chunk[4] 

788 in the 3rd band (i.e. the i band) at TAI = expmjd[11]. 

789 

790 dmag_arr_transpose is dmag_arr with the time and object columns 

791 transposed so that dmag_arr_transpose[4][3][11] == dmag_arr[11][3][4]. 

792 

793 time_arr is an array of integers with shape == (len(chunk), len(obs_valid_dex)). 

794 A -1 in time_arr means that that combination of object and observation did 

795 not yield a valid observation. A +1 means that the object and observation 

796 combination are valid. 

797 """ 

798 

799 ###################################################### 

800 # Calculate the delta_magnitude for all of the sources 

801 # 

802 photometry_catalog._set_current_chunk(chunk) 

803 dmag_arr = photometry_catalog.applyVariability(chunk['varParamStr'], 

804 variability_cache=self._variability_cache, 

805 expmjd=expmjd_list).transpose((2, 0, 1)) 

806 

807 dmag_arr_transpose = dmag_arr.transpose(2, 1, 0) 

808 

809 n_raw_obj = len(chunk) 

810 photometrically_valid = -1*np.ones(n_raw_obj, dtype=int) 

811 for i_obj in range(n_raw_obj): 

812 keep_it = False 

813 for i_filter in range(6): 

814 if np.abs(dmag_arr_transpose[i_obj][i_filter]).max() >= dmag_cutoff: 

815 keep_it = True 

816 break 

817 if keep_it: 

818 photometrically_valid[i_obj] = 1 

819 

820 photometrically_valid = np.where(photometrically_valid >= 0) 

821 

822 if 'properMotionRa'in column_query: 

823 pmra = chunk['properMotionRa'][photometrically_valid] 

824 pmdec = chunk['properMotionDec'][photometrically_valid] 

825 px = chunk['parallax'][photometrically_valid] 

826 vrad = chunk['radialVelocity'][photometrically_valid] 

827 else: 

828 pmra = None 

829 pmdec = None 

830 px = None 

831 vrad = None 

832 

833 ################################################################### 

834 # Figure out which sources actually land on an LSST detector during 

835 # the observations in question 

836 # 

837 chip_name_dict = {} 

838 

839 # time_arr will keep track of which objects appear in which observations; 

840 # 1 means the object appears; -1 means it does not 

841 time_arr_transpose = -1*np.ones((len(obs_valid_dex), len(chunk['raJ2000'])), 

842 dtype=int) 

843 

844 for i_obs, obs_dex in enumerate(obs_valid_dex): 

845 obs = self._obs_list[obs_dex] 

846 chip_name_list = np.array([None]*n_raw_obj) 

847 xpup_list = np.zeros(n_raw_obj, dtype=float) 

848 ypup_list = np.zeros(n_raw_obj, dtype=float) 

849 chip_int_arr = -1*np.ones(len(chip_name_list), dtype=int) 

850 

851 if len(photometrically_valid[0]) > 0: 

852 xpup_list_val, ypup_list_val = _pupilCoordsFromRaDec(chunk['raJ2000'][photometrically_valid], 

853 chunk['decJ2000'][photometrically_valid], 

854 pm_ra=pmra, pm_dec=pmdec, 

855 parallax=px, v_rad=vrad, 

856 obs_metadata=obs) 

857 

858 xpup_list[photometrically_valid] = xpup_list_val 

859 ypup_list[photometrically_valid] = ypup_list_val 

860 

861 chip_name_list[photometrically_valid] = chipNameFromPupilCoordsLSST(xpup_list_val, 

862 ypup_list_val) 

863 

864 for i_chip, name in enumerate(chip_name_list): 

865 if name is not None: 

866 chip_int_arr[i_chip] = 1 

867 

868 valid_obj = np.where(chip_int_arr > 0) 

869 time_arr_transpose[i_obs][valid_obj] = 1 

870 

871 chip_name_dict[i_obs] = (chip_name_list, 

872 xpup_list, 

873 ypup_list, 

874 valid_obj) 

875 

876 time_arr = time_arr_transpose.transpose() 

877 assert len(chip_name_dict) == len(obs_valid_dex) 

878 

879 return chip_name_dict, dmag_arr, dmag_arr_transpose, time_arr 

880 

881 def alert_data_from_htmid(self, htmid, dbobj, 

882 dmag_cutoff=0.005, 

883 chunk_size=1000, write_every=10000, 

884 output_dir='.', output_prefix='', 

885 log_file_name=None, 

886 photometry_class=None, 

887 chunk_cutoff=-1, 

888 lock=None): 

889 

890 """ 

891 Generate an sqlite file with all of the alert data for a given 

892 trixel. 

893 

894 Parameters 

895 ---------- 

896 htmid is an integer denoting the trixel from self.htmid_list that should 

897 be simulated 

898 

899 dbobj is a CatalogDBObject connecting to the data underlying the simulation 

900 

901 dmag_cutoff indicates the minimum change magnitude needed to trigger a 

902 simulated alert 

903 

904 chunk_size denotes the number of objects to query from the database and 

905 process at one time 

906 

907 write_every indicates how often to write to the sqlite file (i.e. the 

908 code will pause the simulation process and write to the sqlite file 

909 when it has accumulated this many valid observations) 

910 

911 output_dir is the directory in which to create the sqlite file 

912 

913 output_prefix is the prefix of the sqlite file's name 

914 

915 log_file_name is the name of a text file where progress will be written 

916 

917 photometry_class is a InstanceCatalog class (not an instantiation) that 

918 contains the methods for calculating the photometry associated with the 

919 simulated alerts (see AlertStellarVariabilityCatalog and 

920 AlertAgnVariabilityCatalog in this module) 

921 

922 chunk_cutoff is an optional int; stop the simulation after this many 

923 chunks have been processed. This is for testing purposes. 

924 If chunk_cutoff == -1, the code will process all of the astrophysical 

925 objects in the trixel. 

926 

927 lock is a multiprocessing.Lock() for use if running multiple 

928 instances of alert_data_from_htmid. This will prevent multiple processes 

929 from writing to the log file or stdout simultaneously. 

930 """ 

931 

932 htmid_level = levelFromHtmid(htmid) 

933 if log_file_name is None: 

934 raise RuntimeError('must specify log_file_name') 

935 

936 if ('_PARAMETRIZED_LC_DMAG_LOOKUP' not in self._variability_cache and 

937 self._dmag_lookup_file_exists): 

938 

939 self._variability_cache['_PARAMETRIZED_LC_DMAG_CUTOFF'] = dmag_cutoff 

940 self._variability_cache['_PARAMETRIZED_LC_DMAG_LOOKUP'] = {} 

941 

942 with open(self._dmag_lookup_file, 'r') as in_file: 

943 for line in in_file: 

944 if line[0] == '#': 

945 continue 

946 params = line.split() 

947 self._variability_cache['_PARAMETRIZED_LC_DMAG_LOOKUP'][int(params[0])] = float(params[1]) 

948 

949 self._stdout_lock = lock 

950 this_pid = os.getpid() 

951 

952 t_start = time.time() # so that we can get a sense of how long the full 

953 # simulation will take 

954 

955 desired_columns = [] 

956 desired_columns.append('simobjid') 

957 desired_columns.append('variabilityParameters') 

958 desired_columns.append('varParamStr') 

959 desired_columns.append('raJ2000') 

960 desired_columns.append('decJ2000') 

961 desired_columns.append('properMotionRa') 

962 desired_columns.append('properMotionDec') 

963 desired_columns.append('parallax') 

964 desired_columns.append('radialVelocity') 

965 desired_columns.append('ebv') 

966 desired_columns.append('redshift') 

967 desired_columns.append('htmid') 

968 

969 if 'umag' in dbobj.columnMap: 

970 desired_columns.append('umag') 

971 desired_columns.append('gmag') 

972 desired_columns.append('rmag') 

973 desired_columns.append('imag') 

974 desired_columns.append('zmag') 

975 desired_columns.append('ymag') 

976 elif 'u_ab' in dbobj.columnMap: 

977 desired_columns.append('u_ab') 

978 desired_columns.append('g_ab') 

979 desired_columns.append('r_ab') 

980 desired_columns.append('i_ab') 

981 desired_columns.append('z_ab') 

982 desired_columns.append('y_ab') 

983 else: 

984 raise RuntimeError('Not sure what quiescent ' 

985 'LSST magnitudes are called ' 

986 'in this CatalogDBObject') 

987 

988 if photometry_class is None: 

989 raise RuntimeError('Must specify photometry_class') 

990 

991 if os.path.exists(output_dir) and not os.path.isdir(output_dir): 

992 raise RuntimeError('%s is not a dir' % output_dir) 

993 if not os.path.exists(output_dir): 

994 os.mkdir(output_dir) 

995 

996 dummy_sed = Sed() 

997 

998 # a dummy call to make sure that the initialization 

999 # is done before we attempt to parallelize calls 

1000 # to chipNameFromRaDecLSST 

1001 chipNameFromPupilCoordsLSST(0.0, 0.0) 

1002 

1003 mag_names = ('u', 'g', 'r', 'i', 'z', 'y') 

1004 

1005 phot_params = PhotometricParameters() 

1006 

1007 # from Table 2 of the overview paper 

1008 obs_mag_cutoff = (23.68, 24.89, 24.43, 24.0, 24.45, 22.60) 

1009 

1010 gamma_template = {} 

1011 for i_filter in range(6): 

1012 gamma_template[i_filter] = None 

1013 

1014 obs_valid_dex = self._htmid_dict[htmid] 

1015 print('n valid obs %d' % len(obs_valid_dex)) 

1016 

1017 cat_list = [] 

1018 expmjd_list = [] 

1019 mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 

1020 'i': 3, 'z': 4, 'y': 5} 

1021 for obs_dex in obs_valid_dex: 

1022 obs = self._obs_list[obs_dex] 

1023 cat = photometry_class(dbobj, obs_metadata=obs) 

1024 cat.lsstBandpassDict = self.bp_dict 

1025 cat_list.append(cat) 

1026 expmjd_list.append(obs.mjd.TAI) 

1027 

1028 expmjd_list = np.array(expmjd_list) 

1029 cat_list = np.array(cat_list) 

1030 sorted_dex = np.argsort(expmjd_list) 

1031 

1032 expmjd_list = expmjd_list[sorted_dex] 

1033 cat_list = cat_list[sorted_dex] 

1034 obs_valid_dex = obs_valid_dex[sorted_dex] 

1035 

1036 available_columns = list(dbobj.columnMap.keys()) 

1037 column_query = [] 

1038 for col in desired_columns: 

1039 if col in available_columns: 

1040 column_query.append(col) 

1041 

1042 n_bits_off = 2*(21-htmid_level) 

1043 

1044 data_iter = dbobj.query_columns_htmid(colnames=column_query, 

1045 htmid=htmid, 

1046 chunk_size=chunk_size) 

1047 

1048 photometry_catalog = photometry_class(dbobj, self._obs_list[obs_valid_dex[0]], 

1049 column_outputs=['lsst_u', 

1050 'lsst_g', 

1051 'lsst_r', 

1052 'lsst_i', 

1053 'lsst_z', 

1054 'lsst_y']) 

1055 

1056 i_chunk = 0 

1057 

1058 output_data_cache = {} 

1059 n_rows_cached = 0 

1060 

1061 n_obj = 0 

1062 n_actual_obj = 0 

1063 n_time_last = 0 

1064 n_rows = 0 

1065 

1066 t_before_obj = time.time() # so that we can get a sense of how long the 

1067 # "iterating over astrophysical objects" part 

1068 # of the simulation will take 

1069 

1070 db_name = os.path.join(output_dir, '%s_%d_sqlite.db' % (output_prefix, htmid)) 

1071 with sqlite3.connect(db_name, isolation_level='EXCLUSIVE') as conn: 

1072 creation_cmd = '''CREATE TABLE alert_data 

1073 (uniqueId int, obshistId int, xPix float, yPix float, 

1074 chipNum int, dflux float, snr float, ra float, dec float)''' 

1075 

1076 cursor = conn.cursor() 

1077 cursor.execute('PRAGMA journal_mode=WAL;') 

1078 conn.commit() 

1079 cursor.execute(creation_cmd) 

1080 conn.commit() 

1081 

1082 creation_cmd = '''CREATE TABLE metadata 

1083 (obshistId int, TAI float, band int)''' 

1084 cursor.execute(creation_cmd) 

1085 conn.commit() 

1086 

1087 for obs_dex in obs_valid_dex: 

1088 obs = self._obs_list[obs_dex] 

1089 cmd = '''INSERT INTO metadata 

1090 VALUES(%d, %.5f, %d)''' % (obs.OpsimMetaData['obsHistID'], 

1091 obs.mjd.TAI, 

1092 mag_name_to_int[obs.bandpass]) 

1093 

1094 cursor.execute(cmd) 

1095 conn.commit() 

1096 

1097 creation_cmd = '''CREATE TABLE quiescent_flux 

1098 (uniqueId int, band int, flux float, snr float)''' 

1099 

1100 cursor.execute(creation_cmd) 

1101 conn.commit() 

1102 

1103 creation_cmd = '''CREATE TABLE baseline_astrometry 

1104 (uniqueId int, ra real, dec real, pmRA real, 

1105 pmDec real, parallax real, TAI real)''' 

1106 

1107 cursor.execute(creation_cmd) 

1108 conn.commit() 

1109 

1110 for chunk in data_iter: 

1111 n_raw_obj = len(chunk) 

1112 i_chunk += 1 

1113 

1114 if chunk_cutoff > 0 and i_chunk >= chunk_cutoff: 

1115 break 

1116 

1117 n_time_last = 0 

1118 # filter the chunk so that we are only considering sources that are in 

1119 # the trixel being considered 

1120 reduced_htmid = chunk['htmid'] >> n_bits_off 

1121 

1122 valid_htmid = np.where(reduced_htmid == htmid) 

1123 if len(valid_htmid[0]) == 0: 

1124 continue 

1125 n_htmid_trim = n_raw_obj-len(valid_htmid[0]) 

1126 chunk = chunk[valid_htmid] 

1127 n_obj += len(valid_htmid[0]) 

1128 

1129 (chip_name_dict, 

1130 dmag_arr, 

1131 dmag_arr_transpose, 

1132 time_arr) = self._filter_on_photometry_then_chip_name(chunk, column_query, 

1133 obs_valid_dex, 

1134 expmjd_list, 

1135 photometry_catalog, 

1136 dmag_cutoff) 

1137 

1138 q_f_dict = {} 

1139 q_m_dict = {} 

1140 

1141 q_m_dict[0] = photometry_catalog.column_by_name('quiescent_lsst_u') 

1142 q_m_dict[1] = photometry_catalog.column_by_name('quiescent_lsst_g') 

1143 q_m_dict[2] = photometry_catalog.column_by_name('quiescent_lsst_r') 

1144 q_m_dict[3] = photometry_catalog.column_by_name('quiescent_lsst_i') 

1145 q_m_dict[4] = photometry_catalog.column_by_name('quiescent_lsst_z') 

1146 q_m_dict[5] = photometry_catalog.column_by_name('quiescent_lsst_y') 

1147 

1148 q_f_dict[0] = dummy_sed.fluxFromMag(q_m_dict[0]) 

1149 q_f_dict[1] = dummy_sed.fluxFromMag(q_m_dict[1]) 

1150 q_f_dict[2] = dummy_sed.fluxFromMag(q_m_dict[2]) 

1151 q_f_dict[3] = dummy_sed.fluxFromMag(q_m_dict[3]) 

1152 q_f_dict[4] = dummy_sed.fluxFromMag(q_m_dict[4]) 

1153 q_f_dict[5] = dummy_sed.fluxFromMag(q_m_dict[5]) 

1154 

1155 q_pmra = 1000.0*arcsecFromRadians(photometry_catalog.column_by_name('properMotionRa')) 

1156 q_pmdec = 1000.0*arcsecFromRadians(photometry_catalog.column_by_name('properMotionDec')) 

1157 q_parallax = 1000.0*arcsecFromRadians(photometry_catalog.column_by_name('parallax')) 

1158 q_ra = np.degrees(photometry_catalog.column_by_name('raICRS')) 

1159 q_dec = np.degrees(photometry_catalog.column_by_name('decICRS')) 

1160 q_tai = photometry_catalog.obs_metadata.mjd.TAI 

1161 

1162 q_snr_dict = {} 

1163 for i_filter in range(6): 

1164 

1165 snr_template, local_gamma = calcSNR_m5(q_m_dict[i_filter], 

1166 self.bp_dict[mag_names[i_filter]], 

1167 obs_mag_cutoff[i_filter], 

1168 phot_params, gamma=gamma_template[i_filter]) 

1169 q_snr_dict[i_filter] = snr_template 

1170 gamma_template[i_filter] = local_gamma 

1171 

1172 unq = photometry_catalog.column_by_name('uniqueId') 

1173 

1174 try: 

1175 assert dmag_arr_transpose.shape == (len(chunk), len(mag_names), len(expmjd_list)) 

1176 except AssertionError: 

1177 print('dmag_arr_transpose_shape %s' % str(dmag_arr_transpose.shape)) 

1178 print('should be (%d, %d, %d)' % (len(chunk), len(mag_names), len(expmjd_list))) 

1179 raise 

1180 

1181 # only include those sources for which np.abs(delta_mag) >= dmag_cutoff 

1182 # at some point in their history (note that delta_mag is defined with 

1183 # respect to the quiescent magnitude) 

1184 # 

1185 # also demand that the magnitude at some point is less than obs_mag_cutoff 

1186 # 

1187 # This is different from the dmag>dmag_cutoff check done in 

1188 # 

1189 # self._filter_on_photometry_then_chip_name() 

1190 # 

1191 # Now we have information about which observations actually detected 

1192 # each object (in self._filter_on_photometry_then_chip_name(), 

1193 # we assumed that every object was detected at every time step). 

1194 

1195 photometrically_valid_obj = [] 

1196 for i_obj in range(len(chunk)): 

1197 keep_it = False 

1198 valid_times = np.where(time_arr[i_obj] > 0) 

1199 if len(valid_times[0]) == 0: 

1200 continue 

1201 for i_filter in range(len(mag_names)): 

1202 if np.abs(dmag_arr_transpose[i_obj][i_filter][valid_times]).max() > dmag_cutoff: 

1203 dmag_min = dmag_arr_transpose[i_obj][i_filter][valid_times].min() 

1204 if q_m_dict[i_filter][i_obj] + dmag_min <= obs_mag_cutoff[i_filter]: 

1205 keep_it = True 

1206 break 

1207 if keep_it: 

1208 photometrically_valid_obj.append(i_obj) 

1209 photometrically_valid_obj = np.array(photometrically_valid_obj) 

1210 

1211 del dmag_arr_transpose 

1212 gc.collect() 

1213 

1214 if np.abs(dmag_arr).max() < dmag_cutoff: 

1215 continue 

1216 

1217 completely_valid = np.zeros(len(chunk), dtype=int) 

1218 

1219 ############################ 

1220 # Process and output sources 

1221 # 

1222 for i_obs, obs_dex in enumerate(obs_valid_dex): 

1223 obs = self._obs_list[obs_dex] 

1224 obshistid = obs.OpsimMetaData['obsHistID'] 

1225 

1226 obs_mag = obs.bandpass 

1227 actual_i_mag = mag_name_to_int[obs_mag] 

1228 assert mag_names[actual_i_mag] == obs_mag 

1229 

1230 # only include those sources which fall on a detector for this pointing 

1231 valid_chip_name, valid_xpup, valid_ypup, chip_valid_obj = chip_name_dict[i_obs] 

1232 

1233 actually_valid_obj = np.intersect1d(photometrically_valid_obj, chip_valid_obj) 

1234 if len(actually_valid_obj) == 0: 

1235 continue 

1236 

1237 try: 

1238 completely_valid[actually_valid_obj] += 1 

1239 except: 

1240 print('failed') 

1241 print(actually_valid_obj) 

1242 print(completely_valid) 

1243 raise 

1244 

1245 valid_sources = chunk[actually_valid_obj] 

1246 local_column_cache = {} 

1247 local_column_cache['deltaMagAvro'] = OrderedDict([('delta_%smag' % mag_names[i_mag], 

1248 dmag_arr[i_obs][i_mag][actually_valid_obj]) 

1249 for i_mag in range(len(mag_names))]) 

1250 

1251 local_column_cache['chipName'] = valid_chip_name[actually_valid_obj] 

1252 local_column_cache['pupilFromSky'] = OrderedDict([('x_pupil', valid_xpup[actually_valid_obj]), 

1253 ('y_pupil', valid_ypup[actually_valid_obj])]) 

1254 

1255 cat = cat_list[i_obs] 

1256 i_valid_chunk = 0 

1257 for valid_chunk, chunk_map in cat.iter_catalog_chunks(query_cache=[valid_sources], 

1258 column_cache=local_column_cache): 

1259 i_valid_chunk += 1 

1260 assert i_valid_chunk == 1 

1261 n_time_last += len(valid_chunk[0]) 

1262 length_of_chunk = len(valid_chunk[chunk_map['uniqueId']]) 

1263 cache_tag = '%d_%d' % (obshistid, i_chunk) 

1264 output_data_cache[cache_tag] = {} 

1265 

1266 for col_name in ('uniqueId', 'raICRS', 'decICRS', 'flux', 'dflux', 'SNR', 

1267 'chipNum', 'xPix', 'yPix'): 

1268 

1269 output_data_cache[cache_tag][col_name] = valid_chunk[chunk_map[col_name]] 

1270 

1271 n_rows_cached += length_of_chunk 

1272 

1273 completely_valid = np.where(completely_valid > 0) 

1274 for i_filter in range(6): 

1275 values = ((int(unq[completely_valid][i_q]), 

1276 i_filter, 

1277 q_f_dict[i_filter][completely_valid][i_q], 

1278 q_snr_dict[i_filter][completely_valid][i_q]) 

1279 for i_q in range(len(completely_valid[0]))) 

1280 cursor.executemany('INSERT INTO quiescent_flux VALUES (?,?,?,?)', values) 

1281 conn.commit() 

1282 

1283 values = ((int(unq[completely_valid][i_q]), 

1284 q_ra[completely_valid][i_q], 

1285 q_dec[completely_valid][i_q], 

1286 q_pmra[completely_valid][i_q], 

1287 q_pmdec[completely_valid][i_q], 

1288 q_parallax[completely_valid][i_q], 

1289 q_tai) 

1290 for i_q in range(len(completely_valid[0]))) 

1291 

1292 cursor.executemany('INSERT INTO baseline_astrometry VALUES (?,?,?,?,?,?,?)', values) 

1293 

1294 if n_rows_cached >= write_every: 

1295 self.acquire_lock() 

1296 with open(log_file_name, 'a') as out_file: 

1297 out_file.write('%d is writing \n' % os.getpid()) 

1298 

1299 print('%d is writing' % os.getpid()) 

1300 

1301 self.release_lock() 

1302 

1303 n_rows += self._output_alert_data(conn, output_data_cache) 

1304 output_data_cache = {} 

1305 n_rows_cached = 0 

1306 

1307 if n_rows > 0: 

1308 self.acquire_lock() 

1309 with open(log_file_name, 'a') as out_file: 

1310 elapsed = (time.time()-t_before_obj)/3600.0 

1311 elapsed_per = elapsed/n_rows 

1312 rows_per_chunk = float(n_rows)/float(i_chunk) 

1313 total_projection = 1000.0*rows_per_chunk*elapsed_per 

1314 out_file.write('\n %d n_obj %d %d trimmed %d\n' % 

1315 (this_pid, n_obj, n_actual_obj, n_htmid_trim)) 

1316 out_file.write(' elapsed %.2e hrs per row %.2e total %2e\n' % 

1317 (elapsed, elapsed_per, total_projection)) 

1318 out_file.write(' n_time_last %d; rows %d\n' % (n_time_last, n_rows)) 

1319 

1320 out_file.write('%d is done writing\n' % os.getpid()) 

1321 

1322 print('\n %d n_obj %d %d trimmed %d' % 

1323 (this_pid, n_obj, n_actual_obj, n_htmid_trim)) 

1324 print(' elapsed %.2e hrs per row %.2e total %2e' % 

1325 (elapsed, elapsed_per, total_projection)) 

1326 print(' n_time_last %d; rows %d\n' % (n_time_last, n_rows)) 

1327 print('%d is done writing' % os.getpid()) 

1328 

1329 self.release_lock() 

1330 

1331 if len(output_data_cache) > 0: 

1332 n_rows += self._output_alert_data(conn, output_data_cache) 

1333 output_data_cache = {} 

1334 

1335 print('htmid %d that took %.2e hours; n_obj %d n_rows %d' % 

1336 (htmid, (time.time()-t_start)/3600.0, n_obj, n_rows)) 

1337 

1338 self.acquire_lock() 

1339 print("INDEXING %d" % htmid) 

1340 self.release_lock() 

1341 

1342 cursor.execute('CREATE INDEX unq_obs ON alert_data (uniqueId, obshistId)') 

1343 cursor.execute('CREATE INDEX unq_flux ON quiescent_flux (uniqueId, band)') 

1344 cursor.execute('CREATE INDEX obs ON metadata (obshistid)') 

1345 cursor.execute('CREATE INDEX unq_ast ON baseline_astrometry (uniqueId)') 

1346 conn.commit() 

1347 

1348 self.acquire_lock() 

1349 with open(log_file_name, 'a') as out_file: 

1350 out_file.write('done with htmid %d -- %e %d\n' % 

1351 (htmid, (time.time()-t_start)/3600.0, n_obj)) 

1352 self.release_lock() 

1353 

1354 return n_rows