Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import numpy as np 

2import os 

3import re 

4import sqlite3 

5from collections import OrderedDict 

6import time 

7import gc 

8from lsst.utils import getPackageDir 

9import lsst.obs.lsst.phosim as obs_lsst_phosim 

10from lsst.sims.catalogs.definitions import InstanceCatalog 

11from lsst.sims.utils import trixelFromHtmid, getAllTrixels 

12from lsst.sims.utils import levelFromHtmid, halfSpaceFromRaDec 

13from lsst.sims.utils import angularSeparation, ObservationMetaData 

14from lsst.sims.utils import arcsecFromRadians 

15from lsst.sims.catUtils.utils import _baseLightCurveCatalog 

16from lsst.sims.utils import _pupilCoordsFromRaDec 

17from lsst.sims.coordUtils import chipNameFromPupilCoords 

18from lsst.sims.coordUtils import pixelCoordsFromPupilCoords 

19 

20from lsst.sims.catalogs.decorators import compound, cached 

21from lsst.sims.photUtils import BandpassDict, Sed, calcSNR_m5 

22from lsst.sims.photUtils import PhotometricParameters 

23from lsst.sims.catUtils.mixins import VariabilityStars, AstrometryStars 

24from lsst.sims.catUtils.mixins import VariabilityGalaxies, AstrometryGalaxies 

25from lsst.sims.catUtils.mixins import CameraCoords, PhotometryBase 

26from lsst.sims.catUtils.mixins import ParametrizedLightCurveMixin 

27from lsst.sims.catUtils.mixins import create_variability_cache 

28 

29from lsst.sims.catUtils.baseCatalogModels import StarObj, GalaxyAgnObj 

30from sqlalchemy.sql import text 

31from lsst.sims.catalogs.db import ChunkIterator 

32 

33__all__ = ["AlertDataGenerator", 

34 "AlertStellarVariabilityCatalog", 

35 "AlertAgnVariabilityCatalog", 

36 "_baseAlertCatalog", 

37 "StellarAlertDBObj", 

38 "AgnAlertDBObj", 

39 "StellarAlertDBObjMixin"] 

40 

41 

42class StellarAlertDBObjMixin(object): 

43 """ 

44 Mimics StarObj class, except it allows you to directly query 

45 all objects in a trixel specified by an htmid. 

46 """ 

47 def query_columns_htmid(self, colnames=None, chunk_size=None, 

48 constraint=None, 

49 limit=None, htmid=None): 

50 """Execute a query from the primary catsim database 

51 

52 Execute a query, taking advantage of the spherical geometry library and 

53 htmid indexes on all catalog tables in the UW catsim database 

54 

55 **Parameters** 

56 

57 * colnames : list or None 

58 a list of valid column names, corresponding to entries in the 

59 `columns` class attribute. If not specified, all columns are 

60 queried. 

61 * chunk_size : int (optional) 

62 if specified, then return an iterator object to query the database, 

63 each time returning the next `chunk_size` elements. If not 

64 specified, all matching results will be returned. 

65 * constraint : str (optional) 

66 a string which is interpreted as SQL and used as a predicate on the query 

67 * limit : int (optional) 

68 limits the number of rows returned by the query 

69 * htmid is the htmid to be queried 

70 

71 **Returns** 

72 

73 * result : list or iterator 

74 If chunk_size is not specified, then result is a list of all 

75 items which match the specified query. If chunk_size is specified, 

76 then result is an iterator over lists of the given size. 

77 """ 

78 

79 # find the minimum and maximum htmid 

80 # (level=21 since that is what is implemented 

81 # on fatboy) that we are asking for 

82 # 

83 # Note that sqlalchemy does not like np.int64 

84 # as a data type 

85 current_level = levelFromHtmid(htmid) 

86 n_bits_off = 2*(21-current_level) 

87 htmid_min = int(htmid << n_bits_off) 

88 htmid_max = int((htmid+1) << n_bits_off) 

89 

90 query = self._get_column_query(colnames) 

91 

92 # add spatial constraints to query. 

93 

94 # Hint sql engine to seek on htmid 

95 if not self.tableid.endswith('forceseek'): 

96 query = query.with_hint(self.table, ' WITH(FORCESEEK)', 'mssql') 

97 

98 # SQL is not case sensitive but python is: 

99 if 'htmID' in self.columnMap: 

100 htmid_name = 'htmID' 

101 elif 'htmid' in self.columnMap: 

102 htmid_name = 'htmid' 

103 else: 

104 htmid_name = 'htmId' 

105 

106 # Range join on htmid ranges 

107 query = query.filter(self.table.c[htmid_name].between(htmid_min, htmid_max)) 

108 

109 if constraint is not None: 

110 query = query.filter(text(constraint)) 

111 

112 if limit is not None: 

113 query = query.limit(limit) 

114 

115 return ChunkIterator(self, query, chunk_size) 

116 

117 

118class StellarAlertDBObj(StellarAlertDBObjMixin, StarObj): 

119 pass 

120 

121 

122class AgnAlertDBObj(GalaxyAgnObj): 

123 """ 

124 Mimics GalaxyAgnObj class, except it allows you to directly query 

125 all objects in a trixel specified by an htmid. 

126 """ 

127 

128 columns = [('htmid', 0, np.int64), 

129 ('galtileid', None, np.int64), 

130 ('galid', None, str, 30), 

131 ('componentra', 'agnra*PI()/180.'), 

132 ('componentdec', 'agndec*PI()/180.'), 

133 #: This is actually a problem with the stored procedure. 

134 #: We need to be able to map columns other than 

135 #: just ra/dec to raJ2000/decJ2000. This gets 

136 #: important when we start perturbing the three galaxy components 

137 ('raJ2000', 'ra'), 

138 ('decJ2000', 'dec'), 

139 ('magNorm', 'magnorm_agn'), 

140 ('magNormAgn', 'magnorm_agn'), 

141 ('sedFilename', 'sedname_agn', str, 40), 

142 ('sedFilenameAgn', 'sedname_agn', str, 40), 

143 ('variabilityParameters', 'varParamStr', str, 256), 

144 ('lsst_u', 'u_ab'), 

145 ('lsst_g', 'g_ab'), 

146 ('lsst_r', 'r_ab'), 

147 ('lsst_i', 'i_ab'), 

148 ('lsst_z', 'z_ab'), 

149 ('lsst_y', 'y_ab')] 

150 

151 def query_columns_htmid(self, colnames=None, chunk_size=None, 

152 constraint=None, 

153 limit=None, htmid=None): 

154 """Execute a query from the primary catsim database 

155 

156 Execute a query, taking advantage of the spherical geometry library and 

157 htmid indexes on all catalog tables in the UW catsim database 

158 

159 **Parameters** 

160 

161 * colnames : list or None 

162 a list of valid column names, corresponding to entries in the 

163 `columns` class attribute. If not specified, all columns are 

164 queried. 

165 * chunk_size : int (optional) 

166 if specified, then return an iterator object to query the database, 

167 each time returning the next `chunk_size` elements. If not 

168 specified, all matching results will be returned. 

169 * constraint : str (optional) 

170 a string which is interpreted as SQL and used as a predicate on the query 

171 * limit : int (optional) 

172 limits the number of rows returned by the query 

173 * htmid is the htmid to be queried 

174 

175 **Returns** 

176 

177 * result : list or iterator 

178 If chunk_size is not specified, then result is a list of all 

179 items which match the specified query. If chunk_size is specified, 

180 then result is an iterator over lists of the given size. 

181 """ 

182 

183 trixel = trixelFromHtmid(htmid) 

184 ra_0, dec_0 = trixel.get_center() 

185 new_obs = ObservationMetaData(pointingRA=ra_0, pointingDec=dec_0, boundType='circle', 

186 boundLength=trixel.get_radius()+0.1) 

187 

188 self._queried_trixel = trixel 

189 self._queried_htmid_level = levelFromHtmid(htmid) 

190 

191 return self.query_columns(colnames=colnames, chunk_size=chunk_size, 

192 obs_metadata=new_obs, constraint=constraint, 

193 limit=limit) 

194 

195 def _final_pass(self, results): 

196 """Modify the results of raJ2000 and decJ2000 to be in radians. 

197 

198 Also filter the results so that any objects outside of the 

199 trixel specified in query_columns_htmid are returned with 

200 htmid=0. 

201 

202 **Parameters** 

203 

204 * results : Structured array of results from query 

205 

206 **Returns** 

207 

208 * results : Modified structured array 

209 

210 """ 

211 

212 if hasattr(self, '_queried_trixel'): 

213 htmid = self._queried_trixel.htmid 

214 htmid_21 = htmid << 2*(21-self._queried_htmid_level) 

215 assert levelFromHtmid(htmid_21) == 21 

216 contains_arr = self._queried_trixel.contains(results['raJ2000'], results['decJ2000']) 

217 results['htmid'] = np.where(contains_arr, htmid_21, 0) 

218 

219 results['raJ2000'] = np.radians(results['raJ2000']) 

220 results['decJ2000'] = np.radians(results['decJ2000']) 

221 return results 

222 

223 

224class _baseAlertCatalog(PhotometryBase, CameraCoords, _baseLightCurveCatalog): 

225 

226 camera = obs_lsst_phosim.PhosimMapper().camera 

227 

228 column_outputs = ['htmid', 'uniqueId', 'raICRS', 'decICRS', 

229 'flux', 'SNR', 'dflux', 

230 'chipNum', 'xPix', 'yPix'] 

231 

232 default_formats = {'f': '%.4g'} 

233 

234 default_columns = [('properMotionRa', 0.0, float), 

235 ('properMotionDec', 0.0, float), 

236 ('parallax', 0.0, float)] 

237 

238 def iter_catalog_chunks(self, chunk_size=None, query_cache=None, column_cache=None): 

239 """ 

240 Returns an iterator over chunks of the catalog. 

241 

242 Parameters 

243 ---------- 

244 chunk_size : int, optional, defaults to None 

245 the number of rows to return from the database at a time. If None, 

246 returns the entire database query in one chunk. 

247 

248 query_cache : iterator over database rows, optional, defaults to None 

249 the result of calling db_obj.query_columns(). If query_cache is not 

250 None, this method will iterate over the rows in query_cache and produce 

251 an appropriate InstanceCatalog. DO NOT set to non-None values 

252 unless you know what you are doing. It is an optional 

253 input for those who want to repeatedly examine the same patch of sky 

254 without actually querying the database over and over again. If it is set 

255 to None (default), this method will handle the database query. 

256 

257 column_cache : a dict that will be copied over into the catalogs self._column_cache. 

258 Should be left as None, unless you know what you are doing. 

259 """ 

260 

261 if query_cache is None: 

262 # Call the original version of iter_catalog defined in the 

263 # InstanceCatalog class. This version of iter_catalog includes 

264 # the call to self.db_obj.query_columns, which the user would have 

265 # used to generate query_cache. 

266 for line in InstanceCatalog.iter_catalog_chunks(self, chunk_size=chunk_size): 

267 yield line 

268 else: 

269 # Otherwise iterate over the query cache 

270 transform_keys = list(self.transformations.keys()) 

271 for chunk in query_cache: 

272 self._set_current_chunk(chunk, column_cache=column_cache) 

273 chunk_cols = [self.transformations[col](self.column_by_name(col)) 

274 if col in transform_keys else 

275 self.column_by_name(col) 

276 for col in self.iter_column_names()] 

277 

278 if not hasattr(self, '_chunkColMap_output'): 

279 

280 self._chunkColMap_output = dict([(col, i) 

281 for i, col in 

282 enumerate(self.iter_column_names())]) 

283 

284 yield chunk_cols, self._chunkColMap_output 

285 

286 self._column_cache = {} 

287 self._current_chunk = None 

288 

289 @cached 

290 def get_chipName(self): 

291 if len(self.column_by_name('uniqueId')) == 0: 

292 return np.array([]) 

293 raise RuntimeError("Should not get this far in get_chipName") 

294 

295 @compound('x_pupil', 'y_pupil') 

296 def get_pupilFromSky(self): 

297 if len(self.column_by_name('uniqueId')) == 0: 

298 return np.array([[], []]) 

299 raise RuntimeError("Should not get this far in get_pupilFromSky") 

300 

301 @cached 

302 def get_chipNum(self): 

303 """ 

304 Concatenate the digits in 'R:i,j S:m,n' to make the chip number ijmn 

305 """ 

306 chip_name = self.column_by_name('chipName') 

307 return np.array([int(''.join(re.findall(r'\d+', name))) if name is not None else 0 

308 for name in chip_name]) 

309 

310 @compound('xPix', 'yPix') 

311 def get_pixelCoordinates(self): 

312 xpup = self.column_by_name('x_pupil') 

313 ypup = self.column_by_name('y_pupil') 

314 chip_name = self.column_by_name('chipName') 

315 xpix, ypix = pixelCoordsFromPupilCoords(xpup, ypup, chipName=chip_name, 

316 includeDistortion=True, 

317 camera=self.camera) 

318 return np.array([xpix, ypix]) 

319 

320 @compound('delta_umag', 'delta_gmag', 'delta_rmag', 

321 'delta_imag', 'delta_zmag', 'delta_ymag') 

322 def get_deltaMagAvro(self): 

323 ra = self.column_by_name('raJ2000') 

324 if len(ra) == 0: 

325 return np.array([[], [], [], [], [], []]) 

326 

327 raise RuntimeError("Should not have gotten this far in delta mag getter") 

328 

329 @compound('lsst_u', 'lsst_g', 'lsst_r', 'lsst_i', 'lsst_z', 'lsst_y') 

330 def get_lsst_magnitudes(self): 

331 """ 

332 getter for LSST stellar magnitudes 

333 """ 

334 

335 magnitudes = np.array([self.column_by_name('quiescent_lsst_u'), 

336 self.column_by_name('quiescent_lsst_g'), 

337 self.column_by_name('quiescent_lsst_r'), 

338 self.column_by_name('quiescent_lsst_i'), 

339 self.column_by_name('quiescent_lsst_z'), 

340 self.column_by_name('quiescent_lsst_y')]) 

341 

342 delta = np.array([self.column_by_name('delta_umag'), 

343 self.column_by_name('delta_gmag'), 

344 self.column_by_name('delta_rmag'), 

345 self.column_by_name('delta_imag'), 

346 self.column_by_name('delta_zmag'), 

347 self.column_by_name('delta_ymag')]) 

348 magnitudes += delta 

349 

350 return magnitudes 

351 

352 @compound('mag', 'dmag', 'quiescent_mag') 

353 def get_alertPhotometry(self): 

354 mag = self.column_by_name('lsst_%s' % self.obs_metadata.bandpass) 

355 quiescent_mag = self.column_by_name('quiescent_lsst_%s' % self.obs_metadata.bandpass) 

356 dmag = mag - quiescent_mag 

357 

358 return np.array([mag, dmag, quiescent_mag]) 

359 

360 @compound('flux', 'dflux', 'SNR') 

361 def get_alertFlux(self): 

362 quiescent_mag = self.column_by_name('quiescent_mag') 

363 mag = self.column_by_name('mag') 

364 if not hasattr(self, '_dummy_sed'): 

365 self._dummy_sed = Sed() 

366 if not hasattr(self, 'lsstBandpassDict'): 

367 self.lsstBandpassDict = BandpassDict.loadTotalBandpassesFromFiles() 

368 if not hasattr(self, 'phot_params'): 

369 self.phot_params = PhotometricParameters() 

370 if not hasattr(self, '_gamma'): 

371 self._gamma = None 

372 

373 quiescent_flux = self._dummy_sed.fluxFromMag(quiescent_mag) 

374 flux = self._dummy_sed.fluxFromMag(mag) 

375 dflux = flux - quiescent_flux 

376 

377 snr_tot, gamma = calcSNR_m5(mag, self.lsstBandpassDict[self.obs_metadata.bandpass], 

378 self.obs_metadata.m5[self.obs_metadata.bandpass], 

379 self.phot_params, gamma=self._gamma) 

380 

381 if self._gamma is None: 

382 self._gamma = gamma 

383 

384 return np.array([flux, dflux, snr_tot]) 

385 

386 

387class AlertStellarVariabilityCatalog(_baseAlertCatalog, 

388 VariabilityStars, 

389 AstrometryStars): 

390 

391 @compound('quiescent_lsst_u', 'quiescent_lsst_g', 'quiescent_lsst_r', 

392 'quiescent_lsst_i', 'quiescent_lsst_z', 'quiescent_lsst_y') 

393 def get_quiescent_lsst_magnitudes(self): 

394 return np.array([self.column_by_name('umag'), self.column_by_name('gmag'), 

395 self.column_by_name('rmag'), self.column_by_name('imag'), 

396 self.column_by_name('zmag'), self.column_by_name('ymag')]) 

397 

398 

399class AlertAgnVariabilityCatalog(_baseAlertCatalog, 

400 VariabilityGalaxies, 

401 AstrometryGalaxies): 

402 

403 @compound('quiescent_lsst_u', 'quiescent_lsst_g', 'quiescent_lsst_r', 

404 'quiescent_lsst_i', 'quiescent_lsst_z', 'quiescent_lsst_y') 

405 def get_quiescent_lsst_magnitudes(self): 

406 return np.array([self.column_by_name('u_ab'), self.column_by_name('g_ab'), 

407 self.column_by_name('r_ab'), self.column_by_name('i_ab'), 

408 self.column_by_name('z_ab'), self.column_by_name('y_ab')]) 

409 

410 

411class AlertDataGenerator(object): 

412 """ 

413 This class will read in astrophysical sources and variability 

414 models from CatSim, observe them with a simulated OpSim 

415 cadence, and write a series of sqlite files containing all 

416 of the simulated observations that could trigger an alert. 

417 

418 In order to make this calculation as efficient as possible, 

419 the class works by partitioning the sky according to the 

420 Hierarchical Triangular Mesh (HTM) of 

421 

422 Kunszt P., Szalay A., Thakar A. (2006) in "Mining The Sky", 

423 Banday A, Zaroubi S, Bartelmann M. eds. 

424 ESO Astrophysics Symposia 

425 https://www.researchgate.net/publication/226072008_The_Hierarchical_Triangular_Mesh 

426 

427 Szalay A. et al. (2007) 

428 "Indexing the Sphere with the Hierarchical Triangular Mesh" 

429 arXiv:cs/0701164 

430 

431 and simulating all of the observations in a given trixel (the 

432 elementary unit of the HTM) at once. Accordintly, the outputs 

433 produced by this class are files named like 

434 

435 prefix_NNNN_sqlite.db 

436 

437 where prefix is specified by theuser and NNNN is the htmid, the 

438 unique identifying integer, corresponding to each simulated trixel. 

439 

440 The proper way to run this class is to instantiate it, run 

441 subdivide_obs on a list of ObservationMetaData corresponding 

442 to the OpSim pointings to be simulated, and then running 

443 alert_data_from_htmid on each of the htmid in the class property 

444 htmid_list. This last step can easily be parallelized using python's 

445 multiprocessing module, with each process handling a different htmid. 

446 

447 The sqlite files produced by alert_data_from_htmid will each contain 

448 four tables. They are as follows. Columns are listed below the 

449 tables. 

450 

451 alert_data 

452 ---------- 

453 uniqueId -- int -- a unique identifier for each astrophysical object 

454 

455 obshistId -- int -- a unique identifier for each OpSim pointing 

456 

457 xPix -- float -- the x pixel coordinate of the source on the focal 

458 plane 

459 

460 yPix -- float -- the y pixel coordinate of the source on the focal 

461 plane 

462 

463 dflux -- float -- the difference in flux between the source's current 

464 flux and its quiescent flux (the source's quiescent flux can be found 

465 in the quiescent_flux table). This is in units of Janskys. 

466 

467 snr -- float -- the signal to noise of the current detection of the 

468 source (not the signal to noise of the source's detection in a simulated 

469 difference image). 

470 

471 ra -- float -- the current RA of the source in degrees 

472 

473 dec -- float -- the current Declination of the source in degrees. 

474 

475 The alert_data table has a mult-column index on uniqueId and obshistId. 

476 

477 metadata 

478 -------- 

479 obshistId -- int --- a unique identifier for each OpSim pointing 

480 

481 TAI -- float -- the International Atomic Time of the observation 

482 as an MJD (in days) 

483 

484 band -- int -- denotes the filter used for the observation 

485 (0=u, 1=g, 2=r, etc.) 

486 

487 The metadata table is indexed on obshistId 

488 

489 quiescent_flux 

490 -------------- 

491 uniqueId -- int -- a unique identifier for each astrophysical source 

492 

493 band -- int -- an integer denoting each LSST filter (0=u, 1=g, 2=r, etc.) 

494 

495 flux -- float -- the flux of the source through the filter specified by 

496 band (in units of Janskys) 

497 

498 snr -- float -- the signal to noise ratio of the source in the given 

499 band with m5 taken from Table 2 of the overview paper (arXiv:0805.2366) 

500 

501 The quiescent_flux table has a multi-column index on uniqueId and band. 

502 

503 baseline_astrometry 

504 ------------------- 

505 uniqueId -- int -- a unique identifier for each astrophysical source 

506 

507 TAI -- float -- the International Atomic Time of the baseline astrometric 

508 measurements below as a MJD (in days) 

509 

510 ra -- float -- the RA of the source at TAI in degrees 

511 

512 dec -- float -- the Declination of the source at TAI in degrees 

513 

514 pmRA -- float -- the RA proper motion of the source in milliarcseconds/year 

515 

516 pmDec -- float -- the Declination proper motion of the source in 

517 milliarcseconds/year 

518 

519 parallax -- float -- the parallax of the source in milliarcseconds 

520 

521 The baseline_astrometry table is indexed on uniqueId 

522 

523 """ 

524 

525 def __init__(self, 

526 testing=False): 

527 """ 

528 Parameters 

529 ---------- 

530 testing as a boolean that should only be True when running the unit tests. 

531 If True, it prevents the AlertDataGenerator from pre-caching variability 

532 models, which aids performance, but uses more memory than we want to use 

533 in a unit test. 

534 """ 

535 

536 self.lsst_camera = obs_lsst_phosim.PhosimMapper().camera 

537 self._variability_cache = create_variability_cache() 

538 self._stdout_lock = None 

539 if not testing: 

540 plm = ParametrizedLightCurveMixin() 

541 plm.load_parametrized_light_curves(variability_cache = self._variability_cache) 

542 self.bp_dict = BandpassDict.loadTotalBandpassesFromFiles() 

543 

544 # This is a file that lists the maximum amplitude of variability 

545 # for each of the Kepler-derived light curve models. It will be 

546 # used by the stellar variability model to figure out which 

547 # stars can be skipped because they will never vary above 

548 # the alert-triggering threshold. 

549 self._dmag_lookup_file_exists = True 

550 self._dmag_lookup_file = os.path.join(getPackageDir('sims_data'), 

551 'catUtilsData', 

552 'kplr_dmag_171204.txt') 

553 

554 if not os.path.exists(self._dmag_lookup_file): 

555 if not testing: 

556 script_name = os.path.join(getPackageDir('sims_catUtils'), 'support_scripts', 

557 'get_kepler_dmag.sh') 

558 raise RuntimeError('\n%s does not exist; run the script\n\n%s\n\n' % 

559 script_name) 

560 else: 

561 self._dmag_lookup_file_exists = False 

562 

563 def acquire_lock(self): 

564 """ 

565 If running with multiprocessing, acquire 

566 the lock. 

567 """ 

568 if self._stdout_lock is not None: 

569 self._stdout_lock.acquire() 

570 

571 def release_lock(self): 

572 """ 

573 If running with multiprocessing, release 

574 the lock. 

575 """ 

576 if self._stdout_lock is not None: 

577 self._stdout_lock.release() 

578 

579 def subdivide_obs(self, obs_list, htmid_level=6): 

580 """ 

581 Take a list of ObservationMetaData and subdivide 

582 them according to which trixels (see htmModule.py 

583 in sims_utils) they intersect. 

584 

585 Parameters 

586 ---------- 

587 obs_list is a list of ObservationMetaData 

588 

589 htmid_level is an int denoting the level of 

590 the HTM mesh you want to use to tile the sky 

591 (higher htmid_level corresponds to a finer 

592 tiling). Default = 6 

593 

594 Returns 

595 ------- 

596 Nothing. 

597 

598 After running this method, this AlertGenerator 

599 will contain the following data. 

600 

601 - a list of the htmid of every trixel intersected 

602 by the fields of view specified in obs_list. This 

603 list is accessible from the property 

604 AlertGenerator.htmid_list 

605 

606 - a dict mapping each htmid to the ObservationMetaData 

607 from obs_list that intersect it. The method 

608 AlertGenerator.obs_from_htmid(htmid) will return a 

609 list of all of the ObservationMetaData that intersect 

610 the trixel specified by htmid. 

611 """ 

612 self._trixel_dict = getAllTrixels(htmid_level) 

613 valid_htmid = [] 

614 for htmid in self._trixel_dict: 

615 if levelFromHtmid(htmid) == htmid_level: 

616 valid_htmid.append(htmid) 

617 

618 obs_list = np.array(obs_list) 

619 self._obs_list = obs_list 

620 obs_ra_list = [] 

621 obs_dec_list = [] 

622 halfspace_list = [] 

623 for obs in obs_list: 

624 obs_ra_list.append(obs.pointingRA) 

625 obs_dec_list.append(obs.pointingDec) 

626 hs = halfSpaceFromRaDec(obs.pointingRA, 

627 obs.pointingDec, 

628 obs.boundLength) 

629 halfspace_list.append(hs) 

630 

631 obs_ra_list = np.array(obs_ra_list) 

632 obs_dec_list = np.array(obs_dec_list) 

633 halfspace_list = np.array(halfspace_list) 

634 

635 self._htmid_dict = {} 

636 self._htmid_list = [] 

637 n_obs_list = [] 

638 fov_radius = 1.75 

639 for i_htmid, htmid in enumerate(valid_htmid): 

640 trixel = self._trixel_dict[htmid] 

641 ra_c, dec_c = trixel.get_center() 

642 radius = trixel.get_radius() 

643 obs_distance = angularSeparation(ra_c, dec_c, obs_ra_list, obs_dec_list) 

644 valid_obs = np.where(obs_distance < radius + fov_radius) 

645 if len(valid_obs[0]) > 0: 

646 final_obs_list = [] 

647 for obs_dex in valid_obs[0]: 

648 hs = halfspace_list[obs_dex] 

649 obs = obs_list[obs_dex] 

650 if hs.contains_trixel(trixel) != 'outside': 

651 final_obs_list.append(obs_dex) 

652 

653 if len(final_obs_list) == 0: 

654 continue 

655 

656 self._htmid_dict[htmid] = np.array(final_obs_list) 

657 self._htmid_list.append(htmid) 

658 n_obs_list.append(len(final_obs_list)) 

659 

660 n_obs_list = np.array(n_obs_list) 

661 self._htmid_list = np.array(self._htmid_list) 

662 sorted_dex = np.argsort(-1.0*n_obs_list) 

663 self._htmid_list = self._htmid_list[sorted_dex] 

664 print('done subdividing obs list -- %d htmid' % 

665 len(self._htmid_list)) 

666 

667 @property 

668 def htmid_list(self): 

669 """ 

670 A list of the unique htmids corresponding to the trixels 

671 that need to be queried to generate the alert data 

672 """ 

673 return self._htmid_list 

674 

675 def n_obs(self, htmid): 

676 """ 

677 Return the number of observations that intersect 

678 the trixel specified by htmid. 

679 

680 Must run subdivide_obs in order for this method to 

681 work. 

682 """ 

683 return len(self._htmid_dict[htmid]) 

684 

685 def obs_from_htmid(self, htmid): 

686 """ 

687 Return a numpy array containing all of the ObservationMetaData 

688 that intersect the trixel specified by htmid. 

689 

690 Must run subdivide_obs in order for this method to 

691 work. 

692 """ 

693 return self._obs_list[self._htmid_dict[htmid]] 

694 

695 def _output_alert_data(self, conn, data_cache): 

696 """ 

697 Write a cache of alert data to the sqlite file currently open. 

698 

699 Parameters 

700 ---------- 

701 conn is the connection to the sqlite file (already open) 

702 

703 data_cache is a dict containing all of the data to be written. 

704 It will keyed on a string like 'i_j' where i is the obshistID 

705 of an OpSim pointing and j is an arbitrary integer. That key 

706 will lead to another dict keyed on the columns being output to 

707 the sqlite file. The values of this second layer of dict are 

708 numpy arrays. 

709 

710 Returns 

711 ------- 

712 The number of rows written to the sqlite file 

713 """ 

714 

715 cursor = conn.cursor() 

716 n_rows_0 = cursor.execute('SELECT COUNT(uniqueId) FROM alert_data').fetchall() 

717 

718 chunk_lengths = np.zeros(len(data_cache)) 

719 

720 for i_cache_tag, cache_tag in enumerate(data_cache): 

721 obshistid = int(cache_tag.split('_')[0]) 

722 n_obj = len(data_cache[cache_tag]['uniqueId']) 

723 chunk_lengths[i_cache_tag] = n_obj 

724 

725 values = ((int(data_cache[cache_tag]['uniqueId'][i_obj]), 

726 obshistid, 

727 data_cache[cache_tag]['xPix'][i_obj], 

728 data_cache[cache_tag]['yPix'][i_obj], 

729 int(data_cache[cache_tag]['chipNum'][i_obj]), 

730 data_cache[cache_tag]['dflux'][i_obj], 

731 data_cache[cache_tag]['SNR'][i_obj], 

732 np.degrees(data_cache[cache_tag]['raICRS'][i_obj]), 

733 np.degrees(data_cache[cache_tag]['decICRS'][i_obj])) 

734 for i_obj in range(n_obj)) 

735 cursor.executemany('INSERT INTO alert_data VALUES (?,?,?,?,?,?,?,?,?)', values) 

736 conn.commit() 

737 

738 n_rows_1 = cursor.execute('SELECT COUNT(uniqueId) FROM alert_data').fetchall() 

739 conn.commit() 

740 n_written = (n_rows_1[0][0]-n_rows_0[0][0]) 

741 

742 return n_written 

743 

744 def _filter_on_photometry_then_chip_name(self, chunk, column_query, 

745 obs_valid_dex, expmjd_list, 

746 photometry_catalog, 

747 dmag_cutoff): 

748 """ 

749 Determine which simulated observations are actually worth storing 

750 by first figuring out which observations of which objects are 

751 photometrically detectable and alert-worthy, then determining 

752 which of those actually fall on an LSST detector. 

753 

754 Parameters 

755 ---------- 

756 chunk is the output yielded from a CatSim ChunkIterator. It is 

757 a numpy recarray representing one chunk_size query from the 

758 underlying simulations database 

759 

760 column_query is a list of the columns that were queried from 

761 the database 

762 

763 obs_valid_dex is a list of integers corresponding to indexes in 

764 self._obs_list of the ObservationMetaData that are actually valid 

765 for the trixel currently being simulated 

766 

767 expmjd_list is a numpy array of the TAI dates of ObservtionMetaData 

768 represented by obs_valid_dex 

769 

770 photometry_catalog is an instantiation of the InstanceCatalog class 

771 being used to calculate magnitudes for these variable sources. 

772 

773 Outputs 

774 ------- 

775 chip_name_dict is a dict keyed on i_obs (which is the index of 

776 an ObservationMetaData's position in obs_valid_dex, NOT its 

777 position in self._obs_list). The values of chip_name_dict are 

778 tuples containing: 

779 - a list of the names of the detectors that objects from chunk 

780 landed on (including Nones for those objects that did not land 

781 on any detector) 

782 

783 - a list of the xPupil coords for every object in chunk 

784 

785 - a list of the yPupil coords for every object in chunk 

786 

787 - a list of the indexes in chunk of those objects which actually 

788 landed on a detector 

789 

790 dmag_arr is a numpy array of the delta_magnitudes of every object 

791 in chunk. dmag_arr[11][3][4] is the delta_magnitude of chunk[4] 

792 in the 3rd band (i.e. the i band) at TAI = expmjd[11]. 

793 

794 dmag_arr_transpose is dmag_arr with the time and object columns 

795 transposed so that dmag_arr_transpose[4][3][11] == dmag_arr[11][3][4]. 

796 

797 time_arr is an array of integers with shape == (len(chunk), len(obs_valid_dex)). 

798 A -1 in time_arr means that that combination of object and observation did 

799 not yield a valid observation. A +1 means that the object and observation 

800 combination are valid. 

801 """ 

802 

803 ###################################################### 

804 # Calculate the delta_magnitude for all of the sources 

805 # 

806 photometry_catalog._set_current_chunk(chunk) 

807 dmag_arr = photometry_catalog.applyVariability(chunk['varParamStr'], 

808 variability_cache=self._variability_cache, 

809 expmjd=expmjd_list).transpose((2, 0, 1)) 

810 

811 dmag_arr_transpose = dmag_arr.transpose(2, 1, 0) 

812 

813 n_raw_obj = len(chunk) 

814 photometrically_valid = -1*np.ones(n_raw_obj, dtype=int) 

815 for i_obj in range(n_raw_obj): 

816 keep_it = False 

817 for i_filter in range(6): 

818 if np.abs(dmag_arr_transpose[i_obj][i_filter]).max() >= dmag_cutoff: 

819 keep_it = True 

820 break 

821 if keep_it: 

822 photometrically_valid[i_obj] = 1 

823 

824 photometrically_valid = np.where(photometrically_valid >= 0) 

825 

826 if 'properMotionRa'in column_query: 

827 pmra = chunk['properMotionRa'][photometrically_valid] 

828 pmdec = chunk['properMotionDec'][photometrically_valid] 

829 px = chunk['parallax'][photometrically_valid] 

830 vrad = chunk['radialVelocity'][photometrically_valid] 

831 else: 

832 pmra = None 

833 pmdec = None 

834 px = None 

835 vrad = None 

836 

837 ################################################################### 

838 # Figure out which sources actually land on an LSST detector during 

839 # the observations in question 

840 # 

841 chip_name_dict = {} 

842 

843 # time_arr will keep track of which objects appear in which observations; 

844 # 1 means the object appears; -1 means it does not 

845 time_arr_transpose = -1*np.ones((len(obs_valid_dex), len(chunk['raJ2000'])), 

846 dtype=int) 

847 

848 for i_obs, obs_dex in enumerate(obs_valid_dex): 

849 obs = self._obs_list[obs_dex] 

850 chip_name_list = np.array([None]*n_raw_obj) 

851 xpup_list = np.zeros(n_raw_obj, dtype=float) 

852 ypup_list = np.zeros(n_raw_obj, dtype=float) 

853 chip_int_arr = -1*np.ones(len(chip_name_list), dtype=int) 

854 

855 if len(photometrically_valid[0]) > 0: 

856 xpup_list_val, ypup_list_val = _pupilCoordsFromRaDec(chunk['raJ2000'][photometrically_valid], 

857 chunk['decJ2000'][photometrically_valid], 

858 pm_ra=pmra, pm_dec=pmdec, 

859 parallax=px, v_rad=vrad, 

860 obs_metadata=obs) 

861 

862 xpup_list[photometrically_valid] = xpup_list_val 

863 ypup_list[photometrically_valid] = ypup_list_val 

864 

865 chip_name_list[photometrically_valid] = chipNameFromPupilCoords(xpup_list_val, 

866 ypup_list_val, 

867 camera=self.lsst_camera) 

868 

869 for i_chip, name in enumerate(chip_name_list): 

870 if name is not None: 

871 chip_int_arr[i_chip] = 1 

872 

873 valid_obj = np.where(chip_int_arr > 0) 

874 time_arr_transpose[i_obs][valid_obj] = 1 

875 

876 chip_name_dict[i_obs] = (chip_name_list, 

877 xpup_list, 

878 ypup_list, 

879 valid_obj) 

880 

881 time_arr = time_arr_transpose.transpose() 

882 assert len(chip_name_dict) == len(obs_valid_dex) 

883 

884 return chip_name_dict, dmag_arr, dmag_arr_transpose, time_arr 

885 

886 def alert_data_from_htmid(self, htmid, dbobj, 

887 dmag_cutoff=0.005, 

888 chunk_size=1000, write_every=10000, 

889 output_dir='.', output_prefix='', 

890 log_file_name=None, 

891 photometry_class=None, 

892 chunk_cutoff=-1, 

893 lock=None): 

894 

895 """ 

896 Generate an sqlite file with all of the alert data for a given 

897 trixel. 

898 

899 Parameters 

900 ---------- 

901 htmid is an integer denoting the trixel from self.htmid_list that should 

902 be simulated 

903 

904 dbobj is a CatalogDBObject connecting to the data underlying the simulation 

905 

906 dmag_cutoff indicates the minimum change magnitude needed to trigger a 

907 simulated alert 

908 

909 chunk_size denotes the number of objects to query from the database and 

910 process at one time 

911 

912 write_every indicates how often to write to the sqlite file (i.e. the 

913 code will pause the simulation process and write to the sqlite file 

914 when it has accumulated this many valid observations) 

915 

916 output_dir is the directory in which to create the sqlite file 

917 

918 output_prefix is the prefix of the sqlite file's name 

919 

920 log_file_name is the name of a text file where progress will be written 

921 

922 photometry_class is a InstanceCatalog class (not an instantiation) that 

923 contains the methods for calculating the photometry associated with the 

924 simulated alerts (see AlertStellarVariabilityCatalog and 

925 AlertAgnVariabilityCatalog in this module) 

926 

927 chunk_cutoff is an optional int; stop the simulation after this many 

928 chunks have been processed. This is for testing purposes. 

929 If chunk_cutoff == -1, the code will process all of the astrophysical 

930 objects in the trixel. 

931 

932 lock is a multiprocessing.Lock() for use if running multiple 

933 instances of alert_data_from_htmid. This will prevent multiple processes 

934 from writing to the log file or stdout simultaneously. 

935 """ 

936 

937 htmid_level = levelFromHtmid(htmid) 

938 if log_file_name is None: 

939 raise RuntimeError('must specify log_file_name') 

940 

941 if ('_PARAMETRIZED_LC_DMAG_LOOKUP' not in self._variability_cache and 

942 self._dmag_lookup_file_exists): 

943 

944 self._variability_cache['_PARAMETRIZED_LC_DMAG_CUTOFF'] = dmag_cutoff 

945 self._variability_cache['_PARAMETRIZED_LC_DMAG_LOOKUP'] = {} 

946 

947 with open(self._dmag_lookup_file, 'r') as in_file: 

948 for line in in_file: 

949 if line[0] == '#': 

950 continue 

951 params = line.split() 

952 self._variability_cache['_PARAMETRIZED_LC_DMAG_LOOKUP'][int(params[0])] = float(params[1]) 

953 

954 self._stdout_lock = lock 

955 this_pid = os.getpid() 

956 

957 t_start = time.time() # so that we can get a sense of how long the full 

958 # simulation will take 

959 

960 desired_columns = [] 

961 desired_columns.append('simobjid') 

962 desired_columns.append('variabilityParameters') 

963 desired_columns.append('varParamStr') 

964 desired_columns.append('raJ2000') 

965 desired_columns.append('decJ2000') 

966 desired_columns.append('properMotionRa') 

967 desired_columns.append('properMotionDec') 

968 desired_columns.append('parallax') 

969 desired_columns.append('radialVelocity') 

970 desired_columns.append('ebv') 

971 desired_columns.append('redshift') 

972 desired_columns.append('htmid') 

973 

974 if 'umag' in dbobj.columnMap: 

975 desired_columns.append('umag') 

976 desired_columns.append('gmag') 

977 desired_columns.append('rmag') 

978 desired_columns.append('imag') 

979 desired_columns.append('zmag') 

980 desired_columns.append('ymag') 

981 elif 'u_ab' in dbobj.columnMap: 

982 desired_columns.append('u_ab') 

983 desired_columns.append('g_ab') 

984 desired_columns.append('r_ab') 

985 desired_columns.append('i_ab') 

986 desired_columns.append('z_ab') 

987 desired_columns.append('y_ab') 

988 else: 

989 raise RuntimeError('Not sure what quiescent ' 

990 'LSST magnitudes are called ' 

991 'in this CatalogDBObject') 

992 

993 if photometry_class is None: 

994 raise RuntimeError('Must specify photometry_class') 

995 

996 if os.path.exists(output_dir) and not os.path.isdir(output_dir): 

997 raise RuntimeError('%s is not a dir' % output_dir) 

998 if not os.path.exists(output_dir): 

999 os.mkdir(output_dir) 

1000 

1001 dummy_sed = Sed() 

1002 

1003 mag_names = ('u', 'g', 'r', 'i', 'z', 'y') 

1004 

1005 phot_params = PhotometricParameters() 

1006 

1007 # from Table 2 of the overview paper 

1008 obs_mag_cutoff = (23.68, 24.89, 24.43, 24.0, 24.45, 22.60) 

1009 

1010 gamma_template = {} 

1011 for i_filter in range(6): 

1012 gamma_template[i_filter] = None 

1013 

1014 obs_valid_dex = self._htmid_dict[htmid] 

1015 print('n valid obs %d' % len(obs_valid_dex)) 

1016 

1017 cat_list = [] 

1018 expmjd_list = [] 

1019 mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 

1020 'i': 3, 'z': 4, 'y': 5} 

1021 for obs_dex in obs_valid_dex: 

1022 obs = self._obs_list[obs_dex] 

1023 cat = photometry_class(dbobj, obs_metadata=obs) 

1024 cat.lsstBandpassDict = self.bp_dict 

1025 cat_list.append(cat) 

1026 expmjd_list.append(obs.mjd.TAI) 

1027 

1028 expmjd_list = np.array(expmjd_list) 

1029 cat_list = np.array(cat_list) 

1030 sorted_dex = np.argsort(expmjd_list) 

1031 

1032 expmjd_list = expmjd_list[sorted_dex] 

1033 cat_list = cat_list[sorted_dex] 

1034 obs_valid_dex = obs_valid_dex[sorted_dex] 

1035 

1036 available_columns = list(dbobj.columnMap.keys()) 

1037 column_query = [] 

1038 for col in desired_columns: 

1039 if col in available_columns: 

1040 column_query.append(col) 

1041 

1042 n_bits_off = 2*(21-htmid_level) 

1043 

1044 data_iter = dbobj.query_columns_htmid(colnames=column_query, 

1045 htmid=htmid, 

1046 chunk_size=chunk_size) 

1047 

1048 photometry_catalog = photometry_class(dbobj, self._obs_list[obs_valid_dex[0]], 

1049 column_outputs=['lsst_u', 

1050 'lsst_g', 

1051 'lsst_r', 

1052 'lsst_i', 

1053 'lsst_z', 

1054 'lsst_y']) 

1055 

1056 i_chunk = 0 

1057 

1058 output_data_cache = {} 

1059 n_rows_cached = 0 

1060 

1061 n_obj = 0 

1062 n_actual_obj = 0 

1063 n_time_last = 0 

1064 n_rows = 0 

1065 

1066 t_before_obj = time.time() # so that we can get a sense of how long the 

1067 # "iterating over astrophysical objects" part 

1068 # of the simulation will take 

1069 

1070 db_name = os.path.join(output_dir, '%s_%d_sqlite.db' % (output_prefix, htmid)) 

1071 with sqlite3.connect(db_name, isolation_level='EXCLUSIVE') as conn: 

1072 creation_cmd = '''CREATE TABLE alert_data 

1073 (uniqueId int, obshistId int, xPix float, yPix float, 

1074 chipNum int, dflux float, snr float, ra float, dec float)''' 

1075 

1076 cursor = conn.cursor() 

1077 cursor.execute('PRAGMA journal_mode=WAL;') 

1078 conn.commit() 

1079 cursor.execute(creation_cmd) 

1080 conn.commit() 

1081 

1082 creation_cmd = '''CREATE TABLE metadata 

1083 (obshistId int, TAI float, band int)''' 

1084 cursor.execute(creation_cmd) 

1085 conn.commit() 

1086 

1087 for obs_dex in obs_valid_dex: 

1088 obs = self._obs_list[obs_dex] 

1089 cmd = '''INSERT INTO metadata 

1090 VALUES(%d, %.5f, %d)''' % (obs.OpsimMetaData['obsHistID'], 

1091 obs.mjd.TAI, 

1092 mag_name_to_int[obs.bandpass]) 

1093 

1094 cursor.execute(cmd) 

1095 conn.commit() 

1096 

1097 creation_cmd = '''CREATE TABLE quiescent_flux 

1098 (uniqueId int, band int, flux float, snr float)''' 

1099 

1100 cursor.execute(creation_cmd) 

1101 conn.commit() 

1102 

1103 creation_cmd = '''CREATE TABLE baseline_astrometry 

1104 (uniqueId int, ra real, dec real, pmRA real, 

1105 pmDec real, parallax real, TAI real)''' 

1106 

1107 cursor.execute(creation_cmd) 

1108 conn.commit() 

1109 

1110 for chunk in data_iter: 

1111 n_raw_obj = len(chunk) 

1112 i_chunk += 1 

1113 

1114 if chunk_cutoff > 0 and i_chunk >= chunk_cutoff: 

1115 break 

1116 

1117 n_time_last = 0 

1118 # filter the chunk so that we are only considering sources that are in 

1119 # the trixel being considered 

1120 reduced_htmid = chunk['htmid'] >> n_bits_off 

1121 

1122 valid_htmid = np.where(reduced_htmid == htmid) 

1123 if len(valid_htmid[0]) == 0: 

1124 continue 

1125 n_htmid_trim = n_raw_obj-len(valid_htmid[0]) 

1126 chunk = chunk[valid_htmid] 

1127 n_obj += len(valid_htmid[0]) 

1128 

1129 (chip_name_dict, 

1130 dmag_arr, 

1131 dmag_arr_transpose, 

1132 time_arr) = self._filter_on_photometry_then_chip_name(chunk, column_query, 

1133 obs_valid_dex, 

1134 expmjd_list, 

1135 photometry_catalog, 

1136 dmag_cutoff) 

1137 

1138 q_f_dict = {} 

1139 q_m_dict = {} 

1140 

1141 q_m_dict[0] = photometry_catalog.column_by_name('quiescent_lsst_u') 

1142 q_m_dict[1] = photometry_catalog.column_by_name('quiescent_lsst_g') 

1143 q_m_dict[2] = photometry_catalog.column_by_name('quiescent_lsst_r') 

1144 q_m_dict[3] = photometry_catalog.column_by_name('quiescent_lsst_i') 

1145 q_m_dict[4] = photometry_catalog.column_by_name('quiescent_lsst_z') 

1146 q_m_dict[5] = photometry_catalog.column_by_name('quiescent_lsst_y') 

1147 

1148 q_f_dict[0] = dummy_sed.fluxFromMag(q_m_dict[0]) 

1149 q_f_dict[1] = dummy_sed.fluxFromMag(q_m_dict[1]) 

1150 q_f_dict[2] = dummy_sed.fluxFromMag(q_m_dict[2]) 

1151 q_f_dict[3] = dummy_sed.fluxFromMag(q_m_dict[3]) 

1152 q_f_dict[4] = dummy_sed.fluxFromMag(q_m_dict[4]) 

1153 q_f_dict[5] = dummy_sed.fluxFromMag(q_m_dict[5]) 

1154 

1155 q_pmra = 1000.0*arcsecFromRadians(photometry_catalog.column_by_name('properMotionRa')) 

1156 q_pmdec = 1000.0*arcsecFromRadians(photometry_catalog.column_by_name('properMotionDec')) 

1157 q_parallax = 1000.0*arcsecFromRadians(photometry_catalog.column_by_name('parallax')) 

1158 q_ra = np.degrees(photometry_catalog.column_by_name('raICRS')) 

1159 q_dec = np.degrees(photometry_catalog.column_by_name('decICRS')) 

1160 q_tai = photometry_catalog.obs_metadata.mjd.TAI 

1161 

1162 q_snr_dict = {} 

1163 for i_filter in range(6): 

1164 

1165 snr_template, local_gamma = calcSNR_m5(q_m_dict[i_filter], 

1166 self.bp_dict[mag_names[i_filter]], 

1167 obs_mag_cutoff[i_filter], 

1168 phot_params, gamma=gamma_template[i_filter]) 

1169 q_snr_dict[i_filter] = snr_template 

1170 gamma_template[i_filter] = local_gamma 

1171 

1172 unq = photometry_catalog.column_by_name('uniqueId') 

1173 

1174 try: 

1175 assert dmag_arr_transpose.shape == (len(chunk), len(mag_names), len(expmjd_list)) 

1176 except AssertionError: 

1177 print('dmag_arr_transpose_shape %s' % str(dmag_arr_transpose.shape)) 

1178 print('should be (%d, %d, %d)' % (len(chunk), len(mag_names), len(expmjd_list))) 

1179 raise 

1180 

1181 # only include those sources for which np.abs(delta_mag) >= dmag_cutoff 

1182 # at some point in their history (note that delta_mag is defined with 

1183 # respect to the quiescent magnitude) 

1184 # 

1185 # also demand that the magnitude at some point is less than obs_mag_cutoff 

1186 # 

1187 # This is different from the dmag>dmag_cutoff check done in 

1188 # 

1189 # self._filter_on_photometry_then_chip_name() 

1190 # 

1191 # Now we have information about which observations actually detected 

1192 # each object (in self._filter_on_photometry_then_chip_name(), 

1193 # we assumed that every object was detected at every time step). 

1194 

1195 photometrically_valid_obj = [] 

1196 for i_obj in range(len(chunk)): 

1197 keep_it = False 

1198 valid_times = np.where(time_arr[i_obj] > 0) 

1199 if len(valid_times[0]) == 0: 

1200 continue 

1201 for i_filter in range(len(mag_names)): 

1202 if np.abs(dmag_arr_transpose[i_obj][i_filter][valid_times]).max() > dmag_cutoff: 

1203 dmag_min = dmag_arr_transpose[i_obj][i_filter][valid_times].min() 

1204 if q_m_dict[i_filter][i_obj] + dmag_min <= obs_mag_cutoff[i_filter]: 

1205 keep_it = True 

1206 break 

1207 if keep_it: 

1208 photometrically_valid_obj.append(i_obj) 

1209 photometrically_valid_obj = np.array(photometrically_valid_obj) 

1210 

1211 del dmag_arr_transpose 

1212 gc.collect() 

1213 

1214 if np.abs(dmag_arr).max() < dmag_cutoff: 

1215 continue 

1216 

1217 completely_valid = np.zeros(len(chunk), dtype=int) 

1218 

1219 ############################ 

1220 # Process and output sources 

1221 # 

1222 for i_obs, obs_dex in enumerate(obs_valid_dex): 

1223 obs = self._obs_list[obs_dex] 

1224 obshistid = obs.OpsimMetaData['obsHistID'] 

1225 

1226 obs_mag = obs.bandpass 

1227 actual_i_mag = mag_name_to_int[obs_mag] 

1228 assert mag_names[actual_i_mag] == obs_mag 

1229 

1230 # only include those sources which fall on a detector for this pointing 

1231 valid_chip_name, valid_xpup, valid_ypup, chip_valid_obj = chip_name_dict[i_obs] 

1232 

1233 actually_valid_obj = np.intersect1d(photometrically_valid_obj, chip_valid_obj) 

1234 if len(actually_valid_obj) == 0: 

1235 continue 

1236 

1237 try: 

1238 completely_valid[actually_valid_obj] += 1 

1239 except: 

1240 print('failed') 

1241 print(actually_valid_obj) 

1242 print(completely_valid) 

1243 raise 

1244 

1245 valid_sources = chunk[actually_valid_obj] 

1246 local_column_cache = {} 

1247 local_column_cache['deltaMagAvro'] = OrderedDict([('delta_%smag' % mag_names[i_mag], 

1248 dmag_arr[i_obs][i_mag][actually_valid_obj]) 

1249 for i_mag in range(len(mag_names))]) 

1250 

1251 local_column_cache['chipName'] = valid_chip_name[actually_valid_obj] 

1252 local_column_cache['pupilFromSky'] = OrderedDict([('x_pupil', valid_xpup[actually_valid_obj]), 

1253 ('y_pupil', valid_ypup[actually_valid_obj])]) 

1254 

1255 cat = cat_list[i_obs] 

1256 i_valid_chunk = 0 

1257 for valid_chunk, chunk_map in cat.iter_catalog_chunks(query_cache=[valid_sources], 

1258 column_cache=local_column_cache): 

1259 i_valid_chunk += 1 

1260 assert i_valid_chunk == 1 

1261 n_time_last += len(valid_chunk[0]) 

1262 length_of_chunk = len(valid_chunk[chunk_map['uniqueId']]) 

1263 cache_tag = '%d_%d' % (obshistid, i_chunk) 

1264 output_data_cache[cache_tag] = {} 

1265 

1266 for col_name in ('uniqueId', 'raICRS', 'decICRS', 'flux', 'dflux', 'SNR', 

1267 'chipNum', 'xPix', 'yPix'): 

1268 

1269 output_data_cache[cache_tag][col_name] = valid_chunk[chunk_map[col_name]] 

1270 

1271 n_rows_cached += length_of_chunk 

1272 

1273 completely_valid = np.where(completely_valid > 0) 

1274 for i_filter in range(6): 

1275 values = ((int(unq[completely_valid][i_q]), 

1276 i_filter, 

1277 q_f_dict[i_filter][completely_valid][i_q], 

1278 q_snr_dict[i_filter][completely_valid][i_q]) 

1279 for i_q in range(len(completely_valid[0]))) 

1280 cursor.executemany('INSERT INTO quiescent_flux VALUES (?,?,?,?)', values) 

1281 conn.commit() 

1282 

1283 values = ((int(unq[completely_valid][i_q]), 

1284 q_ra[completely_valid][i_q], 

1285 q_dec[completely_valid][i_q], 

1286 q_pmra[completely_valid][i_q], 

1287 q_pmdec[completely_valid][i_q], 

1288 q_parallax[completely_valid][i_q], 

1289 q_tai) 

1290 for i_q in range(len(completely_valid[0]))) 

1291 

1292 cursor.executemany('INSERT INTO baseline_astrometry VALUES (?,?,?,?,?,?,?)', values) 

1293 

1294 if n_rows_cached >= write_every: 

1295 self.acquire_lock() 

1296 with open(log_file_name, 'a') as out_file: 

1297 out_file.write('%d is writing \n' % os.getpid()) 

1298 

1299 print('%d is writing' % os.getpid()) 

1300 

1301 self.release_lock() 

1302 

1303 n_rows += self._output_alert_data(conn, output_data_cache) 

1304 output_data_cache = {} 

1305 n_rows_cached = 0 

1306 

1307 if n_rows > 0: 

1308 self.acquire_lock() 

1309 with open(log_file_name, 'a') as out_file: 

1310 elapsed = (time.time()-t_before_obj)/3600.0 

1311 elapsed_per = elapsed/n_rows 

1312 rows_per_chunk = float(n_rows)/float(i_chunk) 

1313 total_projection = 1000.0*rows_per_chunk*elapsed_per 

1314 out_file.write('\n %d n_obj %d %d trimmed %d\n' % 

1315 (this_pid, n_obj, n_actual_obj, n_htmid_trim)) 

1316 out_file.write(' elapsed %.2e hrs per row %.2e total %2e\n' % 

1317 (elapsed, elapsed_per, total_projection)) 

1318 out_file.write(' n_time_last %d; rows %d\n' % (n_time_last, n_rows)) 

1319 

1320 out_file.write('%d is done writing\n' % os.getpid()) 

1321 

1322 print('\n %d n_obj %d %d trimmed %d' % 

1323 (this_pid, n_obj, n_actual_obj, n_htmid_trim)) 

1324 print(' elapsed %.2e hrs per row %.2e total %2e' % 

1325 (elapsed, elapsed_per, total_projection)) 

1326 print(' n_time_last %d; rows %d\n' % (n_time_last, n_rows)) 

1327 print('%d is done writing' % os.getpid()) 

1328 

1329 self.release_lock() 

1330 

1331 if len(output_data_cache) > 0: 

1332 n_rows += self._output_alert_data(conn, output_data_cache) 

1333 output_data_cache = {} 

1334 

1335 print('htmid %d that took %.2e hours; n_obj %d n_rows %d' % 

1336 (htmid, (time.time()-t_start)/3600.0, n_obj, n_rows)) 

1337 

1338 self.acquire_lock() 

1339 print("INDEXING %d" % htmid) 

1340 self.release_lock() 

1341 

1342 cursor.execute('CREATE INDEX unq_obs ON alert_data (uniqueId, obshistId)') 

1343 cursor.execute('CREATE INDEX unq_flux ON quiescent_flux (uniqueId, band)') 

1344 cursor.execute('CREATE INDEX obs ON metadata (obshistid)') 

1345 cursor.execute('CREATE INDEX unq_ast ON baseline_astrometry (uniqueId)') 

1346 conn.commit() 

1347 

1348 self.acquire_lock() 

1349 with open(log_file_name, 'a') as out_file: 

1350 out_file.write('done with htmid %d -- %e %d\n' % 

1351 (htmid, (time.time()-t_start)/3600.0, n_obj)) 

1352 self.release_lock() 

1353 

1354 return n_rows