Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from __future__ import print_function 

2from builtins import zip 

3from builtins import str 

4from builtins import object 

5import numpy as np 

6import copy 

7from collections import OrderedDict 

8 

9from lsst.sims.catUtils.utils import ObservationMetaDataGenerator 

10from lsst.sims.catUtils.mixins import PhotometryStars, VariabilityStars 

11from lsst.sims.catUtils.mixins import PhotometryGalaxies, VariabilityGalaxies 

12from lsst.sims.catalogs.definitions import InstanceCatalog 

13from lsst.sims.catalogs.decorators import compound, cached 

14from lsst.sims.utils import haversine 

15 

16import time 

17 

18__all__ = ["StellarLightCurveGenerator", 

19 "FastStellarLightCurveGenerator", 

20 "AgnLightCurveGenerator", 

21 "FastAgnLightCurveGenerator", 

22 "_baseLightCurveCatalog", 

23 "LightCurveGenerator", 

24 "FastLightCurveGenerator"] 

25 

26# a global cache to store SedLists loaded by the light curve catalogs 

27_sed_cache = {} 

28 

29 

30class _baseLightCurveCatalog(InstanceCatalog): 

31 """ 

32 """ 

33 

34 column_outputs = ["uniqueId", "raJ2000", "decJ2000", 

35 "lightCurveMag", "sigma_lightCurveMag", 

36 "truthInfo", "quiescent_lightCurveMag"] 

37 

38 def iter_catalog(self, chunk_size=None, query_cache=None, column_cache=None): 

39 """ 

40 Returns an iterator over rows of the catalog. 

41 

42 Parameters 

43 ---------- 

44 chunk_size : int, optional, defaults to None 

45 the number of rows to return from the database at a time. If None, 

46 returns the entire database query in one chunk. 

47 

48 query_cache : iterator over database rows, optional, defaults to None 

49 the result of calling db_obj.query_columns(). If query_cache is not 

50 None, this method will iterate over the rows in query_cache and produce 

51 an appropriate InstanceCatalog. DO NOT set to non-None values 

52 unless you know what you are doing. It is an optional 

53 input for those who want to repeatedly examine the same patch of sky 

54 without actually querying the database over and over again. If it is set 

55 to None (default), this method will handle the database query. 

56 

57 column_cache : a dict that will be copied over into the catalogs self._column_cache. 

58 Should be left as None, unless you know what you are doing. 

59 """ 

60 

61 if query_cache is None: 

62 # Call the originalversion of iter_catalog defined in the 

63 # InstanceCatalog class. This version of iter_catalog includes 

64 # the call to self.db_obj.query_columns, which the user would have 

65 # used to generate query_cache. 

66 for line in InstanceCatalog.iter_catalog(self, chunk_size=chunk_size): 

67 yield line 

68 else: 

69 # Otherwise iterate over the query cache 

70 transform_keys = list(self.transformations.keys()) 

71 for chunk in query_cache: 

72 self._set_current_chunk(chunk, column_cache=column_cache) 

73 chunk_cols = [self.transformations[col](self.column_by_name(col)) 

74 if col in transform_keys else 

75 self.column_by_name(col) 

76 for col in self.iter_column_names()] 

77 # iterate over lines in the cache and yield lines augmented by 

78 # values calculated using this catalogs getter methods 

79 for line in zip(*chunk_cols): 

80 yield line 

81 

82 @cached 

83 def get_truthInfo(self): 

84 """ 

85 Default information to be returned as 'truth_dict' by the 

86 LightCurveGenerator. 

87 """ 

88 return self.column_by_name('varParamStr') 

89 

90 

91class _stellarLightCurveCatalog(_baseLightCurveCatalog, VariabilityStars, PhotometryStars): 

92 """ 

93 This class wraps a basic stellar variability InstanceCatalog. It provides its 

94 own photometry getter that 

95 

96 1) only returns the magnitude and uncertainty in the bandpass specified by 

97 self.obs_metadata 

98 

99 2) caches all of the SEDs read in so that they can be reused when sampling the 

100 objects in this catalog at a different MJD. 

101 

102 It should only be used in the context of the LightCurveGenerator class. 

103 """ 

104 

105 def _loadSedList(self, wavelen_match): 

106 """ 

107 Wraps the PhotometryStars._loadSedList method. 

108 

109 If current chunk of objects is not represetned in the global 

110 _sed_cache, this will call the base method defined in 

111 PhotometryStars. 

112 

113 Otherwise, it will read self._sedList from the cache. 

114 That way, the photometry getters defined in PhotometryStars will 

115 not read in SEDs that have already been cached. 

116 """ 

117 

118 global _sed_cache 

119 

120 object_names = self.column_by_name("uniqueId") 

121 

122 if len(object_names) > 0: 

123 cache_name = "stellar_%s_%s" % (object_names[0], object_names[-1]) 

124 else: 

125 cache_name = None 

126 

127 if cache_name not in _sed_cache: 

128 

129 PhotometryStars._loadSedList(self, wavelen_match) 

130 

131 if cache_name is not None: 

132 _sed_cache[cache_name] = copy.copy(self._sedList) 

133 else: 

134 self._sedList = copy.copy(_sed_cache[cache_name]) 

135 

136 @compound("lightCurveMag", "sigma_lightCurveMag", "quiescent_lightCurveMag") 

137 def get_lightCurvePhotometry(self): 

138 """ 

139 A getter which returns the magnitudes and uncertainties in magnitudes 

140 in the bandpass specified by self.obs_metdata. 

141 

142 As it runs, this method will cache the SedLists it reads in so that 

143 they can be used later. 

144 """ 

145 

146 if len(self.obs_metadata.bandpass) != 1: 

147 raise RuntimeError("_stellarLightCurveCatalog cannot handle bandpass " 

148 "%s" % str(self.obs_metadata.bandpass)) 

149 

150 bp = self.obs_metadata.bandpass 

151 

152 return np.array([self.column_by_name("lsst_%s" % bp), 

153 self.column_by_name("sigma_lsst_%s" % bp), 

154 self.column_by_name("lsst_%s" % bp) - self.column_by_name("delta_lsst_%s" % bp)]) 

155 

156 

157class _agnLightCurveCatalog(_baseLightCurveCatalog, VariabilityGalaxies, PhotometryGalaxies): 

158 

159 def _loadAgnSedList(self, wavelen_match): 

160 """ 

161 Wraps the PhotometryGalaxies._loadAgnSedList method. 

162 

163 If current chunk of objects is not represented in the global 

164 _sed_cache, this will call the base method defined in 

165 PhotometryGalaxies. 

166 

167 Otherwise, it will read self._sedList from the cache. 

168 That way, the photometry getters defined in PhotometryGalaxies will 

169 not read in SEDs that have already been cached. 

170 """ 

171 

172 global _sed_cache 

173 

174 object_names = self.column_by_name("uniqueId") 

175 

176 if len(object_names) > 0: 

177 cache_name = "agn_%s_%s" % (object_names[0], object_names[-1]) 

178 else: 

179 cache_name = None 

180 

181 if cache_name not in _sed_cache: 

182 

183 PhotometryGalaxies._loadAgnSedList(self, wavelen_match) 

184 

185 if cache_name is not None: 

186 _sed_cache[cache_name] = copy.copy(self._agnSedList) 

187 else: 

188 self._agnSedList = copy.copy(_sed_cache[cache_name]) 

189 

190 @compound("lightCurveMag", "sigma_lightCurveMag", "quiescent_lightCurveMag") 

191 def get_lightCurvePhotometry(self): 

192 """ 

193 A getter which returns the magnitudes and uncertainties in magnitudes 

194 in the bandpass specified by self.obs_metdata. 

195 

196 As it runs, this method will cache the SedLists it reads in so that 

197 they can be used later. 

198 """ 

199 

200 if len(self.obs_metadata.bandpass) != 1: 

201 raise RuntimeError("_agnLightCurveCatalog cannot handle bandpass " 

202 "%s" % str(self.obs_metadata.bandpass)) 

203 

204 bp = self.obs_metadata.bandpass 

205 

206 return np.array([self.column_by_name("%sAgn" % bp), 

207 self.column_by_name("sigma_%sAgn" % bp), 

208 self.column_by_name("%sAgn" % bp) - self.column_by_name("delta_%sAgn" % bp)]) 

209 

210 

211class LightCurveGenerator(object): 

212 """ 

213 This class will find all of the OpSim pointings in a particular region 

214 of the sky in a particular filter and then return light curves for all 

215 of the objects observed in that region of sky. 

216 

217 Input parameters: 

218 ----------------- 

219 catalogdb is a CatalogDBObject instantiation connecting to the database 

220 of objects to be observed. 

221 

222 opsimdb is the path to the OpSim database of observation. 

223 

224 opsimdriver (optional; default 'sqlite') indicates the database driver to 

225 be used when connecting to opsimdb. 

226 """ 

227 

228 _lightCurveCatalogClass = None 

229 _brightness_name = 'mag' 

230 

231 def __init__(self, catalogdb, opsimdb, opsimdriver="sqlite"): 

232 self._generator = ObservationMetaDataGenerator(database=opsimdb, 

233 driver=opsimdriver) 

234 

235 self._catalogdb = catalogdb 

236 

237 # optional constraint on query to catalog database 

238 # (usually 'varParamStr IS NOT NULL') 

239 if not hasattr(self, '_constraint'): 

240 self._constraint = None 

241 

242 def _filter_chunk(self, chunk): 

243 return chunk 

244 

245 def get_pointings(self, ra, dec, 

246 bandpass=('u', 'g', 'r', 'i', 'z', 'y'), 

247 expMJD=None, 

248 boundLength=1.75): 

249 """ 

250 Inputs 

251 ------- 

252 ra is a tuple indicating the (min, max) values of RA in degrees. 

253 

254 dec is a tuple indicating the (min, max) values of Dec in degrees. 

255 

256 bandpass is a str (i.e. 'u', 'g', 'r', etc.) or an iterable indicating 

257 which filter(s) you want the light curves in. Defaults to all six 

258 LSST bandpasses. 

259 

260 expMJD is an optional tuple indicating a (min, max) range in MJD. 

261 Defaults to None, in which case, the light curves over the entire 

262 10 year survey are returned. 

263 

264 boundLength is the radius in degrees of the field of view of each 

265 returned ObservationMetaData (default=1.75, the radius of the LSST 

266 field of view). 

267 

268 Outputs 

269 ------- 

270 A 2-D list of ObservationMetaData objects. Each row is a list of 

271 ObservationMetaDatas that point to the same patch of sky, sorted by MJD. 

272 Pointings will not be sorted or grouped by filter. 

273 """ 

274 

275 print('parameters', ra, dec, bandpass, expMJD) 

276 if isinstance(bandpass, str): 

277 obs_list = self._generator.getObservationMetaData(fieldRA=ra, 

278 fieldDec=dec, 

279 telescopeFilter=bandpass, 

280 expMJD=expMJD, 

281 boundLength=boundLength) 

282 else: 

283 # obs_list will be populated with a list of lists of ObservationMetaData. 

284 # These ObervationMetaData will have pointingRA, pointingDec, filters, 

285 # and mjd values as specified by the user-input parameters. 

286 # Each row of obs_list will be a list of ObservationMetaData with identical 

287 # pointingRA and pointingDec. 

288 # 

289 obs_list = [] 

290 for bp in bandpass: 

291 sub_list = self._generator.getObservationMetaData(fieldRA=ra, 

292 fieldDec=dec, 

293 telescopeFilter=bp, 

294 expMJD=expMJD, 

295 boundLength=boundLength) 

296 obs_list += sub_list 

297 

298 if len(obs_list) == 0: 

299 print("No observations found matching your criterion") 

300 return None 

301 

302 # Group the OpSim pointings so that all of the pointings centered on the same 

303 # point in the sky are in a list together (this will allow us to generate the 

304 # light curves one pointing at a time without having to query the database for 

305 # the same results more than once. 

306 tol = 1.0e-12 

307 

308 obs_groups = [] # a list of list of the indices of the ObservationMetaDatas 

309 # in obs_list. All of the ObservationMetaData in 

310 # obs_list[i] will point to the same point on the sky. 

311 

312 mjd_groups = [] # a list of lists of the MJDs of the ObservationMetaDatas 

313 # so that they can be sorted into chronological order before 

314 # light curves are calculated 

315 

316 for iobs, obs in enumerate(obs_list): 

317 group_dex = -1 

318 

319 for ix, obs_g in enumerate(obs_groups): 

320 dd = haversine(obs._pointingRA, obs._pointingDec, 

321 obs_list[obs_g[0]]._pointingRA, obs_list[obs_g[0]]._pointingDec) 

322 if dd < tol: 

323 group_dex = ix 

324 break 

325 

326 if group_dex == -1: 

327 obs_groups.append([iobs]) 

328 mjd_groups.append([obs_list[iobs].mjd.TAI]) 

329 else: 

330 obs_groups[group_dex].append(iobs) 

331 mjd_groups[group_dex].append(obs_list[iobs].mjd.TAI) 

332 

333 # rearrange each group of ObservationMetaDatas so that they 

334 # appear in chronological order by MJD 

335 obs_groups_out = [] 

336 for ix, (grp, mjd) in enumerate(zip(obs_groups, mjd_groups)): 

337 oo = np.array(grp) 

338 mm = np.array(mjd) 

339 dexes = np.argsort(mm) 

340 obs_groups_out.append([obs_list[ii] for ii in oo[dexes]]) 

341 

342 return obs_groups_out 

343 

344 def _get_query_from_group(self, grp, chunk_size, lc_per_field=None, constraint=None): 

345 """ 

346 Take a group of ObervationMetaData that all point to the same region 

347 of the sky. Query the CatSim database for all of the celestial objects 

348 in that region, and return it as an iterator over database rows. 

349 

350 grp is a list of ObservationMetaData that all point at the same field 

351 on the sky. 

352 

353 chunk_size is an int specifying th largest chunk of database rows 

354 to be held in memory atthe same time. 

355 

356 lc_per_field specifies the maximum number of light curves to return 

357 per field of view (None implies no constraint). 

358 

359 constraint is a string containing a SQL constraint to be applied to 

360 all database queries associated with generating these light curves 

361 (optional). 

362 """ 

363 

364 if self._constraint is not None and constraint is not None: 

365 master_constraint = self._constraint + ' AND ' + constraint 

366 elif self._constraint is not None and constraint is None: 

367 master_constraint = self._constraint 

368 elif self._constraint is None and constraint is not None: 

369 master_constraint = constraint 

370 else: 

371 master_constraint = None 

372 

373 cat = self._lightCurveCatalogClass(self._catalogdb, obs_metadata=grp[0]) 

374 

375 cat.db_required_columns() 

376 

377 query_result = cat.db_obj.query_columns(colnames=cat._active_columns, 

378 obs_metadata=cat.obs_metadata, 

379 constraint=master_constraint, 

380 limit=lc_per_field, 

381 chunk_size=chunk_size) 

382 

383 return query_result 

384 

385 def _light_curves_from_query(self, cat_dict, query_result, grp, lc_per_field=None): 

386 """ 

387 Read in an iterator over database rows and return light curves for 

388 all of the objects contained. 

389 

390 Input parameters: 

391 ----------------- 

392 cat_dict is a dict of InstanceCatalogs keyed on bandpass name. There 

393 only needs to be one InstanceCatalog per bandpass name. These dummy 

394 catalogs provide the methods needed to calculate synthetic photometry. 

395 

396 query_result is an iterator over database rows that correspond to 

397 celestial objects in our field of view. 

398 

399 grp is a list of ObservationMetaData objects that all point to the 

400 region of sky containing the objects in query_result. cat_dict 

401 should contain an InstanceCatalog for each bandpass represented in 

402 grp. 

403 

404 lc_per_field is an optional int specifying the number of objects per 

405 OpSim field to return. Ordinarily, this is handled at the level of 

406 querying the database, but, when querying our tiled galaxy tables, 

407 it is impossible to impose a row limit on the query. Therefore, 

408 we may have to place the lc_per_field restriction here. 

409 

410 Output 

411 ------ 

412 This method does not output anything. It adds light curves to the 

413 instance member variables self.mjd_dict, self.bright_dict, self.sig_dict, 

414 and self.truth_dict. 

415 """ 

416 

417 global _sed_cache 

418 

419 # local_gamma_cache will cache the InstanceCatalog._gamma_cache 

420 # values used by the photometry mixins to efficiently calculate 

421 # photometric uncertainties in each catalog. 

422 local_gamma_cache = {} 

423 

424 row_ct = 0 

425 

426 for raw_chunk in query_result: 

427 chunk = self._filter_chunk(raw_chunk) 

428 if lc_per_field is not None: 

429 

430 if row_ct >= lc_per_field: 

431 break 

432 

433 if row_ct + len(chunk) > lc_per_field: 

434 chunk = chunk[:lc_per_field-row_ct] 

435 row_ct += len(chunk) 

436 else: 

437 row_ct += len(chunk) 

438 

439 if chunk is not None: 

440 for ix, obs in enumerate(grp): 

441 cat = cat_dict[obs.bandpass] 

442 cat.obs_metadata = obs 

443 if ix in local_gamma_cache: 

444 cat._gamma_cache = local_gamma_cache[ix] 

445 else: 

446 cat._gamma_cache = {} 

447 

448 for star_obj in \ 

449 cat.iter_catalog(query_cache=[chunk]): 

450 

451 if not np.isnan(star_obj[3]) and not np.isinf(star_obj[3]): 

452 

453 if star_obj[0] not in self.truth_dict: 

454 self.truth_dict[star_obj[0]] = star_obj[5] 

455 

456 if star_obj[0] not in self.mjd_dict: 

457 self.mjd_dict[star_obj[0]] = {} 

458 self.bright_dict[star_obj[0]] = {} 

459 self.sig_dict[star_obj[0]] = {} 

460 

461 bp = cat.obs_metadata.bandpass 

462 if bp not in self.mjd_dict[star_obj[0]]: 

463 self.mjd_dict[star_obj[0]][bp] = [] 

464 self.bright_dict[star_obj[0]][bp] = [] 

465 self.sig_dict[star_obj[0]][bp] = [] 

466 

467 self.mjd_dict[star_obj[0]][bp].append(cat.obs_metadata.mjd.TAI) 

468 self.bright_dict[star_obj[0]][bp].append(star_obj[3]) 

469 self.sig_dict[star_obj[0]][bp].append(star_obj[4]) 

470 

471 if ix not in local_gamma_cache: 

472 local_gamma_cache[ix] = cat._gamma_cache 

473 

474 _sed_cache = {} # before moving on to the next chunk of objects 

475 

476 def light_curves_from_pointings(self, pointings, chunk_size=100000, 

477 lc_per_field=None, constraint=None): 

478 """ 

479 Generate light curves for all of the objects in a particular region 

480 of sky in a particular bandpass. 

481 

482 Input parameters: 

483 ----------------- 

484 

485 pointings is a 2-D list of ObservationMetaData objects. Each row 

486 of pointings is a list of ObservationMetaDatas that all point to 

487 the same patch of sky, sorted by MJD. This can be generated with 

488 the method get_pointings(). 

489 

490 chunk_size (optional; default=10000) is an int specifying how many 

491 objects to pull in from the database at a time. Note: the larger 

492 this is, the faster the LightCurveGenerator will run, because it 

493 will be handling more objects in memory at once. 

494 

495 lc_per_field (optional; default None) is an int specifying the maximum 

496 number of light curves to return per field of view (None implies no 

497 constraint). 

498 

499 constraint is a string containing a SQL constraint to be applied to 

500 all database queries associated with generating these light curves 

501 (optional). 

502 

503 Output: 

504 ------- 

505 A dict of light curves. The dict is keyed on the object's uniqueId. 

506 This yields a dict keyed on bandpass, which yields a dict keyed on 

507 'mjd', 'mag', and 'error', i.e. 

508 

509 output[111]['u']['mjd'] is a numpy array of the MJD of observations 

510 of object 111 in the u band. 

511 

512 output[111]['u']['mag'] is a numpy array of the magnitudes of 

513 object 111 in the u band. 

514 

515 output[111]['u']['error'] is a numpy array of the magnitude uncertainties 

516 of object 111 in the u band. 

517 

518 And a dict of truth data for each of the objects (again, keyed on 

519 uniqueId). The contents of this dict will vary, depending on the 

520 variability model being used, but should be sufficient to reconstruct 

521 the actual light curves 'by hand' if necessary (or determine the true 

522 period of a variable source, if attempting to evaluate the performance 

523 of a classification scheme against a proposed observing cadence). 

524 """ 

525 

526 # First get the list of ObservationMetaData objects corresponding 

527 # to the OpSim pointings in the region and bandpass of interest 

528 

529 t_start = time.time() 

530 

531 self.mjd_dict = {} 

532 self.bright_dict = {} 

533 self.sig_dict = {} 

534 self.band_dict = {} 

535 self.truth_dict = {} 

536 

537 cat_dict = {} 

538 for grp in pointings: 

539 for obs in grp: 

540 if obs.bandpass not in cat_dict: 

541 cat_dict[obs.bandpass] = self._lightCurveCatalogClass(self._catalogdb, obs_metadata=obs) 

542 

543 # Loop over the list of groups ObservationMetaData objects, 

544 # querying the database and generating light curves. 

545 for grp in pointings: 

546 

547 self._mjd_min = grp[0].mjd.TAI 

548 self._mjd_max = grp[-1].mjd.TAI 

549 

550 print('starting query') 

551 

552 t_before_query = time.time() 

553 query_result = self._get_query_from_group(grp, chunk_size, lc_per_field=lc_per_field, 

554 constraint=constraint) 

555 

556 print('query took ', time.time()-t_before_query) 

557 

558 self._light_curves_from_query(cat_dict, query_result, grp, lc_per_field=lc_per_field) 

559 

560 output_dict = {} 

561 for unique_id in self.mjd_dict: 

562 output_dict[unique_id] = {} 

563 for bp in self.mjd_dict[unique_id]: 

564 

565 output_dict[unique_id][bp] = {} 

566 

567 # we must sort the MJDs because, if an object appears in multiple 

568 # spatial pointings, its observations will be concatenated out of order 

569 mjd_arr = np.array(self.mjd_dict[unique_id][bp]) 

570 mjd_dexes = np.argsort(mjd_arr) 

571 

572 output_dict[unique_id][bp]['mjd'] = mjd_arr[mjd_dexes] 

573 output_dict[unique_id][bp][self._brightness_name] = \ 

574 np.array(self.bright_dict[unique_id][bp])[mjd_dexes] 

575 output_dict[unique_id][bp]['error'] = np.array(self.sig_dict[unique_id][bp])[mjd_dexes] 

576 

577 print('light curves took %e seconds to generate' % (time.time()-t_start)) 

578 return output_dict, self.truth_dict 

579 

580 

581class FastLightCurveGenerator(LightCurveGenerator): 

582 """ 

583 This LightCurveGenerator sub-class will be specifically designed for variability 

584 models that are implemented as 

585 

586 mag = quiescent_mag + delta_mag(t) 

587 

588 where quiescent_mag does not change as a function of time. 

589 

590 It will re-implement the _light_curves_from_query() method so that quiescent_mag 

591 is only calculated once, and delta_mag(t) is calculated in a vectorized fashion. 

592 """ 

593 

594 def _light_curves_from_query(self, cat_dict, query_result, grp, lc_per_field=None): 

595 """ 

596 Read in an iterator over database rows and return light curves for 

597 all of the objects contained. 

598 

599 Input parameters: 

600 ----------------- 

601 cat_dict is a dict of InstanceCatalogs keyed on bandpass name. There 

602 only needs to be one InstanceCatalog per bandpass name. These dummy 

603 catalogs provide the methods needed to calculate synthetic photometry. 

604 

605 query_result is an iterator over database rows that correspond to 

606 celestial objects in our field of view. 

607 

608 grp is a list of ObservationMetaData objects that all point to the 

609 region of sky containing the objects in query_result. cat_dict 

610 should contain an InstanceCatalog for each bandpass represented in 

611 grp. 

612 

613 lc_per_field is an optional int specifying the number of objects per 

614 OpSim field to return. Ordinarily, this is handled at the level of 

615 querying the database, but, when querying our tiled galaxy tables, 

616 it is impossible to impose a row limit on the query. Therefore, 

617 we may have to place the lc_per_field restriction here. 

618 

619 Output 

620 ------ 

621 This method does not output anything. It adds light curves to the 

622 instance member variables self.mjd_dict, self.bright_dict, self.sig_dict, 

623 and self.truth_dict. 

624 """ 

625 

626 print('using fast light curve generator') 

627 

628 global _sed_cache 

629 

630 # local_gamma_cache will cache the InstanceCatalog._gamma_cache 

631 # values used by the photometry mixins to efficiently calculate 

632 # photometric uncertainties in each catalog. 

633 local_gamma_cache = {} 

634 

635 row_ct = 0 

636 

637 # assemble dicts needed for data precalculation 

638 # 

639 # quiescent_ob_dict contains one ObservationMetaData per bandpass 

640 # 

641 # mjd_arr_dict is a dict of all of the mjd values needed per bandpass 

642 # 

643 # time_lookup_dict is a dict of dicts; for each bandpass it will 

644 # provide a dict that allows you to convert mjd to index in the 

645 # mjd_arr_dict[bp] array. 

646 quiescent_obs_dict = {} 

647 mjd_arr_dict = {} 

648 time_lookup_dict = {} 

649 for obs in grp: 

650 bp = obs.bandpass 

651 if bp not in quiescent_obs_dict: 

652 quiescent_obs_dict[bp] = obs 

653 if bp not in mjd_arr_dict: 

654 mjd_arr_dict[bp] = [] 

655 mjd_arr_dict[bp].append(obs.mjd.TAI) 

656 if bp not in time_lookup_dict: 

657 time_lookup_dict[bp] = {} 

658 time_lookup_dict[bp][obs.mjd.TAI] = len(mjd_arr_dict[obs.bandpass])-1 

659 

660 for bp in mjd_arr_dict: 

661 mjd_arr_dict[bp] = np.array(mjd_arr_dict[bp]) 

662 

663 for raw_chunk in query_result: 

664 chunk = self._filter_chunk(raw_chunk) 

665 if lc_per_field is not None: 

666 

667 if row_ct >= lc_per_field: 

668 break 

669 

670 if row_ct + len(chunk) > lc_per_field: 

671 chunk = chunk[:lc_per_field-row_ct] 

672 row_ct += len(chunk) 

673 else: 

674 row_ct += len(chunk) 

675 

676 if chunk is not None: 

677 quiescent_mags = {} 

678 d_mags = {} 

679 # pre-calculate quiescent magnitudes 

680 # and delta magnitude arrays 

681 for bp in quiescent_obs_dict: 

682 cat = cat_dict[bp] 

683 cat.obs_metadata = quiescent_obs_dict[bp] 

684 local_quiescent_mags = [] 

685 for star_obj in cat.iter_catalog(query_cache=[chunk]): 

686 local_quiescent_mags.append(star_obj[6]) 

687 quiescent_mags[bp] = np.array(local_quiescent_mags) 

688 if self.delta_name_mapper(bp) not in cat._actually_calculated_columns: 

689 cat._actually_calculated_columns.append(self.delta_name_mapper(bp)) 

690 varparamstr = cat.column_by_name('varParamStr') 

691 temp_d_mags = cat.applyVariability(varparamstr, mjd_arr_dict[bp]) 

692 d_mags[bp] = temp_d_mags[{'u':0, 'g':1, 'r':2, 'i':3, 'z':4, 'y':5}[bp]].transpose() 

693 

694 for ix, obs in enumerate(grp): 

695 bp = obs.bandpass 

696 cat = cat_dict[bp] 

697 cat.obs_metadata = obs 

698 time_dex = time_lookup_dict[bp][obs.mjd.TAI] 

699 

700 # build up a column_cache of the pre-calculated 

701 # magnitudes from above that can be passed into 

702 # our catalog iterators so that the catalogs will 

703 # not spend time computing things that have already 

704 # been calculated 

705 local_column_cache = {} 

706 delta_name = self.delta_name_mapper(bp) 

707 total_name = self.total_name_mapper(bp) 

708 

709 if delta_name in cat._compound_column_names: 

710 compound_key = None 

711 for key in cat._column_cache: 

712 if isinstance(cat._column_cache[key], OrderedDict): 

713 if delta_name in cat._column_cache[key]: 

714 compound_key = key 

715 break 

716 local_column_cache[compound_key] = OrderedDict([(delta_name, d_mags[bp][time_dex])]) 

717 else: 

718 local_column_cache[delta_name] = d_mags[bp][time_dex] 

719 

720 if total_name in cat._compound_column_names: 

721 compound_key = None 

722 for key in cat._column_cache: 

723 if isinstance(cat._column_cache[key], OrderedDict): 

724 if total_name in cat._column_cache[key]: 

725 compound_key = key 

726 break 

727 local_column_cache[compound_key] = OrderedDict([(total_name, quiescent_mags[bp]+d_mags[bp][time_dex])]) 

728 else: 

729 local_column_cache[total_name] = quiescent_mags[bp] + d_mags[bp][time_dex] 

730 

731 if ix in local_gamma_cache: 

732 cat._gamma_cache = local_gamma_cache[ix] 

733 else: 

734 cat._gamma_cache = {} 

735 

736 for star_obj in \ 

737 cat.iter_catalog(query_cache=[chunk], column_cache=local_column_cache): 

738 

739 if not np.isnan(star_obj[3]) and not np.isinf(star_obj[3]): 

740 

741 if star_obj[0] not in self.truth_dict: 

742 self.truth_dict[star_obj[0]] = star_obj[5] 

743 

744 if star_obj[0] not in self.mjd_dict: 

745 self.mjd_dict[star_obj[0]] = {} 

746 self.bright_dict[star_obj[0]] = {} 

747 self.sig_dict[star_obj[0]] = {} 

748 

749 bp = cat.obs_metadata.bandpass 

750 if bp not in self.mjd_dict[star_obj[0]]: 

751 self.mjd_dict[star_obj[0]][bp] = [] 

752 self.bright_dict[star_obj[0]][bp] = [] 

753 self.sig_dict[star_obj[0]][bp] = [] 

754 

755 self.mjd_dict[star_obj[0]][bp].append(cat.obs_metadata.mjd.TAI) 

756 self.bright_dict[star_obj[0]][bp].append(star_obj[3]) 

757 self.sig_dict[star_obj[0]][bp].append(star_obj[4]) 

758 

759 if ix not in local_gamma_cache: 

760 local_gamma_cache[ix] = cat._gamma_cache 

761 

762 _sed_cache = {} # before moving on to the next chunk of objects 

763 

764 

765class StellarLightCurveGenerator(LightCurveGenerator): 

766 """ 

767 This class will find all of the OpSim pointings in a particular region 

768 of the sky in a particular filter and then return light curves for all 

769 of the stellar objects observed in that region of sky. 

770 

771 Input parameters: 

772 ----------------- 

773 catalogdb is a CatalogDBObject instantiation connecting to the database 

774 of objects to be observed. 

775 

776 opsimdb is the path to the OpSim database of observation. 

777 

778 opsimdriver (optional; default 'sqlite') indicates the database driver to 

779 be used when connecting to opsimdb. 

780 """ 

781 

782 def __init__(self, *args, **kwargs): 

783 self._lightCurveCatalogClass = _stellarLightCurveCatalog 

784 self._constraint = 'varParamStr IS NOT NULL' 

785 super(StellarLightCurveGenerator, self).__init__(*args, **kwargs) 

786 

787 

788class FastStellarLightCurveGenerator(FastLightCurveGenerator, 

789 StellarLightCurveGenerator): 

790 

791 def delta_name_mapper(self, bp): 

792 return 'delta_lsst_%s' % bp 

793 

794 def total_name_mapper(self, bp): 

795 return 'lsst_%s' % bp 

796 

797 

798class AgnLightCurveGenerator(LightCurveGenerator): 

799 """ 

800 This class will find all of the OpSim pointings in a particular region 

801 of the sky in a particular filter and then return light curves for all 

802 of AGN observed in that region of sky. 

803 

804 Input parameters: 

805 ----------------- 

806 catalogdb is a CatalogDBObject instantiation connecting to the database 

807 of AGN to be observed. 

808 

809 opsimdb is the path to the OpSim database of observation. 

810 

811 opsimdriver (optional; default 'sqlite') indicates the database driver to 

812 be used when connecting to opsimdb. 

813 """ 

814 

815 def __init__(self, *args, **kwargs): 

816 self._lightCurveCatalogClass = _agnLightCurveCatalog 

817 self._constraint = 'varParamStr IS NOT NULL' 

818 super(AgnLightCurveGenerator, self).__init__(*args, **kwargs) 

819 

820 

821class FastAgnLightCurveGenerator(FastLightCurveGenerator, 

822 AgnLightCurveGenerator): 

823 

824 def delta_name_mapper(self, bp): 

825 return 'delta_%sAgn' % bp 

826 

827 def total_name_mapper(self, bp): 

828 return '%sAgn' % bp