Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import numpy as np 

2from lsst.sims.maf.metrics import BaseMetric 

3from lsst.sims.maf.utils.snNSNUtils import Load_Reference, Telescope, LCfast 

4from lsst.sims.maf.utils.snNSNUtils import SN_Rate, CovColor 

5import pandas as pd 

6import numpy.lib.recfunctions as rf 

7import time 

8from scipy.interpolate import interp1d 

9import numpy.lib.recfunctions as nlr 

10 

11 

12class SNNSNMetric(BaseMetric): 

13 """ 

14 Estimate (nSN,zlim) of type Ia supernovae. 

15 

16 Parameters 

17 --------------- 

18 metricName : str, opt 

19 metric name (default : SNSNRMetric) 

20 mjdCol : str, opt 

21 mjd column name (default : observationStartMJD) 

22 RACol : str,opt 

23 Right Ascension column name (default : fieldRA) 

24 DecCol : str,opt 

25 Declinaison column name (default : fieldDec) 

26 filterCol : str,opt 

27 filter column name (default: filter) 

28 m5Col : str, opt 

29 five-sigma depth column name (default : fiveSigmaDepth) 

30 exptimeCol : str,opt 

31 exposure time column name (default : visitExposureTime) 

32 nightCol : str,opt 

33 night column name (default : night) 

34 obsidCol : str,opt 

35 observation id column name (default : observationId) 

36 nexpCol : str,opt 

37 number of exposure column name (default : numExposures) 

38 vistimeCol : str,opt 

39 visit time column name (default : visitTime) 

40 season : list,opt 

41 list of seasons to process (float)(default: -1 = all seasons) 

42 zmin : float,opt 

43 min redshift for the study (default: 0.0) 

44 zmax : float,opt 

45 max redshift for the study (default: 1.2) 

46 pixArea: float, opt 

47 pixel area (default: 9.6) 

48 verbose: bool,opt 

49 verbose mode (default: False) 

50 ploteffi: bool,opt 

51 to plot observing efficiencies vs z (default: False) 

52 n_bef: int, opt 

53 number of LC points LC before T0 (default:5) 

54 n_aft: int, opt 

55 number of LC points after T0 (default: 10) 

56 snr_min: float, opt 

57 minimal SNR of LC points (default: 5.0) 

58 n_phase_min: int, opt 

59 number of LC points with phase<= -5(default:1) 

60 n_phase_max: int, opt 

61 number of LC points with phase>= 20 (default: 1) 

62 """ 

63 

64 def __init__(self, metricName='SNNSNMetric', 

65 mjdCol='observationStartMJD', RACol='fieldRA', DecCol='fieldDec', 

66 filterCol='filter', m5Col='fiveSigmaDepth', exptimeCol='visitExposureTime', 

67 nightCol='night', obsidCol='observationId', nexpCol='numExposures', 

68 vistimeCol='visitTime', season=[-1], zmin=0.0, zmax=1.2, 

69 pixArea=9.6, verbose=False, ploteffi=False, 

70 n_bef=4, n_aft=10, snr_min=5., n_phase_min=1, 

71 n_phase_max=1, templateDir=None, zlim_coeff=-1., **kwargs): 

72 

73 self.mjdCol = mjdCol 

74 self.m5Col = m5Col 

75 self.filterCol = filterCol 

76 self.RACol = RACol 

77 self.DecCol = DecCol 

78 self.exptimeCol = exptimeCol 

79 self.seasonCol = 'season' 

80 self.nightCol = nightCol 

81 self.obsidCol = obsidCol 

82 self.nexpCol = nexpCol 

83 self.vistimeCol = vistimeCol 

84 self.pixArea = pixArea 

85 self.zlim_coeff = zlim_coeff 

86 

87 cols = [self.nightCol, self.m5Col, self.filterCol, self.mjdCol, self.obsidCol, 

88 self.nexpCol, self.vistimeCol, self.exptimeCol, self.seasonCol] 

89 

90 super(SNNSNMetric, self).__init__( 

91 col=cols, metricDtype='object', metricName=metricName, **kwargs) 

92 

93 self.season = season 

94 # LC selection parameters 

95 self.n_bef = n_bef # nb points before peak 

96 self.n_aft = n_aft # nb points after peak 

97 self.snr_min = snr_min # SNR cut for points before/after peak 

98 self.n_phase_min = n_phase_min # nb of point with phase <=-5 

99 self.n_phase_max = n_phase_max # nb of points with phase >=20 

100 

101 # loading reference LC files 

102 lc_reference = Load_Reference(templateDir=templateDir).ref 

103 

104 self.lcFast = {} 

105 telescope = Telescope(airmass=1.2) 

106 for key, vals in lc_reference.items(): 

107 self.lcFast[key] = LCfast(vals, key[0], key[1], telescope, 

108 self.mjdCol, self.RACol, self.DecCol, 

109 self.filterCol, self.exptimeCol, 

110 self.m5Col, self.seasonCol, self.nexpCol, 

111 self.snr_min) 

112 

113 # loading parameters 

114 self.zmin = zmin # zmin for the study 

115 self.zmax = zmax # zmax for the study 

116 self.zStep = 0.05 # zstep 

117 self.daymaxStep = 3. # daymax step 

118 self.min_rf_phase = -20. # min ref phase for LC points selection 

119 self.max_rf_phase = 40. # max ref phase for LC points selection 

120 

121 self.min_rf_phase_qual = -15. # min ref phase for bounds effects 

122 self.max_rf_phase_qual = 25. # max ref phase for bounds effects 

123 

124 # snrate 

125 self.rateSN = SN_Rate( 

126 min_rf_phase=self.min_rf_phase_qual, max_rf_phase=self.max_rf_phase_qual) 

127 

128 # verbose mode - useful for debug and code performance estimation 

129 self.verbose = False 

130 self.ploteffi = False 

131 

132 # supernovae parameters 

133 self.params = ['x0', 'x1', 'daymax', 'color'] 

134 

135 # r = [(-1.0, -1.0)] 

136 self.bad = np.rec.fromrecords([(-1.0, -1.0)], names=['nSN', 'zlim']) 

137 # self.bad = {'nSN': -1.0, 'zlim': -1.0} 

138 

139 def run(self, dataSlice, slicePoint=None): 

140 """ 

141 run method of the metric 

142 

143 Parameters 

144 --------------- 

145 dataSlice: array 

146 data to process 

147 

148 """ 

149 idarray = None 

150 healpixID = -1 

151 if slicePoint is not None: 

152 if 'nside' in slicePoint.keys(): 

153 import healpy as hp 

154 self.pixArea = hp.nside2pixarea( 

155 slicePoint['nside'], degrees=True) 

156 r = [] 

157 names = [] 

158 

159 healpixID = hp.ang2pix( 

160 slicePoint['nside'], np.rad2deg(slicePoint['ra']), np.rad2deg(slicePoint['dec']), nest=True, lonlat=True) 

161 for kk, vv in slicePoint.items(): 

162 r.append(vv) 

163 names.append(kk) 

164 idarray = np.rec.fromrecords([r], names=names) 

165 else: 

166 idarray = np.rec.fromrecords([0., 0.], names=['RA', 'Dec']) 

167 

168 # Two things to do: concatenate data (per band, night) and estimate seasons 

169 dataSlice = rf.drop_fields(dataSlice, ['season']) 

170 

171 dataSlice = self.coadd(pd.DataFrame(dataSlice)) 

172 

173 dataSlice = self.getseason(dataSlice) 

174 

175 # get the seasons 

176 seasons = self.season 

177 

178 # if seasons = -1: process the seasons seen in data 

179 if self.season == [-1]: 

180 seasons = np.unique(dataSlice[self.seasonCol]) 

181 

182 # get redshift range for processing 

183 zRange = list(np.arange(self.zmin, self.zmax, self.zStep)) 

184 if zRange[0] < 1.e-6: 

185 zRange[0] = 0.01 

186 

187 self.zRange = np.unique(zRange) 

188 # season infos 

189 dfa = pd.DataFrame(np.copy(dataSlice)) 

190 season_info = dfa.groupby(['season']).apply( 

191 lambda x: self.seasonInfo(x)).reset_index() 

192 

193 #print('season info', season_info) 

194 # select seasons of at least 30 days 

195 idx = season_info['season_length'] >= 60. 

196 season_info = season_info[idx] 

197 

198 # check wether requested seasons can be processed 

199 test_season = season_info[season_info['season'].isin(seasons)] 

200 # if len(test_season) == 0: 

201 

202 if test_season.empty: 

203 return nlr.merge_arrays([idarray, self.bad], flatten=True) 

204 else: 

205 seasons = test_season['season'] 

206 # print('test_seas', seasons) 

207 # print('hh', season_info) 

208 

209 # get season length depending on the redshift 

210 dur_z = season_info.groupby(['season']).apply( 

211 lambda x: self.duration_z(x)).reset_index() 

212 

213 # remove dur_z with negative season lengths 

214 idx = dur_z['season_length'] >= 10. 

215 dur_z = dur_z[idx] 

216 

217 # generating simulation parameters 

218 gen_par = dur_z.groupby(['z', 'season']).apply( 

219 lambda x: self.calcDaymax(x)).reset_index() 

220 

221 if gen_par.empty: 

222 return nlr.merge_arrays([idarray, self.bad], flatten=True) 

223 resdf = pd.DataFrame() 

224 

225 for seas in seasons: 

226 vara_df = self.run_season( 

227 dataSlice, [seas], gen_par, dur_z) 

228 #print('res', seas, vara_df) 

229 if vara_df is not None: 

230 resdf = pd.concat((resdf, vara_df)) 

231 

232 # final result: median zlim for a faint sn 

233 # and nsn_med for z<zlim 

234 

235 if resdf.empty: 

236 return nlr.merge_arrays([idarray, self.bad], flatten=True) 

237 

238 resdf = resdf.round({'zlim': 3, 'nsn_med': 3}) 

239 x1_ref = -2.0 

240 color_ref = 0.2 

241 

242 idx = np.abs(resdf['x1']-x1_ref) < 1.e-5 

243 idx &= np.abs(resdf['color']-color_ref) < 1.e-5 

244 idx &= resdf['zlim'] > 0 

245 

246 if not resdf[idx].empty: 

247 zlim = resdf[idx]['zlim'].median() 

248 nSN = resdf[idx]['nsn_med'].sum() 

249 

250 resd = np.rec.fromrecords([(nSN, zlim, healpixID)], names=[ 

251 'nSN', 'zlim', 'healpixID']) 

252 

253 res = nlr.merge_arrays([idarray, resd], flatten=True) 

254 

255 else: 

256 

257 res = nlr.merge_arrays([idarray, self.bad], flatten=True) 

258 

259 return res 

260 

261 def reducenSN(self, metricVal): 

262 

263 # At each slicepoint, return the sum nSN value. 

264 

265 return np.sum(metricVal['nSN']) 

266 

267 def reducezlim(self, metricVal): 

268 

269 # At each slicepoint, return the median zlim 

270 

271 return np.median(metricVal['zlim']) 

272 

273 def coadd(self, data): 

274 """ 

275 Method to coadd data per band and per night 

276 

277 Parameters 

278 --------------- 

279 data: pandas df of observations 

280 

281 Returns 

282 ----------- 

283 coadded data (pandas df) 

284 

285 """ 

286 

287 keygroup = [self.filterCol, self.nightCol] 

288 

289 data.sort_values(by=keygroup, ascending=[ 

290 True, True], inplace=True) 

291 

292 coadd_df = data.groupby(keygroup).agg({self.nexpCol: ['sum'], 

293 self.vistimeCol: ['sum'], 

294 self.exptimeCol: ['sum'], 

295 self.mjdCol: ['mean'], 

296 self.RACol: ['mean'], 

297 self.DecCol: ['mean'], 

298 self.m5Col: ['mean']}).reset_index() 

299 

300 coadd_df.columns = [self.filterCol, self.nightCol, self.nexpCol, 

301 self.vistimeCol, self.exptimeCol, self.mjdCol, 

302 self.RACol, self.DecCol, self.m5Col] 

303 

304 coadd_df.loc[:, self.m5Col] += 1.25 * \ 

305 np.log10(coadd_df[self.vistimeCol]/30.) 

306 

307 coadd_df.sort_values(by=[self.filterCol, self.nightCol], ascending=[ 

308 True, True], inplace=True) 

309 

310 return coadd_df.to_records(index=False) 

311 

312 def getseason(self, obs, season_gap=80., mjdCol='observationStartMJD'): 

313 """ 

314 Method to estimate seasons 

315 Parameters 

316 -------------- 

317 obs: numpy array 

318 array of observations 

319 season_gap: float, opt 

320 minimal gap required to define a season (default: 80 days) 

321 mjdCol: str, opt 

322 col name for MJD infos (default: observationStartMJD) 

323 

324 Returns 

325 ---------- 

326 original numpy array with seasonnumber appended 

327 """ 

328 

329 # check wether season has already been estimated 

330 

331 if 'season' in obs.dtype.names: 

332 return obs 

333 

334 obs.sort(order=mjdCol) 

335 

336 seasoncalc = np.ones(obs.size, dtype=int) 

337 

338 if len(obs) > 1: 

339 diff = np.diff(obs[mjdCol]) 

340 flag = np.where(diff > season_gap)[0] 

341 

342 if len(flag) > 0: 

343 for i, indx in enumerate(flag): 

344 seasoncalc[indx+1:] = i+2 

345 

346 obs = rf.append_fields(obs, 'season', seasoncalc) 

347 

348 return obs 

349 

350 def seasonInfo(self, grp): 

351 """ 

352 Method to estimate seasonal info (cadence, season length, ...) 

353 Parameters 

354 -------------- 

355 grp: pandas df group 

356 Returns 

357 --------- 

358 pandas df with the cfollowing cols: 

359 """ 

360 df = pd.DataFrame([len(grp)], columns=['Nvisits']) 

361 df['MJD_min'] = grp[self.mjdCol].min() 

362 df['MJD_max'] = grp[self.mjdCol].max() 

363 df['season_length'] = df['MJD_max']-df['MJD_min'] 

364 df['cadence'] = 0. 

365 

366 if len(grp) > 5: 

367 to = grp.groupby(['night'])[self.mjdCol].median().sort_values() 

368 df['cadence'] = np.mean(to.diff()) 

369 

370 return df 

371 

372 def duration_z(self, grp): 

373 """ 

374 Method to estimate the season length vs redshift 

375 This is necessary to take into account boundary effects 

376 when estimating the number of SN that can be detected 

377 daymin, daymax = min and max MJD of a season 

378 T0_min(z) = daymin-(1+z)*min_rf_phase_qual 

379 T0_max(z) = daymax-(1+z)*max_rf_phase_qual 

380 season_length(z) = T0_max(z)-T0_min(z) 

381 Parameters 

382 -------------- 

383 grp: pandas df group 

384 data to process: season infos 

385 Returns 

386 ---------- 

387 pandas df with season_length, z, T0_min and T0_max cols 

388 """ 

389 

390 daymin = grp['MJD_min'].values 

391 daymax = grp['MJD_max'].values 

392 dur_z = pd.DataFrame(self.zRange, columns=['z']) 

393 dur_z['T0_min'] = daymin-(1.+dur_z['z'])*self.min_rf_phase_qual 

394 dur_z['T0_max'] = daymax-(1.+dur_z['z'])*self.max_rf_phase_qual 

395 dur_z['season_length'] = dur_z['T0_max']-dur_z['T0_min'] 

396 

397 return dur_z 

398 

399 def calcDaymax(self, grp): 

400 """ 

401 Method to estimate T0 (daymax) values for simulation. 

402 Parameters 

403 -------------- 

404 grp: group (pandas df sense) 

405 group of data to process with the following cols: 

406 T0_min: T0 min value (per season) 

407 T0_max: T0 max value (per season) 

408 Returns 

409 ---------- 

410 pandas df with daymax, min_rf_phase, max_rf_phase values 

411 """ 

412 

413 T0_max = grp['T0_max'].values 

414 T0_min = grp['T0_min'].values 

415 num = (T0_max-T0_min)/self.daymaxStep 

416 if T0_max-T0_min > 10: 

417 df = pd.DataFrame(np.linspace( 

418 T0_min, T0_max, int(num)), columns=['daymax']) 

419 else: 

420 df = pd.DataFrame([-1], columns=['daymax']) 

421 

422 df['min_rf_phase'] = self.min_rf_phase_qual 

423 df['max_rf_phase'] = self.max_rf_phase_qual 

424 

425 return df 

426 

427 def run_season(self, dataSlice, season, gen_par, dura_z): 

428 """ 

429 Method to run on seasons 

430 Parameters 

431 -------------- 

432 dataSlice: numpy array, opt 

433 data to process (scheduler simulations) 

434 seasons: list(int) 

435 list of seasons to process 

436 Returns 

437 --------- 

438 effi_seasondf: pandas df 

439 efficiency curves 

440 zlimsdf: pandas df 

441 redshift limits and number of supernovae 

442 """ 

443 

444 time_ref = time.time() 

445 

446 if self.verbose: 

447 print('#### Processing season', season) 

448 

449 groupnames = ['season', 'x1', 'color'] 

450 

451 gen_p = gen_par[gen_par['season'].isin(season)] 

452 

453 if gen_p.empty: 

454 if self.verbose: 

455 print('No generator parameter found') 

456 return None 

457 dur_z = dura_z[dura_z['season'].isin(season)] 

458 obs = pd.DataFrame(np.copy(dataSlice)) 

459 obs = obs[obs['season'].isin(season)] 

460 

461 obs = obs.sort_values(by=['night']) 

462 #print('data here', obs.columns) 

463 #print(obs[['night', 'filter', 'observationStartMJD', 'fieldRA', 'fieldDec']]) 

464 """ 

465 import matplotlib.pyplot as plt 

466 plt.plot(dataSlice['fieldRA'], dataSlice['fieldDec'], 'ko') 

467 print('data', len(dataSlice)) 

468 plt.show() 

469 """ 

470 # simulate supernovae and lc 

471 if self.verbose: 

472 print("SN generation") 

473 print(season, obs) 

474 sn = self.genSN(obs.to_records( 

475 index=False), gen_p.to_records(index=False)) 

476 

477 if self.verbose: 

478 idx = np.abs(sn['x1']+2) < 1.e-5 

479 idx &= np.abs(sn['z']-0.2) < 1.e-5 

480 sel = sn[idx] 

481 sel = sel.sort_values(by=['z', 'daymax']) 

482 

483 print('sn and lc', len(sn), sel.columns, 

484 sel[['x1', 'color', 'z', 'daymax', 'Cov_colorcolor', 'n_bef', 'n_aft']]) 

485 

486 # from these supernovae: estimate observation efficiency vs z 

487 effi_seasondf = self.effidf(sn) 

488 

489 # zlims can only be estimated if efficiencies are ok 

490 idx = effi_seasondf['z'] <= 0.2 

491 x1ref = -2.0 

492 colorref = 0.2 

493 idx &= np.abs(effi_seasondf['x1']-x1ref) < 1.e-5 

494 idx &= np.abs(effi_seasondf['color']-colorref) < 1.e-5 

495 sel = effi_seasondf[idx] 

496 

497 if np.mean(sel['effi']) > 0.02: 

498 # estimate zlims 

499 zlimsdf = self.zlims(effi_seasondf, dur_z, groupnames) 

500 

501 # estimate number of medium supernovae 

502 zlimsdf['nsn_med'], zlimsdf['var_nsn_med'] = zlimsdf.apply(lambda x: self.nsn_typedf( 

503 x, 0.0, 0.0, effi_seasondf, dur_z), axis=1, result_type='expand').T.values 

504 else: 

505 return None 

506 

507 if self.verbose: 

508 print('#### SEASON processed', time.time()-time_ref, 

509 season) 

510 

511 return zlimsdf 

512 

513 def genSN(self, obs, gen_par): 

514 """ 

515 Method to simulate LC and supernovae 

516 Parameters 

517 --------------- 

518 obs: numpy array 

519 array of observations(from scheduler) 

520 gen_par: numpy array 

521 array of parameters for simulation 

522 """ 

523 

524 time_ref = time.time() 

525 # LC estimation 

526 

527 sn_tot = pd.DataFrame() 

528 lc_tot = pd.DataFrame() 

529 for key, vals in self.lcFast.items(): 

530 time_refs = time.time() 

531 gen_par_cp = np.copy(gen_par) 

532 if key == (-2.0, 0.2): 

533 idx = gen_par_cp['z'] < 0.9 

534 gen_par_cp = gen_par_cp[idx] 

535 lc = vals(obs, gen_par_cp, bands='grizy') 

536 if self.verbose: 

537 print('End of simulation', key, time.time()-time_refs) 

538 

539 if self.verbose: 

540 print('End of simulation after concat', 

541 key, time.time()-time_refs) 

542 

543 # estimate SN 

544 

545 sn = pd.DataFrame() 

546 if len(lc) > 0: 

547 sn = self.process(pd.DataFrame(lc)) 

548 

549 if self.verbose: 

550 print('End of supernova', time.time()-time_refs) 

551 

552 if not sn.empty: 

553 sn_tot = pd.concat([sn_tot, pd.DataFrame(sn)], sort=False) 

554 

555 if self.verbose: 

556 print('End of supernova - all', time.time()-time_ref) 

557 

558 return sn_tot 

559 

560 def process(self, tab): 

561 """ 

562 Method to process LC: sigma_color estimation and LC selection 

563 Parameters 

564 -------------- 

565 tab: pandas df of LC points with the following cols: 

566 flux: flux 

567 fluxerr: flux error 

568 phase: phase 

569 snr_m5: Signal-to-Noise Ratio 

570 time: time(MJD) 

571 mag: magnitude 

572 m5: five-sigma depth 

573 magerr: magnitude error 

574 exposuretime: exposure time 

575 band: filter 

576 zp: zero-point 

577 season: season number 

578 healpixID: pixel ID 

579 pixRA: pixel RA 

580 pixDec: pixel Dec 

581 z: redshift 

582 daymax: T0 

583 flux_e_sec: flux(in photoelec/sec) 

584 flux_5: 5-sigma flux(in photoelec/sec) 

585 F_x0x0, ...F_colorcolor: Fisher matrix elements 

586 x1: x1 SN 

587 color: color SN 

588 n_aft: number of LC points before daymax 

589 n_bef: number of LC points after daymax 

590 n_phmin: number of LC points with a phase < -5 

591 n_phmax: number of LC points with a phase > 20 

592 Returns 

593 ---------- 

594 """ 

595 # now groupby 

596 tab = tab.round({'daymax': 3, 

597 'z': 3, 'x1': 2, 'color': 2}) 

598 groups = tab.groupby( 

599 ['daymax', 'season', 'z', 'x1', 'color']) 

600 

601 tosum = [] 

602 for ia, vala in enumerate(self.params): 

603 for jb, valb in enumerate(self.params): 

604 if jb >= ia: 

605 tosum.append('F_'+vala+valb) 

606 tosum += ['n_aft', 'n_bef', 'n_phmin', 'n_phmax'] 

607 # apply the sum on the group 

608 sums = groups[tosum].sum().reset_index() 

609 

610 # select LC according to the number of points bef/aft peak 

611 idx = sums['n_aft'] >= self.n_aft 

612 idx &= sums['n_bef'] >= self.n_bef 

613 idx &= sums['n_phmin'] >= self.n_phase_min 

614 idx &= sums['n_phmax'] >= self.n_phase_max 

615 

616 if self.verbose: 

617 print('selection parameters', self.n_bef, 

618 self.n_aft, self.n_phase_min, self.n_phase_max) 

619 finalsn = pd.DataFrame() 

620 goodsn = pd.DataFrame(sums.loc[idx]) 

621 

622 # estimate the color for SN that passed the selection cuts 

623 if len(goodsn) > 0: 

624 goodsn.loc[:, 'Cov_colorcolor'] = CovColor(goodsn).Cov_colorcolor 

625 finalsn = pd.concat([finalsn, goodsn], sort=False) 

626 

627 badsn = pd.DataFrame(sums.loc[~idx]) 

628 

629 # Supernovae that did not pass the cut have a sigma_color=10 

630 if len(badsn) > 0: 

631 badsn.loc[:, 'Cov_colorcolor'] = 100. 

632 finalsn = pd.concat([finalsn, badsn], sort=False) 

633 

634 return finalsn 

635 

636 def effidf(self, sn_tot, color_cut=0.04): 

637 """ 

638 Method estimating efficiency vs z for a sigma_color cut 

639 Parameters 

640 --------------- 

641 sn_tot: pandas df 

642 data used to estimate efficiencies 

643 color_cut: float, opt 

644 color selection cut(default: 0.04) 

645 Returns 

646 ---------- 

647 effi: pandas df with the following cols: 

648 season: season 

649 pixRA: RA of the pixel 

650 pixDec: Dec of the pixel 

651 healpixID: pixel ID 

652 x1: SN stretch 

653 color: SN color 

654 z: redshift 

655 effi: efficiency 

656 effi_err: efficiency error(binomial) 

657 """ 

658 

659 sndf = pd.DataFrame(sn_tot) 

660 

661 listNames = ['season', 'x1', 'color'] 

662 groups = sndf.groupby(listNames) 

663 

664 # estimating efficiencies 

665 effi = groups[['Cov_colorcolor', 'z']].apply( 

666 lambda x: self.effiObsdf(x, color_cut)).reset_index(level=list(range(len(listNames)))) 

667 

668 # this is to plot efficiencies and also sigma_color vs z 

669 if self.ploteffi: 

670 import matplotlib.pylab as plt 

671 fig, ax = plt.subplots() 

672 figb, axb = plt.subplots() 

673 

674 self.plot(ax, effi, 'effi', 'effi_err', 

675 'Observing Efficiencies', ls='-') 

676 sndf['sigma_color'] = np.sqrt(sndf['Cov_colorcolor']) 

677 self.plot(axb, sndf, 'sigma_color', None, '$\sigma_{color}$') 

678 # get efficiencies vs z 

679 

680 plt.show() 

681 

682 return effi 

683 

684 def plot(self, ax, effi, vary, erry=None, legy='', ls='None'): 

685 """ 

686 Simple method to plot vs z 

687 Parameters 

688 -------------- 

689 ax: 

690 axis where to plot 

691 effi: pandas df 

692 data to plot 

693 vary: str 

694 variable(column of effi) to plot 

695 erry: str, opt 

696 error on y-axis(default: None) 

697 legy: str, opt 

698 y-axis legend(default: '') 

699 """ 

700 grb = effi.groupby(['x1', 'color']) 

701 yerr = None 

702 for key, grp in grb: 

703 x1 = grp['x1'].unique()[0] 

704 color = grp['color'].unique()[0] 

705 if erry is not None: 

706 yerr = grp[erry] 

707 ax.errorbar(grp['z'], grp[vary], yerr=yerr, 

708 marker='o', label='(x1,color)=({},{})'.format(x1, color), lineStyle=ls) 

709 

710 ftsize = 15 

711 ax.set_xlabel('z', fontsize=ftsize) 

712 ax.set_ylabel(legy, fontsize=ftsize) 

713 ax.xaxis.set_tick_params(labelsize=ftsize) 

714 ax.yaxis.set_tick_params(labelsize=ftsize) 

715 ax.legend(fontsize=ftsize) 

716 

717 def effiObsdf(self, data, color_cut=0.04): 

718 """ 

719 Method to estimate observing efficiencies for supernovae 

720 Parameters 

721 -------------- 

722 data: pandas df - grp 

723 data to process 

724 Returns 

725 ---------- 

726 pandas df with the following cols: 

727 - cols used to make the group 

728 - effi, effi_err: observing efficiency and associated error 

729 """ 

730 

731 # reference df to estimate efficiencies 

732 df = data.loc[lambda dfa: np.sqrt(dfa['Cov_colorcolor']) < 100000., :] 

733 

734 # selection on sigma_c<= 0.04 

735 df_sel = df.loc[lambda dfa: np.sqrt( 

736 dfa['Cov_colorcolor']) <= color_cut, :] 

737 

738 # make groups (with z) 

739 group = df.groupby('z') 

740 group_sel = df_sel.groupby('z') 

741 

742 # Take the ratio to get efficiencies 

743 rb = (group_sel.size()/group.size()) 

744 err = np.sqrt(rb*(1.-rb)/group.size()) 

745 var = rb*(1.-rb)*group.size() 

746 

747 rb = rb.array 

748 err = err.array 

749 var = var.array 

750 

751 rb[np.isnan(rb)] = 0. 

752 err[np.isnan(err)] = 0. 

753 var[np.isnan(var)] = 0. 

754 

755 return pd.DataFrame({group.keys: list(group.groups.keys()), 

756 'effi': rb, 

757 'effi_err': err, 

758 'effi_var': var}) 

759 

760 def zlims(self, effi_seasondf, dur_z, groupnames): 

761 """ 

762 Method to estimate redshift limits 

763 Parameters 

764 -------------- 

765 effi_seasondf: pandas df 

766 season: season 

767 pixRA: RA of the pixel 

768 pixDec: Dec of the pixel 

769 healpixID: pixel ID 

770 x1: SN stretch 

771 color: SN color 

772 z: redshift 

773 effi: efficiency 

774 effi_err: efficiency error (binomial) 

775 dur_z: pandas df with the following cols: 

776 season: season 

777 z: redshift 

778 T0_min: min daymax 

779 T0_max: max daymax 

780 season_length: season length 

781 groupnames: list(str) 

782 list of columns to use to define the groups 

783 Returns 

784 ---------- 

785 pandas df with the following cols: pixRA: RA of the pixel 

786 pixDec: Dec of the pixel 

787 healpixID: pixel ID 

788 season: season number 

789 x1: SN stretch 

790 color: SN color 

791 zlim: redshift limit 

792 """ 

793 

794 res = effi_seasondf.groupby(groupnames).apply( 

795 lambda x: self.zlimdf(x, dur_z)).reset_index(level=list(range(len(groupnames)))) 

796 

797 return res 

798 

799 def zlimdf(self, grp, duration_z): 

800 """ 

801 Method to estimate redshift limits 

802 Parameters 

803 -------------- 

804 grp: pandas df group 

805 efficiencies to estimate redshift limits; 

806 columns: 

807 season: season 

808 pixRA: RA of the pixel 

809 pixDec: Dec of the pixel 

810 healpixID: pixel ID 

811 x1: SN stretch 

812 color: SN color 

813 z: redshift 

814 effi: efficiency 

815 effi_err: efficiency error (binomial) 

816 duration_z: pandas df with the following cols: 

817 season: season 

818 z: redshift 

819 T0_min: min daymax 

820 T0_max: max daymax 

821 season_length: season length 

822 Returns 

823 ---------- 

824 pandas df with the following cols: 

825 zlimit: redshift limit 

826 """ 

827 

828 zlimit = 0.0 

829 

830 # z range for the study 

831 zplot = np.arange(self.zmin, self.zmax, 0.01) 

832 

833 # print(grp['z'], grp['effi']) 

834 

835 if len(grp['z']) <= 3: 

836 return pd.DataFrame({'zlim': [zlimit]}) 

837 # 'status': [int(status)]}) 

838 # interpolate efficiencies vs z 

839 effiInterp = interp1d( 

840 grp['z'], grp['effi'], kind='linear', bounds_error=False, fill_value=0.) 

841 

842 if self.zlim_coeff < 0.: 

843 # in that case zlim is estimated from efficiencies 

844 # first step: identify redshift domain with efficiency decrease 

845 zlimit = self.zlim_from_effi(effiInterp, zplot) 

846 #status = self.status['ok'] 

847 

848 else: 

849 zlimit = self.zlim_from_cumul( 

850 grp, duration_z, effiInterp, zplot) 

851 

852 return pd.DataFrame({'zlim': [zlimit]}) 

853 # 'status': [int(status)]}) 

854 

855 def zlim_from_cumul(self, grp, duration_z, effiInterp, zplot, rate='cte'): 

856 """ 

857 Method to estimate the redshift limit from the cumulative 

858 The redshift limit is estimated to be the z value corresponding to: 

859 frac(NSN(z<zlimit))=zlimi_coeff 

860 

861 Parameters 

862 --------------- 

863 grp: pandas group 

864 data to process 

865 duration_z: array 

866 duration as a function of the redshift 

867 effiInterp: interp1d 

868 interpolator for efficiencies 

869 zplot: interp1d 

870 interpolator for redshift values 

871 rate: str, opt 

872 rate to estimate the number of SN to estimate zlimit 

873 rate = cte: rate independent of z 

874 rate = SN_rate: rate from SN_Rate class 

875 

876 Returns 

877 ---------- 

878 zlimit: float 

879 the redshift limit 

880 """ 

881 

882 if rate == 'SN_rate': 

883 # get rate 

884 season = np.median(grp['season']) 

885 idx = duration_z['season'] == season 

886 seas_duration_z = duration_z[idx] 

887 

888 durinterp_z = interp1d( 

889 seas_duration_z['z'], seas_duration_z['season_length'], bounds_error=False, fill_value=0.) 

890 

891 # estimate the rates and nsn vs z 

892 zz, rate, err_rate, nsn, err_nsn = self.rateSN(zmin=self.zmin, 

893 zmax=self.zmax, 

894 duration_z=durinterp_z, 

895 survey_area=self.pixArea) 

896 

897 # rate interpolation 

898 rateInterp = interp1d(zz, nsn, kind='linear', 

899 bounds_error=False, fill_value=0) 

900 else: 

901 # this is for a rate z-independent 

902 nsn = np.ones(len(zplot)) 

903 rateInterp = interp1d(zplot, nsn, kind='linear', 

904 bounds_error=False, fill_value=0) 

905 

906 nsn_cum = np.cumsum(effiInterp(zplot)*rateInterp(zplot)) 

907 

908 if nsn_cum[-1] >= 1.e-5: 

909 nsn_cum_norm = nsn_cum/nsn_cum[-1] # normalize 

910 zlim = interp1d(nsn_cum_norm, zplot) 

911 zlimit = zlim(self.zlim_coeff).item() 

912 

913 if self.ploteffi: 

914 self.plot_NSN_cumul(grp, nsn_cum_norm, zplot) 

915 else: 

916 zlimit = 0. 

917 

918 return zlimit 

919 

920 def plot_NSN_cumul(self, grp, nsn_cum_norm, zplot): 

921 """ 

922 Method to plot the NSN cumulative vs redshift 

923 Parameters 

924 -------------- 

925 grp: pandas group 

926 data to process 

927 """ 

928 

929 import matplotlib.pylab as plt 

930 fig, ax = plt.subplots() 

931 x1 = grp['x1'].unique()[0] 

932 color = grp['color'].unique()[0] 

933 

934 ax.plot(zplot, nsn_cum_norm, 

935 label='(x1,color)=({},{})'.format(x1, color)) 

936 

937 ftsize = 15 

938 ax.set_ylabel('NSN ($z<$)', fontsize=ftsize) 

939 ax.set_xlabel('z', fontsize=ftsize) 

940 ax.xaxis.set_tick_params(labelsize=ftsize) 

941 ax.yaxis.set_tick_params(labelsize=ftsize) 

942 ax.set_xlim((0.0, 0.8)) 

943 ax.set_ylim((0.0, 1.05)) 

944 ax.plot([0., 1.2], [0.95, 0.95], ls='--', color='k') 

945 plt.legend(fontsize=ftsize) 

946 plt.show() 

947 

948 def zlim_from_effi(self, effiInterp, zplot): 

949 """ 

950 Method to estimate the redshift limit from efficiency curves 

951 The redshift limit is defined here as the redshift value beyond 

952 which efficiency decreases up to zero. 

953 Parameters 

954 --------------- 

955 effiInterp: interpolator 

956 use to get efficiencies 

957 zplot: numpy array 

958 redshift values 

959 Returns 

960 ----------- 

961 zlimit: float 

962 the redshift limit 

963 """ 

964 

965 # get efficiencies 

966 effis = effiInterp(zplot) 

967 # select data with efficiency decrease 

968 idx = np.where(np.diff(effis) < -0.005)[0] 

969 

970 # Bail out if there is no data 

971 if np.size(idx) == 0: 

972 return 0 

973 

974 z_effi = np.array(zplot[idx], dtype={ 

975 'names': ['z'], 'formats': [np.float]}) 

976 # from this make some "z-periods" to avoid accidental zdecrease at low z 

977 z_gap = 0.05 

978 seasoncalc = np.ones(z_effi.size, dtype=int) 

979 diffz = np.diff(z_effi['z']) 

980 flag = np.where(diffz > z_gap)[0] 

981 

982 if len(flag) > 0: 

983 for i, indx in enumerate(flag): 

984 seasoncalc[indx+1:] = i+2 

985 z_effi = rf.append_fields(z_effi, 'season', seasoncalc) 

986 

987 # now take the highest season (end of the efficiency curve) 

988 idd = z_effi['season'] == np.max(z_effi['season']) 

989 zlimit = np.min(z_effi[idd]['z']) 

990 

991 return zlimit 

992 

993 def zlimdf_deprecated(self, grp, duration_z): 

994 """ 

995 Method to estimate redshift limits 

996 Parameters 

997 -------------- 

998 grp: pandas df group 

999 efficiencies to estimate redshift limits; 

1000 columns: 

1001 season: season 

1002 pixRA: RA of the pixel 

1003 pixDec: Dec of the pixel 

1004 healpixID: pixel ID 

1005 x1: SN stretch 

1006 color: SN color 

1007 z: redshift 

1008 effi: efficiency 

1009 effi_err: efficiency error (binomial) 

1010 duration_z: pandas df with the following cols: 

1011 season: season 

1012 z: redshift 

1013 T0_min: min daymax 

1014 T0_max: max daymax 

1015 season_length: season length 

1016 Returns 

1017 ---------- 

1018 pandas df with the following cols: 

1019 zlimit: redshift limit 

1020 """ 

1021 

1022 zlimit = 0.0 

1023 

1024 # z range for the study 

1025 zplot = list(np.arange(self.zmin, self.zmax, 0.001)) 

1026 

1027 # print(grp['z'], grp['effi']) 

1028 

1029 if len(grp['z']) <= 3: 

1030 return pd.DataFrame({'zlim': [zlimit]}) 

1031 # 'status': [int(status)]}) 

1032 # interpolate efficiencies vs z 

1033 effiInterp = interp1d( 

1034 grp['z'], grp['effi'], kind='linear', bounds_error=False, fill_value=0.) 

1035 

1036 # get rate 

1037 season = np.median(grp['season']) 

1038 idx = duration_z['season'] == season 

1039 seas_duration_z = duration_z[idx] 

1040 

1041 # print('hhh1', seas_duration_z) 

1042 durinterp_z = interp1d( 

1043 seas_duration_z['z'], seas_duration_z['season_length'], bounds_error=False, fill_value=0.) 

1044 

1045 # estimate the rates and nsn vs z 

1046 zz, rate, err_rate, nsn, err_nsn = self.rateSN(zmin=self.zmin, 

1047 zmax=self.zmax, 

1048 duration_z=durinterp_z, 

1049 survey_area=self.pixArea) 

1050 

1051 # rate interpolation 

1052 rateInterp = interp1d(zz, nsn, kind='linear', 

1053 bounds_error=False, fill_value=0) 

1054 

1055 # estimate the cumulated number of SN vs z 

1056 nsn_cum = np.cumsum(effiInterp(zplot)*rateInterp(zplot)) 

1057 

1058 if nsn_cum[-1] >= 1.e-5: 

1059 nsn_cum_norm = nsn_cum/nsn_cum[-1] # normalize 

1060 zlim = interp1d(nsn_cum_norm, zplot) 

1061 zlimit = zlim(0.95).item() 

1062 # status = self.status['ok'] 

1063 

1064 if self.ploteffi: 

1065 import matplotlib.pylab as plt 

1066 fig, ax = plt.subplots() 

1067 x1 = grp['x1'].unique()[0] 

1068 color = grp['color'].unique()[0] 

1069 

1070 ax.plot(zplot, nsn_cum_norm, 

1071 label='(x1,color)=({},{})'.format(x1, color)) 

1072 

1073 ftsize = 15 

1074 ax.set_ylabel('NSN ($z<$)', fontsize=ftsize) 

1075 ax.set_xlabel('z', fontsize=ftsize) 

1076 ax.xaxis.set_tick_params(labelsize=ftsize) 

1077 ax.yaxis.set_tick_params(labelsize=ftsize) 

1078 ax.set_xlim((0.0, 1.2)) 

1079 ax.set_ylim((0.0, 1.05)) 

1080 ax.plot([0., 1.2], [0.95, 0.95], ls='--', color='k') 

1081 plt.legend(fontsize=ftsize) 

1082 plt.show() 

1083 

1084 return pd.DataFrame({'zlim': [zlimit]}) 

1085 # 'status': [int(status)]}) 

1086 

1087 def nsn_typedf(self, grp, x1, color, effi_tot, duration_z, search=True): 

1088 """ 

1089 Method to estimate the number of supernovae for a given type of SN 

1090 Parameters 

1091 -------------- 

1092 grp: pandas series with the following infos: 

1093 pixRA: pixelRA 

1094 pixDec: pixel Dec 

1095 healpixID: pixel ID 

1096 season: season 

1097 x1: SN stretch 

1098 color: SN color 

1099 zlim: redshift limit 

1100 x1, color: SN params to estimate the number 

1101 effi_tot: pandas df with columns: 

1102 season: season 

1103 pixRA: RA of the pixel 

1104 pixDec: Dec of the pixel 

1105 healpixID: pixel ID 

1106 x1: SN stretch 

1107 color: SN color 

1108 z: redshift 

1109 effi: efficiency 

1110 effi_err: efficiency error (binomial) 

1111 duration_z: pandas df with the following cols: 

1112 season: season 

1113 z: redshift 

1114 T0_min: min daymax 

1115 T0_max: max daymax 

1116 season_length: season length 

1117 Returns 

1118 ---------- 

1119 nsn: float 

1120 number of supernovae 

1121 """ 

1122 

1123 # get rate 

1124 season = np.median(grp['season']) 

1125 idx = duration_z['season'] == season 

1126 seas_duration_z = duration_z[idx] 

1127 

1128 # print('hhh2', seas_duration_z) 

1129 durinterp_z = interp1d( 

1130 seas_duration_z['z'], seas_duration_z['season_length'], bounds_error=False, fill_value=0.) 

1131 

1132 if search: 

1133 effisel = effi_tot.loc[lambda dfa: ( 

1134 dfa['x1'] == x1) & (dfa['color'] == color), :] 

1135 else: 

1136 effisel = effi_tot 

1137 

1138 nsn, var_nsn = self.nsn(effisel, grp['zlim'], durinterp_z) 

1139 

1140 return (nsn, var_nsn) 

1141 

1142 def nsn(self, effi, zlim, duration_z): 

1143 """ 

1144 Method to estimate the number of supernovae 

1145 Parameters 

1146 -------------- 

1147 effi: pandas df grp of efficiencies 

1148 season: season 

1149 pixRA: RA of the pixel 

1150 pixDec: Dec of the pixel 

1151 healpixID: pixel ID 

1152 x1: SN stretch 

1153 color: SN color 

1154 z: redshift 

1155 effi: efficiency 

1156 effi_err: efficiency error (binomial) 

1157 zlim: float 

1158 redshift limit value 

1159 duration_z: pandas df with the following cols: 

1160 season: season 

1161 z: redshift 

1162 T0_min: min daymax 

1163 T0_max: max daymax 

1164 season_length: season length 

1165 Returns 

1166 ---------- 

1167 nsn, var_nsn : float 

1168 number of supernovae (and variance) with z<zlim 

1169 """ 

1170 

1171 if zlim < 1.e-3: 

1172 return (-1.0, -1.0) 

1173 

1174 dz = 0.001 

1175 zplot = list(np.arange(self.zmin, self.zmax, dz)) 

1176 # interpolate efficiencies vs z 

1177 effiInterp = interp1d( 

1178 effi['z'], effi['effi'], kind='linear', bounds_error=False, fill_value=0.) 

1179 # estimate the cumulated number of SN vs z 

1180 zz, rate, err_rate, nsn, err_nsn = self.rateSN(zmin=self.zmin, 

1181 zmax=self.zmax, 

1182 dz=dz, 

1183 duration_z=duration_z, 

1184 survey_area=self.pixArea) 

1185 

1186 nsn_cum = np.cumsum(effiInterp(zplot)*nsn) 

1187 

1188 nsn_interp = interp1d(zplot, nsn_cum) 

1189 

1190 nsn = nsn_interp(zlim).item() 

1191 

1192 return [nsn, 0.0]