Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import numpy as np 

2from lsst.sims.maf.metrics import BaseMetric 

3from lsst.sims.maf.utils.snNSNUtils import Load_Reference, Telescope, LCfast 

4from lsst.sims.maf.utils.snNSNUtils import SN_Rate, CovColor 

5import pandas as pd 

6import numpy.lib.recfunctions as rf 

7import time 

8from scipy.interpolate import interp1d 

9import numpy.lib.recfunctions as nlr 

10 

11__all__ = ['SNNSNMetric'] 

12 

13class SNNSNMetric(BaseMetric): 

14 """ 

15 Estimate (nSN,zlim) of type Ia supernovae. 

16 

17 Parameters 

18 --------------- 

19 metricName : str, opt 

20 metric name (default : SNSNRMetric) 

21 mjdCol : str, opt 

22 mjd column name (default : observationStartMJD) 

23 RACol : str,opt 

24 Right Ascension column name (default : fieldRA) 

25 DecCol : str,opt 

26 Declinaison column name (default : fieldDec) 

27 filterCol : str,opt 

28 filter column name (default: filter) 

29 m5Col : str, opt 

30 five-sigma depth column name (default : fiveSigmaDepth) 

31 exptimeCol : str,opt 

32 exposure time column name (default : visitExposureTime) 

33 nightCol : str,opt 

34 night column name (default : night) 

35 obsidCol : str,opt 

36 observation id column name (default : observationId) 

37 nexpCol : str,opt 

38 number of exposure column name (default : numExposures) 

39 vistimeCol : str,opt 

40 visit time column name (default : visitTime) 

41 season : list,opt 

42 list of seasons to process (float)(default: -1 = all seasons) 

43 zmin : float,opt 

44 min redshift for the study (default: 0.0) 

45 zmax : float,opt 

46 max redshift for the study (default: 1.2) 

47 pixArea: float, opt 

48 pixel area (default: 9.6) 

49 verbose: bool,opt 

50 verbose mode (default: False) 

51 ploteffi: bool,opt 

52 to plot observing efficiencies vs z (default: False) 

53 n_bef: int, opt 

54 number of LC points LC before T0 (default:5) 

55 n_aft: int, opt 

56 number of LC points after T0 (default: 10) 

57 snr_min: float, opt 

58 minimal SNR of LC points (default: 5.0) 

59 n_phase_min: int, opt 

60 number of LC points with phase<= -5(default:1) 

61 n_phase_max: int, opt 

62 number of LC points with phase>= 20 (default: 1) 

63 """ 

64 

65 def __init__(self, metricName='SNNSNMetric', 

66 mjdCol='observationStartMJD', RACol='fieldRA', DecCol='fieldDec', 

67 filterCol='filter', m5Col='fiveSigmaDepth', exptimeCol='visitExposureTime', 

68 nightCol='night', obsidCol='observationId', nexpCol='numExposures', 

69 vistimeCol='visitTime', season=[-1], zmin=0.0, zmax=1.2, 

70 pixArea=9.6, verbose=False, ploteffi=False, 

71 n_bef=4, n_aft=10, snr_min=5., n_phase_min=1, 

72 n_phase_max=1, templateDir=None, zlim_coeff=-1., **kwargs): 

73 

74 self.mjdCol = mjdCol 

75 self.m5Col = m5Col 

76 self.filterCol = filterCol 

77 self.RACol = RACol 

78 self.DecCol = DecCol 

79 self.exptimeCol = exptimeCol 

80 self.seasonCol = 'season' 

81 self.nightCol = nightCol 

82 self.obsidCol = obsidCol 

83 self.nexpCol = nexpCol 

84 self.vistimeCol = vistimeCol 

85 self.pixArea = pixArea 

86 self.zlim_coeff = zlim_coeff 

87 

88 cols = [self.nightCol, self.m5Col, self.filterCol, self.mjdCol, self.obsidCol, 

89 self.nexpCol, self.vistimeCol, self.exptimeCol] 

90 

91 super(SNNSNMetric, self).__init__( 

92 col=cols, metricDtype='object', metricName=metricName, **kwargs) 

93 

94 self.season = season 

95 # LC selection parameters 

96 self.n_bef = n_bef # nb points before peak 

97 self.n_aft = n_aft # nb points after peak 

98 self.snr_min = snr_min # SNR cut for points before/after peak 

99 self.n_phase_min = n_phase_min # nb of point with phase <=-5 

100 self.n_phase_max = n_phase_max # nb of points with phase >=20 

101 

102 # loading reference LC files 

103 lc_reference = Load_Reference(templateDir=templateDir).ref 

104 

105 self.lcFast = {} 

106 telescope = Telescope(airmass=1.2) 

107 for key, vals in lc_reference.items(): 

108 self.lcFast[key] = LCfast(vals, key[0], key[1], telescope, 

109 self.mjdCol, self.RACol, self.DecCol, 

110 self.filterCol, self.exptimeCol, 

111 self.m5Col, self.seasonCol, self.nexpCol, 

112 self.snr_min) 

113 

114 # loading parameters 

115 self.zmin = zmin # zmin for the study 

116 self.zmax = zmax # zmax for the study 

117 self.zStep = 0.05 # zstep 

118 self.daymaxStep = 3. # daymax step 

119 self.min_rf_phase = -20. # min ref phase for LC points selection 

120 self.max_rf_phase = 40. # max ref phase for LC points selection 

121 

122 self.min_rf_phase_qual = -15. # min ref phase for bounds effects 

123 self.max_rf_phase_qual = 25. # max ref phase for bounds effects 

124 

125 # snrate 

126 self.rateSN = SN_Rate( 

127 min_rf_phase=self.min_rf_phase_qual, max_rf_phase=self.max_rf_phase_qual) 

128 

129 # verbose mode - useful for debug and code performance estimation 

130 self.verbose = False 

131 self.ploteffi = False 

132 

133 # supernovae parameters 

134 self.params = ['x0', 'x1', 'daymax', 'color'] 

135 

136 # r = [(-1.0, -1.0)] 

137 self.bad = np.rec.fromrecords([(-1.0, -1.0)], names=['nSN', 'zlim']) 

138 # self.bad = {'nSN': -1.0, 'zlim': -1.0} 

139 

140 def run(self, dataSlice, slicePoint=None): 

141 """ 

142 run method of the metric 

143 

144 Parameters 

145 --------------- 

146 dataSlice: array 

147 data to process 

148 

149 """ 

150 idarray = None 

151 healpixID = -1 

152 if slicePoint is not None: 

153 if 'nside' in slicePoint.keys(): 

154 import healpy as hp 

155 self.pixArea = hp.nside2pixarea( 

156 slicePoint['nside'], degrees=True) 

157 r = [] 

158 names = [] 

159 

160 healpixID = hp.ang2pix( 

161 slicePoint['nside'], np.rad2deg(slicePoint['ra']), np.rad2deg(slicePoint['dec']), nest=True, lonlat=True) 

162 for kk, vv in slicePoint.items(): 

163 r.append(vv) 

164 names.append(kk) 

165 idarray = np.rec.fromrecords([r], names=names) 

166 else: 

167 idarray = np.rec.fromrecords([0., 0.], names=['RA', 'Dec']) 

168 

169 # Two things to do: concatenate data (per band, night) and estimate seasons 

170 dataSlice = rf.drop_fields(dataSlice, ['season']) 

171 

172 dataSlice = self.coadd(pd.DataFrame(dataSlice)) 

173 

174 dataSlice = self.getseason(dataSlice) 

175 

176 # get the seasons 

177 seasons = self.season 

178 

179 # if seasons = -1: process the seasons seen in data 

180 if self.season == [-1]: 

181 seasons = np.unique(dataSlice[self.seasonCol]) 

182 

183 # get redshift range for processing 

184 zRange = list(np.arange(self.zmin, self.zmax, self.zStep)) 

185 if zRange[0] < 1.e-6: 

186 zRange[0] = 0.01 

187 

188 self.zRange = np.unique(zRange) 

189 # season infos 

190 dfa = pd.DataFrame(np.copy(dataSlice)) 

191 season_info = dfa.groupby(['season']).apply( 

192 lambda x: self.seasonInfo(x)).reset_index() 

193 

194 #print('season info', season_info) 

195 # select seasons of at least 30 days 

196 idx = season_info['season_length'] >= 60. 

197 season_info = season_info[idx] 

198 

199 # check wether requested seasons can be processed 

200 test_season = season_info[season_info['season'].isin(seasons)] 

201 # if len(test_season) == 0: 

202 

203 if test_season.empty: 

204 return nlr.merge_arrays([idarray, self.bad], flatten=True) 

205 else: 

206 seasons = test_season['season'] 

207 # print('test_seas', seasons) 

208 # print('hh', season_info) 

209 

210 # get season length depending on the redshift 

211 dur_z = season_info.groupby(['season']).apply( 

212 lambda x: self.duration_z(x)).reset_index() 

213 

214 # remove dur_z with negative season lengths 

215 idx = dur_z['season_length'] >= 10. 

216 dur_z = dur_z[idx] 

217 

218 # generating simulation parameters 

219 gen_par = dur_z.groupby(['z', 'season']).apply( 

220 lambda x: self.calcDaymax(x)).reset_index() 

221 

222 if gen_par.empty: 

223 return nlr.merge_arrays([idarray, self.bad], flatten=True) 

224 resdf = pd.DataFrame() 

225 

226 for seas in seasons: 

227 vara_df = self.run_season( 

228 dataSlice, [seas], gen_par, dur_z) 

229 #print('res', seas, vara_df) 

230 if vara_df is not None: 

231 resdf = pd.concat((resdf, vara_df)) 

232 

233 # final result: median zlim for a faint sn 

234 # and nsn_med for z<zlim 

235 

236 if resdf.empty: 

237 return nlr.merge_arrays([idarray, self.bad], flatten=True) 

238 

239 resdf = resdf.round({'zlim': 3, 'nsn_med': 3}) 

240 x1_ref = -2.0 

241 color_ref = 0.2 

242 

243 idx = np.abs(resdf['x1']-x1_ref) < 1.e-5 

244 idx &= np.abs(resdf['color']-color_ref) < 1.e-5 

245 idx &= resdf['zlim'] > 0 

246 

247 if not resdf[idx].empty: 

248 zlim = resdf[idx]['zlim'].median() 

249 nSN = resdf[idx]['nsn_med'].sum() 

250 

251 resd = np.rec.fromrecords([(nSN, zlim, healpixID)], names=[ 

252 'nSN', 'zlim', 'healpixID']) 

253 res = nlr.merge_arrays([idarray, resd], flatten=True) 

254 

255 else: 

256 

257 res = nlr.merge_arrays([idarray, self.bad], flatten=True) 

258 

259 return res 

260 

261 def reducenSN(self, metricVal): 

262 

263 # At each slicepoint, return the sum nSN value. 

264 

265 return np.sum(metricVal['nSN']) 

266 

267 def reducezlim(self, metricVal): 

268 

269 # At each slicepoint, return the median zlim 

270 

271 return np.median(metricVal['zlim']) 

272 

273 def coadd(self, data): 

274 """ 

275 Method to coadd data per band and per night 

276 

277 Parameters 

278 --------------- 

279 data: pandas df of observations 

280 

281 Returns 

282 ----------- 

283 coadded data (pandas df) 

284 

285 """ 

286 

287 keygroup = [self.filterCol, self.nightCol] 

288 

289 data.sort_values(by=keygroup, ascending=[ 

290 True, True], inplace=True) 

291 

292 coadd_df = data.groupby(keygroup).agg({self.nexpCol: ['sum'], 

293 self.vistimeCol: ['sum'], 

294 self.exptimeCol: ['sum'], 

295 self.mjdCol: ['mean'], 

296 self.RACol: ['mean'], 

297 self.DecCol: ['mean'], 

298 self.m5Col: ['mean']}).reset_index() 

299 

300 coadd_df.columns = [self.filterCol, self.nightCol, self.nexpCol, 

301 self.vistimeCol, self.exptimeCol, self.mjdCol, 

302 self.RACol, self.DecCol, self.m5Col] 

303 

304 coadd_df.loc[:, self.m5Col] += 1.25 * \ 

305 np.log10(coadd_df[self.vistimeCol]/30.) 

306 

307 coadd_df.sort_values(by=[self.filterCol, self.nightCol], ascending=[ 

308 True, True], inplace=True) 

309 

310 return coadd_df.to_records(index=False) 

311 

312 def getseason(self, obs, season_gap=80., mjdCol='observationStartMJD'): 

313 """ 

314 Method to estimate seasons 

315 Parameters 

316 -------------- 

317 obs: numpy array 

318 array of observations 

319 season_gap: float, opt 

320 minimal gap required to define a season (default: 80 days) 

321 mjdCol: str, opt 

322 col name for MJD infos (default: observationStartMJD) 

323 

324 Returns 

325 ---------- 

326 original numpy array with seasonnumber appended 

327 """ 

328 

329 # check wether season has already been estimated 

330 

331 if 'season' in obs.dtype.names: 

332 return obs 

333 

334 obs.sort(order=mjdCol) 

335 

336 seasoncalc = np.ones(obs.size, dtype=int) 

337 

338 if len(obs) > 1: 

339 diff = np.diff(obs[mjdCol]) 

340 flag = np.where(diff > season_gap)[0] 

341 

342 if len(flag) > 0: 

343 for i, indx in enumerate(flag): 

344 seasoncalc[indx+1:] = i+2 

345 

346 obs = rf.append_fields(obs, 'season', seasoncalc) 

347 

348 return obs 

349 

350 def seasonInfo(self, grp): 

351 """ 

352 Method to estimate seasonal info (cadence, season length, ...) 

353 Parameters 

354 -------------- 

355 grp: pandas df group 

356 Returns 

357 --------- 

358 pandas df with the cfollowing cols: 

359 """ 

360 df = pd.DataFrame([len(grp)], columns=['Nvisits']) 

361 df['MJD_min'] = grp[self.mjdCol].min() 

362 df['MJD_max'] = grp[self.mjdCol].max() 

363 df['season_length'] = df['MJD_max']-df['MJD_min'] 

364 df['cadence'] = 0. 

365 

366 if len(grp) > 5: 

367 to = grp.groupby(['night'])[self.mjdCol].median().sort_values() 

368 df['cadence'] = np.mean(to.diff()) 

369 

370 return df 

371 

372 def duration_z(self, grp): 

373 """ 

374 Method to estimate the season length vs redshift 

375 This is necessary to take into account boundary effects 

376 when estimating the number of SN that can be detected 

377 daymin, daymax = min and max MJD of a season 

378 T0_min(z) = daymin-(1+z)*min_rf_phase_qual 

379 T0_max(z) = daymax-(1+z)*max_rf_phase_qual 

380 season_length(z) = T0_max(z)-T0_min(z) 

381 Parameters 

382 -------------- 

383 grp: pandas df group 

384 data to process: season infos 

385 Returns 

386 ---------- 

387 pandas df with season_length, z, T0_min and T0_max cols 

388 """ 

389 

390 daymin = grp['MJD_min'].values 

391 daymax = grp['MJD_max'].values 

392 dur_z = pd.DataFrame(self.zRange, columns=['z']) 

393 dur_z['T0_min'] = daymin-(1.+dur_z['z'])*self.min_rf_phase_qual 

394 dur_z['T0_max'] = daymax-(1.+dur_z['z'])*self.max_rf_phase_qual 

395 dur_z['season_length'] = dur_z['T0_max']-dur_z['T0_min'] 

396 

397 return dur_z 

398 

399 def calcDaymax(self, grp): 

400 """ 

401 Method to estimate T0 (daymax) values for simulation. 

402 Parameters 

403 -------------- 

404 grp: group (pandas df sense) 

405 group of data to process with the following cols: 

406 T0_min: T0 min value (per season) 

407 T0_max: T0 max value (per season) 

408 Returns 

409 ---------- 

410 pandas df with daymax, min_rf_phase, max_rf_phase values 

411 """ 

412 

413 T0_max = grp['T0_max'].values 

414 T0_min = grp['T0_min'].values 

415 num = (T0_max-T0_min)/self.daymaxStep 

416 if T0_max-T0_min > 10: 

417 df = pd.DataFrame(np.linspace( 

418 T0_min, T0_max, int(num)), columns=['daymax']) 

419 else: 

420 df = pd.DataFrame([-1], columns=['daymax']) 

421 

422 df['min_rf_phase'] = self.min_rf_phase_qual 

423 df['max_rf_phase'] = self.max_rf_phase_qual 

424 

425 return df 

426 

427 def run_season(self, dataSlice, season, gen_par, dura_z): 

428 """ 

429 Method to run on seasons 

430 Parameters 

431 -------------- 

432 dataSlice: numpy array, opt 

433 data to process (scheduler simulations) 

434 seasons: list(int) 

435 list of seasons to process 

436 Returns 

437 --------- 

438 effi_seasondf: pandas df 

439 efficiency curves 

440 zlimsdf: pandas df 

441 redshift limits and number of supernovae 

442 """ 

443 

444 time_ref = time.time() 

445 

446 if self.verbose: 

447 print('#### Processing season', season) 

448 

449 groupnames = ['season', 'x1', 'color'] 

450 

451 gen_p = gen_par[gen_par['season'].isin(season)] 

452 

453 if gen_p.empty: 

454 if self.verbose: 

455 print('No generator parameter found') 

456 return None 

457 dur_z = dura_z[dura_z['season'].isin(season)] 

458 obs = pd.DataFrame(np.copy(dataSlice)) 

459 obs = obs[obs['season'].isin(season)] 

460 

461 obs = obs.sort_values(by=['night']) 

462 #print('data here', obs.columns) 

463 #print(obs[['night', 'filter', 'observationStartMJD', 'fieldRA', 'fieldDec']]) 

464 """ 

465 import matplotlib.pyplot as plt 

466 plt.plot(dataSlice['fieldRA'], dataSlice['fieldDec'], 'ko') 

467 print('data', len(dataSlice)) 

468 plt.show() 

469 """ 

470 # simulate supernovae and lc 

471 if self.verbose: 

472 print("SN generation") 

473 print(season, obs) 

474 sn = self.genSN(obs.to_records( 

475 index=False), gen_p.to_records(index=False)) 

476 

477 if self.verbose: 

478 idx = np.abs(sn['x1']+2) < 1.e-5 

479 idx &= np.abs(sn['z']-0.2) < 1.e-5 

480 sel = sn[idx] 

481 sel = sel.sort_values(by=['z', 'daymax']) 

482 

483 print('sn and lc', len(sn), sel.columns, 

484 sel[['x1', 'color', 'z', 'daymax', 'Cov_colorcolor', 'n_bef', 'n_aft']]) 

485 

486 # from these supernovae: estimate observation efficiency vs z 

487 effi_seasondf = self.effidf(sn) 

488 

489 # zlims can only be estimated if efficiencies are ok 

490 idx = effi_seasondf['z'] <= 0.2 

491 x1ref = -2.0 

492 colorref = 0.2 

493 idx &= np.abs(effi_seasondf['x1']-x1ref) < 1.e-5 

494 idx &= np.abs(effi_seasondf['color']-colorref) < 1.e-5 

495 sel = effi_seasondf[idx] 

496 

497 if np.mean(sel['effi']) > 0.02: 

498 # estimate zlims 

499 zlimsdf = self.zlims(effi_seasondf, dur_z, groupnames) 

500 

501 # estimate number of medium supernovae 

502 zlimsdf['nsn_med'], zlimsdf['var_nsn_med'] = zlimsdf.apply(lambda x: self.nsn_typedf( 

503 x, 0.0, 0.0, effi_seasondf, dur_z), axis=1, result_type='expand').T.values 

504 else: 

505 return None 

506 

507 if self.verbose: 

508 print('#### SEASON processed', time.time()-time_ref, 

509 season) 

510 

511 return zlimsdf 

512 

513 def genSN(self, obs, gen_par): 

514 """ 

515 Method to simulate LC and supernovae 

516 Parameters 

517 --------------- 

518 obs: numpy array 

519 array of observations(from scheduler) 

520 gen_par: numpy array 

521 array of parameters for simulation 

522 """ 

523 

524 time_ref = time.time() 

525 # LC estimation 

526 

527 sn_tot = pd.DataFrame() 

528 lc_tot = pd.DataFrame() 

529 for key, vals in self.lcFast.items(): 

530 time_refs = time.time() 

531 gen_par_cp = np.copy(gen_par) 

532 if key == (-2.0, 0.2): 

533 idx = gen_par_cp['z'] < 0.9 

534 gen_par_cp = gen_par_cp[idx] 

535 lc = vals(obs, gen_par_cp, bands='grizy') 

536 if self.verbose: 

537 print('End of simulation', key, time.time()-time_refs) 

538 

539 if self.verbose: 

540 print('End of simulation after concat', 

541 key, time.time()-time_refs) 

542 

543 # estimate SN 

544 

545 sn = pd.DataFrame() 

546 if len(lc) > 0: 

547 sn = self.process(pd.DataFrame(lc)) 

548 

549 if self.verbose: 

550 print('End of supernova', time.time()-time_refs) 

551 

552 if not sn.empty: 

553 sn_tot = pd.concat([sn_tot, pd.DataFrame(sn)], sort=False) 

554 

555 if self.verbose: 

556 print('End of supernova - all', time.time()-time_ref) 

557 

558 return sn_tot 

559 

560 def process(self, tab): 

561 """ 

562 Method to process LC: sigma_color estimation and LC selection 

563 Parameters 

564 -------------- 

565 tab: pandas df of LC points with the following cols: 

566 flux: flux 

567 fluxerr: flux error 

568 phase: phase 

569 snr_m5: Signal-to-Noise Ratio 

570 time: time(MJD) 

571 mag: magnitude 

572 m5: five-sigma depth 

573 magerr: magnitude error 

574 exposuretime: exposure time 

575 band: filter 

576 zp: zero-point 

577 season: season number 

578 healpixID: pixel ID 

579 pixRA: pixel RA 

580 pixDec: pixel Dec 

581 z: redshift 

582 daymax: T0 

583 flux_e_sec: flux(in photoelec/sec) 

584 flux_5: 5-sigma flux(in photoelec/sec) 

585 F_x0x0, ...F_colorcolor: Fisher matrix elements 

586 x1: x1 SN 

587 color: color SN 

588 n_aft: number of LC points before daymax 

589 n_bef: number of LC points after daymax 

590 n_phmin: number of LC points with a phase < -5 

591 n_phmax: number of LC points with a phase > 20 

592 Returns 

593 ---------- 

594 """ 

595 # now groupby 

596 tab = tab.round({'daymax': 3, 

597 'z': 3, 'x1': 2, 'color': 2}) 

598 groups = tab.groupby( 

599 ['daymax', 'season', 'z', 'x1', 'color']) 

600 

601 tosum = [] 

602 for ia, vala in enumerate(self.params): 

603 for jb, valb in enumerate(self.params): 

604 if jb >= ia: 

605 tosum.append('F_'+vala+valb) 

606 tosum += ['n_aft', 'n_bef', 'n_phmin', 'n_phmax'] 

607 # apply the sum on the group 

608 sums = groups[tosum].sum().reset_index() 

609 

610 # select LC according to the number of points bef/aft peak 

611 idx = sums['n_aft'] >= self.n_aft 

612 idx &= sums['n_bef'] >= self.n_bef 

613 idx &= sums['n_phmin'] >= self.n_phase_min 

614 idx &= sums['n_phmax'] >= self.n_phase_max 

615 

616 if self.verbose: 

617 print('selection parameters', self.n_bef, 

618 self.n_aft, self.n_phase_min, self.n_phase_max) 

619 finalsn = pd.DataFrame() 

620 goodsn = pd.DataFrame(sums.loc[idx]) 

621 

622 # estimate the color for SN that passed the selection cuts 

623 if len(goodsn) > 0: 

624 goodsn.loc[:, 'Cov_colorcolor'] = CovColor(goodsn).Cov_colorcolor 

625 finalsn = pd.concat([finalsn, goodsn], sort=False) 

626 

627 badsn = pd.DataFrame(sums.loc[~idx]) 

628 

629 # Supernovae that did not pass the cut have a sigma_color=10 

630 if len(badsn) > 0: 

631 badsn.loc[:, 'Cov_colorcolor'] = 100. 

632 finalsn = pd.concat([finalsn, badsn], sort=False) 

633 

634 return finalsn 

635 

636 def effidf(self, sn_tot, color_cut=0.04): 

637 """ 

638 Method estimating efficiency vs z for a sigma_color cut 

639 Parameters 

640 --------------- 

641 sn_tot: pandas df 

642 data used to estimate efficiencies 

643 color_cut: float, opt 

644 color selection cut(default: 0.04) 

645 Returns 

646 ---------- 

647 effi: pandas df with the following cols: 

648 season: season 

649 pixRA: RA of the pixel 

650 pixDec: Dec of the pixel 

651 healpixID: pixel ID 

652 x1: SN stretch 

653 color: SN color 

654 z: redshift 

655 effi: efficiency 

656 effi_err: efficiency error(binomial) 

657 """ 

658 

659 sndf = pd.DataFrame(sn_tot) 

660 

661 listNames = ['season', 'x1', 'color'] 

662 groups = sndf.groupby(listNames) 

663 

664 # estimating efficiencies 

665 effi = groups[['Cov_colorcolor', 'z']].apply( 

666 lambda x: self.effiObsdf(x, color_cut)).reset_index(level=list(range(len(listNames)))) 

667 

668 # this is to plot efficiencies and also sigma_color vs z 

669 if self.ploteffi: 

670 import matplotlib.pylab as plt 

671 fig, ax = plt.subplots() 

672 figb, axb = plt.subplots() 

673 

674 self.plot(ax, effi, 'effi', 'effi_err', 

675 'Observing Efficiencies', ls='-') 

676 sndf['sigma_color'] = np.sqrt(sndf['Cov_colorcolor']) 

677 self.plot(axb, sndf, 'sigma_color', None, '$\sigma_{color}$') 

678 # get efficiencies vs z 

679 

680 plt.show() 

681 

682 return effi 

683 

684 def plot(self, ax, effi, vary, erry=None, legy='', ls='None'): 

685 """ 

686 Simple method to plot vs z 

687 Parameters 

688 -------------- 

689 ax: 

690 axis where to plot 

691 effi: pandas df 

692 data to plot 

693 vary: str 

694 variable(column of effi) to plot 

695 erry: str, opt 

696 error on y-axis(default: None) 

697 legy: str, opt 

698 y-axis legend(default: '') 

699 """ 

700 grb = effi.groupby(['x1', 'color']) 

701 yerr = None 

702 for key, grp in grb: 

703 x1 = grp['x1'].unique()[0] 

704 color = grp['color'].unique()[0] 

705 if erry is not None: 

706 yerr = grp[erry] 

707 ax.errorbar(grp['z'], grp[vary], yerr=yerr, 

708 marker='o', label='(x1,color)=({},{})'.format(x1, color), lineStyle=ls) 

709 

710 ftsize = 15 

711 ax.set_xlabel('z', fontsize=ftsize) 

712 ax.set_ylabel(legy, fontsize=ftsize) 

713 ax.xaxis.set_tick_params(labelsize=ftsize) 

714 ax.yaxis.set_tick_params(labelsize=ftsize) 

715 ax.legend(fontsize=ftsize) 

716 

717 def effiObsdf(self, data, color_cut=0.04): 

718 """ 

719 Method to estimate observing efficiencies for supernovae 

720 Parameters 

721 -------------- 

722 data: pandas df - grp 

723 data to process 

724 Returns 

725 ---------- 

726 pandas df with the following cols: 

727 - cols used to make the group 

728 - effi, effi_err: observing efficiency and associated error 

729 """ 

730 

731 # reference df to estimate efficiencies 

732 df = data.loc[lambda dfa: np.sqrt(dfa['Cov_colorcolor']) < 100000., :] 

733 

734 # selection on sigma_c<= 0.04 

735 df_sel = df.loc[lambda dfa: np.sqrt( 

736 dfa['Cov_colorcolor']) <= color_cut, :] 

737 

738 # make groups (with z) 

739 group = df.groupby('z') 

740 group_sel = df_sel.groupby('z') 

741 

742 # Take the ratio to get efficiencies 

743 rb = (group_sel.size()/group.size()) 

744 err = np.sqrt(rb*(1.-rb)/group.size()) 

745 var = rb*(1.-rb)*group.size() 

746 

747 rb = rb.array 

748 err = err.array 

749 var = var.array 

750 

751 rb[np.isnan(rb)] = 0. 

752 err[np.isnan(err)] = 0. 

753 var[np.isnan(var)] = 0. 

754 

755 return pd.DataFrame({group.keys: list(group.groups.keys()), 

756 'effi': rb, 

757 'effi_err': err, 

758 'effi_var': var}) 

759 

760 def zlims(self, effi_seasondf, dur_z, groupnames): 

761 """ 

762 Method to estimate redshift limits 

763 Parameters 

764 -------------- 

765 effi_seasondf: pandas df 

766 season: season 

767 pixRA: RA of the pixel 

768 pixDec: Dec of the pixel 

769 healpixID: pixel ID 

770 x1: SN stretch 

771 color: SN color 

772 z: redshift 

773 effi: efficiency 

774 effi_err: efficiency error (binomial) 

775 dur_z: pandas df with the following cols: 

776 season: season 

777 z: redshift 

778 T0_min: min daymax 

779 T0_max: max daymax 

780 season_length: season length 

781 groupnames: list(str) 

782 list of columns to use to define the groups 

783 Returns 

784 ---------- 

785 pandas df with the following cols: pixRA: RA of the pixel 

786 pixDec: Dec of the pixel 

787 healpixID: pixel ID 

788 season: season number 

789 x1: SN stretch 

790 color: SN color 

791 zlim: redshift limit 

792 """ 

793 

794 res = effi_seasondf.groupby(groupnames).apply( 

795 lambda x: self.zlimdf(x, dur_z)).reset_index(level=list(range(len(groupnames)))) 

796 

797 return res 

798 

799 def zlimdf(self, grp, duration_z): 

800 """ 

801 Method to estimate redshift limits 

802 Parameters 

803 -------------- 

804 grp: pandas df group 

805 efficiencies to estimate redshift limits; 

806 columns: 

807 season: season 

808 pixRA: RA of the pixel 

809 pixDec: Dec of the pixel 

810 healpixID: pixel ID 

811 x1: SN stretch 

812 color: SN color 

813 z: redshift 

814 effi: efficiency 

815 effi_err: efficiency error (binomial) 

816 duration_z: pandas df with the following cols: 

817 season: season 

818 z: redshift 

819 T0_min: min daymax 

820 T0_max: max daymax 

821 season_length: season length 

822 Returns 

823 ---------- 

824 pandas df with the following cols: 

825 zlimit: redshift limit 

826 """ 

827 

828 zlimit = 0.0 

829 

830 # z range for the study 

831 zplot = np.arange(self.zmin, self.zmax, 0.01) 

832 

833 # print(grp['z'], grp['effi']) 

834 

835 if len(grp['z']) <= 3: 

836 return pd.DataFrame({'zlim': [zlimit]}) 

837 # 'status': [int(status)]}) 

838 # interpolate efficiencies vs z 

839 effiInterp = interp1d( 

840 grp['z'], grp['effi'], kind='linear', bounds_error=False, fill_value=0.) 

841 

842 if self.zlim_coeff < 0.: 

843 # in that case zlim is estimated from efficiencies 

844 # first step: identify redshift domain with efficiency decrease 

845 zlimit = self.zlim_from_effi(effiInterp, zplot) 

846 #status = self.status['ok'] 

847 

848 else: 

849 zlimit = self.zlim_from_cumul( 

850 grp, duration_z, effiInterp, zplot) 

851 

852 return pd.DataFrame({'zlim': [zlimit]}) 

853 # 'status': [int(status)]}) 

854 

855 def zlim_from_cumul(self, grp, duration_z, effiInterp, zplot, rate='cte'): 

856 """ 

857 Method to estimate the redshift limit from the cumulative 

858 The redshift limit is estimated to be the z value corresponding to: 

859 frac(NSN(z<zlimit))=zlimi_coeff 

860 

861 Parameters 

862 --------------- 

863 grp: pandas group 

864 data to process 

865 duration_z: array 

866 duration as a function of the redshift 

867 effiInterp: interp1d 

868 interpolator for efficiencies 

869 zplot: interp1d 

870 interpolator for redshift values 

871 rate: str, opt 

872 rate to estimate the number of SN to estimate zlimit 

873 rate = cte: rate independent of z 

874 rate = SN_rate: rate from SN_Rate class 

875 

876 Returns 

877 ---------- 

878 zlimit: float 

879 the redshift limit 

880 """ 

881 

882 if rate == 'SN_rate': 

883 # get rate 

884 season = np.median(grp['season']) 

885 idx = duration_z['season'] == season 

886 seas_duration_z = duration_z[idx] 

887 

888 durinterp_z = interp1d( 

889 seas_duration_z['z'], seas_duration_z['season_length'], bounds_error=False, fill_value=0.) 

890 

891 # estimate the rates and nsn vs z 

892 zz, rate, err_rate, nsn, err_nsn = self.rateSN(zmin=self.zmin, 

893 zmax=self.zmax, 

894 duration_z=durinterp_z, 

895 survey_area=self.pixArea) 

896 

897 # rate interpolation 

898 rateInterp = interp1d(zz, nsn, kind='linear', 

899 bounds_error=False, fill_value=0) 

900 else: 

901 # this is for a rate z-independent 

902 nsn = np.ones(len(zplot)) 

903 rateInterp = interp1d(zplot, nsn, kind='linear', 

904 bounds_error=False, fill_value=0) 

905 

906 nsn_cum = np.cumsum(effiInterp(zplot)*rateInterp(zplot)) 

907 

908 if nsn_cum[-1] >= 1.e-5: 

909 nsn_cum_norm = nsn_cum/nsn_cum[-1] # normalize 

910 zlim = interp1d(nsn_cum_norm, zplot) 

911 zlimit = zlim(self.zlim_coeff).item() 

912 

913 if self.ploteffi: 

914 self.plot_NSN_cumul(grp, nsn_cum_norm, zplot) 

915 else: 

916 zlimit = 0. 

917 

918 return zlimit 

919 

920 def plot_NSN_cumul(self, grp, nsn_cum_norm, zplot): 

921 """ 

922 Method to plot the NSN cumulative vs redshift 

923 Parameters 

924 -------------- 

925 grp: pandas group 

926 data to process 

927 """ 

928 

929 import matplotlib.pylab as plt 

930 fig, ax = plt.subplots() 

931 x1 = grp['x1'].unique()[0] 

932 color = grp['color'].unique()[0] 

933 

934 ax.plot(zplot, nsn_cum_norm, 

935 label='(x1,color)=({},{})'.format(x1, color)) 

936 

937 ftsize = 15 

938 ax.set_ylabel('NSN ($z<$)', fontsize=ftsize) 

939 ax.set_xlabel('z', fontsize=ftsize) 

940 ax.xaxis.set_tick_params(labelsize=ftsize) 

941 ax.yaxis.set_tick_params(labelsize=ftsize) 

942 ax.set_xlim((0.0, 0.8)) 

943 ax.set_ylim((0.0, 1.05)) 

944 ax.plot([0., 1.2], [0.95, 0.95], ls='--', color='k') 

945 plt.legend(fontsize=ftsize) 

946 plt.show() 

947 

948 def zlim_from_effi(self, effiInterp, zplot): 

949 """ 

950 Method to estimate the redshift limit from efficiency curves 

951 The redshift limit is defined here as the redshift value beyond 

952 which efficiency decreases up to zero. 

953 Parameters 

954 --------------- 

955 effiInterp: interpolator 

956 use to get efficiencies 

957 zplot: numpy array 

958 redshift values 

959 Returns 

960 ----------- 

961 zlimit: float 

962 the redshift limit 

963 """ 

964 

965 # get efficiencies 

966 effis = effiInterp(zplot) 

967 # select data with efficiency decrease 

968 idx = np.where(np.diff(effis) < -0.005)[0] 

969 

970 # Bail out if there is no data 

971 if np.size(idx) == 0: 

972 return 0 

973 

974 z_effi = np.array(zplot[idx], dtype={ 

975 'names': ['z'], 'formats': [float]}) 

976 # from this make some "z-periods" to avoid accidental zdecrease at low z 

977 z_gap = 0.05 

978 seasoncalc = np.ones(z_effi.size, dtype=int) 

979 diffz = np.diff(z_effi['z']) 

980 flag = np.where(diffz > z_gap)[0] 

981 

982 if len(flag) > 0: 

983 for i, indx in enumerate(flag): 

984 seasoncalc[indx+1:] = i+2 

985 z_effi = rf.append_fields(z_effi, 'season', seasoncalc) 

986 

987 # now take the highest season (end of the efficiency curve) 

988 idd = z_effi['season'] == np.max(z_effi['season']) 

989 zlimit = np.min(z_effi[idd]['z']) 

990 

991 return zlimit 

992 

993 def zlimdf_deprecated(self, grp, duration_z): 

994 """ 

995 Method to estimate redshift limits 

996 Parameters 

997 -------------- 

998 grp: pandas df group 

999 efficiencies to estimate redshift limits; 

1000 columns: 

1001 season: season 

1002 pixRA: RA of the pixel 

1003 pixDec: Dec of the pixel 

1004 healpixID: pixel ID 

1005 x1: SN stretch 

1006 color: SN color 

1007 z: redshift 

1008 effi: efficiency 

1009 effi_err: efficiency error (binomial) 

1010 duration_z: pandas df with the following cols: 

1011 season: season 

1012 z: redshift 

1013 T0_min: min daymax 

1014 T0_max: max daymax 

1015 season_length: season length 

1016 Returns 

1017 ---------- 

1018 pandas df with the following cols: 

1019 zlimit: redshift limit 

1020 """ 

1021 

1022 zlimit = 0.0 

1023 

1024 # z range for the study 

1025 zplot = list(np.arange(self.zmin, self.zmax, 0.001)) 

1026 

1027 # print(grp['z'], grp['effi']) 

1028 

1029 if len(grp['z']) <= 3: 

1030 return pd.DataFrame({'zlim': [zlimit]}) 

1031 # 'status': [int(status)]}) 

1032 # interpolate efficiencies vs z 

1033 effiInterp = interp1d( 

1034 grp['z'], grp['effi'], kind='linear', bounds_error=False, fill_value=0.) 

1035 

1036 # get rate 

1037 season = np.median(grp['season']) 

1038 idx = duration_z['season'] == season 

1039 seas_duration_z = duration_z[idx] 

1040 

1041 # print('hhh1', seas_duration_z) 

1042 durinterp_z = interp1d( 

1043 seas_duration_z['z'], seas_duration_z['season_length'], bounds_error=False, fill_value=0.) 

1044 

1045 # estimate the rates and nsn vs z 

1046 zz, rate, err_rate, nsn, err_nsn = self.rateSN(zmin=self.zmin, 

1047 zmax=self.zmax, 

1048 duration_z=durinterp_z, 

1049 survey_area=self.pixArea) 

1050 

1051 # rate interpolation 

1052 rateInterp = interp1d(zz, nsn, kind='linear', 

1053 bounds_error=False, fill_value=0) 

1054 

1055 # estimate the cumulated number of SN vs z 

1056 nsn_cum = np.cumsum(effiInterp(zplot)*rateInterp(zplot)) 

1057 

1058 if nsn_cum[-1] >= 1.e-5: 

1059 nsn_cum_norm = nsn_cum/nsn_cum[-1] # normalize 

1060 zlim = interp1d(nsn_cum_norm, zplot) 

1061 zlimit = zlim(0.95).item() 

1062 # status = self.status['ok'] 

1063 

1064 if self.ploteffi: 

1065 import matplotlib.pylab as plt 

1066 fig, ax = plt.subplots() 

1067 x1 = grp['x1'].unique()[0] 

1068 color = grp['color'].unique()[0] 

1069 

1070 ax.plot(zplot, nsn_cum_norm, 

1071 label='(x1,color)=({},{})'.format(x1, color)) 

1072 

1073 ftsize = 15 

1074 ax.set_ylabel('NSN ($z<$)', fontsize=ftsize) 

1075 ax.set_xlabel('z', fontsize=ftsize) 

1076 ax.xaxis.set_tick_params(labelsize=ftsize) 

1077 ax.yaxis.set_tick_params(labelsize=ftsize) 

1078 ax.set_xlim((0.0, 1.2)) 

1079 ax.set_ylim((0.0, 1.05)) 

1080 ax.plot([0., 1.2], [0.95, 0.95], ls='--', color='k') 

1081 plt.legend(fontsize=ftsize) 

1082 plt.show() 

1083 

1084 return pd.DataFrame({'zlim': [zlimit]}) 

1085 # 'status': [int(status)]}) 

1086 

1087 def nsn_typedf(self, grp, x1, color, effi_tot, duration_z, search=True): 

1088 """ 

1089 Method to estimate the number of supernovae for a given type of SN 

1090 Parameters 

1091 -------------- 

1092 grp: pandas series with the following infos: 

1093 pixRA: pixelRA 

1094 pixDec: pixel Dec 

1095 healpixID: pixel ID 

1096 season: season 

1097 x1: SN stretch 

1098 color: SN color 

1099 zlim: redshift limit 

1100 x1, color: SN params to estimate the number 

1101 effi_tot: pandas df with columns: 

1102 season: season 

1103 pixRA: RA of the pixel 

1104 pixDec: Dec of the pixel 

1105 healpixID: pixel ID 

1106 x1: SN stretch 

1107 color: SN color 

1108 z: redshift 

1109 effi: efficiency 

1110 effi_err: efficiency error (binomial) 

1111 duration_z: pandas df with the following cols: 

1112 season: season 

1113 z: redshift 

1114 T0_min: min daymax 

1115 T0_max: max daymax 

1116 season_length: season length 

1117 Returns 

1118 ---------- 

1119 nsn: float 

1120 number of supernovae 

1121 """ 

1122 

1123 # get rate 

1124 season = np.median(grp['season']) 

1125 idx = duration_z['season'] == season 

1126 seas_duration_z = duration_z[idx] 

1127 

1128 # print('hhh2', seas_duration_z) 

1129 durinterp_z = interp1d( 

1130 seas_duration_z['z'], seas_duration_z['season_length'], bounds_error=False, fill_value=0.) 

1131 

1132 if search: 

1133 effisel = effi_tot.loc[lambda dfa: ( 

1134 dfa['x1'] == x1) & (dfa['color'] == color), :] 

1135 else: 

1136 effisel = effi_tot 

1137 

1138 nsn, var_nsn = self.nsn(effisel, grp['zlim'], durinterp_z) 

1139 

1140 return (nsn, var_nsn) 

1141 

1142 def nsn(self, effi, zlim, duration_z): 

1143 """ 

1144 Method to estimate the number of supernovae 

1145 Parameters 

1146 -------------- 

1147 effi: pandas df grp of efficiencies 

1148 season: season 

1149 pixRA: RA of the pixel 

1150 pixDec: Dec of the pixel 

1151 healpixID: pixel ID 

1152 x1: SN stretch 

1153 color: SN color 

1154 z: redshift 

1155 effi: efficiency 

1156 effi_err: efficiency error (binomial) 

1157 zlim: float 

1158 redshift limit value 

1159 duration_z: pandas df with the following cols: 

1160 season: season 

1161 z: redshift 

1162 T0_min: min daymax 

1163 T0_max: max daymax 

1164 season_length: season length 

1165 Returns 

1166 ---------- 

1167 nsn, var_nsn : float 

1168 number of supernovae (and variance) with z<zlim 

1169 """ 

1170 

1171 if zlim < 1.e-3: 

1172 return (-1.0, -1.0) 

1173 

1174 dz = 0.001 

1175 zplot = list(np.arange(self.zmin, self.zmax, dz)) 

1176 # interpolate efficiencies vs z 

1177 effiInterp = interp1d( 

1178 effi['z'], effi['effi'], kind='linear', bounds_error=False, fill_value=0.) 

1179 # estimate the cumulated number of SN vs z 

1180 zz, rate, err_rate, nsn, err_nsn = self.rateSN(zmin=self.zmin, 

1181 zmax=self.zmax, 

1182 dz=dz, 

1183 duration_z=duration_z, 

1184 survey_area=self.pixArea) 

1185 

1186 nsn_cum = np.cumsum(effiInterp(zplot)*nsn) 

1187 

1188 nsn_interp = interp1d(zplot, nsn_cum) 

1189 

1190 nsn = nsn_interp(zlim).item() 

1191 

1192 return [nsn, 0.0]