Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from __future__ import print_function 

2from builtins import zip 

3from builtins import str 

4from builtins import range 

5from builtins import object 

6import os 

7import gzip 

8import itertools 

9import numpy as np 

10from astropy.io import fits 

11 

12import lsst.utils 

13from lsst.sims.photUtils.Sed import Sed 

14from lsst.sims.photUtils.Bandpass import Bandpass 

15from lsst.sims.catUtils.matchSED import selectStarSED 

16from lsst.sims.photUtils import BandpassDict 

17 

18__all__ = ["readGalfast"] 

19 

20class readGalfast(object): 

21 

22 def parseGalfast(self, headerLine): 

23 

24 """ 

25 Use galfast header line to organize input 

26 

27 @param [in] headerLine is the first line from a galfast catalog output file 

28 

29 @param [out] galfastDict is a dictionary relating parameter name to input column 

30 

31 """ 

32 

33 galfastDict = {} 

34 header = headerLine.split(' ') 

35 colNo = 0 

36 for title in header: 

37 if title == 'lb[2]': 

38 galfastDict['l'] = colNo 

39 colNo += 1 

40 galfastDict['b'] = colNo 

41 colNo += 1 

42 elif title == 'radec[2]': 

43 galfastDict['ra'] = colNo 

44 colNo += 1 

45 galfastDict['dec'] = colNo 

46 colNo += 1 

47 elif title == 'XYZ[3]': 

48 galfastDict['X'] = colNo 

49 colNo += 1 

50 galfastDict['Y'] = colNo 

51 colNo += 1 

52 galfastDict['Z'] = colNo 

53 colNo += 1 

54 elif title == 'DM': 

55 galfastDict['DM'] = colNo 

56 colNo += 1 

57 elif title == 'absSDSSr{alias=M1;alias=absmag;band=SDSSr;}': 

58 galfastDict['absSDSSr'] = colNo 

59 colNo += 1 

60 elif title == 'comp': 

61 galfastDict['comp'] = colNo 

62 colNo += 1 

63 elif title == 'FeH': 

64 galfastDict['FeH'] = colNo 

65 colNo += 1 

66 elif title == 'vcyl[3]': 

67 galfastDict['Vr'] = colNo 

68 colNo += 1 

69 galfastDict['Vphi'] = colNo 

70 colNo += 1 

71 galfastDict['Vz'] = colNo 

72 colNo += 1 

73 elif title == 'pmlb[3]': 

74 galfastDict['pml'] = colNo 

75 colNo += 1 

76 galfastDict['pmb'] = colNo 

77 colNo += 1 

78 galfastDict['vRadlb'] = colNo 

79 colNo += 1 

80 elif title == 'pmradec[3]': 

81 galfastDict['pmra'] = colNo 

82 colNo += 1 

83 galfastDict['pmdec'] = colNo 

84 colNo += 1 

85 galfastDict['vRad'] = colNo 

86 colNo += 1 

87 elif title == 'Am': 

88 galfastDict['Am'] = colNo 

89 colNo += 1 

90 elif title == 'AmInf': 

91 galfastDict['AmInf'] = colNo 

92 colNo += 1 

93 elif title.startswith('SDSSugriz['): 

94 bandString = title.split('=')[2] 

95 bandString1 = bandString.split(',') 

96 for band in bandString1: 

97 band = band.rstrip(';}') 

98 bandName = band.split(':')[1] 

99 galfastDict[bandName] = colNo 

100 colNo += 1 

101 elif title == 'SDSSugrizPhotoFlags{class=flags;}': 

102 galfastDict['SDSSPhotoFlags'] = colNo 

103 colNo += 1 

104 elif title == '#': pass 

105 elif len(title) < 1: pass 

106 elif title.isspace(): pass 

107 else: 

108 raise RuntimeError('*** Unknown field: %s' % (title)) 

109 return galfastDict 

110 

111 def convDMtoKpc(self, DM): 

112 """ 

113 Change from distance modulus to distance in kiloparsecs 

114 

115 @param [in] DM is the distance modulus 

116 

117 @param [out] distanceKpc is the distance in kiloparsecs 

118 

119 """ 

120 

121 distancePc = 10**((0.2*DM) + 1) 

122 distanceKpc = distancePc / 1000. 

123 return distanceKpc 

124 

125 def loadGalfast(self, filenameList, outFileList, sEDPath = None, kuruczPath = None, 

126 mltPath = None, wdPath = None, kuruczSubset = None, 

127 mltSubset = None, wdSubset = None, chunkSize = 10000): 

128 """ 

129 This is customized for the outputs we currently need for the purposes of consistent output 

130 It will read in a galfast output file and output desired values for database input into a file 

131 

132 @param [in] filenameList is a list of the galfast output files that will be loaded and processed. 

133 Can process fits, gzipped, or txt output from galfast. 

134 

135 @param [in] outFileList is a list of the names of the output files that will be created. If gzipped 

136 output is desired simply write the filenames with .gz at the end. 

137 

138 @param [in] kuruczPath is a place to specify a path to kurucz SED files 

139 

140 @param [in] mltPath is the same as kuruczPath except that it specifies a directory for the 

141 mlt SEDs 

142 

143 @param [in] wdPath is the same as the previous two except that it specifies a path to a 

144 white dwarf SED directory. 

145 

146 @param [in] kuruczSubset is a list which provides a subset of the kurucz files within the 

147 kurucz folder that one wants to use 

148 

149 @param [in] mltSubset is a list which provides a subset of the mlt files within the 

150 mlt folder that one wants to use 

151 

152 @param [in] wdSubset is a list which provides a subset of the wd files within the 

153 wd folder that one wants to use 

154 

155 @param [in] chunkSize is the size of chunks of lines to be read from the catalog at one time. 

156 """ 

157 

158 for filename in filenameList: 

159 #Make sure input file exists and is readable format before doing anything else 

160 if os.path.isfile(filename) == False: 

161 raise RuntimeError('*** File does not exist') 

162 

163 #Process various possible galfast outputs 

164 if filename.endswith(('.txt', '.gz', '.fits')): 

165 continue 

166 else: 

167 raise RuntimeError(str('*** Unsupported File Format in file: ' + str(filename))) 

168 

169 #If all files exist and are in proper formats then load seds 

170 

171 selectStarSED0 = selectStarSED(kuruczDir=kuruczPath, 

172 mltDir=mltPath, 

173 wdDir=wdPath) 

174 

175 if kuruczSubset is None: 

176 kuruczList = selectStarSED0.loadKuruczSEDs() 

177 else: 

178 kuruczList = selectStarSED0.loadKuruczSEDs(subset = kuruczSubset) 

179 

180 #Only need one dictionary since none of the names overlap 

181 positionDict = {} 

182 for kNum, kuruczSED in enumerate(kuruczList): 

183 positionDict[kuruczSED.name] = kNum 

184 

185 if mltSubset is None: 

186 mltList = selectStarSED0.loadmltSEDs() 

187 else: 

188 mltList = selectStarSED0.loadmltSEDs(subset = mltSubset) 

189 

190 for mltNum, mltSED in enumerate(mltList): 

191 positionDict[mltSED.name] = mltNum 

192 

193 if wdSubset is None: 

194 wdListH, wdListHE = selectStarSED0.loadwdSEDs() 

195 else: 

196 wdListH, wdListHE = selectStarSED0.loadwdSEDs(subset = wdSubset) 

197 

198 for hNum, hSED in enumerate(wdListH): 

199 positionDict[hSED.name] = hNum 

200 

201 for heNum, heSED in enumerate(wdListHE): 

202 positionDict[heSED.name] = heNum 

203 

204 #For adding/subtracting extinction when calculating colors 

205 #Numbers below come from Schlafly and Finkbeiner (2011) (ApJ, 737, 103) 

206 #normalized by SDSS r mag value 

207 sdssExtCoeffs = [1.8551, 1.4455, 1.0, 0.7431, 0.5527] 

208 lsstExtCoeffs = [1.8140, 1.4166, 0.9947, 0.7370, 0.5790, 0.4761] 

209 

210 sdssPhot = BandpassDict.loadTotalBandpassesFromFiles(['u','g','r','i','z'], 

211 bandpassDir = os.path.join(lsst.utils.getPackageDir('throughputs'), 

212 'sdss'), 

213 bandpassRoot = 'sdss_') 

214 

215 #Load Bandpasses for LSST colors to get colors from matched SEDs 

216 lsstFilterList = ('u', 'g', 'r', 'i', 'z', 'y') 

217 lsstPhot = BandpassDict.loadTotalBandpassesFromFiles(lsstFilterList) 

218 imSimBand = Bandpass() 

219 imSimBand.imsimBandpass() 

220 

221 #Calculate colors and add them to the SED objects 

222 kuruczColors = selectStarSED0.calcBasicColors(kuruczList, sdssPhot) 

223 mltColors = selectStarSED0.calcBasicColors(mltList, sdssPhot) 

224 hColors = selectStarSED0.calcBasicColors(wdListH, sdssPhot) 

225 heColors = selectStarSED0.calcBasicColors(wdListHE, sdssPhot) 

226 

227 listDict = {'kurucz':kuruczList, 'mlt':mltList, 'H':wdListH, 'HE':wdListHE} 

228 colorDict = {'kurucz':kuruczColors, 'mlt':mltColors, 'H':hColors, 'HE':heColors} 

229 

230 for filename, outFile in zip(filenameList, outFileList): 

231 if filename.endswith('.txt'): 

232 galfastIn = open(filename, 'rt') 

233 inFits = False 

234 gzFile = False 

235 num_lines = sum(1 for line in open(filename)) 

236 elif filename.endswith('.gz'): 

237 galfastIn = gzip.open(filename, 'rt') 

238 inFits = False 

239 gzFile = True 

240 num_lines = sum(1 for line in gzip.open(filename)) 

241 elif filename.endswith('fits'): 

242 hdulist = fits.open(filename) 

243 galfastIn = hdulist[1].data 

244 num_lines = len(galfastIn) 

245 gzFile = False 

246 inFits = True 

247 

248 if outFile.endswith('.txt'): 

249 fOut = open(outFile, 'wt') 

250 elif outFile.endswith('.gz'): 

251 fOut = gzip.open(outFile, 'wt') 

252 fOut.write('#oID, ra, dec, gall, galb, coordX, coordY, coordZ, sEDName, magNorm, ' +\ 

253 'LSSTugrizy, SDSSugriz, absSDSSr, pmRA, pmDec, vRad, pml, pmb, vRadlb, ' +\ 

254 'vR, vPhi, vZ, FeH, pop, distKpc, ebv, ebvInf\n') 

255 header_length = 0 

256 numChunks = 0 

257 if inFits == False: 

258 galfastDict = self.parseGalfast(galfastIn.readline()) 

259 header_length += 1 

260 header_status = True 

261 while header_status == True: 

262 newLine = galfastIn.readline() 

263 if newLine[0] != '#': 

264 header_status = False 

265 else: 

266 header_length += 1 

267 print('Total objects = %i' % (num_lines - header_length)) 

268 numChunks = ((num_lines-header_length)//chunkSize) + 1 

269 

270 for chunk in range(0,numChunks): 

271 if chunk == numChunks-1: 

272 lastChunkSize = (num_lines - header_length) % chunkSize 

273 readSize = lastChunkSize 

274 else: 

275 readSize = chunkSize 

276 oID = np.arange(readSize*chunk, readSize*(chunk+1)) 

277 if inFits: 

278 starData = galfastIn[readSize*chunk:(readSize*chunk + readSize)] 

279 sDSS = starData.field('SDSSugriz') 

280 gall, galb = np.transpose(starData.field('lb')) 

281 ra, dec = np.transpose(starData.field('radec')) 

282 coordX, coordY, coordZ = np.transpose(starData.field('XYZ')) 

283 DM = starData.field('DM') 

284 absSDSSr = starData.field('absSDSSr') 

285 pop = starData.field('comp') 

286 FeH = starData.field('FeH') 

287 vR, vPhi, vZ = np.transpose(starData.field('vcyl')) 

288 pml, pmb, vRadlb = np.transpose(starData.field('pmlb')) 

289 pmRA, pmDec, vRad = np.transpose(starData.field('pmradec')) 

290 am = starData.field('Am') 

291 amInf = starData.field('AmInf') 

292 sdssPhotoFlags = starData.field('SDSSugrizPhotoFlags') 

293 else: 

294 if gzFile == False: 

295 with open(filename) as t_in: 

296 starData = np.loadtxt(itertools.islice(t_in,((readSize*chunk)+header_length), 

297 ((readSize*(chunk+1))+header_length))) 

298 else: 

299 with gzip.open(filename) as t_in: 

300 starData = np.loadtxt(itertools.islice(t_in,((readSize*chunk)+header_length), 

301 ((readSize*(chunk+1))+header_length))) 

302 starData = np.transpose(starData) 

303 gall = starData[galfastDict['l']] 

304 galb = starData[galfastDict['b']] 

305 ra = starData[galfastDict['ra']] 

306 dec = starData[galfastDict['dec']] 

307 coordX = starData[galfastDict['X']] 

308 coordY = starData[galfastDict['Y']] 

309 coordZ = starData[galfastDict['Z']] 

310 DM = starData[galfastDict['DM']] 

311 absSDSSr = starData[galfastDict['absSDSSr']] 

312 pop = starData[galfastDict['comp']] 

313 FeH = starData[galfastDict['FeH']] 

314 vR = starData[galfastDict['Vr']] 

315 vPhi = starData[galfastDict['Vphi']] 

316 vZ = starData[galfastDict['Vz']] 

317 pml = starData[galfastDict['pml']] 

318 pmb = starData[galfastDict['pmb']] 

319 vRadlb = starData[galfastDict['vRadlb']] 

320 pmRA = starData[galfastDict['pmra']] 

321 pmDec = starData[galfastDict['pmdec']] 

322 vRad = starData[galfastDict['vRad']] 

323 am = starData[galfastDict['Am']] 

324 amInf = starData[galfastDict['AmInf']] 

325 sDSS = np.transpose(starData[galfastDict['SDSSu']:galfastDict['SDSSz']+1]) 

326 sDSSPhotoFlags = starData[galfastDict['SDSSPhotoFlags']] 

327 

328 #End of input, now onto processing and output 

329 sDSSunred = selectStarSED0.deReddenMags(am, sDSS, sdssExtCoeffs) 

330 if readSize == 1: 

331 ra = np.array([ra]) 

332 dec = np.array([dec]) 

333 """ 

334 Info about the following population cuts: 

335 From Zeljko: "This color corresponds to the temperature (roughly spectral type M0) where 

336 Kurucz models become increasingly bad, and thus we switch to empirical SEDs (the problem 

337 is that for M and later stars, the effective surface temperature is low enough for 

338 molecules to form, and their opacity is too complex to easily model, especially TiO)." 

339 """ 

340 mIn = np.where(((pop < 10) | (pop >= 20)) & (sDSSunred[:,2] - sDSSunred[:,3] > 0.59)) 

341 kIn = np.where(((pop < 10) | (pop >= 20)) & (sDSSunred[:,2] - sDSSunred[:,3] <= 0.59)) 

342 hIn = np.where((pop >= 10) & (pop < 15)) 

343 heIn = np.where((pop >= 15) & (pop < 20)) 

344 

345 sEDNameK, magNormK, matchErrorK = selectStarSED0.findSED(listDict['kurucz'], 

346 sDSSunred[kIn], ra[kIn], dec[kIn], 

347 reddening = False, 

348 colors = colorDict['kurucz']) 

349 sEDNameM, magNormM, matchErrorM = selectStarSED0.findSED(listDict['mlt'], 

350 sDSSunred[mIn], ra[mIn], dec[mIn], 

351 reddening = False, 

352 colors = colorDict['mlt']) 

353 sEDNameH, magNormH, matchErrorH = selectStarSED0.findSED(listDict['H'], 

354 sDSSunred[hIn], ra[hIn], dec[hIn], 

355 reddening = False, 

356 colors = colorDict['H']) 

357 sEDNameHE, magNormHE, matchErrorHE = selectStarSED0.findSED(listDict['HE'], 

358 sDSSunred[heIn], 

359 ra[heIn], dec[heIn], 

360 reddening = False, 

361 colors = colorDict['HE']) 

362 chunkNames = np.empty(readSize, dtype = 'S32') 

363 chunkTypes = np.empty(readSize, dtype = 'S8') 

364 chunkMagNorms = np.zeros(readSize) 

365 chunkMatchErrors = np.zeros(readSize) 

366 chunkNames[kIn] = sEDNameK 

367 chunkTypes[kIn] = 'kurucz' 

368 chunkMagNorms[kIn] = magNormK 

369 chunkMatchErrors[kIn] = matchErrorK 

370 chunkNames[mIn] = sEDNameM 

371 chunkTypes[mIn] = 'mlt' 

372 chunkMagNorms[mIn] = magNormM 

373 chunkMatchErrors[mIn] = matchErrorM 

374 chunkNames[hIn] = sEDNameH 

375 chunkTypes[hIn] = 'H' 

376 chunkMagNorms[hIn] = magNormH 

377 chunkMatchErrors[hIn] = matchErrorH 

378 chunkNames[heIn] = sEDNameHE 

379 chunkTypes[heIn] = 'HE' 

380 chunkMagNorms[heIn] = magNormHE 

381 chunkMatchErrors[heIn] = matchErrorHE 

382 lsstMagsUnred = [] 

383 for sedName, sedType, magNorm, matchError in zip(chunkNames.astype(str), 

384 chunkTypes.astype(str), 

385 chunkMagNorms, 

386 chunkMatchErrors): 

387 testSED = Sed() 

388 testSED.setSED(listDict[sedType][positionDict[sedName]].wavelen, 

389 flambda = listDict[sedType][positionDict[sedName]].flambda) 

390 fluxNorm = testSED.calcFluxNorm(magNorm, imSimBand) 

391 testSED.multiplyFluxNorm(fluxNorm) 

392 lsstMagsUnred.append(lsstPhot.magListForSed(testSED)) 

393 #If the extinction value is negative then it will add the reddening back in 

394 lsstMags = selectStarSED0.deReddenMags((-1.0*am), lsstMagsUnred, 

395 lsstExtCoeffs) 

396 distKpc = self.convDMtoKpc(DM) 

397 ebv = am / 2.285 #From Schlafly and Finkbeiner 2011, (ApJ, 737, 103) for sdssr 

398 ebvInf = amInf / 2.285 

399 for line in range(0, readSize): 

400 outFmt = '%i,%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,' +\ 

401 '%3.7f,%3.7f,%s,' +\ 

402 '%3.7f,%3.7f,' +\ 

403 '%3.7f,%3.7f,%3.7f,' +\ 

404 '%3.7f,%3.7f,%3.7f,' +\ 

405 '%3.7f,%3.7f,%3.7f,%3.7f,' +\ 

406 '%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,' +\ 

407 '%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,' +\ 

408 '%3.7f,%i,%3.7f,%3.7f,%3.7f\n' 

409 if readSize == 1: 

410 if inFits == True: 

411 sDSS = sDSS[0] 

412 outDat = (oID, ra[line], dec[line], gall, galb, coordX, 

413 coordY, coordZ, chunkNames, 

414 chunkMagNorms, chunkMatchErrors, 

415 lsstMags[line][0], lsstMags[line][1], lsstMags[line][2], 

416 lsstMags[line][3], lsstMags[line][4], lsstMags[line][5], 

417 sDSS[0], sDSS[1], sDSS[2], sDSS[3], 

418 sDSS[4], absSDSSr, pmRA, pmDec, vRad, 

419 pml, pmb, vRadlb, vR, vPhi, vZ, 

420 FeH, pop, distKpc, ebv, ebvInf) 

421 else: 

422 outDat = (oID[line], ra[line], dec[line], gall[line], galb[line], coordX[line], 

423 coordY[line], coordZ[line], chunkNames[line], 

424 chunkMagNorms[line], chunkMatchErrors[line], 

425 lsstMags[line][0], lsstMags[line][1], lsstMags[line][2], 

426 lsstMags[line][3], lsstMags[line][4], lsstMags[line][5], 

427 sDSS[line][0], sDSS[line][1], sDSS[line][2], sDSS[line][3], 

428 sDSS[line][4], absSDSSr[line], pmRA[line], pmDec[line], vRad[line], 

429 pml[line], pmb[line], vRadlb[line], vR[line], vPhi[line], vZ[line], 

430 FeH[line], pop[line], distKpc[line], ebv[line], ebvInf[line]) 

431 fOut.write(outFmt % outDat) 

432 print('Chunk Num Done = %i out of %i' % (chunk+1, numChunks))