Coverage for python/lsst/sims/catUtils/readGalfast/readGalfast.py : 5%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import print_function
2from builtins import zip
3from builtins import str
4from builtins import range
5from builtins import object
6import os
7import gzip
8import itertools
9import numpy as np
10from astropy.io import fits
12import lsst.utils
13from lsst.sims.photUtils.Sed import Sed
14from lsst.sims.photUtils.Bandpass import Bandpass
15from lsst.sims.catUtils.matchSED import selectStarSED
16from lsst.sims.photUtils import BandpassDict
18__all__ = ["readGalfast"]
20class readGalfast(object):
22 def parseGalfast(self, headerLine):
24 """
25 Use galfast header line to organize input
27 @param [in] headerLine is the first line from a galfast catalog output file
29 @param [out] galfastDict is a dictionary relating parameter name to input column
31 """
33 galfastDict = {}
34 header = headerLine.split(' ')
35 colNo = 0
36 for title in header:
37 if title == 'lb[2]':
38 galfastDict['l'] = colNo
39 colNo += 1
40 galfastDict['b'] = colNo
41 colNo += 1
42 elif title == 'radec[2]':
43 galfastDict['ra'] = colNo
44 colNo += 1
45 galfastDict['dec'] = colNo
46 colNo += 1
47 elif title == 'XYZ[3]':
48 galfastDict['X'] = colNo
49 colNo += 1
50 galfastDict['Y'] = colNo
51 colNo += 1
52 galfastDict['Z'] = colNo
53 colNo += 1
54 elif title == 'DM':
55 galfastDict['DM'] = colNo
56 colNo += 1
57 elif title == 'absSDSSr{alias=M1;alias=absmag;band=SDSSr;}':
58 galfastDict['absSDSSr'] = colNo
59 colNo += 1
60 elif title == 'comp':
61 galfastDict['comp'] = colNo
62 colNo += 1
63 elif title == 'FeH':
64 galfastDict['FeH'] = colNo
65 colNo += 1
66 elif title == 'vcyl[3]':
67 galfastDict['Vr'] = colNo
68 colNo += 1
69 galfastDict['Vphi'] = colNo
70 colNo += 1
71 galfastDict['Vz'] = colNo
72 colNo += 1
73 elif title == 'pmlb[3]':
74 galfastDict['pml'] = colNo
75 colNo += 1
76 galfastDict['pmb'] = colNo
77 colNo += 1
78 galfastDict['vRadlb'] = colNo
79 colNo += 1
80 elif title == 'pmradec[3]':
81 galfastDict['pmra'] = colNo
82 colNo += 1
83 galfastDict['pmdec'] = colNo
84 colNo += 1
85 galfastDict['vRad'] = colNo
86 colNo += 1
87 elif title == 'Am':
88 galfastDict['Am'] = colNo
89 colNo += 1
90 elif title == 'AmInf':
91 galfastDict['AmInf'] = colNo
92 colNo += 1
93 elif title.startswith('SDSSugriz['):
94 bandString = title.split('=')[2]
95 bandString1 = bandString.split(',')
96 for band in bandString1:
97 band = band.rstrip(';}')
98 bandName = band.split(':')[1]
99 galfastDict[bandName] = colNo
100 colNo += 1
101 elif title == 'SDSSugrizPhotoFlags{class=flags;}':
102 galfastDict['SDSSPhotoFlags'] = colNo
103 colNo += 1
104 elif title == '#': pass
105 elif len(title) < 1: pass
106 elif title.isspace(): pass
107 else:
108 raise RuntimeError('*** Unknown field: %s' % (title))
109 return galfastDict
111 def convDMtoKpc(self, DM):
112 """
113 Change from distance modulus to distance in kiloparsecs
115 @param [in] DM is the distance modulus
117 @param [out] distanceKpc is the distance in kiloparsecs
119 """
121 distancePc = 10**((0.2*DM) + 1)
122 distanceKpc = distancePc / 1000.
123 return distanceKpc
125 def loadGalfast(self, filenameList, outFileList, sEDPath = None, kuruczPath = None,
126 mltPath = None, wdPath = None, kuruczSubset = None,
127 mltSubset = None, wdSubset = None, chunkSize = 10000):
128 """
129 This is customized for the outputs we currently need for the purposes of consistent output
130 It will read in a galfast output file and output desired values for database input into a file
132 @param [in] filenameList is a list of the galfast output files that will be loaded and processed.
133 Can process fits, gzipped, or txt output from galfast.
135 @param [in] outFileList is a list of the names of the output files that will be created. If gzipped
136 output is desired simply write the filenames with .gz at the end.
138 @param [in] kuruczPath is a place to specify a path to kurucz SED files
140 @param [in] mltPath is the same as kuruczPath except that it specifies a directory for the
141 mlt SEDs
143 @param [in] wdPath is the same as the previous two except that it specifies a path to a
144 white dwarf SED directory.
146 @param [in] kuruczSubset is a list which provides a subset of the kurucz files within the
147 kurucz folder that one wants to use
149 @param [in] mltSubset is a list which provides a subset of the mlt files within the
150 mlt folder that one wants to use
152 @param [in] wdSubset is a list which provides a subset of the wd files within the
153 wd folder that one wants to use
155 @param [in] chunkSize is the size of chunks of lines to be read from the catalog at one time.
156 """
158 for filename in filenameList:
159 #Make sure input file exists and is readable format before doing anything else
160 if os.path.isfile(filename) == False:
161 raise RuntimeError('*** File does not exist')
163 #Process various possible galfast outputs
164 if filename.endswith(('.txt', '.gz', '.fits')):
165 continue
166 else:
167 raise RuntimeError(str('*** Unsupported File Format in file: ' + str(filename)))
169 #If all files exist and are in proper formats then load seds
171 selectStarSED0 = selectStarSED(kuruczDir=kuruczPath,
172 mltDir=mltPath,
173 wdDir=wdPath)
175 if kuruczSubset is None:
176 kuruczList = selectStarSED0.loadKuruczSEDs()
177 else:
178 kuruczList = selectStarSED0.loadKuruczSEDs(subset = kuruczSubset)
180 #Only need one dictionary since none of the names overlap
181 positionDict = {}
182 for kNum, kuruczSED in enumerate(kuruczList):
183 positionDict[kuruczSED.name] = kNum
185 if mltSubset is None:
186 mltList = selectStarSED0.loadmltSEDs()
187 else:
188 mltList = selectStarSED0.loadmltSEDs(subset = mltSubset)
190 for mltNum, mltSED in enumerate(mltList):
191 positionDict[mltSED.name] = mltNum
193 if wdSubset is None:
194 wdListH, wdListHE = selectStarSED0.loadwdSEDs()
195 else:
196 wdListH, wdListHE = selectStarSED0.loadwdSEDs(subset = wdSubset)
198 for hNum, hSED in enumerate(wdListH):
199 positionDict[hSED.name] = hNum
201 for heNum, heSED in enumerate(wdListHE):
202 positionDict[heSED.name] = heNum
204 #For adding/subtracting extinction when calculating colors
205 #Numbers below come from Schlafly and Finkbeiner (2011) (ApJ, 737, 103)
206 #normalized by SDSS r mag value
207 sdssExtCoeffs = [1.8551, 1.4455, 1.0, 0.7431, 0.5527]
208 lsstExtCoeffs = [1.8140, 1.4166, 0.9947, 0.7370, 0.5790, 0.4761]
210 sdssPhot = BandpassDict.loadTotalBandpassesFromFiles(['u','g','r','i','z'],
211 bandpassDir = os.path.join(lsst.utils.getPackageDir('throughputs'),
212 'sdss'),
213 bandpassRoot = 'sdss_')
215 #Load Bandpasses for LSST colors to get colors from matched SEDs
216 lsstFilterList = ('u', 'g', 'r', 'i', 'z', 'y')
217 lsstPhot = BandpassDict.loadTotalBandpassesFromFiles(lsstFilterList)
218 imSimBand = Bandpass()
219 imSimBand.imsimBandpass()
221 #Calculate colors and add them to the SED objects
222 kuruczColors = selectStarSED0.calcBasicColors(kuruczList, sdssPhot)
223 mltColors = selectStarSED0.calcBasicColors(mltList, sdssPhot)
224 hColors = selectStarSED0.calcBasicColors(wdListH, sdssPhot)
225 heColors = selectStarSED0.calcBasicColors(wdListHE, sdssPhot)
227 listDict = {'kurucz':kuruczList, 'mlt':mltList, 'H':wdListH, 'HE':wdListHE}
228 colorDict = {'kurucz':kuruczColors, 'mlt':mltColors, 'H':hColors, 'HE':heColors}
230 for filename, outFile in zip(filenameList, outFileList):
231 if filename.endswith('.txt'):
232 galfastIn = open(filename, 'rt')
233 inFits = False
234 gzFile = False
235 num_lines = sum(1 for line in open(filename))
236 elif filename.endswith('.gz'):
237 galfastIn = gzip.open(filename, 'rt')
238 inFits = False
239 gzFile = True
240 num_lines = sum(1 for line in gzip.open(filename))
241 elif filename.endswith('fits'):
242 hdulist = fits.open(filename)
243 galfastIn = hdulist[1].data
244 num_lines = len(galfastIn)
245 gzFile = False
246 inFits = True
248 if outFile.endswith('.txt'):
249 fOut = open(outFile, 'wt')
250 elif outFile.endswith('.gz'):
251 fOut = gzip.open(outFile, 'wt')
252 fOut.write('#oID, ra, dec, gall, galb, coordX, coordY, coordZ, sEDName, magNorm, ' +\
253 'LSSTugrizy, SDSSugriz, absSDSSr, pmRA, pmDec, vRad, pml, pmb, vRadlb, ' +\
254 'vR, vPhi, vZ, FeH, pop, distKpc, ebv, ebvInf\n')
255 header_length = 0
256 numChunks = 0
257 if inFits == False:
258 galfastDict = self.parseGalfast(galfastIn.readline())
259 header_length += 1
260 header_status = True
261 while header_status == True:
262 newLine = galfastIn.readline()
263 if newLine[0] != '#':
264 header_status = False
265 else:
266 header_length += 1
267 print('Total objects = %i' % (num_lines - header_length))
268 numChunks = ((num_lines-header_length)//chunkSize) + 1
270 for chunk in range(0,numChunks):
271 if chunk == numChunks-1:
272 lastChunkSize = (num_lines - header_length) % chunkSize
273 readSize = lastChunkSize
274 else:
275 readSize = chunkSize
276 oID = np.arange(readSize*chunk, readSize*(chunk+1))
277 if inFits:
278 starData = galfastIn[readSize*chunk:(readSize*chunk + readSize)]
279 sDSS = starData.field('SDSSugriz')
280 gall, galb = np.transpose(starData.field('lb'))
281 ra, dec = np.transpose(starData.field('radec'))
282 coordX, coordY, coordZ = np.transpose(starData.field('XYZ'))
283 DM = starData.field('DM')
284 absSDSSr = starData.field('absSDSSr')
285 pop = starData.field('comp')
286 FeH = starData.field('FeH')
287 vR, vPhi, vZ = np.transpose(starData.field('vcyl'))
288 pml, pmb, vRadlb = np.transpose(starData.field('pmlb'))
289 pmRA, pmDec, vRad = np.transpose(starData.field('pmradec'))
290 am = starData.field('Am')
291 amInf = starData.field('AmInf')
292 sdssPhotoFlags = starData.field('SDSSugrizPhotoFlags')
293 else:
294 if gzFile == False:
295 with open(filename) as t_in:
296 starData = np.loadtxt(itertools.islice(t_in,((readSize*chunk)+header_length),
297 ((readSize*(chunk+1))+header_length)))
298 else:
299 with gzip.open(filename) as t_in:
300 starData = np.loadtxt(itertools.islice(t_in,((readSize*chunk)+header_length),
301 ((readSize*(chunk+1))+header_length)))
302 starData = np.transpose(starData)
303 gall = starData[galfastDict['l']]
304 galb = starData[galfastDict['b']]
305 ra = starData[galfastDict['ra']]
306 dec = starData[galfastDict['dec']]
307 coordX = starData[galfastDict['X']]
308 coordY = starData[galfastDict['Y']]
309 coordZ = starData[galfastDict['Z']]
310 DM = starData[galfastDict['DM']]
311 absSDSSr = starData[galfastDict['absSDSSr']]
312 pop = starData[galfastDict['comp']]
313 FeH = starData[galfastDict['FeH']]
314 vR = starData[galfastDict['Vr']]
315 vPhi = starData[galfastDict['Vphi']]
316 vZ = starData[galfastDict['Vz']]
317 pml = starData[galfastDict['pml']]
318 pmb = starData[galfastDict['pmb']]
319 vRadlb = starData[galfastDict['vRadlb']]
320 pmRA = starData[galfastDict['pmra']]
321 pmDec = starData[galfastDict['pmdec']]
322 vRad = starData[galfastDict['vRad']]
323 am = starData[galfastDict['Am']]
324 amInf = starData[galfastDict['AmInf']]
325 sDSS = np.transpose(starData[galfastDict['SDSSu']:galfastDict['SDSSz']+1])
326 sDSSPhotoFlags = starData[galfastDict['SDSSPhotoFlags']]
328 #End of input, now onto processing and output
329 sDSSunred = selectStarSED0.deReddenMags(am, sDSS, sdssExtCoeffs)
330 if readSize == 1:
331 ra = np.array([ra])
332 dec = np.array([dec])
333 """
334 Info about the following population cuts:
335 From Zeljko: "This color corresponds to the temperature (roughly spectral type M0) where
336 Kurucz models become increasingly bad, and thus we switch to empirical SEDs (the problem
337 is that for M and later stars, the effective surface temperature is low enough for
338 molecules to form, and their opacity is too complex to easily model, especially TiO)."
339 """
340 mIn = np.where(((pop < 10) | (pop >= 20)) & (sDSSunred[:,2] - sDSSunred[:,3] > 0.59))
341 kIn = np.where(((pop < 10) | (pop >= 20)) & (sDSSunred[:,2] - sDSSunred[:,3] <= 0.59))
342 hIn = np.where((pop >= 10) & (pop < 15))
343 heIn = np.where((pop >= 15) & (pop < 20))
345 sEDNameK, magNormK, matchErrorK = selectStarSED0.findSED(listDict['kurucz'],
346 sDSSunred[kIn], ra[kIn], dec[kIn],
347 reddening = False,
348 colors = colorDict['kurucz'])
349 sEDNameM, magNormM, matchErrorM = selectStarSED0.findSED(listDict['mlt'],
350 sDSSunred[mIn], ra[mIn], dec[mIn],
351 reddening = False,
352 colors = colorDict['mlt'])
353 sEDNameH, magNormH, matchErrorH = selectStarSED0.findSED(listDict['H'],
354 sDSSunred[hIn], ra[hIn], dec[hIn],
355 reddening = False,
356 colors = colorDict['H'])
357 sEDNameHE, magNormHE, matchErrorHE = selectStarSED0.findSED(listDict['HE'],
358 sDSSunred[heIn],
359 ra[heIn], dec[heIn],
360 reddening = False,
361 colors = colorDict['HE'])
362 chunkNames = np.empty(readSize, dtype = 'S32')
363 chunkTypes = np.empty(readSize, dtype = 'S8')
364 chunkMagNorms = np.zeros(readSize)
365 chunkMatchErrors = np.zeros(readSize)
366 chunkNames[kIn] = sEDNameK
367 chunkTypes[kIn] = 'kurucz'
368 chunkMagNorms[kIn] = magNormK
369 chunkMatchErrors[kIn] = matchErrorK
370 chunkNames[mIn] = sEDNameM
371 chunkTypes[mIn] = 'mlt'
372 chunkMagNorms[mIn] = magNormM
373 chunkMatchErrors[mIn] = matchErrorM
374 chunkNames[hIn] = sEDNameH
375 chunkTypes[hIn] = 'H'
376 chunkMagNorms[hIn] = magNormH
377 chunkMatchErrors[hIn] = matchErrorH
378 chunkNames[heIn] = sEDNameHE
379 chunkTypes[heIn] = 'HE'
380 chunkMagNorms[heIn] = magNormHE
381 chunkMatchErrors[heIn] = matchErrorHE
382 lsstMagsUnred = []
383 for sedName, sedType, magNorm, matchError in zip(chunkNames.astype(str),
384 chunkTypes.astype(str),
385 chunkMagNorms,
386 chunkMatchErrors):
387 testSED = Sed()
388 testSED.setSED(listDict[sedType][positionDict[sedName]].wavelen,
389 flambda = listDict[sedType][positionDict[sedName]].flambda)
390 fluxNorm = testSED.calcFluxNorm(magNorm, imSimBand)
391 testSED.multiplyFluxNorm(fluxNorm)
392 lsstMagsUnred.append(lsstPhot.magListForSed(testSED))
393 #If the extinction value is negative then it will add the reddening back in
394 lsstMags = selectStarSED0.deReddenMags((-1.0*am), lsstMagsUnred,
395 lsstExtCoeffs)
396 distKpc = self.convDMtoKpc(DM)
397 ebv = am / 2.285 #From Schlafly and Finkbeiner 2011, (ApJ, 737, 103) for sdssr
398 ebvInf = amInf / 2.285
399 for line in range(0, readSize):
400 outFmt = '%i,%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,' +\
401 '%3.7f,%3.7f,%s,' +\
402 '%3.7f,%3.7f,' +\
403 '%3.7f,%3.7f,%3.7f,' +\
404 '%3.7f,%3.7f,%3.7f,' +\
405 '%3.7f,%3.7f,%3.7f,%3.7f,' +\
406 '%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,' +\
407 '%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,%3.7f,' +\
408 '%3.7f,%i,%3.7f,%3.7f,%3.7f\n'
409 if readSize == 1:
410 if inFits == True:
411 sDSS = sDSS[0]
412 outDat = (oID, ra[line], dec[line], gall, galb, coordX,
413 coordY, coordZ, chunkNames,
414 chunkMagNorms, chunkMatchErrors,
415 lsstMags[line][0], lsstMags[line][1], lsstMags[line][2],
416 lsstMags[line][3], lsstMags[line][4], lsstMags[line][5],
417 sDSS[0], sDSS[1], sDSS[2], sDSS[3],
418 sDSS[4], absSDSSr, pmRA, pmDec, vRad,
419 pml, pmb, vRadlb, vR, vPhi, vZ,
420 FeH, pop, distKpc, ebv, ebvInf)
421 else:
422 outDat = (oID[line], ra[line], dec[line], gall[line], galb[line], coordX[line],
423 coordY[line], coordZ[line], chunkNames[line],
424 chunkMagNorms[line], chunkMatchErrors[line],
425 lsstMags[line][0], lsstMags[line][1], lsstMags[line][2],
426 lsstMags[line][3], lsstMags[line][4], lsstMags[line][5],
427 sDSS[line][0], sDSS[line][1], sDSS[line][2], sDSS[line][3],
428 sDSS[line][4], absSDSSr[line], pmRA[line], pmDec[line], vRad[line],
429 pml[line], pmb[line], vRadlb[line], vR[line], vPhi[line], vZ[line],
430 FeH[line], pop[line], distKpc[line], ebv[line], ebvInf[line])
431 fOut.write(outFmt % outDat)
432 print('Chunk Num Done = %i out of %i' % (chunk+1, numChunks))