Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

from __future__ import absolute_import, division, print_function 

 

__all__ = ["getIndexPath", "getConfigFromEnvironment", "AstrometryNetCatalog", "generateCache"] 

 

from builtins import zip 

from builtins import range 

from builtins import object 

import os 

 

import numpy as np 

import pyfits 

 

import lsst.utils 

from lsst.log import Log 

from .astrometry_net import MultiIndex, healpixDistance 

from .astrometryNetDataConfig import AstrometryNetDataConfig 

 

 

def getIndexPath(fn): 

"""!Get the path to the specified astrometry.net index file 

 

No effort is made to confirm that the file exists, so it may be used to locate the 

path to a non-existent file (e.g., to write). 

 

@param[in] fn path to index file; if relative, then relative to astrometry_net_data 

if that product is setup, else relative to the current working directory 

@return the absolute path to the index file 

""" 

29 ↛ 30line 29 didn't jump to line 30, because the condition on line 29 was never true if os.path.isabs(fn): 

return fn 

try: 

andir = lsst.utils.getPackageDir('astrometry_net_data') 

except: 

# Relative to cwd 

return os.path.abspath(fn) 

return os.path.join(andir, fn) 

 

 

def getConfigFromEnvironment(): 

"""Find the config file from the environment 

 

The andConfig.py file is in the astrometry_net_data directory. 

""" 

try: 

anDir = lsst.utils.getPackageDir('astrometry_net_data') 

except: 

anDir = os.getcwd() 

andConfigPath = "andConfig.py" 

if not os.path.exists(andConfigPath): 

raise RuntimeError("Unable to find andConfig.py in the current directory. " 

"Did you forget to setup astrometry_net_data?") 

else: 

andConfigPath = os.path.join(anDir, "andConfig.py") 

54 ↛ 55line 54 didn't jump to line 55, because the condition on line 54 was never true if not os.path.exists(andConfigPath): 

raise RuntimeError("Unable to find andConfig.py in astrometry_net_data directory %s" % (anDir,)) 

 

andConfig = AstrometryNetDataConfig() 

andConfig.load(andConfigPath) 

return andConfig 

 

 

class MultiIndexCache(object): 

"""A wrapper for the multiindex_t, which only reads the data when it needs to 

 

The MultiIndexCache may be instantiated directly, or via the 'fromFilenameList' 

class method, which loads it from a list of filenames. 

""" 

 

def __init__(self, filenameList, healpix, nside): 

"""!Constructor 

 

@param filenameList List of filenames; first is the multiindex, then 

follows the individual index files 

@param healpix Healpix number 

@param nside Healpix nside 

""" 

77 ↛ 78line 77 didn't jump to line 78, because the condition on line 77 was never true if len(filenameList) < 2: 

raise RuntimeError("Insufficient filenames provided for multiindex (%s): expected >= 2" % 

(filenameList,)) 

self._filenameList = filenameList 

self._healpix = int(healpix) 

self._nside = int(nside) 

self._mi = None 

self._loaded = False 

self.log = Log.getDefaultLogger() 

 

@classmethod 

def fromFilenameList(cls, filenameList): 

"""Construct from a list of filenames 

 

The list of filenames should contain the multiindex filename first, 

then the individual index filenames. The healpix and nside are 

determined by reading the indices, so this is not very efficient. 

""" 

self = cls(filenameList, 0, 0) 

self.reload() 

healpixes = set(self[i].healpix for i in range(len(self))) 

nsides = set(self[i].hpnside for i in range(len(self))) 

assert len(healpixes) == 1 

assert len(nsides) == 1 

self._healpix = healpixes.pop() 

self._nside = nsides.pop() 

return self 

 

def read(self): 

"""Read the indices""" 

107 ↛ 108line 107 didn't jump to line 108, because the condition on line 107 was never true if self._mi is not None: 

return 

fn = getIndexPath(self._filenameList[0]) 

110 ↛ 111line 110 didn't jump to line 111, because the condition on line 110 was never true if not os.path.exists(fn): 

raise RuntimeError( 

"Unable to get filename for astrometry star file %s" % (self._filenameList[0],)) 

self._mi = MultiIndex(fn) 

114 ↛ 116line 114 didn't jump to line 116, because the condition on line 114 was never true if self._mi is None: 

# Can't proceed at all without stars 

raise RuntimeError('Failed to read stars from astrometry multiindex filename "%s"' % fn) 

for i, fn in enumerate(self._filenameList[1:]): 

118 ↛ 119line 118 didn't jump to line 119, because the condition on line 118 was never true if fn is None: 

self.log.debug('Unable to find index part of multiindex %s', fn) 

continue 

fn = getIndexPath(fn) 

122 ↛ 123line 122 didn't jump to line 123, because the condition on line 122 was never true if not os.path.exists(fn): 

self.log.warn("Unable to get filename for astrometry index %s", fn) 

continue 

self.log.debug('Reading index from multiindex file "%s"', fn) 

self._mi.addIndex(fn, False) 

ind = self._mi[i] 

self.log.debug(' index %i, hp %i (nside %i), nstars %i, nquads %i', 

ind.indexid, ind.healpix, ind.hpnside, ind.nstars, ind.nquads) 

 

def reload(self): 

"""Reload the indices.""" 

if self._loaded: 

return 

if self._mi is None: 

self.read() 

else: 

self._mi.reload() 

self._loaded = True 

 

def unload(self): 

"""Unload the indices""" 

143 ↛ 144line 143 didn't jump to line 144, because the condition on line 143 was never true if not self._loaded: 

return 

self._mi.unload() 

self._loaded = False 

 

def isWithinRange(self, coord, distance): 

"""!Is the index within range of the provided coordinates? 

 

@param coord ICRS coordinate to check (lsst.afw.geom.SpherPoint) 

@param distance Angular distance (lsst.afw.geom.Angle) 

""" 

return (self._healpix == -1 or healpixDistance(self._healpix, self._nside, coord) <= distance) 

 

def __getitem__(self, i): 

self.reload() 

return self._mi[i] 

 

def __len__(self): 

return len(self._filenameList) - 1 # The first is the multiindex; the rest are the indices 

 

def __iter__(self): 

self.reload() 

return iter(self._mi) 

 

 

class AstrometryNetCatalog(object): 

"""An interface to an astrometry.net catalog 

 

Behaves like a list of MultiIndexCache (or multiindex_t). 

 

These should usually be constructed using the 'fromEnvironment' 

class method, which wraps the 'fromIndexFiles' and 'fromCache' 

alternative class methods. 

""" 

_cacheFilename = "andCache.fits" 

 

def __init__(self, andConfig): 

"""!Constructor 

 

@param andConfig Configuration (an AstrometryNetDataConfig) 

""" 

self.config = andConfig 

cacheName = getIndexPath(self._cacheFilename) 

if self.config.allowCache and os.path.exists(cacheName): 

self._initFromCache(cacheName) 

else: 

self._initFromIndexFiles(self.config) 

 

def _initFromIndexFiles(self, andConfig): 

"""Initialise from the index files in an AstrometryNetDataConfig""" 

indexFiles = list(zip(andConfig.indexFiles, andConfig.indexFiles)) + andConfig.multiIndexFiles 

self._multiInds = [MultiIndexCache.fromFilenameList(fnList) for fnList in indexFiles] 

 

def writeCache(self): 

"""Write a cache file 

 

The cache file is a FITS file with all the required information to build the 

AstrometryNetCatalog quickly. The first table extension contains a row for each multiindex, 

storing the healpix and nside values. The second table extension contains a row 

for each filename in all the multiindexes. The two may be JOINed through the 

'id' column. 

""" 

outName = getIndexPath(self._cacheFilename) 

numFilenames = sum(len(ind._filenameList) for ind in self._multiInds) 

maxLength = max(len(fn) for ind in self._multiInds for fn in ind._filenameList) + 1 

 

# First table 

first = pyfits.new_table([pyfits.Column(name="id", format="K"), 

pyfits.Column(name="healpix", format="K"), 

pyfits.Column(name="nside", format="K"), 

], nrows=len(self._multiInds)) 

first.data.field("id")[:] = np.arange(len(self._multiInds), dtype=int) 

first.data.field("healpix")[:] = np.array([ind._healpix for ind in self._multiInds]) 

first.data.field("nside")[:] = np.array([ind._nside for ind in self._multiInds]) 

 

# Second table 

second = pyfits.new_table([pyfits.Column(name="id", format="K"), 

pyfits.Column(name="filename", format="%dA" % (maxLength)), 

], nrows=numFilenames) 

ident = second.data.field("id") 

filenames = second.data.field("filename") 

i = 0 

for j, ind in enumerate(self._multiInds): 

for fn in ind._filenameList: 

ident[i] = j 

filenames[i] = fn 

i += 1 

 

pyfits.HDUList([pyfits.PrimaryHDU(), first, second]).writeto(outName, clobber=True) 

 

def _initFromCache(self, filename): 

"""Initialise from a cache file 

 

Ingest the cache file written by the 'writeCache' method and 

use that to quickly instantiate the AstrometryNetCatalog. 

""" 

with pyfits.open(filename) as hduList: 

first = hduList[1].data 

second = hduList[2].data 

 

# first JOIN second USING(id) 

filenames = {i: [] for i in first.field("id")} 

for id2, fn in zip(second.field("id"), second.field("filename")): 

filenames[id2].append(fn) 

self._multiInds = [MultiIndexCache(filenames[i], hp, nside) for i, hp, nside in 

zip(first.field("id"), first.field("healpix"), first.field("nside"))] 

 

# Check for consistency 

cacheFiles = set(second.field("filename")) 

configFiles = set(sum(self.config.multiIndexFiles, []) + self.config.indexFiles) 

assert(cacheFiles == configFiles) 

 

def __getitem__(self, ii): 

return self._multiInds[ii] 

 

def __iter__(self): 

return iter(self._multiInds) 

 

def __len__(self): 

return len(self._multiInds) 

 

 

def generateCache(andConfig=None): 

"""Generate a cache file""" 

267 ↛ 268line 267 didn't jump to line 268, because the condition on line 267 was never true if andConfig is None: 

andConfig = getConfigFromEnvironment() 

catalog = AstrometryNetCatalog(andConfig) 

try: 

for index in catalog: 

index.reload() 

catalog.writeCache() 

finally: 

for index in catalog: 

index.unload()