Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

from __future__ import absolute_import, division, print_function 

 

__all__ = ["getIndexPath", "getConfigFromEnvironment", "AstrometryNetCatalog", "generateCache"] 

 

from builtins import zip 

from builtins import range 

from builtins import object 

import os 

 

import numpy as np 

from astropy.io import fits 

 

import lsst.utils 

from lsst.log import Log 

from .astrometry_net import MultiIndex, healpixDistance 

from .astrometryNetDataConfig import AstrometryNetDataConfig 

 

 

def getIndexPath(fn): 

"""!Get the path to the specified astrometry.net index file 

 

No effort is made to confirm that the file exists, so it may be used to 

locate the path to a non-existent file (e.g., to write). 

 

@param[in] fn path to index file; if relative, then relative to 

astrometry_net_data if that product is setup, else relative to the 

current working directory 

@return the absolute path to the index file 

""" 

if os.path.isabs(fn): 

return fn 

try: 

andir = lsst.utils.getPackageDir('astrometry_net_data') 

except Exception: 

# Relative to cwd 

return os.path.abspath(fn) 

return os.path.join(andir, fn) 

 

 

def getConfigFromEnvironment(): 

"""Find the config file from the environment 

 

The andConfig.py file is in the astrometry_net_data directory. 

""" 

try: 

anDir = lsst.utils.getPackageDir('astrometry_net_data') 

except Exception: 

anDir = os.getcwd() 

andConfigPath = "andConfig.py" 

if not os.path.exists(andConfigPath): 

raise RuntimeError("Unable to find andConfig.py in the current directory. " 

"Did you forget to setup astrometry_net_data?") 

else: 

andConfigPath = os.path.join(anDir, "andConfig.py") 

if not os.path.exists(andConfigPath): 

raise RuntimeError("Unable to find andConfig.py in astrometry_net_data directory %s" % (anDir,)) 

 

andConfig = AstrometryNetDataConfig() 

andConfig.load(andConfigPath) 

return andConfig 

 

 

class MultiIndexCache(object): 

"""A wrapper for the multiindex_t, which only reads the data when it 

needs to 

 

The MultiIndexCache may be instantiated directly, or via the 

'fromFilenameList' class method, which loads it from a list of filenames. 

""" 

 

def __init__(self, filenameList, healpix, nside): 

"""!Constructor 

 

@param filenameList List of filenames; first is the multiindex, then 

follows the individual index files 

@param healpix Healpix number 

@param nside Healpix nside 

""" 

if len(filenameList) < 2: 

raise RuntimeError("Insufficient filenames provided for multiindex (%s): expected >= 2" % 

(filenameList,)) 

self._filenameList = filenameList 

self._healpix = int(healpix) 

self._nside = int(nside) 

self._mi = None 

self._loaded = False 

self.log = Log.getDefaultLogger() 

 

@classmethod 

def fromFilenameList(cls, filenameList): 

"""Construct from a list of filenames 

 

The list of filenames should contain the multiindex filename first, 

then the individual index filenames. The healpix and nside are 

determined by reading the indices, so this is not very efficient. 

""" 

self = cls(filenameList, 0, 0) 

self.reload() 

healpixes = set(self[i].healpix for i in range(len(self))) 

nsides = set(self[i].hpnside for i in range(len(self))) 

assert len(healpixes) == 1 

assert len(nsides) == 1 

self._healpix = healpixes.pop() 

self._nside = nsides.pop() 

return self 

 

def read(self): 

"""Read the indices""" 

if self._mi is not None: 

return 

fn = getIndexPath(self._filenameList[0]) 

if not os.path.exists(fn): 

raise RuntimeError( 

"Unable to get filename for astrometry star file %s" % (self._filenameList[0],)) 

self._mi = MultiIndex(fn) 

if self._mi is None: 

# Can't proceed at all without stars 

raise RuntimeError('Failed to read stars from astrometry multiindex filename "%s"' % fn) 

for i, fn in enumerate(self._filenameList[1:]): 

if fn is None: 

self.log.debug('Unable to find index part of multiindex %s', fn) 

continue 

fn = getIndexPath(fn) 

if not os.path.exists(fn): 

self.log.warn("Unable to get filename for astrometry index %s", fn) 

continue 

self.log.debug('Reading index from multiindex file "%s"', fn) 

self._mi.addIndex(fn, False) 

ind = self._mi[i] 

self.log.debug(' index %i, hp %i (nside %i), nstars %i, nquads %i', 

ind.indexid, ind.healpix, ind.hpnside, ind.nstars, ind.nquads) 

 

def reload(self): 

"""Reload the indices.""" 

if self._loaded: 

return 

if self._mi is None: 

self.read() 

else: 

self._mi.reload() 

self._loaded = True 

 

def unload(self): 

"""Unload the indices""" 

if not self._loaded: 

return 

self._mi.unload() 

self._loaded = False 

 

def isWithinRange(self, coord, distance): 

"""!Is the index within range of the provided coordinates? 

 

@param coord ICRS coordinate to check (lsst.afw.geom.SpherPoint) 

@param distance Angular distance (lsst.afw.geom.Angle) 

""" 

return (self._healpix == -1 or healpixDistance(self._healpix, self._nside, coord) <= distance) 

 

def __getitem__(self, i): 

self.reload() 

return self._mi[i] 

 

def __len__(self): 

return len(self._filenameList) - 1 # The first is the multiindex; the rest are the indices 

 

def __iter__(self): 

self.reload() 

return iter(self._mi) 

 

 

class AstrometryNetCatalog(object): 

"""An interface to an astrometry.net catalog 

 

Behaves like a list of MultiIndexCache (or multiindex_t). 

 

These should usually be constructed using the 'fromEnvironment' 

class method, which wraps the 'fromIndexFiles' and 'fromCache' 

alternative class methods. 

""" 

_cacheFilename = "andCache.fits" 

 

def __init__(self, andConfig): 

"""!Constructor 

 

@param andConfig Configuration (an AstrometryNetDataConfig) 

""" 

self.config = andConfig 

cacheName = getIndexPath(self._cacheFilename) 

if self.config.allowCache and os.path.exists(cacheName): 

self._initFromCache(cacheName) 

else: 

self._initFromIndexFiles(self.config) 

 

def _initFromIndexFiles(self, andConfig): 

"""Initialise from the index files in an AstrometryNetDataConfig""" 

indexFiles = list(zip(andConfig.indexFiles, andConfig.indexFiles)) + andConfig.multiIndexFiles 

self._multiInds = [MultiIndexCache.fromFilenameList(fnList) for fnList in indexFiles] 

 

def writeCache(self): 

"""Write a cache file 

 

The cache file is a FITS file with all the required information to 

build the AstrometryNetCatalog quickly. The first table extension 

contains a row for each multiindex, storing the healpix and nside 

values. The second table extension contains a row for each filename 

in all the multiindexes. The two may be JOINed through the 'id' 

column. 

""" 

outName = getIndexPath(self._cacheFilename) 

numFilenames = sum(len(ind._filenameList) for ind in self._multiInds) 

maxLength = max(len(fn) for ind in self._multiInds for fn in ind._filenameList) + 1 

 

# First table 

first = fits.BinTableHDU.from_columns([fits.Column(name="id", format="K"), 

fits.Column(name="healpix", format="K"), 

fits.Column(name="nside", format="K"), 

], nrows=len(self._multiInds)) 

first.data.field("id")[:] = np.arange(len(self._multiInds), dtype=int) 

first.data.field("healpix")[:] = np.array([ind._healpix for ind in self._multiInds]) 

first.data.field("nside")[:] = np.array([ind._nside for ind in self._multiInds]) 

 

# Second table 

second = fits.BinTableHDU.from_columns([fits.Column(name="id", format="K"), 

fits.Column(name="filename", format="%dA" % (maxLength)), 

], nrows=numFilenames) 

ident = second.data.field("id") 

filenames = second.data.field("filename") 

i = 0 

for j, ind in enumerate(self._multiInds): 

for fn in ind._filenameList: 

ident[i] = j 

filenames[i] = fn 

i += 1 

 

fits.HDUList([fits.PrimaryHDU(), first, second]).writeto(outName, overwrite=True) 

 

def _initFromCache(self, filename): 

"""Initialise from a cache file 

 

Ingest the cache file written by the 'writeCache' method and 

use that to quickly instantiate the AstrometryNetCatalog. 

""" 

with fits.open(filename) as hduList: 

first = hduList[1].data 

second = hduList[2].data 

 

# first JOIN second USING(id) 

filenames = {i: [] for i in first.field("id")} 

for id2, fn in zip(second.field("id"), second.field("filename")): 

filenames[id2].append(fn) 

self._multiInds = [MultiIndexCache(filenames[i], hp, nside) for i, hp, nside in 

zip(first.field("id"), first.field("healpix"), first.field("nside"))] 

 

# Check for consistency 

cacheFiles = set(second.field("filename")) 

configFiles = set(sum(self.config.multiIndexFiles, []) + self.config.indexFiles) 

assert(cacheFiles == configFiles) 

 

def __getitem__(self, ii): 

return self._multiInds[ii] 

 

def __iter__(self): 

return iter(self._multiInds) 

 

def __len__(self): 

return len(self._multiInds) 

 

 

def generateCache(andConfig=None): 

"""Generate a cache file""" 

if andConfig is None: 

andConfig = getConfigFromEnvironment() 

catalog = AstrometryNetCatalog(andConfig) 

try: 

for index in catalog: 

index.reload() 

catalog.writeCache() 

finally: 

for index in catalog: 

index.unload()