Coverage for python/lsst/summit/utils/astrometry/anet.py: 19%

178 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-22 12:00 +0000

1# This file is part of summit_utils. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import os 

23import shutil 

24import subprocess 

25import tempfile 

26import numpy as np 

27from astropy.io import fits 

28import time 

29import uuid 

30import warnings 

31 

32from dataclasses import dataclass 

33from functools import cached_property 

34 

35import lsst.geom as geom 

36 

37from .utils import headerToWcs 

38 

39__all__ = ['AstrometryNetResult', 'CommandLineSolver', 'OnlineSolver'] 

40 

41 

42@dataclass(frozen=True) 

43class AstrometryNetResult: 

44 """Minimal wrapper class to construct and return results from the command 

45 line fitter. 

46 

47 Constructs a DM wcs from the output of the command line fitter, and 

48 calculates the plate scale and astrometric scatter measurement in arcsec 

49 and pixels. 

50 

51 Parameters 

52 ---------- 

53 wcsFile : `str` 

54 The path to the .wcs file from the fit. 

55 corrFile : `str`, optional 

56 The path to the .corr file from the fit. 

57 """ 

58 wcsFile: str 

59 corrFile: str = None 

60 

61 def __post_init__(self): 

62 # touch these properties to ensure the files needed to calculate them 

63 # are read immediately, in case they are deleted from temp 

64 self.wcs 

65 self.rmsErrorArsec 

66 

67 @cached_property 

68 def wcs(self): 

69 with fits.open(self.wcsFile) as f: 

70 header = f[0].header 

71 return headerToWcs(header) 

72 

73 @cached_property 

74 def plateScale(self): 

75 return self.wcs.getPixelScale().asArcseconds() 

76 

77 @cached_property 

78 def meanSqErr(self): 

79 if not self.corrFile: 

80 return None 

81 

82 try: 

83 with fits.open(self.corrFile) as f: 

84 data = f[1].data 

85 

86 meanSqErr = 0.0 

87 count = 0 

88 for i in range(data.shape[0]): 

89 row = data[i] 

90 count += 1 

91 error = (row[0] - row[4])**2 + (row[1] - row[5])**2 # square error in pixels 

92 error *= row[10] # multiply by weight 

93 meanSqErr += error 

94 meanSqErr /= count # divide by number of stars 

95 return meanSqErr 

96 except Exception as e: 

97 print(f'Failed for calculate astrometric scatter: {repr(e)}') 

98 

99 @cached_property 

100 def rmsErrorPixels(self): 

101 return np.sqrt(self.meanSqErr) 

102 

103 @cached_property 

104 def rmsErrorArsec(self): 

105 return self.rmsErrorPixels * self.plateScale 

106 

107 

108class CommandLineSolver(): 

109 """An interface for the solve-field command line tool from astrometry.net. 

110 

111 Parameters 

112 ---------- 

113 indexFilePath : `str` 

114 The path to the index files. Do not include the 4100 or 4200 etc. in 

115 the path. This is selected automatically depending on the `isWideField` 

116 flag when calling `run()`. 

117 checkInParallel : `bool`, optional 

118 Do the checks in parallel. Default is True. 

119 timeout : `float`, optional 

120 The timeout for the solve-field command. Default is 300 seconds. 

121 binary : `str`, optional 

122 The path to the solve-field binary. Default is 'solve-field', i.e. it 

123 is assumed to be on the path. 

124 """ 

125 def __init__(self, 

126 indexFilePath=None, 

127 checkInParallel=True, 

128 timeout=300, 

129 binary='solve-field', 

130 fluxSlot='base_CircularApertureFlux_3_0_instFlux', 

131 ): 

132 self.indexFilePath = indexFilePath 

133 self.checkInParallel = checkInParallel 

134 self.timeout = timeout 

135 self.binary = binary 

136 self.fluxSlot = fluxSlot 

137 if not shutil.which(binary): 

138 raise RuntimeError(f"Could not find {binary} in path, please install 'solve-field' and either" 

139 " put it on your PATH or specify the full path to it in the 'binary' argument") 

140 

141 def _writeConfigFile(self, wide, useGaia): 

142 """Write a temporary config file for astrometry.net. 

143 

144 Parameters 

145 ---------- 

146 wide : `bool` 

147 Is this a wide field image? Used to select the 4100 vs 4200 dir in 

148 the index file path. Ignored if ``useGaia`` is ``True``. 

149 useGaia : `bool` 

150 Use the 5200 Gaia catalog? If ``True``, ``wide`` is ignored. 

151 

152 Returns 

153 ------- 

154 filename : `str` 

155 The filename to which the config file was written. 

156 """ 

157 fileSet = '4100' if wide else '4200' 

158 fileSet = '5200/LITE' if useGaia else fileSet 

159 indexFileDir = os.path.join(self.indexFilePath, fileSet) 

160 if not os.path.isdir(indexFileDir): 

161 raise RuntimeError(f"No index files found at {self.indexFilePath}, in {indexFileDir} (you need a" 

162 " 4100 dir for wide field and 4200 dir for narrow field images).") 

163 

164 lines = [] 

165 if self.checkInParallel: 

166 lines.append('inparallel') 

167 

168 lines.append(f"cpulimit {self.timeout}") 

169 lines.append(f"add_path {indexFileDir}") 

170 lines.append("autoindex") 

171 filename = tempfile.mktemp(suffix='.cfg') 

172 with open(filename, 'w') as f: 

173 f.writelines(line + '\n' for line in lines) 

174 return filename 

175 

176 def _writeFitsTable(self, sourceCat): 

177 """Write the source table to a FITS file and return the filename. 

178 

179 Parameters 

180 ---------- 

181 sourceCat : `lsst.afw.table.SourceCatalog` 

182 The source catalog to write to a FITS file for the solver. 

183 

184 Returns 

185 ------- 

186 filename : `str` 

187 The filename to which the catalog was written. 

188 """ 

189 fluxArray = sourceCat[self.fluxSlot] 

190 fluxFinite = np.logical_and(np.isfinite(fluxArray), fluxArray > 0) 

191 fluxArray = fluxArray[fluxFinite] 

192 indices = np.argsort(fluxArray) 

193 x = sourceCat.getColumnView().getX()[fluxFinite] 

194 y = sourceCat.getColumnView().getY()[fluxFinite] 

195 fluxArray = fluxArray[indices][::-1] # brightest finite flux 

196 xArray = x[indices][::-1] 

197 yArray = y[indices][::-1] 

198 x = fits.Column(name='X', format='D', array=xArray) 

199 y = fits.Column(name='Y', format='D', array=yArray) 

200 flux = fits.Column(name='FLUX', format='D', array=fluxArray) 

201 print(f' of which {len(fluxArray)} made it into the fit') 

202 hdu = fits.BinTableHDU.from_columns([flux, x, y]) 

203 

204 filename = tempfile.mktemp(suffix='.fits') 

205 hdu.writeto(filename) 

206 return filename 

207 

208 # try to keep this call sig and the defaults as similar as possible 

209 # to the run method on the OnlineSolver 

210 def run(self, exp, sourceCat, isWideField, *, 

211 useGaia=False, 

212 percentageScaleError=10, 

213 radius=None, 

214 silent=True): 

215 """Get the astrometric solution for an image using astrometry.net using 

216 the binary ``solve-field`` and a set of index files. 

217 

218 Parameters 

219 ---------- 

220 exp : `lsst.afw.image.Exposure` 

221 The input exposure. Only used for its wcs and its dimensions. 

222 sourceCat : `lsst.afw.table.SourceCatalog` 

223 The detected source catalog for the exposure. One produced by a 

224 default run of CharacterizeImageTask is suitable. 

225 isWideField : `bool` 

226 Is this a wide field image? Used to select the correct index files. 

227 Ignored if ``useGaia`` is ``True``. 

228 useGaia : `bool` 

229 Use the Gaia 5200/LITE index files? If set, ``isWideField`` is 

230 ignored. 

231 percentageScaleError : `float`, optional 

232 The percentage scale error to allow in the astrometric solution. 

233 radius : `float`, optional 

234 The search radius from the nominal wcs in degrees. 

235 silent : `bool`, optional 

236 Swallow the output from the command line? The solver is *very* 

237 chatty, so this is recommended. 

238 

239 Returns 

240 ------- 

241 result : `AstrometryNetResult` or `None` 

242 The result of the fit. If the fit was successful, the result will 

243 contain a valid DM wcs, a scatter in arcseconds and a scatter in 

244 pixels. If the fit failed, ``None`` is returned. 

245 """ 

246 wcs = exp.getWcs() 

247 if not wcs: 

248 raise ValueError("No WCS in exposure") 

249 

250 configFile = self._writeConfigFile(wide=isWideField, useGaia=useGaia) 

251 print(f'Fitting image with {len(sourceCat)} sources', end='') 

252 fitsFile = self._writeFitsTable(sourceCat) 

253 

254 plateScale = wcs.getPixelScale().asArcseconds() 

255 scaleMin = plateScale*(1 - percentageScaleError/100) 

256 scaleMax = plateScale*(1 + percentageScaleError/100) 

257 

258 ra, dec = wcs.getSkyOrigin() 

259 

260 # do not use tempfile.TemporaryDirectory() because it must not exist, 

261 # it is made by the solve-field binary and barfs if it exists already! 

262 mainTempDir = tempfile.gettempdir() 

263 tempDirSuffix = str(uuid.uuid1()).split('-')[0] 

264 tempDir = os.path.join(mainTempDir, tempDirSuffix) 

265 

266 cmd = (f"{self.binary} {fitsFile} " # the data 

267 f"--width {exp.getWidth()} " # image dimensions 

268 f"--height {exp.getHeight()} " # image dimensions 

269 f"-3 {ra.asDegrees()} " 

270 f"-4 {dec.asDegrees()} " 

271 f"-5 {radius if radius else 180} " 

272 "-X X -Y Y -v -z 2 -t 2 " # the parts of the bintable to use 

273 f"--scale-low {scaleMin:.3f} " # the scale range 

274 f"--scale-high {scaleMax:.3f} " # the scale range 

275 f"--scale-units arcsecperpix " 

276 f"--crpix-x {wcs.getPixelOrigin()[0]} " # set the pixel origin 

277 f"--crpix-y {wcs.getPixelOrigin()[1]} " # set the pixel origin 

278 f"--config {configFile} " 

279 f"-D {tempDir} " 

280 "--no-plots " # don't make plots 

281 "--overwrite " # shouldn't matter as we're using temp files 

282 ) 

283 

284 t0 = time.time() 

285 with open(os.devnull, 'w') as devnull: 

286 result = subprocess.run(cmd, shell=True, check=True, stdout=devnull if silent else None) 

287 t1 = time.time() 

288 

289 if result.returncode == 0: 

290 print(f"Fitting code ran in {(t1-t0):.2f} seconds") 

291 # output template is /tmpdirname/fitstempname + various suffixes 

292 # for each obj 

293 basename = os.path.basename(fitsFile).removesuffix('.fits') 

294 outputTemplate = os.path.join(tempDir, basename) 

295 wcsFile = outputTemplate + '.wcs' 

296 corrFile = outputTemplate + '.corr' 

297 

298 if not os.path.exists(wcsFile): 

299 print("but failed to find a solution.") 

300 return None 

301 

302 result = AstrometryNetResult(wcsFile, corrFile) 

303 return result 

304 else: 

305 print("Fit failed") 

306 return None 

307 

308 

309class OnlineSolver(): 

310 """A class to solve an image using the Astrometry.net online service. 

311 """ 

312 

313 def __init__(self): 

314 # import seems to spew warnings even if the required key is present 

315 # so we swallow them, and raise on init if the key is missing 

316 with warnings.catch_warnings(): 

317 warnings.simplefilter("ignore") 

318 from astroquery.astrometry_net import AstrometryNet 

319 

320 self.apiKey = self.getApiKey() # raises if not present so do first 

321 self.adn = AstrometryNet() 

322 self.adn.api_key = self.apiKey 

323 

324 @staticmethod 

325 def getApiKey(): 

326 """Get the astrometry.net API key if possible. 

327 

328 Raises a RuntimeError if it isn't found. 

329 

330 Returns 

331 ------- 

332 apiKey : str 

333 The astrometry.net API key, if present. 

334 

335 Raises 

336 ------ 

337 RuntimeError 

338 Raised if the ASTROMETRY_NET_API_KEY is not set. 

339 """ 

340 try: 

341 return os.environ['ASTROMETRY_NET_API_KEY'] 

342 except KeyError as e: 

343 msg = "No AstrometryNet API key found. Sign up and get one, set it to $ASTROMETRY_NET_API_KEY" 

344 raise RuntimeError(msg) from e 

345 

346 # try to keep this call sig and the defaults as similar as possible 

347 # to the run method on the CommandLineSolver 

348 def run(self, exp, sourceCat, *, percentageScaleError=10, radius=None, scaleEstimate=None): 

349 """Get the astrometric solution for an image using the astrometry.net 

350 online solver. 

351 

352 Parameters 

353 ---------- 

354 exp : `lsst.afw.image.Exposure` 

355 The input exposure. Only used for its wcs. 

356 sourceCat : `lsst.afw.table.SourceCatalog` 

357 The detected source catalog for the exposure. One produced by a 

358 default run of CharacterizeImageTask is suitable. 

359 percentageScaleError : `float`, optional 

360 The percentage scale error to allow in the astrometric solution. 

361 radius : `float`, optional 

362 The search radius from the nominal wcs in degrees. 

363 scaleEstimate : `float`, optional 

364 An estimate of the scale in arcseconds per pixel. Only used if 

365 (and required when) the exposure has no wcs. 

366 

367 Returns 

368 ------- 

369 result : `dict` or `None` 

370 The results of the fit, with the following keys, or ``None`` if 

371 the fit failed: 

372 ``nominalRa`` : `lsst.geom.Angle` 

373 The nominal ra from the exposure's boresight. 

374 ``nominalDec`` : `lsst.geom.Angle` 

375 The nominal dec from the exposure's boresight. 

376 ``calculatedRa`` : `lsst.geom.Angle` 

377 The fitted ra. 

378 ``calculatedDec`` : `lsst.geom.Angle` 

379 The fitted dec. 

380 ``deltaRa`` : `lsst.geom.Angle`, 

381 The change in ra, as an Angle. 

382 ``deltaDec`` : `lsst.geom.Angle`, 

383 The change in dec, as an Angle. 

384 ``deltaRaArcsec`` : `float`` 

385 The change in ra in arcseconds, as a float. 

386 ``deltaDecArcsec`` : `float` 

387 The change in dec in arcseconds, as a float. 

388 ``astrometry_net_wcs_header`` : `dict` 

389 The fitted wcs, as a header dict. 

390 """ 

391 nominalWcs = exp.getWcs() 

392 if nominalWcs is not None: 

393 ra, dec = nominalWcs.getSkyOrigin() 

394 scaleEstimate = nominalWcs.getPixelScale().asArcseconds() 

395 else: 

396 print('Trying to process image with None wcs - good luck!') 

397 vi = exp.getInfo().getVisitInfo() 

398 ra, dec = vi.boresightRaDec 

399 if np.isnan(ra.asDegrees()) or np.isnan(dec.asDegrees()): 

400 raise RuntimeError('Exposure has no wcs and did not find nominal ra/dec in visitInfo') 

401 

402 if not scaleEstimate: # must either have a wcs or provide via kwarg 

403 raise RuntimeError('Got no kwarg for scaleEstimate and failed to find one in the nominal wcs.') 

404 

405 image_height, image_width = exp.image.array.shape 

406 scale_units = 'arcsecperpix' 

407 scale_type = 'ev' # ev means submit estimate and % error 

408 scale_err = percentageScaleError # error as percentage 

409 center_ra = ra.asDegrees() 

410 center_dec = dec.asDegrees() 

411 radius = radius if radius else 180 # degrees 

412 try: 

413 wcs_header = self.adn.solve_from_source_list(sourceCat['base_SdssCentroid_x'], 

414 sourceCat['base_SdssCentroid_y'], 

415 image_width, image_height, 

416 scale_units=scale_units, 

417 scale_type=scale_type, 

418 scale_est=scaleEstimate, 

419 scale_err=scale_err, 

420 center_ra=center_ra, 

421 center_dec=center_dec, 

422 radius=radius, 

423 crpix_center=True, # the CRPIX is always the center 

424 solve_timeout=240) 

425 except RuntimeError: 

426 print('Failed to find a solution') 

427 return None 

428 

429 print('Finished solving!') 

430 

431 nominalRa, nominalDec = exp.getInfo().getVisitInfo().getBoresightRaDec() 

432 

433 if 'CRVAL1' not in wcs_header: 

434 raise RuntimeError("Astrometric fit failed.") 

435 calculatedRa = geom.Angle(wcs_header['CRVAL1'], geom.degrees) 

436 calculatedDec = geom.Angle(wcs_header['CRVAL2'], geom.degrees) 

437 

438 deltaRa = geom.Angle(wcs_header['CRVAL1'] - nominalRa.asDegrees(), geom.degrees) 

439 deltaDec = geom.Angle(wcs_header['CRVAL2'] - nominalDec.asDegrees(), geom.degrees) 

440 

441 # TODO: DM-37213 change this to return an AstrometryNetResult class 

442 # like the CommandLineSolver does. 

443 

444 result = {'nominalRa': nominalRa, 

445 'nominalDec': nominalDec, 

446 'calculatedRa': calculatedRa, 

447 'calculatedDec': calculatedDec, 

448 'deltaRa': deltaRa, 

449 'deltaDec': deltaDec, 

450 'deltaRaArcsec': deltaRa.asArcseconds(), 

451 'deltaDecArcsec': deltaDec.asArcseconds(), 

452 'astrometry_net_wcs_header': wcs_header, 

453 'nSources': len(sourceCat), 

454 } 

455 

456 return result