Coverage for python/lsst/meas/astrom/astrometry.py: 16%

145 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-10-18 01:50 -0700

1# 

2# LSST Data Management System 

3# Copyright 2008-2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23__all__ = ["AstrometryConfig", "AstrometryTask"] 

24 

25import numpy as np 

26from astropy import units 

27import scipy.stats 

28 

29import lsst.pex.config as pexConfig 

30import lsst.pipe.base as pipeBase 

31from lsst.utils.timer import timeMethod 

32from .ref_match import RefMatchTask, RefMatchConfig 

33from .fitTanSipWcs import FitTanSipWcsTask 

34from .display import displayAstrometry 

35 

36 

37class AstrometryConfig(RefMatchConfig): 

38 """Config for AstrometryTask. 

39 """ 

40 wcsFitter = pexConfig.ConfigurableField( 

41 target=FitTanSipWcsTask, 

42 doc="WCS fitter", 

43 ) 

44 forceKnownWcs = pexConfig.Field( 

45 dtype=bool, 

46 doc="If True then load reference objects and match sources but do not fit a WCS; " 

47 "this simply controls whether 'run' calls 'solve' or 'loadAndMatch'", 

48 default=False, 

49 ) 

50 maxIter = pexConfig.RangeField( 

51 doc="maximum number of iterations of match sources and fit WCS" 

52 "ignored if not fitting a WCS", 

53 dtype=int, 

54 default=3, 

55 min=1, 

56 ) 

57 minMatchDistanceArcSec = pexConfig.RangeField( 

58 doc="the match distance below which further iteration is pointless (arcsec); " 

59 "ignored if not fitting a WCS", 

60 dtype=float, 

61 default=0.001, 

62 min=0, 

63 ) 

64 maxMeanDistanceArcsec = pexConfig.RangeField( 

65 doc="Maximum mean on-sky distance (in arcsec) between matched source and rerference " 

66 "objects post-fit. A mean distance greater than this threshold raises a TaskError " 

67 "and the WCS fit is considered a failure. The default is set to the maximum tolerated " 

68 "by the external global calibration (e.g. jointcal) step for conceivable recovery. " 

69 "Appropriate value will be dataset and workflow dependent.", 

70 dtype=float, 

71 default=0.5, 

72 min=0, 

73 ) 

74 doMagnitudeOutlierRejection = pexConfig.Field( 

75 dtype=bool, 

76 doc=("If True then a rough zeropoint will be computed from matched sources " 

77 "and outliers will be rejected in the iterations."), 

78 default=False, 

79 ) 

80 magnitudeOutlierRejectionNSigma = pexConfig.Field( 

81 dtype=float, 

82 doc=("Number of sigma (measured from the distribution) in magnitude " 

83 "for a potential reference/source match to be rejected during " 

84 "iteration."), 

85 default=3.0, 

86 ) 

87 

88 def setDefaults(self): 

89 # Override the default source selector for astrometry tasks 

90 self.sourceFluxType = "Ap" 

91 

92 self.sourceSelector.name = "matcher" 

93 self.sourceSelector["matcher"].sourceFluxType = self.sourceFluxType 

94 

95 # Note that if the matcher is MatchOptimisticBTask, then the default 

96 # should be self.sourceSelector['matcher'].excludePixelFlags = False 

97 # However, there is no way to do this automatically. 

98 

99 

100class AstrometryTask(RefMatchTask): 

101 """Match an input source catalog with objects from a reference catalog and 

102 solve for the WCS. 

103 

104 This task is broken into two main subasks: matching and WCS fitting which 

105 are very interactive. The matching here can be considered in part a first 

106 pass WCS fitter due to the fitter's sensitivity to outliers. 

107 

108 Parameters 

109 ---------- 

110 refObjLoader : `lsst.meas.algorithms.ReferenceLoader` 

111 A reference object loader object 

112 schema : `lsst.afw.table.Schema` 

113 Used to set "calib_astrometry_used" flag in output source catalog. 

114 **kwargs 

115 additional keyword arguments for pipe_base 

116 `lsst.pipe.base.Task.__init__` 

117 """ 

118 ConfigClass = AstrometryConfig 

119 _DefaultName = "astrometricSolver" 

120 

121 def __init__(self, refObjLoader, schema=None, **kwargs): 

122 RefMatchTask.__init__(self, refObjLoader, **kwargs) 

123 

124 if schema is not None: 

125 self.usedKey = schema.addField("calib_astrometry_used", type="Flag", 

126 doc="set if source was used in astrometric calibration") 

127 else: 

128 self.usedKey = None 

129 

130 self.makeSubtask("wcsFitter") 

131 

132 @timeMethod 

133 def run(self, sourceCat, exposure): 

134 """Load reference objects, match sources and optionally fit a WCS. 

135 

136 This is a thin layer around solve or loadAndMatch, depending on 

137 config.forceKnownWcs. 

138 

139 Parameters 

140 ---------- 

141 exposure : `lsst.afw.image.Exposure` 

142 exposure whose WCS is to be fit 

143 The following are read only: 

144 

145 - bbox 

146 - filter (may be unset) 

147 - detector (if wcs is pure tangent; may be absent) 

148 

149 The following are updated: 

150 

151 - wcs (the initial value is used as an initial guess, and is 

152 required) 

153 

154 sourceCat : `lsst.afw.table.SourceCatalog` 

155 catalog of sources detected on the exposure 

156 

157 Returns 

158 ------- 

159 result : `lsst.pipe.base.Struct` 

160 with these fields: 

161 

162 - ``refCat`` : reference object catalog of objects that overlap the 

163 exposure (with some margin) (`lsst.afw.table.SimpleCatalog`). 

164 - ``matches`` : astrometric matches 

165 (`list` of `lsst.afw.table.ReferenceMatch`). 

166 - ``scatterOnSky`` : median on-sky separation between reference 

167 objects and sources in "matches" 

168 (`lsst.afw.geom.Angle`) or `None` if config.forceKnownWcs True 

169 - ``matchMeta`` : metadata needed to unpersist matches 

170 (`lsst.daf.base.PropertyList`) 

171 """ 

172 if self.refObjLoader is None: 

173 raise RuntimeError("Running matcher task with no refObjLoader set in __init__ or setRefObjLoader") 

174 if self.config.forceKnownWcs: 

175 res = self.loadAndMatch(exposure=exposure, sourceCat=sourceCat) 

176 res.scatterOnSky = None 

177 else: 

178 res = self.solve(exposure=exposure, sourceCat=sourceCat) 

179 return res 

180 

181 @timeMethod 

182 def solve(self, exposure, sourceCat): 

183 """Load reference objects overlapping an exposure, match to sources and 

184 fit a WCS 

185 

186 Returns 

187 ------- 

188 result : `lsst.pipe.base.Struct` 

189 Result struct with components: 

190 

191 - ``refCat`` : reference object catalog of objects that overlap the 

192 exposure (with some margin) (`lsst::afw::table::SimpleCatalog`). 

193 - ``matches`` : astrometric matches 

194 (`list` of `lsst.afw.table.ReferenceMatch`). 

195 - ``scatterOnSky`` : median on-sky separation between reference 

196 objects and sources in "matches" (`lsst.geom.Angle`) 

197 - ``matchMeta`` : metadata needed to unpersist matches 

198 (`lsst.daf.base.PropertyList`) 

199 

200 Raises 

201 ------ 

202 TaskError 

203 If the measured mean on-sky distance between the matched source and 

204 reference objects is greater than 

205 ``self.config.maxMeanDistanceArcsec``. 

206 

207 Notes 

208 ----- 

209 ignores config.forceKnownWcs 

210 """ 

211 if self.refObjLoader is None: 

212 raise RuntimeError("Running matcher task with no refObjLoader set in __init__ or setRefObjLoader") 

213 import lsstDebug 

214 debug = lsstDebug.Info(__name__) 

215 

216 expMd = self._getExposureMetadata(exposure) 

217 

218 sourceSelection = self.sourceSelector.run(sourceCat) 

219 

220 self.log.info("Purged %d sources, leaving %d good sources", 

221 len(sourceCat) - len(sourceSelection.sourceCat), 

222 len(sourceSelection.sourceCat)) 

223 

224 loadRes = self.refObjLoader.loadPixelBox( 

225 bbox=expMd.bbox, 

226 wcs=expMd.wcs, 

227 filterName=expMd.filterName, 

228 epoch=expMd.epoch, 

229 ) 

230 

231 refSelection = self.referenceSelector.run(loadRes.refCat) 

232 

233 matchMeta = self.refObjLoader.getMetadataBox( 

234 bbox=expMd.bbox, 

235 wcs=expMd.wcs, 

236 filterName=expMd.filterName, 

237 epoch=expMd.epoch, 

238 ) 

239 

240 if debug.display: 

241 frame = int(debug.frame) 

242 displayAstrometry( 

243 refCat=refSelection.sourceCat, 

244 sourceCat=sourceSelection.sourceCat, 

245 exposure=exposure, 

246 bbox=expMd.bbox, 

247 frame=frame, 

248 title="Reference catalog", 

249 ) 

250 

251 res = None 

252 wcs = expMd.wcs 

253 match_tolerance = None 

254 fitFailed = False 

255 for i in range(self.config.maxIter): 

256 if not fitFailed: 

257 iterNum = i + 1 

258 try: 

259 tryRes = self._matchAndFitWcs( 

260 refCat=refSelection.sourceCat, 

261 sourceCat=sourceCat, 

262 goodSourceCat=sourceSelection.sourceCat, 

263 refFluxField=loadRes.fluxField, 

264 bbox=expMd.bbox, 

265 wcs=wcs, 

266 exposure=exposure, 

267 match_tolerance=match_tolerance, 

268 ) 

269 except Exception as e: 

270 # If we have had a succeessful iteration then use that; 

271 # otherwise fail. 

272 if i > 0: 

273 self.log.info("Fit WCS iter %d failed; using previous iteration: %s", iterNum, e) 

274 iterNum -= 1 

275 break 

276 else: 

277 self.log.info("Fit WCS iter %d failed: %s" % (iterNum, e)) 

278 fitFailed = True 

279 

280 if not fitFailed: 

281 match_tolerance = tryRes.match_tolerance 

282 tryMatchDist = self._computeMatchStatsOnSky(tryRes.matches) 

283 self.log.debug( 

284 "Match and fit WCS iteration %d: found %d matches with on-sky distance mean and " 

285 "scatter = %0.3f +- %0.3f arcsec; max match distance = %0.3f arcsec", 

286 iterNum, len(tryRes.matches), tryMatchDist.distMean.asArcseconds(), 

287 tryMatchDist.distStdDev.asArcseconds(), tryMatchDist.maxMatchDist.asArcseconds()) 

288 

289 maxMatchDist = tryMatchDist.maxMatchDist 

290 res = tryRes 

291 wcs = res.wcs 

292 if maxMatchDist.asArcseconds() < self.config.minMatchDistanceArcSec: 

293 self.log.debug( 

294 "Max match distance = %0.3f arcsec < %0.3f = config.minMatchDistanceArcSec; " 

295 "that's good enough", 

296 maxMatchDist.asArcseconds(), self.config.minMatchDistanceArcSec) 

297 break 

298 match_tolerance.maxMatchDist = maxMatchDist 

299 

300 if not fitFailed: 

301 self.log.info("Matched and fit WCS in %d iterations; " 

302 "found %d matches with mean and scatter = %0.3f +- %0.3f arcsec" % 

303 (iterNum, len(tryRes.matches), tryMatchDist.distMean.asArcseconds(), 

304 tryMatchDist.distStdDev.asArcseconds())) 

305 if tryMatchDist.distMean.asArcseconds() > self.config.maxMeanDistanceArcsec: 

306 self.log.info("Assigning as a fit failure: mean on-sky distance = %0.3f arcsec > %0.3f " 

307 "(maxMeanDistanceArcsec)" % (tryMatchDist.distMean.asArcseconds(), 

308 self.config.maxMeanDistanceArcsec)) 

309 fitFailed = True 

310 

311 if fitFailed: 

312 self.log.warning("WCS fit failed. Setting exposure's WCS to None and coord_ra & coord_dec " 

313 "cols in sourceCat to nan.") 

314 sourceCat["coord_ra"] = np.nan 

315 sourceCat["coord_dec"] = np.nan 

316 exposure.setWcs(None) 

317 matches = None 

318 scatterOnSky = None 

319 else: 

320 for m in res.matches: 

321 if self.usedKey: 

322 m.second.set(self.usedKey, True) 

323 exposure.setWcs(res.wcs) 

324 matches = res.matches 

325 scatterOnSky = res.scatterOnSky 

326 

327 # If fitter converged, record the scatter in the exposure metadata 

328 # even if the fit was deemed a failure according to the value of 

329 # the maxMeanDistanceArcsec config. 

330 if res is not None: 

331 md = exposure.getMetadata() 

332 md['SFM_ASTROM_OFFSET_MEAN'] = tryMatchDist.distMean.asArcseconds() 

333 md['SFM_ASTROM_OFFSET_STD'] = tryMatchDist.distStdDev.asArcseconds() 

334 

335 return pipeBase.Struct( 

336 refCat=refSelection.sourceCat, 

337 matches=matches, 

338 scatterOnSky=scatterOnSky, 

339 matchMeta=matchMeta, 

340 ) 

341 

342 @timeMethod 

343 def _matchAndFitWcs(self, refCat, sourceCat, goodSourceCat, refFluxField, bbox, wcs, match_tolerance, 

344 exposure=None): 

345 """Match sources to reference objects and fit a WCS. 

346 

347 Parameters 

348 ---------- 

349 refCat : `lsst.afw.table.SimpleCatalog` 

350 catalog of reference objects 

351 sourceCat : `lsst.afw.table.SourceCatalog` 

352 catalog of sources detected on the exposure 

353 goodSourceCat : `lsst.afw.table.SourceCatalog` 

354 catalog of down-selected good sources detected on the exposure 

355 refFluxField : 'str' 

356 field of refCat to use for flux 

357 bbox : `lsst.geom.Box2I` 

358 bounding box of exposure 

359 wcs : `lsst.afw.geom.SkyWcs` 

360 initial guess for WCS of exposure 

361 match_tolerance : `lsst.meas.astrom.MatchTolerance` 

362 a MatchTolerance object (or None) specifying 

363 internal tolerances to the matcher. See the MatchTolerance 

364 definition in the respective matcher for the class definition. 

365 exposure : `lsst.afw.image.Exposure` 

366 exposure whose WCS is to be fit, or None; used only for the debug 

367 display. 

368 

369 Returns 

370 ------- 

371 result : `lsst.pipe.base.Struct` 

372 Result struct with components: 

373 

374 - ``matches``: astrometric matches 

375 (`list` of `lsst.afw.table.ReferenceMatch`). 

376 - ``wcs``: the fit WCS (lsst.afw.geom.SkyWcs). 

377 - ``scatterOnSky`` : median on-sky separation between reference 

378 objects and sources in "matches" (`lsst.afw.geom.Angle`). 

379 """ 

380 import lsstDebug 

381 debug = lsstDebug.Info(__name__) 

382 

383 sourceFluxField = "slot_%sFlux_instFlux" % (self.config.sourceFluxType) 

384 

385 matchRes = self.matcher.matchObjectsToSources( 

386 refCat=refCat, 

387 sourceCat=goodSourceCat, 

388 wcs=wcs, 

389 sourceFluxField=sourceFluxField, 

390 refFluxField=refFluxField, 

391 match_tolerance=match_tolerance, 

392 ) 

393 self.log.debug("Found %s matches", len(matchRes.matches)) 

394 if debug.display: 

395 frame = int(debug.frame) 

396 displayAstrometry( 

397 refCat=refCat, 

398 sourceCat=matchRes.usableSourceCat, 

399 matches=matchRes.matches, 

400 exposure=exposure, 

401 bbox=bbox, 

402 frame=frame + 1, 

403 title="Initial WCS", 

404 ) 

405 

406 if self.config.doMagnitudeOutlierRejection: 

407 matches = self._removeMagnitudeOutliers(sourceFluxField, refFluxField, matchRes.matches) 

408 else: 

409 matches = matchRes.matches 

410 

411 self.log.debug("Fitting WCS") 

412 fitRes = self.wcsFitter.fitWcs( 

413 matches=matches, 

414 initWcs=wcs, 

415 bbox=bbox, 

416 refCat=refCat, 

417 sourceCat=sourceCat, 

418 exposure=exposure, 

419 ) 

420 fitWcs = fitRes.wcs 

421 scatterOnSky = fitRes.scatterOnSky 

422 if debug.display: 

423 frame = int(debug.frame) 

424 displayAstrometry( 

425 refCat=refCat, 

426 sourceCat=matchRes.usableSourceCat, 

427 matches=matches, 

428 exposure=exposure, 

429 bbox=bbox, 

430 frame=frame + 2, 

431 title="Fit TAN-SIP WCS", 

432 ) 

433 

434 return pipeBase.Struct( 

435 matches=matches, 

436 wcs=fitWcs, 

437 scatterOnSky=scatterOnSky, 

438 match_tolerance=matchRes.match_tolerance, 

439 ) 

440 

441 def _removeMagnitudeOutliers(self, sourceFluxField, refFluxField, matchesIn): 

442 """Remove magnitude outliers, computing a simple zeropoint. 

443 

444 Parameters 

445 ---------- 

446 sourceFluxField : `str` 

447 Field in source catalog for instrumental fluxes. 

448 refFluxField : `str` 

449 Field in reference catalog for fluxes (nJy). 

450 matchesIn : `list` [`lsst.afw.table.ReferenceMatch`] 

451 List of source/reference matches input 

452 

453 Returns 

454 ------- 

455 matchesOut : `list` [`lsst.afw.table.ReferenceMatch`] 

456 List of source/reference matches with magnitude 

457 outliers removed. 

458 """ 

459 nMatch = len(matchesIn) 

460 sourceMag = np.zeros(nMatch) 

461 refMag = np.zeros(nMatch) 

462 for i, match in enumerate(matchesIn): 

463 sourceMag[i] = -2.5*np.log10(match[1][sourceFluxField]) 

464 refMag[i] = (match[0][refFluxField]*units.nJy).to_value(units.ABmag) 

465 

466 deltaMag = refMag - sourceMag 

467 # Protect against negative fluxes and nans in the reference catalog. 

468 goodDelta, = np.where(np.isfinite(deltaMag)) 

469 zp = np.median(deltaMag[goodDelta]) 

470 # Use median absolute deviation (MAD) for zpSigma. 

471 # Also require a minimum scatter to prevent floating-point errors from 

472 # rejecting objects in zero-noise tests. 

473 zpSigma = np.clip(scipy.stats.median_abs_deviation(deltaMag[goodDelta], scale='normal'), 

474 1e-3, 

475 None) 

476 

477 self.log.info("Rough zeropoint from astrometry matches is %.4f +/- %.4f.", 

478 zp, zpSigma) 

479 

480 goodStars = goodDelta[(np.abs(deltaMag[goodDelta] - zp) 

481 <= self.config.magnitudeOutlierRejectionNSigma*zpSigma)] 

482 

483 nOutlier = nMatch - goodStars.size 

484 self.log.info("Removed %d magnitude outliers out of %d total astrometry matches.", 

485 nOutlier, nMatch) 

486 

487 matchesOut = [] 

488 for matchInd in goodStars: 

489 matchesOut.append(matchesIn[matchInd]) 

490 

491 return matchesOut