Coverage for python/lsst/meas/astrom/astrometry.py: 19%

145 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-08-05 02:33 +0000

1# 

2# LSST Data Management System 

3# Copyright 2008-2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23__all__ = ["AstrometryConfig", "AstrometryTask"] 

24 

25import numpy as np 

26from astropy import units 

27import scipy.stats 

28 

29import lsst.pex.config as pexConfig 

30import lsst.pipe.base as pipeBase 

31from lsst.utils.timer import timeMethod 

32from .ref_match import RefMatchTask, RefMatchConfig 

33from .fitTanSipWcs import FitTanSipWcsTask 

34from .display import displayAstrometry 

35 

36 

37class AstrometryConfig(RefMatchConfig): 

38 """Config for AstrometryTask. 

39 """ 

40 wcsFitter = pexConfig.ConfigurableField( 

41 target=FitTanSipWcsTask, 

42 doc="WCS fitter", 

43 ) 

44 forceKnownWcs = pexConfig.Field( 

45 dtype=bool, 

46 doc="If True then load reference objects and match sources but do not fit a WCS; " 

47 "this simply controls whether 'run' calls 'solve' or 'loadAndMatch'", 

48 default=False, 

49 ) 

50 maxIter = pexConfig.RangeField( 

51 doc="maximum number of iterations of match sources and fit WCS" 

52 "ignored if not fitting a WCS", 

53 dtype=int, 

54 default=3, 

55 min=1, 

56 ) 

57 minMatchDistanceArcSec = pexConfig.RangeField( 

58 doc="the match distance below which further iteration is pointless (arcsec); " 

59 "ignored if not fitting a WCS", 

60 dtype=float, 

61 default=0.001, 

62 min=0, 

63 ) 

64 maxMeanDistanceArcsec = pexConfig.RangeField( 

65 doc="Maximum mean on-sky distance (in arcsec) between matched source and rerference " 

66 "objects post-fit. A mean distance greater than this threshold raises a TaskError " 

67 "and the WCS fit is considered a failure. The default is set to the maximum tolerated " 

68 "by the external global calibration (e.g. jointcal) step for conceivable recovery. " 

69 "Appropriate value will be dataset and workflow dependent.", 

70 dtype=float, 

71 default=0.5, 

72 min=0, 

73 ) 

74 doMagnitudeOutlierRejection = pexConfig.Field( 

75 dtype=bool, 

76 doc=("If True then a rough zeropoint will be computed from matched sources " 

77 "and outliers will be rejected in the iterations."), 

78 default=False, 

79 ) 

80 magnitudeOutlierRejectionNSigma = pexConfig.Field( 

81 dtype=float, 

82 doc=("Number of sigma (measured from the distribution) in magnitude " 

83 "for a potential reference/source match to be rejected during " 

84 "iteration."), 

85 default=3.0, 

86 ) 

87 

88 def setDefaults(self): 

89 # Override the default source selector for astrometry tasks 

90 self.sourceFluxType = "Ap" 

91 

92 self.sourceSelector.name = "matcher" 

93 self.sourceSelector["matcher"].sourceFluxType = self.sourceFluxType 

94 

95 # Note that if the matcher is MatchOptimisticBTask, then the default 

96 # should be self.sourceSelector['matcher'].excludePixelFlags = False 

97 # However, there is no way to do this automatically. 

98 

99 

100class AstrometryTask(RefMatchTask): 

101 """Match an input source catalog with objects from a reference catalog and 

102 solve for the WCS. 

103 

104 This task is broken into two main subasks: matching and WCS fitting which 

105 are very interactive. The matching here can be considered in part a first 

106 pass WCS fitter due to the fitter's sensitivity to outliers. 

107 

108 Parameters 

109 ---------- 

110 refObjLoader : `lsst.meas.algorithms.ReferenceLoader` 

111 A reference object loader object; gen3 pipeline tasks will pass `None` 

112 and call `setRefObjLoader` in `runQuantum`. 

113 schema : `lsst.afw.table.Schema` 

114 Used to set "calib_astrometry_used" flag in output source catalog. 

115 **kwargs 

116 Additional keyword arguments for pipe_base 

117 `lsst.pipe.base.Task.__init__`. 

118 """ 

119 ConfigClass = AstrometryConfig 

120 _DefaultName = "astrometricSolver" 

121 

122 def __init__(self, refObjLoader=None, schema=None, **kwargs): 

123 RefMatchTask.__init__(self, refObjLoader=refObjLoader, **kwargs) 

124 

125 if schema is not None: 

126 self.usedKey = schema.addField("calib_astrometry_used", type="Flag", 

127 doc="set if source was used in astrometric calibration") 

128 else: 

129 self.usedKey = None 

130 

131 self.makeSubtask("wcsFitter") 

132 

133 @timeMethod 

134 def run(self, sourceCat, exposure): 

135 """Load reference objects, match sources and optionally fit a WCS. 

136 

137 This is a thin layer around solve or loadAndMatch, depending on 

138 config.forceKnownWcs. 

139 

140 Parameters 

141 ---------- 

142 exposure : `lsst.afw.image.Exposure` 

143 exposure whose WCS is to be fit 

144 The following are read only: 

145 

146 - bbox 

147 - filter (may be unset) 

148 - detector (if wcs is pure tangent; may be absent) 

149 

150 The following are updated: 

151 

152 - wcs (the initial value is used as an initial guess, and is 

153 required) 

154 

155 sourceCat : `lsst.afw.table.SourceCatalog` 

156 catalog of sources detected on the exposure 

157 

158 Returns 

159 ------- 

160 result : `lsst.pipe.base.Struct` 

161 with these fields: 

162 

163 - ``refCat`` : reference object catalog of objects that overlap the 

164 exposure (with some margin) (`lsst.afw.table.SimpleCatalog`). 

165 - ``matches`` : astrometric matches 

166 (`list` of `lsst.afw.table.ReferenceMatch`). 

167 - ``scatterOnSky`` : median on-sky separation between reference 

168 objects and sources in "matches" 

169 (`lsst.afw.geom.Angle`) or `None` if config.forceKnownWcs True 

170 - ``matchMeta`` : metadata needed to unpersist matches 

171 (`lsst.daf.base.PropertyList`) 

172 """ 

173 if self.refObjLoader is None: 

174 raise RuntimeError("Running matcher task with no refObjLoader set in __init__ or setRefObjLoader") 

175 if self.config.forceKnownWcs: 

176 res = self.loadAndMatch(exposure=exposure, sourceCat=sourceCat) 

177 res.scatterOnSky = None 

178 else: 

179 res = self.solve(exposure=exposure, sourceCat=sourceCat) 

180 return res 

181 

182 @timeMethod 

183 def solve(self, exposure, sourceCat): 

184 """Load reference objects overlapping an exposure, match to sources and 

185 fit a WCS 

186 

187 Returns 

188 ------- 

189 result : `lsst.pipe.base.Struct` 

190 Result struct with components: 

191 

192 - ``refCat`` : reference object catalog of objects that overlap the 

193 exposure (with some margin) (`lsst::afw::table::SimpleCatalog`). 

194 - ``matches`` : astrometric matches 

195 (`list` of `lsst.afw.table.ReferenceMatch`). 

196 - ``scatterOnSky`` : median on-sky separation between reference 

197 objects and sources in "matches" (`lsst.geom.Angle`) 

198 - ``matchMeta`` : metadata needed to unpersist matches 

199 (`lsst.daf.base.PropertyList`) 

200 

201 Raises 

202 ------ 

203 TaskError 

204 If the measured mean on-sky distance between the matched source and 

205 reference objects is greater than 

206 ``self.config.maxMeanDistanceArcsec``. 

207 

208 Notes 

209 ----- 

210 ignores config.forceKnownWcs 

211 """ 

212 if self.refObjLoader is None: 

213 raise RuntimeError("Running matcher task with no refObjLoader set in __init__ or setRefObjLoader") 

214 import lsstDebug 

215 debug = lsstDebug.Info(__name__) 

216 

217 expMd = self._getExposureMetadata(exposure) 

218 

219 sourceSelection = self.sourceSelector.run(sourceCat) 

220 

221 self.log.info("Purged %d sources, leaving %d good sources", 

222 len(sourceCat) - len(sourceSelection.sourceCat), 

223 len(sourceSelection.sourceCat)) 

224 

225 loadRes = self.refObjLoader.loadPixelBox( 

226 bbox=expMd.bbox, 

227 wcs=expMd.wcs, 

228 filterName=expMd.filterName, 

229 epoch=expMd.epoch, 

230 ) 

231 

232 refSelection = self.referenceSelector.run(loadRes.refCat) 

233 

234 matchMeta = self.refObjLoader.getMetadataBox( 

235 bbox=expMd.bbox, 

236 wcs=expMd.wcs, 

237 filterName=expMd.filterName, 

238 epoch=expMd.epoch, 

239 ) 

240 

241 if debug.display: 

242 frame = int(debug.frame) 

243 displayAstrometry( 

244 refCat=refSelection.sourceCat, 

245 sourceCat=sourceSelection.sourceCat, 

246 exposure=exposure, 

247 bbox=expMd.bbox, 

248 frame=frame, 

249 title="Reference catalog", 

250 ) 

251 

252 res = None 

253 wcs = expMd.wcs 

254 match_tolerance = None 

255 fitFailed = False 

256 for i in range(self.config.maxIter): 

257 if not fitFailed: 

258 iterNum = i + 1 

259 try: 

260 tryRes = self._matchAndFitWcs( 

261 refCat=refSelection.sourceCat, 

262 sourceCat=sourceCat, 

263 goodSourceCat=sourceSelection.sourceCat, 

264 refFluxField=loadRes.fluxField, 

265 bbox=expMd.bbox, 

266 wcs=wcs, 

267 exposure=exposure, 

268 match_tolerance=match_tolerance, 

269 ) 

270 except Exception as e: 

271 # If we have had a succeessful iteration then use that; 

272 # otherwise fail. 

273 if i > 0: 

274 self.log.info("Fit WCS iter %d failed; using previous iteration: %s", iterNum, e) 

275 iterNum -= 1 

276 break 

277 else: 

278 self.log.info("Fit WCS iter %d failed: %s" % (iterNum, e)) 

279 fitFailed = True 

280 

281 if not fitFailed: 

282 match_tolerance = tryRes.match_tolerance 

283 tryMatchDist = self._computeMatchStatsOnSky(tryRes.matches) 

284 self.log.debug( 

285 "Match and fit WCS iteration %d: found %d matches with on-sky distance mean and " 

286 "scatter = %0.3f +- %0.3f arcsec; max match distance = %0.3f arcsec", 

287 iterNum, len(tryRes.matches), tryMatchDist.distMean.asArcseconds(), 

288 tryMatchDist.distStdDev.asArcseconds(), tryMatchDist.maxMatchDist.asArcseconds()) 

289 

290 maxMatchDist = tryMatchDist.maxMatchDist 

291 res = tryRes 

292 wcs = res.wcs 

293 if maxMatchDist.asArcseconds() < self.config.minMatchDistanceArcSec: 

294 self.log.debug( 

295 "Max match distance = %0.3f arcsec < %0.3f = config.minMatchDistanceArcSec; " 

296 "that's good enough", 

297 maxMatchDist.asArcseconds(), self.config.minMatchDistanceArcSec) 

298 break 

299 match_tolerance.maxMatchDist = maxMatchDist 

300 

301 if not fitFailed: 

302 self.log.info("Matched and fit WCS in %d iterations; " 

303 "found %d matches with mean and scatter = %0.3f +- %0.3f arcsec" % 

304 (iterNum, len(tryRes.matches), tryMatchDist.distMean.asArcseconds(), 

305 tryMatchDist.distStdDev.asArcseconds())) 

306 if tryMatchDist.distMean.asArcseconds() > self.config.maxMeanDistanceArcsec: 

307 self.log.info("Assigning as a fit failure: mean on-sky distance = %0.3f arcsec > %0.3f " 

308 "(maxMeanDistanceArcsec)" % (tryMatchDist.distMean.asArcseconds(), 

309 self.config.maxMeanDistanceArcsec)) 

310 fitFailed = True 

311 

312 if fitFailed: 

313 self.log.warning("WCS fit failed. Setting exposure's WCS to None and coord_ra & coord_dec " 

314 "cols in sourceCat to nan.") 

315 sourceCat["coord_ra"] = np.nan 

316 sourceCat["coord_dec"] = np.nan 

317 exposure.setWcs(None) 

318 matches = None 

319 scatterOnSky = None 

320 else: 

321 for m in res.matches: 

322 if self.usedKey: 

323 m.second.set(self.usedKey, True) 

324 exposure.setWcs(res.wcs) 

325 matches = res.matches 

326 scatterOnSky = res.scatterOnSky 

327 

328 # If fitter converged, record the scatter in the exposure metadata 

329 # even if the fit was deemed a failure according to the value of 

330 # the maxMeanDistanceArcsec config. 

331 if res is not None: 

332 md = exposure.getMetadata() 

333 md['SFM_ASTROM_OFFSET_MEAN'] = tryMatchDist.distMean.asArcseconds() 

334 md['SFM_ASTROM_OFFSET_STD'] = tryMatchDist.distStdDev.asArcseconds() 

335 

336 return pipeBase.Struct( 

337 refCat=refSelection.sourceCat, 

338 matches=matches, 

339 scatterOnSky=scatterOnSky, 

340 matchMeta=matchMeta, 

341 ) 

342 

343 @timeMethod 

344 def _matchAndFitWcs(self, refCat, sourceCat, goodSourceCat, refFluxField, bbox, wcs, match_tolerance, 

345 exposure=None): 

346 """Match sources to reference objects and fit a WCS. 

347 

348 Parameters 

349 ---------- 

350 refCat : `lsst.afw.table.SimpleCatalog` 

351 catalog of reference objects 

352 sourceCat : `lsst.afw.table.SourceCatalog` 

353 catalog of sources detected on the exposure 

354 goodSourceCat : `lsst.afw.table.SourceCatalog` 

355 catalog of down-selected good sources detected on the exposure 

356 refFluxField : 'str' 

357 field of refCat to use for flux 

358 bbox : `lsst.geom.Box2I` 

359 bounding box of exposure 

360 wcs : `lsst.afw.geom.SkyWcs` 

361 initial guess for WCS of exposure 

362 match_tolerance : `lsst.meas.astrom.MatchTolerance` 

363 a MatchTolerance object (or None) specifying 

364 internal tolerances to the matcher. See the MatchTolerance 

365 definition in the respective matcher for the class definition. 

366 exposure : `lsst.afw.image.Exposure` 

367 exposure whose WCS is to be fit, or None; used only for the debug 

368 display. 

369 

370 Returns 

371 ------- 

372 result : `lsst.pipe.base.Struct` 

373 Result struct with components: 

374 

375 - ``matches``: astrometric matches 

376 (`list` of `lsst.afw.table.ReferenceMatch`). 

377 - ``wcs``: the fit WCS (lsst.afw.geom.SkyWcs). 

378 - ``scatterOnSky`` : median on-sky separation between reference 

379 objects and sources in "matches" (`lsst.afw.geom.Angle`). 

380 """ 

381 import lsstDebug 

382 debug = lsstDebug.Info(__name__) 

383 

384 sourceFluxField = "slot_%sFlux_instFlux" % (self.config.sourceFluxType) 

385 

386 matchRes = self.matcher.matchObjectsToSources( 

387 refCat=refCat, 

388 sourceCat=goodSourceCat, 

389 wcs=wcs, 

390 sourceFluxField=sourceFluxField, 

391 refFluxField=refFluxField, 

392 match_tolerance=match_tolerance, 

393 ) 

394 self.log.debug("Found %s matches", len(matchRes.matches)) 

395 if debug.display: 

396 frame = int(debug.frame) 

397 displayAstrometry( 

398 refCat=refCat, 

399 sourceCat=matchRes.usableSourceCat, 

400 matches=matchRes.matches, 

401 exposure=exposure, 

402 bbox=bbox, 

403 frame=frame + 1, 

404 title="Initial WCS", 

405 ) 

406 

407 if self.config.doMagnitudeOutlierRejection: 

408 matches = self._removeMagnitudeOutliers(sourceFluxField, refFluxField, matchRes.matches) 

409 else: 

410 matches = matchRes.matches 

411 

412 self.log.debug("Fitting WCS") 

413 fitRes = self.wcsFitter.fitWcs( 

414 matches=matches, 

415 initWcs=wcs, 

416 bbox=bbox, 

417 refCat=refCat, 

418 sourceCat=sourceCat, 

419 exposure=exposure, 

420 ) 

421 fitWcs = fitRes.wcs 

422 scatterOnSky = fitRes.scatterOnSky 

423 if debug.display: 

424 frame = int(debug.frame) 

425 displayAstrometry( 

426 refCat=refCat, 

427 sourceCat=matchRes.usableSourceCat, 

428 matches=matches, 

429 exposure=exposure, 

430 bbox=bbox, 

431 frame=frame + 2, 

432 title="Fit TAN-SIP WCS", 

433 ) 

434 

435 return pipeBase.Struct( 

436 matches=matches, 

437 wcs=fitWcs, 

438 scatterOnSky=scatterOnSky, 

439 match_tolerance=matchRes.match_tolerance, 

440 ) 

441 

442 def _removeMagnitudeOutliers(self, sourceFluxField, refFluxField, matchesIn): 

443 """Remove magnitude outliers, computing a simple zeropoint. 

444 

445 Parameters 

446 ---------- 

447 sourceFluxField : `str` 

448 Field in source catalog for instrumental fluxes. 

449 refFluxField : `str` 

450 Field in reference catalog for fluxes (nJy). 

451 matchesIn : `list` [`lsst.afw.table.ReferenceMatch`] 

452 List of source/reference matches input 

453 

454 Returns 

455 ------- 

456 matchesOut : `list` [`lsst.afw.table.ReferenceMatch`] 

457 List of source/reference matches with magnitude 

458 outliers removed. 

459 """ 

460 nMatch = len(matchesIn) 

461 sourceMag = np.zeros(nMatch) 

462 refMag = np.zeros(nMatch) 

463 for i, match in enumerate(matchesIn): 

464 sourceMag[i] = -2.5*np.log10(match[1][sourceFluxField]) 

465 refMag[i] = (match[0][refFluxField]*units.nJy).to_value(units.ABmag) 

466 

467 deltaMag = refMag - sourceMag 

468 # Protect against negative fluxes and nans in the reference catalog. 

469 goodDelta, = np.where(np.isfinite(deltaMag)) 

470 zp = np.median(deltaMag[goodDelta]) 

471 # Use median absolute deviation (MAD) for zpSigma. 

472 # Also require a minimum scatter to prevent floating-point errors from 

473 # rejecting objects in zero-noise tests. 

474 zpSigma = np.clip(scipy.stats.median_abs_deviation(deltaMag[goodDelta], scale='normal'), 

475 1e-3, 

476 None) 

477 

478 self.log.info("Rough zeropoint from astrometry matches is %.4f +/- %.4f.", 

479 zp, zpSigma) 

480 

481 goodStars = goodDelta[(np.abs(deltaMag[goodDelta] - zp) 

482 <= self.config.magnitudeOutlierRejectionNSigma*zpSigma)] 

483 

484 nOutlier = nMatch - goodStars.size 

485 self.log.info("Removed %d magnitude outliers out of %d total astrometry matches.", 

486 nOutlier, nMatch) 

487 

488 matchesOut = [] 

489 for matchInd in goodStars: 

490 matchesOut.append(matchesIn[matchInd]) 

491 

492 return matchesOut