Coverage for python/lsst/meas/astrom/astrometry.py: 18%

149 statements  

« prev     ^ index     » next       coverage.py v7.4.1, created at 2024-02-09 11:42 +0000

1# This file is part of meas_astrom. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["AstrometryConfig", "AstrometryTask"] 

23 

24import numpy as np 

25from astropy import units 

26import scipy.stats 

27 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30from lsst.utils.timer import timeMethod 

31from .ref_match import RefMatchTask, RefMatchConfig 

32from .fitTanSipWcs import FitTanSipWcsTask 

33from .display import displayAstrometry 

34 

35 

36class AstrometryConfig(RefMatchConfig): 

37 """Config for AstrometryTask. 

38 """ 

39 wcsFitter = pexConfig.ConfigurableField( 

40 target=FitTanSipWcsTask, 

41 doc="WCS fitter", 

42 ) 

43 forceKnownWcs = pexConfig.Field( 

44 dtype=bool, 

45 doc="If True then load reference objects and match sources but do not fit a WCS; " 

46 "this simply controls whether 'run' calls 'solve' or 'loadAndMatch'", 

47 default=False, 

48 ) 

49 maxIter = pexConfig.RangeField( 

50 doc="maximum number of iterations of match sources and fit WCS" 

51 "ignored if not fitting a WCS", 

52 dtype=int, 

53 default=3, 

54 min=1, 

55 ) 

56 minMatchDistanceArcSec = pexConfig.RangeField( 

57 doc="the match distance below which further iteration is pointless (arcsec); " 

58 "ignored if not fitting a WCS", 

59 dtype=float, 

60 default=0.001, 

61 min=0, 

62 ) 

63 maxMeanDistanceArcsec = pexConfig.RangeField( 

64 doc="Maximum mean on-sky distance (in arcsec) between matched source and rerference " 

65 "objects post-fit. A mean distance greater than this threshold raises a TaskError " 

66 "and the WCS fit is considered a failure. The default is set to the maximum tolerated " 

67 "by the external global calibration (e.g. jointcal) step for conceivable recovery. " 

68 "Appropriate value will be dataset and workflow dependent.", 

69 dtype=float, 

70 default=0.5, 

71 min=0, 

72 ) 

73 doMagnitudeOutlierRejection = pexConfig.Field( 

74 dtype=bool, 

75 doc=("If True then a rough zeropoint will be computed from matched sources " 

76 "and outliers will be rejected in the iterations."), 

77 default=True, 

78 ) 

79 magnitudeOutlierRejectionNSigma = pexConfig.Field( 

80 dtype=float, 

81 doc=("Number of sigma (measured from the distribution) in magnitude " 

82 "for a potential reference/source match to be rejected during " 

83 "iteration."), 

84 default=3.0, 

85 ) 

86 

87 def setDefaults(self): 

88 # Override the default source selector for astrometry tasks 

89 self.sourceFluxType = "Psf" 

90 # Configured to match the deprecated "matcher" selector: isolated, 

91 # SN > 40, some bad flags, valid centroids. 

92 self.sourceSelector["science"].doSignalToNoise = True 

93 self.sourceSelector["science"].signalToNoise.minimum = 40 

94 self.sourceSelector["science"].signalToNoise.fluxField = f"slot_{self.sourceFluxType}Flux_instFlux" 

95 self.sourceSelector["science"].signalToNoise.errField = f"slot_{self.sourceFluxType}Flux_instFluxErr" 

96 self.sourceSelector["science"].doFlags = True 

97 self.sourceSelector["science"].flags.bad = ["base_PixelFlags_flag_edge", 

98 "base_PixelFlags_flag_interpolatedCenter", 

99 "base_PixelFlags_flag_saturated", 

100 "base_SdssCentroid_flag", 

101 ] 

102 self.sourceSelector["science"].doRequirePrimary = True 

103 self.sourceSelector["science"].doIsolated = True 

104 

105 

106class AstrometryTask(RefMatchTask): 

107 """Match an input source catalog with objects from a reference catalog and 

108 solve for the WCS. 

109 

110 This task is broken into two main subasks: matching and WCS fitting which 

111 are very interactive. The matching here can be considered in part a first 

112 pass WCS fitter due to the fitter's sensitivity to outliers. 

113 

114 Parameters 

115 ---------- 

116 refObjLoader : `lsst.meas.algorithms.ReferenceLoader` 

117 A reference object loader object; gen3 pipeline tasks will pass `None` 

118 and call `setRefObjLoader` in `runQuantum`. 

119 schema : `lsst.afw.table.Schema` 

120 Used to set "calib_astrometry_used" flag in output source catalog. 

121 **kwargs 

122 Additional keyword arguments for pipe_base 

123 `lsst.pipe.base.Task.__init__`. 

124 """ 

125 ConfigClass = AstrometryConfig 

126 _DefaultName = "astrometricSolver" 

127 

128 def __init__(self, refObjLoader=None, schema=None, **kwargs): 

129 RefMatchTask.__init__(self, refObjLoader=refObjLoader, **kwargs) 

130 

131 if schema is not None: 

132 self.usedKey = schema.addField("calib_astrometry_used", type="Flag", 

133 doc="set if source was used in astrometric calibration") 

134 else: 

135 self.usedKey = None 

136 

137 self.makeSubtask("wcsFitter") 

138 

139 @timeMethod 

140 def run(self, sourceCat, exposure): 

141 """Load reference objects, match sources and optionally fit a WCS. 

142 

143 This is a thin layer around solve or loadAndMatch, depending on 

144 config.forceKnownWcs. 

145 

146 Parameters 

147 ---------- 

148 exposure : `lsst.afw.image.Exposure` 

149 exposure whose WCS is to be fit 

150 The following are read only: 

151 

152 - bbox 

153 - filter (may be unset) 

154 - detector (if wcs is pure tangent; may be absent) 

155 

156 The following are updated: 

157 

158 - wcs (the initial value is used as an initial guess, and is 

159 required) 

160 

161 sourceCat : `lsst.afw.table.SourceCatalog` 

162 catalog of sources detected on the exposure 

163 

164 Returns 

165 ------- 

166 result : `lsst.pipe.base.Struct` 

167 with these fields: 

168 

169 - ``refCat`` : reference object catalog of objects that overlap the 

170 exposure (with some margin) (`lsst.afw.table.SimpleCatalog`). 

171 - ``matches`` : astrometric matches 

172 (`list` of `lsst.afw.table.ReferenceMatch`). 

173 - ``scatterOnSky`` : median on-sky separation between reference 

174 objects and sources in "matches" 

175 (`lsst.afw.geom.Angle`) or `None` if config.forceKnownWcs True 

176 - ``matchMeta`` : metadata needed to unpersist matches 

177 (`lsst.daf.base.PropertyList`) 

178 """ 

179 if self.refObjLoader is None: 

180 raise RuntimeError("Running matcher task with no refObjLoader set in __init__ or setRefObjLoader") 

181 if self.config.forceKnownWcs: 

182 res = self.loadAndMatch(exposure=exposure, sourceCat=sourceCat) 

183 res.scatterOnSky = None 

184 else: 

185 res = self.solve(exposure=exposure, sourceCat=sourceCat) 

186 return res 

187 

188 @timeMethod 

189 def solve(self, exposure, sourceCat): 

190 """Load reference objects overlapping an exposure, match to sources and 

191 fit a WCS 

192 

193 Returns 

194 ------- 

195 result : `lsst.pipe.base.Struct` 

196 Result struct with components: 

197 

198 - ``refCat`` : reference object catalog of objects that overlap the 

199 exposure (with some margin) (`lsst::afw::table::SimpleCatalog`). 

200 - ``matches`` : astrometric matches 

201 (`list` of `lsst.afw.table.ReferenceMatch`). 

202 - ``scatterOnSky`` : median on-sky separation between reference 

203 objects and sources in "matches" (`lsst.geom.Angle`) 

204 - ``matchMeta`` : metadata needed to unpersist matches 

205 (`lsst.daf.base.PropertyList`) 

206 

207 Raises 

208 ------ 

209 TaskError 

210 If the measured mean on-sky distance between the matched source and 

211 reference objects is greater than 

212 ``self.config.maxMeanDistanceArcsec``. 

213 

214 Notes 

215 ----- 

216 ignores config.forceKnownWcs 

217 """ 

218 if self.refObjLoader is None: 

219 raise RuntimeError("Running matcher task with no refObjLoader set in __init__ or setRefObjLoader") 

220 import lsstDebug 

221 debug = lsstDebug.Info(__name__) 

222 

223 expMd = self._getExposureMetadata(exposure) 

224 

225 sourceSelection = self.sourceSelector.run(sourceCat) 

226 

227 self.log.info("Purged %d sources, leaving %d good sources", 

228 len(sourceCat) - len(sourceSelection.sourceCat), 

229 len(sourceSelection.sourceCat)) 

230 

231 loadRes = self.refObjLoader.loadPixelBox( 

232 bbox=expMd.bbox, 

233 wcs=expMd.wcs, 

234 filterName=expMd.filterName, 

235 epoch=expMd.epoch, 

236 ) 

237 

238 refSelection = self.referenceSelector.run(loadRes.refCat) 

239 

240 matchMeta = self.refObjLoader.getMetadataBox( 

241 bbox=expMd.bbox, 

242 wcs=expMd.wcs, 

243 filterName=expMd.filterName, 

244 epoch=expMd.epoch, 

245 ) 

246 

247 if debug.display: 

248 frame = int(debug.frame) 

249 displayAstrometry( 

250 refCat=refSelection.sourceCat, 

251 sourceCat=sourceSelection.sourceCat, 

252 exposure=exposure, 

253 bbox=expMd.bbox, 

254 frame=frame, 

255 title="Reference catalog", 

256 ) 

257 

258 res = None 

259 wcs = expMd.wcs 

260 match_tolerance = None 

261 fitFailed = False 

262 for i in range(self.config.maxIter): 

263 if not fitFailed: 

264 iterNum = i + 1 

265 try: 

266 tryRes = self._matchAndFitWcs( 

267 refCat=refSelection.sourceCat, 

268 sourceCat=sourceCat, 

269 goodSourceCat=sourceSelection.sourceCat, 

270 refFluxField=loadRes.fluxField, 

271 bbox=expMd.bbox, 

272 wcs=wcs, 

273 exposure=exposure, 

274 match_tolerance=match_tolerance, 

275 ) 

276 except Exception as e: 

277 # If we have had a succeessful iteration then use that; 

278 # otherwise fail. 

279 if i > 0: 

280 self.log.info("Fit WCS iter %d failed; using previous iteration: %s", iterNum, e) 

281 iterNum -= 1 

282 break 

283 else: 

284 self.log.info("Fit WCS iter %d failed: %s" % (iterNum, e)) 

285 fitFailed = True 

286 

287 if not fitFailed: 

288 match_tolerance = tryRes.match_tolerance 

289 tryMatchDist = self._computeMatchStatsOnSky(tryRes.matches) 

290 self.log.debug( 

291 "Match and fit WCS iteration %d: found %d matches with on-sky distance mean and " 

292 "scatter = %0.3f +- %0.3f arcsec; max match distance = %0.3f arcsec", 

293 iterNum, len(tryRes.matches), tryMatchDist.distMean.asArcseconds(), 

294 tryMatchDist.distStdDev.asArcseconds(), tryMatchDist.maxMatchDist.asArcseconds()) 

295 

296 maxMatchDist = tryMatchDist.maxMatchDist 

297 res = tryRes 

298 wcs = res.wcs 

299 if maxMatchDist.asArcseconds() < self.config.minMatchDistanceArcSec: 

300 self.log.debug( 

301 "Max match distance = %0.3f arcsec < %0.3f = config.minMatchDistanceArcSec; " 

302 "that's good enough", 

303 maxMatchDist.asArcseconds(), self.config.minMatchDistanceArcSec) 

304 break 

305 match_tolerance.maxMatchDist = maxMatchDist 

306 

307 if not fitFailed: 

308 self.log.info("Matched and fit WCS in %d iterations; " 

309 "found %d matches with mean and scatter = %0.3f +- %0.3f arcsec" % 

310 (iterNum, len(tryRes.matches), tryMatchDist.distMean.asArcseconds(), 

311 tryMatchDist.distStdDev.asArcseconds())) 

312 if tryMatchDist.distMean.asArcseconds() > self.config.maxMeanDistanceArcsec: 

313 self.log.info("Assigning as a fit failure: mean on-sky distance = %0.3f arcsec > %0.3f " 

314 "(maxMeanDistanceArcsec)" % (tryMatchDist.distMean.asArcseconds(), 

315 self.config.maxMeanDistanceArcsec)) 

316 fitFailed = True 

317 

318 if fitFailed: 

319 self.log.warning("WCS fit failed. Setting exposure's WCS to None and coord_ra & coord_dec " 

320 "cols in sourceCat to nan.") 

321 sourceCat["coord_ra"] = np.nan 

322 sourceCat["coord_dec"] = np.nan 

323 exposure.setWcs(None) 

324 matches = None 

325 scatterOnSky = None 

326 else: 

327 for m in res.matches: 

328 if self.usedKey: 

329 m.second.set(self.usedKey, True) 

330 exposure.setWcs(res.wcs) 

331 matches = res.matches 

332 scatterOnSky = res.scatterOnSky 

333 

334 # If fitter converged, record the scatter in the exposure metadata 

335 # even if the fit was deemed a failure according to the value of 

336 # the maxMeanDistanceArcsec config. 

337 if res is not None: 

338 md = exposure.getMetadata() 

339 md['SFM_ASTROM_OFFSET_MEAN'] = tryMatchDist.distMean.asArcseconds() 

340 md['SFM_ASTROM_OFFSET_STD'] = tryMatchDist.distStdDev.asArcseconds() 

341 

342 return pipeBase.Struct( 

343 refCat=refSelection.sourceCat, 

344 matches=matches, 

345 scatterOnSky=scatterOnSky, 

346 matchMeta=matchMeta, 

347 ) 

348 

349 @timeMethod 

350 def _matchAndFitWcs(self, refCat, sourceCat, goodSourceCat, refFluxField, bbox, wcs, match_tolerance, 

351 exposure=None): 

352 """Match sources to reference objects and fit a WCS. 

353 

354 Parameters 

355 ---------- 

356 refCat : `lsst.afw.table.SimpleCatalog` 

357 catalog of reference objects 

358 sourceCat : `lsst.afw.table.SourceCatalog` 

359 catalog of sources detected on the exposure 

360 goodSourceCat : `lsst.afw.table.SourceCatalog` 

361 catalog of down-selected good sources detected on the exposure 

362 refFluxField : 'str' 

363 field of refCat to use for flux 

364 bbox : `lsst.geom.Box2I` 

365 bounding box of exposure 

366 wcs : `lsst.afw.geom.SkyWcs` 

367 initial guess for WCS of exposure 

368 match_tolerance : `lsst.meas.astrom.MatchTolerance` 

369 a MatchTolerance object (or None) specifying 

370 internal tolerances to the matcher. See the MatchTolerance 

371 definition in the respective matcher for the class definition. 

372 exposure : `lsst.afw.image.Exposure` 

373 exposure whose WCS is to be fit, or None; used only for the debug 

374 display. 

375 

376 Returns 

377 ------- 

378 result : `lsst.pipe.base.Struct` 

379 Result struct with components: 

380 

381 - ``matches``: astrometric matches 

382 (`list` of `lsst.afw.table.ReferenceMatch`). 

383 - ``wcs``: the fit WCS (lsst.afw.geom.SkyWcs). 

384 - ``scatterOnSky`` : median on-sky separation between reference 

385 objects and sources in "matches" (`lsst.afw.geom.Angle`). 

386 """ 

387 import lsstDebug 

388 debug = lsstDebug.Info(__name__) 

389 

390 sourceFluxField = "slot_%sFlux_instFlux" % (self.config.sourceFluxType) 

391 

392 matchRes = self.matcher.matchObjectsToSources( 

393 refCat=refCat, 

394 sourceCat=goodSourceCat, 

395 wcs=wcs, 

396 sourceFluxField=sourceFluxField, 

397 refFluxField=refFluxField, 

398 match_tolerance=match_tolerance, 

399 ) 

400 self.log.debug("Found %s matches", len(matchRes.matches)) 

401 if debug.display: 

402 frame = int(debug.frame) 

403 displayAstrometry( 

404 refCat=refCat, 

405 sourceCat=matchRes.usableSourceCat, 

406 matches=matchRes.matches, 

407 exposure=exposure, 

408 bbox=bbox, 

409 frame=frame + 1, 

410 title="Initial WCS", 

411 ) 

412 

413 if self.config.doMagnitudeOutlierRejection: 

414 matches = self._removeMagnitudeOutliers(sourceFluxField, refFluxField, matchRes.matches) 

415 else: 

416 matches = matchRes.matches 

417 

418 self.log.debug("Fitting WCS") 

419 fitRes = self.wcsFitter.fitWcs( 

420 matches=matches, 

421 initWcs=wcs, 

422 bbox=bbox, 

423 refCat=refCat, 

424 sourceCat=sourceCat, 

425 exposure=exposure, 

426 ) 

427 fitWcs = fitRes.wcs 

428 scatterOnSky = fitRes.scatterOnSky 

429 if debug.display: 

430 frame = int(debug.frame) 

431 displayAstrometry( 

432 refCat=refCat, 

433 sourceCat=matchRes.usableSourceCat, 

434 matches=matches, 

435 exposure=exposure, 

436 bbox=bbox, 

437 frame=frame + 2, 

438 title=f"Fitter: {self.wcsFitter._DefaultName}", 

439 ) 

440 

441 return pipeBase.Struct( 

442 matches=matches, 

443 wcs=fitWcs, 

444 scatterOnSky=scatterOnSky, 

445 match_tolerance=matchRes.match_tolerance, 

446 ) 

447 

448 def _removeMagnitudeOutliers(self, sourceFluxField, refFluxField, matchesIn): 

449 """Remove magnitude outliers, computing a simple zeropoint. 

450 

451 Parameters 

452 ---------- 

453 sourceFluxField : `str` 

454 Field in source catalog for instrumental fluxes. 

455 refFluxField : `str` 

456 Field in reference catalog for fluxes (nJy). 

457 matchesIn : `list` [`lsst.afw.table.ReferenceMatch`] 

458 List of source/reference matches input 

459 

460 Returns 

461 ------- 

462 matchesOut : `list` [`lsst.afw.table.ReferenceMatch`] 

463 List of source/reference matches with magnitude 

464 outliers removed. 

465 """ 

466 nMatch = len(matchesIn) 

467 sourceMag = np.zeros(nMatch) 

468 refMag = np.zeros(nMatch) 

469 for i, match in enumerate(matchesIn): 

470 sourceMag[i] = -2.5*np.log10(match[1][sourceFluxField]) 

471 refMag[i] = (match[0][refFluxField]*units.nJy).to_value(units.ABmag) 

472 

473 deltaMag = refMag - sourceMag 

474 # Protect against negative fluxes and nans in the reference catalog. 

475 goodDelta, = np.where(np.isfinite(deltaMag)) 

476 zp = np.median(deltaMag[goodDelta]) 

477 # Use median absolute deviation (MAD) for zpSigma. 

478 # Also require a minimum scatter to prevent floating-point errors from 

479 # rejecting objects in zero-noise tests. 

480 zpSigma = np.clip(scipy.stats.median_abs_deviation(deltaMag[goodDelta], scale='normal'), 

481 1e-3, 

482 None) 

483 

484 self.log.info("Rough zeropoint from astrometry matches is %.4f +/- %.4f.", 

485 zp, zpSigma) 

486 

487 goodStars = goodDelta[(np.abs(deltaMag[goodDelta] - zp) 

488 <= self.config.magnitudeOutlierRejectionNSigma*zpSigma)] 

489 

490 nOutlier = nMatch - goodStars.size 

491 self.log.info("Removed %d magnitude outliers out of %d total astrometry matches.", 

492 nOutlier, nMatch) 

493 

494 matchesOut = [matchesIn[idx] for idx in goodStars] 

495 

496 return matchesOut