Coverage for python/lsst/jointcal/utils.py: 7%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

313 statements  

1# This file is part of jointcal. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22""" 

23Statistics of jointcal vs. single-frame procesing and diagnostic plots. 

24 

25Notes 

26----- 

27Some of the algorithms and data structures in this code are temporary 

28kludges and will no longer be necessary once the following are available: 

29 

30- a composite data structure that contains all ccds from a single visit 

31- an n-way matching system that preserves the separations between sources 

32""" 

33import collections 

34import os 

35 

36import numpy as np 

37from astropy import units as u 

38 

39import lsst.log 

40import lsst.afw.table 

41import lsst.afw.image 

42from lsst.geom import arcseconds 

43 

44__all__ = ['JointcalStatistics'] 

45 

46MatchDict = collections.namedtuple('MatchDict', ['relative', 'absolute']) 

47_LOG = lsst.log.Log.getLogger(__name__) 

48 

49 

50class JointcalStatistics: 

51 """ 

52 Compute statistics on jointcal-processed data, and optionally generate plots. 

53 

54 Notes 

55 ----- 

56 Instantiate JointcalStatistics and call compute_rms() to get the relevant 

57 statistics for e.g. unittests, and call make_plots() to generate a suite of 

58 diagnostic plots. 

59 """ 

60 

61 def __init__(self, match_radius=0.1*arcseconds, flux_limit=100.0, 

62 do_photometry=True, do_astrometry=True, 

63 verbose=False): 

64 """ 

65 Parameters 

66 ---------- 

67 match_radius : lsst.geom.Angle 

68 match sources within this radius for RMS statistics 

69 flux_limit : float 

70 Signal/Noise (flux/fluxErr) for sources to be included in the RMS cross-match. 

71 100 is a balance between good centroids and enough sources. 

72 do_photometry : bool, optional 

73 Perform calculations/make plots for photometric metrics. 

74 do_astrometry : bool, optional 

75 Perform calculations/make plots for astrometric metrics. 

76 verbose : bool, optional 

77 Print extra things 

78 """ 

79 self.match_radius = match_radius 

80 self.flux_limit = flux_limit 

81 self.do_photometry = do_photometry 

82 self.do_astrometry = do_astrometry 

83 self.verbose = verbose 

84 self.log = _LOG.getChild('JointcalStatistics') 

85 

86 def compute_rms(self, data_refs, reference): 

87 """ 

88 Match all data_refs to compute the RMS, for all detections above self.flux_limit. 

89 

90 Parameters 

91 ---------- 

92 data_refs : list of lsst.daf.persistence.butlerSubset.ButlerDataRef 

93 A list of data refs to do the calculations between. 

94 reference : lsst reference catalog 

95 reference catalog to do absolute matching against. 

96 

97 Returns 

98 ------- 

99 namedtuple: 

100 astropy.Quantity 

101 Post-jointcal relative RMS of the matched sources. 

102 astropy.Quantity 

103 Post-jointcal absolute RMS of matched sources. 

104 float 

105 Post-jointcal photometric repeatability (PA1 from the SRD). 

106 """ 

107 

108 # DECAM doesn't have "filter" in its registry, so we have to get the filter names directly. 

109 self.filters = [ref.get('calexp_filterLabel').bandLabel for ref in data_refs] 

110 self.visits_per_dataRef = [ref.dataId['visit'] for ref in data_refs] 

111 

112 def compute(catalogs, photoCalibs): 

113 """Compute the relative and absolute matches in distance and flux.""" 

114 visit_catalogs = self._make_visit_catalogs(catalogs, self.visits_per_dataRef) 

115 catalogs = [visit_catalogs[x] for x in self.visits_per_dataRef] 

116 # use the first catalog as the relative reference catalog 

117 # NOTE: The "first" catalog depends on the original ordering of the data_refs. 

118 # NOTE: Thus, because I'm doing a many-1 match in _make_match_dict, 

119 # the number of matches (and thus the details of the match statistics) 

120 # will change if the data_refs are ordered differently. 

121 # All the more reason to use a proper n-way matcher here. See: DM-8664 

122 refcat = catalogs[0] 

123 refcalib = photoCalibs[0] if photoCalibs != [] else None 

124 dist_rel, flux_rel, ref_flux_rel, source_rel = self._make_match_dict(refcat, 

125 catalogs[1:], 

126 photoCalibs[1:], 

127 refcalib=refcalib) 

128 dist_abs, flux_abs, ref_flux_abs, source_abs = self._make_match_dict(reference, 

129 catalogs, 

130 photoCalibs) 

131 dist = MatchDict(dist_rel, dist_abs) 

132 flux = MatchDict(flux_rel, flux_abs) 

133 ref_flux = MatchDict(ref_flux_rel, ref_flux_abs) 

134 source = MatchDict(source_rel, source_abs) 

135 return dist, flux, ref_flux, source 

136 

137 old_cats = [ref.get('src') for ref in data_refs] 

138 # NOTE: build photoCalibs from existing old Calib objects. 

139 # TODO: we can make this a listcomp again once DM-10153 is finished. 

140 old_calibs = [] 

141 if self.do_photometry: 

142 old_calibs = [ref.get('calexp_photoCalib') for ref in data_refs] 

143 

144 self.old_dist, self.old_flux, self.old_ref_flux, self.old_source = compute(old_cats, old_calibs) 

145 

146 # Update coordinates with the new wcs, and get the new photoCalibs. 

147 new_cats = [ref.get('src') for ref in data_refs] 

148 new_wcss = [] 

149 if self.do_astrometry: 

150 new_wcss = [ref.get('jointcal_wcs') for ref in data_refs] 

151 new_calibs = [] 

152 if self.do_photometry: 

153 new_calibs = [ref.get('jointcal_photoCalib') for ref in data_refs] 

154 if self.do_astrometry: 

155 for wcs, cat in zip(new_wcss, new_cats): 

156 # update in-place the object coordinates based on the new wcs 

157 lsst.afw.table.updateSourceCoords(wcs, cat) 

158 

159 self.new_dist, self.new_flux, self.new_ref_flux, self.new_source = compute(new_cats, new_calibs) 

160 

161 if self.verbose: 

162 print('old, new relative distance matches:', 

163 len(self.old_dist.relative), len(self.new_dist.relative)) 

164 print('old, new absolute distance matches:', 

165 len(self.old_dist.absolute), len(self.new_dist.absolute)) 

166 print('old, new relative flux matches:', 

167 len(self.old_flux.relative), len(self.new_flux.relative)) 

168 print('old, new absolute flux matches:', 

169 len(self.old_flux.absolute), len(self.new_flux.absolute)) 

170 

171 if self.do_photometry: 

172 self._photometric_rms() 

173 else: 

174 self.new_PA1 = None 

175 

176 def rms_total(data): 

177 """Compute the total rms across all sources.""" 

178 total = sum(sum(dd**2) for dd in data.values()) 

179 n = sum(len(dd) for dd in data.values()) 

180 return np.sqrt(total/n) 

181 

182 if self.do_astrometry: 

183 self.old_dist_total = MatchDict(*(tuple(map(rms_total, self.old_dist))*u.radian).to(u.arcsecond)) 

184 self.new_dist_total = MatchDict(*(tuple(map(rms_total, self.new_dist))*u.radian).to(u.arcsecond)) 

185 else: 

186 self.old_dist_total = MatchDict(None, None) 

187 self.new_dist_total = MatchDict(None, None) 

188 

189 if self.verbose: 

190 if self.do_astrometry: 

191 print("relative, absolute astrometry:", 

192 self.new_dist_total.relative, 

193 self.new_dist_total.absolute) 

194 if self.do_photometry: 

195 print("Photometric Accuracy (PA1):", self.new_PA1) 

196 Rms_result = collections.namedtuple("Rms_result", ["dist_relative", "dist_absolute", "pa1"]) 

197 return Rms_result(self.new_dist_total.relative, self.new_dist_total.absolute, self.new_PA1) 

198 

199 def make_plots(self, data_refs, old_wcs_list, 

200 name='', interactive=False, per_ccd_plot=False, outdir='.plots'): 

201 """ 

202 Make plots of various quantites to help with debugging. 

203 Requires that `compute_rms()` was run first. 

204 

205 Parameters 

206 ---------- 

207 data_refs : list of lsst.daf.persistence.butlerSubset.ButlerDataRef 

208 A list of data refs to do the calculations between. 

209 old_wcs_list : list of lsst.afw.image.wcs.Wcs 

210 A list of the old (pre-jointcal) WCSs, one-to-one corresponding to data_refs. 

211 name : str 

212 Name to include in plot titles and save files. 

213 interactive : bool 

214 Turn on matplotlib interactive mode and drop into a debugger when 

215 plotting is finished. Otherwise, use a non-interactive backend. 

216 per_ccd_plot : bool 

217 Plot the WCS per CCD (takes longer and generates many plots for a large camera) 

218 outdir : str 

219 directory to save plots to. 

220 """ 

221 import matplotlib 

222 

223 if not interactive: 

224 # Use a non-interactive backend for faster plotting. 

225 matplotlib.use('pdf') 

226 

227 import matplotlib.pyplot as plt 

228 import astropy.visualization 

229 # make quantities behave nicely when plotted. 

230 astropy.visualization.quantity_support() 

231 if interactive: 

232 plt.ion() 

233 

234 self.log.info("N data_refs: %d", len(data_refs)) 

235 

236 if self.do_photometry: 

237 plot_flux_distributions(plt, self.old_mag, self.new_mag, 

238 self.old_weighted_rms, self.new_weighted_rms, 

239 self.faint, self.bright, self.old_PA1, self.new_PA1, 

240 name=name, outdir=outdir) 

241 self.log.info("Photometric accuracy (old, new): {:.2e} {:.2e}".format(self.old_PA1, 

242 self.new_PA1)) 

243 

244 def rms_per_source(data): 

245 """Each element of data must already be the "delta" of whatever measurement.""" 

246 return (np.sqrt([np.mean(dd**2) for dd in data.values()])*u.radian).to(u.arcsecond) 

247 

248 if self.do_astrometry: 

249 old_dist_rms = MatchDict(*(tuple(map(rms_per_source, self.old_dist)))) 

250 new_dist_rms = MatchDict(*(tuple(map(rms_per_source, self.new_dist)))) 

251 

252 self.log.info("relative RMS (old, new): {:.2e} {:.2e}".format(self.old_dist_total.relative, 

253 self.new_dist_total.relative)) 

254 self.log.info("absolute RMS (old, new): {:.2e} {:.2e}".format(self.old_dist_total.absolute, 

255 self.new_dist_total.absolute)) 

256 plot_rms_histogram(plt, old_dist_rms.relative, old_dist_rms.absolute, 

257 new_dist_rms.relative, new_dist_rms.absolute, 

258 self.old_dist_total.relative, self.old_dist_total.absolute, 

259 self.new_dist_total.relative, self.new_dist_total.absolute, 

260 name=name, outdir=outdir) 

261 

262 plot_all_wcs_deltas(plt, data_refs, self.visits_per_dataRef, old_wcs_list, 

263 per_ccd_plot=per_ccd_plot, 

264 name=name, outdir=outdir) 

265 

266 if interactive: 

267 plt.show() 

268 import pdb 

269 pdb.set_trace() 

270 

271 def _photometric_rms(self, sn_cut=300, magnitude_range=3): 

272 """ 

273 Compute the photometric RMS and the photometric repeatablity values (PA1). 

274 

275 Parameters 

276 ---------- 

277 sn_cut : float 

278 The minimum signal/noise for sources to be included in the PA1 calculation. 

279 magnitude_range : float 

280 The range of magnitudes above sn_cut to include in the PA1 calculation. 

281 """ 

282 def rms(flux, ref_flux): 

283 return np.sqrt([np.mean((ref_flux[dd] - flux[dd])**2) for dd in flux]) 

284 

285 self.old_rms = MatchDict(*map(rms, self.old_flux, self.old_ref_flux)) 

286 self.new_rms = MatchDict(*map(rms, self.new_flux, self.new_ref_flux)) 

287 

288 # we want to use the absolute fluxes for all of these calculations. 

289 self.old_ref = np.fromiter(self.old_ref_flux.absolute.values(), dtype=float) 

290 self.new_ref = np.fromiter(self.new_ref_flux.absolute.values(), dtype=float) 

291 self.old_mag = u.Quantity(self.old_ref, u.nJy).to_value(u.ABmag) 

292 self.new_mag = u.Quantity(self.new_ref, u.nJy).to_value(u.ABmag) 

293 

294 def signal_to_noise(sources, flux_key='slot_PsfFlux_instFlux', sigma_key='slot_PsfFlux_instFluxErr'): 

295 """Compute the mean signal/noise per source from a MatchDict of SourceRecords.""" 

296 result = np.empty(len(sources)) 

297 for i, src in enumerate(sources.values()): 

298 result[i] = np.mean([x[flux_key]/x[sigma_key] for x in src]) 

299 return result 

300 

301 old_sn = signal_to_noise(self.old_source.absolute) 

302 # Find the faint/bright magnitude limits that are the "flat" part of the rms/magnitude relation. 

303 self.faint = self.old_mag[old_sn > sn_cut].max() 

304 self.bright = self.faint - magnitude_range 

305 if self.verbose: 

306 print("PA1 Magnitude range: {:.3f}, {:.3f}".format(self.bright, self.faint)) 

307 old_good = (self.old_mag < self.faint) & (self.old_mag > self.bright) 

308 new_good = (self.new_mag < self.faint) & (self.new_mag > self.bright) 

309 self.old_weighted_rms = self.old_rms.absolute/self.old_ref 

310 self.new_weighted_rms = self.new_rms.absolute/self.new_ref 

311 self.old_PA1 = np.median(self.old_weighted_rms[old_good]) 

312 self.new_PA1 = np.median(self.new_weighted_rms[new_good]) 

313 

314 def _make_match_dict(self, reference, visit_catalogs, photoCalibs, refcalib=None): 

315 """ 

316 Return several dicts of sourceID:[values] over the catalogs, to be used in RMS calculations. 

317 

318 Parameters 

319 ---------- 

320 reference : lsst.afw.table.SourceCatalog 

321 Catalog to do the matching against. 

322 visit_catalogs : list of lsst.afw.table.SourceCatalog 

323 Visit source catalogs (values() produced by _make_visit_catalogs) 

324 to cross-match against reference. 

325 photoCalibs : list of lsst.afw.image.PhotoCalib 

326 Exposure PhotoCalibs, 1-1 coorespondent with visit_catalogs. 

327 refcalib : lsst.afw.image.PhotoCalib or None 

328 Pass a PhotoCalib here to use it to compute nanojansky from the 

329 reference catalog ADU slot_flux. 

330 

331 Returns 

332 ------- 

333 distances : dict 

334 dict of sourceID: array(separation distances for that source) 

335 fluxes : dict 

336 dict of sourceID: array(fluxes (nJy) for that source) 

337 ref_fluxes : dict 

338 dict of sourceID: flux (nJy) of the reference object 

339 sources : dict 

340 dict of sourceID: list(each SourceRecord that was position-matched 

341 to this sourceID) 

342 """ 

343 # If we have no photoCalibs, make it the same length as the others for zipping. 

344 if photoCalibs == []: 

345 photoCalibs = [[]]*len(visit_catalogs) 

346 

347 distances = collections.defaultdict(list) 

348 fluxes = collections.defaultdict(list) 

349 ref_fluxes = {} 

350 sources = collections.defaultdict(list) 

351 if 'slot_CalibFlux_instFlux' in reference.schema: 

352 ref_flux_key = 'slot_CalibFlux' 

353 else: 

354 ref_flux_key = '{}_flux' 

355 

356 def get_fluxes(photoCalib, match): 

357 """Return (flux, ref_flux) or None if either is invalid.""" 

358 # NOTE: Protect against negative fluxes: ignore this match if we find one. 

359 flux = match[1]['slot_CalibFlux_instFlux'] 

360 if flux < 0: 

361 return None 

362 else: 

363 flux = photoCalib.instFluxToNanojansky(match[1], "slot_CalibFlux").value 

364 

365 # NOTE: Have to protect against negative "reference" fluxes too. 

366 if 'slot' in ref_flux_key: 

367 ref_flux = match[0][ref_flux_key+'_instFlux'] # relative reference flux 

368 if ref_flux < 0: 

369 return None 

370 else: 

371 ref_flux = refcalib.instFluxToNanojansky(match[0], ref_flux_key).value 

372 else: 

373 # refcat fluxes are already in nanojansky. 

374 ref_flux = match[0][ref_flux_key.format(filt)] 

375 if ref_flux < 0: 

376 return None 

377 

378 Flux = collections.namedtuple('Flux', ('flux', 'ref_flux')) 

379 return Flux(flux, ref_flux) 

380 

381 for cat, photoCalib, filt in zip(visit_catalogs, photoCalibs, self.filters): 

382 good = (cat.get('base_PsfFlux_instFlux')/cat.get('base_PsfFlux_instFluxErr')) > self.flux_limit 

383 # things the classifier called sources are not extended. 

384 good &= (cat.get('base_ClassificationExtendedness_value') == 0) 

385 matches = lsst.afw.table.matchRaDec(reference, cat[good], self.match_radius) 

386 for m in matches: 

387 if self.do_photometry: 

388 flux = get_fluxes(photoCalib, m) 

389 if flux is None: 

390 continue 

391 else: 

392 fluxes[m[0].getId()].append(flux.flux) 

393 # we can just use assignment here, since the value is always the same. 

394 ref_fluxes[m[0].getId()] = flux.ref_flux 

395 

396 if self.do_astrometry: 

397 # Just use the computed separation distance directly. 

398 distances[m[0].getId()].append(m[2]) 

399 

400 sources[m[0].getId()].append(m[1]) 

401 # Convert to numpy array for easier math 

402 for source in distances: 

403 distances[source] = np.array(distances[source]) 

404 for source in fluxes: 

405 fluxes[source] = np.array(fluxes[source]) 

406 

407 return distances, fluxes, ref_fluxes, sources 

408 

409 def _make_visit_catalogs(self, catalogs, visits): 

410 """ 

411 Merge all catalogs from the each visit. 

412 NOTE: creating this structure is somewhat slow, and will be unnecessary 

413 once a full-visit composite dataset is available. 

414 

415 Parameters 

416 ---------- 

417 catalogs : list of lsst.afw.table.SourceCatalog 

418 Catalogs to combine into per-visit catalogs. 

419 visits : list of visit id (usually int) 

420 list of visit identifiers, one-to-one correspondent with catalogs. 

421 

422 Returns 

423 ------- 

424 dict 

425 dict of visit: catalog of all sources from all CCDs of that visit. 

426 """ 

427 visit_dict = {v: lsst.afw.table.SourceCatalog(catalogs[0].schema) for v in visits} 

428 for v, cat in zip(visits, catalogs): 

429 visit_dict[v].extend(cat) 

430 # We want catalog contiguity to do object selection later. 

431 for v in visit_dict: 

432 visit_dict[v] = visit_dict[v].copy(deep=True) 

433 

434 return visit_dict 

435 

436 

437def plot_flux_distributions(plt, old_mag, new_mag, old_weighted_rms, new_weighted_rms, 

438 faint, bright, old_PA1, new_PA1, 

439 name='', outdir='.plots'): 

440 """Plot various distributions of fluxes and magnitudes. 

441 

442 Parameters 

443 ---------- 

444 plt : matplotlib.pyplot instance 

445 pyplot instance to plot with 

446 old_mag : np.array 

447 old magnitudes 

448 new_mag : np.array 

449 new magnitudes 

450 old_weighted_rms : np.array 

451 old rms weighted by the mean (rms(data)/mean(data)) 

452 new_weighted_rms : np.array 

453 old rms weighted by the mean (rms(data)/mean(data)) 

454 faint : float 

455 Faint end of range that PA1 was computed from. 

456 bright : float 

457 Bright end of range that PA1 was computed from. 

458 old_PA1 : float 

459 Old value of PA1, to plot as horizontal line. 

460 new_PA1 : float 

461 New value of PA1, to plot as horizontal line. 

462 name : str 

463 Name to include in plot titles and save files. 

464 outdir : str, optional 

465 Directory to write the saved plots to. 

466 """ 

467 

468 import seaborn 

469 seaborn.set_style('whitegrid') 

470 import scipy.stats 

471 

472 old_color = 'blue' 

473 new_color = 'red' 

474 plt.figure() 

475 plt.plot(old_mag, old_weighted_rms, '.', color=old_color, label='old') 

476 plt.plot(new_mag, new_weighted_rms, '.', color=new_color, label='new') 

477 plt.axvline(faint, ls=':', color=old_color) 

478 plt.axvline(bright, ls=':', color=old_color) 

479 plt.axhline(old_PA1, ls='--', color=old_color) 

480 plt.axhline(new_PA1, ls='--', color=new_color) 

481 plt.legend(loc='upper left') 

482 plt.title('Where is the systematic flux rms limit?') 

483 plt.xlabel('magnitude') 

484 plt.ylabel('rms/mean per source') 

485 filename = os.path.join(outdir, '{}-photometry-PA1.pdf') 

486 plt.savefig(filename.format(name)) 

487 

488 plt.figure() 

489 seaborn.distplot(old_weighted_rms, fit=scipy.stats.lognorm, kde=False, label="old", color=old_color) 

490 seaborn.distplot(new_weighted_rms, fit=scipy.stats.lognorm, kde=False, label="new", color=new_color) 

491 plt.title('Source RMS pre/post-jointcal') 

492 plt.xlabel('rms(flux)/mean(flux)') 

493 plt.ylabel('number') 

494 plt.legend(loc='upper right') 

495 filename = os.path.join(outdir, '{}-photometry-rms.pdf') 

496 plt.savefig(filename.format(name)) 

497 

498 

499def plot_all_wcs_deltas(plt, data_refs, visits, old_wcs_list, per_ccd_plot=False, 

500 name='', outdir='.plots'): 

501 """ 

502 Various plots of the difference between old and new Wcs. 

503 

504 Parameters 

505 ---------- 

506 plt : matplotlib.pyplot instance 

507 pyplot instance to plot with. 

508 data_refs : list of lsst.daf.persistence.butlerSubset.ButlerDataRef 

509 A list of data refs to plot. 

510 visits : list of visit id (usually int) 

511 list of visit identifiers, one-to-one correspondent with catalogs. 

512 old_wcs_list : list of lsst.afw.image.wcs.Wcs 

513 A list of the old (pre-jointcal) WCSs, one-to-one corresponding to data_refs. 

514 per_ccd_plot : bool, optional 

515 Make per-ccd plots of the "wcs different" (warning: slow!) 

516 name : str 

517 Name to include in plot titles and save files. 

518 outdir : str, optional 

519 Directory to write the saved plots to. 

520 """ 

521 

522 plot_wcs_magnitude(plt, data_refs, visits, old_wcs_list, name, outdir=outdir) 

523 plot_all_wcs_quivers(plt, data_refs, visits, old_wcs_list, name, outdir=outdir) 

524 

525 if per_ccd_plot: 

526 for i, ref in enumerate(data_refs): 

527 md = ref.get('calexp_md') 

528 dims = lsst.afw.image.bboxFromMetadata(md).getDimensions() 

529 plot_wcs(plt, old_wcs_list[i], ref.get('jointcal_wcs'), 

530 dims.getX(), dims.getY(), 

531 center=(md.getScalar('CRVAL1'), md.getScalar('CRVAL2')), name='dataRef %d'%i, 

532 outdir=outdir) 

533 

534 

535def make_xy_wcs_grid(x_dim, y_dim, wcs1, wcs2, num=50): 

536 """Return num x/y grid coordinates for wcs1 and wcs2.""" 

537 x = np.linspace(0, x_dim, num) 

538 y = np.linspace(0, y_dim, num) 

539 x1, y1 = wcs_convert(x, y, wcs1) 

540 x2, y2 = wcs_convert(x, y, wcs2) 

541 return x1, y1, x2, y2 

542 

543 

544def wcs_convert(xv, yv, wcs): 

545 """Convert two arrays of x/y points into an on-sky grid.""" 

546 xout = np.zeros((xv.shape[0], yv.shape[0])) 

547 yout = np.zeros((xv.shape[0], yv.shape[0])) 

548 for i, x in enumerate(xv): 

549 for j, y in enumerate(yv): 

550 sky = wcs.pixelToSky(x, y) 

551 xout[i, j] = sky.getRa() 

552 yout[i, j] = sky.getDec() 

553 return xout, yout 

554 

555 

556def plot_all_wcs_quivers(plt, data_refs, visits, old_wcs_list, name, outdir='.plots'): 

557 """ 

558 Make quiver plots of the WCS deltas for each CCD in each visit. 

559 

560 Parameters 

561 ---------- 

562 plt : matplotlib.pyplot instance 

563 pyplot instance to plot with. 

564 data_refs : list of lsst.daf.persistence.butlerSubset.ButlerDataRef 

565 A list of data refs to plot. 

566 visits : list of visit id (usually int) 

567 list of visit identifiers, one-to-one correspondent with catalogs. 

568 old_wcs_list : list of lsst.afw.image.wcs.Wcs 

569 A list of the old (pre-jointcal) WCSs, one-to-one corresponding to data_refs. 

570 name : str 

571 Name to include in plot titles and save files. 

572 outdir : str, optional 

573 Directory to write the saved plots to. 

574 """ 

575 

576 for visit in visits: 

577 fig = plt.figure() 

578 # fig.set_tight_layout(True) 

579 ax = fig.add_subplot(111) 

580 for old_wcs, ref in zip(old_wcs_list, data_refs): 

581 if ref.dataId['visit'] != visit: 

582 continue 

583 md = ref.get('calexp_md') 

584 dims = lsst.afw.image.bboxFromMetadata(md).getDimensions() 

585 Q = plot_wcs_quivers(ax, old_wcs, ref.get('jointcal_wcs'), 

586 dims.getX(), dims.getY()) 

587 # TODO: add CCD bounding boxes to plot once DM-5503 is finished. 

588 # TODO: add a circle for the full focal plane. 

589 length = (0.1*u.arcsecond).to(u.radian).value 

590 ax.quiverkey(Q, 0.9, 0.95, length, '0.1 arcsec', coordinates='figure', labelpos='W') 

591 plt.xlabel('RA') 

592 plt.ylabel('Dec') 

593 plt.title('visit: {}'.format(visit)) 

594 filename = os.path.join(outdir, '{}-{}-quivers.pdf') 

595 plt.savefig(filename.format(name, visit)) 

596 

597 

598def plot_wcs_quivers(ax, wcs1, wcs2, x_dim, y_dim): 

599 """ 

600 Plot the delta between wcs1 and wcs2 as vector arrows. 

601 

602 Parameters 

603 ---------- 

604 ax : matplotlib.axis 

605 Matplotlib axis instance to plot to. 

606 wcs1 : lsst.afw.image.wcs.Wcs 

607 First WCS to compare. 

608 wcs2 : lsst.afw.image.wcs.Wcs 

609 Second WCS to compare. 

610 x_dim : int 

611 Size of array in X-coordinate to make the grid over. 

612 y_dim : int 

613 Size of array in Y-coordinate to make the grid over. 

614 """ 

615 

616 x1, y1, x2, y2 = make_xy_wcs_grid(x_dim, y_dim, wcs1, wcs2) 

617 uu = x2 - x1 

618 vv = y2 - y1 

619 return ax.quiver(x1, y1, uu, vv, units='x', pivot='tail', scale=1e-3, width=1e-5) 

620 

621 

622def plot_wcs_magnitude(plt, data_refs, visits, old_wcs_list, name, outdir='.plots'): 

623 """Plot the magnitude of the WCS change between old and new visits as a heat map. 

624 

625 Parameters 

626 ---------- 

627 plt : matplotlib.pyplot instance 

628 pyplot instance to plot with. 

629 data_refs : list of lsst.daf.persistence.butlerSubset.ButlerDataRef 

630 A list of data refs to plot. 

631 visits : list of visit id (usually int) 

632 list of visit identifiers, one-to-one correspondent with catalogs. 

633 old_wcs_list : list of lsst.afw.image.wcs.Wcs 

634 A list of the old (pre-jointcal) WCSs, one-to-one corresponding to data_refs. 

635 name : str 

636 Name to include in plot titles and save files. 

637 outdir : str, optional 

638 Directory to write the saved plots to. 

639 """ 

640 for visit in visits: 

641 fig = plt.figure() 

642 fig.set_tight_layout(True) 

643 ax = fig.add_subplot(111) 

644 # Start min/max at the "opposite" ends so they always get the first valid value. 

645 xmin = np.inf 

646 ymin = np.inf 

647 xmax = -np.inf 

648 ymax = -np.inf 

649 for old_wcs, ref in zip(old_wcs_list, data_refs): 

650 if ref.dataId['visit'] != visit: 

651 continue 

652 md = ref.get('calexp_md') 

653 dims = lsst.afw.image.bboxFromMetadata(md).getDimensions() 

654 x1, y1, x2, y2 = make_xy_wcs_grid(dims.getX(), dims.getY(), 

655 old_wcs, ref.get('jointcal_wcs')) 

656 uu = x2 - x1 

657 vv = y2 - y1 

658 extent = (x1[0, 0], x1[-1, -1], y1[0, 0], y1[-1, -1]) 

659 xmin = min(x1.min(), xmin) 

660 ymin = min(y1.min(), ymin) 

661 xmax = max(x1.max(), xmax) 

662 ymax = max(y1.max(), ymax) 

663 magnitude = (np.linalg.norm((uu, vv), axis=0)*u.radian).to(u.arcsecond).value 

664 img = ax.imshow(magnitude, vmin=0, vmax=0.3, 

665 aspect='auto', extent=extent, cmap=plt.get_cmap('magma')) 

666 # TODO: add CCD bounding boxes to the plot once DM-5503 is finished. 

667 # TODO: add a circle for the full focal plane. 

668 

669 # We're reusing only one of the returned images here for colorbar scaling, 

670 # but it doesn't matter because we set vmin/vmax so they are all scaled the same. 

671 cbar = plt.colorbar(img) 

672 cbar.ax.set_ylabel('distortion (arcseconds)') 

673 plt.xlim(xmin, xmax) 

674 plt.ylim(ymin, ymax) 

675 plt.xlabel('RA') 

676 plt.ylabel('Dec') 

677 plt.title('visit: {}'.format(visit)) 

678 filename = os.path.join(outdir, '{}-{}-heatmap.pdf') 

679 plt.savefig(filename.format(name, visit)) 

680 

681 

682def plot_wcs(plt, wcs1, wcs2, x_dim, y_dim, center=(0, 0), name="", outdir='.plots'): 

683 """Plot the "distortion map": wcs1-wcs2 delta of points in the CCD grid. 

684 

685 Parameters 

686 ---------- 

687 plt : matplotlib.pyplot instance 

688 pyplot instance to plot with. 

689 wcs1 : lsst.afw.image.wcs.Wcs 

690 First WCS to compare. 

691 wcs2 : lsst.afw.image.wcs.Wcs 

692 Second WCS to compare. 

693 x_dim : int 

694 Size of array in X-coordinate to make the grid over. 

695 y_dim : int 

696 Size of array in Y-coordinate to make the grid over. 

697 center : tuple, optional 

698 Center of the data, in on-chip coordinates. 

699 name : str 

700 Name to include in plot titles and save files. 

701 outdir : str, optional 

702 Directory to write the saved plots to. 

703 """ 

704 

705 plt.figure() 

706 

707 x1, y1, x2, y2 = make_xy_wcs_grid(x_dim, y_dim, wcs1, wcs2, num=50) 

708 plt.plot((x1 - x2) + center[0], (y1 - y2) + center[1], '-') 

709 plt.xlabel('delta RA (arcsec)') 

710 plt.ylabel('delta Dec (arcsec)') 

711 plt.title(name) 

712 filename = os.path.join(outdir, '{}-wcs.pdf') 

713 plt.savefig(filename.format(name)) 

714 

715 

716def plot_rms_histogram(plt, old_rms_relative, old_rms_absolute, 

717 new_rms_relative, new_rms_absolute, 

718 old_rel_total, old_abs_total, new_rel_total, new_abs_total, 

719 name="", outdir='.plots'): 

720 """Plot histograms of the source separations and their RMS values. 

721 

722 Parameters 

723 ---------- 

724 plt : matplotlib.pyplot instance 

725 pyplot instance to plot with. 

726 old_rms_relative : np.array 

727 old relative rms/star 

728 old_rms_absolute : np.array 

729 old absolute rms/star 

730 new_rms_relative : np.array 

731 new relative rms/star 

732 new_rms_absolute : np.array 

733 new absolute rms/star 

734 old_rel_total : float 

735 old relative rms over all stars 

736 old_abs_total : float 

737 old absolute rms over all stars 

738 new_rel_total : float 

739 new relative rms over all stars 

740 new_abs_total : float 

741 new absolute rms over all stars 

742 name : str 

743 Name to include in plot titles and save files. 

744 outdir : str, optional 

745 Directory to write the saved plots to. 

746 """ 

747 plt.figure() 

748 

749 color_rel = 'black' 

750 ls_old = 'dotted' 

751 color_abs = 'green' 

752 ls_new = 'dashed' 

753 plotOptions = {'lw': 2, 'range': (0, 0.1)*u.arcsecond, 'normed': True, 

754 'bins': 30, 'histtype': 'step'} 

755 

756 plt.title('relative vs. absolute: %d vs. %d'%(len(old_rms_relative), len(old_rms_absolute))) 

757 

758 plt.hist(old_rms_absolute, color=color_abs, ls=ls_old, label='old abs', **plotOptions) 

759 plt.hist(new_rms_absolute, color=color_abs, ls=ls_new, label='new abs', **plotOptions) 

760 

761 plt.hist(old_rms_relative, color=color_rel, ls=ls_old, label='old rel', **plotOptions) 

762 plt.hist(new_rms_relative, color=color_rel, ls=ls_new, label='new rel', **plotOptions) 

763 

764 plt.axvline(x=old_abs_total.value, linewidth=1.5, color=color_abs, ls=ls_old) 

765 plt.axvline(x=new_abs_total.value, linewidth=1.5, color=color_abs, ls=ls_new) 

766 plt.axvline(x=old_rel_total.value, linewidth=1.5, color=color_rel, ls=ls_old) 

767 plt.axvline(x=new_rel_total.value, linewidth=1.5, color=color_rel, ls=ls_new) 

768 

769 plt.xlim(plotOptions['range']) 

770 plt.xlabel('arcseconds') 

771 plt.legend(loc='best') 

772 filename = os.path.join(outdir, '{}-histogram.pdf') 

773 plt.savefig(filename.format(name))