lsst.jointcal main-g6000b2fdb9+cc219f2c90
utils.py
Go to the documentation of this file.
1# This file is part of jointcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22"""
23Statistics of jointcal vs. single-frame procesing and diagnostic plots.
24
25Notes
26-----
27Some of the algorithms and data structures in this code are temporary
28kludges and will no longer be necessary once the following are available:
29
30- a composite data structure that contains all ccds from a single visit
31- an n-way matching system that preserves the separations between sources
32"""
33import collections
34import os
35
36import numpy as np
37from astropy import units as u
38
39import lsst.log
40import lsst.afw.table
41import lsst.afw.image
42from lsst.geom import arcseconds
43
44__all__ = ['JointcalStatistics']
45
46MatchDict = collections.namedtuple('MatchDict', ['relative', 'absolute'])
47_LOG = lsst.log.Log.getLogger(__name__)
48
49
51 """
52 Compute statistics on jointcal-processed data, and optionally generate plots.
53
54 Notes
55 -----
56 Instantiate JointcalStatistics and call compute_rms() to get the relevant
57 statistics for e.g. unittests, and call make_plots() to generate a suite of
58 diagnostic plots.
59 """
60
61 def __init__(self, match_radius=0.1*arcseconds, flux_limit=100.0,
62 do_photometry=True, do_astrometry=True,
63 verbose=False):
64 """
65 Parameters
66 ----------
67 match_radius : lsst.geom.Angle
68 match sources within this radius for RMS statistics
69 flux_limit : float
70 Signal/Noise (flux/fluxErr) for sources to be included in the RMS cross-match.
71 100 is a balance between good centroids and enough sources.
72 do_photometry : bool, optional
73 Perform calculations/make plots for photometric metrics.
74 do_astrometry : bool, optional
75 Perform calculations/make plots for astrometric metrics.
76 verbose : bool, optional
77 Print extra things
78 """
79 self.match_radiusmatch_radius = match_radius
80 self.flux_limitflux_limit = flux_limit
81 self.do_photometrydo_photometry = do_photometry
82 self.do_astrometrydo_astrometry = do_astrometry
83 self.verboseverbose = verbose
84 self.loglog = _LOG.getChild('JointcalStatistics')
85
86 def compute_rms(self, data_refs, reference):
87 """
88 Match all data_refs to compute the RMS, for all detections above self.flux_limitflux_limit.
89
90 Parameters
91 ----------
93 A list of data refs to do the calculations between.
94 reference : lsst reference catalog
95 reference catalog to do absolute matching against.
96
97 Returns
98 -------
99 namedtuple:
100 astropy.Quantity
101 Post-jointcal relative RMS of the matched sources.
102 astropy.Quantity
103 Post-jointcal absolute RMS of matched sources.
104 float
105 Post-jointcal photometric repeatability (PA1 from the SRD).
106 """
107
108 # DECAM doesn't have "filter" in its registry, so we have to get the filter names directly.
109 self.filtersfilters = [ref.get('calexp_filterLabel').bandLabel for ref in data_refs]
110 self.visits_per_dataRefvisits_per_dataRef = [ref.dataId['visit'] for ref in data_refs]
111
112 def compute(catalogs, photoCalibs):
113 """Compute the relative and absolute matches in distance and flux."""
114 visit_catalogs = self._make_visit_catalogs_make_visit_catalogs(catalogs, self.visits_per_dataRefvisits_per_dataRef)
115 catalogs = [visit_catalogs[x] for x in self.visits_per_dataRefvisits_per_dataRef]
116 # use the first catalog as the relative reference catalog
117 # NOTE: The "first" catalog depends on the original ordering of the data_refs.
118 # NOTE: Thus, because I'm doing a many-1 match in _make_match_dict,
119 # the number of matches (and thus the details of the match statistics)
120 # will change if the data_refs are ordered differently.
121 # All the more reason to use a proper n-way matcher here. See: DM-8664
122 refcat = catalogs[0]
123 refcalib = photoCalibs[0] if photoCalibs != [] else None
124 dist_rel, flux_rel, ref_flux_rel, source_rel = self._make_match_dict_make_match_dict(refcat,
125 catalogs[1:],
126 photoCalibs[1:],
127 refcalib=refcalib)
128 dist_abs, flux_abs, ref_flux_abs, source_abs = self._make_match_dict_make_match_dict(reference,
129 catalogs,
130 photoCalibs)
131 dist = MatchDict(dist_rel, dist_abs)
132 flux = MatchDict(flux_rel, flux_abs)
133 ref_flux = MatchDict(ref_flux_rel, ref_flux_abs)
134 source = MatchDict(source_rel, source_abs)
135 return dist, flux, ref_flux, source
136
137 old_cats = [ref.get('src') for ref in data_refs]
138 # NOTE: build photoCalibs from existing old Calib objects.
139 # TODO: we can make this a listcomp again once DM-10153 is finished.
140 old_calibs = []
141 if self.do_photometrydo_photometry:
142 old_calibs = [ref.get('calexp_photoCalib') for ref in data_refs]
143
144 self.old_dist, self.old_flux, self.old_ref_flux, self.old_sourceold_source = compute(old_cats, old_calibs)
145
146 # Update coordinates with the new wcs, and get the new photoCalibs.
147 new_cats = [ref.get('src') for ref in data_refs]
148 new_wcss = []
149 if self.do_astrometrydo_astrometry:
150 new_wcss = [ref.get('jointcal_wcs') for ref in data_refs]
151 new_calibs = []
152 if self.do_photometrydo_photometry:
153 new_calibs = [ref.get('jointcal_photoCalib') for ref in data_refs]
154 if self.do_astrometrydo_astrometry:
155 for wcs, cat in zip(new_wcss, new_cats):
156 # update in-place the object coordinates based on the new wcs
158
159 self.new_dist, self.new_flux, self.new_ref_flux, self.new_sourcenew_source = compute(new_cats, new_calibs)
160
161 if self.verboseverbose:
162 print('old, new relative distance matches:',
163 len(self.old_dist.relative), len(self.new_dist.relative))
164 print('old, new absolute distance matches:',
165 len(self.old_dist.absolute), len(self.new_dist.absolute))
166 print('old, new relative flux matches:',
167 len(self.old_flux.relative), len(self.new_flux.relative))
168 print('old, new absolute flux matches:',
169 len(self.old_flux.absolute), len(self.new_flux.absolute))
170
171 if self.do_photometrydo_photometry:
172 self._photometric_rms_photometric_rms()
173 else:
174 self.new_PA1new_PA1 = None
175
176 def rms_total(data):
177 """Compute the total rms across all sources."""
178 total = sum(sum(dd**2) for dd in data.values())
179 n = sum(len(dd) for dd in data.values())
180 return np.sqrt(total/n)
181
182 if self.do_astrometrydo_astrometry:
183 self.old_dist_totalold_dist_total = MatchDict(*(tuple(map(rms_total, self.old_dist))*u.radian).to(u.arcsecond))
184 self.new_dist_totalnew_dist_total = MatchDict(*(tuple(map(rms_total, self.new_dist))*u.radian).to(u.arcsecond))
185 else:
186 self.old_dist_totalold_dist_total = MatchDict(None, None)
187 self.new_dist_totalnew_dist_total = MatchDict(None, None)
188
189 if self.verboseverbose:
190 if self.do_astrometrydo_astrometry:
191 print("relative, absolute astrometry:",
192 self.new_dist_totalnew_dist_total.relative,
193 self.new_dist_totalnew_dist_total.absolute)
194 if self.do_photometrydo_photometry:
195 print("Photometric Accuracy (PA1):", self.new_PA1new_PA1)
196 Rms_result = collections.namedtuple("Rms_result", ["dist_relative", "dist_absolute", "pa1"])
197 return Rms_result(self.new_dist_totalnew_dist_total.relative, self.new_dist_totalnew_dist_total.absolute, self.new_PA1new_PA1)
198
199 def make_plots(self, data_refs, old_wcs_list,
200 name='', interactive=False, per_ccd_plot=False, outdir='.plots'):
201 """
202 Make plots of various quantites to help with debugging.
203 Requires that `compute_rms()` was run first.
204
205 Parameters
206 ----------
208 A list of data refs to do the calculations between.
209 old_wcs_list : list of lsst.afw.image.wcs.Wcs
210 A list of the old (pre-jointcal) WCSs, one-to-one corresponding to data_refs.
211 name : str
212 Name to include in plot titles and save files.
213 interactive : bool
214 Turn on matplotlib interactive mode and drop into a debugger when
215 plotting is finished. Otherwise, use a non-interactive backend.
216 per_ccd_plot : bool
217 Plot the WCS per CCD (takes longer and generates many plots for a large camera)
218 outdir : str
219 directory to save plots to.
220 """
221 import matplotlib
222
223 if not interactive:
224 # Use a non-interactive backend for faster plotting.
225 matplotlib.use('pdf')
226
227 import matplotlib.pyplot as plt
228 import astropy.visualization
229 # make quantities behave nicely when plotted.
230 astropy.visualization.quantity_support()
231 if interactive:
232 plt.ion()
233
234 self.loglog.info("N data_refs: %d", len(data_refs))
235
236 if self.do_photometrydo_photometry:
237 plot_flux_distributions(plt, self.old_magold_mag, self.new_magnew_mag,
238 self.old_weighted_rmsold_weighted_rms, self.new_weighted_rmsnew_weighted_rms,
239 self.faintfaint, self.brightbright, self.old_PA1old_PA1, self.new_PA1new_PA1,
240 name=name, outdir=outdir)
241 self.loglog.info("Photometric accuracy (old, new): {:.2e} {:.2e}".format(self.old_PA1old_PA1,
242 self.new_PA1new_PA1))
243
244 def rms_per_source(data):
245 """Each element of data must already be the "delta" of whatever measurement."""
246 return (np.sqrt([np.mean(dd**2) for dd in data.values()])*u.radian).to(u.arcsecond)
247
248 if self.do_astrometrydo_astrometry:
249 old_dist_rms = MatchDict(*(tuple(map(rms_per_source, self.old_dist))))
250 new_dist_rms = MatchDict(*(tuple(map(rms_per_source, self.new_dist))))
251
252 self.loglog.info("relative RMS (old, new): {:.2e} {:.2e}".format(self.old_dist_totalold_dist_total.relative,
253 self.new_dist_totalnew_dist_total.relative))
254 self.loglog.info("absolute RMS (old, new): {:.2e} {:.2e}".format(self.old_dist_totalold_dist_total.absolute,
255 self.new_dist_totalnew_dist_total.absolute))
256 plot_rms_histogram(plt, old_dist_rms.relative, old_dist_rms.absolute,
257 new_dist_rms.relative, new_dist_rms.absolute,
258 self.old_dist_totalold_dist_total.relative, self.old_dist_totalold_dist_total.absolute,
259 self.new_dist_totalnew_dist_total.relative, self.new_dist_totalnew_dist_total.absolute,
260 name=name, outdir=outdir)
261
262 plot_all_wcs_deltas(plt, data_refs, self.visits_per_dataRefvisits_per_dataRef, old_wcs_list,
263 per_ccd_plot=per_ccd_plot,
264 name=name, outdir=outdir)
265
266 if interactive:
267 plt.show()
268 import pdb
269 pdb.set_trace()
270
271 def _photometric_rms(self, sn_cut=300, magnitude_range=3):
272 """
273 Compute the photometric RMS and the photometric repeatablity values (PA1).
274
275 Parameters
276 ----------
277 sn_cut : float
278 The minimum signal/noise for sources to be included in the PA1 calculation.
279 magnitude_range : float
280 The range of magnitudes above sn_cut to include in the PA1 calculation.
281 """
282 def rms(flux, ref_flux):
283 return np.sqrt([np.mean((ref_flux[dd] - flux[dd])**2) for dd in flux])
284
285 self.old_rmsold_rms = MatchDict(*map(rms, self.old_flux, self.old_ref_flux))
286 self.new_rmsnew_rms = MatchDict(*map(rms, self.new_flux, self.new_ref_flux))
287
288 # we want to use the absolute fluxes for all of these calculations.
289 self.old_refold_ref = np.fromiter(self.old_ref_flux.absolute.values(), dtype=float)
290 self.new_refnew_ref = np.fromiter(self.new_ref_flux.absolute.values(), dtype=float)
291 self.old_magold_mag = u.Quantity(self.old_refold_ref, u.nJy).to_value(u.ABmag)
292 self.new_magnew_mag = u.Quantity(self.new_refnew_ref, u.nJy).to_value(u.ABmag)
293
294 def signal_to_noise(sources, flux_key='slot_PsfFlux_instFlux', sigma_key='slot_PsfFlux_instFluxErr'):
295 """Compute the mean signal/noise per source from a MatchDict of SourceRecords."""
296 result = np.empty(len(sources))
297 for i, src in enumerate(sources.values()):
298 result[i] = np.mean([x[flux_key]/x[sigma_key] for x in src])
299 return result
300
301 old_sn = signal_to_noise(self.old_sourceold_source.absolute)
302 # Find the faint/bright magnitude limits that are the "flat" part of the rms/magnitude relation.
303 self.faintfaint = self.old_magold_mag[old_sn > sn_cut].max()
304 self.brightbright = self.faintfaint - magnitude_range
305 if self.verboseverbose:
306 print("PA1 Magnitude range: {:.3f}, {:.3f}".format(self.brightbright, self.faintfaint))
307 old_good = (self.old_magold_mag < self.faintfaint) & (self.old_magold_mag > self.brightbright)
308 new_good = (self.new_magnew_mag < self.faintfaint) & (self.new_magnew_mag > self.brightbright)
309 self.old_weighted_rmsold_weighted_rms = self.old_rmsold_rms.absolute/self.old_refold_ref
310 self.new_weighted_rmsnew_weighted_rms = self.new_rmsnew_rms.absolute/self.new_refnew_ref
311 self.old_PA1old_PA1 = np.median(self.old_weighted_rmsold_weighted_rms[old_good])
312 self.new_PA1new_PA1 = np.median(self.new_weighted_rmsnew_weighted_rms[new_good])
313
314 def _make_match_dict(self, reference, visit_catalogs, photoCalibs, refcalib=None):
315 """
316 Return several dicts of sourceID:[values] over the catalogs, to be used in RMS calculations.
317
318 Parameters
319 ----------
321 Catalog to do the matching against.
322 visit_catalogs : list of lsst.afw.table.SourceCatalog
323 Visit source catalogs (values() produced by _make_visit_catalogs)
324 to cross-match against reference.
325 photoCalibs : list of lsst.afw.image.PhotoCalib
326 Exposure PhotoCalibs, 1-1 coorespondent with visit_catalogs.
327 refcalib : lsst.afw.image.PhotoCalib or None
328 Pass a PhotoCalib here to use it to compute nanojansky from the
329 reference catalog ADU slot_flux.
330
331 Returns
332 -------
333 distances : dict
334 dict of sourceID: array(separation distances for that source)
335 fluxes : dict
336 dict of sourceID: array(fluxes (nJy) for that source)
337 ref_fluxes : dict
338 dict of sourceID: flux (nJy) of the reference object
339 sources : dict
340 dict of sourceID: list(each SourceRecord that was position-matched
341 to this sourceID)
342 """
343 # If we have no photoCalibs, make it the same length as the others for zipping.
344 if photoCalibs == []:
345 photoCalibs = [[]]*len(visit_catalogs)
346
347 distances = collections.defaultdict(list)
348 fluxes = collections.defaultdict(list)
349 ref_fluxes = {}
350 sources = collections.defaultdict(list)
351 if 'slot_CalibFlux_instFlux' in reference.schema:
352 ref_flux_key = 'slot_CalibFlux'
353 else:
354 ref_flux_key = '{}_flux'
355
356 def get_fluxes(photoCalib, match):
357 """Return (flux, ref_flux) or None if either is invalid."""
358 # NOTE: Protect against negative fluxes: ignore this match if we find one.
359 flux = match[1]['slot_CalibFlux_instFlux']
360 if flux < 0:
361 return None
362 else:
363 flux = photoCalib.instFluxToNanojansky(match[1], "slot_CalibFlux").value
364
365 # NOTE: Have to protect against negative "reference" fluxes too.
366 if 'slot' in ref_flux_key:
367 ref_flux = match[0][ref_flux_key+'_instFlux'] # relative reference flux
368 if ref_flux < 0:
369 return None
370 else:
371 ref_flux = refcalib.instFluxToNanojansky(match[0], ref_flux_key).value
372 else:
373 # refcat fluxes are already in nanojansky.
374 ref_flux = match[0][ref_flux_key.format(filt)]
375 if ref_flux < 0:
376 return None
377
378 Flux = collections.namedtuple('Flux', ('flux', 'ref_flux'))
379 return Flux(flux, ref_flux)
380
381 for cat, photoCalib, filt in zip(visit_catalogs, photoCalibs, self.filtersfilters):
382 good = (cat.get('base_PsfFlux_instFlux')/cat.get('base_PsfFlux_instFluxErr')) > self.flux_limitflux_limit
383 # things the classifier called sources are not extended.
384 good &= (cat.get('base_ClassificationExtendedness_value') == 0)
385 matches = lsst.afw.table.matchRaDec(reference, cat[good], self.match_radiusmatch_radius)
386 for m in matches:
387 if self.do_photometrydo_photometry:
388 flux = get_fluxes(photoCalib, m)
389 if flux is None:
390 continue
391 else:
392 fluxes[m[0].getId()].append(flux.flux)
393 # we can just use assignment here, since the value is always the same.
394 ref_fluxes[m[0].getId()] = flux.ref_flux
395
396 if self.do_astrometrydo_astrometry:
397 # Just use the computed separation distance directly.
398 distances[m[0].getId()].append(m[2])
399
400 sources[m[0].getId()].append(m[1])
401 # Convert to numpy array for easier math
402 for source in distances:
403 distances[source] = np.array(distances[source])
404 for source in fluxes:
405 fluxes[source] = np.array(fluxes[source])
406
407 return distances, fluxes, ref_fluxes, sources
408
409 def _make_visit_catalogs(self, catalogs, visits):
410 """
411 Merge all catalogs from the each visit.
412 NOTE: creating this structure is somewhat slow, and will be unnecessary
413 once a full-visit composite dataset is available.
414
415 Parameters
416 ----------
417 catalogs : list of lsst.afw.table.SourceCatalog
418 Catalogs to combine into per-visit catalogs.
419 visits : list of visit id (usually int)
420 list of visit identifiers, one-to-one correspondent with catalogs.
421
422 Returns
423 -------
424 dict
425 dict of visit: catalog of all sources from all CCDs of that visit.
426 """
427 visit_dict = {v: lsst.afw.table.SourceCatalog(catalogs[0].schema) for v in visits}
428 for v, cat in zip(visits, catalogs):
429 visit_dict[v].extend(cat)
430 # We want catalog contiguity to do object selection later.
431 for v in visit_dict:
432 visit_dict[v] = visit_dict[v].copy(deep=True)
433
434 return visit_dict
435
436
437def plot_flux_distributions(plt, old_mag, new_mag, old_weighted_rms, new_weighted_rms,
438 faint, bright, old_PA1, new_PA1,
439 name='', outdir='.plots'):
440 """Plot various distributions of fluxes and magnitudes.
441
442 Parameters
443 ----------
444 plt : matplotlib.pyplot instance
445 pyplot instance to plot with
446 old_mag : np.array
447 old magnitudes
448 new_mag : np.array
449 new magnitudes
450 old_weighted_rms : np.array
451 old rms weighted by the mean (rms(data)/mean(data))
452 new_weighted_rms : np.array
453 old rms weighted by the mean (rms(data)/mean(data))
454 faint : float
455 Faint end of range that PA1 was computed from.
456 bright : float
457 Bright end of range that PA1 was computed from.
458 old_PA1 : float
459 Old value of PA1, to plot as horizontal line.
460 new_PA1 : float
461 New value of PA1, to plot as horizontal line.
462 name : str
463 Name to include in plot titles and save files.
464 outdir : str, optional
465 Directory to write the saved plots to.
466 """
467
468 import seaborn
469 seaborn.set_style('whitegrid')
470 import scipy.stats
471
472 old_color = 'blue'
473 new_color = 'red'
474 plt.figure()
475 plt.plot(old_mag, old_weighted_rms, '.', color=old_color, label='old')
476 plt.plot(new_mag, new_weighted_rms, '.', color=new_color, label='new')
477 plt.axvline(faint, ls=':', color=old_color)
478 plt.axvline(bright, ls=':', color=old_color)
479 plt.axhline(old_PA1, ls='--', color=old_color)
480 plt.axhline(new_PA1, ls='--', color=new_color)
481 plt.legend(loc='upper left')
482 plt.title('Where is the systematic flux rms limit?')
483 plt.xlabel('magnitude')
484 plt.ylabel('rms/mean per source')
485 filename = os.path.join(outdir, '{}-photometry-PA1.pdf')
486 plt.savefig(filename.format(name))
487
488 plt.figure()
489 seaborn.distplot(old_weighted_rms, fit=scipy.stats.lognorm, kde=False, label="old", color=old_color)
490 seaborn.distplot(new_weighted_rms, fit=scipy.stats.lognorm, kde=False, label="new", color=new_color)
491 plt.title('Source RMS pre/post-jointcal')
492 plt.xlabel('rms(flux)/mean(flux)')
493 plt.ylabel('number')
494 plt.legend(loc='upper right')
495 filename = os.path.join(outdir, '{}-photometry-rms.pdf')
496 plt.savefig(filename.format(name))
497
498
499def plot_all_wcs_deltas(plt, data_refs, visits, old_wcs_list, per_ccd_plot=False,
500 name='', outdir='.plots'):
501 """
502 Various plots of the difference between old and new Wcs.
503
504 Parameters
505 ----------
506 plt : matplotlib.pyplot instance
507 pyplot instance to plot with.
509 A list of data refs to plot.
510 visits : list of visit id (usually int)
511 list of visit identifiers, one-to-one correspondent with catalogs.
512 old_wcs_list : list of lsst.afw.image.wcs.Wcs
513 A list of the old (pre-jointcal) WCSs, one-to-one corresponding to data_refs.
514 per_ccd_plot : bool, optional
515 Make per-ccd plots of the "wcs different" (warning: slow!)
516 name : str
517 Name to include in plot titles and save files.
518 outdir : str, optional
519 Directory to write the saved plots to.
520 """
521
522 plot_wcs_magnitude(plt, data_refs, visits, old_wcs_list, name, outdir=outdir)
523 plot_all_wcs_quivers(plt, data_refs, visits, old_wcs_list, name, outdir=outdir)
524
525 if per_ccd_plot:
526 for i, ref in enumerate(data_refs):
527 md = ref.get('calexp_md')
528 dims = lsst.afw.image.bboxFromMetadata(md).getDimensions()
529 plot_wcs(plt, old_wcs_list[i], ref.get('jointcal_wcs'),
530 dims.getX(), dims.getY(),
531 center=(md.getScalar('CRVAL1'), md.getScalar('CRVAL2')), name='dataRef %d'%i,
532 outdir=outdir)
533
534
535def make_xy_wcs_grid(x_dim, y_dim, wcs1, wcs2, num=50):
536 """Return num x/y grid coordinates for wcs1 and wcs2."""
537 x = np.linspace(0, x_dim, num)
538 y = np.linspace(0, y_dim, num)
539 x1, y1 = wcs_convert(x, y, wcs1)
540 x2, y2 = wcs_convert(x, y, wcs2)
541 return x1, y1, x2, y2
542
543
544def wcs_convert(xv, yv, wcs):
545 """Convert two arrays of x/y points into an on-sky grid."""
546 xout = np.zeros((xv.shape[0], yv.shape[0]))
547 yout = np.zeros((xv.shape[0], yv.shape[0]))
548 for i, x in enumerate(xv):
549 for j, y in enumerate(yv):
550 sky = wcs.pixelToSky(x, y)
551 xout[i, j] = sky.getRa()
552 yout[i, j] = sky.getDec()
553 return xout, yout
554
555
556def plot_all_wcs_quivers(plt, data_refs, visits, old_wcs_list, name, outdir='.plots'):
557 """
558 Make quiver plots of the WCS deltas for each CCD in each visit.
559
560 Parameters
561 ----------
562 plt : matplotlib.pyplot instance
563 pyplot instance to plot with.
565 A list of data refs to plot.
566 visits : list of visit id (usually int)
567 list of visit identifiers, one-to-one correspondent with catalogs.
568 old_wcs_list : list of lsst.afw.image.wcs.Wcs
569 A list of the old (pre-jointcal) WCSs, one-to-one corresponding to data_refs.
570 name : str
571 Name to include in plot titles and save files.
572 outdir : str, optional
573 Directory to write the saved plots to.
574 """
575
576 for visit in visits:
577 fig = plt.figure()
578 # fig.set_tight_layout(True)
579 ax = fig.add_subplot(111)
580 for old_wcs, ref in zip(old_wcs_list, data_refs):
581 if ref.dataId['visit'] != visit:
582 continue
583 md = ref.get('calexp_md')
584 dims = lsst.afw.image.bboxFromMetadata(md).getDimensions()
585 Q = plot_wcs_quivers(ax, old_wcs, ref.get('jointcal_wcs'),
586 dims.getX(), dims.getY())
587 # TODO: add CCD bounding boxes to plot once DM-5503 is finished.
588 # TODO: add a circle for the full focal plane.
589 length = (0.1*u.arcsecond).to(u.radian).value
590 ax.quiverkey(Q, 0.9, 0.95, length, '0.1 arcsec', coordinates='figure', labelpos='W')
591 plt.xlabel('RA')
592 plt.ylabel('Dec')
593 plt.title('visit: {}'.format(visit))
594 filename = os.path.join(outdir, '{}-{}-quivers.pdf')
595 plt.savefig(filename.format(name, visit))
596
597
598def plot_wcs_quivers(ax, wcs1, wcs2, x_dim, y_dim):
599 """
600 Plot the delta between wcs1 and wcs2 as vector arrows.
601
602 Parameters
603 ----------
604 ax : matplotlib.axis
605 Matplotlib axis instance to plot to.
606 wcs1 : lsst.afw.image.wcs.Wcs
607 First WCS to compare.
608 wcs2 : lsst.afw.image.wcs.Wcs
609 Second WCS to compare.
610 x_dim : int
611 Size of array in X-coordinate to make the grid over.
612 y_dim : int
613 Size of array in Y-coordinate to make the grid over.
614 """
615
616 x1, y1, x2, y2 = make_xy_wcs_grid(x_dim, y_dim, wcs1, wcs2)
617 uu = x2 - x1
618 vv = y2 - y1
619 return ax.quiver(x1, y1, uu, vv, units='x', pivot='tail', scale=1e-3, width=1e-5)
620
621
622def plot_wcs_magnitude(plt, data_refs, visits, old_wcs_list, name, outdir='.plots'):
623 """Plot the magnitude of the WCS change between old and new visits as a heat map.
624
625 Parameters
626 ----------
627 plt : matplotlib.pyplot instance
628 pyplot instance to plot with.
630 A list of data refs to plot.
631 visits : list of visit id (usually int)
632 list of visit identifiers, one-to-one correspondent with catalogs.
633 old_wcs_list : list of lsst.afw.image.wcs.Wcs
634 A list of the old (pre-jointcal) WCSs, one-to-one corresponding to data_refs.
635 name : str
636 Name to include in plot titles and save files.
637 outdir : str, optional
638 Directory to write the saved plots to.
639 """
640 for visit in visits:
641 fig = plt.figure()
642 fig.set_tight_layout(True)
643 ax = fig.add_subplot(111)
644 # Start min/max at the "opposite" ends so they always get the first valid value.
645 xmin = np.inf
646 ymin = np.inf
647 xmax = -np.inf
648 ymax = -np.inf
649 for old_wcs, ref in zip(old_wcs_list, data_refs):
650 if ref.dataId['visit'] != visit:
651 continue
652 md = ref.get('calexp_md')
653 dims = lsst.afw.image.bboxFromMetadata(md).getDimensions()
654 x1, y1, x2, y2 = make_xy_wcs_grid(dims.getX(), dims.getY(),
655 old_wcs, ref.get('jointcal_wcs'))
656 uu = x2 - x1
657 vv = y2 - y1
658 extent = (x1[0, 0], x1[-1, -1], y1[0, 0], y1[-1, -1])
659 xmin = min(x1.min(), xmin)
660 ymin = min(y1.min(), ymin)
661 xmax = max(x1.max(), xmax)
662 ymax = max(y1.max(), ymax)
663 magnitude = (np.linalg.norm((uu, vv), axis=0)*u.radian).to(u.arcsecond).value
664 img = ax.imshow(magnitude, vmin=0, vmax=0.3,
665 aspect='auto', extent=extent, cmap=plt.get_cmap('magma'))
666 # TODO: add CCD bounding boxes to the plot once DM-5503 is finished.
667 # TODO: add a circle for the full focal plane.
668
669 # We're reusing only one of the returned images here for colorbar scaling,
670 # but it doesn't matter because we set vmin/vmax so they are all scaled the same.
671 cbar = plt.colorbar(img)
672 cbar.ax.set_ylabel('distortion (arcseconds)')
673 plt.xlim(xmin, xmax)
674 plt.ylim(ymin, ymax)
675 plt.xlabel('RA')
676 plt.ylabel('Dec')
677 plt.title('visit: {}'.format(visit))
678 filename = os.path.join(outdir, '{}-{}-heatmap.pdf')
679 plt.savefig(filename.format(name, visit))
680
681
682def plot_wcs(plt, wcs1, wcs2, x_dim, y_dim, center=(0, 0), name="", outdir='.plots'):
683 """Plot the "distortion map": wcs1-wcs2 delta of points in the CCD grid.
684
685 Parameters
686 ----------
687 plt : matplotlib.pyplot instance
688 pyplot instance to plot with.
689 wcs1 : lsst.afw.image.wcs.Wcs
690 First WCS to compare.
691 wcs2 : lsst.afw.image.wcs.Wcs
692 Second WCS to compare.
693 x_dim : int
694 Size of array in X-coordinate to make the grid over.
695 y_dim : int
696 Size of array in Y-coordinate to make the grid over.
697 center : tuple, optional
698 Center of the data, in on-chip coordinates.
699 name : str
700 Name to include in plot titles and save files.
701 outdir : str, optional
702 Directory to write the saved plots to.
703 """
704
705 plt.figure()
706
707 x1, y1, x2, y2 = make_xy_wcs_grid(x_dim, y_dim, wcs1, wcs2, num=50)
708 plt.plot((x1 - x2) + center[0], (y1 - y2) + center[1], '-')
709 plt.xlabel('delta RA (arcsec)')
710 plt.ylabel('delta Dec (arcsec)')
711 plt.title(name)
712 filename = os.path.join(outdir, '{}-wcs.pdf')
713 plt.savefig(filename.format(name))
714
715
716def plot_rms_histogram(plt, old_rms_relative, old_rms_absolute,
717 new_rms_relative, new_rms_absolute,
718 old_rel_total, old_abs_total, new_rel_total, new_abs_total,
719 name="", outdir='.plots'):
720 """Plot histograms of the source separations and their RMS values.
721
722 Parameters
723 ----------
724 plt : matplotlib.pyplot instance
725 pyplot instance to plot with.
726 old_rms_relative : np.array
727 old relative rms/star
728 old_rms_absolute : np.array
729 old absolute rms/star
730 new_rms_relative : np.array
731 new relative rms/star
732 new_rms_absolute : np.array
733 new absolute rms/star
734 old_rel_total : float
735 old relative rms over all stars
736 old_abs_total : float
737 old absolute rms over all stars
738 new_rel_total : float
739 new relative rms over all stars
740 new_abs_total : float
741 new absolute rms over all stars
742 name : str
743 Name to include in plot titles and save files.
744 outdir : str, optional
745 Directory to write the saved plots to.
746 """
747 plt.figure()
748
749 color_rel = 'black'
750 ls_old = 'dotted'
751 color_abs = 'green'
752 ls_new = 'dashed'
753 plotOptions = {'lw': 2, 'range': (0, 0.1)*u.arcsecond, 'normed': True,
754 'bins': 30, 'histtype': 'step'}
755
756 plt.title('relative vs. absolute: %d vs. %d'%(len(old_rms_relative), len(old_rms_absolute)))
757
758 plt.hist(old_rms_absolute, color=color_abs, ls=ls_old, label='old abs', **plotOptions)
759 plt.hist(new_rms_absolute, color=color_abs, ls=ls_new, label='new abs', **plotOptions)
760
761 plt.hist(old_rms_relative, color=color_rel, ls=ls_old, label='old rel', **plotOptions)
762 plt.hist(new_rms_relative, color=color_rel, ls=ls_new, label='new rel', **plotOptions)
763
764 plt.axvline(x=old_abs_total.value, linewidth=1.5, color=color_abs, ls=ls_old)
765 plt.axvline(x=new_abs_total.value, linewidth=1.5, color=color_abs, ls=ls_new)
766 plt.axvline(x=old_rel_total.value, linewidth=1.5, color=color_rel, ls=ls_old)
767 plt.axvline(x=new_rel_total.value, linewidth=1.5, color=color_rel, ls=ls_new)
768
769 plt.xlim(plotOptions['range'])
770 plt.xlabel('arcseconds')
771 plt.legend(loc='best')
772 filename = os.path.join(outdir, '{}-histogram.pdf')
773 plt.savefig(filename.format(name))
def _make_match_dict(self, reference, visit_catalogs, photoCalibs, refcalib=None)
Definition: utils.py:314
def compute_rms(self, data_refs, reference)
Definition: utils.py:86
def _photometric_rms(self, sn_cut=300, magnitude_range=3)
Definition: utils.py:271
def make_plots(self, data_refs, old_wcs_list, name='', interactive=False, per_ccd_plot=False, outdir='.plots')
Definition: utils.py:200
def _make_visit_catalogs(self, catalogs, visits)
Definition: utils.py:409
def __init__(self, match_radius=0.1 *arcseconds, flux_limit=100.0, do_photometry=True, do_astrometry=True, verbose=False)
Definition: utils.py:63
static Log getLogger(std::string const &loggername)
lsst::geom::Box2I bboxFromMetadata(daf::base::PropertySet &metadata)
void updateSourceCoords(geom::SkyWcs const &wcs, SourceCollection &sourceList)
std::vector< Match< typename Cat1::Record, typename Cat2::Record > > matchRaDec(Cat1 const &cat1, Cat2 const &cat2, lsst::geom::Angle radius, MatchControl const &mc=MatchControl())
def plot_all_wcs_deltas(plt, data_refs, visits, old_wcs_list, per_ccd_plot=False, name='', outdir='.plots')
Definition: utils.py:500
def plot_wcs_quivers(ax, wcs1, wcs2, x_dim, y_dim)
Definition: utils.py:598
def plot_wcs(plt, wcs1, wcs2, x_dim, y_dim, center=(0, 0), name="", outdir='.plots')
Definition: utils.py:682
def wcs_convert(xv, yv, wcs)
Definition: utils.py:544
def make_xy_wcs_grid(x_dim, y_dim, wcs1, wcs2, num=50)
Definition: utils.py:535
def plot_flux_distributions(plt, old_mag, new_mag, old_weighted_rms, new_weighted_rms, faint, bright, old_PA1, new_PA1, name='', outdir='.plots')
Definition: utils.py:439
def plot_rms_histogram(plt, old_rms_relative, old_rms_absolute, new_rms_relative, new_rms_absolute, old_rel_total, old_abs_total, new_rel_total, new_abs_total, name="", outdir='.plots')
Definition: utils.py:719
def plot_all_wcs_quivers(plt, data_refs, visits, old_wcs_list, name, outdir='.plots')
Definition: utils.py:556
def plot_wcs_magnitude(plt, data_refs, visits, old_wcs_list, name, outdir='.plots')
Definition: utils.py:622