lsst.jointcal  master-ga8493ae4fe+6
utils.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 """
3 Statistics of jointcal vs. single-frame procesing and diagnostic plots.
4 
5 NOTE: some of the algorithms and data structures in this code are temporary
6 kludges and will no longer be necessary once the following are available:
7  * a composite data structure that contains all ccds from a single visit
8  * an n-way matching system that preserves the separations between sources
9 """
10 from __future__ import division, print_function, absolute_import
11 from builtins import zip
12 from builtins import object
13 import collections
14 import os
15 
16 import numpy as np
17 from astropy import units as u
18 
19 import lsst.log
20 import lsst.afw.table
21 from lsst.afw.image import fluxFromABMag, abMagFromFlux, bboxFromMetadata
22 from lsst.afw.geom import arcseconds
23 
24 MatchDict = collections.namedtuple('MatchDict', ['relative', 'absolute'])
25 
26 
27 class JointcalStatistics(object):
28  """
29  Compute statistics on jointcal-processed data, and optionally generate plots.
30 
31  Notes
32  -----
33  Instantiate JointcalStatistics and call compute_rms() to get the relevant
34  statistics for e.g. unittests, and call make_plots() to generate a suite of
35  diagnostic plots.
36  """
37 
38  def __init__(self, match_radius=0.1*arcseconds, flux_limit=100.0,
39  do_photometry=True, do_astrometry=True,
40  verbose=False):
41  """
42  Parameters
43  ----------
44  match_radius : lsst.afw.Angle
45  match sources within this radius for RMS statistics
46  flux_limit : float
47  Signal/Noise (flux/fluxSigma) for sources to be included in the RMS cross-match.
48  100 is a balance between good centroids and enough sources.
49  do_photometry : bool, optional
50  Perform calculations/make plots for photometric metrics.
51  do_astrometry : bool, optional
52  Perform calculations/make plots for astrometric metrics.
53  verbose : bool, optional
54  Print extra things
55  """
56  self.match_radius = match_radius
57  self.flux_limit = flux_limit
58  self.do_photometry = do_photometry
59  self.do_astrometry = do_astrometry
60  self.verbose = verbose
61  self.log = lsst.log.Log.getLogger('JointcalStatistics')
62 
63  def compute_rms(self, data_refs, reference):
64  """
65  Match all data_refs to compute the RMS, for all detections above self.flux_limit.
66 
67  Parameters
68  ----------
69  data_refs : list of lsst.daf.persistence.butlerSubset.ButlerDataRef
70  A list of data refs to do the calculations between.
71  reference : lsst reference catalog
72  reference catalog to do absolute matching against.
73 
74  Return
75  ------
76  namedtuple:
77  astropy.Quantity
78  Post-jointcal relative RMS of the matched sources.
79  astropy.Quantity
80  Post-jointcal absolute RMS of matched sources.
81  float
82  Post-jointcal photometric repeatability (PA1 from the SRD).
83  """
84 
85  # DECAM doesn't have "filter" in its registry, so we have to get filter names from VisitInfo.
86  self.filters = [ref.get('calexp').getInfo().getFilter().getName() for ref in data_refs]
87  self.visits_per_dataRef = [ref.dataId['visit'] for ref in data_refs]
88 
89  def compute(catalogs, calibs):
90  """Compute the relative and absolute matches in distance and flux."""
91  visit_catalogs = self._make_visit_catalogs(catalogs, self.visits_per_dataRef)
92  catalogs = [visit_catalogs[x] for x in self.visits_per_dataRef]
93  # use the first catalog as the relative reference catalog
94  # NOTE: The "first" catalog depends on the original ordering of the data_refs.
95  # NOTE: Thus, because I'm doing a many-1 match in _make_match_dict,
96  # the number of matches (and thus the details of the match statistics)
97  # will change if the data_refs are ordered differently.
98  # All the more reason to use a proper n-way matcher here. See: DM-8664
99  refcat = catalogs[0]
100  refcalib = calibs[0]
101  dist_rel, flux_rel, ref_flux_rel, source_rel = self._make_match_dict(refcat,
102  catalogs[1:],
103  calibs[1:],
104  refcalib=refcalib)
105  dist_abs, flux_abs, ref_flux_abs, source_abs = self._make_match_dict(reference, catalogs, calibs)
106  dist = MatchDict(dist_rel, dist_abs)
107  flux = MatchDict(flux_rel, flux_abs)
108  ref_flux = MatchDict(ref_flux_rel, ref_flux_abs)
109  source = MatchDict(source_rel, source_abs)
110  return dist, flux, ref_flux, source
111 
112  old_cats = [ref.get('src') for ref in data_refs]
113  old_calibs = [ref.get('calexp').getCalib() for ref in data_refs]
114  self.old_dist, self.old_flux, self.old_ref_flux, self.old_source = compute(old_cats, old_calibs)
115 
116  # Update coordinates with the new wcs, and get the new Calibs.
117  new_cats = [ref.get('src') for ref in data_refs]
118  new_wcss = [ref.get('wcs') for ref in data_refs]
119  new_calibs = [wcs.getCalib() for wcs in new_wcss]
120  if self.do_astrometry:
121  for wcs, cat in zip(new_wcss, new_cats):
122  # update in-place the object coordinates based on the new wcs
123  lsst.afw.table.utils.updateSourceCoords(wcs.getWcs(), cat)
124 
125  self.new_dist, self.new_flux, self.new_ref_flux, self.new_source = compute(new_cats, new_calibs)
126 
127  if self.verbose:
128  print('old, new relative distance matches:',
129  len(self.old_dist.relative), len(self.new_dist.relative))
130  print('old, new absolute distance matches:',
131  len(self.old_dist.absolute), len(self.new_dist.absolute))
132  print('old, new relative flux matches:',
133  len(self.old_flux.relative), len(self.new_flux.relative))
134  print('old, new absolute flux matches:',
135  len(self.old_flux.absolute), len(self.new_flux.absolute))
136 
137  if self.do_photometry:
138  self._photometric_rms()
139  if self.verbose:
140  print('"photometric factor" for each data ref:')
141  for ref, old, new in zip(data_refs, old_calibs, new_calibs):
142  print(tuple(ref.dataId.values()), new.getFluxMag0()[0]/old.getFluxMag0()[0])
143  else:
144  self.new_PA1 = None
145 
146  def rms_total(data):
147  """Compute the total rms across all sources."""
148  total = sum(sum(dd**2) for dd in data.values())
149  n = sum(len(dd) for dd in data.values())
150  return np.sqrt(total/n)
151 
152  if self.do_astrometry:
153  self.old_dist_total = MatchDict(*(tuple(map(rms_total, self.old_dist))*u.radian).to(u.arcsecond))
154  self.new_dist_total = MatchDict(*(tuple(map(rms_total, self.new_dist))*u.radian).to(u.arcsecond))
155  else:
156  self.old_dist_total = MatchDict(None, None)
157  self.new_dist_total = MatchDict(None, None)
158 
159  Rms_result = collections.namedtuple("Rms_result", ["dist_relative", "dist_absolute", "pa1"])
160  return Rms_result(self.new_dist_total.relative, self.new_dist_total.absolute, self.new_PA1)
161 
162  def make_plots(self, data_refs, old_wcs_list,
163  name='', interactive=False, per_ccd_plot=False, outdir='.plots'):
164  """
165  Make plots of various quantites to help with debugging.
166  Requires that `compute_rms()` was run first.
167 
168  Parameters
169  ----------
170  data_refs : list of lsst.daf.persistence.butlerSubset.ButlerDataRef
171  A list of data refs to do the calculations between.
172  old_wcs_list : list of lsst.afw.image.wcs.Wcs
173  A list of the old (pre-jointcal) WCSs, one-to-one corresponding to data_refs.
174  name : str
175  Name to include in plot titles and save files.
176  interactive : bool
177  Turn on matplotlib interactive mode and drop into a debugger when
178  plotting is finished. Otherwise, use a non-interactive backend.
179  per_ccd_plot : bool
180  Plot the WCS per CCD (takes longer and generates many plots for a large camera)
181  outdir : str
182  directory to save plots to.
183  """
184  import matplotlib
185 
186  if not interactive:
187  # Use a non-interactive backend for faster plotting.
188  matplotlib.use('pdf')
189 
190  import matplotlib.pyplot as plt
191  import astropy.visualization
192  # make quantities behave nicely when plotted.
193  astropy.visualization.quantity_support()
194  if interactive:
195  plt.ion()
196 
197  self.log.info("N data_refs: %d", len(data_refs))
198 
199  if self.do_photometry:
200  plot_flux_distributions(plt, self.old_mag, self.new_mag,
202  self.faint, self.bright, self.old_PA1, self.new_PA1,
203  name=name, outdir=outdir)
204 
205  def rms_per_source(data):
206  """Each element of data must already be the "delta" of whatever measurement."""
207  return (np.sqrt([np.mean(dd**2) for dd in data.values()])*u.radian).to(u.arcsecond)
208 
209  if self.do_astrometry:
210  old_dist_rms = MatchDict(*(tuple(map(rms_per_source, self.old_dist))))
211  new_dist_rms = MatchDict(*(tuple(map(rms_per_source, self.new_dist))))
212 
213  self.log.info("relative RMS (old, new): {:.2e} {:.2e}".format(self.old_dist_total.relative,
214  self.new_dist_total.relative))
215  self.log.info("absolute RMS (old, new): {:.2e} {:.2e}".format(self.old_dist_total.absolute,
216  self.new_dist_total.absolute))
217  plot_rms_histogram(plt, old_dist_rms.relative, old_dist_rms.absolute,
218  new_dist_rms.relative, new_dist_rms.absolute,
219  self.old_dist_total.relative, self.old_dist_total.absolute,
220  self.new_dist_total.relative, self.new_dist_total.absolute,
221  name=name, outdir=outdir)
222 
223  plot_all_wcs_deltas(plt, data_refs, self.visits_per_dataRef, old_wcs_list,
224  per_ccd_plot=per_ccd_plot,
225  name=name, outdir=outdir)
226 
227  if interactive:
228  plt.show()
229  import pdb
230  pdb.set_trace()
231 
232  def _photometric_rms(self, sn_cut=300, magnitude_range=3):
233  """
234  Compute the photometric RMS and the photometric repeatablity values (PA1).
235 
236  Parameters
237  ----------
238  sn_cut : float
239  The minimum signal/noise for sources to be included in the PA1 calculation.
240  magnitude_range : float
241  The range of magnitudes above sn_cut to include in the PA1 calculation.
242  """
243  def rms(flux, ref_flux):
244  return np.sqrt([np.mean((ref_flux[dd] - flux[dd])**2) for dd in flux])
245 
246  self.old_rms = MatchDict(*map(rms, self.old_flux, self.old_ref_flux))
247  self.new_rms = MatchDict(*map(rms, self.new_flux, self.new_ref_flux))
248 
249  # we want to use the absolute fluxes for all of these calculations.
250  self.old_ref = np.fromiter(self.old_ref_flux.absolute.values(), dtype=float)
251  self.new_ref = np.fromiter(self.new_ref_flux.absolute.values(), dtype=float)
252  self.old_mag = np.fromiter((abMagFromFlux(r) for r in self.old_ref), dtype=float)
253  self.new_mag = np.fromiter((abMagFromFlux(r) for r in self.new_ref), dtype=float)
254 
255  def signal_to_noise(sources, flux_key='slot_PsfFlux_flux', sigma_key='slot_PsfFlux_fluxSigma'):
256  """Compute the mean signal/noise per source from a MatchDict of SourceRecords."""
257  result = np.empty(len(sources))
258  for i, src in enumerate(sources.values()):
259  result[i] = np.mean([x[flux_key]/x[sigma_key] for x in src])
260  return result
261 
262  old_sn = signal_to_noise(self.old_source.absolute)
263  # Find the faint/bright magnitude limits that are the "flat" part of the rms/magnitude relation.
264  self.faint = self.old_mag[old_sn > sn_cut].max()
265  self.bright = self.faint - magnitude_range
266  if self.verbose:
267  print("PA1 Magnitude range: {:.3f}, {:.3f}".format(self.bright, self.faint))
268  old_good = (self.old_mag < self.faint) & (self.old_mag > self.bright)
269  new_good = (self.new_mag < self.faint) & (self.new_mag > self.bright)
270  self.old_weighted_rms = self.old_rms.absolute/self.old_ref
271  self.new_weighted_rms = self.new_rms.absolute/self.new_ref
272  self.old_PA1 = np.median(self.old_weighted_rms[old_good])
273  self.new_PA1 = np.median(self.new_weighted_rms[new_good])
274 
275  def _make_match_dict(self, reference, visit_catalogs, calibs, refcalib=None):
276  """
277  Return several dicts of sourceID:[values] over the catalogs, to be used in RMS calculations.
278 
279  Parameters
280  ----------
281  reference : lsst.afw.table.SourceCatalog
282  Catalog to do the matching against.
283  visit_catalogs : list of lsst.afw.table.SourceCatalog
284  Visit source catalogs (values() produced by _make_visit_catalogs)
285  to cross-match against reference.
286  calibs : list of lsst.afw.image.Calib
287  Exposure calibs, 1-1 coorespondent with visit_catalogs.
288  refcalib : lsst.afw.image.Calib or None
289  Pass a Calib here to use it to compute Janskys from the reference catalog ADU slot_flux.
290 
291  Returns
292  -------
293  distances : dict
294  dict of sourceID: array(separation distances for that source)
295  fluxes : dict
296  dict of sourceID: array(fluxes (Jy) for that source)
297  ref_fluxes : dict
298  dict of sourceID: flux (Jy) of the reference object
299  sources : dict
300  dict of sourceID: list(each SourceRecord that was position-matched to this sourceID)
301  """
302 
303  distances = collections.defaultdict(list)
304  fluxes = collections.defaultdict(list)
305  ref_fluxes = {}
306  sources = collections.defaultdict(list)
307  if 'slot_CalibFlux_flux' in reference.schema:
308  ref_flux_key = 'slot_CalibFlux_flux'
309  else:
310  ref_flux_key = '{}_flux'
311 
312  def get_fluxes(match):
313  """Return (flux, ref_flux) or None if either is invalid."""
314  # NOTE: Protect against negative fluxes: ignore this match if we find one.
315  flux = match[1]['slot_CalibFlux_flux']
316  if flux < 0:
317  return None
318  else:
319  # convert to magnitudes and then Janskys, for a useable flux.
320  flux = fluxFromABMag(calib.getMagnitude(flux))
321 
322  # NOTE: Have to protect against negative reference fluxes too.
323  if 'slot' in ref_flux_key:
324  ref_flux = match[0][ref_flux_key]
325  if ref_flux < 0:
326  return None
327  else:
328  ref_flux = fluxFromABMag(refcalib.getMagnitude(ref_flux))
329  else:
330  # a.net fluxes are already in Janskys.
331  ref_flux = match[0][ref_flux_key.format(filt)]
332  if ref_flux < 0:
333  return None
334 
335  Flux = collections.namedtuple('Flux', ('flux', 'ref_flux'))
336  return Flux(flux, ref_flux)
337 
338  for cat, calib, filt in zip(visit_catalogs, calibs, self.filters):
339  good = (cat.get('base_PsfFlux_flux')/cat.get('base_PsfFlux_fluxSigma')) > self.flux_limit
340  # things the classifier called sources are not extended.
341  good &= (cat.get('base_ClassificationExtendedness_value') == 0)
342  matches = lsst.afw.table.matchRaDec(reference, cat[good], self.match_radius)
343  for m in matches:
344  if self.do_photometry:
345  flux = get_fluxes(m)
346  if flux is None:
347  continue
348  else:
349  fluxes[m[0].getId()].append(flux.flux)
350  # we can just use assignment here, since the value is always the same.
351  ref_fluxes[m[0].getId()] = flux.ref_flux
352 
353  if self.do_astrometry:
354  # Just use the computed separation distance directly.
355  distances[m[0].getId()].append(m[2])
356 
357  sources[m[0].getId()].append(m[1])
358  # Convert to numpy array for easier math
359  for source in distances:
360  distances[source] = np.array(distances[source])
361  for source in fluxes:
362  fluxes[source] = np.array(fluxes[source])
363 
364  return distances, fluxes, ref_fluxes, sources
365 
366  def _make_visit_catalogs(self, catalogs, visits):
367  """
368  Merge all catalogs from the each visit.
369  NOTE: creating this structure is somewhat slow, and will be unnecessary
370  once a full-visit composite dataset is available.
371 
372  Parameters
373  ----------
374  catalogs : list of lsst.afw.table.SourceCatalog
375  Catalogs to combine into per-visit catalogs.
376  visits : list of visit id (usually int)
377  list of visit identifiers, one-to-one correspondent with catalogs.
378 
379  Returns
380  -------
381  dict
382  dict of visit: catalog of all sources from all CCDs of that visit.
383  """
384  visit_dict = {v: lsst.afw.table.SourceCatalog(catalogs[0].schema) for v in visits}
385  for v, cat in zip(visits, catalogs):
386  visit_dict[v].extend(cat)
387  # We want catalog contiguity to do object selection later.
388  for v in visit_dict:
389  visit_dict[v] = visit_dict[v].copy(deep=True)
390 
391  return visit_dict
392 
393 
394 def plot_flux_distributions(plt, old_mag, new_mag, old_weighted_rms, new_weighted_rms,
395  faint, bright, old_PA1, new_PA1,
396  name='', outdir='.plots'):
397  """Plot various distributions of fluxes and magnitudes.
398 
399  Parameters
400  ----------
401  plt : matplotlib.pyplot instance
402  pyplot instance to plot with
403  old_mag : np.array
404  old magnitudes
405  new_mag : np.array
406  new magnitudes
407  old_weighted_rms : np.array
408  old rms weighted by the mean (rms(data)/mean(data))
409  new_weighted_rms : np.array
410  old rms weighted by the mean (rms(data)/mean(data))
411  faint : float
412  Faint end of range that PA1 was computed from.
413  bright : float
414  Bright end of range that PA1 was computed from.
415  old_PA1 : float
416  Old value of PA1, to plot as horizontal line.
417  new_PA1 : float
418  New value of PA1, to plot as horizontal line.
419  name : str
420  Name to include in plot titles and save files.
421  outdir : str, optional
422  Directory to write the saved plots to.
423  """
424 
425  import seaborn
426  seaborn.set_style('whitegrid')
427  import scipy.stats
428 
429  old_color = 'blue'
430  new_color = 'red'
431  plt.figure()
432  plt.plot(old_mag, old_weighted_rms, '.', color=old_color, label='old')
433  plt.plot(new_mag, new_weighted_rms, '.', color=new_color, label='new')
434  plt.axvline(faint, ls=':', color=old_color)
435  plt.axvline(bright, ls=':', color=old_color)
436  plt.axhline(old_PA1, ls='--', color=old_color)
437  plt.axhline(new_PA1, ls='--', color=new_color)
438  plt.legend(loc='upper left')
439  plt.title('Where is the systematic flux rms limit?')
440  plt.xlabel('magnitude')
441  plt.ylabel('rms/mean per source')
442  filename = os.path.join(outdir, '{}-photometry-PA1.pdf')
443  plt.savefig(filename.format(name))
444 
445  plt.figure()
446  seaborn.distplot(old_weighted_rms, fit=scipy.stats.lognorm, kde=False, label="old", color=old_color)
447  seaborn.distplot(new_weighted_rms, fit=scipy.stats.lognorm, kde=False, label="new", color=new_color)
448  plt.title('Source RMS pre/post-jointcal')
449  plt.xlabel('rms(flux)/mean(flux)')
450  plt.ylabel('number')
451  plt.legend(loc='upper right')
452  filename = os.path.join(outdir, '{}-photometry-rms.pdf')
453  plt.savefig(filename.format(name))
454 
455 
456 def plot_all_wcs_deltas(plt, data_refs, visits, old_wcs_list, per_ccd_plot=False,
457  name='', outdir='.plots'):
458  """
459  Various plots of the difference between old and new Wcs.
460 
461  Parameters
462  ----------
463  plt : matplotlib.pyplot instance
464  pyplot instance to plot with.
465  data_refs : list of lsst.daf.persistence.butlerSubset.ButlerDataRef
466  A list of data refs to plot.
467  visits : list of visit id (usually int)
468  list of visit identifiers, one-to-one correspondent with catalogs.
469  old_wcs_list : list of lsst.afw.image.wcs.Wcs
470  A list of the old (pre-jointcal) WCSs, one-to-one corresponding to data_refs.
471  per_ccd_plot : bool, optional
472  Make per-ccd plots of the "wcs different" (warning: slow!)
473  name : str
474  Name to include in plot titles and save files.
475  outdir : str, optional
476  Directory to write the saved plots to.
477  """
478 
479  plot_wcs_magnitude(plt, data_refs, visits, old_wcs_list, name, outdir=outdir)
480  plot_all_wcs_quivers(plt, data_refs, visits, old_wcs_list, name, outdir=outdir)
481 
482  if per_ccd_plot:
483  for i, ref in enumerate(data_refs):
484  md = ref.get('calexp_md')
485  dims = bboxFromMetadata(md).getDimensions()
486  plot_wcs(plt, old_wcs_list[i], ref.get('wcs').getWcs(),
487  dims.getX(), dims.getY(),
488  center=(md.get('CRVAL1'), md.get('CRVAL2')), name='dataRef %d'%i,
489  outdir=outdir)
490 
491 
492 def make_xy_wcs_grid(x_dim, y_dim, wcs1, wcs2, num=50):
493  """Return num x/y grid coordinates for wcs1 and wcs2."""
494  x = np.linspace(0, x_dim, num)
495  y = np.linspace(0, y_dim, num)
496  x1, y1 = wcs_convert(x, y, wcs1)
497  x2, y2 = wcs_convert(x, y, wcs2)
498  return x1, y1, x2, y2
499 
500 
501 def wcs_convert(xv, yv, wcs):
502  """Convert two arrays of x/y points into an on-sky grid."""
503  xout = np.zeros((xv.shape[0], yv.shape[0]))
504  yout = np.zeros((xv.shape[0], yv.shape[0]))
505  for i, x in enumerate(xv):
506  for j, y in enumerate(yv):
507  sky = wcs.pixelToSky(x, y).toFk5()
508  xout[i, j] = sky.getRa()
509  yout[i, j] = sky.getDec()
510  return xout, yout
511 
512 
513 def plot_all_wcs_quivers(plt, data_refs, visits, old_wcs_list, name, outdir='.plots'):
514  """
515  Make quiver plots of the WCS deltas for each CCD in each visit.
516 
517  Parameters
518  ----------
519  plt : matplotlib.pyplot instance
520  pyplot instance to plot with.
521  data_refs : list of lsst.daf.persistence.butlerSubset.ButlerDataRef
522  A list of data refs to plot.
523  visits : list of visit id (usually int)
524  list of visit identifiers, one-to-one correspondent with catalogs.
525  old_wcs_list : list of lsst.afw.image.wcs.Wcs
526  A list of the old (pre-jointcal) WCSs, one-to-one corresponding to data_refs.
527  name : str
528  Name to include in plot titles and save files.
529  outdir : str, optional
530  Directory to write the saved plots to.
531  """
532 
533  for visit in visits:
534  fig = plt.figure()
535  # fig.set_tight_layout(True)
536  ax = fig.add_subplot(111)
537  for old_wcs, ref in zip(old_wcs_list, data_refs):
538  if ref.dataId['visit'] != visit:
539  continue
540  md = ref.get('calexp_md')
541  dims = bboxFromMetadata(md).getDimensions()
542  Q = plot_wcs_quivers(ax, old_wcs, ref.get('wcs').getWcs(),
543  dims.getX(), dims.getY())
544  # TODO: add CCD bounding boxes to plot once DM-5503 is finished.
545  # TODO: add a circle for the full focal plane.
546  length = (0.1*u.arcsecond).to(u.radian).value
547  ax.quiverkey(Q, 0.9, 0.95, length, '0.1 arcsec', coordinates='figure', labelpos='W')
548  plt.xlabel('RA')
549  plt.ylabel('Dec')
550  plt.title('visit: {}'.format(visit))
551  filename = os.path.join(outdir, '{}-{}-quivers.pdf')
552  plt.savefig(filename.format(name, visit))
553 
554 
555 def plot_wcs_quivers(ax, wcs1, wcs2, x_dim, y_dim):
556  """
557  Plot the delta between wcs1 and wcs2 as vector arrows.
558 
559  Parameters
560  ----------
561  ax : matplotlib.axis
562  Matplotlib axis instance to plot to.
563  wcs1 : lsst.afw.image.wcs.Wcs
564  First WCS to compare.
565  wcs2 : lsst.afw.image.wcs.Wcs
566  Second WCS to compare.
567  x_dim : int
568  Size of array in X-coordinate to make the grid over.
569  y_dim : int
570  Size of array in Y-coordinate to make the grid over.
571  """
572 
573  x1, y1, x2, y2 = make_xy_wcs_grid(x_dim, y_dim, wcs1, wcs2)
574  uu = x2 - x1
575  vv = y2 - y1
576  return ax.quiver(x1, y1, uu, vv, units='x', pivot='tail', scale=1e-3, width=1e-5)
577 
578 
579 def plot_wcs_magnitude(plt, data_refs, visits, old_wcs_list, name, outdir='.plots'):
580  """Plot the magnitude of the WCS change between old and new visits as a heat map.
581 
582  Parameters
583  ----------
584  plt : matplotlib.pyplot instance
585  pyplot instance to plot with.
586  data_refs : list of lsst.daf.persistence.butlerSubset.ButlerDataRef
587  A list of data refs to plot.
588  visits : list of visit id (usually int)
589  list of visit identifiers, one-to-one correspondent with catalogs.
590  old_wcs_list : list of lsst.afw.image.wcs.Wcs
591  A list of the old (pre-jointcal) WCSs, one-to-one corresponding to data_refs.
592  name : str
593  Name to include in plot titles and save files.
594  outdir : str, optional
595  Directory to write the saved plots to.
596  """
597  for visit in visits:
598  fig = plt.figure()
599  fig.set_tight_layout(True)
600  ax = fig.add_subplot(111)
601  # Start min/max at the "opposite" ends so they always get the first valid value.
602  xmin = np.inf
603  ymin = np.inf
604  xmax = -np.inf
605  ymax = -np.inf
606  for old_wcs, ref in zip(old_wcs_list, data_refs):
607  if ref.dataId['visit'] != visit:
608  continue
609  md = ref.get('calexp_md')
610  dims = bboxFromMetadata(md).getDimensions()
611  x1, y1, x2, y2 = make_xy_wcs_grid(dims.getX(), dims.getY(),
612  old_wcs, ref.get('wcs').getWcs())
613  uu = x2 - x1
614  vv = y2 - y1
615  extent = (x1[0, 0], x1[-1, -1], y1[0, 0], y1[-1, -1])
616  xmin = min(x1.min(), xmin)
617  ymin = min(y1.min(), ymin)
618  xmax = max(x1.max(), xmax)
619  ymax = max(y1.max(), ymax)
620  magnitude = (np.linalg.norm((uu, vv), axis=0)*u.radian).to(u.arcsecond).value
621  img = ax.imshow(magnitude, vmin=0, vmax=0.3,
622  aspect='auto', extent=extent, cmap=plt.get_cmap('magma'))
623  # TODO: add CCD bounding boxes to the plot once DM-5503 is finished.
624  # TODO: add a circle for the full focal plane.
625 
626  # We're reusing only one of the returned images here for colorbar scaling,
627  # but it doesn't matter because we set vmin/vmax so they are all scaled the same.
628  cbar = plt.colorbar(img)
629  cbar.ax.set_ylabel('distortion (arcseconds)')
630  plt.xlim(xmin, xmax)
631  plt.ylim(ymin, ymax)
632  plt.xlabel('RA')
633  plt.ylabel('Dec')
634  plt.title('visit: {}'.format(visit))
635  filename = os.path.join(outdir, '{}-{}-heatmap.pdf')
636  plt.savefig(filename.format(name, visit))
637 
638 
639 def plot_wcs(plt, wcs1, wcs2, x_dim, y_dim, center=(0, 0), name="", outdir='.plots'):
640  """Plot the "distortion map": wcs1-wcs2 delta of points in the CCD grid.
641 
642  Parameters
643  ----------
644  plt : matplotlib.pyplot instance
645  pyplot instance to plot with.
646  wcs1 : lsst.afw.image.wcs.Wcs
647  First WCS to compare.
648  wcs2 : lsst.afw.image.wcs.Wcs
649  Second WCS to compare.
650  x_dim : int
651  Size of array in X-coordinate to make the grid over.
652  y_dim : int
653  Size of array in Y-coordinate to make the grid over.
654  center : tuple, optional
655  Center of the data, in on-chip coordinates.
656  name : str
657  Name to include in plot titles and save files.
658  outdir : str, optional
659  Directory to write the saved plots to.
660  """
661 
662  plt.figure()
663 
664  x1, y1, x2, y2 = make_xy_wcs_grid(x_dim, y_dim, wcs1, wcs2, num=50)
665  plt.plot((x1 - x2) + center[0], (y1 - y2) + center[1], '-')
666  plt.xlabel('delta RA (arcsec)')
667  plt.ylabel('delta Dec (arcsec)')
668  plt.title(name)
669  filename = os.path.join(outdir, '{}-wcs.pdf')
670  plt.savefig(filename.format(name))
671 
672 
673 def plot_rms_histogram(plt, old_rms_relative, old_rms_absolute,
674  new_rms_relative, new_rms_absolute,
675  old_rel_total, old_abs_total, new_rel_total, new_abs_total,
676  name="", outdir='.plots'):
677  """Plot histograms of the source separations and their RMS values.
678 
679  Parameters
680  ----------
681  plt : matplotlib.pyplot instance
682  pyplot instance to plot with.
683  old_rms_relative : np.array
684  old relative rms/star
685  old_rms_absolute : np.array
686  old absolute rms/star
687  new_rms_relative : np.array
688  new relative rms/star
689  new_rms_absolute : np.array
690  new absolute rms/star
691  old_rel_total : float
692  old relative rms over all stars
693  old_abs_total : float
694  old absolute rms over all stars
695  new_rel_total : float
696  new relative rms over all stars
697  new_abs_total : float
698  new absolute rms over all stars
699  name : str
700  Name to include in plot titles and save files.
701  outdir : str, optional
702  Directory to write the saved plots to.
703  """
704  plt.figure()
705 
706  color_rel = 'black'
707  ls_old = 'dotted'
708  color_abs = 'green'
709  ls_new = 'dashed'
710  plotOptions = {'lw': 2, 'range': (0, 0.1)*u.arcsecond, 'normed': True,
711  'bins': 30, 'histtype': 'step'}
712 
713  plt.title('relative vs. absolute: %d vs. %d'%(len(old_rms_relative), len(old_rms_absolute)))
714 
715  plt.hist(old_rms_absolute, color=color_abs, ls=ls_old, label='old abs', **plotOptions)
716  plt.hist(new_rms_absolute, color=color_abs, ls=ls_new, label='new abs', **plotOptions)
717 
718  plt.hist(old_rms_relative, color=color_rel, ls=ls_old, label='old rel', **plotOptions)
719  plt.hist(new_rms_relative, color=color_rel, ls=ls_new, label='new rel', **plotOptions)
720 
721  plt.axvline(x=old_abs_total.value, linewidth=1.5, color=color_abs, ls=ls_old)
722  plt.axvline(x=new_abs_total.value, linewidth=1.5, color=color_abs, ls=ls_new)
723  plt.axvline(x=old_rel_total.value, linewidth=1.5, color=color_rel, ls=ls_old)
724  plt.axvline(x=new_rel_total.value, linewidth=1.5, color=color_rel, ls=ls_new)
725 
726  plt.xlim(plotOptions['range'])
727  plt.xlabel('arcseconds')
728  plt.legend(loc='best')
729  filename = os.path.join(outdir, '{}-histogram.pdf')
730  plt.savefig(filename.format(name))
def _photometric_rms(self, sn_cut=300, magnitude_range=3)
Definition: utils.py:232
def make_plots(self, data_refs, old_wcs_list, name='', interactive=False, per_ccd_plot=False, outdir='.plots')
Definition: utils.py:163
def __init__(self, match_radius=0.1 *arcseconds, flux_limit=100.0, do_photometry=True, do_astrometry=True, verbose=False)
Definition: utils.py:40
def wcs_convert(xv, yv, wcs)
Definition: utils.py:501
def _make_match_dict(self, reference, visit_catalogs, calibs, refcalib=None)
Definition: utils.py:275
def plot_rms_histogram(plt, old_rms_relative, old_rms_absolute, new_rms_relative, new_rms_absolute, old_rel_total, old_abs_total, new_rel_total, new_abs_total, name="", outdir='.plots')
Definition: utils.py:676
def plot_all_wcs_deltas(plt, data_refs, visits, old_wcs_list, per_ccd_plot=False, name='', outdir='.plots')
Definition: utils.py:457
def plot_wcs(plt, wcs1, wcs2, x_dim, y_dim, center=(0, 0), name="", outdir='.plots')
Definition: utils.py:639
def plot_wcs_quivers(ax, wcs1, wcs2, x_dim, y_dim)
Definition: utils.py:555
def make_xy_wcs_grid(x_dim, y_dim, wcs1, wcs2, num=50)
Definition: utils.py:492
def plot_flux_distributions(plt, old_mag, new_mag, old_weighted_rms, new_weighted_rms, faint, bright, old_PA1, new_PA1, name='', outdir='.plots')
Definition: utils.py:396
def compute_rms(self, data_refs, reference)
Definition: utils.py:63
def plot_wcs_magnitude(plt, data_refs, visits, old_wcs_list, name, outdir='.plots')
Definition: utils.py:579
def plot_all_wcs_quivers(plt, data_refs, visits, old_wcs_list, name, outdir='.plots')
Definition: utils.py:513
def _make_visit_catalogs(self, catalogs, visits)
Definition: utils.py:366