lsst.ip.diffim  14.0-9-g330837b+3
utils.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2016 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 """Support utilities for Measuring sources"""
23 from __future__ import absolute_import, division, print_function
24 
25 from builtins import zip
26 from builtins import str
27 from builtins import range
28 from builtins import object
29 import numpy as np
30 
31 import lsst.afw.detection as afwDet
32 import lsst.afw.geom as afwGeom
33 import lsst.afw.image as afwImage
34 import lsst.afw.math as afwMath
35 import lsst.afw.table as afwTable
36 import lsst.afw.display.ds9 as ds9
37 import lsst.afw.display.utils as displayUtils
38 from lsst.log import Log
39 import lsst.meas.algorithms as measAlg
40 import lsst.meas.base as measBase
41 from .dipoleFitTask import DipoleFitAlgorithm
42 from . import diffimLib
43 from . import diffimTools
44 
45 # Export DipoleTestImage to expose fake image generating funcs
46 __all__ = ("DipoleTestImage")
47 
48 
49 keptPlots = False # Have we arranged to keep spatial plots open?
50 
51 
52 def showSourceSet(sSet, xy0=(0, 0), frame=0, ctype=ds9.GREEN, symb="+", size=2):
53  """Draw the (XAstrom, YAstrom) positions of a set of Sources. Image has the given XY0"""
54 
55  with ds9.Buffering():
56  for s in sSet:
57  xc, yc = s.getXAstrom() - xy0[0], s.getYAstrom() - xy0[1]
58 
59  if symb == "id":
60  ds9.dot(str(s.getId()), xc, yc, frame=frame, ctype=ctype, size=size)
61  else:
62  ds9.dot(symb, xc, yc, frame=frame, ctype=ctype, size=size)
63 
64 
65 # Kernel display utilities
66 #
67 
68 
69 def showKernelSpatialCells(maskedIm, kernelCellSet, showChi2=False, symb="o",
70  ctype=None, ctypeUnused=None, ctypeBad=None, size=3,
71  frame=None, title="Spatial Cells"):
72  """Show the SpatialCells. If symb is something that ds9.dot
73  understands (e.g. "o"), the top nMaxPerCell candidates will be
74  indicated with that symbol, using ctype and size"""
75 
76  ds9.mtv(maskedIm, frame=frame, title=title)
77  with ds9.Buffering():
78  origin = [-maskedIm.getX0(), -maskedIm.getY0()]
79  for cell in kernelCellSet.getCellList():
80  displayUtils.drawBBox(cell.getBBox(), origin=origin, frame=frame)
81 
82  goodies = ctypeBad is None
83  for cand in cell.begin(goodies):
84  xc, yc = cand.getXCenter() + origin[0], cand.getYCenter() + origin[1]
85  if cand.getStatus() == afwMath.SpatialCellCandidate.BAD:
86  color = ctypeBad
87  elif cand.getStatus() == afwMath.SpatialCellCandidate.GOOD:
88  color = ctype
89  elif cand.getStatus() == afwMath.SpatialCellCandidate.UNKNOWN:
90  color = ctypeUnused
91  else:
92  continue
93 
94  if color:
95  ds9.dot(symb, xc, yc, frame=frame, ctype=color, size=size)
96 
97  if showChi2:
98  rchi2 = cand.getChi2()
99  if rchi2 > 1e100:
100  rchi2 = np.nan
101  ds9.dot("%d %.1f" % (cand.getId(), rchi2),
102  xc - size, yc - size - 4, frame=frame, ctype=color, size=size)
103 
104 
105 def showDiaSources(sources, exposure, isFlagged, isDipole, frame=None):
106  """Display Dia Sources
107  """
108  #
109  # Show us the ccandidates
110  #
111  # Too many mask planes in diffims
112  for plane in ("BAD", "CR", "EDGE", "INTERPOlATED", "INTRP", "SAT", "SATURATED"):
113  ds9.setMaskPlaneVisibility(plane, False)
114 
115  mos = displayUtils.Mosaic()
116  for i in range(len(sources)):
117  source = sources[i]
118  badFlag = isFlagged[i]
119  dipoleFlag = isDipole[i]
120  bbox = source.getFootprint().getBBox()
121  stamp = exposure.Factory(exposure, bbox, True)
122  im = displayUtils.Mosaic(gutter=1, background=0, mode="x")
123  im.append(stamp.getMaskedImage())
124  lab = "%.1f,%.1f:" % (source.getX(), source.getY())
125  if badFlag:
126  ctype = ds9.RED
127  lab += "BAD"
128  if dipoleFlag:
129  ctype = ds9.YELLOW
130  lab += "DIPOLE"
131  if not badFlag and not dipoleFlag:
132  ctype = ds9.GREEN
133  lab += "OK"
134  mos.append(im.makeMosaic(), lab, ctype)
135  title = "Dia Sources"
136  mosaicImage = mos.makeMosaic(frame=frame, title=title)
137  return mosaicImage
138 
139 
140 def showKernelCandidates(kernelCellSet, kernel, background, frame=None, showBadCandidates=True,
141  resids=False, kernels=False):
142  """Display the Kernel candidates.
143  If kernel is provided include spatial model and residuals;
144  If chi is True, generate a plot of residuals/sqrt(variance), i.e. chi
145  """
146 
147  #
148  # Show us the ccandidates
149  #
150  if kernels:
151  mos = displayUtils.Mosaic(gutter=5, background=0)
152  else:
153  mos = displayUtils.Mosaic(gutter=5, background=-1)
154  #
155  candidateCenters = []
156  candidateCentersBad = []
157  candidateIndex = 0
158  for cell in kernelCellSet.getCellList():
159  for cand in cell.begin(False): # include bad candidates
160  # Original difference image; if does not exist, skip candidate
161  try:
162  resid = cand.getDifferenceImage(diffimLib.KernelCandidateF.ORIG)
163  except Exception:
164  continue
165 
166  rchi2 = cand.getChi2()
167  if rchi2 > 1e100:
168  rchi2 = np.nan
169 
170  if not showBadCandidates and cand.isBad():
171  continue
172 
173  im_resid = displayUtils.Mosaic(gutter=1, background=-0.5, mode="x")
174 
175  try:
176  im = cand.getScienceMaskedImage()
177  im = im.Factory(im, True)
178  im.setXY0(cand.getScienceMaskedImage().getXY0())
179  except Exception:
180  continue
181  if (not resids and not kernels):
182  im_resid.append(im.Factory(im, True))
183  try:
184  im = cand.getTemplateMaskedImage()
185  im = im.Factory(im, True)
186  im.setXY0(cand.getTemplateMaskedImage().getXY0())
187  except Exception:
188  continue
189  if (not resids and not kernels):
190  im_resid.append(im.Factory(im, True))
191 
192  # Difference image with original basis
193  if resids:
194  var = resid.getVariance()
195  var = var.Factory(var, True)
196  np.sqrt(var.getArray(), var.getArray()) # inplace sqrt
197  resid = resid.getImage()
198  resid /= var
199  bbox = kernel.shrinkBBox(resid.getBBox())
200  resid = resid.Factory(resid, bbox, True)
201  elif kernels:
202  kim = cand.getKernelImage(diffimLib.KernelCandidateF.ORIG).convertF()
203  resid = kim.Factory(kim, True)
204  im_resid.append(resid)
205 
206  # residuals using spatial model
207  ski = afwImage.ImageD(kernel.getDimensions())
208  kernel.computeImage(ski, False, int(cand.getXCenter()), int(cand.getYCenter()))
209  sk = afwMath.FixedKernel(ski)
210  sbg = 0.0
211  if background:
212  sbg = background(int(cand.getXCenter()), int(cand.getYCenter()))
213  sresid = cand.getDifferenceImage(sk, sbg)
214  resid = sresid
215  if resids:
216  resid = sresid.getImage()
217  resid /= var
218  bbox = kernel.shrinkBBox(resid.getBBox())
219  resid = resid.Factory(resid, bbox, True)
220  elif kernels:
221  kim = ski.convertF()
222  resid = kim.Factory(kim, True)
223  im_resid.append(resid)
224 
225  im = im_resid.makeMosaic()
226 
227  lab = "%d chi^2 %.1f" % (cand.getId(), rchi2)
228  ctype = ds9.RED if cand.isBad() else ds9.GREEN
229 
230  mos.append(im, lab, ctype)
231 
232  if False and np.isnan(rchi2):
233  ds9.mtv(cand.getScienceMaskedImage.getImage(), title="candidate", frame=1)
234  print("rating", cand.getCandidateRating())
235 
236  im = cand.getScienceMaskedImage()
237  center = (candidateIndex, cand.getXCenter() - im.getX0(), cand.getYCenter() - im.getY0())
238  candidateIndex += 1
239  if cand.isBad():
240  candidateCentersBad.append(center)
241  else:
242  candidateCenters.append(center)
243 
244  if resids:
245  title = "chi Diffim"
246  elif kernels:
247  title = "Kernels"
248  else:
249  title = "Candidates & residuals"
250  mosaicImage = mos.makeMosaic(frame=frame, title=title)
251 
252  return mosaicImage
253 
254 
255 def showKernelBasis(kernel, frame=None):
256  """Display a Kernel's basis images
257  """
258  mos = displayUtils.Mosaic()
259 
260  for k in kernel.getKernelList():
261  im = afwImage.ImageD(k.getDimensions())
262  k.computeImage(im, False)
263  mos.append(im)
264  mos.makeMosaic(frame=frame, title="Kernel Basis Images")
265 
266  return mos
267 
268 
269 
270 
271 def plotKernelSpatialModel(kernel, kernelCellSet, showBadCandidates=True,
272  numSample=128, keepPlots=True, maxCoeff=10):
273  """Plot the Kernel spatial model."""
274 
275  try:
276  import matplotlib.pyplot as plt
277  import matplotlib.colors
278  except ImportError as e:
279  print("Unable to import numpy and matplotlib: %s" % e)
280  return
281 
282  x0 = kernelCellSet.getBBox().getBeginX()
283  y0 = kernelCellSet.getBBox().getBeginY()
284 
285  candPos = list()
286  candFits = list()
287  badPos = list()
288  badFits = list()
289  candAmps = list()
290  badAmps = list()
291  for cell in kernelCellSet.getCellList():
292  for cand in cell.begin(False):
293  if not showBadCandidates and cand.isBad():
294  continue
295  candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter())
296  try:
297  im = cand.getTemplateMaskedImage()
298  except Exception as e:
299  continue
300 
301  targetFits = badFits if cand.isBad() else candFits
302  targetPos = badPos if cand.isBad() else candPos
303  targetAmps = badAmps if cand.isBad() else candAmps
304 
305  # compare original and spatial kernel coefficients
306  kp0 = np.array(cand.getKernel(diffimLib.KernelCandidateF.ORIG).getKernelParameters())
307  amp = cand.getCandidateRating()
308 
309  targetFits = badFits if cand.isBad() else candFits
310  targetPos = badPos if cand.isBad() else candPos
311  targetAmps = badAmps if cand.isBad() else candAmps
312 
313  targetFits.append(kp0)
314  targetPos.append(candCenter)
315  targetAmps.append(amp)
316 
317  xGood = np.array([pos.getX() for pos in candPos]) - x0
318  yGood = np.array([pos.getY() for pos in candPos]) - y0
319  zGood = np.array(candFits)
320 
321  xBad = np.array([pos.getX() for pos in badPos]) - x0
322  yBad = np.array([pos.getY() for pos in badPos]) - y0
323  zBad = np.array(badFits)
324  numBad = len(badPos)
325 
326  xRange = np.linspace(0, kernelCellSet.getBBox().getWidth(), num=numSample)
327  yRange = np.linspace(0, kernelCellSet.getBBox().getHeight(), num=numSample)
328 
329  if maxCoeff:
330  maxCoeff = min(maxCoeff, kernel.getNKernelParameters())
331  else:
332  maxCoeff = kernel.getNKernelParameters()
333 
334  for k in range(maxCoeff):
335  func = kernel.getSpatialFunction(k)
336  dfGood = zGood[:, k] - np.array([func(pos.getX(), pos.getY()) for pos in candPos])
337  yMin = dfGood.min()
338  yMax = dfGood.max()
339  if numBad > 0:
340  dfBad = zBad[:, k] - np.array([func(pos.getX(), pos.getY()) for pos in badPos])
341  # Can really screw up the range...
342  yMin = min([yMin, dfBad.min()])
343  yMax = max([yMax, dfBad.max()])
344  yMin -= 0.05 * (yMax - yMin)
345  yMax += 0.05 * (yMax - yMin)
346 
347  fRange = np.ndarray((len(xRange), len(yRange)))
348  for j, yVal in enumerate(yRange):
349  for i, xVal in enumerate(xRange):
350  fRange[j][i] = func(xVal, yVal)
351 
352  fig = plt.figure(k)
353 
354  fig.clf()
355  try:
356  fig.canvas._tkcanvas._root().lift() # == Tk's raise, but raise is a python reserved word
357  except Exception: # protect against API changes
358  pass
359 
360  fig.suptitle('Kernel component %d' % k)
361 
362  # LL
363  ax = fig.add_axes((0.1, 0.05, 0.35, 0.35))
364  vmin = fRange.min() # - 0.05 * np.fabs(fRange.min())
365  vmax = fRange.max() # + 0.05 * np.fabs(fRange.max())
366  norm = matplotlib.colors.Normalize(vmin=vmin, vmax=vmax)
367  im = ax.imshow(fRange, aspect='auto', norm=norm,
368  extent=[0, kernelCellSet.getBBox().getWidth()-1,
369  0, kernelCellSet.getBBox().getHeight()-1])
370  ax.set_title('Spatial polynomial')
371  plt.colorbar(im, orientation='horizontal', ticks=[vmin, vmax])
372 
373  # UL
374  ax = fig.add_axes((0.1, 0.55, 0.35, 0.35))
375  ax.plot(-2.5*np.log10(candAmps), zGood[:, k], 'b+')
376  if numBad > 0:
377  ax.plot(-2.5*np.log10(badAmps), zBad[:, k], 'r+')
378  ax.set_title("Basis Coefficients")
379  ax.set_xlabel("Instr mag")
380  ax.set_ylabel("Coeff")
381 
382  # LR
383  ax = fig.add_axes((0.55, 0.05, 0.35, 0.35))
384  ax.set_autoscale_on(False)
385  ax.set_xbound(lower=0, upper=kernelCellSet.getBBox().getHeight())
386  ax.set_ybound(lower=yMin, upper=yMax)
387  ax.plot(yGood, dfGood, 'b+')
388  if numBad > 0:
389  ax.plot(yBad, dfBad, 'r+')
390  ax.axhline(0.0)
391  ax.set_title('dCoeff (indiv-spatial) vs. y')
392 
393  # UR
394  ax = fig.add_axes((0.55, 0.55, 0.35, 0.35))
395  ax.set_autoscale_on(False)
396  ax.set_xbound(lower=0, upper=kernelCellSet.getBBox().getWidth())
397  ax.set_ybound(lower=yMin, upper=yMax)
398  ax.plot(xGood, dfGood, 'b+')
399  if numBad > 0:
400  ax.plot(xBad, dfBad, 'r+')
401  ax.axhline(0.0)
402  ax.set_title('dCoeff (indiv-spatial) vs. x')
403 
404  fig.show()
405 
406  global keptPlots
407  if keepPlots and not keptPlots:
408  # Keep plots open when done
409  def show():
410  print("%s: Please close plots when done." % __name__)
411  try:
412  plt.show()
413  except Exception:
414  pass
415  print("Plots closed, exiting...")
416  import atexit
417  atexit.register(show)
418  keptPlots = True
419 
420 
421 def showKernelMosaic(bbox, kernel, nx=7, ny=None, frame=None, title=None,
422  showCenter=True, showEllipticity=True):
423  """Show a mosaic of Kernel images.
424  """
425  mos = displayUtils.Mosaic()
426 
427  x0 = bbox.getBeginX()
428  y0 = bbox.getBeginY()
429  width = bbox.getWidth()
430  height = bbox.getHeight()
431 
432  if not ny:
433  ny = int(nx*float(height)/width + 0.5)
434  if not ny:
435  ny = 1
436 
437  schema = afwTable.SourceTable.makeMinimalSchema()
438  centroidName = "base_SdssCentroid"
439  shapeName = "base_SdssShape"
440  control = measBase.SdssCentroidControl()
441  schema.getAliasMap().set("slot_Centroid", centroidName)
442  schema.getAliasMap().set("slot_Centroid_flag", centroidName+"_flag")
443  centroider = measBase.SdssCentroidAlgorithm(control, centroidName, schema)
444  sdssShape = measBase.SdssShapeControl()
445  shaper = measBase.SdssShapeAlgorithm(sdssShape, shapeName, schema)
446  table = afwTable.SourceTable.make(schema)
447  table.defineCentroid(centroidName)
448  table.defineShape(shapeName)
449 
450  centers = []
451  shapes = []
452  for iy in range(ny):
453  for ix in range(nx):
454  x = int(ix*(width-1)/(nx-1)) + x0
455  y = int(iy*(height-1)/(ny-1)) + y0
456 
457  im = afwImage.ImageD(kernel.getDimensions())
458  ksum = kernel.computeImage(im, False, x, y)
459  lab = "Kernel(%d,%d)=%.2f" % (x, y, ksum) if False else ""
460  mos.append(im, lab)
461 
462  # SdssCentroidAlgorithm.measure requires an exposure of floats
463  exp = afwImage.makeExposure(afwImage.makeMaskedImage(im.convertF()))
464  w, h = im.getWidth(), im.getHeight()
465  centerX = im.getX0() + w//2
466  centerY = im.getY0() + h//2
467  src = table.makeRecord()
468  foot = afwDet.Footprint(exp.getBBox())
469  foot.addPeak(centerX, centerY, 1)
470  src.setFootprint(foot)
471 
472  centroider.measure(src, exp)
473  centers.append((src.getX(), src.getY()))
474 
475  shaper.measure(src, exp)
476  shapes.append((src.getIxx(), src.getIxy(), src.getIyy()))
477 
478  mos.makeMosaic(frame=frame, title=title if title else "Model Kernel", mode=nx)
479 
480  if centers and frame is not None:
481  i = 0
482  with ds9.Buffering():
483  for cen, shape in zip(centers, shapes):
484  bbox = mos.getBBox(i)
485  i += 1
486  xc, yc = cen[0] + bbox.getMinX(), cen[1] + bbox.getMinY()
487  if showCenter:
488  ds9.dot("+", xc, yc, ctype=ds9.BLUE, frame=frame)
489 
490  if showEllipticity:
491  ixx, ixy, iyy = shape
492  ds9.dot("@:%g,%g,%g" % (ixx, ixy, iyy), xc, yc, frame=frame, ctype=ds9.RED)
493 
494  return mos
495 
496 
497 def plotPixelResiduals(exposure, warpedTemplateExposure, diffExposure, kernelCellSet,
498  kernel, background, testSources, config,
499  origVariance=False, nptsFull=1e6, keepPlots=True, titleFs=14):
500  """Plot diffim residuals for LOCAL and SPATIAL models"""
501  candidateResids = []
502  spatialResids = []
503  nonfitResids = []
504 
505  for cell in kernelCellSet.getCellList():
506  for cand in cell.begin(True): # only look at good ones
507  # Be sure
508  if not (cand.getStatus() == afwMath.SpatialCellCandidate.GOOD):
509  continue
510 
511  diffim = cand.getDifferenceImage(diffimLib.KernelCandidateF.ORIG)
512  orig = cand.getScienceMaskedImage()
513 
514  ski = afwImage.ImageD(kernel.getDimensions())
515  kernel.computeImage(ski, False, int(cand.getXCenter()), int(cand.getYCenter()))
516  sk = afwMath.FixedKernel(ski)
517  sbg = background(int(cand.getXCenter()), int(cand.getYCenter()))
518  sdiffim = cand.getDifferenceImage(sk, sbg)
519 
520  # trim edgs due to convolution
521  bbox = kernel.shrinkBBox(diffim.getBBox())
522  tdiffim = diffim.Factory(diffim, bbox)
523  torig = orig.Factory(orig, bbox)
524  tsdiffim = sdiffim.Factory(sdiffim, bbox)
525 
526  if origVariance:
527  candidateResids.append(np.ravel(tdiffim.getImage().getArray() /
528  np.sqrt(torig.getVariance().getArray())))
529  spatialResids.append(np.ravel(tsdiffim.getImage().getArray() /
530  np.sqrt(torig.getVariance().getArray())))
531  else:
532  candidateResids.append(np.ravel(tdiffim.getImage().getArray() /
533  np.sqrt(tdiffim.getVariance().getArray())))
534  spatialResids.append(np.ravel(tsdiffim.getImage().getArray() /
535  np.sqrt(tsdiffim.getVariance().getArray())))
536 
537  fullIm = diffExposure.getMaskedImage().getImage().getArray()
538  fullMask = diffExposure.getMaskedImage().getMask().getArray()
539  if origVariance:
540  fullVar = exposure.getMaskedImage().getVariance().getArray()
541  else:
542  fullVar = diffExposure.getMaskedImage().getVariance().getArray()
543 
544  bitmaskBad = 0
545  bitmaskBad |= afwImage.Mask.getPlaneBitMask('NO_DATA')
546  bitmaskBad |= afwImage.Mask.getPlaneBitMask('SAT')
547  idx = np.where((fullMask & bitmaskBad) == 0)
548  stride = int(len(idx[0]) // nptsFull)
549  sidx = idx[0][::stride], idx[1][::stride]
550  allResids = fullIm[sidx] / np.sqrt(fullVar[sidx])
551 
552  testFootprints = diffimTools.sourceToFootprintList(testSources, warpedTemplateExposure,
553  exposure, config, Log.getDefaultLogger())
554  for fp in testFootprints:
555  subexp = diffExposure.Factory(diffExposure, fp["footprint"].getBBox())
556  subim = subexp.getMaskedImage().getImage()
557  if origVariance:
558  subvar = afwImage.ExposureF(exposure, fp["footprint"].getBBox()).getMaskedImage().getVariance()
559  else:
560  subvar = subexp.getMaskedImage().getVariance()
561  nonfitResids.append(np.ravel(subim.getArray() / np.sqrt(subvar.getArray())))
562 
563  candidateResids = np.ravel(np.array(candidateResids))
564  spatialResids = np.ravel(np.array(spatialResids))
565  nonfitResids = np.ravel(np.array(nonfitResids))
566 
567  try:
568  import pylab
569  from matplotlib.font_manager import FontProperties
570  except ImportError as e:
571  print("Unable to import pylab: %s" % e)
572  return
573 
574  fig = pylab.figure()
575  fig.clf()
576  try:
577  fig.canvas._tkcanvas._root().lift() # == Tk's raise, but raise is a python reserved word
578  except Exception: # protect against API changes
579  pass
580  if origVariance:
581  fig.suptitle("Diffim residuals: Normalized by sqrt(input variance)", fontsize=titleFs)
582  else:
583  fig.suptitle("Diffim residuals: Normalized by sqrt(diffim variance)", fontsize=titleFs)
584 
585  sp1 = pylab.subplot(221)
586  sp2 = pylab.subplot(222, sharex=sp1, sharey=sp1)
587  sp3 = pylab.subplot(223, sharex=sp1, sharey=sp1)
588  sp4 = pylab.subplot(224, sharex=sp1, sharey=sp1)
589  xs = np.arange(-5, 5.05, 0.1)
590  ys = 1. / np.sqrt(2*np.pi)*np.exp(-0.5*xs**2)
591 
592  sp1.hist(candidateResids, bins=xs, normed=True, alpha=0.5, label="N(%.2f, %.2f)"
593  % (np.mean(candidateResids), np.var(candidateResids)))
594  sp1.plot(xs, ys, "r-", lw=2, label="N(0,1)")
595  sp1.set_title("Candidates: basis fit", fontsize=titleFs-2)
596  sp1.legend(loc=1, fancybox=True, shadow=True, prop=FontProperties(size=titleFs-6))
597 
598  sp2.hist(spatialResids, bins=xs, normed=True, alpha=0.5, label="N(%.2f, %.2f)"
599  % (np.mean(spatialResids), np.var(spatialResids)))
600  sp2.plot(xs, ys, "r-", lw=2, label="N(0,1)")
601  sp2.set_title("Candidates: spatial fit", fontsize=titleFs-2)
602  sp2.legend(loc=1, fancybox=True, shadow=True, prop=FontProperties(size=titleFs-6))
603 
604  sp3.hist(nonfitResids, bins=xs, normed=True, alpha=0.5, label="N(%.2f, %.2f)"
605  % (np.mean(nonfitResids), np.var(nonfitResids)))
606  sp3.plot(xs, ys, "r-", lw=2, label="N(0,1)")
607  sp3.set_title("Control sample: spatial fit", fontsize=titleFs-2)
608  sp3.legend(loc=1, fancybox=True, shadow=True, prop=FontProperties(size=titleFs-6))
609 
610  sp4.hist(allResids, bins=xs, normed=True, alpha=0.5, label="N(%.2f, %.2f)"
611  % (np.mean(allResids), np.var(allResids)))
612  sp4.plot(xs, ys, "r-", lw=2, label="N(0,1)")
613  sp4.set_title("Full image (subsampled)", fontsize=titleFs-2)
614  sp4.legend(loc=1, fancybox=True, shadow=True, prop=FontProperties(size=titleFs-6))
615 
616  pylab.setp(sp1.get_xticklabels()+sp1.get_yticklabels(), fontsize=titleFs-4)
617  pylab.setp(sp2.get_xticklabels()+sp2.get_yticklabels(), fontsize=titleFs-4)
618  pylab.setp(sp3.get_xticklabels()+sp3.get_yticklabels(), fontsize=titleFs-4)
619  pylab.setp(sp4.get_xticklabels()+sp4.get_yticklabels(), fontsize=titleFs-4)
620 
621  sp1.set_xlim(-5, 5)
622  sp1.set_ylim(0, 0.5)
623  fig.show()
624 
625  global keptPlots
626  if keepPlots and not keptPlots:
627  # Keep plots open when done
628  def show():
629  print("%s: Please close plots when done." % __name__)
630  try:
631  pylab.show()
632  except Exception:
633  pass
634  print("Plots closed, exiting...")
635  import atexit
636  atexit.register(show)
637  keptPlots = True
638 
639 
640 def calcCentroid(arr):
641  """Calculate first moment of a (kernel) image"""
642  y, x = arr.shape
643  sarr = arr*arr
644  xarr = np.asarray([[el for el in range(x)] for el2 in range(y)])
645  yarr = np.asarray([[el2 for el in range(x)] for el2 in range(y)])
646  narr = xarr*sarr
647  sarrSum = sarr.sum()
648  centx = narr.sum()/sarrSum
649  narr = yarr*sarr
650  centy = narr.sum()/sarrSum
651  return centx, centy
652 
653 
654 def calcWidth(arr, centx, centy):
655  """Calculate second moment of a (kernel) image"""
656  y, x = arr.shape
657  # Square the flux so we don't have to deal with negatives
658  sarr = arr*arr
659  xarr = np.asarray([[el for el in range(x)] for el2 in range(y)])
660  yarr = np.asarray([[el2 for el in range(x)] for el2 in range(y)])
661  narr = sarr*np.power((xarr - centx), 2.)
662  sarrSum = sarr.sum()
663  xstd = np.sqrt(narr.sum()/sarrSum)
664  narr = sarr*np.power((yarr - centy), 2.)
665  ystd = np.sqrt(narr.sum()/sarrSum)
666  return xstd, ystd
667 
668 
669 def printSkyDiffs(sources, wcs):
670  """Print differences in sky coordinates between source Position and its Centroid mapped through Wcs"""
671  for s in sources:
672  sCentroid = s.getCentroid()
673  sPosition = s.getCoord().getPosition()
674  dra = 3600*(sPosition.getX() - wcs.pixelToSky(sCentroid.getX(),
675  sCentroid.getY()).getPosition().getX())/0.2
676  ddec = 3600*(sPosition.getY() - wcs.pixelToSky(sCentroid.getX(),
677  sCentroid.getY()).getPosition().getY())/0.2
678  if np.isfinite(dra) and np.isfinite(ddec):
679  print(dra, ddec)
680 
681 
682 def makeRegions(sources, outfilename, wcs=None):
683  """Create regions file for ds9 from input source list"""
684  fh = open(outfilename, "w")
685  fh.write("global color=red font=\"helvetica 10 normal\" "
686  "select=1 highlite=1 edit=1 move=1 delete=1 include=1 fixed=0 source\nfk5\n")
687  for s in sources:
688  if wcs:
689  (ra, dec) = wcs.pixelToSky(s.getCentroid().getX(), s.getCentroid().getY()).getPosition()
690  else:
691  (ra, dec) = s.getCoord().getPosition()
692  if np.isfinite(ra) and np.isfinite(dec):
693  fh.write("circle(%f,%f,2\")\n"%(ra, dec))
694  fh.flush()
695  fh.close()
696 
697 
698 def showSourceSetSky(sSet, wcs, xy0, frame=0, ctype=ds9.GREEN, symb="+", size=2):
699  """Draw the (RA, Dec) positions of a set of Sources. Image has the XY0."""
700  with ds9.Buffering():
701  for s in sSet:
702  (xc, yc) = wcs.skyToPixel(s.getCoord().getRa(), s.getCoord().getDec())
703  xc -= xy0[0]
704  yc -= xy0[1]
705  ds9.dot(symb, xc, yc, frame=frame, ctype=ctype, size=size)
706 
707 
708 def plotWhisker(results, newWcs):
709  """Plot whisker diagram of astromeric offsets between results.matches"""
710  refCoordKey = results.matches[0].first.getTable().getCoordKey()
711  inCentroidKey = results.matches[0].second.getTable().getCentroidKey()
712  positions = [m.first.get(refCoordKey) for m in results.matches]
713  residuals = [m.first.get(refCoordKey).getOffsetFrom(
714  newWcs.pixelToSky(m.second.get(inCentroidKey))) for
715  m in results.matches]
716  import matplotlib.pyplot as plt
717  fig = plt.figure()
718  sp = fig.add_subplot(1, 1, 0)
719  xpos = [x[0].asDegrees() for x in positions]
720  ypos = [x[1].asDegrees() for x in positions]
721  xpos.append(0.02*(max(xpos) - min(xpos)) + min(xpos))
722  ypos.append(0.98*(max(ypos) - min(ypos)) + min(ypos))
723  xidxs = np.isfinite(xpos)
724  yidxs = np.isfinite(ypos)
725  X = np.asarray(xpos)[xidxs]
726  Y = np.asarray(ypos)[yidxs]
727  distance = [x[1].asArcseconds() for x in residuals]
728  distance.append(0.2)
729  distance = np.asarray(distance)[xidxs]
730  # NOTE: This assumes that the bearing is measured positive from +RA through North.
731  # From the documentation this is not clear.
732  bearing = [x[0].asRadians() for x in residuals]
733  bearing.append(0)
734  bearing = np.asarray(bearing)[xidxs]
735  U = (distance*np.cos(bearing))
736  V = (distance*np.sin(bearing))
737  sp.quiver(X, Y, U, V)
738  sp.set_title("WCS Residual")
739  plt.show()
740 
741 
742 class DipoleTestImage(object):
743 
744  """!Utility class for dipole measurement testing
745 
746  Generate an image with simulated dipoles and noise; store the original "pre-subtraction" images
747  and catalogs as well.
748  Used to generate test data for DMTN-007 (http://dmtn-007.lsst.io).
749  """
750 
751  def __init__(self, w=101, h=101, xcenPos=[27.], ycenPos=[25.], xcenNeg=[23.], ycenNeg=[25.],
752  psfSigma=2., flux=[30000.], fluxNeg=None, noise=10., gradientParams=None):
753  self.w = w
754  self.h = h
755  self.xcenPos = xcenPos
756  self.ycenPos = ycenPos
757  self.xcenNeg = xcenNeg
758  self.ycenNeg = ycenNeg
759  self.psfSigma = psfSigma
760  self.flux = flux
761  self.fluxNeg = fluxNeg
762  if fluxNeg is None:
763  self.fluxNeg = self.flux
764  self.noise = noise
765  self.gradientParams = gradientParams
766  self._makeDipoleImage()
767 
768  def _makeDipoleImage(self):
769  """!Generate an exposure and catalog with the given dipole source(s)"""
770 
771  # Must seed the pos/neg images with different values to ensure they get different noise realizations
772  posImage, posCatalog = self._makeStarImage(
773  xc=self.xcenPos, yc=self.ycenPos, flux=self.flux, randomSeed=111)
774 
775  negImage, negCatalog = self._makeStarImage(
776  xc=self.xcenNeg, yc=self.ycenNeg, flux=self.fluxNeg, randomSeed=222)
777 
778  dipole = posImage.clone()
779  di = dipole.getMaskedImage()
780  di -= negImage.getMaskedImage()
781 
782  # Carry through pos/neg detection masks to new planes in diffim
783  dm = di.getMask()
784  posDetectedBits = posImage.getMaskedImage().getMask().getArray() == dm.getPlaneBitMask("DETECTED")
785  negDetectedBits = negImage.getMaskedImage().getMask().getArray() == dm.getPlaneBitMask("DETECTED")
786  pos_det = dm.addMaskPlane("DETECTED_POS") # new mask plane -- different from "DETECTED"
787  neg_det = dm.addMaskPlane("DETECTED_NEG") # new mask plane -- different from "DETECTED_NEGATIVE"
788  dma = dm.getArray()
789  # set the two custom mask planes to these new masks
790  dma[:, :] = posDetectedBits*pos_det + negDetectedBits*neg_det
791  self.diffim, self.posImage, self.posCatalog, self.negImage, self.negCatalog \
792  = dipole, posImage, posCatalog, negImage, negCatalog
793 
794  def _makeStarImage(self, xc=[15.3], yc=[18.6], flux=[2500], schema=None, randomSeed=None):
795  """!Generate an exposure and catalog with the given stellar source(s)"""
796 
797  from lsst.meas.base.tests import TestDataset
798  bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Point2I(self.w-1, self.h-1))
799  dataset = TestDataset(bbox, psfSigma=self.psfSigma, threshold=1.)
800 
801  for i in range(len(xc)):
802  dataset.addSource(flux=flux[i], centroid=afwGeom.Point2D(xc[i], yc[i]))
803 
804  if schema is None:
805  schema = TestDataset.makeMinimalSchema()
806  exposure, catalog = dataset.realize(noise=self.noise, schema=schema, randomSeed=randomSeed)
807 
808  if self.gradientParams is not None:
809  y, x = np.mgrid[:self.w, :self.h]
810  gp = self.gradientParams
811  gradient = gp[0] + gp[1] * x + gp[2] * y
812  if len(self.gradientParams) > 3: # it includes a set of 2nd-order polynomial params
813  gradient += gp[3] * x*y + gp[4] * x*x + gp[5] * y*y
814  imgArr = exposure.getMaskedImage().getArrays()[0]
815  imgArr += gradient
816 
817  return exposure, catalog
818 
819  def fitDipoleSource(self, source, **kwds):
820  alg = DipoleFitAlgorithm(self.diffim, self.posImage, self.negImage)
821  fitResult = alg.fitDipole(source, **kwds)
822  return fitResult
823 
824  def detectDipoleSources(self, doMerge=True, diffim=None, detectSigma=5.5, grow=3, minBinSize=32):
825  """!Utility function for detecting dipoles.
826 
827  Detect pos/neg sources in the diffim, then merge them. A
828  bigger "grow" parameter leads to a larger footprint which
829  helps with dipole measurement for faint dipoles.
830 
831  Parameters
832  ----------
833  doMerge : `bool`
834  Whether to merge the positive and negagive detections into a single source table
835  diffim : `lsst.afw.image.exposure.exposure.ExposureF`
836  Difference image on which to perform detection
837  detectSigma : `float`
838  Threshold for object detection
839  grow : `int`
840  Number of pixels to grow the footprints before merging
841  minBinSize : `int`
842  Minimum bin size for the background (re)estimation (only applies if the default leads to
843  min(nBinX, nBinY) < fit order so the default config parameter needs to be decreased, but not
844  to a value smaller than minBinSize, in which case the fitting algorithm will take over and
845  decrease the fit order appropriately.)
846 
847  Returns
848  -------
849  sources : `lsst.afw.table.SourceCatalog`
850  If doMerge=True, the merged source catalog is returned OR
851  detectTask : `lsst.meas.algorithms.SourceDetectionTask`
852  schema : `lsst.afw.table.Schema`
853  If doMerge=False, the source detection task and its schema are returned
854  """
855  if diffim is None:
856  diffim = self.diffim
857 
858  # Start with a minimal schema - only the fields all SourceCatalogs need
859  schema = afwTable.SourceTable.makeMinimalSchema()
860 
861  # Customize the detection task a bit (optional)
862  detectConfig = measAlg.SourceDetectionConfig()
863  detectConfig.returnOriginalFootprints = False # should be the default
864 
865  psfSigma = diffim.getPsf().computeShape().getDeterminantRadius()
866 
867  # code from imageDifference.py:
868  detectConfig.thresholdPolarity = "both"
869  detectConfig.thresholdValue = detectSigma
870  # detectConfig.nSigmaToGrow = psfSigma
871  detectConfig.reEstimateBackground = True # if False, will fail often for faint sources on gradients?
872  detectConfig.thresholdType = "pixel_stdev"
873  # Test images are often quite small, so may need to adjust background binSize
874  while ((min(diffim.getWidth(), diffim.getHeight()))//detectConfig.background.binSize <
875  detectConfig.background.approxOrderX and detectConfig.background.binSize > minBinSize):
876  detectConfig.background.binSize = max(minBinSize, detectConfig.background.binSize//2)
877 
878  # Create the detection task. We pass the schema so the task can declare a few flag fields
879  detectTask = measAlg.SourceDetectionTask(schema, config=detectConfig)
880 
881  table = afwTable.SourceTable.make(schema)
882  catalog = detectTask.makeSourceCatalog(table, diffim, sigma=psfSigma)
883 
884  # Now do the merge.
885  if doMerge:
886  fpSet = catalog.fpSets.positive
887  fpSet.merge(catalog.fpSets.negative, grow, grow, False)
888  sources = afwTable.SourceCatalog(table)
889  fpSet.makeSources(sources)
890 
891  return sources
892 
893  else:
894  return detectTask, schema
def _makeDipoleImage(self)
Generate an exposure and catalog with the given dipole source(s)
Definition: utils.py:768
def showDiaSources(sources, exposure, isFlagged, isDipole, frame=None)
Definition: utils.py:105
Lightweight class containing methods for fitting a dipole model in a diffim, used by DipoleFitPlugin...
def makeRegions(sources, outfilename, wcs=None)
Definition: utils.py:682
def __init__(self, w=101, h=101, xcenPos=[27.], ycenPos=[25.], xcenNeg=[23.], ycenNeg=[25.], psfSigma=2., flux=[30000.], fluxNeg=None, noise=10., gradientParams=None)
Definition: utils.py:752
Utility class for dipole measurement testing.
Definition: utils.py:742
MaskedImage< ImagePixelT, MaskPixelT, VariancePixelT > * makeMaskedImage(typename std::shared_ptr< Image< ImagePixelT >> image, typename std::shared_ptr< Mask< MaskPixelT >> mask=Mask< MaskPixelT >(), typename std::shared_ptr< Image< VariancePixelT >> variance=Image< VariancePixelT >())
def showKernelSpatialCells(maskedIm, kernelCellSet, showChi2=False, symb="o", ctype=None, ctypeUnused=None, ctypeBad=None, size=3, frame=None, title="Spatial Cells")
Definition: utils.py:71
def _makeStarImage(self, xc=[15.3], yc=[18.6], flux=[2500], schema=None, randomSeed=None)
Generate an exposure and catalog with the given stellar source(s)
Definition: utils.py:794
std::shared_ptr< Exposure< ImagePixelT, MaskPixelT, VariancePixelT > > makeExposure(MaskedImage< ImagePixelT, MaskPixelT, VariancePixelT > &mimage, std::shared_ptr< Wcs const > wcs=std::shared_ptr< Wcs const >())
def plotKernelSpatialModel(kernel, kernelCellSet, showBadCandidates=True, numSample=128, keepPlots=True, maxCoeff=10)
Definition: utils.py:272
def showSourceSetSky(sSet, wcs, xy0, frame=0, ctype=ds9.GREEN, symb="+", size=2)
Definition: utils.py:698
def plotPixelResiduals(exposure, warpedTemplateExposure, diffExposure, kernelCellSet, kernel, background, testSources, config, origVariance=False, nptsFull=1e6, keepPlots=True, titleFs=14)
Definition: utils.py:499
def printSkyDiffs(sources, wcs)
Definition: utils.py:669
def showKernelCandidates(kernelCellSet, kernel, background, frame=None, showBadCandidates=True, resids=False, kernels=False)
Definition: utils.py:141
def showSourceSet(sSet, xy0=(0, 0), frame=0, ctype=ds9.GREEN, symb="+", size=2)
Definition: utils.py:52
def detectDipoleSources(self, doMerge=True, diffim=None, detectSigma=5.5, grow=3, minBinSize=32)
Utility function for detecting dipoles.
Definition: utils.py:824
def calcCentroid(arr)
Definition: utils.py:640
def calcWidth(arr, centx, centy)
Definition: utils.py:654
def plotWhisker(results, newWcs)
Definition: utils.py:708
def showKernelMosaic(bbox, kernel, nx=7, ny=None, frame=None, title=None, showCenter=True, showEllipticity=True)
Definition: utils.py:422
def showKernelBasis(kernel, frame=None)
Definition: utils.py:255
def fitDipoleSource(self, source, kwds)
Definition: utils.py:819