lsst.ip.diffim  16.0-22-g847a80f+1
utils.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2016 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 """Support utilities for Measuring sources"""
23 
24 # Export DipoleTestImage to expose fake image generating funcs
25 __all__ = ["DipoleTestImage"]
26 
27 import numpy as np
28 
29 import lsst.afw.detection as afwDet
30 import lsst.afw.geom as afwGeom
31 import lsst.afw.image as afwImage
32 import lsst.afw.math as afwMath
33 import lsst.afw.table as afwTable
34 import lsst.afw.display.ds9 as ds9
35 import lsst.afw.display.utils as displayUtils
36 from lsst.log import Log
37 import lsst.meas.algorithms as measAlg
38 import lsst.meas.base as measBase
39 from .dipoleFitTask import DipoleFitAlgorithm
40 from . import diffimLib
41 from . import diffimTools
42 
43 keptPlots = False # Have we arranged to keep spatial plots open?
44 
45 
46 def showSourceSet(sSet, xy0=(0, 0), frame=0, ctype=ds9.GREEN, symb="+", size=2):
47  """Draw the (XAstrom, YAstrom) positions of a set of Sources. Image has the given XY0"""
48 
49  with ds9.Buffering():
50  for s in sSet:
51  xc, yc = s.getXAstrom() - xy0[0], s.getYAstrom() - xy0[1]
52 
53  if symb == "id":
54  ds9.dot(str(s.getId()), xc, yc, frame=frame, ctype=ctype, size=size)
55  else:
56  ds9.dot(symb, xc, yc, frame=frame, ctype=ctype, size=size)
57 
58 
59 # Kernel display utilities
60 #
61 
62 
63 def showKernelSpatialCells(maskedIm, kernelCellSet, showChi2=False, symb="o",
64  ctype=None, ctypeUnused=None, ctypeBad=None, size=3,
65  frame=None, title="Spatial Cells"):
66  """Show the SpatialCells. If symb is something that ds9.dot
67  understands (e.g. "o"), the top nMaxPerCell candidates will be
68  indicated with that symbol, using ctype and size"""
69 
70  ds9.mtv(maskedIm, frame=frame, title=title)
71  with ds9.Buffering():
72  origin = [-maskedIm.getX0(), -maskedIm.getY0()]
73  for cell in kernelCellSet.getCellList():
74  displayUtils.drawBBox(cell.getBBox(), origin=origin, frame=frame)
75 
76  goodies = ctypeBad is None
77  for cand in cell.begin(goodies):
78  xc, yc = cand.getXCenter() + origin[0], cand.getYCenter() + origin[1]
79  if cand.getStatus() == afwMath.SpatialCellCandidate.BAD:
80  color = ctypeBad
81  elif cand.getStatus() == afwMath.SpatialCellCandidate.GOOD:
82  color = ctype
83  elif cand.getStatus() == afwMath.SpatialCellCandidate.UNKNOWN:
84  color = ctypeUnused
85  else:
86  continue
87 
88  if color:
89  ds9.dot(symb, xc, yc, frame=frame, ctype=color, size=size)
90 
91  if showChi2:
92  rchi2 = cand.getChi2()
93  if rchi2 > 1e100:
94  rchi2 = np.nan
95  ds9.dot("%d %.1f" % (cand.getId(), rchi2),
96  xc - size, yc - size - 4, frame=frame, ctype=color, size=size)
97 
98 
99 def showDiaSources(sources, exposure, isFlagged, isDipole, frame=None):
100  """Display Dia Sources
101  """
102  #
103  # Show us the ccandidates
104  #
105  # Too many mask planes in diffims
106  for plane in ("BAD", "CR", "EDGE", "INTERPOlATED", "INTRP", "SAT", "SATURATED"):
107  ds9.setMaskPlaneVisibility(plane, False)
108 
109  mos = displayUtils.Mosaic()
110  for i in range(len(sources)):
111  source = sources[i]
112  badFlag = isFlagged[i]
113  dipoleFlag = isDipole[i]
114  bbox = source.getFootprint().getBBox()
115  stamp = exposure.Factory(exposure, bbox, True)
116  im = displayUtils.Mosaic(gutter=1, background=0, mode="x")
117  im.append(stamp.getMaskedImage())
118  lab = "%.1f,%.1f:" % (source.getX(), source.getY())
119  if badFlag:
120  ctype = ds9.RED
121  lab += "BAD"
122  if dipoleFlag:
123  ctype = ds9.YELLOW
124  lab += "DIPOLE"
125  if not badFlag and not dipoleFlag:
126  ctype = ds9.GREEN
127  lab += "OK"
128  mos.append(im.makeMosaic(), lab, ctype)
129  title = "Dia Sources"
130  mosaicImage = mos.makeMosaic(frame=frame, title=title)
131  return mosaicImage
132 
133 
134 def showKernelCandidates(kernelCellSet, kernel, background, frame=None, showBadCandidates=True,
135  resids=False, kernels=False):
136  """Display the Kernel candidates.
137  If kernel is provided include spatial model and residuals;
138  If chi is True, generate a plot of residuals/sqrt(variance), i.e. chi
139  """
140 
141  #
142  # Show us the ccandidates
143  #
144  if kernels:
145  mos = displayUtils.Mosaic(gutter=5, background=0)
146  else:
147  mos = displayUtils.Mosaic(gutter=5, background=-1)
148  #
149  candidateCenters = []
150  candidateCentersBad = []
151  candidateIndex = 0
152  for cell in kernelCellSet.getCellList():
153  for cand in cell.begin(False): # include bad candidates
154  # Original difference image; if does not exist, skip candidate
155  try:
156  resid = cand.getDifferenceImage(diffimLib.KernelCandidateF.ORIG)
157  except Exception:
158  continue
159 
160  rchi2 = cand.getChi2()
161  if rchi2 > 1e100:
162  rchi2 = np.nan
163 
164  if not showBadCandidates and cand.isBad():
165  continue
166 
167  im_resid = displayUtils.Mosaic(gutter=1, background=-0.5, mode="x")
168 
169  try:
170  im = cand.getScienceMaskedImage()
171  im = im.Factory(im, True)
172  im.setXY0(cand.getScienceMaskedImage().getXY0())
173  except Exception:
174  continue
175  if (not resids and not kernels):
176  im_resid.append(im.Factory(im, True))
177  try:
178  im = cand.getTemplateMaskedImage()
179  im = im.Factory(im, True)
180  im.setXY0(cand.getTemplateMaskedImage().getXY0())
181  except Exception:
182  continue
183  if (not resids and not kernels):
184  im_resid.append(im.Factory(im, True))
185 
186  # Difference image with original basis
187  if resids:
188  var = resid.getVariance()
189  var = var.Factory(var, True)
190  np.sqrt(var.getArray(), var.getArray()) # inplace sqrt
191  resid = resid.getImage()
192  resid /= var
193  bbox = kernel.shrinkBBox(resid.getBBox())
194  resid = resid.Factory(resid, bbox, True)
195  elif kernels:
196  kim = cand.getKernelImage(diffimLib.KernelCandidateF.ORIG).convertF()
197  resid = kim.Factory(kim, True)
198  im_resid.append(resid)
199 
200  # residuals using spatial model
201  ski = afwImage.ImageD(kernel.getDimensions())
202  kernel.computeImage(ski, False, int(cand.getXCenter()), int(cand.getYCenter()))
203  sk = afwMath.FixedKernel(ski)
204  sbg = 0.0
205  if background:
206  sbg = background(int(cand.getXCenter()), int(cand.getYCenter()))
207  sresid = cand.getDifferenceImage(sk, sbg)
208  resid = sresid
209  if resids:
210  resid = sresid.getImage()
211  resid /= var
212  bbox = kernel.shrinkBBox(resid.getBBox())
213  resid = resid.Factory(resid, bbox, True)
214  elif kernels:
215  kim = ski.convertF()
216  resid = kim.Factory(kim, True)
217  im_resid.append(resid)
218 
219  im = im_resid.makeMosaic()
220 
221  lab = "%d chi^2 %.1f" % (cand.getId(), rchi2)
222  ctype = ds9.RED if cand.isBad() else ds9.GREEN
223 
224  mos.append(im, lab, ctype)
225 
226  if False and np.isnan(rchi2):
227  ds9.mtv(cand.getScienceMaskedImage.getImage(), title="candidate", frame=1)
228  print("rating", cand.getCandidateRating())
229 
230  im = cand.getScienceMaskedImage()
231  center = (candidateIndex, cand.getXCenter() - im.getX0(), cand.getYCenter() - im.getY0())
232  candidateIndex += 1
233  if cand.isBad():
234  candidateCentersBad.append(center)
235  else:
236  candidateCenters.append(center)
237 
238  if resids:
239  title = "chi Diffim"
240  elif kernels:
241  title = "Kernels"
242  else:
243  title = "Candidates & residuals"
244  mosaicImage = mos.makeMosaic(frame=frame, title=title)
245 
246  return mosaicImage
247 
248 
249 def showKernelBasis(kernel, frame=None):
250  """Display a Kernel's basis images
251  """
252  mos = displayUtils.Mosaic()
253 
254  for k in kernel.getKernelList():
255  im = afwImage.ImageD(k.getDimensions())
256  k.computeImage(im, False)
257  mos.append(im)
258  mos.makeMosaic(frame=frame, title="Kernel Basis Images")
259 
260  return mos
261 
262 
263 
264 
265 def plotKernelSpatialModel(kernel, kernelCellSet, showBadCandidates=True,
266  numSample=128, keepPlots=True, maxCoeff=10):
267  """Plot the Kernel spatial model."""
268 
269  try:
270  import matplotlib.pyplot as plt
271  import matplotlib.colors
272  except ImportError as e:
273  print("Unable to import numpy and matplotlib: %s" % e)
274  return
275 
276  x0 = kernelCellSet.getBBox().getBeginX()
277  y0 = kernelCellSet.getBBox().getBeginY()
278 
279  candPos = list()
280  candFits = list()
281  badPos = list()
282  badFits = list()
283  candAmps = list()
284  badAmps = list()
285  for cell in kernelCellSet.getCellList():
286  for cand in cell.begin(False):
287  if not showBadCandidates and cand.isBad():
288  continue
289  candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter())
290  try:
291  im = cand.getTemplateMaskedImage()
292  except Exception:
293  continue
294 
295  targetFits = badFits if cand.isBad() else candFits
296  targetPos = badPos if cand.isBad() else candPos
297  targetAmps = badAmps if cand.isBad() else candAmps
298 
299  # compare original and spatial kernel coefficients
300  kp0 = np.array(cand.getKernel(diffimLib.KernelCandidateF.ORIG).getKernelParameters())
301  amp = cand.getCandidateRating()
302 
303  targetFits = badFits if cand.isBad() else candFits
304  targetPos = badPos if cand.isBad() else candPos
305  targetAmps = badAmps if cand.isBad() else candAmps
306 
307  targetFits.append(kp0)
308  targetPos.append(candCenter)
309  targetAmps.append(amp)
310 
311  xGood = np.array([pos.getX() for pos in candPos]) - x0
312  yGood = np.array([pos.getY() for pos in candPos]) - y0
313  zGood = np.array(candFits)
314 
315  xBad = np.array([pos.getX() for pos in badPos]) - x0
316  yBad = np.array([pos.getY() for pos in badPos]) - y0
317  zBad = np.array(badFits)
318  numBad = len(badPos)
319 
320  xRange = np.linspace(0, kernelCellSet.getBBox().getWidth(), num=numSample)
321  yRange = np.linspace(0, kernelCellSet.getBBox().getHeight(), num=numSample)
322 
323  if maxCoeff:
324  maxCoeff = min(maxCoeff, kernel.getNKernelParameters())
325  else:
326  maxCoeff = kernel.getNKernelParameters()
327 
328  for k in range(maxCoeff):
329  func = kernel.getSpatialFunction(k)
330  dfGood = zGood[:, k] - np.array([func(pos.getX(), pos.getY()) for pos in candPos])
331  yMin = dfGood.min()
332  yMax = dfGood.max()
333  if numBad > 0:
334  dfBad = zBad[:, k] - np.array([func(pos.getX(), pos.getY()) for pos in badPos])
335  # Can really screw up the range...
336  yMin = min([yMin, dfBad.min()])
337  yMax = max([yMax, dfBad.max()])
338  yMin -= 0.05 * (yMax - yMin)
339  yMax += 0.05 * (yMax - yMin)
340 
341  fRange = np.ndarray((len(xRange), len(yRange)))
342  for j, yVal in enumerate(yRange):
343  for i, xVal in enumerate(xRange):
344  fRange[j][i] = func(xVal, yVal)
345 
346  fig = plt.figure(k)
347 
348  fig.clf()
349  try:
350  fig.canvas._tkcanvas._root().lift() # == Tk's raise, but raise is a python reserved word
351  except Exception: # protect against API changes
352  pass
353 
354  fig.suptitle('Kernel component %d' % k)
355 
356  # LL
357  ax = fig.add_axes((0.1, 0.05, 0.35, 0.35))
358  vmin = fRange.min() # - 0.05 * np.fabs(fRange.min())
359  vmax = fRange.max() # + 0.05 * np.fabs(fRange.max())
360  norm = matplotlib.colors.Normalize(vmin=vmin, vmax=vmax)
361  im = ax.imshow(fRange, aspect='auto', norm=norm,
362  extent=[0, kernelCellSet.getBBox().getWidth()-1,
363  0, kernelCellSet.getBBox().getHeight()-1])
364  ax.set_title('Spatial polynomial')
365  plt.colorbar(im, orientation='horizontal', ticks=[vmin, vmax])
366 
367  # UL
368  ax = fig.add_axes((0.1, 0.55, 0.35, 0.35))
369  ax.plot(-2.5*np.log10(candAmps), zGood[:, k], 'b+')
370  if numBad > 0:
371  ax.plot(-2.5*np.log10(badAmps), zBad[:, k], 'r+')
372  ax.set_title("Basis Coefficients")
373  ax.set_xlabel("Instr mag")
374  ax.set_ylabel("Coeff")
375 
376  # LR
377  ax = fig.add_axes((0.55, 0.05, 0.35, 0.35))
378  ax.set_autoscale_on(False)
379  ax.set_xbound(lower=0, upper=kernelCellSet.getBBox().getHeight())
380  ax.set_ybound(lower=yMin, upper=yMax)
381  ax.plot(yGood, dfGood, 'b+')
382  if numBad > 0:
383  ax.plot(yBad, dfBad, 'r+')
384  ax.axhline(0.0)
385  ax.set_title('dCoeff (indiv-spatial) vs. y')
386 
387  # UR
388  ax = fig.add_axes((0.55, 0.55, 0.35, 0.35))
389  ax.set_autoscale_on(False)
390  ax.set_xbound(lower=0, upper=kernelCellSet.getBBox().getWidth())
391  ax.set_ybound(lower=yMin, upper=yMax)
392  ax.plot(xGood, dfGood, 'b+')
393  if numBad > 0:
394  ax.plot(xBad, dfBad, 'r+')
395  ax.axhline(0.0)
396  ax.set_title('dCoeff (indiv-spatial) vs. x')
397 
398  fig.show()
399 
400  global keptPlots
401  if keepPlots and not keptPlots:
402  # Keep plots open when done
403  def show():
404  print("%s: Please close plots when done." % __name__)
405  try:
406  plt.show()
407  except Exception:
408  pass
409  print("Plots closed, exiting...")
410  import atexit
411  atexit.register(show)
412  keptPlots = True
413 
414 
415 def showKernelMosaic(bbox, kernel, nx=7, ny=None, frame=None, title=None,
416  showCenter=True, showEllipticity=True):
417  """Show a mosaic of Kernel images.
418  """
419  mos = displayUtils.Mosaic()
420 
421  x0 = bbox.getBeginX()
422  y0 = bbox.getBeginY()
423  width = bbox.getWidth()
424  height = bbox.getHeight()
425 
426  if not ny:
427  ny = int(nx*float(height)/width + 0.5)
428  if not ny:
429  ny = 1
430 
431  schema = afwTable.SourceTable.makeMinimalSchema()
432  centroidName = "base_SdssCentroid"
433  shapeName = "base_SdssShape"
434  control = measBase.SdssCentroidControl()
435  schema.getAliasMap().set("slot_Centroid", centroidName)
436  schema.getAliasMap().set("slot_Centroid_flag", centroidName+"_flag")
437  centroider = measBase.SdssCentroidAlgorithm(control, centroidName, schema)
438  sdssShape = measBase.SdssShapeControl()
439  shaper = measBase.SdssShapeAlgorithm(sdssShape, shapeName, schema)
440  table = afwTable.SourceTable.make(schema)
441  table.defineCentroid(centroidName)
442  table.defineShape(shapeName)
443 
444  centers = []
445  shapes = []
446  for iy in range(ny):
447  for ix in range(nx):
448  x = int(ix*(width-1)/(nx-1)) + x0
449  y = int(iy*(height-1)/(ny-1)) + y0
450 
451  im = afwImage.ImageD(kernel.getDimensions())
452  ksum = kernel.computeImage(im, False, x, y)
453  lab = "Kernel(%d,%d)=%.2f" % (x, y, ksum) if False else ""
454  mos.append(im, lab)
455 
456  # SdssCentroidAlgorithm.measure requires an exposure of floats
457  exp = afwImage.makeExposure(afwImage.makeMaskedImage(im.convertF()))
458  w, h = im.getWidth(), im.getHeight()
459  centerX = im.getX0() + w//2
460  centerY = im.getY0() + h//2
461  src = table.makeRecord()
462  foot = afwDet.Footprint(exp.getBBox())
463  foot.addPeak(centerX, centerY, 1)
464  src.setFootprint(foot)
465 
466  centroider.measure(src, exp)
467  centers.append((src.getX(), src.getY()))
468 
469  shaper.measure(src, exp)
470  shapes.append((src.getIxx(), src.getIxy(), src.getIyy()))
471 
472  mos.makeMosaic(frame=frame, title=title if title else "Model Kernel", mode=nx)
473 
474  if centers and frame is not None:
475  i = 0
476  with ds9.Buffering():
477  for cen, shape in zip(centers, shapes):
478  bbox = mos.getBBox(i)
479  i += 1
480  xc, yc = cen[0] + bbox.getMinX(), cen[1] + bbox.getMinY()
481  if showCenter:
482  ds9.dot("+", xc, yc, ctype=ds9.BLUE, frame=frame)
483 
484  if showEllipticity:
485  ixx, ixy, iyy = shape
486  ds9.dot("@:%g,%g,%g" % (ixx, ixy, iyy), xc, yc, frame=frame, ctype=ds9.RED)
487 
488  return mos
489 
490 
491 def plotPixelResiduals(exposure, warpedTemplateExposure, diffExposure, kernelCellSet,
492  kernel, background, testSources, config,
493  origVariance=False, nptsFull=1e6, keepPlots=True, titleFs=14):
494  """Plot diffim residuals for LOCAL and SPATIAL models"""
495  candidateResids = []
496  spatialResids = []
497  nonfitResids = []
498 
499  for cell in kernelCellSet.getCellList():
500  for cand in cell.begin(True): # only look at good ones
501  # Be sure
502  if not (cand.getStatus() == afwMath.SpatialCellCandidate.GOOD):
503  continue
504 
505  diffim = cand.getDifferenceImage(diffimLib.KernelCandidateF.ORIG)
506  orig = cand.getScienceMaskedImage()
507 
508  ski = afwImage.ImageD(kernel.getDimensions())
509  kernel.computeImage(ski, False, int(cand.getXCenter()), int(cand.getYCenter()))
510  sk = afwMath.FixedKernel(ski)
511  sbg = background(int(cand.getXCenter()), int(cand.getYCenter()))
512  sdiffim = cand.getDifferenceImage(sk, sbg)
513 
514  # trim edgs due to convolution
515  bbox = kernel.shrinkBBox(diffim.getBBox())
516  tdiffim = diffim.Factory(diffim, bbox)
517  torig = orig.Factory(orig, bbox)
518  tsdiffim = sdiffim.Factory(sdiffim, bbox)
519 
520  if origVariance:
521  candidateResids.append(np.ravel(tdiffim.getImage().getArray() /
522  np.sqrt(torig.getVariance().getArray())))
523  spatialResids.append(np.ravel(tsdiffim.getImage().getArray() /
524  np.sqrt(torig.getVariance().getArray())))
525  else:
526  candidateResids.append(np.ravel(tdiffim.getImage().getArray() /
527  np.sqrt(tdiffim.getVariance().getArray())))
528  spatialResids.append(np.ravel(tsdiffim.getImage().getArray() /
529  np.sqrt(tsdiffim.getVariance().getArray())))
530 
531  fullIm = diffExposure.getMaskedImage().getImage().getArray()
532  fullMask = diffExposure.getMaskedImage().getMask().getArray()
533  if origVariance:
534  fullVar = exposure.getMaskedImage().getVariance().getArray()
535  else:
536  fullVar = diffExposure.getMaskedImage().getVariance().getArray()
537 
538  bitmaskBad = 0
539  bitmaskBad |= afwImage.Mask.getPlaneBitMask('NO_DATA')
540  bitmaskBad |= afwImage.Mask.getPlaneBitMask('SAT')
541  idx = np.where((fullMask & bitmaskBad) == 0)
542  stride = int(len(idx[0]) // nptsFull)
543  sidx = idx[0][::stride], idx[1][::stride]
544  allResids = fullIm[sidx] / np.sqrt(fullVar[sidx])
545 
546  testFootprints = diffimTools.sourceToFootprintList(testSources, warpedTemplateExposure,
547  exposure, config, Log.getDefaultLogger())
548  for fp in testFootprints:
549  subexp = diffExposure.Factory(diffExposure, fp["footprint"].getBBox())
550  subim = subexp.getMaskedImage().getImage()
551  if origVariance:
552  subvar = afwImage.ExposureF(exposure, fp["footprint"].getBBox()).getMaskedImage().getVariance()
553  else:
554  subvar = subexp.getMaskedImage().getVariance()
555  nonfitResids.append(np.ravel(subim.getArray() / np.sqrt(subvar.getArray())))
556 
557  candidateResids = np.ravel(np.array(candidateResids))
558  spatialResids = np.ravel(np.array(spatialResids))
559  nonfitResids = np.ravel(np.array(nonfitResids))
560 
561  try:
562  import pylab
563  from matplotlib.font_manager import FontProperties
564  except ImportError as e:
565  print("Unable to import pylab: %s" % e)
566  return
567 
568  fig = pylab.figure()
569  fig.clf()
570  try:
571  fig.canvas._tkcanvas._root().lift() # == Tk's raise, but raise is a python reserved word
572  except Exception: # protect against API changes
573  pass
574  if origVariance:
575  fig.suptitle("Diffim residuals: Normalized by sqrt(input variance)", fontsize=titleFs)
576  else:
577  fig.suptitle("Diffim residuals: Normalized by sqrt(diffim variance)", fontsize=titleFs)
578 
579  sp1 = pylab.subplot(221)
580  sp2 = pylab.subplot(222, sharex=sp1, sharey=sp1)
581  sp3 = pylab.subplot(223, sharex=sp1, sharey=sp1)
582  sp4 = pylab.subplot(224, sharex=sp1, sharey=sp1)
583  xs = np.arange(-5, 5.05, 0.1)
584  ys = 1. / np.sqrt(2*np.pi)*np.exp(-0.5*xs**2)
585 
586  sp1.hist(candidateResids, bins=xs, normed=True, alpha=0.5, label="N(%.2f, %.2f)"
587  % (np.mean(candidateResids), np.var(candidateResids)))
588  sp1.plot(xs, ys, "r-", lw=2, label="N(0,1)")
589  sp1.set_title("Candidates: basis fit", fontsize=titleFs-2)
590  sp1.legend(loc=1, fancybox=True, shadow=True, prop=FontProperties(size=titleFs-6))
591 
592  sp2.hist(spatialResids, bins=xs, normed=True, alpha=0.5, label="N(%.2f, %.2f)"
593  % (np.mean(spatialResids), np.var(spatialResids)))
594  sp2.plot(xs, ys, "r-", lw=2, label="N(0,1)")
595  sp2.set_title("Candidates: spatial fit", fontsize=titleFs-2)
596  sp2.legend(loc=1, fancybox=True, shadow=True, prop=FontProperties(size=titleFs-6))
597 
598  sp3.hist(nonfitResids, bins=xs, normed=True, alpha=0.5, label="N(%.2f, %.2f)"
599  % (np.mean(nonfitResids), np.var(nonfitResids)))
600  sp3.plot(xs, ys, "r-", lw=2, label="N(0,1)")
601  sp3.set_title("Control sample: spatial fit", fontsize=titleFs-2)
602  sp3.legend(loc=1, fancybox=True, shadow=True, prop=FontProperties(size=titleFs-6))
603 
604  sp4.hist(allResids, bins=xs, normed=True, alpha=0.5, label="N(%.2f, %.2f)"
605  % (np.mean(allResids), np.var(allResids)))
606  sp4.plot(xs, ys, "r-", lw=2, label="N(0,1)")
607  sp4.set_title("Full image (subsampled)", fontsize=titleFs-2)
608  sp4.legend(loc=1, fancybox=True, shadow=True, prop=FontProperties(size=titleFs-6))
609 
610  pylab.setp(sp1.get_xticklabels()+sp1.get_yticklabels(), fontsize=titleFs-4)
611  pylab.setp(sp2.get_xticklabels()+sp2.get_yticklabels(), fontsize=titleFs-4)
612  pylab.setp(sp3.get_xticklabels()+sp3.get_yticklabels(), fontsize=titleFs-4)
613  pylab.setp(sp4.get_xticklabels()+sp4.get_yticklabels(), fontsize=titleFs-4)
614 
615  sp1.set_xlim(-5, 5)
616  sp1.set_ylim(0, 0.5)
617  fig.show()
618 
619  global keptPlots
620  if keepPlots and not keptPlots:
621  # Keep plots open when done
622  def show():
623  print("%s: Please close plots when done." % __name__)
624  try:
625  pylab.show()
626  except Exception:
627  pass
628  print("Plots closed, exiting...")
629  import atexit
630  atexit.register(show)
631  keptPlots = True
632 
633 
634 def calcCentroid(arr):
635  """Calculate first moment of a (kernel) image"""
636  y, x = arr.shape
637  sarr = arr*arr
638  xarr = np.asarray([[el for el in range(x)] for el2 in range(y)])
639  yarr = np.asarray([[el2 for el in range(x)] for el2 in range(y)])
640  narr = xarr*sarr
641  sarrSum = sarr.sum()
642  centx = narr.sum()/sarrSum
643  narr = yarr*sarr
644  centy = narr.sum()/sarrSum
645  return centx, centy
646 
647 
648 def calcWidth(arr, centx, centy):
649  """Calculate second moment of a (kernel) image"""
650  y, x = arr.shape
651  # Square the flux so we don't have to deal with negatives
652  sarr = arr*arr
653  xarr = np.asarray([[el for el in range(x)] for el2 in range(y)])
654  yarr = np.asarray([[el2 for el in range(x)] for el2 in range(y)])
655  narr = sarr*np.power((xarr - centx), 2.)
656  sarrSum = sarr.sum()
657  xstd = np.sqrt(narr.sum()/sarrSum)
658  narr = sarr*np.power((yarr - centy), 2.)
659  ystd = np.sqrt(narr.sum()/sarrSum)
660  return xstd, ystd
661 
662 
663 def printSkyDiffs(sources, wcs):
664  """Print differences in sky coordinates between source Position and its Centroid mapped through Wcs"""
665  for s in sources:
666  sCentroid = s.getCentroid()
667  sPosition = s.getCoord().getPosition(afwGeom.degrees)
668  dra = 3600*(sPosition.getX() - wcs.pixelToSky(sCentroid).getPosition(afwGeom.degrees).getX())/0.2
669  ddec = 3600*(sPosition.getY() - wcs.pixelToSky(sCentroid).getPosition(afwGeom.degrees).getY())/0.2
670  if np.isfinite(dra) and np.isfinite(ddec):
671  print(dra, ddec)
672 
673 
674 def makeRegions(sources, outfilename, wcs=None):
675  """Create regions file for ds9 from input source list"""
676  fh = open(outfilename, "w")
677  fh.write("global color=red font=\"helvetica 10 normal\" "
678  "select=1 highlite=1 edit=1 move=1 delete=1 include=1 fixed=0 source\nfk5\n")
679  for s in sources:
680  if wcs:
681  (ra, dec) = wcs.pixelToSky(s.getCentroid()).getPosition(afwGeom.degrees)
682  else:
683  (ra, dec) = s.getCoord().getPosition(afwGeom.degrees)
684  if np.isfinite(ra) and np.isfinite(dec):
685  fh.write("circle(%f,%f,2\")\n"%(ra, dec))
686  fh.flush()
687  fh.close()
688 
689 
690 def showSourceSetSky(sSet, wcs, xy0, frame=0, ctype=ds9.GREEN, symb="+", size=2):
691  """Draw the (RA, Dec) positions of a set of Sources. Image has the XY0."""
692  with ds9.Buffering():
693  for s in sSet:
694  (xc, yc) = wcs.skyToPixel(s.getCoord().getRa(), s.getCoord().getDec())
695  xc -= xy0[0]
696  yc -= xy0[1]
697  ds9.dot(symb, xc, yc, frame=frame, ctype=ctype, size=size)
698 
699 
700 def plotWhisker(results, newWcs):
701  """Plot whisker diagram of astromeric offsets between results.matches"""
702  refCoordKey = results.matches[0].first.getTable().getCoordKey()
703  inCentroidKey = results.matches[0].second.getTable().getCentroidKey()
704  positions = [m.first.get(refCoordKey) for m in results.matches]
705  residuals = [m.first.get(refCoordKey).getOffsetFrom(
706  newWcs.pixelToSky(m.second.get(inCentroidKey))) for
707  m in results.matches]
708  import matplotlib.pyplot as plt
709  fig = plt.figure()
710  sp = fig.add_subplot(1, 1, 0)
711  xpos = [x[0].asDegrees() for x in positions]
712  ypos = [x[1].asDegrees() for x in positions]
713  xpos.append(0.02*(max(xpos) - min(xpos)) + min(xpos))
714  ypos.append(0.98*(max(ypos) - min(ypos)) + min(ypos))
715  xidxs = np.isfinite(xpos)
716  yidxs = np.isfinite(ypos)
717  X = np.asarray(xpos)[xidxs]
718  Y = np.asarray(ypos)[yidxs]
719  distance = [x[1].asArcseconds() for x in residuals]
720  distance.append(0.2)
721  distance = np.asarray(distance)[xidxs]
722  # NOTE: This assumes that the bearing is measured positive from +RA through North.
723  # From the documentation this is not clear.
724  bearing = [x[0].asRadians() for x in residuals]
725  bearing.append(0)
726  bearing = np.asarray(bearing)[xidxs]
727  U = (distance*np.cos(bearing))
728  V = (distance*np.sin(bearing))
729  sp.quiver(X, Y, U, V)
730  sp.set_title("WCS Residual")
731  plt.show()
732 
733 
734 class DipoleTestImage(object):
735 
736  """Utility class for dipole measurement testing
737 
738  Generate an image with simulated dipoles and noise; store the original "pre-subtraction" images
739  and catalogs as well.
740  Used to generate test data for DMTN-007 (http://dmtn-007.lsst.io).
741  """
742 
743  def __init__(self, w=101, h=101, xcenPos=[27.], ycenPos=[25.], xcenNeg=[23.], ycenNeg=[25.],
744  psfSigma=2., flux=[30000.], fluxNeg=None, noise=10., gradientParams=None):
745  self.w = w
746  self.h = h
747  self.xcenPos = xcenPos
748  self.ycenPos = ycenPos
749  self.xcenNeg = xcenNeg
750  self.ycenNeg = ycenNeg
751  self.psfSigma = psfSigma
752  self.flux = flux
753  self.fluxNeg = fluxNeg
754  if fluxNeg is None:
755  self.fluxNeg = self.flux
756  self.noise = noise
757  self.gradientParams = gradientParams
758  self._makeDipoleImage()
759 
760  def _makeDipoleImage(self):
761  """!Generate an exposure and catalog with the given dipole source(s)"""
762 
763  # Must seed the pos/neg images with different values to ensure they get different noise realizations
764  posImage, posCatalog = self._makeStarImage(
765  xc=self.xcenPos, yc=self.ycenPos, flux=self.flux, randomSeed=111)
766 
767  negImage, negCatalog = self._makeStarImage(
768  xc=self.xcenNeg, yc=self.ycenNeg, flux=self.fluxNeg, randomSeed=222)
769 
770  dipole = posImage.clone()
771  di = dipole.getMaskedImage()
772  di -= negImage.getMaskedImage()
773 
774  # Carry through pos/neg detection masks to new planes in diffim
775  dm = di.getMask()
776  posDetectedBits = posImage.getMaskedImage().getMask().getArray() == dm.getPlaneBitMask("DETECTED")
777  negDetectedBits = negImage.getMaskedImage().getMask().getArray() == dm.getPlaneBitMask("DETECTED")
778  pos_det = dm.addMaskPlane("DETECTED_POS") # new mask plane -- different from "DETECTED"
779  neg_det = dm.addMaskPlane("DETECTED_NEG") # new mask plane -- different from "DETECTED_NEGATIVE"
780  dma = dm.getArray()
781  # set the two custom mask planes to these new masks
782  dma[:, :] = posDetectedBits*pos_det + negDetectedBits*neg_det
783  self.diffim, self.posImage, self.posCatalog, self.negImage, self.negCatalog \
784  = dipole, posImage, posCatalog, negImage, negCatalog
785 
786  def _makeStarImage(self, xc=[15.3], yc=[18.6], flux=[2500], schema=None, randomSeed=None):
787  """!Generate an exposure and catalog with the given stellar source(s)"""
788 
789  from lsst.meas.base.tests import TestDataset
790  bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Point2I(self.w-1, self.h-1))
791  dataset = TestDataset(bbox, psfSigma=self.psfSigma, threshold=1.)
792 
793  for i in range(len(xc)):
794  dataset.addSource(instFlux=flux[i], centroid=afwGeom.Point2D(xc[i], yc[i]))
795 
796  if schema is None:
797  schema = TestDataset.makeMinimalSchema()
798  exposure, catalog = dataset.realize(noise=self.noise, schema=schema, randomSeed=randomSeed)
799 
800  if self.gradientParams is not None:
801  y, x = np.mgrid[:self.w, :self.h]
802  gp = self.gradientParams
803  gradient = gp[0] + gp[1] * x + gp[2] * y
804  if len(self.gradientParams) > 3: # it includes a set of 2nd-order polynomial params
805  gradient += gp[3] * x*y + gp[4] * x*x + gp[5] * y*y
806  imgArr = exposure.getMaskedImage().getArrays()[0]
807  imgArr += gradient
808 
809  return exposure, catalog
810 
811  def fitDipoleSource(self, source, **kwds):
812  alg = DipoleFitAlgorithm(self.diffim, self.posImage, self.negImage)
813  fitResult = alg.fitDipole(source, **kwds)
814  return fitResult
815 
816  def detectDipoleSources(self, doMerge=True, diffim=None, detectSigma=5.5, grow=3, minBinSize=32):
817  """Utility function for detecting dipoles.
818 
819  Detect pos/neg sources in the diffim, then merge them. A
820  bigger "grow" parameter leads to a larger footprint which
821  helps with dipole measurement for faint dipoles.
822 
823  Parameters
824  ----------
825  doMerge : `bool`
826  Whether to merge the positive and negagive detections into a single source table
827  diffim : `lsst.afw.image.exposure.exposure.ExposureF`
828  Difference image on which to perform detection
829  detectSigma : `float`
830  Threshold for object detection
831  grow : `int`
832  Number of pixels to grow the footprints before merging
833  minBinSize : `int`
834  Minimum bin size for the background (re)estimation (only applies if the default leads to
835  min(nBinX, nBinY) < fit order so the default config parameter needs to be decreased, but not
836  to a value smaller than minBinSize, in which case the fitting algorithm will take over and
837  decrease the fit order appropriately.)
838 
839  Returns
840  -------
841  sources : `lsst.afw.table.SourceCatalog`
842  If doMerge=True, the merged source catalog is returned OR
843  detectTask : `lsst.meas.algorithms.SourceDetectionTask`
844  schema : `lsst.afw.table.Schema`
845  If doMerge=False, the source detection task and its schema are returned
846  """
847  if diffim is None:
848  diffim = self.diffim
849 
850  # Start with a minimal schema - only the fields all SourceCatalogs need
851  schema = afwTable.SourceTable.makeMinimalSchema()
852 
853  # Customize the detection task a bit (optional)
854  detectConfig = measAlg.SourceDetectionConfig()
855  detectConfig.returnOriginalFootprints = False # should be the default
856 
857  psfSigma = diffim.getPsf().computeShape().getDeterminantRadius()
858 
859  # code from imageDifference.py:
860  detectConfig.thresholdPolarity = "both"
861  detectConfig.thresholdValue = detectSigma
862  # detectConfig.nSigmaToGrow = psfSigma
863  detectConfig.reEstimateBackground = True # if False, will fail often for faint sources on gradients?
864  detectConfig.thresholdType = "pixel_stdev"
865  # Test images are often quite small, so may need to adjust background binSize
866  while ((min(diffim.getWidth(), diffim.getHeight()))//detectConfig.background.binSize <
867  detectConfig.background.approxOrderX and detectConfig.background.binSize > minBinSize):
868  detectConfig.background.binSize = max(minBinSize, detectConfig.background.binSize//2)
869 
870  # Create the detection task. We pass the schema so the task can declare a few flag fields
871  detectTask = measAlg.SourceDetectionTask(schema, config=detectConfig)
872 
873  table = afwTable.SourceTable.make(schema)
874  catalog = detectTask.makeSourceCatalog(table, diffim, sigma=psfSigma)
875 
876  # Now do the merge.
877  if doMerge:
878  fpSet = catalog.fpSets.positive
879  fpSet.merge(catalog.fpSets.negative, grow, grow, False)
880  sources = afwTable.SourceCatalog(table)
881  fpSet.makeSources(sources)
882 
883  return sources
884 
885  else:
886  return detectTask, schema
def _makeDipoleImage(self)
Generate an exposure and catalog with the given dipole source(s)
Definition: utils.py:760
def showDiaSources(sources, exposure, isFlagged, isDipole, frame=None)
Definition: utils.py:99
def makeRegions(sources, outfilename, wcs=None)
Definition: utils.py:674
def __init__(self, w=101, h=101, xcenPos=[27.], ycenPos=[25.], xcenNeg=[23.], ycenNeg=[25.], psfSigma=2., flux=[30000.], fluxNeg=None, noise=10., gradientParams=None)
Definition: utils.py:744
MaskedImage< ImagePixelT, MaskPixelT, VariancePixelT > * makeMaskedImage(typename std::shared_ptr< Image< ImagePixelT >> image, typename std::shared_ptr< Mask< MaskPixelT >> mask=Mask< MaskPixelT >(), typename std::shared_ptr< Image< VariancePixelT >> variance=Image< VariancePixelT >())
def showKernelSpatialCells(maskedIm, kernelCellSet, showChi2=False, symb="o", ctype=None, ctypeUnused=None, ctypeBad=None, size=3, frame=None, title="Spatial Cells")
Definition: utils.py:65
std::shared_ptr< Exposure< ImagePixelT, MaskPixelT, VariancePixelT > > makeExposure(MaskedImage< ImagePixelT, MaskPixelT, VariancePixelT > &mimage, std::shared_ptr< geom::SkyWcs const > wcs=std::shared_ptr< geom::SkyWcs const >())
def _makeStarImage(self, xc=[15.3], yc=[18.6], flux=[2500], schema=None, randomSeed=None)
Generate an exposure and catalog with the given stellar source(s)
Definition: utils.py:786
def plotKernelSpatialModel(kernel, kernelCellSet, showBadCandidates=True, numSample=128, keepPlots=True, maxCoeff=10)
Definition: utils.py:266
def showSourceSetSky(sSet, wcs, xy0, frame=0, ctype=ds9.GREEN, symb="+", size=2)
Definition: utils.py:690
def plotPixelResiduals(exposure, warpedTemplateExposure, diffExposure, kernelCellSet, kernel, background, testSources, config, origVariance=False, nptsFull=1e6, keepPlots=True, titleFs=14)
Definition: utils.py:493
def printSkyDiffs(sources, wcs)
Definition: utils.py:663
def showKernelCandidates(kernelCellSet, kernel, background, frame=None, showBadCandidates=True, resids=False, kernels=False)
Definition: utils.py:135
def showSourceSet(sSet, xy0=(0, 0), frame=0, ctype=ds9.GREEN, symb="+", size=2)
Definition: utils.py:46
def detectDipoleSources(self, doMerge=True, diffim=None, detectSigma=5.5, grow=3, minBinSize=32)
Definition: utils.py:816
def calcCentroid(arr)
Definition: utils.py:634
def calcWidth(arr, centx, centy)
Definition: utils.py:648
def plotWhisker(results, newWcs)
Definition: utils.py:700
def showKernelMosaic(bbox, kernel, nx=7, ny=None, frame=None, title=None, showCenter=True, showEllipticity=True)
Definition: utils.py:416
def showKernelBasis(kernel, frame=None)
Definition: utils.py:249
def fitDipoleSource(self, source, kwds)
Definition: utils.py:811