lsst.ip.diffim  15.0-3-ge6a6747+2
utils.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2016 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 """Support utilities for Measuring sources"""
23 from __future__ import absolute_import, division, print_function
24 
25 from builtins import zip
26 from builtins import str
27 from builtins import range
28 from builtins import object
29 import numpy as np
30 
31 import lsst.afw.detection as afwDet
32 import lsst.afw.geom as afwGeom
33 import lsst.afw.image as afwImage
34 import lsst.afw.math as afwMath
35 import lsst.afw.table as afwTable
36 import lsst.afw.display.ds9 as ds9
37 import lsst.afw.display.utils as displayUtils
38 from lsst.log import Log
39 import lsst.meas.algorithms as measAlg
40 import lsst.meas.base as measBase
41 from .dipoleFitTask import DipoleFitAlgorithm
42 from . import diffimLib
43 from . import diffimTools
44 
45 # Export DipoleTestImage to expose fake image generating funcs
46 __all__ = ("DipoleTestImage")
47 
48 
49 keptPlots = False # Have we arranged to keep spatial plots open?
50 
51 
52 def showSourceSet(sSet, xy0=(0, 0), frame=0, ctype=ds9.GREEN, symb="+", size=2):
53  """Draw the (XAstrom, YAstrom) positions of a set of Sources. Image has the given XY0"""
54 
55  with ds9.Buffering():
56  for s in sSet:
57  xc, yc = s.getXAstrom() - xy0[0], s.getYAstrom() - xy0[1]
58 
59  if symb == "id":
60  ds9.dot(str(s.getId()), xc, yc, frame=frame, ctype=ctype, size=size)
61  else:
62  ds9.dot(symb, xc, yc, frame=frame, ctype=ctype, size=size)
63 
64 
65 # Kernel display utilities
66 #
67 
68 
69 def showKernelSpatialCells(maskedIm, kernelCellSet, showChi2=False, symb="o",
70  ctype=None, ctypeUnused=None, ctypeBad=None, size=3,
71  frame=None, title="Spatial Cells"):
72  """Show the SpatialCells. If symb is something that ds9.dot
73  understands (e.g. "o"), the top nMaxPerCell candidates will be
74  indicated with that symbol, using ctype and size"""
75 
76  ds9.mtv(maskedIm, frame=frame, title=title)
77  with ds9.Buffering():
78  origin = [-maskedIm.getX0(), -maskedIm.getY0()]
79  for cell in kernelCellSet.getCellList():
80  displayUtils.drawBBox(cell.getBBox(), origin=origin, frame=frame)
81 
82  goodies = ctypeBad is None
83  for cand in cell.begin(goodies):
84  xc, yc = cand.getXCenter() + origin[0], cand.getYCenter() + origin[1]
85  if cand.getStatus() == afwMath.SpatialCellCandidate.BAD:
86  color = ctypeBad
87  elif cand.getStatus() == afwMath.SpatialCellCandidate.GOOD:
88  color = ctype
89  elif cand.getStatus() == afwMath.SpatialCellCandidate.UNKNOWN:
90  color = ctypeUnused
91  else:
92  continue
93 
94  if color:
95  ds9.dot(symb, xc, yc, frame=frame, ctype=color, size=size)
96 
97  if showChi2:
98  rchi2 = cand.getChi2()
99  if rchi2 > 1e100:
100  rchi2 = np.nan
101  ds9.dot("%d %.1f" % (cand.getId(), rchi2),
102  xc - size, yc - size - 4, frame=frame, ctype=color, size=size)
103 
104 
105 def showDiaSources(sources, exposure, isFlagged, isDipole, frame=None):
106  """Display Dia Sources
107  """
108  #
109  # Show us the ccandidates
110  #
111  # Too many mask planes in diffims
112  for plane in ("BAD", "CR", "EDGE", "INTERPOlATED", "INTRP", "SAT", "SATURATED"):
113  ds9.setMaskPlaneVisibility(plane, False)
114 
115  mos = displayUtils.Mosaic()
116  for i in range(len(sources)):
117  source = sources[i]
118  badFlag = isFlagged[i]
119  dipoleFlag = isDipole[i]
120  bbox = source.getFootprint().getBBox()
121  stamp = exposure.Factory(exposure, bbox, True)
122  im = displayUtils.Mosaic(gutter=1, background=0, mode="x")
123  im.append(stamp.getMaskedImage())
124  lab = "%.1f,%.1f:" % (source.getX(), source.getY())
125  if badFlag:
126  ctype = ds9.RED
127  lab += "BAD"
128  if dipoleFlag:
129  ctype = ds9.YELLOW
130  lab += "DIPOLE"
131  if not badFlag and not dipoleFlag:
132  ctype = ds9.GREEN
133  lab += "OK"
134  mos.append(im.makeMosaic(), lab, ctype)
135  title = "Dia Sources"
136  mosaicImage = mos.makeMosaic(frame=frame, title=title)
137  return mosaicImage
138 
139 
140 def showKernelCandidates(kernelCellSet, kernel, background, frame=None, showBadCandidates=True,
141  resids=False, kernels=False):
142  """Display the Kernel candidates.
143  If kernel is provided include spatial model and residuals;
144  If chi is True, generate a plot of residuals/sqrt(variance), i.e. chi
145  """
146 
147  #
148  # Show us the ccandidates
149  #
150  if kernels:
151  mos = displayUtils.Mosaic(gutter=5, background=0)
152  else:
153  mos = displayUtils.Mosaic(gutter=5, background=-1)
154  #
155  candidateCenters = []
156  candidateCentersBad = []
157  candidateIndex = 0
158  for cell in kernelCellSet.getCellList():
159  for cand in cell.begin(False): # include bad candidates
160  # Original difference image; if does not exist, skip candidate
161  try:
162  resid = cand.getDifferenceImage(diffimLib.KernelCandidateF.ORIG)
163  except Exception:
164  continue
165 
166  rchi2 = cand.getChi2()
167  if rchi2 > 1e100:
168  rchi2 = np.nan
169 
170  if not showBadCandidates and cand.isBad():
171  continue
172 
173  im_resid = displayUtils.Mosaic(gutter=1, background=-0.5, mode="x")
174 
175  try:
176  im = cand.getScienceMaskedImage()
177  im = im.Factory(im, True)
178  im.setXY0(cand.getScienceMaskedImage().getXY0())
179  except Exception:
180  continue
181  if (not resids and not kernels):
182  im_resid.append(im.Factory(im, True))
183  try:
184  im = cand.getTemplateMaskedImage()
185  im = im.Factory(im, True)
186  im.setXY0(cand.getTemplateMaskedImage().getXY0())
187  except Exception:
188  continue
189  if (not resids and not kernels):
190  im_resid.append(im.Factory(im, True))
191 
192  # Difference image with original basis
193  if resids:
194  var = resid.getVariance()
195  var = var.Factory(var, True)
196  np.sqrt(var.getArray(), var.getArray()) # inplace sqrt
197  resid = resid.getImage()
198  resid /= var
199  bbox = kernel.shrinkBBox(resid.getBBox())
200  resid = resid.Factory(resid, bbox, True)
201  elif kernels:
202  kim = cand.getKernelImage(diffimLib.KernelCandidateF.ORIG).convertF()
203  resid = kim.Factory(kim, True)
204  im_resid.append(resid)
205 
206  # residuals using spatial model
207  ski = afwImage.ImageD(kernel.getDimensions())
208  kernel.computeImage(ski, False, int(cand.getXCenter()), int(cand.getYCenter()))
209  sk = afwMath.FixedKernel(ski)
210  sbg = 0.0
211  if background:
212  sbg = background(int(cand.getXCenter()), int(cand.getYCenter()))
213  sresid = cand.getDifferenceImage(sk, sbg)
214  resid = sresid
215  if resids:
216  resid = sresid.getImage()
217  resid /= var
218  bbox = kernel.shrinkBBox(resid.getBBox())
219  resid = resid.Factory(resid, bbox, True)
220  elif kernels:
221  kim = ski.convertF()
222  resid = kim.Factory(kim, True)
223  im_resid.append(resid)
224 
225  im = im_resid.makeMosaic()
226 
227  lab = "%d chi^2 %.1f" % (cand.getId(), rchi2)
228  ctype = ds9.RED if cand.isBad() else ds9.GREEN
229 
230  mos.append(im, lab, ctype)
231 
232  if False and np.isnan(rchi2):
233  ds9.mtv(cand.getScienceMaskedImage.getImage(), title="candidate", frame=1)
234  print("rating", cand.getCandidateRating())
235 
236  im = cand.getScienceMaskedImage()
237  center = (candidateIndex, cand.getXCenter() - im.getX0(), cand.getYCenter() - im.getY0())
238  candidateIndex += 1
239  if cand.isBad():
240  candidateCentersBad.append(center)
241  else:
242  candidateCenters.append(center)
243 
244  if resids:
245  title = "chi Diffim"
246  elif kernels:
247  title = "Kernels"
248  else:
249  title = "Candidates & residuals"
250  mosaicImage = mos.makeMosaic(frame=frame, title=title)
251 
252  return mosaicImage
253 
254 
255 def showKernelBasis(kernel, frame=None):
256  """Display a Kernel's basis images
257  """
258  mos = displayUtils.Mosaic()
259 
260  for k in kernel.getKernelList():
261  im = afwImage.ImageD(k.getDimensions())
262  k.computeImage(im, False)
263  mos.append(im)
264  mos.makeMosaic(frame=frame, title="Kernel Basis Images")
265 
266  return mos
267 
268 
269 
270 
271 def plotKernelSpatialModel(kernel, kernelCellSet, showBadCandidates=True,
272  numSample=128, keepPlots=True, maxCoeff=10):
273  """Plot the Kernel spatial model."""
274 
275  try:
276  import matplotlib.pyplot as plt
277  import matplotlib.colors
278  except ImportError as e:
279  print("Unable to import numpy and matplotlib: %s" % e)
280  return
281 
282  x0 = kernelCellSet.getBBox().getBeginX()
283  y0 = kernelCellSet.getBBox().getBeginY()
284 
285  candPos = list()
286  candFits = list()
287  badPos = list()
288  badFits = list()
289  candAmps = list()
290  badAmps = list()
291  for cell in kernelCellSet.getCellList():
292  for cand in cell.begin(False):
293  if not showBadCandidates and cand.isBad():
294  continue
295  candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter())
296  try:
297  im = cand.getTemplateMaskedImage()
298  except Exception as e:
299  continue
300 
301  targetFits = badFits if cand.isBad() else candFits
302  targetPos = badPos if cand.isBad() else candPos
303  targetAmps = badAmps if cand.isBad() else candAmps
304 
305  # compare original and spatial kernel coefficients
306  kp0 = np.array(cand.getKernel(diffimLib.KernelCandidateF.ORIG).getKernelParameters())
307  amp = cand.getCandidateRating()
308 
309  targetFits = badFits if cand.isBad() else candFits
310  targetPos = badPos if cand.isBad() else candPos
311  targetAmps = badAmps if cand.isBad() else candAmps
312 
313  targetFits.append(kp0)
314  targetPos.append(candCenter)
315  targetAmps.append(amp)
316 
317  xGood = np.array([pos.getX() for pos in candPos]) - x0
318  yGood = np.array([pos.getY() for pos in candPos]) - y0
319  zGood = np.array(candFits)
320 
321  xBad = np.array([pos.getX() for pos in badPos]) - x0
322  yBad = np.array([pos.getY() for pos in badPos]) - y0
323  zBad = np.array(badFits)
324  numBad = len(badPos)
325 
326  xRange = np.linspace(0, kernelCellSet.getBBox().getWidth(), num=numSample)
327  yRange = np.linspace(0, kernelCellSet.getBBox().getHeight(), num=numSample)
328 
329  if maxCoeff:
330  maxCoeff = min(maxCoeff, kernel.getNKernelParameters())
331  else:
332  maxCoeff = kernel.getNKernelParameters()
333 
334  for k in range(maxCoeff):
335  func = kernel.getSpatialFunction(k)
336  dfGood = zGood[:, k] - np.array([func(pos.getX(), pos.getY()) for pos in candPos])
337  yMin = dfGood.min()
338  yMax = dfGood.max()
339  if numBad > 0:
340  dfBad = zBad[:, k] - np.array([func(pos.getX(), pos.getY()) for pos in badPos])
341  # Can really screw up the range...
342  yMin = min([yMin, dfBad.min()])
343  yMax = max([yMax, dfBad.max()])
344  yMin -= 0.05 * (yMax - yMin)
345  yMax += 0.05 * (yMax - yMin)
346 
347  fRange = np.ndarray((len(xRange), len(yRange)))
348  for j, yVal in enumerate(yRange):
349  for i, xVal in enumerate(xRange):
350  fRange[j][i] = func(xVal, yVal)
351 
352  fig = plt.figure(k)
353 
354  fig.clf()
355  try:
356  fig.canvas._tkcanvas._root().lift() # == Tk's raise, but raise is a python reserved word
357  except Exception: # protect against API changes
358  pass
359 
360  fig.suptitle('Kernel component %d' % k)
361 
362  # LL
363  ax = fig.add_axes((0.1, 0.05, 0.35, 0.35))
364  vmin = fRange.min() # - 0.05 * np.fabs(fRange.min())
365  vmax = fRange.max() # + 0.05 * np.fabs(fRange.max())
366  norm = matplotlib.colors.Normalize(vmin=vmin, vmax=vmax)
367  im = ax.imshow(fRange, aspect='auto', norm=norm,
368  extent=[0, kernelCellSet.getBBox().getWidth()-1,
369  0, kernelCellSet.getBBox().getHeight()-1])
370  ax.set_title('Spatial polynomial')
371  plt.colorbar(im, orientation='horizontal', ticks=[vmin, vmax])
372 
373  # UL
374  ax = fig.add_axes((0.1, 0.55, 0.35, 0.35))
375  ax.plot(-2.5*np.log10(candAmps), zGood[:, k], 'b+')
376  if numBad > 0:
377  ax.plot(-2.5*np.log10(badAmps), zBad[:, k], 'r+')
378  ax.set_title("Basis Coefficients")
379  ax.set_xlabel("Instr mag")
380  ax.set_ylabel("Coeff")
381 
382  # LR
383  ax = fig.add_axes((0.55, 0.05, 0.35, 0.35))
384  ax.set_autoscale_on(False)
385  ax.set_xbound(lower=0, upper=kernelCellSet.getBBox().getHeight())
386  ax.set_ybound(lower=yMin, upper=yMax)
387  ax.plot(yGood, dfGood, 'b+')
388  if numBad > 0:
389  ax.plot(yBad, dfBad, 'r+')
390  ax.axhline(0.0)
391  ax.set_title('dCoeff (indiv-spatial) vs. y')
392 
393  # UR
394  ax = fig.add_axes((0.55, 0.55, 0.35, 0.35))
395  ax.set_autoscale_on(False)
396  ax.set_xbound(lower=0, upper=kernelCellSet.getBBox().getWidth())
397  ax.set_ybound(lower=yMin, upper=yMax)
398  ax.plot(xGood, dfGood, 'b+')
399  if numBad > 0:
400  ax.plot(xBad, dfBad, 'r+')
401  ax.axhline(0.0)
402  ax.set_title('dCoeff (indiv-spatial) vs. x')
403 
404  fig.show()
405 
406  global keptPlots
407  if keepPlots and not keptPlots:
408  # Keep plots open when done
409  def show():
410  print("%s: Please close plots when done." % __name__)
411  try:
412  plt.show()
413  except Exception:
414  pass
415  print("Plots closed, exiting...")
416  import atexit
417  atexit.register(show)
418  keptPlots = True
419 
420 
421 def showKernelMosaic(bbox, kernel, nx=7, ny=None, frame=None, title=None,
422  showCenter=True, showEllipticity=True):
423  """Show a mosaic of Kernel images.
424  """
425  mos = displayUtils.Mosaic()
426 
427  x0 = bbox.getBeginX()
428  y0 = bbox.getBeginY()
429  width = bbox.getWidth()
430  height = bbox.getHeight()
431 
432  if not ny:
433  ny = int(nx*float(height)/width + 0.5)
434  if not ny:
435  ny = 1
436 
437  schema = afwTable.SourceTable.makeMinimalSchema()
438  centroidName = "base_SdssCentroid"
439  shapeName = "base_SdssShape"
440  control = measBase.SdssCentroidControl()
441  schema.getAliasMap().set("slot_Centroid", centroidName)
442  schema.getAliasMap().set("slot_Centroid_flag", centroidName+"_flag")
443  centroider = measBase.SdssCentroidAlgorithm(control, centroidName, schema)
444  sdssShape = measBase.SdssShapeControl()
445  shaper = measBase.SdssShapeAlgorithm(sdssShape, shapeName, schema)
446  table = afwTable.SourceTable.make(schema)
447  table.defineCentroid(centroidName)
448  table.defineShape(shapeName)
449 
450  centers = []
451  shapes = []
452  for iy in range(ny):
453  for ix in range(nx):
454  x = int(ix*(width-1)/(nx-1)) + x0
455  y = int(iy*(height-1)/(ny-1)) + y0
456 
457  im = afwImage.ImageD(kernel.getDimensions())
458  ksum = kernel.computeImage(im, False, x, y)
459  lab = "Kernel(%d,%d)=%.2f" % (x, y, ksum) if False else ""
460  mos.append(im, lab)
461 
462  # SdssCentroidAlgorithm.measure requires an exposure of floats
463  exp = afwImage.makeExposure(afwImage.makeMaskedImage(im.convertF()))
464  w, h = im.getWidth(), im.getHeight()
465  centerX = im.getX0() + w//2
466  centerY = im.getY0() + h//2
467  src = table.makeRecord()
468  foot = afwDet.Footprint(exp.getBBox())
469  foot.addPeak(centerX, centerY, 1)
470  src.setFootprint(foot)
471 
472  centroider.measure(src, exp)
473  centers.append((src.getX(), src.getY()))
474 
475  shaper.measure(src, exp)
476  shapes.append((src.getIxx(), src.getIxy(), src.getIyy()))
477 
478  mos.makeMosaic(frame=frame, title=title if title else "Model Kernel", mode=nx)
479 
480  if centers and frame is not None:
481  i = 0
482  with ds9.Buffering():
483  for cen, shape in zip(centers, shapes):
484  bbox = mos.getBBox(i)
485  i += 1
486  xc, yc = cen[0] + bbox.getMinX(), cen[1] + bbox.getMinY()
487  if showCenter:
488  ds9.dot("+", xc, yc, ctype=ds9.BLUE, frame=frame)
489 
490  if showEllipticity:
491  ixx, ixy, iyy = shape
492  ds9.dot("@:%g,%g,%g" % (ixx, ixy, iyy), xc, yc, frame=frame, ctype=ds9.RED)
493 
494  return mos
495 
496 
497 def plotPixelResiduals(exposure, warpedTemplateExposure, diffExposure, kernelCellSet,
498  kernel, background, testSources, config,
499  origVariance=False, nptsFull=1e6, keepPlots=True, titleFs=14):
500  """Plot diffim residuals for LOCAL and SPATIAL models"""
501  candidateResids = []
502  spatialResids = []
503  nonfitResids = []
504 
505  for cell in kernelCellSet.getCellList():
506  for cand in cell.begin(True): # only look at good ones
507  # Be sure
508  if not (cand.getStatus() == afwMath.SpatialCellCandidate.GOOD):
509  continue
510 
511  diffim = cand.getDifferenceImage(diffimLib.KernelCandidateF.ORIG)
512  orig = cand.getScienceMaskedImage()
513 
514  ski = afwImage.ImageD(kernel.getDimensions())
515  kernel.computeImage(ski, False, int(cand.getXCenter()), int(cand.getYCenter()))
516  sk = afwMath.FixedKernel(ski)
517  sbg = background(int(cand.getXCenter()), int(cand.getYCenter()))
518  sdiffim = cand.getDifferenceImage(sk, sbg)
519 
520  # trim edgs due to convolution
521  bbox = kernel.shrinkBBox(diffim.getBBox())
522  tdiffim = diffim.Factory(diffim, bbox)
523  torig = orig.Factory(orig, bbox)
524  tsdiffim = sdiffim.Factory(sdiffim, bbox)
525 
526  if origVariance:
527  candidateResids.append(np.ravel(tdiffim.getImage().getArray() /
528  np.sqrt(torig.getVariance().getArray())))
529  spatialResids.append(np.ravel(tsdiffim.getImage().getArray() /
530  np.sqrt(torig.getVariance().getArray())))
531  else:
532  candidateResids.append(np.ravel(tdiffim.getImage().getArray() /
533  np.sqrt(tdiffim.getVariance().getArray())))
534  spatialResids.append(np.ravel(tsdiffim.getImage().getArray() /
535  np.sqrt(tsdiffim.getVariance().getArray())))
536 
537  fullIm = diffExposure.getMaskedImage().getImage().getArray()
538  fullMask = diffExposure.getMaskedImage().getMask().getArray()
539  if origVariance:
540  fullVar = exposure.getMaskedImage().getVariance().getArray()
541  else:
542  fullVar = diffExposure.getMaskedImage().getVariance().getArray()
543 
544  bitmaskBad = 0
545  bitmaskBad |= afwImage.Mask.getPlaneBitMask('NO_DATA')
546  bitmaskBad |= afwImage.Mask.getPlaneBitMask('SAT')
547  idx = np.where((fullMask & bitmaskBad) == 0)
548  stride = int(len(idx[0]) // nptsFull)
549  sidx = idx[0][::stride], idx[1][::stride]
550  allResids = fullIm[sidx] / np.sqrt(fullVar[sidx])
551 
552  testFootprints = diffimTools.sourceToFootprintList(testSources, warpedTemplateExposure,
553  exposure, config, Log.getDefaultLogger())
554  for fp in testFootprints:
555  subexp = diffExposure.Factory(diffExposure, fp["footprint"].getBBox())
556  subim = subexp.getMaskedImage().getImage()
557  if origVariance:
558  subvar = afwImage.ExposureF(exposure, fp["footprint"].getBBox()).getMaskedImage().getVariance()
559  else:
560  subvar = subexp.getMaskedImage().getVariance()
561  nonfitResids.append(np.ravel(subim.getArray() / np.sqrt(subvar.getArray())))
562 
563  candidateResids = np.ravel(np.array(candidateResids))
564  spatialResids = np.ravel(np.array(spatialResids))
565  nonfitResids = np.ravel(np.array(nonfitResids))
566 
567  try:
568  import pylab
569  from matplotlib.font_manager import FontProperties
570  except ImportError as e:
571  print("Unable to import pylab: %s" % e)
572  return
573 
574  fig = pylab.figure()
575  fig.clf()
576  try:
577  fig.canvas._tkcanvas._root().lift() # == Tk's raise, but raise is a python reserved word
578  except Exception: # protect against API changes
579  pass
580  if origVariance:
581  fig.suptitle("Diffim residuals: Normalized by sqrt(input variance)", fontsize=titleFs)
582  else:
583  fig.suptitle("Diffim residuals: Normalized by sqrt(diffim variance)", fontsize=titleFs)
584 
585  sp1 = pylab.subplot(221)
586  sp2 = pylab.subplot(222, sharex=sp1, sharey=sp1)
587  sp3 = pylab.subplot(223, sharex=sp1, sharey=sp1)
588  sp4 = pylab.subplot(224, sharex=sp1, sharey=sp1)
589  xs = np.arange(-5, 5.05, 0.1)
590  ys = 1. / np.sqrt(2*np.pi)*np.exp(-0.5*xs**2)
591 
592  sp1.hist(candidateResids, bins=xs, normed=True, alpha=0.5, label="N(%.2f, %.2f)"
593  % (np.mean(candidateResids), np.var(candidateResids)))
594  sp1.plot(xs, ys, "r-", lw=2, label="N(0,1)")
595  sp1.set_title("Candidates: basis fit", fontsize=titleFs-2)
596  sp1.legend(loc=1, fancybox=True, shadow=True, prop=FontProperties(size=titleFs-6))
597 
598  sp2.hist(spatialResids, bins=xs, normed=True, alpha=0.5, label="N(%.2f, %.2f)"
599  % (np.mean(spatialResids), np.var(spatialResids)))
600  sp2.plot(xs, ys, "r-", lw=2, label="N(0,1)")
601  sp2.set_title("Candidates: spatial fit", fontsize=titleFs-2)
602  sp2.legend(loc=1, fancybox=True, shadow=True, prop=FontProperties(size=titleFs-6))
603 
604  sp3.hist(nonfitResids, bins=xs, normed=True, alpha=0.5, label="N(%.2f, %.2f)"
605  % (np.mean(nonfitResids), np.var(nonfitResids)))
606  sp3.plot(xs, ys, "r-", lw=2, label="N(0,1)")
607  sp3.set_title("Control sample: spatial fit", fontsize=titleFs-2)
608  sp3.legend(loc=1, fancybox=True, shadow=True, prop=FontProperties(size=titleFs-6))
609 
610  sp4.hist(allResids, bins=xs, normed=True, alpha=0.5, label="N(%.2f, %.2f)"
611  % (np.mean(allResids), np.var(allResids)))
612  sp4.plot(xs, ys, "r-", lw=2, label="N(0,1)")
613  sp4.set_title("Full image (subsampled)", fontsize=titleFs-2)
614  sp4.legend(loc=1, fancybox=True, shadow=True, prop=FontProperties(size=titleFs-6))
615 
616  pylab.setp(sp1.get_xticklabels()+sp1.get_yticklabels(), fontsize=titleFs-4)
617  pylab.setp(sp2.get_xticklabels()+sp2.get_yticklabels(), fontsize=titleFs-4)
618  pylab.setp(sp3.get_xticklabels()+sp3.get_yticklabels(), fontsize=titleFs-4)
619  pylab.setp(sp4.get_xticklabels()+sp4.get_yticklabels(), fontsize=titleFs-4)
620 
621  sp1.set_xlim(-5, 5)
622  sp1.set_ylim(0, 0.5)
623  fig.show()
624 
625  global keptPlots
626  if keepPlots and not keptPlots:
627  # Keep plots open when done
628  def show():
629  print("%s: Please close plots when done." % __name__)
630  try:
631  pylab.show()
632  except Exception:
633  pass
634  print("Plots closed, exiting...")
635  import atexit
636  atexit.register(show)
637  keptPlots = True
638 
639 
640 def calcCentroid(arr):
641  """Calculate first moment of a (kernel) image"""
642  y, x = arr.shape
643  sarr = arr*arr
644  xarr = np.asarray([[el for el in range(x)] for el2 in range(y)])
645  yarr = np.asarray([[el2 for el in range(x)] for el2 in range(y)])
646  narr = xarr*sarr
647  sarrSum = sarr.sum()
648  centx = narr.sum()/sarrSum
649  narr = yarr*sarr
650  centy = narr.sum()/sarrSum
651  return centx, centy
652 
653 
654 def calcWidth(arr, centx, centy):
655  """Calculate second moment of a (kernel) image"""
656  y, x = arr.shape
657  # Square the flux so we don't have to deal with negatives
658  sarr = arr*arr
659  xarr = np.asarray([[el for el in range(x)] for el2 in range(y)])
660  yarr = np.asarray([[el2 for el in range(x)] for el2 in range(y)])
661  narr = sarr*np.power((xarr - centx), 2.)
662  sarrSum = sarr.sum()
663  xstd = np.sqrt(narr.sum()/sarrSum)
664  narr = sarr*np.power((yarr - centy), 2.)
665  ystd = np.sqrt(narr.sum()/sarrSum)
666  return xstd, ystd
667 
668 
669 def printSkyDiffs(sources, wcs):
670  """Print differences in sky coordinates between source Position and its Centroid mapped through Wcs"""
671  for s in sources:
672  sCentroid = s.getCentroid()
673  sPosition = s.getCoord().getPosition(afwGeom.degrees)
674  dra = 3600*(sPosition.getX() - wcs.pixelToSky(sCentroid).getPosition(afwGeom.degrees).getX())/0.2
675  ddec = 3600*(sPosition.getY() - wcs.pixelToSky(sCentroid).getPosition(afwGeom.degrees).getY())/0.2
676  if np.isfinite(dra) and np.isfinite(ddec):
677  print(dra, ddec)
678 
679 
680 def makeRegions(sources, outfilename, wcs=None):
681  """Create regions file for ds9 from input source list"""
682  fh = open(outfilename, "w")
683  fh.write("global color=red font=\"helvetica 10 normal\" "
684  "select=1 highlite=1 edit=1 move=1 delete=1 include=1 fixed=0 source\nfk5\n")
685  for s in sources:
686  if wcs:
687  (ra, dec) = wcs.pixelToSky(s.getCentroid()).getPosition(afwGeom.degrees)
688  else:
689  (ra, dec) = s.getCoord().getPosition(afwGeom.degrees)
690  if np.isfinite(ra) and np.isfinite(dec):
691  fh.write("circle(%f,%f,2\")\n"%(ra, dec))
692  fh.flush()
693  fh.close()
694 
695 
696 def showSourceSetSky(sSet, wcs, xy0, frame=0, ctype=ds9.GREEN, symb="+", size=2):
697  """Draw the (RA, Dec) positions of a set of Sources. Image has the XY0."""
698  with ds9.Buffering():
699  for s in sSet:
700  (xc, yc) = wcs.skyToPixel(s.getCoord().getRa(), s.getCoord().getDec())
701  xc -= xy0[0]
702  yc -= xy0[1]
703  ds9.dot(symb, xc, yc, frame=frame, ctype=ctype, size=size)
704 
705 
706 def plotWhisker(results, newWcs):
707  """Plot whisker diagram of astromeric offsets between results.matches"""
708  refCoordKey = results.matches[0].first.getTable().getCoordKey()
709  inCentroidKey = results.matches[0].second.getTable().getCentroidKey()
710  positions = [m.first.get(refCoordKey) for m in results.matches]
711  residuals = [m.first.get(refCoordKey).getOffsetFrom(
712  newWcs.pixelToSky(m.second.get(inCentroidKey))) for
713  m in results.matches]
714  import matplotlib.pyplot as plt
715  fig = plt.figure()
716  sp = fig.add_subplot(1, 1, 0)
717  xpos = [x[0].asDegrees() for x in positions]
718  ypos = [x[1].asDegrees() for x in positions]
719  xpos.append(0.02*(max(xpos) - min(xpos)) + min(xpos))
720  ypos.append(0.98*(max(ypos) - min(ypos)) + min(ypos))
721  xidxs = np.isfinite(xpos)
722  yidxs = np.isfinite(ypos)
723  X = np.asarray(xpos)[xidxs]
724  Y = np.asarray(ypos)[yidxs]
725  distance = [x[1].asArcseconds() for x in residuals]
726  distance.append(0.2)
727  distance = np.asarray(distance)[xidxs]
728  # NOTE: This assumes that the bearing is measured positive from +RA through North.
729  # From the documentation this is not clear.
730  bearing = [x[0].asRadians() for x in residuals]
731  bearing.append(0)
732  bearing = np.asarray(bearing)[xidxs]
733  U = (distance*np.cos(bearing))
734  V = (distance*np.sin(bearing))
735  sp.quiver(X, Y, U, V)
736  sp.set_title("WCS Residual")
737  plt.show()
738 
739 
740 class DipoleTestImage(object):
741 
742  """!Utility class for dipole measurement testing
743 
744  Generate an image with simulated dipoles and noise; store the original "pre-subtraction" images
745  and catalogs as well.
746  Used to generate test data for DMTN-007 (http://dmtn-007.lsst.io).
747  """
748 
749  def __init__(self, w=101, h=101, xcenPos=[27.], ycenPos=[25.], xcenNeg=[23.], ycenNeg=[25.],
750  psfSigma=2., flux=[30000.], fluxNeg=None, noise=10., gradientParams=None):
751  self.w = w
752  self.h = h
753  self.xcenPos = xcenPos
754  self.ycenPos = ycenPos
755  self.xcenNeg = xcenNeg
756  self.ycenNeg = ycenNeg
757  self.psfSigma = psfSigma
758  self.flux = flux
759  self.fluxNeg = fluxNeg
760  if fluxNeg is None:
761  self.fluxNeg = self.flux
762  self.noise = noise
763  self.gradientParams = gradientParams
764  self._makeDipoleImage()
765 
766  def _makeDipoleImage(self):
767  """!Generate an exposure and catalog with the given dipole source(s)"""
768 
769  # Must seed the pos/neg images with different values to ensure they get different noise realizations
770  posImage, posCatalog = self._makeStarImage(
771  xc=self.xcenPos, yc=self.ycenPos, flux=self.flux, randomSeed=111)
772 
773  negImage, negCatalog = self._makeStarImage(
774  xc=self.xcenNeg, yc=self.ycenNeg, flux=self.fluxNeg, randomSeed=222)
775 
776  dipole = posImage.clone()
777  di = dipole.getMaskedImage()
778  di -= negImage.getMaskedImage()
779 
780  # Carry through pos/neg detection masks to new planes in diffim
781  dm = di.getMask()
782  posDetectedBits = posImage.getMaskedImage().getMask().getArray() == dm.getPlaneBitMask("DETECTED")
783  negDetectedBits = negImage.getMaskedImage().getMask().getArray() == dm.getPlaneBitMask("DETECTED")
784  pos_det = dm.addMaskPlane("DETECTED_POS") # new mask plane -- different from "DETECTED"
785  neg_det = dm.addMaskPlane("DETECTED_NEG") # new mask plane -- different from "DETECTED_NEGATIVE"
786  dma = dm.getArray()
787  # set the two custom mask planes to these new masks
788  dma[:, :] = posDetectedBits*pos_det + negDetectedBits*neg_det
789  self.diffim, self.posImage, self.posCatalog, self.negImage, self.negCatalog \
790  = dipole, posImage, posCatalog, negImage, negCatalog
791 
792  def _makeStarImage(self, xc=[15.3], yc=[18.6], flux=[2500], schema=None, randomSeed=None):
793  """!Generate an exposure and catalog with the given stellar source(s)"""
794 
795  from lsst.meas.base.tests import TestDataset
796  bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Point2I(self.w-1, self.h-1))
797  dataset = TestDataset(bbox, psfSigma=self.psfSigma, threshold=1.)
798 
799  for i in range(len(xc)):
800  dataset.addSource(flux=flux[i], centroid=afwGeom.Point2D(xc[i], yc[i]))
801 
802  if schema is None:
803  schema = TestDataset.makeMinimalSchema()
804  exposure, catalog = dataset.realize(noise=self.noise, schema=schema, randomSeed=randomSeed)
805 
806  if self.gradientParams is not None:
807  y, x = np.mgrid[:self.w, :self.h]
808  gp = self.gradientParams
809  gradient = gp[0] + gp[1] * x + gp[2] * y
810  if len(self.gradientParams) > 3: # it includes a set of 2nd-order polynomial params
811  gradient += gp[3] * x*y + gp[4] * x*x + gp[5] * y*y
812  imgArr = exposure.getMaskedImage().getArrays()[0]
813  imgArr += gradient
814 
815  return exposure, catalog
816 
817  def fitDipoleSource(self, source, **kwds):
818  alg = DipoleFitAlgorithm(self.diffim, self.posImage, self.negImage)
819  fitResult = alg.fitDipole(source, **kwds)
820  return fitResult
821 
822  def detectDipoleSources(self, doMerge=True, diffim=None, detectSigma=5.5, grow=3, minBinSize=32):
823  """!Utility function for detecting dipoles.
824 
825  Detect pos/neg sources in the diffim, then merge them. A
826  bigger "grow" parameter leads to a larger footprint which
827  helps with dipole measurement for faint dipoles.
828 
829  Parameters
830  ----------
831  doMerge : `bool`
832  Whether to merge the positive and negagive detections into a single source table
833  diffim : `lsst.afw.image.exposure.exposure.ExposureF`
834  Difference image on which to perform detection
835  detectSigma : `float`
836  Threshold for object detection
837  grow : `int`
838  Number of pixels to grow the footprints before merging
839  minBinSize : `int`
840  Minimum bin size for the background (re)estimation (only applies if the default leads to
841  min(nBinX, nBinY) < fit order so the default config parameter needs to be decreased, but not
842  to a value smaller than minBinSize, in which case the fitting algorithm will take over and
843  decrease the fit order appropriately.)
844 
845  Returns
846  -------
847  sources : `lsst.afw.table.SourceCatalog`
848  If doMerge=True, the merged source catalog is returned OR
849  detectTask : `lsst.meas.algorithms.SourceDetectionTask`
850  schema : `lsst.afw.table.Schema`
851  If doMerge=False, the source detection task and its schema are returned
852  """
853  if diffim is None:
854  diffim = self.diffim
855 
856  # Start with a minimal schema - only the fields all SourceCatalogs need
857  schema = afwTable.SourceTable.makeMinimalSchema()
858 
859  # Customize the detection task a bit (optional)
860  detectConfig = measAlg.SourceDetectionConfig()
861  detectConfig.returnOriginalFootprints = False # should be the default
862 
863  psfSigma = diffim.getPsf().computeShape().getDeterminantRadius()
864 
865  # code from imageDifference.py:
866  detectConfig.thresholdPolarity = "both"
867  detectConfig.thresholdValue = detectSigma
868  # detectConfig.nSigmaToGrow = psfSigma
869  detectConfig.reEstimateBackground = True # if False, will fail often for faint sources on gradients?
870  detectConfig.thresholdType = "pixel_stdev"
871  # Test images are often quite small, so may need to adjust background binSize
872  while ((min(diffim.getWidth(), diffim.getHeight()))//detectConfig.background.binSize <
873  detectConfig.background.approxOrderX and detectConfig.background.binSize > minBinSize):
874  detectConfig.background.binSize = max(minBinSize, detectConfig.background.binSize//2)
875 
876  # Create the detection task. We pass the schema so the task can declare a few flag fields
877  detectTask = measAlg.SourceDetectionTask(schema, config=detectConfig)
878 
879  table = afwTable.SourceTable.make(schema)
880  catalog = detectTask.makeSourceCatalog(table, diffim, sigma=psfSigma)
881 
882  # Now do the merge.
883  if doMerge:
884  fpSet = catalog.fpSets.positive
885  fpSet.merge(catalog.fpSets.negative, grow, grow, False)
886  sources = afwTable.SourceCatalog(table)
887  fpSet.makeSources(sources)
888 
889  return sources
890 
891  else:
892  return detectTask, schema
def _makeDipoleImage(self)
Generate an exposure and catalog with the given dipole source(s)
Definition: utils.py:766
def showDiaSources(sources, exposure, isFlagged, isDipole, frame=None)
Definition: utils.py:105
Lightweight class containing methods for fitting a dipole model in a diffim, used by DipoleFitPlugin...
def makeRegions(sources, outfilename, wcs=None)
Definition: utils.py:680
def __init__(self, w=101, h=101, xcenPos=[27.], ycenPos=[25.], xcenNeg=[23.], ycenNeg=[25.], psfSigma=2., flux=[30000.], fluxNeg=None, noise=10., gradientParams=None)
Definition: utils.py:750
Utility class for dipole measurement testing.
Definition: utils.py:740
MaskedImage< ImagePixelT, MaskPixelT, VariancePixelT > * makeMaskedImage(typename std::shared_ptr< Image< ImagePixelT >> image, typename std::shared_ptr< Mask< MaskPixelT >> mask=Mask< MaskPixelT >(), typename std::shared_ptr< Image< VariancePixelT >> variance=Image< VariancePixelT >())
def showKernelSpatialCells(maskedIm, kernelCellSet, showChi2=False, symb="o", ctype=None, ctypeUnused=None, ctypeBad=None, size=3, frame=None, title="Spatial Cells")
Definition: utils.py:71
std::shared_ptr< Exposure< ImagePixelT, MaskPixelT, VariancePixelT > > makeExposure(MaskedImage< ImagePixelT, MaskPixelT, VariancePixelT > &mimage, std::shared_ptr< geom::SkyWcs const > wcs=std::shared_ptr< geom::SkyWcs const >())
def _makeStarImage(self, xc=[15.3], yc=[18.6], flux=[2500], schema=None, randomSeed=None)
Generate an exposure and catalog with the given stellar source(s)
Definition: utils.py:792
def plotKernelSpatialModel(kernel, kernelCellSet, showBadCandidates=True, numSample=128, keepPlots=True, maxCoeff=10)
Definition: utils.py:272
def showSourceSetSky(sSet, wcs, xy0, frame=0, ctype=ds9.GREEN, symb="+", size=2)
Definition: utils.py:696
def plotPixelResiduals(exposure, warpedTemplateExposure, diffExposure, kernelCellSet, kernel, background, testSources, config, origVariance=False, nptsFull=1e6, keepPlots=True, titleFs=14)
Definition: utils.py:499
def printSkyDiffs(sources, wcs)
Definition: utils.py:669
def showKernelCandidates(kernelCellSet, kernel, background, frame=None, showBadCandidates=True, resids=False, kernels=False)
Definition: utils.py:141
def showSourceSet(sSet, xy0=(0, 0), frame=0, ctype=ds9.GREEN, symb="+", size=2)
Definition: utils.py:52
def detectDipoleSources(self, doMerge=True, diffim=None, detectSigma=5.5, grow=3, minBinSize=32)
Utility function for detecting dipoles.
Definition: utils.py:822
def calcCentroid(arr)
Definition: utils.py:640
def calcWidth(arr, centx, centy)
Definition: utils.py:654
def plotWhisker(results, newWcs)
Definition: utils.py:706
def showKernelMosaic(bbox, kernel, nx=7, ny=None, frame=None, title=None, showCenter=True, showEllipticity=True)
Definition: utils.py:422
def showKernelBasis(kernel, frame=None)
Definition: utils.py:255
def fitDipoleSource(self, source, kwds)
Definition: utils.py:817