lsst.ip.diffim  18.0.0-4-gd60ad06
utils.py
Go to the documentation of this file.
1 # This file is part of ip_diffim.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 """Support utilities for Measuring sources"""
23 
24 # Export DipoleTestImage to expose fake image generating funcs
25 __all__ = ["DipoleTestImage"]
26 
27 import numpy as np
28 
29 import lsst.afw.detection as afwDet
30 import lsst.afw.display as afwDisplay
31 import lsst.afw.geom as afwGeom
32 import lsst.afw.image as afwImage
33 import lsst.afw.math as afwMath
34 import lsst.afw.table as afwTable
35 from lsst.log import Log
36 import lsst.meas.algorithms as measAlg
37 import lsst.meas.base as measBase
38 from .dipoleFitTask import DipoleFitAlgorithm
39 from . import diffimLib
40 from . import diffimTools
41 
42 afwDisplay.setDefaultMaskTransparency(75)
43 keptPlots = False # Have we arranged to keep spatial plots open?
44 
45 
46 def showSourceSet(sSet, xy0=(0, 0), frame=0, ctype=afwDisplay.GREEN, symb="+", size=2):
47  """Draw the (XAstrom, YAstrom) positions of a set of Sources.
48 
49  Image has the given XY0.
50  """
51  disp = afwDisplay.afwDisplay(frame=frame)
52  with disp.Buffering():
53  for s in sSet:
54  xc, yc = s.getXAstrom() - xy0[0], s.getYAstrom() - xy0[1]
55 
56  if symb == "id":
57  disp.dot(str(s.getId()), xc, yc, ctype=ctype, size=size)
58  else:
59  disp.dot(symb, xc, yc, ctype=ctype, size=size)
60 
61 
62 # Kernel display utilities
63 #
64 
65 
66 def showKernelSpatialCells(maskedIm, kernelCellSet, showChi2=False, symb="o",
67  ctype=None, ctypeUnused=None, ctypeBad=None, size=3,
68  frame=None, title="Spatial Cells"):
69  """Show the SpatialCells.
70 
71  If symb is something that display.dot understands (e.g. "o"), the top
72  nMaxPerCell candidates will be indicated with that symbol, using ctype
73  and size.
74  """
75  disp = afwDisplay.Display(frame=frame)
76  disp.mtv(maskedIm, title=title)
77  with disp.Buffering():
78  origin = [-maskedIm.getX0(), -maskedIm.getY0()]
79  for cell in kernelCellSet.getCellList():
80  afwDisplay.utils.drawBBox(cell.getBBox(), origin=origin, display=disp)
81 
82  goodies = ctypeBad is None
83  for cand in cell.begin(goodies):
84  xc, yc = cand.getXCenter() + origin[0], cand.getYCenter() + origin[1]
85  if cand.getStatus() == afwMath.SpatialCellCandidate.BAD:
86  color = ctypeBad
87  elif cand.getStatus() == afwMath.SpatialCellCandidate.GOOD:
88  color = ctype
89  elif cand.getStatus() == afwMath.SpatialCellCandidate.UNKNOWN:
90  color = ctypeUnused
91  else:
92  continue
93 
94  if color:
95  disp.dot(symb, xc, yc, ctype=color, size=size)
96 
97  if showChi2:
98  rchi2 = cand.getChi2()
99  if rchi2 > 1e100:
100  rchi2 = np.nan
101  disp.dot("%d %.1f" % (cand.getId(), rchi2),
102  xc - size, yc - size - 4, ctype=color, size=size)
103 
104 
105 def showDiaSources(sources, exposure, isFlagged, isDipole, frame=None):
106  """Display Dia Sources.
107  """
108  #
109  # Show us the ccandidates
110  #
111  # Too many mask planes in diffims
112  disp = afwDisplay.Display(frame=frame)
113  for plane in ("BAD", "CR", "EDGE", "INTERPOlATED", "INTRP", "SAT", "SATURATED"):
114  disp.setMaskPlaneColor(plane, color="ignore")
115 
116  mos = afwDisplay.utils.Mosaic()
117  for i in range(len(sources)):
118  source = sources[i]
119  badFlag = isFlagged[i]
120  dipoleFlag = isDipole[i]
121  bbox = source.getFootprint().getBBox()
122  stamp = exposure.Factory(exposure, bbox, True)
123  im = afwDisplay.utils.Mosaic(gutter=1, background=0, mode="x")
124  im.append(stamp.getMaskedImage())
125  lab = "%.1f,%.1f:" % (source.getX(), source.getY())
126  if badFlag:
127  ctype = afwDisplay.RED
128  lab += "BAD"
129  if dipoleFlag:
130  ctype = afwDisplay.YELLOW
131  lab += "DIPOLE"
132  if not badFlag and not dipoleFlag:
133  ctype = afwDisplay.GREEN
134  lab += "OK"
135  mos.append(im.makeMosaic(), lab, ctype)
136  title = "Dia Sources"
137  mosaicImage = mos.makeMosaic(frame=frame, title=title)
138  return mosaicImage
139 
140 
141 def showKernelCandidates(kernelCellSet, kernel, background, frame=None, showBadCandidates=True,
142  resids=False, kernels=False):
143  """Display the Kernel candidates.
144 
145  If kernel is provided include spatial model and residuals;
146  If chi is True, generate a plot of residuals/sqrt(variance), i.e. chi.
147  """
148  #
149  # Show us the ccandidates
150  #
151  if kernels:
152  mos = afwDisplay.utils.Mosaic(gutter=5, background=0)
153  else:
154  mos = afwDisplay.utils.Mosaic(gutter=5, background=-1)
155  #
156  candidateCenters = []
157  candidateCentersBad = []
158  candidateIndex = 0
159  for cell in kernelCellSet.getCellList():
160  for cand in cell.begin(False): # include bad candidates
161  # Original difference image; if does not exist, skip candidate
162  try:
163  resid = cand.getDifferenceImage(diffimLib.KernelCandidateF.ORIG)
164  except Exception:
165  continue
166 
167  rchi2 = cand.getChi2()
168  if rchi2 > 1e100:
169  rchi2 = np.nan
170 
171  if not showBadCandidates and cand.isBad():
172  continue
173 
174  im_resid = afwDisplay.utils.Mosaic(gutter=1, background=-0.5, mode="x")
175 
176  try:
177  im = cand.getScienceMaskedImage()
178  im = im.Factory(im, True)
179  im.setXY0(cand.getScienceMaskedImage().getXY0())
180  except Exception:
181  continue
182  if (not resids and not kernels):
183  im_resid.append(im.Factory(im, True))
184  try:
185  im = cand.getTemplateMaskedImage()
186  im = im.Factory(im, True)
187  im.setXY0(cand.getTemplateMaskedImage().getXY0())
188  except Exception:
189  continue
190  if (not resids and not kernels):
191  im_resid.append(im.Factory(im, True))
192 
193  # Difference image with original basis
194  if resids:
195  var = resid.getVariance()
196  var = var.Factory(var, True)
197  np.sqrt(var.getArray(), var.getArray()) # inplace sqrt
198  resid = resid.getImage()
199  resid /= var
200  bbox = kernel.shrinkBBox(resid.getBBox())
201  resid = resid.Factory(resid, bbox, deep=True)
202  elif kernels:
203  kim = cand.getKernelImage(diffimLib.KernelCandidateF.ORIG).convertF()
204  resid = kim.Factory(kim, True)
205  im_resid.append(resid)
206 
207  # residuals using spatial model
208  ski = afwImage.ImageD(kernel.getDimensions())
209  kernel.computeImage(ski, False, int(cand.getXCenter()), int(cand.getYCenter()))
210  sk = afwMath.FixedKernel(ski)
211  sbg = 0.0
212  if background:
213  sbg = background(int(cand.getXCenter()), int(cand.getYCenter()))
214  sresid = cand.getDifferenceImage(sk, sbg)
215  resid = sresid
216  if resids:
217  resid = sresid.getImage()
218  resid /= var
219  bbox = kernel.shrinkBBox(resid.getBBox())
220  resid = resid.Factory(resid, bbox, deep=True)
221  elif kernels:
222  kim = ski.convertF()
223  resid = kim.Factory(kim, True)
224  im_resid.append(resid)
225 
226  im = im_resid.makeMosaic()
227 
228  lab = "%d chi^2 %.1f" % (cand.getId(), rchi2)
229  ctype = afwDisplay.RED if cand.isBad() else afwDisplay.GREEN
230 
231  mos.append(im, lab, ctype)
232 
233  if False and np.isnan(rchi2):
234  disp = afwDisplay.Display(frame=1)
235  disp.mtv(cand.getScienceMaskedImage.getImage(), title="candidate")
236  print("rating", cand.getCandidateRating())
237 
238  im = cand.getScienceMaskedImage()
239  center = (candidateIndex, cand.getXCenter() - im.getX0(), cand.getYCenter() - im.getY0())
240  candidateIndex += 1
241  if cand.isBad():
242  candidateCentersBad.append(center)
243  else:
244  candidateCenters.append(center)
245 
246  if resids:
247  title = "chi Diffim"
248  elif kernels:
249  title = "Kernels"
250  else:
251  title = "Candidates & residuals"
252  mosaicImage = mos.makeMosaic(frame=frame, title=title)
253 
254  return mosaicImage
255 
256 
257 def showKernelBasis(kernel, frame=None):
258  """Display a Kernel's basis images.
259  """
260  mos = afwDisplay.utils.Mosaic()
261 
262  for k in kernel.getKernelList():
263  im = afwImage.ImageD(k.getDimensions())
264  k.computeImage(im, False)
265  mos.append(im)
266  mos.makeMosaic(frame=frame, title="Kernel Basis Images")
267 
268  return mos
269 
270 
271 
272 
273 def plotKernelSpatialModel(kernel, kernelCellSet, showBadCandidates=True,
274  numSample=128, keepPlots=True, maxCoeff=10):
275  """Plot the Kernel spatial model.
276  """
277  try:
278  import matplotlib.pyplot as plt
279  import matplotlib.colors
280  except ImportError as e:
281  print("Unable to import numpy and matplotlib: %s" % e)
282  return
283 
284  x0 = kernelCellSet.getBBox().getBeginX()
285  y0 = kernelCellSet.getBBox().getBeginY()
286 
287  candPos = list()
288  candFits = list()
289  badPos = list()
290  badFits = list()
291  candAmps = list()
292  badAmps = list()
293  for cell in kernelCellSet.getCellList():
294  for cand in cell.begin(False):
295  if not showBadCandidates and cand.isBad():
296  continue
297  candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter())
298  try:
299  im = cand.getTemplateMaskedImage()
300  except Exception:
301  continue
302 
303  targetFits = badFits if cand.isBad() else candFits
304  targetPos = badPos if cand.isBad() else candPos
305  targetAmps = badAmps if cand.isBad() else candAmps
306 
307  # compare original and spatial kernel coefficients
308  kp0 = np.array(cand.getKernel(diffimLib.KernelCandidateF.ORIG).getKernelParameters())
309  amp = cand.getCandidateRating()
310 
311  targetFits = badFits if cand.isBad() else candFits
312  targetPos = badPos if cand.isBad() else candPos
313  targetAmps = badAmps if cand.isBad() else candAmps
314 
315  targetFits.append(kp0)
316  targetPos.append(candCenter)
317  targetAmps.append(amp)
318 
319  xGood = np.array([pos.getX() for pos in candPos]) - x0
320  yGood = np.array([pos.getY() for pos in candPos]) - y0
321  zGood = np.array(candFits)
322 
323  xBad = np.array([pos.getX() for pos in badPos]) - x0
324  yBad = np.array([pos.getY() for pos in badPos]) - y0
325  zBad = np.array(badFits)
326  numBad = len(badPos)
327 
328  xRange = np.linspace(0, kernelCellSet.getBBox().getWidth(), num=numSample)
329  yRange = np.linspace(0, kernelCellSet.getBBox().getHeight(), num=numSample)
330 
331  if maxCoeff:
332  maxCoeff = min(maxCoeff, kernel.getNKernelParameters())
333  else:
334  maxCoeff = kernel.getNKernelParameters()
335 
336  for k in range(maxCoeff):
337  func = kernel.getSpatialFunction(k)
338  dfGood = zGood[:, k] - np.array([func(pos.getX(), pos.getY()) for pos in candPos])
339  yMin = dfGood.min()
340  yMax = dfGood.max()
341  if numBad > 0:
342  dfBad = zBad[:, k] - np.array([func(pos.getX(), pos.getY()) for pos in badPos])
343  # Can really screw up the range...
344  yMin = min([yMin, dfBad.min()])
345  yMax = max([yMax, dfBad.max()])
346  yMin -= 0.05*(yMax - yMin)
347  yMax += 0.05*(yMax - yMin)
348 
349  fRange = np.ndarray((len(xRange), len(yRange)))
350  for j, yVal in enumerate(yRange):
351  for i, xVal in enumerate(xRange):
352  fRange[j][i] = func(xVal, yVal)
353 
354  fig = plt.figure(k)
355 
356  fig.clf()
357  try:
358  fig.canvas._tkcanvas._root().lift() # == Tk's raise, but raise is a python reserved word
359  except Exception: # protect against API changes
360  pass
361 
362  fig.suptitle('Kernel component %d' % k)
363 
364  # LL
365  ax = fig.add_axes((0.1, 0.05, 0.35, 0.35))
366  vmin = fRange.min() # - 0.05*np.fabs(fRange.min())
367  vmax = fRange.max() # + 0.05*np.fabs(fRange.max())
368  norm = matplotlib.colors.Normalize(vmin=vmin, vmax=vmax)
369  im = ax.imshow(fRange, aspect='auto', norm=norm,
370  extent=[0, kernelCellSet.getBBox().getWidth() - 1,
371  0, kernelCellSet.getBBox().getHeight() - 1])
372  ax.set_title('Spatial polynomial')
373  plt.colorbar(im, orientation='horizontal', ticks=[vmin, vmax])
374 
375  # UL
376  ax = fig.add_axes((0.1, 0.55, 0.35, 0.35))
377  ax.plot(-2.5*np.log10(candAmps), zGood[:, k], 'b+')
378  if numBad > 0:
379  ax.plot(-2.5*np.log10(badAmps), zBad[:, k], 'r+')
380  ax.set_title("Basis Coefficients")
381  ax.set_xlabel("Instr mag")
382  ax.set_ylabel("Coeff")
383 
384  # LR
385  ax = fig.add_axes((0.55, 0.05, 0.35, 0.35))
386  ax.set_autoscale_on(False)
387  ax.set_xbound(lower=0, upper=kernelCellSet.getBBox().getHeight())
388  ax.set_ybound(lower=yMin, upper=yMax)
389  ax.plot(yGood, dfGood, 'b+')
390  if numBad > 0:
391  ax.plot(yBad, dfBad, 'r+')
392  ax.axhline(0.0)
393  ax.set_title('dCoeff (indiv-spatial) vs. y')
394 
395  # UR
396  ax = fig.add_axes((0.55, 0.55, 0.35, 0.35))
397  ax.set_autoscale_on(False)
398  ax.set_xbound(lower=0, upper=kernelCellSet.getBBox().getWidth())
399  ax.set_ybound(lower=yMin, upper=yMax)
400  ax.plot(xGood, dfGood, 'b+')
401  if numBad > 0:
402  ax.plot(xBad, dfBad, 'r+')
403  ax.axhline(0.0)
404  ax.set_title('dCoeff (indiv-spatial) vs. x')
405 
406  fig.show()
407 
408  global keptPlots
409  if keepPlots and not keptPlots:
410  # Keep plots open when done
411  def show():
412  print("%s: Please close plots when done." % __name__)
413  try:
414  plt.show()
415  except Exception:
416  pass
417  print("Plots closed, exiting...")
418  import atexit
419  atexit.register(show)
420  keptPlots = True
421 
422 
423 def plotKernelCoefficients(spatialKernel, kernelCellSet, showBadCandidates=False, keepPlots=True):
424  """Plot the individual kernel candidate and the spatial kernel solution coefficients.
425 
426  Parameters
427  ----------
428 
429  spatialKernel : `lsst.afw.math.LinearCombinationKernel`
430  The spatial spatialKernel solution model which is a spatially varying linear combination
431  of the spatialKernel basis functions.
432  Typically returned by `lsst.ip.diffim.SpatialKernelSolution.getSolutionPair()`.
433 
434  kernelCellSet : `lsst.afw.math.SpatialCellSet`
435  The spatial cells that was used for solution for the spatialKernel. They contain the
436  local solutions of the AL kernel for the selected sources.
437 
438  showBadCandidates : `bool`, optional
439  If True, plot the coefficient values for kernel candidates where the solution was marked
440  bad by the numerical algorithm. Defaults to False.
441 
442  keepPlots: `bool`, optional
443  If True, sets ``plt.show()`` to be called before the task terminates, so that the plots
444  can be explored interactively. Defaults to True.
445 
446  Notes
447  -----
448  This function produces 3 figures per image subtraction operation.
449  * A grid plot of the local solutions. Each grid cell corresponds to a proportional area in
450  the image. In each cell, local kernel solution coefficients are plotted of kernel candidates (color)
451  that fall into this area as a function of the kernel basis function number.
452  * A grid plot of the spatial solution. Each grid cell corresponds to a proportional area in
453  the image. In each cell, the spatial solution coefficients are evaluated for the center of the cell.
454  * Histogram of the local solution coefficients. Red line marks the spatial solution value at
455  center of the image.
456 
457  This function is called if ``lsst.ip.diffim.psfMatch.plotKernelCoefficients==True`` in lsstDebug. This
458  function was implemented as part of DM-17825.
459  """
460  try:
461  import matplotlib.pyplot as plt
462  except ImportError as e:
463  print("Unable to import matplotlib: %s" % e)
464  return
465 
466  # Image dimensions
467  imgBBox = kernelCellSet.getBBox()
468  x0 = imgBBox.getBeginX()
469  y0 = imgBBox.getBeginY()
470  wImage = imgBBox.getWidth()
471  hImage = imgBBox.getHeight()
472  imgCenterX = imgBBox.getCenterX()
473  imgCenterY = imgBBox.getCenterY()
474 
475  # Plot the local solutions
476  # ----
477 
478  # Grid size
479  nX = 8
480  nY = 8
481  wCell = wImage / nX
482  hCell = hImage / nY
483 
484  fig = plt.figure()
485  fig.suptitle("Kernel candidate parameters on an image grid")
486  arrAx = fig.subplots(nrows=nY, ncols=nX, sharex=True, sharey=True, gridspec_kw=dict(
487  wspace=0, hspace=0))
488 
489  # Bottom left panel is for bottom left part of the image
490  arrAx = arrAx[::-1, :]
491 
492  allParams = []
493  for cell in kernelCellSet.getCellList():
494  cellBBox = afwGeom.Box2D(cell.getBBox())
495  # Determine which panel this spatial cell belongs to
496  iX = int((cellBBox.getCenterX() - x0)//wCell)
497  iY = int((cellBBox.getCenterY() - y0)//hCell)
498 
499  for cand in cell.begin(False):
500  try:
501  kernel = cand.getKernel(cand.ORIG)
502  except Exception:
503  continue
504 
505  if not showBadCandidates and cand.isBad():
506  continue
507 
508  nKernelParams = kernel.getNKernelParameters()
509  kernelParams = np.array(kernel.getKernelParameters())
510  allParams.append(kernelParams)
511 
512  if cand.isBad():
513  color = 'red'
514  else:
515  color = None
516  arrAx[iY, iX].plot(np.arange(nKernelParams), kernelParams, '.-',
517  color=color, drawstyle='steps-mid', linewidth=0.1)
518  for ax in arrAx.ravel():
519  ax.grid(True, axis='y')
520 
521  # Plot histogram of the local parameters and the global solution at the image center
522  # ----
523 
524  spatialFuncs = spatialKernel.getSpatialFunctionList()
525  nKernelParams = spatialKernel.getNKernelParameters()
526  nX = 8
527  fig = plt.figure()
528  fig.suptitle("Hist. of parameters marked with spatial solution at img center")
529  arrAx = fig.subplots(nrows=int(nKernelParams//nX)+1, ncols=nX)
530  arrAx = arrAx[::-1, :]
531  allParams = np.array(allParams)
532  for k in range(nKernelParams):
533  ax = arrAx.ravel()[k]
534  ax.hist(allParams[:, k], bins=20, edgecolor='black')
535  ax.set_xlabel('P{}'.format(k))
536  valueParam = spatialFuncs[k](imgCenterX, imgCenterY)
537  ax.axvline(x=valueParam, color='red')
538  ax.text(0.1, 0.9, '{:.1f}'.format(valueParam),
539  transform=ax.transAxes, backgroundcolor='lightsteelblue')
540 
541  # Plot grid of the spatial solution
542  # ----
543 
544  nX = 8
545  nY = 8
546  wCell = wImage / nX
547  hCell = hImage / nY
548  x0 += wCell / 2
549  y0 += hCell / 2
550 
551  fig = plt.figure()
552  fig.suptitle("Spatial solution of kernel parameters on an image grid")
553  arrAx = fig.subplots(nrows=nY, ncols=nX, sharex=True, sharey=True, gridspec_kw=dict(
554  wspace=0, hspace=0))
555  arrAx = arrAx[::-1, :]
556  kernelParams = np.zeros(nKernelParams, dtype=float)
557 
558  for iX in range(nX):
559  for iY in range(nY):
560  x = x0 + iX * wCell
561  y = y0 + iY * hCell
562  # Evaluate the spatial solution functions for this x,y location
563  kernelParams = [f(x, y) for f in spatialFuncs]
564  arrAx[iY, iX].plot(np.arange(nKernelParams), kernelParams, '.-', drawstyle='steps-mid')
565  arrAx[iY, iX].grid(True, axis='y')
566 
567  global keptPlots
568  if keepPlots and not keptPlots:
569  # Keep plots open when done
570  def show():
571  print("%s: Please close plots when done." % __name__)
572  try:
573  plt.show()
574  except Exception:
575  pass
576  print("Plots closed, exiting...")
577  import atexit
578  atexit.register(show)
579  keptPlots = True
580 
581 
582 def showKernelMosaic(bbox, kernel, nx=7, ny=None, frame=None, title=None,
583  showCenter=True, showEllipticity=True):
584  """Show a mosaic of Kernel images.
585  """
586  mos = afwDisplay.utils.Mosaic()
587 
588  x0 = bbox.getBeginX()
589  y0 = bbox.getBeginY()
590  width = bbox.getWidth()
591  height = bbox.getHeight()
592 
593  if not ny:
594  ny = int(nx*float(height)/width + 0.5)
595  if not ny:
596  ny = 1
597 
598  schema = afwTable.SourceTable.makeMinimalSchema()
599  centroidName = "base_SdssCentroid"
600  shapeName = "base_SdssShape"
601  control = measBase.SdssCentroidControl()
602  schema.getAliasMap().set("slot_Centroid", centroidName)
603  schema.getAliasMap().set("slot_Centroid_flag", centroidName + "_flag")
604  centroider = measBase.SdssCentroidAlgorithm(control, centroidName, schema)
605  sdssShape = measBase.SdssShapeControl()
606  shaper = measBase.SdssShapeAlgorithm(sdssShape, shapeName, schema)
607  table = afwTable.SourceTable.make(schema)
608  table.defineCentroid(centroidName)
609  table.defineShape(shapeName)
610 
611  centers = []
612  shapes = []
613  for iy in range(ny):
614  for ix in range(nx):
615  x = int(ix*(width - 1)/(nx - 1)) + x0
616  y = int(iy*(height - 1)/(ny - 1)) + y0
617 
618  im = afwImage.ImageD(kernel.getDimensions())
619  ksum = kernel.computeImage(im, False, x, y)
620  lab = "Kernel(%d,%d)=%.2f" % (x, y, ksum) if False else ""
621  mos.append(im, lab)
622 
623  # SdssCentroidAlgorithm.measure requires an exposure of floats
624  exp = afwImage.makeExposure(afwImage.makeMaskedImage(im.convertF()))
625 
626  w, h = im.getWidth(), im.getHeight()
627  centerX = im.getX0() + w//2
628  centerY = im.getY0() + h//2
629  src = table.makeRecord()
630  spans = afwGeom.SpanSet(exp.getBBox())
631  foot = afwDet.Footprint(spans)
632  foot.addPeak(centerX, centerY, 1)
633  src.setFootprint(foot)
634 
635  try: # The centroider requires a psf, so this will fail if none is attached to exp
636  centroider.measure(src, exp)
637  centers.append((src.getX(), src.getY()))
638 
639  shaper.measure(src, exp)
640  shapes.append((src.getIxx(), src.getIxy(), src.getIyy()))
641  except Exception:
642  pass
643 
644  mos.makeMosaic(frame=frame, title=title if title else "Model Kernel", mode=nx)
645 
646  if centers and frame is not None:
647  disp = afwDisplay.Display(frame=frame)
648  i = 0
649  with disp.Buffering():
650  for cen, shape in zip(centers, shapes):
651  bbox = mos.getBBox(i)
652  i += 1
653  xc, yc = cen[0] + bbox.getMinX(), cen[1] + bbox.getMinY()
654  if showCenter:
655  disp.dot("+", xc, yc, ctype=afwDisplay.BLUE)
656 
657  if showEllipticity:
658  ixx, ixy, iyy = shape
659  disp.dot("@:%g,%g,%g" % (ixx, ixy, iyy), xc, yc, ctype=afwDisplay.RED)
660 
661  return mos
662 
663 
664 def plotPixelResiduals(exposure, warpedTemplateExposure, diffExposure, kernelCellSet,
665  kernel, background, testSources, config,
666  origVariance=False, nptsFull=1e6, keepPlots=True, titleFs=14):
667  """Plot diffim residuals for LOCAL and SPATIAL models.
668  """
669  candidateResids = []
670  spatialResids = []
671  nonfitResids = []
672 
673  for cell in kernelCellSet.getCellList():
674  for cand in cell.begin(True): # only look at good ones
675  # Be sure
676  if not (cand.getStatus() == afwMath.SpatialCellCandidate.GOOD):
677  continue
678 
679  diffim = cand.getDifferenceImage(diffimLib.KernelCandidateF.ORIG)
680  orig = cand.getScienceMaskedImage()
681 
682  ski = afwImage.ImageD(kernel.getDimensions())
683  kernel.computeImage(ski, False, int(cand.getXCenter()), int(cand.getYCenter()))
684  sk = afwMath.FixedKernel(ski)
685  sbg = background(int(cand.getXCenter()), int(cand.getYCenter()))
686  sdiffim = cand.getDifferenceImage(sk, sbg)
687 
688  # trim edgs due to convolution
689  bbox = kernel.shrinkBBox(diffim.getBBox())
690  tdiffim = diffim.Factory(diffim, bbox)
691  torig = orig.Factory(orig, bbox)
692  tsdiffim = sdiffim.Factory(sdiffim, bbox)
693 
694  if origVariance:
695  candidateResids.append(np.ravel(tdiffim.getImage().getArray() /
696  np.sqrt(torig.getVariance().getArray())))
697  spatialResids.append(np.ravel(tsdiffim.getImage().getArray() /
698  np.sqrt(torig.getVariance().getArray())))
699  else:
700  candidateResids.append(np.ravel(tdiffim.getImage().getArray() /
701  np.sqrt(tdiffim.getVariance().getArray())))
702  spatialResids.append(np.ravel(tsdiffim.getImage().getArray() /
703  np.sqrt(tsdiffim.getVariance().getArray())))
704 
705  fullIm = diffExposure.getMaskedImage().getImage().getArray()
706  fullMask = diffExposure.getMaskedImage().getMask().getArray()
707  if origVariance:
708  fullVar = exposure.getMaskedImage().getVariance().getArray()
709  else:
710  fullVar = diffExposure.getMaskedImage().getVariance().getArray()
711 
712  bitmaskBad = 0
713  bitmaskBad |= afwImage.Mask.getPlaneBitMask('NO_DATA')
714  bitmaskBad |= afwImage.Mask.getPlaneBitMask('SAT')
715  idx = np.where((fullMask & bitmaskBad) == 0)
716  stride = int(len(idx[0])//nptsFull)
717  sidx = idx[0][::stride], idx[1][::stride]
718  allResids = fullIm[sidx]/np.sqrt(fullVar[sidx])
719 
720  testFootprints = diffimTools.sourceToFootprintList(testSources, warpedTemplateExposure,
721  exposure, config, Log.getDefaultLogger())
722  for fp in testFootprints:
723  subexp = diffExposure.Factory(diffExposure, fp["footprint"].getBBox())
724  subim = subexp.getMaskedImage().getImage()
725  if origVariance:
726  subvar = afwImage.ExposureF(exposure, fp["footprint"].getBBox()).getMaskedImage().getVariance()
727  else:
728  subvar = subexp.getMaskedImage().getVariance()
729  nonfitResids.append(np.ravel(subim.getArray()/np.sqrt(subvar.getArray())))
730 
731  candidateResids = np.ravel(np.array(candidateResids))
732  spatialResids = np.ravel(np.array(spatialResids))
733  nonfitResids = np.ravel(np.array(nonfitResids))
734 
735  try:
736  import pylab
737  from matplotlib.font_manager import FontProperties
738  except ImportError as e:
739  print("Unable to import pylab: %s" % e)
740  return
741 
742  fig = pylab.figure()
743  fig.clf()
744  try:
745  fig.canvas._tkcanvas._root().lift() # == Tk's raise, but raise is a python reserved word
746  except Exception: # protect against API changes
747  pass
748  if origVariance:
749  fig.suptitle("Diffim residuals: Normalized by sqrt(input variance)", fontsize=titleFs)
750  else:
751  fig.suptitle("Diffim residuals: Normalized by sqrt(diffim variance)", fontsize=titleFs)
752 
753  sp1 = pylab.subplot(221)
754  sp2 = pylab.subplot(222, sharex=sp1, sharey=sp1)
755  sp3 = pylab.subplot(223, sharex=sp1, sharey=sp1)
756  sp4 = pylab.subplot(224, sharex=sp1, sharey=sp1)
757  xs = np.arange(-5, 5.05, 0.1)
758  ys = 1./np.sqrt(2*np.pi)*np.exp(-0.5*xs**2)
759 
760  sp1.hist(candidateResids, bins=xs, normed=True, alpha=0.5, label="N(%.2f, %.2f)"
761  % (np.mean(candidateResids), np.var(candidateResids)))
762  sp1.plot(xs, ys, "r-", lw=2, label="N(0,1)")
763  sp1.set_title("Candidates: basis fit", fontsize=titleFs - 2)
764  sp1.legend(loc=1, fancybox=True, shadow=True, prop=FontProperties(size=titleFs - 6))
765 
766  sp2.hist(spatialResids, bins=xs, normed=True, alpha=0.5, label="N(%.2f, %.2f)"
767  % (np.mean(spatialResids), np.var(spatialResids)))
768  sp2.plot(xs, ys, "r-", lw=2, label="N(0,1)")
769  sp2.set_title("Candidates: spatial fit", fontsize=titleFs - 2)
770  sp2.legend(loc=1, fancybox=True, shadow=True, prop=FontProperties(size=titleFs - 6))
771 
772  sp3.hist(nonfitResids, bins=xs, normed=True, alpha=0.5, label="N(%.2f, %.2f)"
773  % (np.mean(nonfitResids), np.var(nonfitResids)))
774  sp3.plot(xs, ys, "r-", lw=2, label="N(0,1)")
775  sp3.set_title("Control sample: spatial fit", fontsize=titleFs - 2)
776  sp3.legend(loc=1, fancybox=True, shadow=True, prop=FontProperties(size=titleFs - 6))
777 
778  sp4.hist(allResids, bins=xs, normed=True, alpha=0.5, label="N(%.2f, %.2f)"
779  % (np.mean(allResids), np.var(allResids)))
780  sp4.plot(xs, ys, "r-", lw=2, label="N(0,1)")
781  sp4.set_title("Full image (subsampled)", fontsize=titleFs - 2)
782  sp4.legend(loc=1, fancybox=True, shadow=True, prop=FontProperties(size=titleFs - 6))
783 
784  pylab.setp(sp1.get_xticklabels() + sp1.get_yticklabels(), fontsize=titleFs - 4)
785  pylab.setp(sp2.get_xticklabels() + sp2.get_yticklabels(), fontsize=titleFs - 4)
786  pylab.setp(sp3.get_xticklabels() + sp3.get_yticklabels(), fontsize=titleFs - 4)
787  pylab.setp(sp4.get_xticklabels() + sp4.get_yticklabels(), fontsize=titleFs - 4)
788 
789  sp1.set_xlim(-5, 5)
790  sp1.set_ylim(0, 0.5)
791  fig.show()
792 
793  global keptPlots
794  if keepPlots and not keptPlots:
795  # Keep plots open when done
796  def show():
797  print("%s: Please close plots when done." % __name__)
798  try:
799  pylab.show()
800  except Exception:
801  pass
802  print("Plots closed, exiting...")
803  import atexit
804  atexit.register(show)
805  keptPlots = True
806 
807 
808 def calcCentroid(arr):
809  """Calculate first moment of a (kernel) image.
810  """
811  y, x = arr.shape
812  sarr = arr*arr
813  xarr = np.asarray([[el for el in range(x)] for el2 in range(y)])
814  yarr = np.asarray([[el2 for el in range(x)] for el2 in range(y)])
815  narr = xarr*sarr
816  sarrSum = sarr.sum()
817  centx = narr.sum()/sarrSum
818  narr = yarr*sarr
819  centy = narr.sum()/sarrSum
820  return centx, centy
821 
822 
823 def calcWidth(arr, centx, centy):
824  """Calculate second moment of a (kernel) image.
825  """
826  y, x = arr.shape
827  # Square the flux so we don't have to deal with negatives
828  sarr = arr*arr
829  xarr = np.asarray([[el for el in range(x)] for el2 in range(y)])
830  yarr = np.asarray([[el2 for el in range(x)] for el2 in range(y)])
831  narr = sarr*np.power((xarr - centx), 2.)
832  sarrSum = sarr.sum()
833  xstd = np.sqrt(narr.sum()/sarrSum)
834  narr = sarr*np.power((yarr - centy), 2.)
835  ystd = np.sqrt(narr.sum()/sarrSum)
836  return xstd, ystd
837 
838 
839 def printSkyDiffs(sources, wcs):
840  """Print differences in sky coordinates.
841 
842  The difference is that between the source Position and its Centroid mapped
843  through Wcs.
844  """
845  for s in sources:
846  sCentroid = s.getCentroid()
847  sPosition = s.getCoord().getPosition(afwGeom.degrees)
848  dra = 3600*(sPosition.getX() - wcs.pixelToSky(sCentroid).getPosition(afwGeom.degrees).getX())/0.2
849  ddec = 3600*(sPosition.getY() - wcs.pixelToSky(sCentroid).getPosition(afwGeom.degrees).getY())/0.2
850  if np.isfinite(dra) and np.isfinite(ddec):
851  print(dra, ddec)
852 
853 
854 def makeRegions(sources, outfilename, wcs=None):
855  """Create regions file for display from input source list.
856  """
857  fh = open(outfilename, "w")
858  fh.write("global color=red font=\"helvetica 10 normal\" "
859  "select=1 highlite=1 edit=1 move=1 delete=1 include=1 fixed=0 source\nfk5\n")
860  for s in sources:
861  if wcs:
862  (ra, dec) = wcs.pixelToSky(s.getCentroid()).getPosition(afwGeom.degrees)
863  else:
864  (ra, dec) = s.getCoord().getPosition(afwGeom.degrees)
865  if np.isfinite(ra) and np.isfinite(dec):
866  fh.write("circle(%f,%f,2\")\n"%(ra, dec))
867  fh.flush()
868  fh.close()
869 
870 
871 def showSourceSetSky(sSet, wcs, xy0, frame=0, ctype=afwDisplay.GREEN, symb="+", size=2):
872  """Draw the (RA, Dec) positions of a set of Sources. Image has the XY0.
873  """
874  disp = afwDisplay.Display(frame=frame)
875  with disp.Buffering():
876  for s in sSet:
877  (xc, yc) = wcs.skyToPixel(s.getCoord().getRa(), s.getCoord().getDec())
878  xc -= xy0[0]
879  yc -= xy0[1]
880  disp.dot(symb, xc, yc, ctype=ctype, size=size)
881 
882 
883 def plotWhisker(results, newWcs):
884  """Plot whisker diagram of astromeric offsets between results.matches.
885  """
886  refCoordKey = results.matches[0].first.getTable().getCoordKey()
887  inCentroidKey = results.matches[0].second.getTable().getCentroidKey()
888  positions = [m.first.get(refCoordKey) for m in results.matches]
889  residuals = [m.first.get(refCoordKey).getOffsetFrom(
890  newWcs.pixelToSky(m.second.get(inCentroidKey))) for
891  m in results.matches]
892  import matplotlib.pyplot as plt
893  fig = plt.figure()
894  sp = fig.add_subplot(1, 1, 0)
895  xpos = [x[0].asDegrees() for x in positions]
896  ypos = [x[1].asDegrees() for x in positions]
897  xpos.append(0.02*(max(xpos) - min(xpos)) + min(xpos))
898  ypos.append(0.98*(max(ypos) - min(ypos)) + min(ypos))
899  xidxs = np.isfinite(xpos)
900  yidxs = np.isfinite(ypos)
901  X = np.asarray(xpos)[xidxs]
902  Y = np.asarray(ypos)[yidxs]
903  distance = [x[1].asArcseconds() for x in residuals]
904  distance.append(0.2)
905  distance = np.asarray(distance)[xidxs]
906  # NOTE: This assumes that the bearing is measured positive from +RA through North.
907  # From the documentation this is not clear.
908  bearing = [x[0].asRadians() for x in residuals]
909  bearing.append(0)
910  bearing = np.asarray(bearing)[xidxs]
911  U = (distance*np.cos(bearing))
912  V = (distance*np.sin(bearing))
913  sp.quiver(X, Y, U, V)
914  sp.set_title("WCS Residual")
915  plt.show()
916 
917 
918 class DipoleTestImage(object):
919  """Utility class for dipole measurement testing.
920 
921  Generate an image with simulated dipoles and noise; store the original
922  "pre-subtraction" images and catalogs as well.
923  Used to generate test data for DMTN-007 (http://dmtn-007.lsst.io).
924  """
925 
926  def __init__(self, w=101, h=101, xcenPos=[27.], ycenPos=[25.], xcenNeg=[23.], ycenNeg=[25.],
927  psfSigma=2., flux=[30000.], fluxNeg=None, noise=10., gradientParams=None):
928  self.w = w
929  self.h = h
930  self.xcenPos = xcenPos
931  self.ycenPos = ycenPos
932  self.xcenNeg = xcenNeg
933  self.ycenNeg = ycenNeg
934  self.psfSigma = psfSigma
935  self.flux = flux
936  self.fluxNeg = fluxNeg
937  if fluxNeg is None:
938  self.fluxNeg = self.flux
939  self.noise = noise
940  self.gradientParams = gradientParams
941  self._makeDipoleImage()
942 
943  def _makeDipoleImage(self):
944  """Generate an exposure and catalog with the given dipole source(s).
945  """
946  # Must seed the pos/neg images with different values to ensure they get different noise realizations
947  posImage, posCatalog = self._makeStarImage(
948  xc=self.xcenPos, yc=self.ycenPos, flux=self.flux, randomSeed=111)
949 
950  negImage, negCatalog = self._makeStarImage(
951  xc=self.xcenNeg, yc=self.ycenNeg, flux=self.fluxNeg, randomSeed=222)
952 
953  dipole = posImage.clone()
954  di = dipole.getMaskedImage()
955  di -= negImage.getMaskedImage()
956 
957  # Carry through pos/neg detection masks to new planes in diffim
958  dm = di.getMask()
959  posDetectedBits = posImage.getMaskedImage().getMask().getArray() == dm.getPlaneBitMask("DETECTED")
960  negDetectedBits = negImage.getMaskedImage().getMask().getArray() == dm.getPlaneBitMask("DETECTED")
961  pos_det = dm.addMaskPlane("DETECTED_POS") # new mask plane -- different from "DETECTED"
962  neg_det = dm.addMaskPlane("DETECTED_NEG") # new mask plane -- different from "DETECTED_NEGATIVE"
963  dma = dm.getArray()
964  # set the two custom mask planes to these new masks
965  dma[:, :] = posDetectedBits*pos_det + negDetectedBits*neg_det
966  self.diffim, self.posImage, self.posCatalog, self.negImage, self.negCatalog \
967  = dipole, posImage, posCatalog, negImage, negCatalog
968 
969  def _makeStarImage(self, xc=[15.3], yc=[18.6], flux=[2500], schema=None, randomSeed=None):
970  """Generate an exposure and catalog with the given stellar source(s).
971  """
972  from lsst.meas.base.tests import TestDataset
973  bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Point2I(self.w - 1, self.h - 1))
974  dataset = TestDataset(bbox, psfSigma=self.psfSigma, threshold=1.)
975 
976  for i in range(len(xc)):
977  dataset.addSource(instFlux=flux[i], centroid=afwGeom.Point2D(xc[i], yc[i]))
978 
979  if schema is None:
980  schema = TestDataset.makeMinimalSchema()
981  exposure, catalog = dataset.realize(noise=self.noise, schema=schema, randomSeed=randomSeed)
982 
983  if self.gradientParams is not None:
984  y, x = np.mgrid[:self.w, :self.h]
985  gp = self.gradientParams
986  gradient = gp[0] + gp[1]*x + gp[2]*y
987  if len(self.gradientParams) > 3: # it includes a set of 2nd-order polynomial params
988  gradient += gp[3]*x*y + gp[4]*x*x + gp[5]*y*y
989  imgArr = exposure.getMaskedImage().getArrays()[0]
990  imgArr += gradient
991 
992  return exposure, catalog
993 
994  def fitDipoleSource(self, source, **kwds):
995  alg = DipoleFitAlgorithm(self.diffim, self.posImage, self.negImage)
996  fitResult = alg.fitDipole(source, **kwds)
997  return fitResult
998 
999  def detectDipoleSources(self, doMerge=True, diffim=None, detectSigma=5.5, grow=3, minBinSize=32):
1000  """Utility function for detecting dipoles.
1001 
1002  Detect pos/neg sources in the diffim, then merge them. A
1003  bigger "grow" parameter leads to a larger footprint which
1004  helps with dipole measurement for faint dipoles.
1005 
1006  Parameters
1007  ----------
1008  doMerge : `bool`
1009  Whether to merge the positive and negagive detections into a single
1010  source table.
1011  diffim : `lsst.afw.image.exposure.exposure.ExposureF`
1012  Difference image on which to perform detection.
1013  detectSigma : `float`
1014  Threshold for object detection.
1015  grow : `int`
1016  Number of pixels to grow the footprints before merging.
1017  minBinSize : `int`
1018  Minimum bin size for the background (re)estimation (only applies if
1019  the default leads to min(nBinX, nBinY) < fit order so the default
1020  config parameter needs to be decreased, but not to a value smaller
1021  than ``minBinSize``, in which case the fitting algorithm will take
1022  over and decrease the fit order appropriately.)
1023 
1024  Returns
1025  -------
1026  sources : `lsst.afw.table.SourceCatalog`
1027  If doMerge=True, the merged source catalog is returned OR
1028  detectTask : `lsst.meas.algorithms.SourceDetectionTask`
1029  schema : `lsst.afw.table.Schema`
1030  If doMerge=False, the source detection task and its schema are
1031  returned.
1032  """
1033  if diffim is None:
1034  diffim = self.diffim
1035 
1036  # Start with a minimal schema - only the fields all SourceCatalogs need
1037  schema = afwTable.SourceTable.makeMinimalSchema()
1038 
1039  # Customize the detection task a bit (optional)
1040  detectConfig = measAlg.SourceDetectionConfig()
1041  detectConfig.returnOriginalFootprints = False # should be the default
1042 
1043  psfSigma = diffim.getPsf().computeShape().getDeterminantRadius()
1044 
1045  # code from imageDifference.py:
1046  detectConfig.thresholdPolarity = "both"
1047  detectConfig.thresholdValue = detectSigma
1048  # detectConfig.nSigmaToGrow = psfSigma
1049  detectConfig.reEstimateBackground = True # if False, will fail often for faint sources on gradients?
1050  detectConfig.thresholdType = "pixel_stdev"
1051  # Test images are often quite small, so may need to adjust background binSize
1052  while ((min(diffim.getWidth(), diffim.getHeight()))//detectConfig.background.binSize <
1053  detectConfig.background.approxOrderX and detectConfig.background.binSize > minBinSize):
1054  detectConfig.background.binSize = max(minBinSize, detectConfig.background.binSize//2)
1055 
1056  # Create the detection task. We pass the schema so the task can declare a few flag fields
1057  detectTask = measAlg.SourceDetectionTask(schema, config=detectConfig)
1058 
1059  table = afwTable.SourceTable.make(schema)
1060  catalog = detectTask.makeSourceCatalog(table, diffim, sigma=psfSigma)
1061 
1062  # Now do the merge.
1063  if doMerge:
1064  fpSet = catalog.fpSets.positive
1065  fpSet.merge(catalog.fpSets.negative, grow, grow, False)
1066  sources = afwTable.SourceCatalog(table)
1067  fpSet.makeSources(sources)
1068 
1069  return sources
1070 
1071  else:
1072  return detectTask, schema
def showDiaSources(sources, exposure, isFlagged, isDipole, frame=None)
Definition: utils.py:105
def plotKernelCoefficients(spatialKernel, kernelCellSet, showBadCandidates=False, keepPlots=True)
Definition: utils.py:423
def makeRegions(sources, outfilename, wcs=None)
Definition: utils.py:854
def __init__(self, w=101, h=101, xcenPos=[27.], ycenPos=[25.], xcenNeg=[23.], ycenNeg=[25.], psfSigma=2., flux=[30000.], fluxNeg=None, noise=10., gradientParams=None)
Definition: utils.py:927
MaskedImage< ImagePixelT, MaskPixelT, VariancePixelT > * makeMaskedImage(typename std::shared_ptr< Image< ImagePixelT >> image, typename std::shared_ptr< Mask< MaskPixelT >> mask=Mask< MaskPixelT >(), typename std::shared_ptr< Image< VariancePixelT >> variance=Image< VariancePixelT >())
def showKernelSpatialCells(maskedIm, kernelCellSet, showChi2=False, symb="o", ctype=None, ctypeUnused=None, ctypeBad=None, size=3, frame=None, title="Spatial Cells")
Definition: utils.py:68
std::shared_ptr< Exposure< ImagePixelT, MaskPixelT, VariancePixelT > > makeExposure(MaskedImage< ImagePixelT, MaskPixelT, VariancePixelT > &mimage, std::shared_ptr< geom::SkyWcs const > wcs=std::shared_ptr< geom::SkyWcs const >())
def _makeStarImage(self, xc=[15.3], yc=[18.6], flux=[2500], schema=None, randomSeed=None)
Definition: utils.py:969
def plotKernelSpatialModel(kernel, kernelCellSet, showBadCandidates=True, numSample=128, keepPlots=True, maxCoeff=10)
Definition: utils.py:274
def plotPixelResiduals(exposure, warpedTemplateExposure, diffExposure, kernelCellSet, kernel, background, testSources, config, origVariance=False, nptsFull=1e6, keepPlots=True, titleFs=14)
Definition: utils.py:666
def printSkyDiffs(sources, wcs)
Definition: utils.py:839
def showKernelCandidates(kernelCellSet, kernel, background, frame=None, showBadCandidates=True, resids=False, kernels=False)
Definition: utils.py:142
def detectDipoleSources(self, doMerge=True, diffim=None, detectSigma=5.5, grow=3, minBinSize=32)
Definition: utils.py:999
def calcCentroid(arr)
Definition: utils.py:808
def calcWidth(arr, centx, centy)
Definition: utils.py:823
def showSourceSet(sSet, xy0=(0, 0), frame=0, ctype=afwDisplay.GREEN, symb="+", size=2)
Definition: utils.py:46
def plotWhisker(results, newWcs)
Definition: utils.py:883
def showKernelMosaic(bbox, kernel, nx=7, ny=None, frame=None, title=None, showCenter=True, showEllipticity=True)
Definition: utils.py:583
def showKernelBasis(kernel, frame=None)
Definition: utils.py:257
def fitDipoleSource(self, source, kwds)
Definition: utils.py:994
def showSourceSetSky(sSet, wcs, xy0, frame=0, ctype=afwDisplay.GREEN, symb="+", size=2)
Definition: utils.py:871