lsst.ip.diffim  18.1.0-9-gae7190a+1
utils.py
Go to the documentation of this file.
1 # This file is part of ip_diffim.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 """Support utilities for Measuring sources"""
23 
24 # Export DipoleTestImage to expose fake image generating funcs
25 __all__ = ["DipoleTestImage"]
26 
27 import numpy as np
28 
29 import lsst.geom as geom
30 import lsst.afw.detection as afwDet
31 import lsst.afw.display as afwDisplay
32 import lsst.afw.geom as afwGeom
33 import lsst.afw.image as afwImage
34 import lsst.afw.math as afwMath
35 import lsst.afw.table as afwTable
36 from lsst.log import Log
37 import lsst.meas.algorithms as measAlg
38 import lsst.meas.base as measBase
39 from .dipoleFitTask import DipoleFitAlgorithm
40 from . import diffimLib
41 from . import diffimTools
42 
43 afwDisplay.setDefaultMaskTransparency(75)
44 keptPlots = False # Have we arranged to keep spatial plots open?
45 
46 
47 def showSourceSet(sSet, xy0=(0, 0), frame=0, ctype=afwDisplay.GREEN, symb="+", size=2):
48  """Draw the (XAstrom, YAstrom) positions of a set of Sources.
49 
50  Image has the given XY0.
51  """
52  disp = afwDisplay.afwDisplay(frame=frame)
53  with disp.Buffering():
54  for s in sSet:
55  xc, yc = s.getXAstrom() - xy0[0], s.getYAstrom() - xy0[1]
56 
57  if symb == "id":
58  disp.dot(str(s.getId()), xc, yc, ctype=ctype, size=size)
59  else:
60  disp.dot(symb, xc, yc, ctype=ctype, size=size)
61 
62 
63 # Kernel display utilities
64 #
65 
66 
67 def showKernelSpatialCells(maskedIm, kernelCellSet, showChi2=False, symb="o",
68  ctype=None, ctypeUnused=None, ctypeBad=None, size=3,
69  frame=None, title="Spatial Cells"):
70  """Show the SpatialCells.
71 
72  If symb is something that display.dot understands (e.g. "o"), the top
73  nMaxPerCell candidates will be indicated with that symbol, using ctype
74  and size.
75  """
76  disp = afwDisplay.Display(frame=frame)
77  disp.mtv(maskedIm, title=title)
78  with disp.Buffering():
79  origin = [-maskedIm.getX0(), -maskedIm.getY0()]
80  for cell in kernelCellSet.getCellList():
81  afwDisplay.utils.drawBBox(cell.getBBox(), origin=origin, display=disp)
82 
83  goodies = ctypeBad is None
84  for cand in cell.begin(goodies):
85  xc, yc = cand.getXCenter() + origin[0], cand.getYCenter() + origin[1]
86  if cand.getStatus() == afwMath.SpatialCellCandidate.BAD:
87  color = ctypeBad
88  elif cand.getStatus() == afwMath.SpatialCellCandidate.GOOD:
89  color = ctype
90  elif cand.getStatus() == afwMath.SpatialCellCandidate.UNKNOWN:
91  color = ctypeUnused
92  else:
93  continue
94 
95  if color:
96  disp.dot(symb, xc, yc, ctype=color, size=size)
97 
98  if showChi2:
99  rchi2 = cand.getChi2()
100  if rchi2 > 1e100:
101  rchi2 = np.nan
102  disp.dot("%d %.1f" % (cand.getId(), rchi2),
103  xc - size, yc - size - 4, ctype=color, size=size)
104 
105 
106 def showDiaSources(sources, exposure, isFlagged, isDipole, frame=None):
107  """Display Dia Sources.
108  """
109  #
110  # Show us the ccandidates
111  #
112  # Too many mask planes in diffims
113  disp = afwDisplay.Display(frame=frame)
114  for plane in ("BAD", "CR", "EDGE", "INTERPOlATED", "INTRP", "SAT", "SATURATED"):
115  disp.setMaskPlaneColor(plane, color="ignore")
116 
117  mos = afwDisplay.utils.Mosaic()
118  for i in range(len(sources)):
119  source = sources[i]
120  badFlag = isFlagged[i]
121  dipoleFlag = isDipole[i]
122  bbox = source.getFootprint().getBBox()
123  stamp = exposure.Factory(exposure, bbox, True)
124  im = afwDisplay.utils.Mosaic(gutter=1, background=0, mode="x")
125  im.append(stamp.getMaskedImage())
126  lab = "%.1f,%.1f:" % (source.getX(), source.getY())
127  if badFlag:
128  ctype = afwDisplay.RED
129  lab += "BAD"
130  if dipoleFlag:
131  ctype = afwDisplay.YELLOW
132  lab += "DIPOLE"
133  if not badFlag and not dipoleFlag:
134  ctype = afwDisplay.GREEN
135  lab += "OK"
136  mos.append(im.makeMosaic(), lab, ctype)
137  title = "Dia Sources"
138  mosaicImage = mos.makeMosaic(frame=frame, title=title)
139  return mosaicImage
140 
141 
142 def showKernelCandidates(kernelCellSet, kernel, background, frame=None, showBadCandidates=True,
143  resids=False, kernels=False):
144  """Display the Kernel candidates.
145 
146  If kernel is provided include spatial model and residuals;
147  If chi is True, generate a plot of residuals/sqrt(variance), i.e. chi.
148  """
149  #
150  # Show us the ccandidates
151  #
152  if kernels:
153  mos = afwDisplay.utils.Mosaic(gutter=5, background=0)
154  else:
155  mos = afwDisplay.utils.Mosaic(gutter=5, background=-1)
156  #
157  candidateCenters = []
158  candidateCentersBad = []
159  candidateIndex = 0
160  for cell in kernelCellSet.getCellList():
161  for cand in cell.begin(False): # include bad candidates
162  # Original difference image; if does not exist, skip candidate
163  try:
164  resid = cand.getDifferenceImage(diffimLib.KernelCandidateF.ORIG)
165  except Exception:
166  continue
167 
168  rchi2 = cand.getChi2()
169  if rchi2 > 1e100:
170  rchi2 = np.nan
171 
172  if not showBadCandidates and cand.isBad():
173  continue
174 
175  im_resid = afwDisplay.utils.Mosaic(gutter=1, background=-0.5, mode="x")
176 
177  try:
178  im = cand.getScienceMaskedImage()
179  im = im.Factory(im, True)
180  im.setXY0(cand.getScienceMaskedImage().getXY0())
181  except Exception:
182  continue
183  if (not resids and not kernels):
184  im_resid.append(im.Factory(im, True))
185  try:
186  im = cand.getTemplateMaskedImage()
187  im = im.Factory(im, True)
188  im.setXY0(cand.getTemplateMaskedImage().getXY0())
189  except Exception:
190  continue
191  if (not resids and not kernels):
192  im_resid.append(im.Factory(im, True))
193 
194  # Difference image with original basis
195  if resids:
196  var = resid.getVariance()
197  var = var.Factory(var, True)
198  np.sqrt(var.getArray(), var.getArray()) # inplace sqrt
199  resid = resid.getImage()
200  resid /= var
201  bbox = kernel.shrinkBBox(resid.getBBox())
202  resid = resid.Factory(resid, bbox, deep=True)
203  elif kernels:
204  kim = cand.getKernelImage(diffimLib.KernelCandidateF.ORIG).convertF()
205  resid = kim.Factory(kim, True)
206  im_resid.append(resid)
207 
208  # residuals using spatial model
209  ski = afwImage.ImageD(kernel.getDimensions())
210  kernel.computeImage(ski, False, int(cand.getXCenter()), int(cand.getYCenter()))
211  sk = afwMath.FixedKernel(ski)
212  sbg = 0.0
213  if background:
214  sbg = background(int(cand.getXCenter()), int(cand.getYCenter()))
215  sresid = cand.getDifferenceImage(sk, sbg)
216  resid = sresid
217  if resids:
218  resid = sresid.getImage()
219  resid /= var
220  bbox = kernel.shrinkBBox(resid.getBBox())
221  resid = resid.Factory(resid, bbox, deep=True)
222  elif kernels:
223  kim = ski.convertF()
224  resid = kim.Factory(kim, True)
225  im_resid.append(resid)
226 
227  im = im_resid.makeMosaic()
228 
229  lab = "%d chi^2 %.1f" % (cand.getId(), rchi2)
230  ctype = afwDisplay.RED if cand.isBad() else afwDisplay.GREEN
231 
232  mos.append(im, lab, ctype)
233 
234  if False and np.isnan(rchi2):
235  disp = afwDisplay.Display(frame=1)
236  disp.mtv(cand.getScienceMaskedImage.getImage(), title="candidate")
237  print("rating", cand.getCandidateRating())
238 
239  im = cand.getScienceMaskedImage()
240  center = (candidateIndex, cand.getXCenter() - im.getX0(), cand.getYCenter() - im.getY0())
241  candidateIndex += 1
242  if cand.isBad():
243  candidateCentersBad.append(center)
244  else:
245  candidateCenters.append(center)
246 
247  if resids:
248  title = "chi Diffim"
249  elif kernels:
250  title = "Kernels"
251  else:
252  title = "Candidates & residuals"
253  mosaicImage = mos.makeMosaic(frame=frame, title=title)
254 
255  return mosaicImage
256 
257 
258 def showKernelBasis(kernel, frame=None):
259  """Display a Kernel's basis images.
260  """
261  mos = afwDisplay.utils.Mosaic()
262 
263  for k in kernel.getKernelList():
264  im = afwImage.ImageD(k.getDimensions())
265  k.computeImage(im, False)
266  mos.append(im)
267  mos.makeMosaic(frame=frame, title="Kernel Basis Images")
268 
269  return mos
270 
271 
272 
273 
274 def plotKernelSpatialModel(kernel, kernelCellSet, showBadCandidates=True,
275  numSample=128, keepPlots=True, maxCoeff=10):
276  """Plot the Kernel spatial model.
277  """
278  try:
279  import matplotlib.pyplot as plt
280  import matplotlib.colors
281  except ImportError as e:
282  print("Unable to import numpy and matplotlib: %s" % e)
283  return
284 
285  x0 = kernelCellSet.getBBox().getBeginX()
286  y0 = kernelCellSet.getBBox().getBeginY()
287 
288  candPos = list()
289  candFits = list()
290  badPos = list()
291  badFits = list()
292  candAmps = list()
293  badAmps = list()
294  for cell in kernelCellSet.getCellList():
295  for cand in cell.begin(False):
296  if not showBadCandidates and cand.isBad():
297  continue
298  candCenter = geom.PointD(cand.getXCenter(), cand.getYCenter())
299  try:
300  im = cand.getTemplateMaskedImage()
301  except Exception:
302  continue
303 
304  targetFits = badFits if cand.isBad() else candFits
305  targetPos = badPos if cand.isBad() else candPos
306  targetAmps = badAmps if cand.isBad() else candAmps
307 
308  # compare original and spatial kernel coefficients
309  kp0 = np.array(cand.getKernel(diffimLib.KernelCandidateF.ORIG).getKernelParameters())
310  amp = cand.getCandidateRating()
311 
312  targetFits = badFits if cand.isBad() else candFits
313  targetPos = badPos if cand.isBad() else candPos
314  targetAmps = badAmps if cand.isBad() else candAmps
315 
316  targetFits.append(kp0)
317  targetPos.append(candCenter)
318  targetAmps.append(amp)
319 
320  xGood = np.array([pos.getX() for pos in candPos]) - x0
321  yGood = np.array([pos.getY() for pos in candPos]) - y0
322  zGood = np.array(candFits)
323 
324  xBad = np.array([pos.getX() for pos in badPos]) - x0
325  yBad = np.array([pos.getY() for pos in badPos]) - y0
326  zBad = np.array(badFits)
327  numBad = len(badPos)
328 
329  xRange = np.linspace(0, kernelCellSet.getBBox().getWidth(), num=numSample)
330  yRange = np.linspace(0, kernelCellSet.getBBox().getHeight(), num=numSample)
331 
332  if maxCoeff:
333  maxCoeff = min(maxCoeff, kernel.getNKernelParameters())
334  else:
335  maxCoeff = kernel.getNKernelParameters()
336 
337  for k in range(maxCoeff):
338  func = kernel.getSpatialFunction(k)
339  dfGood = zGood[:, k] - np.array([func(pos.getX(), pos.getY()) for pos in candPos])
340  yMin = dfGood.min()
341  yMax = dfGood.max()
342  if numBad > 0:
343  dfBad = zBad[:, k] - np.array([func(pos.getX(), pos.getY()) for pos in badPos])
344  # Can really screw up the range...
345  yMin = min([yMin, dfBad.min()])
346  yMax = max([yMax, dfBad.max()])
347  yMin -= 0.05*(yMax - yMin)
348  yMax += 0.05*(yMax - yMin)
349 
350  fRange = np.ndarray((len(xRange), len(yRange)))
351  for j, yVal in enumerate(yRange):
352  for i, xVal in enumerate(xRange):
353  fRange[j][i] = func(xVal, yVal)
354 
355  fig = plt.figure(k)
356 
357  fig.clf()
358  try:
359  fig.canvas._tkcanvas._root().lift() # == Tk's raise, but raise is a python reserved word
360  except Exception: # protect against API changes
361  pass
362 
363  fig.suptitle('Kernel component %d' % k)
364 
365  # LL
366  ax = fig.add_axes((0.1, 0.05, 0.35, 0.35))
367  vmin = fRange.min() # - 0.05*np.fabs(fRange.min())
368  vmax = fRange.max() # + 0.05*np.fabs(fRange.max())
369  norm = matplotlib.colors.Normalize(vmin=vmin, vmax=vmax)
370  im = ax.imshow(fRange, aspect='auto', norm=norm,
371  extent=[0, kernelCellSet.getBBox().getWidth() - 1,
372  0, kernelCellSet.getBBox().getHeight() - 1])
373  ax.set_title('Spatial polynomial')
374  plt.colorbar(im, orientation='horizontal', ticks=[vmin, vmax])
375 
376  # UL
377  ax = fig.add_axes((0.1, 0.55, 0.35, 0.35))
378  ax.plot(-2.5*np.log10(candAmps), zGood[:, k], 'b+')
379  if numBad > 0:
380  ax.plot(-2.5*np.log10(badAmps), zBad[:, k], 'r+')
381  ax.set_title("Basis Coefficients")
382  ax.set_xlabel("Instr mag")
383  ax.set_ylabel("Coeff")
384 
385  # LR
386  ax = fig.add_axes((0.55, 0.05, 0.35, 0.35))
387  ax.set_autoscale_on(False)
388  ax.set_xbound(lower=0, upper=kernelCellSet.getBBox().getHeight())
389  ax.set_ybound(lower=yMin, upper=yMax)
390  ax.plot(yGood, dfGood, 'b+')
391  if numBad > 0:
392  ax.plot(yBad, dfBad, 'r+')
393  ax.axhline(0.0)
394  ax.set_title('dCoeff (indiv-spatial) vs. y')
395 
396  # UR
397  ax = fig.add_axes((0.55, 0.55, 0.35, 0.35))
398  ax.set_autoscale_on(False)
399  ax.set_xbound(lower=0, upper=kernelCellSet.getBBox().getWidth())
400  ax.set_ybound(lower=yMin, upper=yMax)
401  ax.plot(xGood, dfGood, 'b+')
402  if numBad > 0:
403  ax.plot(xBad, dfBad, 'r+')
404  ax.axhline(0.0)
405  ax.set_title('dCoeff (indiv-spatial) vs. x')
406 
407  fig.show()
408 
409  global keptPlots
410  if keepPlots and not keptPlots:
411  # Keep plots open when done
412  def show():
413  print("%s: Please close plots when done." % __name__)
414  try:
415  plt.show()
416  except Exception:
417  pass
418  print("Plots closed, exiting...")
419  import atexit
420  atexit.register(show)
421  keptPlots = True
422 
423 
424 def plotKernelCoefficients(spatialKernel, kernelCellSet, showBadCandidates=False, keepPlots=True):
425  """Plot the individual kernel candidate and the spatial kernel solution coefficients.
426 
427  Parameters
428  ----------
429 
430  spatialKernel : `lsst.afw.math.LinearCombinationKernel`
431  The spatial spatialKernel solution model which is a spatially varying linear combination
432  of the spatialKernel basis functions.
433  Typically returned by `lsst.ip.diffim.SpatialKernelSolution.getSolutionPair()`.
434 
435  kernelCellSet : `lsst.afw.math.SpatialCellSet`
436  The spatial cells that was used for solution for the spatialKernel. They contain the
437  local solutions of the AL kernel for the selected sources.
438 
439  showBadCandidates : `bool`, optional
440  If True, plot the coefficient values for kernel candidates where the solution was marked
441  bad by the numerical algorithm. Defaults to False.
442 
443  keepPlots: `bool`, optional
444  If True, sets ``plt.show()`` to be called before the task terminates, so that the plots
445  can be explored interactively. Defaults to True.
446 
447  Notes
448  -----
449  This function produces 3 figures per image subtraction operation.
450  * A grid plot of the local solutions. Each grid cell corresponds to a proportional area in
451  the image. In each cell, local kernel solution coefficients are plotted of kernel candidates (color)
452  that fall into this area as a function of the kernel basis function number.
453  * A grid plot of the spatial solution. Each grid cell corresponds to a proportional area in
454  the image. In each cell, the spatial solution coefficients are evaluated for the center of the cell.
455  * Histogram of the local solution coefficients. Red line marks the spatial solution value at
456  center of the image.
457 
458  This function is called if ``lsst.ip.diffim.psfMatch.plotKernelCoefficients==True`` in lsstDebug. This
459  function was implemented as part of DM-17825.
460  """
461  try:
462  import matplotlib.pyplot as plt
463  except ImportError as e:
464  print("Unable to import matplotlib: %s" % e)
465  return
466 
467  # Image dimensions
468  imgBBox = kernelCellSet.getBBox()
469  x0 = imgBBox.getBeginX()
470  y0 = imgBBox.getBeginY()
471  wImage = imgBBox.getWidth()
472  hImage = imgBBox.getHeight()
473  imgCenterX = imgBBox.getCenterX()
474  imgCenterY = imgBBox.getCenterY()
475 
476  # Plot the local solutions
477  # ----
478 
479  # Grid size
480  nX = 8
481  nY = 8
482  wCell = wImage / nX
483  hCell = hImage / nY
484 
485  fig = plt.figure()
486  fig.suptitle("Kernel candidate parameters on an image grid")
487  arrAx = fig.subplots(nrows=nY, ncols=nX, sharex=True, sharey=True, gridspec_kw=dict(
488  wspace=0, hspace=0))
489 
490  # Bottom left panel is for bottom left part of the image
491  arrAx = arrAx[::-1, :]
492 
493  allParams = []
494  for cell in kernelCellSet.getCellList():
495  cellBBox = afwGeom.Box2D(cell.getBBox())
496  # Determine which panel this spatial cell belongs to
497  iX = int((cellBBox.getCenterX() - x0)//wCell)
498  iY = int((cellBBox.getCenterY() - y0)//hCell)
499 
500  for cand in cell.begin(False):
501  try:
502  kernel = cand.getKernel(cand.ORIG)
503  except Exception:
504  continue
505 
506  if not showBadCandidates and cand.isBad():
507  continue
508 
509  nKernelParams = kernel.getNKernelParameters()
510  kernelParams = np.array(kernel.getKernelParameters())
511  allParams.append(kernelParams)
512 
513  if cand.isBad():
514  color = 'red'
515  else:
516  color = None
517  arrAx[iY, iX].plot(np.arange(nKernelParams), kernelParams, '.-',
518  color=color, drawstyle='steps-mid', linewidth=0.1)
519  for ax in arrAx.ravel():
520  ax.grid(True, axis='y')
521 
522  # Plot histogram of the local parameters and the global solution at the image center
523  # ----
524 
525  spatialFuncs = spatialKernel.getSpatialFunctionList()
526  nKernelParams = spatialKernel.getNKernelParameters()
527  nX = 8
528  fig = plt.figure()
529  fig.suptitle("Hist. of parameters marked with spatial solution at img center")
530  arrAx = fig.subplots(nrows=int(nKernelParams//nX)+1, ncols=nX)
531  arrAx = arrAx[::-1, :]
532  allParams = np.array(allParams)
533  for k in range(nKernelParams):
534  ax = arrAx.ravel()[k]
535  ax.hist(allParams[:, k], bins=20, edgecolor='black')
536  ax.set_xlabel('P{}'.format(k))
537  valueParam = spatialFuncs[k](imgCenterX, imgCenterY)
538  ax.axvline(x=valueParam, color='red')
539  ax.text(0.1, 0.9, '{:.1f}'.format(valueParam),
540  transform=ax.transAxes, backgroundcolor='lightsteelblue')
541 
542  # Plot grid of the spatial solution
543  # ----
544 
545  nX = 8
546  nY = 8
547  wCell = wImage / nX
548  hCell = hImage / nY
549  x0 += wCell / 2
550  y0 += hCell / 2
551 
552  fig = plt.figure()
553  fig.suptitle("Spatial solution of kernel parameters on an image grid")
554  arrAx = fig.subplots(nrows=nY, ncols=nX, sharex=True, sharey=True, gridspec_kw=dict(
555  wspace=0, hspace=0))
556  arrAx = arrAx[::-1, :]
557  kernelParams = np.zeros(nKernelParams, dtype=float)
558 
559  for iX in range(nX):
560  for iY in range(nY):
561  x = x0 + iX * wCell
562  y = y0 + iY * hCell
563  # Evaluate the spatial solution functions for this x,y location
564  kernelParams = [f(x, y) for f in spatialFuncs]
565  arrAx[iY, iX].plot(np.arange(nKernelParams), kernelParams, '.-', drawstyle='steps-mid')
566  arrAx[iY, iX].grid(True, axis='y')
567 
568  global keptPlots
569  if keepPlots and not keptPlots:
570  # Keep plots open when done
571  def show():
572  print("%s: Please close plots when done." % __name__)
573  try:
574  plt.show()
575  except Exception:
576  pass
577  print("Plots closed, exiting...")
578  import atexit
579  atexit.register(show)
580  keptPlots = True
581 
582 
583 def showKernelMosaic(bbox, kernel, nx=7, ny=None, frame=None, title=None,
584  showCenter=True, showEllipticity=True):
585  """Show a mosaic of Kernel images.
586  """
587  mos = afwDisplay.utils.Mosaic()
588 
589  x0 = bbox.getBeginX()
590  y0 = bbox.getBeginY()
591  width = bbox.getWidth()
592  height = bbox.getHeight()
593 
594  if not ny:
595  ny = int(nx*float(height)/width + 0.5)
596  if not ny:
597  ny = 1
598 
599  schema = afwTable.SourceTable.makeMinimalSchema()
600  centroidName = "base_SdssCentroid"
601  shapeName = "base_SdssShape"
602  control = measBase.SdssCentroidControl()
603  schema.getAliasMap().set("slot_Centroid", centroidName)
604  schema.getAliasMap().set("slot_Centroid_flag", centroidName + "_flag")
605  centroider = measBase.SdssCentroidAlgorithm(control, centroidName, schema)
606  sdssShape = measBase.SdssShapeControl()
607  shaper = measBase.SdssShapeAlgorithm(sdssShape, shapeName, schema)
608  table = afwTable.SourceTable.make(schema)
609  table.defineCentroid(centroidName)
610  table.defineShape(shapeName)
611 
612  centers = []
613  shapes = []
614  for iy in range(ny):
615  for ix in range(nx):
616  x = int(ix*(width - 1)/(nx - 1)) + x0
617  y = int(iy*(height - 1)/(ny - 1)) + y0
618 
619  im = afwImage.ImageD(kernel.getDimensions())
620  ksum = kernel.computeImage(im, False, x, y)
621  lab = "Kernel(%d,%d)=%.2f" % (x, y, ksum) if False else ""
622  mos.append(im, lab)
623 
624  # SdssCentroidAlgorithm.measure requires an exposure of floats
625  exp = afwImage.makeExposure(afwImage.makeMaskedImage(im.convertF()))
626 
627  w, h = im.getWidth(), im.getHeight()
628  centerX = im.getX0() + w//2
629  centerY = im.getY0() + h//2
630  src = table.makeRecord()
631  spans = afwGeom.SpanSet(exp.getBBox())
632  foot = afwDet.Footprint(spans)
633  foot.addPeak(centerX, centerY, 1)
634  src.setFootprint(foot)
635 
636  try: # The centroider requires a psf, so this will fail if none is attached to exp
637  centroider.measure(src, exp)
638  centers.append((src.getX(), src.getY()))
639 
640  shaper.measure(src, exp)
641  shapes.append((src.getIxx(), src.getIxy(), src.getIyy()))
642  except Exception:
643  pass
644 
645  mos.makeMosaic(frame=frame, title=title if title else "Model Kernel", mode=nx)
646 
647  if centers and frame is not None:
648  disp = afwDisplay.Display(frame=frame)
649  i = 0
650  with disp.Buffering():
651  for cen, shape in zip(centers, shapes):
652  bbox = mos.getBBox(i)
653  i += 1
654  xc, yc = cen[0] + bbox.getMinX(), cen[1] + bbox.getMinY()
655  if showCenter:
656  disp.dot("+", xc, yc, ctype=afwDisplay.BLUE)
657 
658  if showEllipticity:
659  ixx, ixy, iyy = shape
660  disp.dot("@:%g,%g,%g" % (ixx, ixy, iyy), xc, yc, ctype=afwDisplay.RED)
661 
662  return mos
663 
664 
665 def plotPixelResiduals(exposure, warpedTemplateExposure, diffExposure, kernelCellSet,
666  kernel, background, testSources, config,
667  origVariance=False, nptsFull=1e6, keepPlots=True, titleFs=14):
668  """Plot diffim residuals for LOCAL and SPATIAL models.
669  """
670  candidateResids = []
671  spatialResids = []
672  nonfitResids = []
673 
674  for cell in kernelCellSet.getCellList():
675  for cand in cell.begin(True): # only look at good ones
676  # Be sure
677  if not (cand.getStatus() == afwMath.SpatialCellCandidate.GOOD):
678  continue
679 
680  diffim = cand.getDifferenceImage(diffimLib.KernelCandidateF.ORIG)
681  orig = cand.getScienceMaskedImage()
682 
683  ski = afwImage.ImageD(kernel.getDimensions())
684  kernel.computeImage(ski, False, int(cand.getXCenter()), int(cand.getYCenter()))
685  sk = afwMath.FixedKernel(ski)
686  sbg = background(int(cand.getXCenter()), int(cand.getYCenter()))
687  sdiffim = cand.getDifferenceImage(sk, sbg)
688 
689  # trim edgs due to convolution
690  bbox = kernel.shrinkBBox(diffim.getBBox())
691  tdiffim = diffim.Factory(diffim, bbox)
692  torig = orig.Factory(orig, bbox)
693  tsdiffim = sdiffim.Factory(sdiffim, bbox)
694 
695  if origVariance:
696  candidateResids.append(np.ravel(tdiffim.getImage().getArray() /
697  np.sqrt(torig.getVariance().getArray())))
698  spatialResids.append(np.ravel(tsdiffim.getImage().getArray() /
699  np.sqrt(torig.getVariance().getArray())))
700  else:
701  candidateResids.append(np.ravel(tdiffim.getImage().getArray() /
702  np.sqrt(tdiffim.getVariance().getArray())))
703  spatialResids.append(np.ravel(tsdiffim.getImage().getArray() /
704  np.sqrt(tsdiffim.getVariance().getArray())))
705 
706  fullIm = diffExposure.getMaskedImage().getImage().getArray()
707  fullMask = diffExposure.getMaskedImage().getMask().getArray()
708  if origVariance:
709  fullVar = exposure.getMaskedImage().getVariance().getArray()
710  else:
711  fullVar = diffExposure.getMaskedImage().getVariance().getArray()
712 
713  bitmaskBad = 0
714  bitmaskBad |= afwImage.Mask.getPlaneBitMask('NO_DATA')
715  bitmaskBad |= afwImage.Mask.getPlaneBitMask('SAT')
716  idx = np.where((fullMask & bitmaskBad) == 0)
717  stride = int(len(idx[0])//nptsFull)
718  sidx = idx[0][::stride], idx[1][::stride]
719  allResids = fullIm[sidx]/np.sqrt(fullVar[sidx])
720 
721  testFootprints = diffimTools.sourceToFootprintList(testSources, warpedTemplateExposure,
722  exposure, config, Log.getDefaultLogger())
723  for fp in testFootprints:
724  subexp = diffExposure.Factory(diffExposure, fp["footprint"].getBBox())
725  subim = subexp.getMaskedImage().getImage()
726  if origVariance:
727  subvar = afwImage.ExposureF(exposure, fp["footprint"].getBBox()).getMaskedImage().getVariance()
728  else:
729  subvar = subexp.getMaskedImage().getVariance()
730  nonfitResids.append(np.ravel(subim.getArray()/np.sqrt(subvar.getArray())))
731 
732  candidateResids = np.ravel(np.array(candidateResids))
733  spatialResids = np.ravel(np.array(spatialResids))
734  nonfitResids = np.ravel(np.array(nonfitResids))
735 
736  try:
737  import pylab
738  from matplotlib.font_manager import FontProperties
739  except ImportError as e:
740  print("Unable to import pylab: %s" % e)
741  return
742 
743  fig = pylab.figure()
744  fig.clf()
745  try:
746  fig.canvas._tkcanvas._root().lift() # == Tk's raise, but raise is a python reserved word
747  except Exception: # protect against API changes
748  pass
749  if origVariance:
750  fig.suptitle("Diffim residuals: Normalized by sqrt(input variance)", fontsize=titleFs)
751  else:
752  fig.suptitle("Diffim residuals: Normalized by sqrt(diffim variance)", fontsize=titleFs)
753 
754  sp1 = pylab.subplot(221)
755  sp2 = pylab.subplot(222, sharex=sp1, sharey=sp1)
756  sp3 = pylab.subplot(223, sharex=sp1, sharey=sp1)
757  sp4 = pylab.subplot(224, sharex=sp1, sharey=sp1)
758  xs = np.arange(-5, 5.05, 0.1)
759  ys = 1./np.sqrt(2*np.pi)*np.exp(-0.5*xs**2)
760 
761  sp1.hist(candidateResids, bins=xs, normed=True, alpha=0.5, label="N(%.2f, %.2f)"
762  % (np.mean(candidateResids), np.var(candidateResids)))
763  sp1.plot(xs, ys, "r-", lw=2, label="N(0,1)")
764  sp1.set_title("Candidates: basis fit", fontsize=titleFs - 2)
765  sp1.legend(loc=1, fancybox=True, shadow=True, prop=FontProperties(size=titleFs - 6))
766 
767  sp2.hist(spatialResids, bins=xs, normed=True, alpha=0.5, label="N(%.2f, %.2f)"
768  % (np.mean(spatialResids), np.var(spatialResids)))
769  sp2.plot(xs, ys, "r-", lw=2, label="N(0,1)")
770  sp2.set_title("Candidates: spatial fit", fontsize=titleFs - 2)
771  sp2.legend(loc=1, fancybox=True, shadow=True, prop=FontProperties(size=titleFs - 6))
772 
773  sp3.hist(nonfitResids, bins=xs, normed=True, alpha=0.5, label="N(%.2f, %.2f)"
774  % (np.mean(nonfitResids), np.var(nonfitResids)))
775  sp3.plot(xs, ys, "r-", lw=2, label="N(0,1)")
776  sp3.set_title("Control sample: spatial fit", fontsize=titleFs - 2)
777  sp3.legend(loc=1, fancybox=True, shadow=True, prop=FontProperties(size=titleFs - 6))
778 
779  sp4.hist(allResids, bins=xs, normed=True, alpha=0.5, label="N(%.2f, %.2f)"
780  % (np.mean(allResids), np.var(allResids)))
781  sp4.plot(xs, ys, "r-", lw=2, label="N(0,1)")
782  sp4.set_title("Full image (subsampled)", fontsize=titleFs - 2)
783  sp4.legend(loc=1, fancybox=True, shadow=True, prop=FontProperties(size=titleFs - 6))
784 
785  pylab.setp(sp1.get_xticklabels() + sp1.get_yticklabels(), fontsize=titleFs - 4)
786  pylab.setp(sp2.get_xticklabels() + sp2.get_yticklabels(), fontsize=titleFs - 4)
787  pylab.setp(sp3.get_xticklabels() + sp3.get_yticklabels(), fontsize=titleFs - 4)
788  pylab.setp(sp4.get_xticklabels() + sp4.get_yticklabels(), fontsize=titleFs - 4)
789 
790  sp1.set_xlim(-5, 5)
791  sp1.set_ylim(0, 0.5)
792  fig.show()
793 
794  global keptPlots
795  if keepPlots and not keptPlots:
796  # Keep plots open when done
797  def show():
798  print("%s: Please close plots when done." % __name__)
799  try:
800  pylab.show()
801  except Exception:
802  pass
803  print("Plots closed, exiting...")
804  import atexit
805  atexit.register(show)
806  keptPlots = True
807 
808 
809 def calcCentroid(arr):
810  """Calculate first moment of a (kernel) image.
811  """
812  y, x = arr.shape
813  sarr = arr*arr
814  xarr = np.asarray([[el for el in range(x)] for el2 in range(y)])
815  yarr = np.asarray([[el2 for el in range(x)] for el2 in range(y)])
816  narr = xarr*sarr
817  sarrSum = sarr.sum()
818  centx = narr.sum()/sarrSum
819  narr = yarr*sarr
820  centy = narr.sum()/sarrSum
821  return centx, centy
822 
823 
824 def calcWidth(arr, centx, centy):
825  """Calculate second moment of a (kernel) image.
826  """
827  y, x = arr.shape
828  # Square the flux so we don't have to deal with negatives
829  sarr = arr*arr
830  xarr = np.asarray([[el for el in range(x)] for el2 in range(y)])
831  yarr = np.asarray([[el2 for el in range(x)] for el2 in range(y)])
832  narr = sarr*np.power((xarr - centx), 2.)
833  sarrSum = sarr.sum()
834  xstd = np.sqrt(narr.sum()/sarrSum)
835  narr = sarr*np.power((yarr - centy), 2.)
836  ystd = np.sqrt(narr.sum()/sarrSum)
837  return xstd, ystd
838 
839 
840 def printSkyDiffs(sources, wcs):
841  """Print differences in sky coordinates.
842 
843  The difference is that between the source Position and its Centroid mapped
844  through Wcs.
845  """
846  for s in sources:
847  sCentroid = s.getCentroid()
848  sPosition = s.getCoord().getPosition(geom.degrees)
849  dra = 3600*(sPosition.getX() - wcs.pixelToSky(sCentroid).getPosition(geom.degrees).getX())/0.2
850  ddec = 3600*(sPosition.getY() - wcs.pixelToSky(sCentroid).getPosition(geom.degrees).getY())/0.2
851  if np.isfinite(dra) and np.isfinite(ddec):
852  print(dra, ddec)
853 
854 
855 def makeRegions(sources, outfilename, wcs=None):
856  """Create regions file for display from input source list.
857  """
858  fh = open(outfilename, "w")
859  fh.write("global color=red font=\"helvetica 10 normal\" "
860  "select=1 highlite=1 edit=1 move=1 delete=1 include=1 fixed=0 source\nfk5\n")
861  for s in sources:
862  if wcs:
863  (ra, dec) = wcs.pixelToSky(s.getCentroid()).getPosition(geom.degrees)
864  else:
865  (ra, dec) = s.getCoord().getPosition(geom.degrees)
866  if np.isfinite(ra) and np.isfinite(dec):
867  fh.write("circle(%f,%f,2\")\n"%(ra, dec))
868  fh.flush()
869  fh.close()
870 
871 
872 def showSourceSetSky(sSet, wcs, xy0, frame=0, ctype=afwDisplay.GREEN, symb="+", size=2):
873  """Draw the (RA, Dec) positions of a set of Sources. Image has the XY0.
874  """
875  disp = afwDisplay.Display(frame=frame)
876  with disp.Buffering():
877  for s in sSet:
878  (xc, yc) = wcs.skyToPixel(s.getCoord().getRa(), s.getCoord().getDec())
879  xc -= xy0[0]
880  yc -= xy0[1]
881  disp.dot(symb, xc, yc, ctype=ctype, size=size)
882 
883 
884 def plotWhisker(results, newWcs):
885  """Plot whisker diagram of astromeric offsets between results.matches.
886  """
887  refCoordKey = results.matches[0].first.getTable().getCoordKey()
888  inCentroidKey = results.matches[0].second.getTable().getCentroidKey()
889  positions = [m.first.get(refCoordKey) for m in results.matches]
890  residuals = [m.first.get(refCoordKey).getOffsetFrom(
891  newWcs.pixelToSky(m.second.get(inCentroidKey))) for
892  m in results.matches]
893  import matplotlib.pyplot as plt
894  fig = plt.figure()
895  sp = fig.add_subplot(1, 1, 0)
896  xpos = [x[0].asDegrees() for x in positions]
897  ypos = [x[1].asDegrees() for x in positions]
898  xpos.append(0.02*(max(xpos) - min(xpos)) + min(xpos))
899  ypos.append(0.98*(max(ypos) - min(ypos)) + min(ypos))
900  xidxs = np.isfinite(xpos)
901  yidxs = np.isfinite(ypos)
902  X = np.asarray(xpos)[xidxs]
903  Y = np.asarray(ypos)[yidxs]
904  distance = [x[1].asArcseconds() for x in residuals]
905  distance.append(0.2)
906  distance = np.asarray(distance)[xidxs]
907  # NOTE: This assumes that the bearing is measured positive from +RA through North.
908  # From the documentation this is not clear.
909  bearing = [x[0].asRadians() for x in residuals]
910  bearing.append(0)
911  bearing = np.asarray(bearing)[xidxs]
912  U = (distance*np.cos(bearing))
913  V = (distance*np.sin(bearing))
914  sp.quiver(X, Y, U, V)
915  sp.set_title("WCS Residual")
916  plt.show()
917 
918 
919 class DipoleTestImage(object):
920  """Utility class for dipole measurement testing.
921 
922  Generate an image with simulated dipoles and noise; store the original
923  "pre-subtraction" images and catalogs as well.
924  Used to generate test data for DMTN-007 (http://dmtn-007.lsst.io).
925  """
926 
927  def __init__(self, w=101, h=101, xcenPos=[27.], ycenPos=[25.], xcenNeg=[23.], ycenNeg=[25.],
928  psfSigma=2., flux=[30000.], fluxNeg=None, noise=10., gradientParams=None):
929  self.w = w
930  self.h = h
931  self.xcenPos = xcenPos
932  self.ycenPos = ycenPos
933  self.xcenNeg = xcenNeg
934  self.ycenNeg = ycenNeg
935  self.psfSigma = psfSigma
936  self.flux = flux
937  self.fluxNeg = fluxNeg
938  if fluxNeg is None:
939  self.fluxNeg = self.flux
940  self.noise = noise
941  self.gradientParams = gradientParams
942  self._makeDipoleImage()
943 
944  def _makeDipoleImage(self):
945  """Generate an exposure and catalog with the given dipole source(s).
946  """
947  # Must seed the pos/neg images with different values to ensure they get different noise realizations
948  posImage, posCatalog = self._makeStarImage(
949  xc=self.xcenPos, yc=self.ycenPos, flux=self.flux, randomSeed=111)
950 
951  negImage, negCatalog = self._makeStarImage(
952  xc=self.xcenNeg, yc=self.ycenNeg, flux=self.fluxNeg, randomSeed=222)
953 
954  dipole = posImage.clone()
955  di = dipole.getMaskedImage()
956  di -= negImage.getMaskedImage()
957 
958  # Carry through pos/neg detection masks to new planes in diffim
959  dm = di.getMask()
960  posDetectedBits = posImage.getMaskedImage().getMask().getArray() == dm.getPlaneBitMask("DETECTED")
961  negDetectedBits = negImage.getMaskedImage().getMask().getArray() == dm.getPlaneBitMask("DETECTED")
962  pos_det = dm.addMaskPlane("DETECTED_POS") # new mask plane -- different from "DETECTED"
963  neg_det = dm.addMaskPlane("DETECTED_NEG") # new mask plane -- different from "DETECTED_NEGATIVE"
964  dma = dm.getArray()
965  # set the two custom mask planes to these new masks
966  dma[:, :] = posDetectedBits*pos_det + negDetectedBits*neg_det
967  self.diffim, self.posImage, self.posCatalog, self.negImage, self.negCatalog \
968  = dipole, posImage, posCatalog, negImage, negCatalog
969 
970  def _makeStarImage(self, xc=[15.3], yc=[18.6], flux=[2500], schema=None, randomSeed=None):
971  """Generate an exposure and catalog with the given stellar source(s).
972  """
973  from lsst.meas.base.tests import TestDataset
974  bbox = geom.Box2I(geom.Point2I(0, 0), geom.Point2I(self.w - 1, self.h - 1))
975  dataset = TestDataset(bbox, psfSigma=self.psfSigma, threshold=1.)
976 
977  for i in range(len(xc)):
978  dataset.addSource(instFlux=flux[i], centroid=geom.Point2D(xc[i], yc[i]))
979 
980  if schema is None:
981  schema = TestDataset.makeMinimalSchema()
982  exposure, catalog = dataset.realize(noise=self.noise, schema=schema, randomSeed=randomSeed)
983 
984  if self.gradientParams is not None:
985  y, x = np.mgrid[:self.w, :self.h]
986  gp = self.gradientParams
987  gradient = gp[0] + gp[1]*x + gp[2]*y
988  if len(self.gradientParams) > 3: # it includes a set of 2nd-order polynomial params
989  gradient += gp[3]*x*y + gp[4]*x*x + gp[5]*y*y
990  imgArr = exposure.getMaskedImage().getArrays()[0]
991  imgArr += gradient
992 
993  return exposure, catalog
994 
995  def fitDipoleSource(self, source, **kwds):
996  alg = DipoleFitAlgorithm(self.diffim, self.posImage, self.negImage)
997  fitResult = alg.fitDipole(source, **kwds)
998  return fitResult
999 
1000  def detectDipoleSources(self, doMerge=True, diffim=None, detectSigma=5.5, grow=3, minBinSize=32):
1001  """Utility function for detecting dipoles.
1002 
1003  Detect pos/neg sources in the diffim, then merge them. A
1004  bigger "grow" parameter leads to a larger footprint which
1005  helps with dipole measurement for faint dipoles.
1006 
1007  Parameters
1008  ----------
1009  doMerge : `bool`
1010  Whether to merge the positive and negagive detections into a single
1011  source table.
1012  diffim : `lsst.afw.image.exposure.exposure.ExposureF`
1013  Difference image on which to perform detection.
1014  detectSigma : `float`
1015  Threshold for object detection.
1016  grow : `int`
1017  Number of pixels to grow the footprints before merging.
1018  minBinSize : `int`
1019  Minimum bin size for the background (re)estimation (only applies if
1020  the default leads to min(nBinX, nBinY) < fit order so the default
1021  config parameter needs to be decreased, but not to a value smaller
1022  than ``minBinSize``, in which case the fitting algorithm will take
1023  over and decrease the fit order appropriately.)
1024 
1025  Returns
1026  -------
1027  sources : `lsst.afw.table.SourceCatalog`
1028  If doMerge=True, the merged source catalog is returned OR
1029  detectTask : `lsst.meas.algorithms.SourceDetectionTask`
1030  schema : `lsst.afw.table.Schema`
1031  If doMerge=False, the source detection task and its schema are
1032  returned.
1033  """
1034  if diffim is None:
1035  diffim = self.diffim
1036 
1037  # Start with a minimal schema - only the fields all SourceCatalogs need
1038  schema = afwTable.SourceTable.makeMinimalSchema()
1039 
1040  # Customize the detection task a bit (optional)
1041  detectConfig = measAlg.SourceDetectionConfig()
1042  detectConfig.returnOriginalFootprints = False # should be the default
1043 
1044  psfSigma = diffim.getPsf().computeShape().getDeterminantRadius()
1045 
1046  # code from imageDifference.py:
1047  detectConfig.thresholdPolarity = "both"
1048  detectConfig.thresholdValue = detectSigma
1049  # detectConfig.nSigmaToGrow = psfSigma
1050  detectConfig.reEstimateBackground = True # if False, will fail often for faint sources on gradients?
1051  detectConfig.thresholdType = "pixel_stdev"
1052  # Test images are often quite small, so may need to adjust background binSize
1053  while ((min(diffim.getWidth(), diffim.getHeight()))//detectConfig.background.binSize <
1054  detectConfig.background.approxOrderX and detectConfig.background.binSize > minBinSize):
1055  detectConfig.background.binSize = max(minBinSize, detectConfig.background.binSize//2)
1056 
1057  # Create the detection task. We pass the schema so the task can declare a few flag fields
1058  detectTask = measAlg.SourceDetectionTask(schema, config=detectConfig)
1059 
1060  table = afwTable.SourceTable.make(schema)
1061  catalog = detectTask.makeSourceCatalog(table, diffim, sigma=psfSigma)
1062 
1063  # Now do the merge.
1064  if doMerge:
1065  fpSet = catalog.fpSets.positive
1066  fpSet.merge(catalog.fpSets.negative, grow, grow, False)
1067  sources = afwTable.SourceCatalog(table)
1068  fpSet.makeSources(sources)
1069 
1070  return sources
1071 
1072  else:
1073  return detectTask, schema
def showDiaSources(sources, exposure, isFlagged, isDipole, frame=None)
Definition: utils.py:106
def plotKernelCoefficients(spatialKernel, kernelCellSet, showBadCandidates=False, keepPlots=True)
Definition: utils.py:424
def makeRegions(sources, outfilename, wcs=None)
Definition: utils.py:855
def __init__(self, w=101, h=101, xcenPos=[27.], ycenPos=[25.], xcenNeg=[23.], ycenNeg=[25.], psfSigma=2., flux=[30000.], fluxNeg=None, noise=10., gradientParams=None)
Definition: utils.py:928
MaskedImage< ImagePixelT, MaskPixelT, VariancePixelT > * makeMaskedImage(typename std::shared_ptr< Image< ImagePixelT >> image, typename std::shared_ptr< Mask< MaskPixelT >> mask=Mask< MaskPixelT >(), typename std::shared_ptr< Image< VariancePixelT >> variance=Image< VariancePixelT >())
def showKernelSpatialCells(maskedIm, kernelCellSet, showChi2=False, symb="o", ctype=None, ctypeUnused=None, ctypeBad=None, size=3, frame=None, title="Spatial Cells")
Definition: utils.py:69
std::shared_ptr< Exposure< ImagePixelT, MaskPixelT, VariancePixelT > > makeExposure(MaskedImage< ImagePixelT, MaskPixelT, VariancePixelT > &mimage, std::shared_ptr< geom::SkyWcs const > wcs=std::shared_ptr< geom::SkyWcs const >())
def _makeStarImage(self, xc=[15.3], yc=[18.6], flux=[2500], schema=None, randomSeed=None)
Definition: utils.py:970
def plotKernelSpatialModel(kernel, kernelCellSet, showBadCandidates=True, numSample=128, keepPlots=True, maxCoeff=10)
Definition: utils.py:275
def plotPixelResiduals(exposure, warpedTemplateExposure, diffExposure, kernelCellSet, kernel, background, testSources, config, origVariance=False, nptsFull=1e6, keepPlots=True, titleFs=14)
Definition: utils.py:667
def printSkyDiffs(sources, wcs)
Definition: utils.py:840
def showKernelCandidates(kernelCellSet, kernel, background, frame=None, showBadCandidates=True, resids=False, kernels=False)
Definition: utils.py:143
def detectDipoleSources(self, doMerge=True, diffim=None, detectSigma=5.5, grow=3, minBinSize=32)
Definition: utils.py:1000
def calcCentroid(arr)
Definition: utils.py:809
def calcWidth(arr, centx, centy)
Definition: utils.py:824
def showSourceSet(sSet, xy0=(0, 0), frame=0, ctype=afwDisplay.GREEN, symb="+", size=2)
Definition: utils.py:47
def plotWhisker(results, newWcs)
Definition: utils.py:884
def showKernelMosaic(bbox, kernel, nx=7, ny=None, frame=None, title=None, showCenter=True, showEllipticity=True)
Definition: utils.py:584
def showKernelBasis(kernel, frame=None)
Definition: utils.py:258
def fitDipoleSource(self, source, kwds)
Definition: utils.py:995
def showSourceSetSky(sSet, wcs, xy0, frame=0, ctype=afwDisplay.GREEN, symb="+", size=2)
Definition: utils.py:872