lsst.ip.diffim  15.0-4-gd76abed
diffimTools.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2016 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 __all__ = ["backgroundSubtract", "writeKernelCellSet", "sourceToFootprintList", "NbasisEvaluator"]
24 
25 from builtins import range
26 from builtins import object
27 
28 # python
29 import time
30 import os
31 from collections import Counter
32 import numpy as np
33 
34 # all the c++ level classes and routines
35 from . import diffimLib
36 
37 # all the other LSST packages
38 import lsst.afw.geom as afwGeom
39 import lsst.afw.image as afwImage
40 import lsst.afw.table as afwTable
41 import lsst.afw.detection as afwDetect
42 import lsst.afw.math.mathLib as afwMath
43 from lsst.log import Log
44 import lsst.pex.config as pexConfig
45 from .makeKernelBasisList import makeKernelBasisList
46 
47 # Helper functions for ipDiffim; mostly viewing of results and writing
48 # debugging info to disk.
49 
50 
53 
54 
55 def makeFlatNoiseImage(mi, seedStat=afwMath.MAX):
56  img = mi.getImage()
57  seed = int(10. * afwMath.makeStatistics(mi.getImage(), seedStat).getValue() + 1)
58  rdm = afwMath.Random(afwMath.Random.MT19937, seed)
59  rdmImage = img.Factory(img.getDimensions())
60  afwMath.randomGaussianImage(rdmImage, rdm)
61  return rdmImage
62 
63 
65  """Return a Poisson noise image based on im
66 
67  Uses numpy.random; you may wish to call numpy.random.seed first.
68 
69  @warning This uses an undocumented numpy API (the documented API
70  uses a single float expectation value instead of an array).
71 
72  @param[in] im image; the output image has the same dimensions and shape
73  and its expectation value is the value of im at each pixel
74  """
75  import numpy.random as rand
76  imArr = im.getArray()
77  noiseIm = im.Factory(im.getBBox())
78  noiseArr = noiseIm.getArray()
79 
80  with np.errstate(invalid='ignore'):
81  intNoiseArr = rand.poisson(imArr)
82 
83  noiseArr[:, :] = intNoiseArr.astype(noiseArr.dtype)
84  return noiseIm
85 
86 
91 
92 
93 def fakeCoeffs():
94  kCoeffs = ((1.0, 0.0, 0.0),
95  (0.005, -0.000001, 0.000001),
96  (0.005, 0.000004, 0.000004),
97  (-0.001, -0.000030, 0.000030),
98  (-0.001, 0.000015, 0.000015),
99  (-0.005, -0.000050, 0.000050))
100  return kCoeffs
101 
102 
103 def makeFakeKernelSet(sizeCell=128, nCell=3,
104  deltaFunctionCounts=1.e4, tGaussianWidth=1.0,
105  addNoise=True, bgValue=100., display=False):
106 
107  from . import imagePsfMatch
108  configFake = imagePsfMatch.ImagePsfMatchConfig()
109  configFake.kernel.name = "AL"
110  subconfigFake = configFake.kernel.active
111  subconfigFake.alardNGauss = 1
112  subconfigFake.alardSigGauss = [2.5, ]
113  subconfigFake.alardDegGauss = [2, ]
114  subconfigFake.sizeCellX = sizeCell
115  subconfigFake.sizeCellY = sizeCell
116  subconfigFake.spatialKernelOrder = 1
117  subconfigFake.spatialModelType = "polynomial"
118  subconfigFake.singleKernelClipping = False # variance is a hack
119  subconfigFake.spatialKernelClipping = False # variance is a hack
120  if bgValue > 0.0:
121  subconfigFake.fitForBackground = True
122 
123  policyFake = pexConfig.makePolicy(subconfigFake)
124 
125  basisList = makeKernelBasisList(subconfigFake)
126  kSize = subconfigFake.kernelSize
127 
128  # This sets the final extent of each convolved delta function
129  gaussKernelWidth = sizeCell//2
130 
131  # This sets the scale over which pixels are correlated in the
132  # spatial convolution; should be at least as big as the kernel you
133  # are trying to fit for
134  spatialKernelWidth = kSize
135 
136  # Number of bad pixels due to convolutions
137  border = (gaussKernelWidth + spatialKernelWidth)//2
138 
139  # Make a fake image with a matrix of delta functions
140  totalSize = nCell * sizeCell + 2*border
141  tim = afwImage.ImageF(afwGeom.Extent2I(totalSize, totalSize))
142  for x in range(nCell):
143  for y in range(nCell):
144  tim.set(x*sizeCell + sizeCell//2 + border - 1,
145  y*sizeCell + sizeCell//2 + border - 1,
146  deltaFunctionCounts)
147 
148  # Turn this into stars with a narrow width; conserve counts
149  gaussFunction = afwMath.GaussianFunction2D(tGaussianWidth, tGaussianWidth)
150  gaussKernel = afwMath.AnalyticKernel(gaussKernelWidth, gaussKernelWidth, gaussFunction)
151  cim = afwImage.ImageF(tim.getDimensions())
152  afwMath.convolve(cim, tim, gaussKernel, True)
153  tim = cim
154 
155  # Trim off border pixels
156  bbox = gaussKernel.shrinkBBox(tim.getBBox(afwImage.LOCAL))
157  tim = afwImage.ImageF(tim, bbox, afwImage.LOCAL)
158 
159  # Now make a science image which is this convolved with some
160  # spatial function. Use input basis list.
161  polyFunc = afwMath.PolynomialFunction2D(1)
162  kCoeffs = fakeCoeffs()
163  nToUse = min(len(kCoeffs), len(basisList))
164 
165  # Make the full convolved science image
166  sKernel = afwMath.LinearCombinationKernel(basisList[:nToUse], polyFunc)
167  sKernel.setSpatialParameters(kCoeffs[:nToUse])
168  sim = afwImage.ImageF(tim.getDimensions())
169  afwMath.convolve(sim, tim, sKernel, True)
170 
171  # Get the good subregion
172  bbox = sKernel.shrinkBBox(sim.getBBox(afwImage.LOCAL))
173 
174  # Add background
175  sim += bgValue
176 
177  # Watch out for negative values
178  tim += 2 * np.abs(np.min(tim.getArray()))
179 
180  # Add noise?
181  if addNoise:
182  sim = makePoissonNoiseImage(sim)
183  tim = makePoissonNoiseImage(tim)
184 
185  # And turn into MaskedImages
186  sim = afwImage.ImageF(sim, bbox, afwImage.LOCAL)
187  svar = afwImage.ImageF(sim, True)
188  smask = afwImage.Mask(sim.getDimensions())
189  smask.set(0x0)
190  sMi = afwImage.MaskedImageF(sim, smask, svar)
191 
192  tim = afwImage.ImageF(tim, bbox, afwImage.LOCAL)
193  tvar = afwImage.ImageF(tim, True)
194  tmask = afwImage.Mask(tim.getDimensions())
195  tmask.set(0x0)
196  tMi = afwImage.MaskedImageF(tim, tmask, tvar)
197 
198  if display:
199  import lsst.afw.display.ds9 as ds9
200  ds9.mtv(tMi, frame=1)
201  ds9.mtv(sMi, frame=2)
202 
203  # Finally, make a kernelSet from these 2 images
205  afwGeom.Extent2I(sizeCell * nCell,
206  sizeCell * nCell)),
207  sizeCell,
208  sizeCell)
209  stampHalfWidth = 2 * kSize
210  for x in range(nCell):
211  for y in range(nCell):
212  xCoord = x * sizeCell + sizeCell // 2
213  yCoord = y * sizeCell + sizeCell // 2
214  p0 = afwGeom.Point2I(xCoord - stampHalfWidth,
215  yCoord - stampHalfWidth)
216  p1 = afwGeom.Point2I(xCoord + stampHalfWidth,
217  yCoord + stampHalfWidth)
218  bbox = afwGeom.Box2I(p0, p1)
219  tsi = afwImage.MaskedImageF(tMi, bbox, origin=afwImage.LOCAL)
220  ssi = afwImage.MaskedImageF(sMi, bbox, origin=afwImage.LOCAL)
221 
222  kc = diffimLib.makeKernelCandidate(xCoord, yCoord, tsi, ssi, policyFake)
223  kernelCellSet.insertCandidate(kc)
224 
225  tMi.setXY0(0, 0)
226  sMi.setXY0(0, 0)
227  return tMi, sMi, sKernel, kernelCellSet, configFake
228 
229 
230 
233 
234 def backgroundSubtract(config, maskedImages):
235  backgrounds = []
236  t0 = time.time()
237  algorithm = config.algorithm
238  binsize = config.binSize
239  undersample = config.undersampleStyle
240  bctrl = afwMath.BackgroundControl(algorithm)
241  bctrl.setUndersampleStyle(undersample)
242  for maskedImage in maskedImages:
243  bctrl.setNxSample(maskedImage.getWidth()//binsize + 1)
244  bctrl.setNySample(maskedImage.getHeight()//binsize + 1)
245  image = maskedImage.getImage()
246  backobj = afwMath.makeBackground(image, bctrl)
247 
248  image -= backobj.getImageF()
249  backgrounds.append(backobj.getImageF())
250  del backobj
251 
252  t1 = time.time()
253  logger = Log.getLogger("ip.diffim.backgroundSubtract")
254  logger.debug("Total time for background subtraction : %.2f s", (t1-t0))
255  return backgrounds
256 
257 
260 
261 
262 def writeKernelCellSet(kernelCellSet, psfMatchingKernel, backgroundModel, outdir):
263  if not os.path.isdir(outdir):
264  os.makedirs(outdir)
265 
266  for cell in kernelCellSet.getCellList():
267  for cand in cell.begin(False): # False = include bad candidates
268  if cand.getStatus() == afwMath.SpatialCellCandidate.GOOD:
269  xCand = int(cand.getXCenter())
270  yCand = int(cand.getYCenter())
271  idCand = cand.getId()
272  diffIm = cand.getDifferenceImage(diffimLib.KernelCandidateF.ORIG)
273  kernel = cand.getKernelImage(diffimLib.KernelCandidateF.ORIG)
274  diffIm.writeFits(os.path.join(outdir, 'diffim_c%d_x%d_y%d.fits' % (idCand, xCand, yCand)))
275  kernel.writeFits(os.path.join(outdir, 'kernel_c%d_x%d_y%d.fits' % (idCand, xCand, yCand)))
276 
277  # Diffim from spatial model
278  ski = afwImage.ImageD(kernel.getDimensions())
279  psfMatchingKernel.computeImage(ski, False, xCand, yCand)
280  sk = afwMath.FixedKernel(ski)
281  sbg = backgroundModel(xCand, yCand)
282  sdmi = cand.getDifferenceImage(sk, sbg)
283  sdmi.writeFits(os.path.join(outdir, 'sdiffim_c%d_x%d_y%d.fits' % (idCand, xCand, yCand)))
284 
285 
288 
289 
290 def sourceToFootprintList(candidateInList, templateExposure, scienceExposure, kernelSize, config, log):
291  """ Takes an input list of Sources that were selected to constrain
292  the Psf-matching Kernel and turns them into a List of Footprints,
293  which are used to seed a set of KernelCandidates. The function
294  checks both the template and science image for masked pixels,
295  rejecting the Source if certain Mask bits (defined in config) are
296  set within the Footprint.
297 
298  @param candidateInList: Input list of Sources
299  @param templateExposure: Template image, to be checked for Mask bits in Source Footprint
300  @param scienceExposure: Science image, to be checked for Mask bits in Source Footprint
301  @param config: Config that defines the Mask planes that indicate an invalid Source and Bbox grow radius
302  @param log: Log for output
303 
304  @return a list of dicts having a "source" and "footprint" field, to be used for Psf-matching
305  """
306 
307  candidateOutList = []
308  fsb = diffimLib.FindSetBitsU()
309  badBitMask = 0
310  for mp in config.badMaskPlanes:
311  badBitMask |= afwImage.Mask.getPlaneBitMask(mp)
312  bbox = scienceExposure.getBBox()
313 
314  # Size to grow Sources
315  if config.scaleByFwhm:
316  fpGrowPix = int(config.fpGrowKernelScaling * kernelSize + 0.5)
317  else:
318  fpGrowPix = config.fpGrowPix
319  log.info("Growing %d kernel candidate stars by %d pixels", len(candidateInList), fpGrowPix)
320 
321  for kernelCandidate in candidateInList:
322  if not type(kernelCandidate) == afwTable.SourceRecord:
323  raise RuntimeError("Candiate not of type afwTable.SourceRecord")
324  bm1 = 0
325  bm2 = 0
326  center = afwGeom.Point2I(scienceExposure.getWcs().skyToPixel(kernelCandidate.getCoord()))
327  if center[0] < bbox.getMinX() or center[0] > bbox.getMaxX():
328  continue
329  if center[1] < bbox.getMinY() or center[1] > bbox.getMaxY():
330  continue
331 
332  xmin = center[0] - fpGrowPix
333  xmax = center[0] + fpGrowPix
334  ymin = center[1] - fpGrowPix
335  ymax = center[1] + fpGrowPix
336 
337  # Keep object centered
338  if (xmin - bbox.getMinX()) < 0:
339  xmax += (xmin - bbox.getMinX())
340  xmin -= (xmin - bbox.getMinX())
341  if (ymin - bbox.getMinY()) < 0:
342  ymax += (ymin - bbox.getMinY())
343  ymin -= (ymin - bbox.getMinY())
344  if (bbox.getMaxX() - xmax) < 0:
345  xmin -= (bbox.getMaxX() - xmax)
346  xmax += (bbox.getMaxX() - xmax)
347  if (bbox.getMaxY() - ymax) < 0:
348  ymin -= (bbox.getMaxY() - ymax)
349  ymax += (bbox.getMaxY() - ymax)
350  if xmin > xmax or ymin > ymax:
351  continue
352 
353  kbbox = afwGeom.Box2I(afwGeom.Point2I(xmin, ymin), afwGeom.Point2I(xmax, ymax))
354  try:
355  fsb.apply(afwImage.MaskedImageF(templateExposure.getMaskedImage(), kbbox, deep=False).getMask())
356  bm1 = fsb.getBits()
357  fsb.apply(afwImage.MaskedImageF(scienceExposure.getMaskedImage(), kbbox, deep=False).getMask())
358  bm2 = fsb.getBits()
359  except Exception:
360  pass
361  else:
362  if not((bm1 & badBitMask) or (bm2 & badBitMask)):
363  candidateOutList.append({'source': kernelCandidate,
364  'footprint': afwDetect.Footprint(afwGeom.SpanSet(kbbox))})
365  log.info("Selected %d / %d sources for KernelCandidacy", len(candidateOutList), len(candidateInList))
366  return candidateOutList
367 
368 
369 def sourceTableToCandidateList(sourceTable, templateExposure, scienceExposure, kConfig, dConfig, log,
370  basisList, doBuild=False):
371  """Takes an input list of Sources, and turns them into
372  KernelCandidates for fitting of the Psf-matching kernel."""
373  kernelSize = basisList[0].getWidth()
374  footprintList = sourceToFootprintList(list(sourceTable), templateExposure, scienceExposure,
375  kernelSize, dConfig, log)
376  candList = []
377 
378  if doBuild and not basisList:
379  doBuild = False
380  else:
381  policy = pexConfig.makePolicy(kConfig)
382  visitor = diffimLib.BuildSingleKernelVisitorF(basisList, policy)
383 
384  policy = pexConfig.makePolicy(kConfig)
385  for cand in footprintList:
386  bbox = cand['footprint'].getBBox()
387  tmi = afwImage.MaskedImageF(templateExposure.getMaskedImage(), bbox)
388  smi = afwImage.MaskedImageF(scienceExposure.getMaskedImage(), bbox)
389  kCand = diffimLib.makeKernelCandidate(cand['source'], tmi, smi, policy)
390  if doBuild:
391  visitor.processCandidate(kCand)
392  kCand.setStatus(afwMath.SpatialCellCandidate.UNKNOWN)
393  candList.append(kCand)
394  return candList
395 
396 
397 
400 
401 
402 class NbasisEvaluator(object):
403  """A functor to evaluate the Bayesian Information Criterion for the number of basis sets
404  going into the kernel fitting"""
405 
406  def __init__(self, psfMatchConfig, psfFwhmPixTc, psfFwhmPixTnc):
407  self.psfMatchConfig = psfMatchConfig
408  self.psfFwhmPixTc = psfFwhmPixTc
409  self.psfFwhmPixTnc = psfFwhmPixTnc
410  if not self.psfMatchConfig.kernelBasisSet == "alard-lupton":
411  raise RuntimeError("BIC only implemnted for AL (alard lupton) basis")
412 
413  def __call__(self, kernelCellSet, log):
414  d1, d2, d3 = self.psfMatchConfig.alardDegGauss
415  bicArray = {}
416  for d1i in range(1, d1+1):
417  for d2i in range(1, d2+1):
418  for d3i in range(1, d3+1):
419  dList = [d1i, d2i, d3i]
420  bicConfig = type(self.psfMatchConfig)(self.psfMatchConfig, alardDegGauss=dList)
421  kList = makeKernelBasisList(bicConfig, self.psfFwhmPixTc, self.psfFwhmPixTnc)
422  k = len(kList)
423  visitor = diffimLib.BuildSingleKernelVisitorF(kList, pexConfig.makePolicy(bicConfig))
424  visitor.setSkipBuilt(False)
425  kernelCellSet.visitCandidates(visitor, bicConfig.nStarPerCell)
426 
427  for cell in kernelCellSet.getCellList():
428  for cand in cell.begin(False): # False = include bad candidates
429  if cand.getStatus() != afwMath.SpatialCellCandidate.GOOD:
430  continue
431  diffIm = cand.getDifferenceImage(diffimLib.KernelCandidateF.RECENT)
432  bbox = cand.getKernel(diffimLib.KernelCandidateF.RECENT).shrinkBBox(
433  diffIm.getBBox(afwImage.LOCAL))
434  diffIm = type(diffIm)(diffIm, bbox, True)
435  chi2 = diffIm.getImage().getArray()**2 / diffIm.getVariance().getArray()
436  n = chi2.shape[0] * chi2.shape[1]
437  bic = np.sum(chi2) + k * np.log(n)
438  if cand.getId() not in bicArray:
439  bicArray[cand.getId()] = {}
440  bicArray[cand.getId()][(d1i, d2i, d3i)] = bic
441 
442  bestConfigs = []
443  for candId in bicArray:
444  cconfig, cvals = list(bicArray[candId].keys()), list(bicArray[candId].values())
445  idx = np.argsort(cvals)
446  bestConfig = cconfig[idx[0]]
447  bestConfigs.append(bestConfig)
448 
449  counter = Counter(bestConfigs).most_common(3)
450  log.info("B.I.C. prefers basis complexity %s %d times; %s %d times; %s %d times",
451  counter[0][0], counter[0][1],
452  counter[1][0], counter[1][1],
453  counter[2][0], counter[2][1])
454  return counter[0][0], counter[1][0], counter[2][0]
def makeFlatNoiseImage(mi, seedStat=afwMath.MAX)
Add noise.
Definition: diffimTools.py:55
Configuration for image-to-image Psf matching.
def makeKernelBasisList(config, targetFwhmPix=None, referenceFwhmPix=None, basisDegGauss=None, metadata=None)
def fakeCoeffs()
Make fake images for testing; one is a delta function (or narrow gaussian) and the other is a convolu...
Definition: diffimTools.py:93
std::shared_ptr< Background > makeBackground(ImageT const &img, BackgroundControl const &bgCtrl)
def backgroundSubtract(config, maskedImages)
Background subtraction for ip_diffim.
Definition: diffimTools.py:234
def makeFakeKernelSet(sizeCell=128, nCell=3, deltaFunctionCounts=1.e4, tGaussianWidth=1.0, addNoise=True, bgValue=100., display=False)
Definition: diffimTools.py:105
Statistics makeStatistics(lsst::afw::math::MaskedVector< EntryT > const &mv, std::vector< WeightPixel > const &vweights, int const flags, StatisticsControl const &sctrl=StatisticsControl())
def writeKernelCellSet(kernelCellSet, psfMatchingKernel, backgroundModel, outdir)
More coarse debugging.
Definition: diffimTools.py:262
void randomGaussianImage(ImageT *image, Random &rand)
def sourceTableToCandidateList(sourceTable, templateExposure, scienceExposure, kConfig, dConfig, log, basisList, doBuild=False)
Definition: diffimTools.py:370
def __init__(self, psfMatchConfig, psfFwhmPixTc, psfFwhmPixTnc)
Definition: diffimTools.py:406
void convolve(OutImageT &convolvedImage, InImageT const &inImage, KernelT const &kernel, bool doNormalize, bool doCopyEdge=false)
def __call__(self, kernelCellSet, log)
Definition: diffimTools.py:413
def sourceToFootprintList(candidateInList, templateExposure, scienceExposure, kernelSize, config, log)
Converting types.
Definition: diffimTools.py:290