Coverage for python/lsst/ip/diffim/diffimTools.py : 8%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["backgroundSubtract", "writeKernelCellSet", "sourceToFootprintList", "NbasisEvaluator"]
24# python
25import time
26import os
27from collections import Counter
28import numpy as np
30# all the c++ level classes and routines
31from . import diffimLib
33# all the other LSST packages
34import lsst.afw.geom as afwGeom
35import lsst.afw.image as afwImage
36import lsst.afw.table as afwTable
37import lsst.afw.detection as afwDetect
38import lsst.afw.math.mathLib as afwMath
39import lsst.geom as geom
40from lsst.log import Log
41import lsst.pex.config as pexConfig
42from .makeKernelBasisList import makeKernelBasisList
44# Helper functions for ipDiffim; mostly viewing of results and writing
45# debugging info to disk.
47#######
48# Add noise
49#######
52def makeFlatNoiseImage(mi, seedStat=afwMath.MAX):
53 img = mi.getImage()
54 seed = int(10.*afwMath.makeStatistics(mi.getImage(), seedStat).getValue() + 1)
55 rdm = afwMath.Random(afwMath.Random.MT19937, seed)
56 rdmImage = img.Factory(img.getDimensions())
57 afwMath.randomGaussianImage(rdmImage, rdm)
58 return rdmImage
61def makePoissonNoiseImage(im):
62 """Return a Poisson noise image based on im
64 Parameters
65 ----------
66 im : `lsst.afw.image.Image`
67 image; the output image has the same dtype, dimensions, and shape
68 and its expectation value is the value of ``im`` at each pixel
70 Returns
71 -------
72 noiseIm : `lsst.afw.image.Image`
73 Newly constructed image instance, same type as ``im``.
75 Notes
76 -----
77 - Warning: This uses an undocumented numpy API (the documented API
78 uses a single float expectation value instead of an array).
80 - Uses numpy.random; you may wish to call numpy.random.seed first.
81 """
82 import numpy.random as rand
83 imArr = im.getArray()
84 noiseIm = im.Factory(im.getBBox())
85 noiseArr = noiseIm.getArray()
87 intNoiseArr = rand.poisson(np.where(np.isfinite(imArr), imArr, 0.0))
89 noiseArr[:, :] = intNoiseArr.astype(noiseArr.dtype)
90 return noiseIm
92#######
93# Make fake images for testing; one is a delta function (or narrow
94# gaussian) and the other is a convolution of this with a spatially
95# varying kernel.
96#######
99def fakeCoeffs():
100 kCoeffs = ((1.0, 0.0, 0.0),
101 (0.005, -0.000001, 0.000001),
102 (0.005, 0.000004, 0.000004),
103 (-0.001, -0.000030, 0.000030),
104 (-0.001, 0.000015, 0.000015),
105 (-0.005, -0.000050, 0.000050))
106 return kCoeffs
109def makeFakeKernelSet(sizeCell=128, nCell=3,
110 deltaFunctionCounts=1.e4, tGaussianWidth=1.0,
111 addNoise=True, bgValue=100., display=False):
112 """Generate test template and science images with sources.
114 Parameters
115 ----------
116 sizeCell : `int`, optional
117 Size of the square spatial cells in pixels.
118 nCell : `int`, optional
119 Number of adjacent spatial cells in both direction in both images.
120 deltaFunctionCounts : `float`, optional
121 Flux value for the template image sources.
122 tGaussianWidth : `float`, optional
123 Sigma of the generated Gaussian PSF sources in the template image.
124 addNoise : `bool`, optional
125 If `True`, Poisson noise is added to both the generated template
126 and science images.
127 bgValue : `float`, optional
128 Background level to be added to the generated science image.
129 display : `bool`, optional
130 If `True` displays the generated template and science images by
131 `lsst.afw.display.Display`.
133 Notes
134 -----
135 - The generated images consist of adjacent ``nCell x nCell`` cells, each
136 of pixel size ``sizeCell x sizeCell``.
137 - The sources in the science image are generated by convolving the
138 template by ``sKernel``. ``sKernel`` is a spatial `LinearCombinationKernel`
139 of hard wired kernel bases functions. The linear combination has first
140 order polynomial spatial dependence with polynomial parameters from ``fakeCoeffs()``.
141 - The template image sources are generated in the center of each spatial
142 cell from one pixel, set to `deltaFunctionCounts` counts, then convolved
143 by a 2D Gaussian with sigma of `tGaussianWidth` along each axis.
144 - The sources are also returned in ``kernelCellSet`` each source is "detected"
145 exactly at the center of a cell.
147 Returns
148 -------
149 tMi : `lsst.afw.image.MaskedImage`
150 Generated template image.
151 sMi : `lsst.afw.image.MaskedImage`
152 Generated science image.
153 sKernel : `lsst.afw.math.LinearCombinationKernel`
154 The spatial kernel used to generate the sources in the science image.
155 kernelCellSet : `lsst.afw.math.SpatialCellSet`
156 Cell grid of `lsst.afw.math.SpatialCell` instances, containing
157 `lsst.ip.diffim.KernelCandidate` instances around all the generated sources
158 in the science image.
159 configFake : `lsst.ip.diffim.ImagePsfMatchConfig`
160 Config instance used in the image generation.
161 """
162 from . import imagePsfMatch
163 configFake = imagePsfMatch.ImagePsfMatchConfig()
164 configFake.kernel.name = "AL"
165 subconfigFake = configFake.kernel.active
166 subconfigFake.alardNGauss = 1
167 subconfigFake.alardSigGauss = [2.5, ]
168 subconfigFake.alardDegGauss = [2, ]
169 subconfigFake.sizeCellX = sizeCell
170 subconfigFake.sizeCellY = sizeCell
171 subconfigFake.spatialKernelOrder = 1
172 subconfigFake.spatialModelType = "polynomial"
173 subconfigFake.singleKernelClipping = False # variance is a hack
174 subconfigFake.spatialKernelClipping = False # variance is a hack
175 if bgValue > 0.0:
176 subconfigFake.fitForBackground = True
178 psFake = pexConfig.makePropertySet(subconfigFake)
180 basisList = makeKernelBasisList(subconfigFake)
181 kSize = subconfigFake.kernelSize
183 # This sets the final extent of each convolved delta function
184 gaussKernelWidth = sizeCell//2
186 # This sets the scale over which pixels are correlated in the
187 # spatial convolution; should be at least as big as the kernel you
188 # are trying to fit for
189 spatialKernelWidth = kSize
191 # Number of bad pixels due to convolutions
192 border = (gaussKernelWidth + spatialKernelWidth)//2
194 # Make a fake image with a matrix of delta functions
195 totalSize = nCell*sizeCell + 2*border
196 tim = afwImage.ImageF(geom.Extent2I(totalSize, totalSize))
197 for x in range(nCell):
198 for y in range(nCell):
199 tim[x*sizeCell + sizeCell//2 + border - 1,
200 y*sizeCell + sizeCell//2 + border - 1,
201 afwImage.LOCAL] = deltaFunctionCounts
203 # Turn this into stars with a narrow width; conserve counts
204 gaussFunction = afwMath.GaussianFunction2D(tGaussianWidth, tGaussianWidth)
205 gaussKernel = afwMath.AnalyticKernel(gaussKernelWidth, gaussKernelWidth, gaussFunction)
206 cim = afwImage.ImageF(tim.getDimensions())
207 afwMath.convolve(cim, tim, gaussKernel, True)
208 tim = cim
210 # Trim off border pixels
211 bbox = gaussKernel.shrinkBBox(tim.getBBox(afwImage.LOCAL))
212 tim = afwImage.ImageF(tim, bbox, afwImage.LOCAL)
214 # Now make a science image which is this convolved with some
215 # spatial function. Use input basis list.
216 polyFunc = afwMath.PolynomialFunction2D(1)
217 kCoeffs = fakeCoeffs()
218 nToUse = min(len(kCoeffs), len(basisList))
220 # Make the full convolved science image
221 sKernel = afwMath.LinearCombinationKernel(basisList[:nToUse], polyFunc)
222 sKernel.setSpatialParameters(kCoeffs[:nToUse])
223 sim = afwImage.ImageF(tim.getDimensions())
224 afwMath.convolve(sim, tim, sKernel, True)
226 # Get the good subregion
227 bbox = sKernel.shrinkBBox(sim.getBBox(afwImage.LOCAL))
229 # Add background
230 sim += bgValue
232 # Watch out for negative values
233 tim += 2*np.abs(np.min(tim.getArray()))
235 # Add noise?
236 if addNoise:
237 sim = makePoissonNoiseImage(sim)
238 tim = makePoissonNoiseImage(tim)
240 # And turn into MaskedImages
241 sim = afwImage.ImageF(sim, bbox, afwImage.LOCAL)
242 svar = afwImage.ImageF(sim, True)
243 smask = afwImage.Mask(sim.getDimensions())
244 smask.set(0x0)
245 sMi = afwImage.MaskedImageF(sim, smask, svar)
247 tim = afwImage.ImageF(tim, bbox, afwImage.LOCAL)
248 tvar = afwImage.ImageF(tim, True)
249 tmask = afwImage.Mask(tim.getDimensions())
250 tmask.set(0x0)
251 tMi = afwImage.MaskedImageF(tim, tmask, tvar)
253 if display:
254 import lsst.afw.display as afwDisplay
255 afwDisplay.Display(frame=1).mtv(tMi)
256 afwDisplay.Display(frame=2).mtv(sMi)
258 # Finally, make a kernelSet from these 2 images
259 kernelCellSet = afwMath.SpatialCellSet(geom.Box2I(geom.Point2I(0, 0),
260 geom.Extent2I(sizeCell*nCell,
261 sizeCell*nCell)),
262 sizeCell,
263 sizeCell)
264 stampHalfWidth = 2*kSize
265 for x in range(nCell):
266 for y in range(nCell):
267 xCoord = x*sizeCell + sizeCell//2
268 yCoord = y*sizeCell + sizeCell//2
269 p0 = geom.Point2I(xCoord - stampHalfWidth,
270 yCoord - stampHalfWidth)
271 p1 = geom.Point2I(xCoord + stampHalfWidth,
272 yCoord + stampHalfWidth)
273 bbox = geom.Box2I(p0, p1)
274 tsi = afwImage.MaskedImageF(tMi, bbox, origin=afwImage.LOCAL)
275 ssi = afwImage.MaskedImageF(sMi, bbox, origin=afwImage.LOCAL)
277 kc = diffimLib.makeKernelCandidate(xCoord, yCoord, tsi, ssi, psFake)
278 kernelCellSet.insertCandidate(kc)
280 tMi.setXY0(0, 0)
281 sMi.setXY0(0, 0)
282 return tMi, sMi, sKernel, kernelCellSet, configFake
285#######
286# Background subtraction for ip_diffim
287#######
289def backgroundSubtract(config, maskedImages):
290 """Subtract the background from masked images.
292 Parameters
293 ----------
294 config : TODO: DM-17458
295 TODO: DM-17458
296 maskedImages : `list` of `lsst.afw.image.MaskedImage`
297 TODO: DM-17458
299 Returns
300 -------
301 TODO: DM-17458
302 TODO: DM-17458
303 """
304 backgrounds = []
305 t0 = time.time()
306 algorithm = config.algorithm
307 binsize = config.binSize
308 undersample = config.undersampleStyle
309 bctrl = afwMath.BackgroundControl(algorithm)
310 bctrl.setUndersampleStyle(undersample)
311 for maskedImage in maskedImages:
312 bctrl.setNxSample(maskedImage.getWidth()//binsize + 1)
313 bctrl.setNySample(maskedImage.getHeight()//binsize + 1)
314 image = maskedImage.getImage()
315 backobj = afwMath.makeBackground(image, bctrl)
317 image -= backobj.getImageF()
318 backgrounds.append(backobj.getImageF())
319 del backobj
321 t1 = time.time()
322 logger = Log.getLogger("ip.diffim.backgroundSubtract")
323 logger.debug("Total time for background subtraction : %.2f s", (t1 - t0))
324 return backgrounds
326#######
327# More coarse debugging
328#######
331def writeKernelCellSet(kernelCellSet, psfMatchingKernel, backgroundModel, outdir):
332 """TODO: DM-17458
334 Parameters
335 ----------
336 kernelCellSet : TODO: DM-17458
337 TODO: DM-17458
338 psfMatchingKernel : TODO: DM-17458
339 TODO: DM-17458
340 backgroundModel : TODO: DM-17458
341 TODO: DM-17458
342 outdir : TODO: DM-17458
343 TODO: DM-17458
344 """
345 if not os.path.isdir(outdir):
346 os.makedirs(outdir)
348 for cell in kernelCellSet.getCellList():
349 for cand in cell.begin(False): # False = include bad candidates
350 if cand.getStatus() == afwMath.SpatialCellCandidate.GOOD:
351 xCand = int(cand.getXCenter())
352 yCand = int(cand.getYCenter())
353 idCand = cand.getId()
354 diffIm = cand.getDifferenceImage(diffimLib.KernelCandidateF.ORIG)
355 kernel = cand.getKernelImage(diffimLib.KernelCandidateF.ORIG)
356 diffIm.writeFits(os.path.join(outdir, 'diffim_c%d_x%d_y%d.fits' % (idCand, xCand, yCand)))
357 kernel.writeFits(os.path.join(outdir, 'kernel_c%d_x%d_y%d.fits' % (idCand, xCand, yCand)))
359 # Diffim from spatial model
360 ski = afwImage.ImageD(kernel.getDimensions())
361 psfMatchingKernel.computeImage(ski, False, xCand, yCand)
362 sk = afwMath.FixedKernel(ski)
363 sbg = backgroundModel(xCand, yCand)
364 sdmi = cand.getDifferenceImage(sk, sbg)
365 sdmi.writeFits(os.path.join(outdir, 'sdiffim_c%d_x%d_y%d.fits' % (idCand, xCand, yCand)))
367#######
368# Converting types
369#######
372def sourceToFootprintList(candidateInList, templateExposure, scienceExposure, kernelSize, config, log):
373 """Convert a list of sources for the PSF-matching Kernel to Footprints.
375 Parameters
376 ----------
377 candidateInList : TODO: DM-17458
378 Input list of Sources
379 templateExposure : TODO: DM-17458
380 Template image, to be checked for Mask bits in Source Footprint
381 scienceExposure : TODO: DM-17458
382 Science image, to be checked for Mask bits in Source Footprint
383 kernelSize : TODO: DM-17458
384 TODO: DM-17458
385 config : TODO: DM-17458
386 Config that defines the Mask planes that indicate an invalid Source and Bbox grow radius
387 log : TODO: DM-17458
388 Log for output
390 Returns
391 -------
392 candidateOutList : `list`
393 a list of dicts having a "source" and "footprint" field, to be used for Psf-matching
395 Raises
396 ------
397 RuntimeError
398 TODO: DM-17458
400 Notes
401 -----
402 Takes an input list of Sources that were selected to constrain
403 the Psf-matching Kernel and turns them into a List of Footprints,
404 which are used to seed a set of KernelCandidates. The function
405 checks both the template and science image for masked pixels,
406 rejecting the Source if certain Mask bits (defined in config) are
407 set within the Footprint.
408 """
410 candidateOutList = []
411 fsb = diffimLib.FindSetBitsU()
412 badBitMask = 0
413 for mp in config.badMaskPlanes:
414 badBitMask |= afwImage.Mask.getPlaneBitMask(mp)
415 bbox = scienceExposure.getBBox()
417 # Size to grow Sources
418 if config.scaleByFwhm:
419 fpGrowPix = int(config.fpGrowKernelScaling*kernelSize + 0.5)
420 else:
421 fpGrowPix = config.fpGrowPix
422 log.info("Growing %d kernel candidate stars by %d pixels", len(candidateInList), fpGrowPix)
424 for kernelCandidate in candidateInList:
425 if not type(kernelCandidate) == afwTable.SourceRecord:
426 raise RuntimeError("Candiate not of type afwTable.SourceRecord")
427 bm1 = 0
428 bm2 = 0
429 center = geom.Point2I(scienceExposure.getWcs().skyToPixel(kernelCandidate.getCoord()))
430 if center[0] < bbox.getMinX() or center[0] > bbox.getMaxX():
431 continue
432 if center[1] < bbox.getMinY() or center[1] > bbox.getMaxY():
433 continue
435 xmin = center[0] - fpGrowPix
436 xmax = center[0] + fpGrowPix
437 ymin = center[1] - fpGrowPix
438 ymax = center[1] + fpGrowPix
440 # Keep object centered
441 if (xmin - bbox.getMinX()) < 0:
442 xmax += (xmin - bbox.getMinX())
443 xmin -= (xmin - bbox.getMinX())
444 if (ymin - bbox.getMinY()) < 0:
445 ymax += (ymin - bbox.getMinY())
446 ymin -= (ymin - bbox.getMinY())
447 if (bbox.getMaxX() - xmax) < 0:
448 xmin -= (bbox.getMaxX() - xmax)
449 xmax += (bbox.getMaxX() - xmax)
450 if (bbox.getMaxY() - ymax) < 0:
451 ymin -= (bbox.getMaxY() - ymax)
452 ymax += (bbox.getMaxY() - ymax)
453 if xmin > xmax or ymin > ymax:
454 continue
456 kbbox = geom.Box2I(geom.Point2I(xmin, ymin), geom.Point2I(xmax, ymax))
457 try:
458 fsb.apply(afwImage.MaskedImageF(templateExposure.getMaskedImage(), kbbox, deep=False).getMask())
459 bm1 = fsb.getBits()
460 fsb.apply(afwImage.MaskedImageF(scienceExposure.getMaskedImage(), kbbox, deep=False).getMask())
461 bm2 = fsb.getBits()
462 except Exception:
463 pass
464 else:
465 if not((bm1 & badBitMask) or (bm2 & badBitMask)):
466 candidateOutList.append({'source': kernelCandidate,
467 'footprint': afwDetect.Footprint(afwGeom.SpanSet(kbbox))})
468 log.info("Selected %d / %d sources for KernelCandidacy", len(candidateOutList), len(candidateInList))
469 return candidateOutList
472def sourceTableToCandidateList(sourceTable, templateExposure, scienceExposure, kConfig, dConfig, log,
473 basisList, doBuild=False):
474 """Convert a list of Sources into KernelCandidates.
476 The KernelCandidates are used for fitting the Psf-matching kernel.
478 Parameters
479 ----------
480 sourceTable : TODO: DM-17458
481 TODO: DM-17458
482 templateExposure : TODO: DM-17458
483 TODO: DM-17458
484 scienceExposure : TODO: DM-17458
485 TODO: DM-17458
486 kConfig : TODO: DM-17458
487 TODO: DM-17458
488 dConfig : TODO: DM-17458
489 TODO: DM-17458
490 log : TODO: DM-17458
491 TODO: DM-17458
492 basisList : TODO: DM-17458
493 TODO: DM-17458
494 doBuild : `bool`, optional
495 TODO: DM-17458
497 Returns
498 -------
499 TODO: DM-17458
500 TODO: DM-17458
501 """
502 kernelSize = basisList[0].getWidth()
503 footprintList = sourceToFootprintList(list(sourceTable), templateExposure, scienceExposure,
504 kernelSize, dConfig, log)
505 candList = []
507 if doBuild and not basisList:
508 doBuild = False
509 else:
510 ps = pexConfig.makePropertySet(kConfig)
511 visitor = diffimLib.BuildSingleKernelVisitorF(basisList, ps)
513 ps = pexConfig.makePropertySet(kConfig)
514 for cand in footprintList:
515 bbox = cand['footprint'].getBBox()
516 tmi = afwImage.MaskedImageF(templateExposure.getMaskedImage(), bbox)
517 smi = afwImage.MaskedImageF(scienceExposure.getMaskedImage(), bbox)
518 kCand = diffimLib.makeKernelCandidate(cand['source'], tmi, smi, ps)
519 if doBuild:
520 visitor.processCandidate(kCand)
521 kCand.setStatus(afwMath.SpatialCellCandidate.UNKNOWN)
522 candList.append(kCand)
523 return candList
526#######
527#
528#######
531class NbasisEvaluator(object):
532 """A functor to evaluate the Bayesian Information Criterion for the number of basis sets
533 going into the kernel fitting"""
535 def __init__(self, psfMatchConfig, psfFwhmPixTc, psfFwhmPixTnc):
536 self.psfMatchConfig = psfMatchConfig
537 self.psfFwhmPixTc = psfFwhmPixTc
538 self.psfFwhmPixTnc = psfFwhmPixTnc
539 if not self.psfMatchConfig.kernelBasisSet == "alard-lupton":
540 raise RuntimeError("BIC only implemnted for AL (alard lupton) basis")
542 def __call__(self, kernelCellSet, log):
543 d1, d2, d3 = self.psfMatchConfig.alardDegGauss
544 bicArray = {}
545 for d1i in range(1, d1 + 1):
546 for d2i in range(1, d2 + 1):
547 for d3i in range(1, d3 + 1):
548 dList = [d1i, d2i, d3i]
549 bicConfig = type(self.psfMatchConfig)(self.psfMatchConfig, alardDegGauss=dList)
550 kList = makeKernelBasisList(bicConfig, self.psfFwhmPixTc, self.psfFwhmPixTnc)
551 k = len(kList)
552 visitor = diffimLib.BuildSingleKernelVisitorF(kList,
553 pexConfig.makePropertySet(bicConfig))
554 visitor.setSkipBuilt(False)
555 kernelCellSet.visitCandidates(visitor, bicConfig.nStarPerCell)
557 for cell in kernelCellSet.getCellList():
558 for cand in cell.begin(False): # False = include bad candidates
559 if cand.getStatus() != afwMath.SpatialCellCandidate.GOOD:
560 continue
561 diffIm = cand.getDifferenceImage(diffimLib.KernelCandidateF.RECENT)
562 bbox = cand.getKernel(diffimLib.KernelCandidateF.RECENT).shrinkBBox(
563 diffIm.getBBox(afwImage.LOCAL))
564 diffIm = type(diffIm)(diffIm, bbox, True)
565 chi2 = diffIm.getImage().getArray()**2/diffIm.getVariance().getArray()
566 n = chi2.shape[0]*chi2.shape[1]
567 bic = np.sum(chi2) + k*np.log(n)
568 if cand.getId() not in bicArray:
569 bicArray[cand.getId()] = {}
570 bicArray[cand.getId()][(d1i, d2i, d3i)] = bic
572 bestConfigs = []
573 for candId in bicArray:
574 cconfig, cvals = list(bicArray[candId].keys()), list(bicArray[candId].values())
575 idx = np.argsort(cvals)
576 bestConfig = cconfig[idx[0]]
577 bestConfigs.append(bestConfig)
579 counter = Counter(bestConfigs).most_common(3)
580 log.info("B.I.C. prefers basis complexity %s %d times; %s %d times; %s %d times",
581 counter[0][0], counter[0][1],
582 counter[1][0], counter[1][1],
583 counter[2][0], counter[2][1])
584 return counter[0][0], counter[1][0], counter[2][0]