23 from __future__
import print_function
25 __all__ = [
"PcaPsfDeterminerConfig",
"PcaPsfDeterminerTask"]
27 from builtins
import input
28 from builtins
import zip
29 from builtins
import range
35 import lsst.pex.config
as pexConfig
36 import lsst.pex.exceptions
as pexExceptions
38 import lsst.afw.geom.ellipses
as afwEll
39 import lsst.afw.display.ds9
as ds9
40 import lsst.afw.math
as afwMath
41 from .psfDeterminer
import BasePsfDeterminerTask, psfDeterminerRegistry
42 from .psfCandidate
import PsfCandidateF
43 from .spatialModelPsf
import createKernelFromPsfCandidates, countPsfCandidates, fitSpatialKernelFromPsfCandidates, fitKernelParamsToImage
44 from .pcaPsf
import PcaPsf
45 from .
import utils
as maUtils
49 """Return the number of PSF candidates to be rejected.
51 The number of candidates being rejected on each iteration gradually
52 increases, so that on the Nth of M iterations we reject N/M of the bad
57 numBadCandidates : int
58 Number of bad candidates under consideration.
61 The number of the current PSF iteration.
64 The total number of PSF iterations.
69 Number of candidates to reject.
71 return int(numBadCandidates * (numIter + 1) // totalIter + 0.5)
74 nonLinearSpatialFit = pexConfig.Field(
75 doc=
"Use non-linear fitter for spatial variation of Kernel",
79 nEigenComponents = pexConfig.Field(
80 doc=
"number of eigen components for PSF kernel creation",
84 spatialOrder = pexConfig.Field(
85 doc=
"specify spatial order for PSF kernel creation",
89 sizeCellX = pexConfig.Field(
90 doc=
"size of cell used to determine PSF (pixels, column direction)",
94 check=
lambda x: x >= 10,
96 sizeCellY = pexConfig.Field(
97 doc=
"size of cell used to determine PSF (pixels, row direction)",
99 default=sizeCellX.default,
101 check=
lambda x: x >= 10,
103 nStarPerCell = pexConfig.Field(
104 doc=
"number of stars per psf cell for PSF kernel creation",
108 borderWidth = pexConfig.Field(
109 doc=
"Number of pixels to ignore around the edge of PSF candidate postage stamps",
113 nStarPerCellSpatialFit = pexConfig.Field(
114 doc=
"number of stars per psf Cell for spatial fitting",
118 constantWeight = pexConfig.Field(
119 doc=
"Should each PSF candidate be given the same weight, independent of magnitude?",
123 nIterForPsf = pexConfig.Field(
124 doc=
"number of iterations of PSF candidate star list",
128 tolerance = pexConfig.Field(
129 doc=
"tolerance of spatial fitting",
133 lam = pexConfig.Field(
134 doc=
"floor for variance is lam*data",
138 reducedChi2ForPsfCandidates = pexConfig.Field(
139 doc=
"for psf candidate evaluation",
143 spatialReject = pexConfig.Field(
144 doc=
"Rejection threshold (stdev) for candidates based on spatial fit",
148 pixelThreshold = pexConfig.Field(
149 doc=
"Threshold (stdev) for rejecting extraneous pixels around candidate; applied if positive",
153 doRejectBlends = pexConfig.Field(
154 doc=
"Reject candidates that are blended?",
158 doMaskBlends = pexConfig.Field(
159 doc=
"Mask blends in image?",
167 A measurePsfTask psf estimator
169 ConfigClass = PcaPsfDeterminerConfig
172 def _fitPsf(self, exposure, psfCellSet, kernelSize, nEigenComponents):
173 PsfCandidateF.setPixelThreshold(self.config.pixelThreshold)
174 PsfCandidateF.setMaskBlends(self.config.doMaskBlends)
178 for nEigen
in range(nEigenComponents, 0, -1):
182 psfCellSet, exposure.getDimensions(), exposure.getXY0(), nEigen,
183 self.config.spatialOrder, kernelSize, self.config.nStarPerCell,
184 bool(self.config.constantWeight))
187 except pexExceptions.LengthError
as e:
189 raise IndexError(
"No viable PSF candidates survive")
191 self.log.warn(
"%s: reducing number of eigen components" % e.what())
196 size = kernelSize + 2*self.config.borderWidth
199 for l
in eigenValues]
203 kernel, psfCellSet, bool(self.config.nonLinearSpatialFit),
204 self.config.nStarPerCellSpatialFit, self.config.tolerance, self.config.lam)
208 return psf, eigenValues, nEigen, chi2
210 def determinePsf(self, exposure, psfCandidateList, metadata=None, flagKey=None):
211 """!Determine a PCA PSF model for an exposure given a list of PSF candidates
213 \param[in] exposure exposure containing the psf candidates (lsst.afw.image.Exposure)
214 \param[in] psfCandidateList a sequence of PSF candidates (each an lsst.meas.algorithms.PsfCandidate);
215 typically obtained by detecting sources and then running them through a star selector
216 \param[in,out] metadata a home for interesting tidbits of information
217 \param[in] flagKey schema key used to mark sources actually used in PSF determination
220 - psf: the measured PSF, an lsst.meas.algorithms.PcaPsf
221 - cellSet: an lsst.afw.math.SpatialCellSet containing the PSF candidates
224 display = lsstDebug.Info(__name__).display
225 displayExposure = lsstDebug.Info(__name__).displayExposure
226 displayPsfCandidates = lsstDebug.Info(__name__).displayPsfCandidates
227 displayIterations = lsstDebug.Info(__name__).displayIterations
228 displayPsfComponents = lsstDebug.Info(__name__).displayPsfComponents
229 displayResiduals = lsstDebug.Info(__name__).displayResiduals
230 displayPsfMosaic = lsstDebug.Info(__name__).displayPsfMosaic
232 matchKernelAmplitudes = lsstDebug.Info(__name__).matchKernelAmplitudes
234 keepMatplotlibPlots = lsstDebug.Info(__name__).keepMatplotlibPlots
235 displayPsfSpatialModel = lsstDebug.Info(__name__).displayPsfSpatialModel
236 showBadCandidates = lsstDebug.Info(__name__).showBadCandidates
238 normalizeResiduals = lsstDebug.Info(__name__).normalizeResiduals
239 pause = lsstDebug.Info(__name__).pause
244 mi = exposure.getMaskedImage()
246 if len(psfCandidateList) == 0:
247 raise RuntimeError(
"No PSF candidates supplied.")
251 psfCellSet = afwMath.SpatialCellSet(bbox, self.config.sizeCellX, self.config.sizeCellY)
253 for i, psfCandidate
in enumerate(psfCandidateList):
254 if psfCandidate.getSource().getPsfFluxFlag():
258 psfCellSet.insertCandidate(psfCandidate)
259 except Exception
as e:
260 self.log.debug(
"Skipping PSF candidate %d of %d: %s", i, len(psfCandidateList), e)
262 source = psfCandidate.getSource()
264 quad = afwEll.Quadrupole(source.getIxx(), source.getIyy(), source.getIxy())
265 axes = afwEll.Axes(quad)
266 sizes.append(axes.getA())
268 raise RuntimeError(
"No usable PSF candidates supplied")
269 nEigenComponents = self.config.nEigenComponents
271 if self.config.kernelSize >= 15:
272 self.log.warn(
"WARNING: NOT scaling kernelSize by stellar quadrupole moment " +
273 "because config.kernelSize=%s >= 15; using config.kernelSize as as the width, instead",
274 self.config.kernelSize)
275 actualKernelSize = int(self.config.kernelSize)
277 medSize = numpy.median(sizes)
278 actualKernelSize = 2 * int(self.config.kernelSize * math.sqrt(medSize) + 0.5) + 1
279 if actualKernelSize < self.config.kernelSizeMin:
280 actualKernelSize = self.config.kernelSizeMin
281 if actualKernelSize > self.config.kernelSizeMax:
282 actualKernelSize = self.config.kernelSizeMax
285 print(
"Median size=%s" % (medSize,))
286 self.log.trace(
"Kernel size=%s", actualKernelSize)
289 psfCandidateList[0].setHeight(actualKernelSize)
290 psfCandidateList[0].setWidth(actualKernelSize)
292 if self.config.doRejectBlends:
294 blendedCandidates = []
296 if len(cand.getSource().getFootprint().getPeaks()) > 1:
297 blendedCandidates.append((cell, cand))
300 print(
"Removing %d blended Psf candidates" % len(blendedCandidates))
301 for cell, cand
in blendedCandidates:
302 cell.removeCandidate(cand)
304 raise RuntimeError(
"All PSF candidates removed as blends")
309 ds9.mtv(exposure, frame=frame, title=
"psf determination")
310 maUtils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell,
311 symb=
"o", ctype=ds9.CYAN, ctypeUnused=ds9.YELLOW,
318 for iterNum
in range(self.config.nIterForPsf):
319 if display
and displayPsfCandidates:
321 import lsst.afw.display.utils
as displayUtils
324 for cell
in psfCellSet.getCellList():
325 for cand
in cell.begin(
not showBadCandidates):
327 im = cand.getMaskedImage()
329 chi2 = cand.getChi2()
333 stamps.append((im,
"%d%s" %
334 (maUtils.splitId(cand.getSource().getId(),
True)[
"objId"], chi2),
336 except Exception
as e:
340 print(
"WARNING: No PSF candidates to show; try setting showBadCandidates=True")
342 mos = displayUtils.Mosaic()
343 for im, label, status
in stamps:
344 im = type(im)(im,
True)
346 im /= afwMath.makeStatistics(im, afwMath.MAX).getValue()
347 except NotImplementedError:
350 mos.append(im, label,
351 ds9.GREEN
if status == afwMath.SpatialCellCandidate.GOOD
else
352 ds9.YELLOW
if status == afwMath.SpatialCellCandidate.UNKNOWN
else ds9.RED)
354 mos.makeMosaic(frame=8, title=
"Psf Candidates")
363 psf, eigenValues, nEigenComponents, fitChi2 = \
364 self.
_fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents)
369 for cell
in psfCellSet.getCellList():
371 for cand
in cell.begin(
False):
372 cand.setStatus(afwMath.SpatialCellCandidate.UNKNOWN)
373 rchi2 = cand.getChi2()
374 if not numpy.isfinite(rchi2)
or rchi2 <= 0:
376 awfulCandidates.append(cand)
378 self.log.debug(
"chi^2=%s; id=%s",
379 cand.getChi2(), cand.getSource().getId())
380 for cand
in awfulCandidates:
382 print(
"Removing bad candidate: id=%d, chi^2=%f" % \
383 (cand.getSource().getId(), cand.getChi2()))
384 cell.removeCandidate(cand)
389 badCandidates = list()
390 for cell
in psfCellSet.getCellList():
391 for cand
in cell.begin(
False):
392 rchi2 = cand.getChi2()
394 if rchi2 > self.config.reducedChi2ForPsfCandidates:
395 badCandidates.append(cand)
397 badCandidates.sort(key=
lambda x: x.getChi2(), reverse=
True)
399 self.config.nIterForPsf)
400 for i, c
in zip(range(numBad), badCandidates):
406 print(
"Chi^2 clipping %-4d %.2g" % (c.getSource().getId(), chi2))
407 c.setStatus(afwMath.SpatialCellCandidate.BAD)
420 kernel = psf.getKernel()
421 noSpatialKernel = psf.getKernel()
422 for cell
in psfCellSet.getCellList():
423 for cand
in cell.begin(
False):
424 candCenter = afwGeom.PointD(cand.getXCenter(), cand.getYCenter())
426 im = cand.getMaskedImage(kernel.getWidth(), kernel.getHeight())
427 except Exception
as e:
434 for p, k
in zip(params, kernels):
435 amp += p * k.getSum()
437 predict = [kernel.getSpatialFunction(k)(candCenter.getX(), candCenter.getY())
for
438 k
in range(kernel.getNKernelParameters())]
442 residuals.append([a / amp - p
for a, p
in zip(params, predict)])
443 candidates.append(cand)
445 residuals = numpy.array(residuals)
447 for k
in range(kernel.getNKernelParameters()):
450 mean = residuals[:, k].mean()
451 rms = residuals[:, k].
std()
454 sr = numpy.sort(residuals[:, k])
455 mean = sr[int(0.5*len(sr))]
if len(sr) % 2
else \
456 0.5 * (sr[int(0.5*len(sr))] + sr[int(0.5*len(sr))+1])
457 rms = 0.74 * (sr[int(0.75*len(sr))] - sr[int(0.25*len(sr))])
459 stats = afwMath.makeStatistics(residuals[:, k], afwMath.MEANCLIP | afwMath.STDEVCLIP)
460 mean = stats.getValue(afwMath.MEANCLIP)
461 rms = stats.getValue(afwMath.STDEVCLIP)
463 rms = max(1.0e-4, rms)
466 print(
"Mean for component %d is %f" % (k, mean))
467 print(
"RMS for component %d is %f" % (k, rms))
468 badCandidates = list()
469 for i, cand
in enumerate(candidates):
470 if numpy.fabs(residuals[i, k] - mean) > self.config.spatialReject * rms:
471 badCandidates.append(i)
473 badCandidates.sort(key=
lambda x: numpy.fabs(residuals[x, k] - mean), reverse=
True)
476 self.config.nIterForPsf)
478 for i, c
in zip(range(min(len(badCandidates), numBad)), badCandidates):
481 print(
"Spatial clipping %d (%f,%f) based on %d: %f vs %f" % \
482 (cand.getSource().getId(), cand.getXCenter(), cand.getYCenter(), k,
483 residuals[badCandidates[i], k], self.config.spatialReject * rms))
484 cand.setStatus(afwMath.SpatialCellCandidate.BAD)
489 if display
and displayIterations:
492 ds9.erase(frame=frame)
493 maUtils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, showChi2=
True,
494 symb=
"o", size=8, frame=frame,
495 ctype=ds9.YELLOW, ctypeBad=ds9.RED, ctypeUnused=ds9.MAGENTA)
496 if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell:
497 maUtils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCellSpatialFit,
498 symb=
"o", size=10, frame=frame,
499 ctype=ds9.YELLOW, ctypeBad=ds9.RED)
503 maUtils.showPsfCandidates(exposure, psfCellSet, psf=psf, frame=4,
504 normalize=normalizeResiduals,
505 showBadCandidates=showBadCandidates)
506 maUtils.showPsfCandidates(exposure, psfCellSet, psf=psf, frame=5,
507 normalize=normalizeResiduals,
508 showBadCandidates=showBadCandidates,
511 if not showBadCandidates:
512 showBadCandidates =
True
516 if displayPsfComponents:
517 maUtils.showPsf(psf, eigenValues, frame=6)
519 maUtils.showPsfMosaic(exposure, psf, frame=7, showFwhm=
True)
520 ds9.scale(
'linear', 0, 1, frame=7)
521 if displayPsfSpatialModel:
522 maUtils.plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=
True,
523 matchKernelAmplitudes=matchKernelAmplitudes,
524 keepPlots=keepMatplotlibPlots)
529 reply = input(
"Next iteration? [ynchpqQs] ").strip()
533 reply = reply.split()
535 reply, args = reply[0], reply[1:]
539 if reply
in (
"",
"c",
"h",
"n",
"p",
"q",
"Q",
"s",
"y"):
543 print(
"c[ontinue without prompting] h[elp] n[o] p[db] q[uit displaying] " \
544 "s[ave fileName] y[es]")
554 fileName = args.pop(0)
556 print(
"Please provide a filename")
559 print(
"Saving to %s" % fileName)
560 maUtils.saveSpatialCellSet(psfCellSet, fileName=fileName)
564 print(
"Unrecognised response: %s" % reply, file=sys.stderr)
570 psf, eigenValues, nEigenComponents, fitChi2 = \
571 self.
_fitPsf(exposure, psfCellSet, actualKernelSize, nEigenComponents)
576 if display
and reply !=
"n":
578 maUtils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCell, showChi2=
True,
579 symb=
"o", ctype=ds9.YELLOW, ctypeBad=ds9.RED, size=8, frame=frame)
580 if self.config.nStarPerCellSpatialFit != self.config.nStarPerCell:
581 maUtils.showPsfSpatialCells(exposure, psfCellSet, self.config.nStarPerCellSpatialFit,
582 symb=
"o", ctype=ds9.YELLOW, ctypeBad=ds9.RED,
583 size=10, frame=frame)
585 maUtils.showPsfCandidates(exposure, psfCellSet, psf=psf, frame=4,
586 normalize=normalizeResiduals,
587 showBadCandidates=showBadCandidates)
589 if displayPsfComponents:
590 maUtils.showPsf(psf, eigenValues, frame=6)
593 maUtils.showPsfMosaic(exposure, psf, frame=7, showFwhm=
True)
594 ds9.scale(
"linear", 0, 1, frame=7)
595 if displayPsfSpatialModel:
596 maUtils.plotPsfSpatialModel(exposure, psf, psfCellSet, showBadCandidates=
True,
597 matchKernelAmplitudes=matchKernelAmplitudes,
598 keepPlots=keepMatplotlibPlots)
610 for cell
in psfCellSet.getCellList():
611 for cand
in cell.begin(
False):
614 for cand
in cell.begin(
True):
615 src = cand.getSource()
616 if flagKey
is not None:
617 src.set(flagKey,
True)
625 if metadata
is not None:
626 metadata.set(
"spatialFitChi2", fitChi2)
627 metadata.set(
"numGoodStars", numGoodStars)
628 metadata.set(
"numAvailStars", numAvailStars)
629 metadata.set(
"avgX", avgX)
630 metadata.set(
"avgY", avgY)
632 psf = PcaPsf(psf.getKernel(), afwGeom.Point2D(avgX, avgY))
634 return psf, psfCellSet
638 """!Generator for Psf candidates
640 This allows two 'for' loops to be reduced to one.
642 \param psfCellSet SpatialCellSet of PSF candidates
643 \param ignoreBad Ignore candidates flagged as BAD?
644 \return SpatialCell, PsfCandidate
646 for cell
in psfCellSet.getCellList():
647 for cand
in cell.begin(ignoreBad):
650 psfDeterminerRegistry.register(
"pca", PcaPsfDeterminerTask)
int countPsfCandidates(lsst::afw::math::SpatialCellSet const &psfCells, int const nStarPerCell=-1)
def numCandidatesToReject
A measurePsfTask psf estimator.
std::pair< std::vector< double >, lsst::afw::math::KernelList > fitKernelParamsToImage(lsst::afw::math::LinearCombinationKernel const &kernel, Image const &image, lsst::afw::geom::Point2D const &pos)
std::pair< bool, double > fitSpatialKernelFromPsfCandidates(lsst::afw::math::Kernel *kernel, lsst::afw::math::SpatialCellSet const &psfCells, int const nStarPerCell=-1, double const tolerance=1e-5, double const lambda=0.0)
std::pair< std::shared_ptr< lsst::afw::math::LinearCombinationKernel >, std::vector< double > > createKernelFromPsfCandidates(lsst::afw::math::SpatialCellSet const &psfCells, lsst::afw::geom::Extent2I const &dims, lsst::afw::geom::Point2I const &xy0, int const nEigenComponents, int const spatialOrder, int const ksize, int const nStarPerCell=-1, bool const constantWeight=true, int const border=3)
def determinePsf
Determine a PCA PSF model for an exposure given a list of PSF candidates.
def candidatesIter
Generator for Psf candidates.