664 def _diagnostic(self, kernelCellSet, spatialSolution, spatialKernel, spatialBg):
665 """Provide logging diagnostics on quality of spatial kernel fit
669 kernelCellSet : `lsst.afw.math.SpatialCellSet`
670 Cellset that contains the KernelCandidates used in the fitting
671 spatialSolution : `lsst.ip.diffim.SpatialKernelSolution`
672 KernelSolution of best-fit
673 spatialKernel : `lsst.afw.math.LinearCombinationKernel`
674 Best-fit spatial Kernel model
675 spatialBg : `lsst.afw.math.Function2D`
676 Best-fit spatial background model
678 # What is the final kernel sum
679 kImage = afwImage.ImageD(spatialKernel.getDimensions())
680 kSum = spatialKernel.computeImage(kImage, False)
681 self.log.info("Final spatial kernel sum %.3f", kSum)
683 # Look at how well conditioned the matrix is
684 conditionNum = spatialSolution.getConditionNumber(
685 getattr(diffimLib.KernelSolution, self.kConfig.conditionNumberType))
686 self.log.info("Spatial model condition number %.3e", conditionNum)
688 if conditionNum < 0.0:
689 self.log.warning("Condition number is negative (%.3e)", conditionNum)
690 if conditionNum > self.kConfig.maxSpatialConditionNumber:
691 self.log.warning("Spatial solution exceeds max condition number (%.3e > %.3e)",
692 conditionNum, self.kConfig.maxSpatialConditionNumber)
694 self.metadata["spatialConditionNum"] = conditionNum
695 self.metadata["spatialKernelSum"] = kSum
697 # Look at how well the solution is constrained
698 nBasisKernels = spatialKernel.getNBasisKernels()
699 nKernelTerms = spatialKernel.getNSpatialParameters()
700 if nKernelTerms == 0: # order 0
704 nBgTerms = spatialBg.getNParameters()
706 if spatialBg.getParameters()[0] == 0.0:
712 for cell in kernelCellSet.getCellList():
713 for cand in cell.begin(False): # False = include bad candidates
715 if cand.getStatus() == afwMath.SpatialCellCandidate.GOOD:
717 if cand.getStatus() == afwMath.SpatialCellCandidate.BAD:
720 self.log.info("Doing stats of kernel candidates used in the spatial fit.")
722 # Counting statistics
724 self.log.warning("Many more candidates rejected than accepted; %d total, %d rejected, %d used",
727 self.log.info("%d candidates total, %d rejected, %d used", nTot, nBad, nGood)
729 # Some judgements on the quality of the spatial models
730 if nGood < nKernelTerms:
731 self.log.warning("Spatial kernel model underconstrained; %d candidates, %d terms, %d bases",
732 nGood, nKernelTerms, nBasisKernels)
733 self.log.warning("Consider lowering the spatial order")
734 elif nGood <= 2*nKernelTerms:
735 self.log.warning("Spatial kernel model poorly constrained; %d candidates, %d terms, %d bases",
736 nGood, nKernelTerms, nBasisKernels)
737 self.log.warning("Consider lowering the spatial order")
739 self.log.info("Spatial kernel model well constrained; %d candidates, %d terms, %d bases",
740 nGood, nKernelTerms, nBasisKernels)
743 self.log.warning("Spatial background model underconstrained; %d candidates, %d terms",
745 self.log.warning("Consider lowering the spatial order")
746 elif nGood <= 2*nBgTerms:
747 self.log.warning("Spatial background model poorly constrained; %d candidates, %d terms",
749 self.log.warning("Consider lowering the spatial order")
751 self.log.info("Spatial background model appears well constrained; %d candidates, %d terms",
754 def _displayDebug(self, kernelCellSet, spatialKernel, spatialBackground):
755 """Provide visualization of the inputs and ouputs to the Psf-matching code
759 kernelCellSet : `lsst.afw.math.SpatialCellSet`
760 The SpatialCellSet used in determining the matching kernel and background
761 spatialKernel : `lsst.afw.math.LinearCombinationKernel`
762 Spatially varying Psf-matching kernel
763 spatialBackground : `lsst.afw.math.Function2D`
764 Spatially varying background-matching function
767 displayCandidates = lsstDebug.Info(__name__).displayCandidates
768 displayKernelBasis = lsstDebug.Info(__name__).displayKernelBasis
769 displayKernelMosaic = lsstDebug.Info(__name__).displayKernelMosaic
770 plotKernelSpatialModel = lsstDebug.Info(__name__).plotKernelSpatialModel
771 plotKernelCoefficients = lsstDebug.Info(__name__).plotKernelCoefficients
772 showBadCandidates = lsstDebug.Info(__name__).showBadCandidates
773 maskTransparency = lsstDebug.Info(__name__).maskTransparency
774 if not maskTransparency:
776 afwDisplay.setDefaultMaskTransparency(maskTransparency)
778 if displayCandidates:
779 diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground,
780 frame=lsstDebug.frame,
781 showBadCandidates=showBadCandidates)
783 diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground,
784 frame=lsstDebug.frame,
785 showBadCandidates=showBadCandidates,
788 diutils.showKernelCandidates(kernelCellSet, kernel=spatialKernel, background=spatialBackground,
789 frame=lsstDebug.frame,
790 showBadCandidates=showBadCandidates,
794 if displayKernelBasis:
795 diutils.showKernelBasis(spatialKernel, frame=lsstDebug.frame)
798 if displayKernelMosaic:
799 diutils.showKernelMosaic(kernelCellSet.getBBox(), spatialKernel, frame=lsstDebug.frame)
802 if plotKernelSpatialModel:
803 diutils.plotKernelSpatialModel(spatialKernel, kernelCellSet, showBadCandidates=showBadCandidates)
805 if plotKernelCoefficients:
806 diutils.plotKernelCoefficients(spatialKernel, kernelCellSet)
808 def _createPcaBasis(self, kernelCellSet, nStarPerCell, ps):
809 """Create Principal Component basis
811 If a principal component analysis is requested, typically when using a delta function basis,
812 perform the PCA here and return a new basis list containing the new principal components.
816 kernelCellSet : `lsst.afw.math.SpatialCellSet`
817 a SpatialCellSet containing KernelCandidates, from which components are derived
819 the number of stars per cell to visit when doing the PCA
820 ps : `lsst.daf.base.PropertySet`
821 input property set controlling the single kernel visitor
826 number of KernelCandidates rejected during PCA loop
827 spatialBasisList : `list` of `lsst.afw.math.kernel.FixedKernel`
828 basis list containing the principal shapes as Kernels
833 If the Eigenvalues sum to zero.
835 nComponents = self.kConfig.numPrincipalComponents
836 imagePca = diffimLib.KernelPcaD()
837 importStarVisitor = diffimLib.KernelPcaVisitorF(imagePca)
838 kernelCellSet.visitCandidates(importStarVisitor, nStarPerCell)
839 if self.kConfig.subtractMeanForPca:
840 importStarVisitor.subtractMean()
843 eigenValues = imagePca.getEigenValues()
844 pcaBasisList = importStarVisitor.getEigenKernels()
846 eSum = np.sum(eigenValues)
848 raise RuntimeError("Eigenvalues sum to zero")
849 trace_logger = getTraceLogger(self.log.getChild("_solve"), 5)
850 for j in range(len(eigenValues)):
851 trace_logger.debug("Eigenvalue %d : %f (%f)", j, eigenValues[j], eigenValues[j]/eSum)
853 nToUse = min(nComponents, len(eigenValues))
855 for j in range(nToUse):
857 kimage = afwImage.ImageD(pcaBasisList[j].getDimensions())
858 pcaBasisList[j].computeImage(kimage, False)
859 if not (True in np.isnan(kimage.array)):
860 trimBasisList.append(pcaBasisList[j])
862 # Put all the power in the first kernel, which will not vary spatially
863 spatialBasisList = diffimLib.renormalizeKernelList(trimBasisList)
865 # New Kernel visitor for this new basis list (no regularization explicitly)
866 singlekvPca = diffimLib.BuildSingleKernelVisitorF(spatialBasisList, ps)
867 singlekvPca.setSkipBuilt(False)
868 kernelCellSet.visitCandidates(singlekvPca, nStarPerCell)
869 singlekvPca.setSkipBuilt(True)
870 nRejectedPca = singlekvPca.getNRejected()
872 return nRejectedPca, spatialBasisList
881 def _solve(self, kernelCellSet, basisList, returnOnExcept=False):
882 """Solve for the PSF matching kernel
886 kernelCellSet : `lsst.afw.math.SpatialCellSet`
887 a SpatialCellSet to use in determining the matching kernel
888 (typically as provided by _buildCellSet)
889 basisList : `list` of `lsst.afw.math.kernel.FixedKernel`
890 list of Kernels to be used in the decomposition of the spatially varying kernel
891 (typically as provided by makeKernelBasisList)
892 returnOnExcept : `bool`, optional
893 if True then return (None, None) if an error occurs, else raise the exception
897 psfMatchingKernel : `lsst.afw.math.LinearCombinationKernel`
898 Spatially varying Psf-matching kernel
899 backgroundModel : `lsst.afw.math.Function2D`
900 Spatially varying background-matching function
905 If unable to determine PSF matching kernel and ``returnOnExcept==False``.
909 display = lsstDebug.Info(__name__).display
911 maxSpatialIterations = self.kConfig.maxSpatialIterations
912 nStarPerCell = self.kConfig.nStarPerCell
913 usePcaForSpatialKernel = self.kConfig.usePcaForSpatialKernel
915 # Visitor for the single kernel fit
916 ps = pexConfig.makePropertySet(self.kConfig)
917 if self.useRegularization:
918 singlekv = diffimLib.BuildSingleKernelVisitorF(basisList, ps, self.hMat)
920 singlekv = diffimLib.BuildSingleKernelVisitorF(basisList, ps)
922 # Visitor for the kernel sum rejection
923 ksv = diffimLib.KernelSumVisitorF(ps)
926 trace_loggers = [getTraceLogger(self.log.getChild("_solve"), i) for i in range(5)]
931 while (thisIteration < maxSpatialIterations):
933 # Make sure there are no uninitialized candidates as active occupants of Cell
935 while (nRejectedSkf != 0):
936 trace_loggers[1].debug("Building single kernels...")
937 kernelCellSet.visitCandidates(singlekv, nStarPerCell)
938 nRejectedSkf = singlekv.getNRejected()
939 trace_loggers[1].debug(
940 "Iteration %d, rejected %d candidates due to initial kernel fit",
941 thisIteration, nRejectedSkf
944 # Reject outliers in kernel sum
946 ksv.setMode(diffimLib.KernelSumVisitorF.AGGREGATE)
947 kernelCellSet.visitCandidates(ksv, nStarPerCell)
948 ksv.processKsumDistribution()
949 ksv.setMode(diffimLib.KernelSumVisitorF.REJECT)
950 kernelCellSet.visitCandidates(ksv, nStarPerCell)
952 nRejectedKsum = ksv.getNRejected()
953 trace_loggers[1].debug(
954 "Iteration %d, rejected %d candidates due to kernel sum",
955 thisIteration, nRejectedKsum
958 # Do we jump back to the top without incrementing thisIteration?
959 if nRejectedKsum > 0:
963 # At this stage we can either apply the spatial fit to
964 # the kernels, or we run a PCA, use these as a *new*
965 # basis set with lower dimensionality, and then apply
966 # the spatial fit to these kernels
968 if (usePcaForSpatialKernel):
969 trace_loggers[0].debug("Building Pca basis")
971 nRejectedPca, spatialBasisList = self._createPcaBasis(kernelCellSet, nStarPerCell, ps)
972 trace_loggers[1].debug(
973 "Iteration %d, rejected %d candidates due to Pca kernel fit",
974 thisIteration, nRejectedPca
977 # We don't want to continue on (yet) with the
978 # spatial modeling, because we have bad objects
979 # contributing to the Pca basis. We basically
980 # need to restart from the beginning of this loop,
981 # since the cell-mates of those objects that were
982 # rejected need their original Kernels built by
983 # singleKernelFitter.
985 # Don't count against thisIteration
986 if (nRejectedPca > 0):
990 spatialBasisList = basisList
992 # We have gotten on to the spatial modeling part
993 regionBBox = kernelCellSet.getBBox()
994 spatialkv = diffimLib.BuildSpatialKernelVisitorF(spatialBasisList, regionBBox, ps)
995 kernelCellSet.visitCandidates(spatialkv, nStarPerCell)
996 spatialkv.solveLinearEquation()
997 trace_loggers[2].debug("Spatial kernel built with %d candidates", spatialkv.getNCandidates())
998 spatialKernel, spatialBackground = spatialkv.getSolutionPair()
1000 # Check the quality of the spatial fit (look at residuals)
1001 assesskv = diffimLib.AssessSpatialKernelVisitorF(spatialKernel, spatialBackground, ps)
1002 kernelCellSet.visitCandidates(assesskv, nStarPerCell)
1003 nRejectedSpatial = assesskv.getNRejected()
1004 nGoodSpatial = assesskv.getNGood()
1005 trace_loggers[1].debug(
1006 "Iteration %d, rejected %d candidates due to spatial kernel fit",
1007 thisIteration, nRejectedSpatial
1009 trace_loggers[1].debug("%d candidates used in fit", nGoodSpatial)
1011 # If only nGoodSpatial == 0, might be other candidates in the cells
1012 if nGoodSpatial == 0 and nRejectedSpatial == 0:
1013 raise RuntimeError("No kernel candidates for spatial fit")
1015 if nRejectedSpatial == 0:
1016 # Nothing rejected, finished with spatial fit
1019 # Otherwise, iterate on...
1022 # Final fit if above did not converge
1023 if (nRejectedSpatial > 0) and (thisIteration == maxSpatialIterations):
1024 trace_loggers[1].debug("Final spatial fit")
1025 if (usePcaForSpatialKernel):
1026 nRejectedPca, spatialBasisList = self._createPcaBasis(kernelCellSet, nStarPerCell, ps)
1027 regionBBox = kernelCellSet.getBBox()
1028 spatialkv = diffimLib.BuildSpatialKernelVisitorF(spatialBasisList, regionBBox, ps)
1029 kernelCellSet.visitCandidates(spatialkv, nStarPerCell)
1030 spatialkv.solveLinearEquation()
1031 trace_loggers[2].debug("Spatial kernel built with %d candidates", spatialkv.getNCandidates())
1032 spatialKernel, spatialBackground = spatialkv.getSolutionPair()
1034 spatialSolution = spatialkv.getKernelSolution()
1036 except Exception as e:
1037 self.log.error("ERROR: Unable to calculate psf matching kernel")
1039 trace_loggers[1].debug("%s", e)
1043 trace_loggers[0].debug("Total time to compute the spatial kernel : %.2f s", (t1 - t0))
1046 self._displayDebug(kernelCellSet, spatialKernel, spatialBackground)
1048 self._diagnostic(kernelCellSet, spatialSolution, spatialKernel, spatialBackground)
1050 return spatialSolution, spatialKernel, spatialBackground