4 #include <boost/math/tools/minima.hpp> 39 size_t nval = chi2List.
size();
40 if (nval == 0)
return 0;
42 double median = (nval & 1) ? chi2List[nval / 2].chi2
43 : 0.5 * (chi2List[nval / 2 - 1].chi2 + chi2List[nval / 2].chi2);
45 LOGLS_DEBUG(
_log,
"RemoveOutliers chi2 stat: mean/median/sigma " << averageAndSigma.first <<
'/' << median
46 <<
'/' << averageAndSigma.second);
47 double cut = averageAndSigma.first + nSigmaCut * averageAndSigma.second;
53 Eigen::VectorXi affectedParams(
_nParTot);
54 affectedParams.setZero();
56 unsigned nOutliers = 0;
58 for (
auto chi2 = chi2List.
rbegin(); chi2 != chi2List.
rend(); ++chi2) {
59 if (chi2->chi2 < cut)
break;
66 if (measuredStar ==
nullptr) {
69 if (fittedStar->getMeasurementCount() == 0) {
70 LOGLS_WARN(
_log,
"FittedStar with no measuredStars found as an outlier: " << *fittedStar);
75 indices.
push_back(fittedStar->getIndexInMatrix());
80 auto tempFittedStar = measuredStar->getFittedStar();
81 if (tempFittedStar->getMeasurementCount() == 1 && tempFittedStar->getRefStar() ==
nullptr) {
82 LOGLS_WARN(
_log,
"FittedStar with 1 measuredStar and no refStar found as an outlier: " 94 for (
auto const &i : indices) {
95 if (affectedParams(i) != 0) {
102 if (measuredStar ==
nullptr) {
110 for (
auto const &i : indices) {
116 LOGLS_INFO(
_log,
"findOutliers: found " << msOutliers.
size() <<
" meas outliers and " << fsOutliers.
size()
117 <<
" ref outliers ");
126 jacobian.setFromTriplets(tripletList.
begin(), tripletList.
end());
127 return jacobian * jacobian.transpose();
131 void dumpMatrixAndGradient(
SparseMatrixD const &matrix, Eigen::VectorXd
const &grad,
134 Eigen::MatrixXd matrixDense(matrix);
135 std::string dumpMatrixPath = dumpFile +
"-mat" + ext;
138 std::string dumpGradPath = dumpFile +
"-grad" + ext;
141 LOGLS_INFO(_log,
"Dumped Hessian, gradient to: '" << dumpMatrixPath <<
"', '" << dumpGradPath <<
"'");
146 bool const doLineSearch,
std::string const &dumpMatrixFile) {
168 << hessian.rows() <<
" non-zeros=" << hessian.nonZeros()
169 <<
" filling-frac = " << hessian.nonZeros() /
std::pow(hessian.rows(), 2));
171 if (dumpMatrixFile !=
"") {
172 if (hessian.rows() * hessian.cols() > 2e8) {
173 LOGLS_WARN(
_log,
"Hessian matrix is too big to dump to file, with rows, columns: " 174 << hessian.rows() <<
", " << hessian.cols());
176 dumpMatrixAndGradient(hessian, grad, dumpMatrixFile,
_log);
181 if (chol.info() != Eigen::Success) {
186 unsigned totalMeasOutliers = 0;
187 unsigned totalRefOutliers = 0;
191 Eigen::VectorXd delta = chol.solve(grad);
193 scale = _lineSearch(delta);
199 LOGL_ERROR(
_log,
"chi2 is not finite. Aborting outlier rejection.");
203 if (currentChi2.
chi2 > oldChi2 && totalMeasOutliers + totalRefOutliers != 0) {
204 LOGL_WARN(
_log,
"chi2 went up, skipping outlier rejection loop");
208 oldChi2 = currentChi2.
chi2;
210 if (nSigmaCut == 0)
break;
214 int nOutliers =
findOutliers(nSigmaCut, msOutliers, fsOutliers);
215 totalMeasOutliers += msOutliers.
size();
216 totalRefOutliers += fsOutliers.
size();
217 if (nOutliers == 0)
break;
228 H.setFromTriplets(outlierTriplets.
begin(), outlierTriplets.
end());
242 hessian = createHessian(
_nParTot, nextTripletList);
243 nextTripletList.
clear();
246 "Restarting factorization, hessian: dim=" 247 << hessian.rows() <<
" non-zeros=" << hessian.nonZeros()
248 <<
" filling-frac = " << hessian.nonZeros() /
std::pow(hessian.rows(), 2));
249 chol.compute(hessian);
250 if (chol.info() != Eigen::Success) {
258 if (nSigmaCut != 0) {
259 LOGLS_INFO(
_log,
"Number of outliers (Measured + Reference = Total): " 260 << totalMeasOutliers <<
" + " << totalRefOutliers <<
" = " 261 << totalMeasOutliers + totalRefOutliers);
268 for (
auto &outlier : msOutliers) {
278 for (
auto &measuredStar : outliers) {
279 auto fittedStar = measuredStar->getFittedStar();
280 measuredStar->setValid(
false);
281 fittedStar->getMeasurementCount()--;
286 for (
auto &fittedStar : outliers) {
287 fittedStar->setRefStar(
nullptr);
293 for (
auto const &
ccdImage : ccdImageList) {
303 size_t slash = baseName.
rfind(
'/');
304 if (dot == std::string::npos || (slash != std::string::npos && dot < slash)) dot = baseName.
size();
306 measTuple.
insert(dot,
"-meas");
309 refTuple.
insert(dot,
"-ref");
313 double FitterBase::_lineSearch(Eigen::VectorXd
const &delta) {
314 auto func = [
this, &delta](
double scale) {
315 auto offset =
scale * delta;
324 auto result = boost::math::tools::brent_find_minima(func, -1.0, 2.0, bits);
#define LOGLS_WARN(logger, message)
virtual void accumulateStatRefStars(Chi2Accumulator &accum) const =0
Compute the chi2 (per star or total, depending on which Chi2Accumulator is used) for RefStars...
virtual void leastSquareDerivativesReference(FittedStarList const &fittedStarList, TripletList &tripletList, Eigen::VectorXd &grad) const =0
Compute the derivatives of the reference terms.
Simple structure to accumulate chi2 and ndof.
Eigen::SparseMatrix< double > SparseMatrixD
virtual void accumulateStatImageList(CcdImageList const &ccdImageList, Chi2Accumulator &accum) const =0
Compute the chi2 (per star or total, depending on which Chi2Accumulator is used) for measurements...
unsigned findOutliers(double nSigmaCut, MeasuredStarList &msOutliers, FittedStarList &fsOutliers) const
Find Measurements and references contributing more than a cut, computed as The outliers are NOT remo...
def scale(algorithm, min, max=None, frame=None)
virtual void getIndicesOfMeasuredStar(MeasuredStar const &measuredStar, std::vector< unsigned > &indices) const =0
Set the indices of a measured star from the full matrix, for outlier removal.
#define LOGL_ERROR(logger, message...)
void removeMeasOutliers(MeasuredStarList &outliers)
Remove measuredStar outliers from the fit. No Refit done.
void update(SparseMatrixD const &H, bool UpOrDown)
A list of MeasuredStar. They are usually filled in Associations::createCcdImage.
virtual void saveChi2Contributions(std::string const &baseName) const
Save the full chi2 term per star that was used in the minimization, for debugging.
MinimizeResult
Return value of minimize()
def dot(symb, c, r, frame=None, size=2, ctype=None, origin=afwImage.PARENT, args, kwargs)
MinimizeResult minimize(std::string const &whatToFit, double const nSigmaCut=0, bool const doRankUpdate=true, bool const doLineSearch=false, std::string const &dumpMatrixFile="")
Does a 1 step minimization, assuming a linear model.
virtual void saveChi2MeasContributions(std::string const &baseName) const =0
Save a CSV file containing residuals of measurement terms.
Structure to accumulate the chi2 contributions per each star (to help find outliers).
#define LOGLS_DEBUG(logger, message)
Class for a simple mapping implementing a generic Gtransfo.
A list of FittedStar s. Such a list is typically constructed by Associations.
void removeRefOutliers(FittedStarList &outliers)
Remove refStar outliers from the fit. No Refit done.
objects measured on actual images.
T dynamic_pointer_cast(T... args)
#define LOGL_WARN(logger, message...)
virtual void saveChi2RefContributions(std::string const &baseName) const =0
Save a CSV file containing residuals of reference terms.
std::shared_ptr< Associations > _associations
virtual void assignIndices(std::string const &whatToFit)=0
Set parameters to fit and assign indices in the big matrix.
std::unique_ptr< SchemaItem< U > > result
#define LOGLS_INFO(logger, message)
Chi2Statistic computeChi2() const
Returns the chi2 for the current state.
void leastSquareDerivatives(TripletList &tripletList, Eigen::VectorXd &grad) const
Evaluates the chI^2 derivatives (Jacobian and gradient) for the current whatToFit setting...
Handler of an actual image from a single CCD.
virtual void offsetParams(Eigen::VectorXd const &delta)=0
Offset the parameters by the requested quantities.
#define LOGLS_ERROR(logger, message)
unsigned getNextFreeIndex() const
void outliersContributions(MeasuredStarList &msOutliers, FittedStarList &fsOutliers, TripletList &tripletList, Eigen::VectorXd &grad)
Contributions to derivatives from (presumably) outlier terms.
The objects which have been measured several times.
std::pair< double, double > computeAverageAndSigma()
Compute the average and std-deviation of these chisq values.
virtual void leastSquareDerivativesMeasurement(CcdImage const &ccdImage, TripletList &tripletList, Eigen::VectorXd &grad, MeasuredStarList const *measuredStarList=nullptr) const =0
Compute the derivatives of the measured stars and model for one CcdImage.