lsst.jointcal  master-g52a623bc1f+3
FitterBase.cc
Go to the documentation of this file.
1 #include <vector>
2 
3 #include "lsst/log/Log.h"
4 
5 #include "lsst/jointcal/Chi2.h"
11 
12 static double sqr(double x) { return x * x; }
13 
14 namespace {
15 LOG_LOGGER _log = LOG_GET("jointcal.Fitter");
16 }
17 
18 namespace lsst {
19 namespace jointcal {
20 
22  Chi2Statistic chi2;
23  accumulateStatImageList(_associations->getCcdImageList(), chi2);
25  // chi2.ndof contains the number of squares.
26  // So subtract the number of parameters.
27  chi2.ndof -= _nParTot;
28  return chi2;
29 }
30 
31 unsigned FitterBase::findOutliers(double nSigmaCut, MeasuredStarList &msOutliers,
32  FittedStarList &fsOutliers) const {
33  // collect chi2 contributions
34  Chi2List chi2List;
35  chi2List.reserve(_nMeasuredStars + _associations->refStarList.size());
36  // contributions from measurement terms:
37  accumulateStatImageList(_associations->ccdImageList, chi2List);
38  // and from reference terms
39  accumulateStatRefStars(chi2List);
40 
41  // compute some statistics
42  size_t nval = chi2List.size();
43  if (nval == 0) return 0;
44  sort(chi2List.begin(), chi2List.end());
45  double median = (nval & 1) ? chi2List[nval / 2].chi2
46  : 0.5 * (chi2List[nval / 2 - 1].chi2 + chi2List[nval / 2].chi2);
47  auto averageAndSigma = chi2List.computeAverageAndSigma();
48  LOGLS_DEBUG(_log, "RemoveOutliers chi2 stat: mean/median/sigma " << averageAndSigma.first << '/' << median
49  << '/' << averageAndSigma.second);
50  double cut = averageAndSigma.first + nSigmaCut * averageAndSigma.second;
51  /* For each of the parameters, we will not remove more than 1
52  measurement that contributes to constraining it. Keep track using
53  of what we are touching using an integer vector. This is the
54  trick that Marc Betoule came up to for outlier removals in "star
55  flats" fits. */
56  Eigen::VectorXi affectedParams(_nParTot);
57  affectedParams.setZero();
58 
59  unsigned nOutliers = 0; // returned to the caller
60  // start from the strongest outliers.
61  for (auto chi2 = chi2List.rbegin(); chi2 != chi2List.rend(); ++chi2) {
62  if (chi2->chi2 < cut) break; // because the array is sorted.
63  std::vector<unsigned> indices;
64  /* now, we want to get the indices of the parameters this chi2
65  term depends on. We have to figure out which kind of term it
66  is; we use for that the type of the star attached to the Chi2Star. */
67  auto ms = std::dynamic_pointer_cast<MeasuredStar>(chi2->star);
68  std::shared_ptr<FittedStar> fs;
69  if (!ms) {
70  // it is reference term.
71  fs = std::dynamic_pointer_cast<FittedStar>(chi2->star);
72  indices.push_back(fs->getIndexInMatrix());
73  indices.push_back(fs->getIndexInMatrix() + 1); // probably useless
74  /* One might think it would be useful to account for PM
75  parameters here, but it is just useless */
76  } else { // it is a measurement term.
77  getIndicesOfMeasuredStar(*ms, indices);
78  }
79 
80  /* Find out if we already discarded a stronger outlier
81  constraining some parameter this one constrains as well. If
82  yes, we keep this one, because this stronger outlier could be
83  causing the large chi2 we have in hand. */
84  bool drop_it = true;
85  for (auto const &i : indices) {
86  if (affectedParams(i) != 0) {
87  drop_it = false;
88  }
89  }
90 
91  if (drop_it) // store the outlier in one of the lists:
92  {
93  if (ms) {
94  // measurement term
95  msOutliers.push_back(ms);
96  } else {
97  // ref term
98  fsOutliers.push_back(fs);
99  }
100  // mark the parameters as directly changed when we discard this chi2 term.
101  for (auto const &i : indices) {
102  affectedParams(i)++;
103  }
104  nOutliers++;
105  }
106  } // end loop on measurements/references
107  LOGLS_INFO(_log, "findOutliers: found " << msOutliers.size() << " meas outliers and " << fsOutliers.size()
108  << " ref outliers ");
109 
110  return nOutliers;
111 }
112 
113 MinimizeResult FitterBase::minimize(std::string const &whatToFit, double nSigmaCut) {
114  assignIndices(whatToFit);
115 
117 
118  // TODO : write a guesser for the number of triplets
119  unsigned nTrip = (_lastNTrip) ? _lastNTrip : 1e6;
120  TripletList tripletList(nTrip);
121  Eigen::VectorXd grad(_nParTot);
122  grad.setZero();
123 
124  // Fill the triplets
125  leastSquareDerivatives(tripletList, grad);
126  _lastNTrip = tripletList.size();
127 
128  LOGLS_DEBUG(_log, "End of triplet filling, ntrip = " << tripletList.size());
129 
130  SpMat hessian;
131  {
132  SpMat jacobian(_nParTot, tripletList.getNextFreeIndex());
133  jacobian.setFromTriplets(tripletList.begin(), tripletList.end());
134  // release memory shrink_to_fit is C++11
135  tripletList.clear(); // tripletList.shrink_to_fit();
136  hessian = jacobian * jacobian.transpose();
137  } // release the Jacobian
138 
139  LOGLS_DEBUG(_log, "Starting factorization, hessian: dim="
140  << hessian.rows() << " non-zeros=" << hessian.nonZeros()
141  << " filling-frac = " << hessian.nonZeros() / sqr(hessian.rows()));
142 
143  CholmodSimplicialLDLT2<SpMat> chol(hessian);
144  if (chol.info() != Eigen::Success) {
145  LOGLS_ERROR(_log, "minimize: factorization failed ");
146  return MinimizeResult::Failed;
147  }
148 
149  unsigned totalOutliers = 0;
150  double oldChi2 = computeChi2().chi2;
151 
152  while (true) {
153  Eigen::VectorXd delta = chol.solve(grad);
154  offsetParams(delta);
155  Chi2Statistic currentChi2(computeChi2());
156  LOGLS_DEBUG(_log, currentChi2);
157  if (currentChi2.chi2 > oldChi2) {
158  LOGL_WARN(_log, "chi2 went up, skipping outlier rejection loop");
159  returnCode = MinimizeResult::Chi2Increased;
160  break;
161  }
162  oldChi2 = currentChi2.chi2;
163 
164  if (nSigmaCut == 0) break; // no rejection step to perform
165  MeasuredStarList msOutliers;
166  FittedStarList fsOutliers;
167  int nOutliers = findOutliers(nSigmaCut, msOutliers, fsOutliers);
168  totalOutliers += nOutliers;
169  if (nOutliers == 0) break;
170  TripletList tripletList(nOutliers);
171  grad.setZero(); // recycle the gradient
172  // compute the contributions of outliers to derivatives
173  outliersContributions(msOutliers, fsOutliers, tripletList, grad);
174  // Remove significant outliers
175  removeMeasOutliers(msOutliers);
176  removeRefOutliers(fsOutliers);
177  // convert triplet list to eigen internal format
178  SpMat H(_nParTot, tripletList.getNextFreeIndex());
179  H.setFromTriplets(tripletList.begin(), tripletList.end());
180  int update_status = chol.update(H, false /* means downdate */);
181  LOGLS_DEBUG(_log, "cholmod update_status " << update_status);
182  // The contribution of outliers to the gradient is the opposite
183  // of the contribution of all other terms, because they add up to 0
184  grad *= -1;
185  }
186 
187  LOGLS_INFO(_log, "Total number of outliers " << totalOutliers);
188  return returnCode;
189 }
190 
192  TripletList &tripletList, Eigen::VectorXd &grad) {
193  for (auto &outlier : msOutliers) {
194  MeasuredStarList tmp;
195  tmp.push_back(outlier);
196  const CcdImage &ccdImage = outlier->getCcdImage();
197  leastSquareDerivativesMeasurement(ccdImage, tripletList, grad, &tmp);
198  }
199  leastSquareDerivativesReference(fsOutliers, tripletList, grad);
200 }
201 
203  for (auto &measuredStar : outliers) {
204  auto fittedStar = std::const_pointer_cast<FittedStar>(measuredStar->getFittedStar());
205  measuredStar->setValid(false);
206  fittedStar->getMeasurementCount()--; // could be put in setValid
207  }
208 }
209 
211  for (auto &fittedStar : outliers) {
212  fittedStar->setRefStar(nullptr);
213  }
214 }
215 
216 void FitterBase::leastSquareDerivatives(TripletList &tripletList, Eigen::VectorXd &grad) const {
217  auto ccdImageList = _associations->getCcdImageList();
218  for (auto const &ccdImage : ccdImageList) {
219  leastSquareDerivativesMeasurement(*ccdImage, tripletList, grad);
220  }
221  leastSquareDerivativesReference(_associations->fittedStarList, tripletList, grad);
222 }
223 
224 } // namespace jointcal
225 } // namespace lsst
virtual void accumulateStatRefStars(Chi2Accumulator &accum) const =0
Compute the chi2 (per star or total, depending on which Chi2Accumulator is used) for RefStars...
virtual void leastSquareDerivativesReference(FittedStarList const &fittedStarList, TripletList &tripletList, Eigen::VectorXd &grad) const =0
Compute the derivatives of the reference terms.
Simple structure to accumulate chi2 and ndof.
Definition: Chi2.h:29
int update(SpMat const &H, bool UpOrDown)
Definition: Eigenstuff.h:41
virtual void accumulateStatImageList(CcdImageList const &ccdImageList, Chi2Accumulator &accum) const =0
Compute the chi2 (per star or total, depending on which Chi2Accumulator is used) for measurements...
unsigned findOutliers(double nSigmaCut, MeasuredStarList &msOutliers, FittedStarList &fsOutliers) const
Find Measurements and references contributing more than a cut, computed as The outliers are NOT remo...
Definition: FitterBase.cc:31
int getMeasurementCount() const
Definition: FittedStar.h:92
virtual void getIndicesOfMeasuredStar(MeasuredStar const &measuredStar, std::vector< unsigned > &indices) const =0
Set the indices of a measured star from the full matrix, for outlier removal.
MinimizeResult minimize(std::string const &whatToFit, double nSigmaCut=0)
Does a 1 step minimization, assuming a linear model.
Definition: FitterBase.cc:113
void removeMeasOutliers(MeasuredStarList &outliers)
Remove measuredStar outliers from the fit. No Refit done.
Definition: FitterBase.cc:202
A list of MeasuredStar. They are usually filled in Associations::AddImage.
Definition: MeasuredStar.h:95
MinimizeResult
Return value of minimize()
Definition: FitterBase.h:16
Structure to accumulate the chi2 contributions per each star (to help find outliers).
Definition: Chi2.h:84
Class for a simple mapping implementing a generic Gtransfo.
Definition: Associations.h:24
Eigen::SparseMatrix< double > SpMat
Definition: Eigenstuff.h:11
A list of FittedStar s. Such a list is typically constructed by Associations.
Definition: FittedStar.h:121
void removeRefOutliers(FittedStarList &outliers)
Remove refStar outliers from the fit. No Refit done.
Definition: FitterBase.cc:210
objects measured on actual images.
Definition: MeasuredStar.h:18
std::shared_ptr< Associations > _associations
Definition: FitterBase.h:107
virtual void assignIndices(std::string const &whatToFit)=0
Set parameters to fit and assign indices in the big matrix.
Chi2Statistic computeChi2() const
Returns the chi2 for the current state.
Definition: FitterBase.cc:21
void leastSquareDerivatives(TripletList &tripletList, Eigen::VectorXd &grad) const
Evaluates the chI^2 derivatives (Jacobian and gradient) for the current whatToFit setting...
Definition: FitterBase.cc:216
Handler of an actual image from a single CCD.
Definition: CcdImage.h:31
virtual void offsetParams(Eigen::VectorXd const &delta)=0
Offset the parameters by the requested quantities.
unsigned getNextFreeIndex() const
Definition: Tripletlist.h:25
void outliersContributions(MeasuredStarList &msOutliers, FittedStarList &fsOutliers, TripletList &tripletList, Eigen::VectorXd &grad)
Contributions to derivatives from (presumably) outlier terms.
Definition: FitterBase.cc:191
The objects which have been measured several times.
Definition: FittedStar.h:37
std::pair< double, double > computeAverageAndSigma()
Compute the average and std-deviation of these chisq values.
Definition: Chi2.cc:33
virtual void leastSquareDerivativesMeasurement(CcdImage const &ccdImage, TripletList &tripletList, Eigen::VectorXd &grad, MeasuredStarList const *measuredStarList=nullptr) const =0
Compute the derivatives of the measured stars and model for one CcdImage.