lsst.jointcal  14.0-17-ge3cc87b+5
FitterBase.cc
Go to the documentation of this file.
1 #include <vector>
2 
3 #include "lsst/log/Log.h"
4 
5 #include "lsst/jointcal/Chi2.h"
11 
12 namespace {
13 LOG_LOGGER _log = LOG_GET("jointcal.Fitter");
14 }
15 
16 namespace lsst {
17 namespace jointcal {
18 
20  Chi2Statistic chi2;
21  accumulateStatImageList(_associations->getCcdImageList(), chi2);
23  // chi2.ndof contains the number of squares.
24  // So subtract the number of parameters.
25  chi2.ndof -= _nParTot;
26  return chi2;
27 }
28 
29 unsigned FitterBase::findOutliers(double nSigmaCut, MeasuredStarList &msOutliers,
30  FittedStarList &fsOutliers) const {
31  // collect chi2 contributions
32  Chi2List chi2List;
33  chi2List.reserve(_nMeasuredStars + _associations->refStarList.size());
34  // contributions from measurement terms:
35  accumulateStatImageList(_associations->ccdImageList, chi2List);
36  // and from reference terms
37  accumulateStatRefStars(chi2List);
38 
39  // compute some statistics
40  size_t nval = chi2List.size();
41  if (nval == 0) return 0;
42  sort(chi2List.begin(), chi2List.end());
43  double median = (nval & 1) ? chi2List[nval / 2].chi2
44  : 0.5 * (chi2List[nval / 2 - 1].chi2 + chi2List[nval / 2].chi2);
45  auto averageAndSigma = chi2List.computeAverageAndSigma();
46  LOGLS_DEBUG(_log, "RemoveOutliers chi2 stat: mean/median/sigma " << averageAndSigma.first << '/' << median
47  << '/' << averageAndSigma.second);
48  double cut = averageAndSigma.first + nSigmaCut * averageAndSigma.second;
49  /* For each of the parameters, we will not remove more than 1
50  measurement that contributes to constraining it. Keep track using
51  of what we are touching using an integer vector. This is the
52  trick that Marc Betoule came up to for outlier removals in "star
53  flats" fits. */
54  Eigen::VectorXi affectedParams(_nParTot);
55  affectedParams.setZero();
56 
57  unsigned nOutliers = 0; // returned to the caller
58  // start from the strongest outliers.
59  for (auto chi2 = chi2List.rbegin(); chi2 != chi2List.rend(); ++chi2) {
60  if (chi2->chi2 < cut) break; // because the array is sorted.
61  std::vector<unsigned> indices;
62  /* now, we want to get the indices of the parameters this chi2
63  term depends on. We have to figure out which kind of term it
64  is; we use for that the type of the star attached to the Chi2Star. */
65  auto ms = std::dynamic_pointer_cast<MeasuredStar>(chi2->star);
67  if (!ms) {
68  // it is reference term.
69  fs = std::dynamic_pointer_cast<FittedStar>(chi2->star);
70  // NOTE: Stars contribute twice to astrometry (x,y), but once to photometry (flux),
71  // NOTE: but we only need to mark one index here because both will be removed with that star.
72  indices.push_back(fs->getIndexInMatrix());
73  /* One might think it would be useful to account for PM
74  parameters here, but it is just useless */
75  } else { // it is a measurement term.
76  getIndicesOfMeasuredStar(*ms, indices);
77  }
78 
79  /* Find out if we already discarded a stronger outlier
80  constraining some parameter this one constrains as well. If
81  yes, we keep this one, because this stronger outlier could be
82  causing the large chi2 we have in hand. */
83  bool drop_it = true;
84  for (auto const &i : indices) {
85  if (affectedParams(i) != 0) {
86  drop_it = false;
87  }
88  }
89 
90  if (drop_it) // store the outlier in one of the lists:
91  {
92  if (ms) {
93  // measurement term
94  msOutliers.push_back(ms);
95  } else {
96  // ref term
97  fsOutliers.push_back(fs);
98  }
99  // mark the parameters as directly changed when we discard this chi2 term.
100  for (auto const &i : indices) {
101  affectedParams(i)++;
102  }
103  nOutliers++;
104  }
105  } // end loop on measurements/references
106  LOGLS_INFO(_log, "findOutliers: found " << msOutliers.size() << " meas outliers and " << fsOutliers.size()
107  << " ref outliers ");
108 
109  return nOutliers;
110 }
111 
112 MinimizeResult FitterBase::minimize(std::string const &whatToFit, double nSigmaCut) {
113  assignIndices(whatToFit);
114 
116 
117  // TODO : write a guesser for the number of triplets
118  unsigned nTrip = (_lastNTrip) ? _lastNTrip : 1e6;
119  TripletList tripletList(nTrip);
120  Eigen::VectorXd grad(_nParTot);
121  grad.setZero();
122 
123  // Fill the triplets
124  leastSquareDerivatives(tripletList, grad);
125  _lastNTrip = tripletList.size();
126 
127  LOGLS_DEBUG(_log, "End of triplet filling, ntrip = " << tripletList.size());
128 
129  SpMat hessian;
130  {
131  SpMat jacobian(_nParTot, tripletList.getNextFreeIndex());
132  jacobian.setFromTriplets(tripletList.begin(), tripletList.end());
133  // release memory shrink_to_fit is C++11
134  tripletList.clear(); // tripletList.shrink_to_fit();
135  hessian = jacobian * jacobian.transpose();
136  } // release the Jacobian
137 
138  LOGLS_DEBUG(_log, "Starting factorization, hessian: dim="
139  << hessian.rows() << " non-zeros=" << hessian.nonZeros()
140  << " filling-frac = " << hessian.nonZeros() / std::pow(hessian.rows(), 2));
141 
142  CholmodSimplicialLDLT2<SpMat> chol(hessian);
143  if (chol.info() != Eigen::Success) {
144  LOGLS_ERROR(_log, "minimize: factorization failed ");
145  return MinimizeResult::Failed;
146  }
147 
148  unsigned totalOutliers = 0;
149  double oldChi2 = computeChi2().chi2;
150 
151  while (true) {
152  Eigen::VectorXd delta = chol.solve(grad);
153  offsetParams(delta);
154  Chi2Statistic currentChi2(computeChi2());
155  LOGLS_DEBUG(_log, currentChi2);
156  if (currentChi2.chi2 > oldChi2 && totalOutliers != 0) {
157  LOGL_WARN(_log, "chi2 went up, skipping outlier rejection loop");
158  returnCode = MinimizeResult::Chi2Increased;
159  break;
160  }
161  oldChi2 = currentChi2.chi2;
162 
163  if (nSigmaCut == 0) break; // no rejection step to perform
164  MeasuredStarList msOutliers;
165  FittedStarList fsOutliers;
166  int nOutliers = findOutliers(nSigmaCut, msOutliers, fsOutliers);
167  totalOutliers += nOutliers;
168  if (nOutliers == 0) break;
169  TripletList tripletList(nOutliers);
170  grad.setZero(); // recycle the gradient
171  // compute the contributions of outliers to derivatives
172  outliersContributions(msOutliers, fsOutliers, tripletList, grad);
173  // Remove significant outliers
174  removeMeasOutliers(msOutliers);
175  removeRefOutliers(fsOutliers);
176  // convert triplet list to eigen internal format
177  SpMat H(_nParTot, tripletList.getNextFreeIndex());
178  H.setFromTriplets(tripletList.begin(), tripletList.end());
179  int update_status = chol.update(H, false /* means downdate */);
180  LOGLS_DEBUG(_log, "cholmod update_status " << update_status);
181  // The contribution of outliers to the gradient is the opposite
182  // of the contribution of all other terms, because they add up to 0
183  grad *= -1;
184  }
185 
186  LOGLS_INFO(_log, "Total number of outliers " << totalOutliers);
187  return returnCode;
188 }
189 
191  TripletList &tripletList, Eigen::VectorXd &grad) {
192  for (auto &outlier : msOutliers) {
193  MeasuredStarList tmp;
194  tmp.push_back(outlier);
195  const CcdImage &ccdImage = outlier->getCcdImage();
196  leastSquareDerivativesMeasurement(ccdImage, tripletList, grad, &tmp);
197  }
198  leastSquareDerivativesReference(fsOutliers, tripletList, grad);
199 }
200 
202  for (auto &measuredStar : outliers) {
203  auto fittedStar = std::const_pointer_cast<FittedStar>(measuredStar->getFittedStar());
204  measuredStar->setValid(false);
205  fittedStar->getMeasurementCount()--; // could be put in setValid
206  }
207 }
208 
210  for (auto &fittedStar : outliers) {
211  fittedStar->setRefStar(nullptr);
212  }
213 }
214 
215 void FitterBase::leastSquareDerivatives(TripletList &tripletList, Eigen::VectorXd &grad) const {
216  auto ccdImageList = _associations->getCcdImageList();
217  for (auto const &ccdImage : ccdImageList) {
218  leastSquareDerivativesMeasurement(*ccdImage, tripletList, grad);
219  }
220  leastSquareDerivativesReference(_associations->fittedStarList, tripletList, grad);
221 }
222 
223 void FitterBase::saveChi2Contributions(std::string const &baseName) const {
224  /* cook-up 2 different file names by inserting something just before
225  the dot (if any), and within the actual file name. */
226  size_t dot = baseName.rfind('.');
227  size_t slash = baseName.rfind('/');
228  if (dot == std::string::npos || (slash != std::string::npos && dot < slash)) dot = baseName.size();
229  std::string measTuple(baseName);
230  measTuple.insert(dot, "-meas");
231  saveChi2MeasContributions(measTuple);
232  std::string refTuple(baseName);
233  refTuple.insert(dot, "-ref");
234  saveChi2RefContributions(refTuple);
235 }
236 
237 } // namespace jointcal
238 } // namespace lsst
virtual void accumulateStatRefStars(Chi2Accumulator &accum) const =0
Compute the chi2 (per star or total, depending on which Chi2Accumulator is used) for RefStars...
virtual void leastSquareDerivativesReference(FittedStarList const &fittedStarList, TripletList &tripletList, Eigen::VectorXd &grad) const =0
Compute the derivatives of the reference terms.
Simple structure to accumulate chi2 and ndof.
Definition: Chi2.h:29
int update(SpMat const &H, bool UpOrDown)
Definition: Eigenstuff.h:41
virtual void accumulateStatImageList(CcdImageList const &ccdImageList, Chi2Accumulator &accum) const =0
Compute the chi2 (per star or total, depending on which Chi2Accumulator is used) for measurements...
unsigned findOutliers(double nSigmaCut, MeasuredStarList &msOutliers, FittedStarList &fsOutliers) const
Find Measurements and references contributing more than a cut, computed as The outliers are NOT remo...
Definition: FitterBase.cc:29
T rend(T... args)
int getMeasurementCount() const
Definition: FittedStar.h:82
T rfind(T... args)
virtual void getIndicesOfMeasuredStar(MeasuredStar const &measuredStar, std::vector< unsigned > &indices) const =0
Set the indices of a measured star from the full matrix, for outlier removal.
MinimizeResult minimize(std::string const &whatToFit, double nSigmaCut=0)
Does a 1 step minimization, assuming a linear model.
Definition: FitterBase.cc:112
T end(T... args)
void removeMeasOutliers(MeasuredStarList &outliers)
Remove measuredStar outliers from the fit. No Refit done.
Definition: FitterBase.cc:201
A list of MeasuredStar. They are usually filled in Associations::AddImage.
Definition: MeasuredStar.h:111
virtual void saveChi2Contributions(std::string const &baseName) const
Save the full chi2 term per star that was used in the minimization, for debugging.
Definition: FitterBase.cc:223
MinimizeResult
Return value of minimize()
Definition: FitterBase.h:16
#define LOGLS_INFO(logger, message)
def dot(symb, c, r, frame=None, size=2, ctype=None, origin=afwImage.PARENT, args, kwargs)
STL class.
virtual void saveChi2MeasContributions(std::string const &baseName) const =0
Save a CSV file containing residuals of measurement terms.
Structure to accumulate the chi2 contributions per each star (to help find outliers).
Definition: Chi2.h:77
T push_back(T... args)
Class for a simple mapping implementing a generic Gtransfo.
Eigen::SparseMatrix< double > SpMat
Definition: Eigenstuff.h:11
A list of FittedStar s. Such a list is typically constructed by Associations.
Definition: FittedStar.h:121
void removeRefOutliers(FittedStarList &outliers)
Remove refStar outliers from the fit. No Refit done.
Definition: FitterBase.cc:209
objects measured on actual images.
Definition: MeasuredStar.h:18
T dynamic_pointer_cast(T... args)
T clear(T... args)
virtual void saveChi2RefContributions(std::string const &baseName) const =0
Save a CSV file containing residuals of reference terms.
std::shared_ptr< Associations > _associations
Definition: FitterBase.h:113
virtual void assignIndices(std::string const &whatToFit)=0
Set parameters to fit and assign indices in the big matrix.
T insert(T... args)
T size(T... args)
Chi2Statistic computeChi2() const
Returns the chi2 for the current state.
Definition: FitterBase.cc:19
T begin(T... args)
T pow(T... args)
#define LOGLS_DEBUG(logger, message)
void leastSquareDerivatives(TripletList &tripletList, Eigen::VectorXd &grad) const
Evaluates the chI^2 derivatives (Jacobian and gradient) for the current whatToFit setting...
Definition: FitterBase.cc:215
T sort(T... args)
Handler of an actual image from a single CCD.
Definition: CcdImage.h:34
#define LOGL_WARN(logger, message...)
virtual void offsetParams(Eigen::VectorXd const &delta)=0
Offset the parameters by the requested quantities.
unsigned getNextFreeIndex() const
Definition: Tripletlist.h:22
#define LOG_GET(logger)
void outliersContributions(MeasuredStarList &msOutliers, FittedStarList &fsOutliers, TripletList &tripletList, Eigen::VectorXd &grad)
Contributions to derivatives from (presumably) outlier terms.
Definition: FitterBase.cc:190
The objects which have been measured several times.
Definition: FittedStar.h:37
std::pair< double, double > computeAverageAndSigma()
Compute the average and std-deviation of these chisq values.
Definition: Chi2.cc:31
#define LOGLS_ERROR(logger, message)
T reserve(T... args)
virtual void leastSquareDerivativesMeasurement(CcdImage const &ccdImage, TripletList &tripletList, Eigen::VectorXd &grad, MeasuredStarList const *measuredStarList=nullptr) const =0
Compute the derivatives of the measured stars and model for one CcdImage.
T rbegin(T... args)