lsst.meas.astrom  14.0-4-g4cc409d+26
ScaledPolynomialTransformFitter.cc
Go to the documentation of this file.
1 // -*- LSST-C++ -*-
2 
3 /*
4  * LSST Data Management System
5  * Copyright 2016 LSST/AURA
6  *
7  * This product includes software developed by the
8  * LSST Project (http://www.lsst.org/).
9  *
10  * This program is free software: you can redistribute it and/or modify
11  * it under the terms of the GNU General Public License as published by
12  * the Free Software Foundation, either version 3 of the License, or
13  * (at your option) any later version.
14  *
15  * This program is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18  * GNU General Public License for more details.
19  *
20  * You should have received a copy of the LSST License Statement and
21  * the GNU General Public License along with this program. If not,
22  * see <http://www.lsstcorp.org/LegalNotices/>.
23  */
24 
25 #include <map>
26 
27 #include "Eigen/LU" // for determinant, even though it's a 2x2 that doesn't use actual LU implementation
28 
29 #include "boost/math/tools/minima.hpp"
30 
34 #include "lsst/afw/table/Match.h"
36 
37 // When developing this code, it was used to add an in-line check for the
38 // correctness of a particularly complex calculation that was difficult to
39 // factor out into something unit-testable (the larger context that code is in
40 // *is* unit tested, which should be guard against regressions). The test
41 // code is still in place, guarded by a check against this preprocessor macro;
42 // set it to 1 to re-enable that test during development, but be sure to
43 // set it back to zero before committing.
44 #define LSST_ScaledPolynomialTransformFitter_TEST_IN_PLACE 0
45 
46 namespace lsst { namespace meas { namespace astrom {
47 
48 // A singleton struct that manages the schema and keys for polynomial fitting catalogs.
50 public:
59  // We use uint16 instead of Flag since it's the only bool we have here, we
60  // may want NumPy views, and afw::table doesn't support [u]int8 fields.
62 
63  Keys(Keys const &) = delete;
64  Keys(Keys &&) = delete;
65  Keys & operator=(Keys const &) = delete;
66  Keys & operator=(Keys &&) = delete;
67 
68  static Keys const & forMatches() {
69  static Keys const it(0);
70  return it;
71  }
72 
73  static Keys const & forGrid() {
74  static Keys const it;
75  return it;
76  }
77 
78 private:
79 
80  Keys(int) :
81  schema(),
82  refId(schema.addField<afw::table::RecordId>("ref_id", "ID of reference object in this match.")),
83  srcId(schema.addField<afw::table::RecordId>("src_id", "ID of source object in this match.")),
84  output(
86  schema, "src", "source positions in pixel coordinates.", "pix"
87  )
88  ),
89  input(
91  schema, "intermediate", "reference positions in intermediate world coordinates", "deg"
92  )
93  ),
94  initial(
96  schema, "initial", "reference positions transformed by initial WCS", "pix"
97  )
98  ),
99  model(
101  schema, "model", "result of applying transform to reference positions", "pix"
102  )
103  ),
104  outputErr(
106  schema, "src", {"x", "y"}, "pix"
107  )
108  ),
109  rejected(
110  schema.addField<std::uint16_t>(
111  "rejected",
112  "True if the match should be rejected from the fit."
113  )
114  )
115  {
116  schema.getCitizen().markPersistent();
117  }
118 
119  Keys() :
120  schema(),
121  output(
123  schema, "output", "grid output positions in intermediate world coordinates", "deg"
124  )
125  ),
126  input(
128  schema, "input", "grid input positions in pixel coordinates.", "pix"
129  )
130  ),
131  model(
133  schema, "model", "result of applying transform to input positions", "deg"
134  )
135  )
136  {
137  schema.getCitizen().markPersistent();
138  }
139 
140 };
141 
142 namespace {
143 
144 // Return the AffineTransforms that maps the given (x,y) coordinates to lie within (-1, 1)x(-1, 1)
145 afw::geom::AffineTransform computeScaling(
146  afw::table::BaseCatalog const & data,
147  afw::table::Point2DKey const & key
148 ) {
149  afw::geom::Box2D bbox;
150  for (auto const & record : data) {
151  bbox.include(afw::geom::Point2D(record.get(key)));
152  };
156 }
157 
158 } // anonymous
159 
161  int maxOrder,
162  afw::table::ReferenceMatchVector const & matches,
163  afw::image::Wcs const & initialWcs,
164  double intrinsicScatter
165 ) {
166  Keys const & keys = Keys::forMatches();
167  afw::table::BaseCatalog catalog(keys.schema);
168  catalog.reserve(matches.size());
169  float var2 = intrinsicScatter*intrinsicScatter;
170  for (auto const & match : matches) {
171  auto record = catalog.addNew();
172  record->set(keys.refId, match.first->getId());
173  record->set(keys.srcId, match.second->getId());
174  record->set(keys.input, initialWcs.skyToIntermediateWorldCoord(match.first->getCoord()));
175  record->set(keys.initial, initialWcs.skyToPixel(match.first->getCoord()));
176  record->set(keys.output, match.second->getCentroid());
177  record->set(keys.outputErr, match.second->getCentroidErr() + var2*Eigen::Matrix2f::Identity());
178  record->set(keys.rejected, false);
179  }
181  catalog,
182  keys,
183  maxOrder,
184  intrinsicScatter,
185  computeScaling(catalog, keys.input),
186  computeScaling(catalog, keys.output)
187  );
188 }
189 
191  int maxOrder,
192  afw::geom::Box2D const & bbox,
193  int nGridX, int nGridY,
194  ScaledPolynomialTransform const & toInvert
195 ) {
196  Keys const & keys = Keys::forGrid();
197  afw::table::BaseCatalog catalog(keys.schema);
198  catalog.reserve(nGridX*nGridY);
199  afw::geom::Extent2D dx(bbox.getWidth()/nGridX, 0.0);
200  afw::geom::Extent2D dy(0.0, bbox.getHeight()/nGridY);
201  for (int iy = 0; iy < nGridY; ++iy) {
202  for (int ix = 0; ix < nGridX; ++ix) {
203  afw::geom::Point2D point = bbox.getMin() + dx*ix + dy*iy;
204  auto record = catalog.addNew();
205  record->set(keys.output, point);
206  record->set(keys.input, toInvert(point));
207  }
208  }
210  catalog,
211  keys,
212  maxOrder,
213  0.0,
214  computeScaling(catalog, keys.input),
215  computeScaling(catalog, keys.output)
216  );
217 }
218 
219 ScaledPolynomialTransformFitter::ScaledPolynomialTransformFitter(
220  afw::table::BaseCatalog const & data,
221  Keys const & keys,
222  int maxOrder,
223  double intrinsicScatter,
224  afw::geom::AffineTransform const & inputScaling,
225  afw::geom::AffineTransform const & outputScaling
226 ) :
227  _keys(keys),
228  _intrinsicScatter(intrinsicScatter),
229  _data(data),
230  _outputScaling(outputScaling),
231  _transform(
232  PolynomialTransform(maxOrder),
233  inputScaling,
234  outputScaling.invert()
235  ),
236  _vandermonde(data.size(), detail::computePackedSize(maxOrder))
237 {
238  // Create a matrix that evaluates the max-order polynomials of all the (scaled) input positions;
239  // we'll extract subsets of this later when fitting to a subset of the matches and a lower order.
240  for (std::size_t i = 0; i < data.size(); ++i) {
241  afw::geom::Point2D input = getInputScaling()(_data[i].get(_keys.input));
242  // x[k] == pow(x, k), y[k] == pow(y, k)
243  detail::computePowers(_transform._poly._u, input.getX());
244  detail::computePowers(_transform._poly._v, input.getY());
245  // We pack coefficients in the following order:
246  // (0,0), (0,1), (1,0), (0,2), (1,1), (2,0)
247  // Note that this lets us choose the just first N(N+1)/2 columns to
248  // evaluate an Nth order polynomial, even if N < maxOrder.
249  for (int n = 0, j = 0; n <= maxOrder; ++n) {
250  for (int p = 0, q = n; p <= n; ++p, --q, ++j) {
251  _vandermonde(i, j) = _transform._poly._u[p] * _transform._poly._v[q];
252  }
253  }
254  }
255 }
256 
258  int maxOrder = _transform.getPoly().getOrder();
259  if (order < 0) {
260  order = maxOrder;
261  }
262  if (order > maxOrder) {
263  throw LSST_EXCEPT(
265  (boost::format("Order (%d) exceeded maximum order for the fitter (%d)")
266  % order % maxOrder).str()
267  );
268  }
269 
270  int const packedSize = detail::computePackedSize(order);
271  std::size_t nGood = 0;
272  if (_keys.rejected.isValid()) {
273  for (auto const & record : _data) {
274  if (!record.get(_keys.rejected)) {
275  ++nGood;
276  }
277  }
278  } else {
279  nGood = _data.size();
280  }
281  // One block of the block-diagonal (2x2) unweighted design matrix M;
282  // m[i,j] = u_i^{p(j)} v_i^{q(j)}. The two nonzero blocks are the same,
283  // because we're using the same polynomial basis for x and y.
284  Eigen::MatrixXd m = Eigen::MatrixXd::Zero(nGood, packedSize);
285  // vx, vy: (2x1) blocks of the unweighted data vector v
286  Eigen::VectorXd vx = Eigen::VectorXd::Zero(nGood);
287  Eigen::VectorXd vy = Eigen::VectorXd::Zero(nGood);
288  // sxx, syy, sxy: (2x2) blocks of the covariance matrix S, each of which is
289  // individually diagonal.
290  Eigen::ArrayXd sxx(nGood);
291  Eigen::ArrayXd syy(nGood);
292  Eigen::ArrayXd sxy(nGood);
293  Eigen::Matrix2d outS = _outputScaling.getLinear().getMatrix();
294  for (std::size_t i1 = 0, i2 = 0; i1 < _data.size(); ++i1) {
295  // check that the 'rejected' field (== 'not outlier-rejected') is both
296  // present in the schema and not rejected.
297  if (!_keys.rejected.isValid() || !_data[i1].get(_keys.rejected)) {
298  afw::geom::Point2D output = _outputScaling(_data[i1].get(_keys.output));
299  vx[i2] = output.getX();
300  vy[i2] = output.getY();
301  m.row(i2) = _vandermonde.row(i1).head(packedSize);
302  if (_keys.outputErr.isValid()) {
303  Eigen::Matrix2d modelErr = outS*_data[i1].get(_keys.outputErr).cast<double>()*outS.adjoint();
304  sxx[i2] = modelErr(0, 0);
305  sxy[i2] = modelErr(0, 1);
306  syy[i2] = modelErr(1, 1);
307  } else {
308  sxx[i2] = 1.0;
309  sxy[i2] = 0.0;
310  syy[i2] = 1.0;
311  }
312  ++i2;
313  }
314  }
315  // Do a blockwise inverse of S. Note that the result F is still symmetric
316  Eigen::ArrayXd fxx = 1.0/(sxx - sxy.square()/syy);
317  Eigen::ArrayXd fyy = 1.0/(syy - sxy.square()/sxx);
318  Eigen::ArrayXd fxy = -(sxy/sxx)*fyy;
319 #ifdef LSST_ScaledPolynomialTransformFitter_TEST_IN_PLACE
320  assert((sxx*fxx + sxy*fxy).isApproxToConstant(1.0));
321  assert((syy*fyy + sxy*fxy).isApproxToConstant(1.0));
322  assert((sxx*fxy).isApprox(-sxy*fyy));
323  assert((sxy*fxx).isApprox(-syy*fxy));
324 #endif
325  // Now that we've got all the block quantities, we'll form the full normal equations matrix.
326  // That's H = M^T F M:
327  Eigen::MatrixXd h(2*packedSize, 2*packedSize);
328  h.topLeftCorner(packedSize, packedSize) = m.adjoint() * fxx.matrix().asDiagonal() * m;
329  h.topRightCorner(packedSize, packedSize) = m.adjoint() * fxy.matrix().asDiagonal() * m;
330  h.bottomLeftCorner(packedSize, packedSize) = h.topRightCorner(packedSize, packedSize).adjoint();
331  h.bottomRightCorner(packedSize, packedSize) = m.adjoint() * fyy.matrix().asDiagonal() * m;
332  // And here's the corresponding RHS vector, g = M^T F v
333  Eigen::VectorXd g(2*packedSize);
334  g.head(packedSize) = m.adjoint() * (fxx.matrix().asDiagonal()*vx + fxy.matrix().asDiagonal()*vy);
335  g.tail(packedSize) = m.adjoint() * (fxy.matrix().asDiagonal()*vx + fyy.matrix().asDiagonal()*vy);
336  // Solve the normal equations.
338  auto solution = lstsq.getSolution();
339  // Unpack the solution vector back into the polynomial coefficient matrices.
340  for (int n = 0, j = 0; n <= order; ++n) {
341  for (int p = 0, q = n; p <= n; ++p, --q, ++j) {
342  _transform._poly._xCoeffs(p, q) = solution[j];
343  _transform._poly._yCoeffs(p, q) = solution[j + packedSize];
344  }
345  }
346 }
347 
349  for (auto & record : _data) {
350  record.set(
351  _keys.model,
352  _transform(record.get(_keys.input))
353  );
354  }
355 }
356 
358  if (!_keys.rejected.isValid()) {
359  throw LSST_EXCEPT(
361  "Cannot compute intrinsic scatter on fitter initialized with fromGrid."
362  );
363  }
364  double newIntrinsicScatter = computeIntrinsicScatter();
365  float varDiff = newIntrinsicScatter*newIntrinsicScatter - _intrinsicScatter*_intrinsicScatter;
366  for (auto & record : _data) {
367  record.set(_keys.outputErr, record.get(_keys.outputErr) + varDiff*Eigen::Matrix2f::Identity());
368  }
369  _intrinsicScatter = newIntrinsicScatter;
370  return _intrinsicScatter;
371 }
372 
373 double ScaledPolynomialTransformFitter::computeIntrinsicScatter() const {
374  // We model the variance matrix of each match as the sum of the intrinsic variance (which we're
375  // trying to fit) and the per-source measurement uncertainty.
376  // We start by computing the variance directly, which yields the sum.
377  // At the same time, we find the maximum per-source variance. Since the per-source uncertainties
378  // are actually 2x2 matrices, we use the square of the major axis of that ellipse.
379  double directVariance = 0.0; // direct estimate of total scatter (includes measurement errors)
380  double maxMeasurementVariance = 0.0; // maximum of the per-match measurement uncertainties
381  double oldIntrinsicVariance = _intrinsicScatter*_intrinsicScatter;
382  std::size_t nGood = 0;
383  for (auto const & record : _data) {
384  if (!_keys.rejected.isValid() || !record.get(_keys.rejected)) {
385  auto delta = record.get(_keys.output) - record.get(_keys.model);
386  directVariance += 0.5*delta.computeSquaredNorm();
387  double cxx = _keys.outputErr.getElement(record, 0, 0) - oldIntrinsicVariance;
388  double cyy = _keys.outputErr.getElement(record, 1, 1) - oldIntrinsicVariance;
389  double cxy = _keys.outputErr.getElement(record, 0, 1);
390  // square of semimajor axis of uncertainty error ellipse
391  double ca2 = 0.5*(cxx + cyy + std::sqrt(cxx*cxx + cyy*cyy + 4*cxy*cxy - 2*cxx*cyy));
392  maxMeasurementVariance = std::max(maxMeasurementVariance, ca2);
393  ++nGood;
394  }
395  }
396  directVariance /= nGood;
397 
398  // Function that computes the -log likelihood of the current deltas with
399  // the variance modeled as described above.
400  auto logLikelihood = [this](double intrinsicVariance) {
401  // Uncertainties in the table right now include the old intrinsic scatter; need to
402  // subtract it off as we add the new one in.
403  double varDiff = intrinsicVariance - this->_intrinsicScatter*this->_intrinsicScatter;
404  double q = 0.0;
405  for (auto & record : this->_data) {
406  double x = record.get(this->_keys.output.getX()) - record.get(_keys.model.getX());
407  double y = record.get(this->_keys.output.getY()) - record.get(_keys.model.getY());
408  double cxx = this->_keys.outputErr.getElement(record, 0, 0) + varDiff;
409  double cyy = this->_keys.outputErr.getElement(record, 1, 1) + varDiff;
410  double cxy = this->_keys.outputErr.getElement(record, 0, 1);
411  double det = cxx*cyy - cxy*cxy;
412  q += (x*x*cyy - 2*x*y*cxy + y*y*cxx)/det + std::log(det);
413  }
414  return q;
415  };
416 
417  // directVariance brackets the intrinsic variance from above, and this quantity
418  // brackets it from below:
419  double minIntrinsicVariance = std::max(0.0, directVariance - maxMeasurementVariance);
420 
421  // Minimize the negative log likelihood to find the best-fit intrinsic variance.
422  static constexpr int BITS_REQUIRED = 16; // solution good to ~1E-4
423  boost::uintmax_t maxIterations = 20;
424  auto result = boost::math::tools::brent_find_minima(
425  logLikelihood,
426  minIntrinsicVariance,
427  directVariance,
428  BITS_REQUIRED,
429  maxIterations
430  );
431  return std::sqrt(result.first); // return RMS instead of variance
432 }
433 
434 
436  OutlierRejectionControl const & ctrl
437 ) {
438  // If the 'rejected' field isn't present in the schema (because the fitter
439  // was constructed with fromGrid), we can't do outlier rejection.
440  if (!_keys.rejected.isValid()) {
441  throw LSST_EXCEPT(
443  "Cannot reject outliers on fitter initialized with fromGrid."
444  );
445  }
446  if (static_cast<std::size_t>(ctrl.nClipMin) >= _data.size()) {
447  throw LSST_EXCEPT(
449  (boost::format("Not enough values (%d) to clip %d.")
450  % _data.size() % ctrl.nClipMin).str()
451  );
452  }
454  for (auto & record : _data) {
455  Eigen::Matrix2d cov = record.get(_keys.outputErr).cast<double>();
456  Eigen::Vector2d d = (record.get(_keys.output) - record.get(_keys.model)).asEigen();
457  double r2 = d.dot(cov.inverse() * d);
458  rankings.insert(std::make_pair(r2, &record));
459  }
460  auto cutoff = rankings.upper_bound(ctrl.nSigma * ctrl.nSigma);
461  int nClip = 0, nGood = 0;
462  for (auto iter = rankings.begin(); iter != cutoff; ++iter) {
463  iter->second->set(_keys.rejected, false);
464  ++nGood;
465  }
466  for (auto iter = cutoff; iter != rankings.end(); ++iter) {
467  iter->second->set(_keys.rejected, true);
468  ++nClip;
469  }
470  assert(static_cast<std::size_t>(nGood + nClip) == _data.size());
471  while (nClip < ctrl.nClipMin) {
472  --cutoff;
473  cutoff->second->set(_keys.rejected, true);
474  ++nClip;
475  }
476  while (nClip > ctrl.nClipMax && cutoff != rankings.end()) {
477  cutoff->second->set(_keys.rejected, false);
478  ++cutoff;
479  --nClip;
480  }
481  std::pair<double,std::size_t> result(ctrl.nSigma, nClip);
482  if (cutoff != rankings.end()) {
483  result.first = std::sqrt(cutoff->first);
484  }
485  return result;
486 }
487 
488 
489 
490 }}} // namespace lsst::meas::astrom
daf::base::Citizen & getCitizen()
double updateIntrinsicScatter()
Infer the intrinsic scatter in the offset between the data points and the best-fit model...
static PointKey addFields(Schema &schema, std::string const &name, std::string const &doc, std::string const &unit)
int nClipMax
"Never clip more than this many matches." ;
static LeastSquares fromNormalEquations(ndarray::Array< T1, 2, C1 > const &fisher, ndarray::Array< T2, 1, C2 > const &rhs, Factorization factorization=NORMAL_EIGENSYSTEM)
geom::Point2D skyToPixel(geom::Angle sky1, geom::Angle sky2) const
int nClipMin
"Always clip at least this many matches." ;
T log(T... args)
static LinearTransform makeScaling(double s)
std::pair< double, size_t > rejectOutliers(OutlierRejectionControl const &ctrl)
Mark outliers in the data catalog using sigma clipping.
Key< T > addField(Field< T > const &field, bool doReplace=false)
A fitter class for scaled polynomial transforms.
STL class.
void fit(int order=-1)
Perform a linear least-squares fit of the polynomial coefficients.
int computePackedSize(int order)
Compute this size of a packed 2-d polynomial coefficient array.
void updateModel()
Update the &#39;model&#39; field in the data catalog using the current best- fit transform.
Point2D const getCenter() const
double getHeight() const
void include(Point2D const &point)
T make_pair(T... args)
double x
geom::Point2D skyToIntermediateWorldCoord(coord::Coord const &coord) const
T max(T... args)
A 2-d coordinate transform represented by a lazy composition of an AffineTransform, a PolynomialTransform, and another AffineTransform.
T insert(T... args)
void reserve(size_type n)
static ScaledPolynomialTransformFitter fromGrid(int maxOrder, afw::geom::Box2D const &bbox, int nGridX, int nGridY, ScaledPolynomialTransform const &toInvert)
Initialize a fit that inverts an existing transform by evaluating and fitting to points on a grid...
T size(T... args)
#define LSST_EXCEPT(type,...)
size_type size() const
T sqrt(T... args)
m
Control object for outlier rejection in ScaledPolynomialTransformFitter.
afw::geom::AffineTransform const & getInputScaling() const
Return the input scaling transform that maps input data points to [-1, 1].
static ScaledPolynomialTransformFitter fromMatches(int maxOrder, afw::table::ReferenceMatchVector const &matches, afw::image::Wcs const &initialWcs, double intrinsicScatter)
Initialize a fit from intermediate world coordinates to pixels using source/reference matches...
AffineTransform const invert() const
std::shared_ptr< RecordT > addNew()
double getWidth() const
void computePowers(Eigen::VectorXd &r, double x)
Fill an array with integer powers of x, so .
A 2-d coordinate transform represented by a pair of standard polynomials (one for each coordinate)...
Point2D const getMin() const