lsst.meas.astrom  14.0-7-g0d69b06+3
ScaledPolynomialTransformFitter.cc
Go to the documentation of this file.
1 // -*- LSST-C++ -*-
2 
3 /*
4  * LSST Data Management System
5  * Copyright 2016 LSST/AURA
6  *
7  * This product includes software developed by the
8  * LSST Project (http://www.lsst.org/).
9  *
10  * This program is free software: you can redistribute it and/or modify
11  * it under the terms of the GNU General Public License as published by
12  * the Free Software Foundation, either version 3 of the License, or
13  * (at your option) any later version.
14  *
15  * This program is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18  * GNU General Public License for more details.
19  *
20  * You should have received a copy of the LSST License Statement and
21  * the GNU General Public License along with this program. If not,
22  * see <http://www.lsstcorp.org/LegalNotices/>.
23  */
24 
25 #include <map>
26 
27 #include "Eigen/LU" // for determinant, even though it's a 2x2 that doesn't use actual LU implementation
28 
29 #include "boost/math/tools/minima.hpp"
30 
34 #include "lsst/afw/table/Match.h"
36 
37 // When developing this code, it was used to add an in-line check for the
38 // correctness of a particularly complex calculation that was difficult to
39 // factor out into something unit-testable (the larger context that code is in
40 // *is* unit tested, which should be guard against regressions). The test
41 // code is still in place, guarded by a check against this preprocessor macro;
42 // set it to 1 to re-enable that test during development, but be sure to
43 // set it back to zero before committing.
44 #define LSST_ScaledPolynomialTransformFitter_TEST_IN_PLACE 0
45 
46 namespace lsst { namespace meas { namespace astrom {
47 
48 // A singleton struct that manages the schema and keys for polynomial fitting catalogs.
50 public:
59  // We use uint16 instead of Flag since it's the only bool we have here, we
60  // may want NumPy views, and afw::table doesn't support [u]int8 fields.
62 
63  Keys(Keys const &) = delete;
64  Keys(Keys &&) = delete;
65  Keys & operator=(Keys const &) = delete;
66  Keys & operator=(Keys &&) = delete;
67 
68  static Keys const & forMatches() {
69  static Keys const it(0);
70  return it;
71  }
72 
73  static Keys const & forGrid() {
74  static Keys const it;
75  return it;
76  }
77 
78 private:
79 
80  Keys(int) :
81  schema(),
82  refId(schema.addField<afw::table::RecordId>("ref_id", "ID of reference object in this match.")),
83  srcId(schema.addField<afw::table::RecordId>("src_id", "ID of source object in this match.")),
84  output(
86  schema, "src", "source positions in pixel coordinates.", "pix"
87  )
88  ),
89  input(
91  schema, "intermediate", "reference positions in intermediate world coordinates", "deg"
92  )
93  ),
94  initial(
96  schema, "initial", "reference positions transformed by initial WCS", "pix"
97  )
98  ),
99  model(
101  schema, "model", "result of applying transform to reference positions", "pix"
102  )
103  ),
104  outputErr(
106  schema, "src", {"x", "y"}, "pix"
107  )
108  ),
109  rejected(
110  schema.addField<std::uint16_t>(
111  "rejected",
112  "True if the match should be rejected from the fit."
113  )
114  )
115  {
116  schema.getCitizen().markPersistent();
117  }
118 
119  Keys() :
120  schema(),
121  output(
123  schema, "output", "grid output positions in intermediate world coordinates", "deg"
124  )
125  ),
126  input(
128  schema, "input", "grid input positions in pixel coordinates.", "pix"
129  )
130  ),
131  model(
133  schema, "model", "result of applying transform to input positions", "deg"
134  )
135  )
136  {
137  schema.getCitizen().markPersistent();
138  }
139 
140 };
141 
142 namespace {
143 
144 // Return the AffineTransforms that maps the given (x,y) coordinates to lie within (-1, 1)x(-1, 1)
145 afw::geom::AffineTransform computeScaling(
146  afw::table::BaseCatalog const & data,
147  afw::table::Point2DKey const & key
148 ) {
150  for (auto const & record : data) {
151  bbox.include(afw::geom::Point2D(record.get(key)));
152  };
156 }
157 
158 } // anonymous
159 
161  int maxOrder,
162  afw::table::ReferenceMatchVector const & matches,
163  afw::geom::SkyWcs const & initialWcs,
164  double intrinsicScatter
165 ) {
166  Keys const & keys = Keys::forMatches();
167  afw::table::BaseCatalog catalog(keys.schema);
168  catalog.reserve(matches.size());
169  float var2 = intrinsicScatter*intrinsicScatter;
170  auto initialIwcToSky = getIntermediateWorldCoordsToSky(initialWcs);
171  for (auto const & match : matches) {
172  auto record = catalog.addNew();
173  record->set(keys.refId, match.first->getId());
174  record->set(keys.srcId, match.second->getId());
175  record->set(keys.input, initialIwcToSky->applyInverse(match.first->getCoord()));
176  record->set(keys.initial, initialWcs.skyToPixel(match.first->getCoord()));
177  record->set(keys.output, match.second->getCentroid());
178  record->set(keys.outputErr, match.second->getCentroidErr() + var2*Eigen::Matrix2f::Identity());
179  record->set(keys.rejected, false);
180  }
182  catalog,
183  keys,
184  maxOrder,
185  intrinsicScatter,
186  computeScaling(catalog, keys.input),
187  computeScaling(catalog, keys.output)
188  );
189 }
190 
192  int maxOrder,
193  afw::geom::Box2D const & bbox,
194  int nGridX, int nGridY,
195  ScaledPolynomialTransform const & toInvert
196 ) {
197  Keys const & keys = Keys::forGrid();
198  afw::table::BaseCatalog catalog(keys.schema);
199  catalog.reserve(nGridX*nGridY);
200  afw::geom::Extent2D dx(bbox.getWidth()/nGridX, 0.0);
201  afw::geom::Extent2D dy(0.0, bbox.getHeight()/nGridY);
202  for (int iy = 0; iy < nGridY; ++iy) {
203  for (int ix = 0; ix < nGridX; ++ix) {
204  afw::geom::Point2D point = bbox.getMin() + dx*ix + dy*iy;
205  auto record = catalog.addNew();
206  record->set(keys.output, point);
207  record->set(keys.input, toInvert(point));
208  }
209  }
211  catalog,
212  keys,
213  maxOrder,
214  0.0,
215  computeScaling(catalog, keys.input),
216  computeScaling(catalog, keys.output)
217  );
218 }
219 
220 ScaledPolynomialTransformFitter::ScaledPolynomialTransformFitter(
221  afw::table::BaseCatalog const & data,
222  Keys const & keys,
223  int maxOrder,
224  double intrinsicScatter,
225  afw::geom::AffineTransform const & inputScaling,
226  afw::geom::AffineTransform const & outputScaling
227 ) :
228  _keys(keys),
229  _intrinsicScatter(intrinsicScatter),
230  _data(data),
231  _outputScaling(outputScaling),
232  _transform(
233  PolynomialTransform(maxOrder),
234  inputScaling,
235  outputScaling.invert()
236  ),
237  _vandermonde(data.size(), detail::computePackedSize(maxOrder))
238 {
239  // Create a matrix that evaluates the max-order polynomials of all the (scaled) input positions;
240  // we'll extract subsets of this later when fitting to a subset of the matches and a lower order.
241  for (std::size_t i = 0; i < data.size(); ++i) {
242  afw::geom::Point2D input = getInputScaling()(_data[i].get(_keys.input));
243  // x[k] == pow(x, k), y[k] == pow(y, k)
244  detail::computePowers(_transform._poly._u, input.getX());
245  detail::computePowers(_transform._poly._v, input.getY());
246  // We pack coefficients in the following order:
247  // (0,0), (0,1), (1,0), (0,2), (1,1), (2,0)
248  // Note that this lets us choose the just first N(N+1)/2 columns to
249  // evaluate an Nth order polynomial, even if N < maxOrder.
250  for (int n = 0, j = 0; n <= maxOrder; ++n) {
251  for (int p = 0, q = n; p <= n; ++p, --q, ++j) {
252  _vandermonde(i, j) = _transform._poly._u[p] * _transform._poly._v[q];
253  }
254  }
255  }
256 }
257 
259  int maxOrder = _transform.getPoly().getOrder();
260  if (order < 0) {
261  order = maxOrder;
262  }
263  if (order > maxOrder) {
264  throw LSST_EXCEPT(
266  (boost::format("Order (%d) exceeded maximum order for the fitter (%d)")
267  % order % maxOrder).str()
268  );
269  }
270 
271  int const packedSize = detail::computePackedSize(order);
272  std::size_t nGood = 0;
273  if (_keys.rejected.isValid()) {
274  for (auto const & record : _data) {
275  if (!record.get(_keys.rejected)) {
276  ++nGood;
277  }
278  }
279  } else {
280  nGood = _data.size();
281  }
282  // One block of the block-diagonal (2x2) unweighted design matrix M;
283  // m[i,j] = u_i^{p(j)} v_i^{q(j)}. The two nonzero blocks are the same,
284  // because we're using the same polynomial basis for x and y.
285  Eigen::MatrixXd m = Eigen::MatrixXd::Zero(nGood, packedSize);
286  // vx, vy: (2x1) blocks of the unweighted data vector v
287  Eigen::VectorXd vx = Eigen::VectorXd::Zero(nGood);
288  Eigen::VectorXd vy = Eigen::VectorXd::Zero(nGood);
289  // sxx, syy, sxy: (2x2) blocks of the covariance matrix S, each of which is
290  // individually diagonal.
291  Eigen::ArrayXd sxx(nGood);
292  Eigen::ArrayXd syy(nGood);
293  Eigen::ArrayXd sxy(nGood);
294  Eigen::Matrix2d outS = _outputScaling.getLinear().getMatrix();
295  for (std::size_t i1 = 0, i2 = 0; i1 < _data.size(); ++i1) {
296  // check that the 'rejected' field (== 'not outlier-rejected') is both
297  // present in the schema and not rejected.
298  if (!_keys.rejected.isValid() || !_data[i1].get(_keys.rejected)) {
299  afw::geom::Point2D output = _outputScaling(_data[i1].get(_keys.output));
300  vx[i2] = output.getX();
301  vy[i2] = output.getY();
302  m.row(i2) = _vandermonde.row(i1).head(packedSize);
303  if (_keys.outputErr.isValid()) {
304  Eigen::Matrix2d modelErr = outS*_data[i1].get(_keys.outputErr).cast<double>()*outS.adjoint();
305  sxx[i2] = modelErr(0, 0);
306  sxy[i2] = modelErr(0, 1);
307  syy[i2] = modelErr(1, 1);
308  } else {
309  sxx[i2] = 1.0;
310  sxy[i2] = 0.0;
311  syy[i2] = 1.0;
312  }
313  ++i2;
314  }
315  }
316  // Do a blockwise inverse of S. Note that the result F is still symmetric
317  Eigen::ArrayXd fxx = 1.0/(sxx - sxy.square()/syy);
318  Eigen::ArrayXd fyy = 1.0/(syy - sxy.square()/sxx);
319  Eigen::ArrayXd fxy = -(sxy/sxx)*fyy;
320 #ifdef LSST_ScaledPolynomialTransformFitter_TEST_IN_PLACE
321  assert((sxx*fxx + sxy*fxy).isApproxToConstant(1.0));
322  assert((syy*fyy + sxy*fxy).isApproxToConstant(1.0));
323  assert((sxx*fxy).isApprox(-sxy*fyy));
324  assert((sxy*fxx).isApprox(-syy*fxy));
325 #endif
326  // Now that we've got all the block quantities, we'll form the full normal equations matrix.
327  // That's H = M^T F M:
328  Eigen::MatrixXd h(2*packedSize, 2*packedSize);
329  h.topLeftCorner(packedSize, packedSize) = m.adjoint() * fxx.matrix().asDiagonal() * m;
330  h.topRightCorner(packedSize, packedSize) = m.adjoint() * fxy.matrix().asDiagonal() * m;
331  h.bottomLeftCorner(packedSize, packedSize) = h.topRightCorner(packedSize, packedSize).adjoint();
332  h.bottomRightCorner(packedSize, packedSize) = m.adjoint() * fyy.matrix().asDiagonal() * m;
333  // And here's the corresponding RHS vector, g = M^T F v
334  Eigen::VectorXd g(2*packedSize);
335  g.head(packedSize) = m.adjoint() * (fxx.matrix().asDiagonal()*vx + fxy.matrix().asDiagonal()*vy);
336  g.tail(packedSize) = m.adjoint() * (fxy.matrix().asDiagonal()*vx + fyy.matrix().asDiagonal()*vy);
337  // Solve the normal equations.
339  auto solution = lstsq.getSolution();
340  // Unpack the solution vector back into the polynomial coefficient matrices.
341  for (int n = 0, j = 0; n <= order; ++n) {
342  for (int p = 0, q = n; p <= n; ++p, --q, ++j) {
343  _transform._poly._xCoeffs(p, q) = solution[j];
344  _transform._poly._yCoeffs(p, q) = solution[j + packedSize];
345  }
346  }
347 }
348 
350  for (auto & record : _data) {
351  record.set(
352  _keys.model,
353  _transform(record.get(_keys.input))
354  );
355  }
356 }
357 
359  if (!_keys.rejected.isValid()) {
360  throw LSST_EXCEPT(
362  "Cannot compute intrinsic scatter on fitter initialized with fromGrid."
363  );
364  }
365  double newIntrinsicScatter = computeIntrinsicScatter();
366  float varDiff = newIntrinsicScatter*newIntrinsicScatter - _intrinsicScatter*_intrinsicScatter;
367  for (auto & record : _data) {
368  record.set(_keys.outputErr, record.get(_keys.outputErr) + varDiff*Eigen::Matrix2f::Identity());
369  }
370  _intrinsicScatter = newIntrinsicScatter;
371  return _intrinsicScatter;
372 }
373 
374 double ScaledPolynomialTransformFitter::computeIntrinsicScatter() const {
375  // We model the variance matrix of each match as the sum of the intrinsic variance (which we're
376  // trying to fit) and the per-source measurement uncertainty.
377  // We start by computing the variance directly, which yields the sum.
378  // At the same time, we find the maximum per-source variance. Since the per-source uncertainties
379  // are actually 2x2 matrices, we use the square of the major axis of that ellipse.
380  double directVariance = 0.0; // direct estimate of total scatter (includes measurement errors)
381  double maxMeasurementVariance = 0.0; // maximum of the per-match measurement uncertainties
382  double oldIntrinsicVariance = _intrinsicScatter*_intrinsicScatter;
383  std::size_t nGood = 0;
384  for (auto const & record : _data) {
385  if (!_keys.rejected.isValid() || !record.get(_keys.rejected)) {
386  auto delta = record.get(_keys.output) - record.get(_keys.model);
387  directVariance += 0.5*delta.computeSquaredNorm();
388  double cxx = _keys.outputErr.getElement(record, 0, 0) - oldIntrinsicVariance;
389  double cyy = _keys.outputErr.getElement(record, 1, 1) - oldIntrinsicVariance;
390  double cxy = _keys.outputErr.getElement(record, 0, 1);
391  // square of semimajor axis of uncertainty error ellipse
392  double ca2 = 0.5*(cxx + cyy + std::sqrt(cxx*cxx + cyy*cyy + 4*cxy*cxy - 2*cxx*cyy));
393  maxMeasurementVariance = std::max(maxMeasurementVariance, ca2);
394  ++nGood;
395  }
396  }
397  directVariance /= nGood;
398 
399  // Function that computes the -log likelihood of the current deltas with
400  // the variance modeled as described above.
401  auto logLikelihood = [this](double intrinsicVariance) {
402  // Uncertainties in the table right now include the old intrinsic scatter; need to
403  // subtract it off as we add the new one in.
404  double varDiff = intrinsicVariance - this->_intrinsicScatter*this->_intrinsicScatter;
405  double q = 0.0;
406  for (auto & record : this->_data) {
407  double x = record.get(this->_keys.output.getX()) - record.get(_keys.model.getX());
408  double y = record.get(this->_keys.output.getY()) - record.get(_keys.model.getY());
409  double cxx = this->_keys.outputErr.getElement(record, 0, 0) + varDiff;
410  double cyy = this->_keys.outputErr.getElement(record, 1, 1) + varDiff;
411  double cxy = this->_keys.outputErr.getElement(record, 0, 1);
412  double det = cxx*cyy - cxy*cxy;
413  q += (x*x*cyy - 2*x*y*cxy + y*y*cxx)/det + std::log(det);
414  }
415  return q;
416  };
417 
418  // directVariance brackets the intrinsic variance from above, and this quantity
419  // brackets it from below:
420  double minIntrinsicVariance = std::max(0.0, directVariance - maxMeasurementVariance);
421 
422  // Minimize the negative log likelihood to find the best-fit intrinsic variance.
423  static constexpr int BITS_REQUIRED = 16; // solution good to ~1E-4
424  boost::uintmax_t maxIterations = 20;
425  auto result = boost::math::tools::brent_find_minima(
426  logLikelihood,
427  minIntrinsicVariance,
428  directVariance,
429  BITS_REQUIRED,
430  maxIterations
431  );
432  return std::sqrt(result.first); // return RMS instead of variance
433 }
434 
435 
437  OutlierRejectionControl const & ctrl
438 ) {
439  // If the 'rejected' field isn't present in the schema (because the fitter
440  // was constructed with fromGrid), we can't do outlier rejection.
441  if (!_keys.rejected.isValid()) {
442  throw LSST_EXCEPT(
444  "Cannot reject outliers on fitter initialized with fromGrid."
445  );
446  }
447  if (static_cast<std::size_t>(ctrl.nClipMin) >= _data.size()) {
448  throw LSST_EXCEPT(
450  (boost::format("Not enough values (%d) to clip %d.")
451  % _data.size() % ctrl.nClipMin).str()
452  );
453  }
455  for (auto & record : _data) {
456  Eigen::Matrix2d cov = record.get(_keys.outputErr).cast<double>();
457  Eigen::Vector2d d = (record.get(_keys.output) - record.get(_keys.model)).asEigen();
458  double r2 = d.dot(cov.inverse() * d);
459  rankings.insert(std::make_pair(r2, &record));
460  }
461  auto cutoff = rankings.upper_bound(ctrl.nSigma * ctrl.nSigma);
462  int nClip = 0, nGood = 0;
463  for (auto iter = rankings.begin(); iter != cutoff; ++iter) {
464  iter->second->set(_keys.rejected, false);
465  ++nGood;
466  }
467  for (auto iter = cutoff; iter != rankings.end(); ++iter) {
468  iter->second->set(_keys.rejected, true);
469  ++nClip;
470  }
471  assert(static_cast<std::size_t>(nGood + nClip) == _data.size());
472  while (nClip < ctrl.nClipMin) {
473  --cutoff;
474  cutoff->second->set(_keys.rejected, true);
475  ++nClip;
476  }
477  while (nClip > ctrl.nClipMax && cutoff != rankings.end()) {
478  cutoff->second->set(_keys.rejected, false);
479  ++cutoff;
480  --nClip;
481  }
482  std::pair<double,std::size_t> result(ctrl.nSigma, nClip);
483  if (cutoff != rankings.end()) {
484  result.first = std::sqrt(cutoff->first);
485  }
486  return result;
487 }
488 
489 
490 
491 }}} // namespace lsst::meas::astrom
daf::base::Citizen & getCitizen()
double updateIntrinsicScatter()
Infer the intrinsic scatter in the offset between the data points and the best-fit model...
static PointKey addFields(Schema &schema, std::string const &name, std::string const &doc, std::string const &unit)
int nClipMax
"Never clip more than this many matches." ;
static LeastSquares fromNormalEquations(ndarray::Array< T1, 2, C1 > const &fisher, ndarray::Array< T2, 1, C2 > const &rhs, Factorization factorization=NORMAL_EIGENSYSTEM)
int nClipMin
"Always clip at least this many matches." ;
T log(T... args)
static LinearTransform makeScaling(double s)
std::pair< double, size_t > rejectOutliers(OutlierRejectionControl const &ctrl)
Mark outliers in the data catalog using sigma clipping.
Key< T > addField(Field< T > const &field, bool doReplace=false)
A fitter class for scaled polynomial transforms.
STL class.
void fit(int order=-1)
Perform a linear least-squares fit of the polynomial coefficients.
int computePackedSize(int order)
Compute this size of a packed 2-d polynomial coefficient array.
void updateModel()
Update the &#39;model&#39; field in the data catalog using the current best- fit transform.
static ScaledPolynomialTransformFitter fromMatches(int maxOrder, afw::table::ReferenceMatchVector const &matches, afw::geom::SkyWcs const &initialWcs, double intrinsicScatter)
Initialize a fit from intermediate world coordinates to pixels using source/reference matches...
Point2D const getCenter() const
double getHeight() const
void include(Point2D const &point)
T make_pair(T... args)
double x
Point2D skyToPixel(coord::IcrsCoord const &sky) const
T max(T... args)
A 2-d coordinate transform represented by a lazy composition of an AffineTransform, a PolynomialTransform, and another AffineTransform.
T insert(T... args)
void reserve(size_type n)
static ScaledPolynomialTransformFitter fromGrid(int maxOrder, afw::geom::Box2D const &bbox, int nGridX, int nGridY, ScaledPolynomialTransform const &toInvert)
Initialize a fit that inverts an existing transform by evaluating and fitting to points on a grid...
T size(T... args)
#define LSST_EXCEPT(type,...)
table::Box2IKey bbox
size_type size() const
T sqrt(T... args)
m
Control object for outlier rejection in ScaledPolynomialTransformFitter.
afw::geom::AffineTransform const & getInputScaling() const
Return the input scaling transform that maps input data points to [-1, 1].
AffineTransform const invert() const
std::shared_ptr< TransformPoint2ToIcrsCoord > getIntermediateWorldCoordsToSky(SkyWcs const &wcs, bool simplify=true)
std::shared_ptr< RecordT > addNew()
double getWidth() const
void computePowers(Eigen::VectorXd &r, double x)
Fill an array with integer powers of x, so .
A 2-d coordinate transform represented by a pair of standard polynomials (one for each coordinate)...
Point2D const getMin() const