lsst.afw  22.0.1-29-g184b6e44e+8b185d4e2d
Statistics.cc
Go to the documentation of this file.
1 // -*- LSST-C++ -*-
2 
3 /*
4  * LSST Data Management System
5  * Copyright 2008, 2009, 2010 LSST Corporation.
6  *
7  * This product includes software developed by the
8  * LSST Project (http://www.lsst.org/).
9  *
10  * This program is free software: you can redistribute it and/or modify
11  * it under the terms of the GNU General Public License as published by
12  * the Free Software Foundation, either version 3 of the License, or
13  * (at your option) any later version.
14  *
15  * This program is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18  * GNU General Public License for more details.
19  *
20  * You should have received a copy of the LSST License Statement and
21  * the GNU General Public License along with this program. If not,
22  * see <http://www.lsstcorp.org/LegalNotices/>.
23  */
24 
25 /*
26  * Support statistical operations on images
27  */
28 #include <cassert>
29 #include <cmath>
30 #include <cstdint>
31 #include <iostream>
32 #include <limits>
33 #include <memory>
34 #include <tuple>
35 #include <type_traits>
36 
37 #include "lsst/pex/exceptions.h"
38 #include "lsst/afw/image/Image.h"
40 #include "lsst/geom/Angle.h"
41 
42 using namespace std;
44 
45 namespace lsst {
46 namespace afw {
47 namespace math {
48 
49 namespace {
50 double const NaN = std::numeric_limits<double>::quiet_NaN();
51 double const MAX_DOUBLE = std::numeric_limits<double>::max();
52 double const IQ_TO_STDEV = 0.741301109252802; // 1 sigma in units of iqrange (assume Gaussian)
53 
55 class AlwaysTrue {
56 public:
57  template <typename T>
58  bool operator()(T) const {
59  return true;
60  }
61  template <typename Ta, typename Tb>
62  bool operator()(Ta, Tb) const {
63  return true;
64  }
65  template <typename Ta, typename Tb, typename Tc>
66  bool operator()(Ta, Tb, Tc) const {
67  return true;
68  }
69 };
70 
72 class AlwaysFalse {
73 public:
74  template <typename T>
75  bool operator()(T) const {
76  return false;
77  }
78  template <typename Ta, typename Tb>
79  bool operator()(Ta, Tb) const {
80  return false;
81  }
82  template <typename Ta, typename Tb, typename Tc>
83  bool operator()(Ta, Tb, Tc) const {
84  return false;
85  }
86 };
87 
89 class CheckFinite {
90 public:
91  template <typename T>
92  bool operator()(T val) const {
93  return std::isfinite(static_cast<float>(val));
94  }
95 };
96 
98 class CheckValueLtMin {
99 public:
100  template <typename Tval, typename Tmin>
101  bool operator()(Tval val, Tmin min) const {
102  return (static_cast<Tmin>(val) < min);
103  }
104 };
105 
107 class CheckValueGtMax {
108 public:
109  template <typename Tval, typename Tmax>
110  bool operator()(Tval val, Tmax max) const {
111  return (static_cast<Tmax>(val) > max);
112  }
113 };
114 
116 class CheckClipRange {
117 public:
118  template <typename Tval, typename Tcen, typename Tmax>
119  bool operator()(Tval val, Tcen center, Tmax cliplimit) const {
120  Tmax tmp = fabs(val - center);
121  return (tmp <= cliplimit);
122  }
123 };
124 
125 // define some abbreviated typenames for the test templates
126 using ChkFin = CheckFinite;
127 using ChkMin = CheckValueLtMin;
128 using ChkMax = CheckValueGtMax;
129 using ChkClip = CheckClipRange;
130 using AlwaysT = AlwaysTrue;
131 using AlwaysF = AlwaysFalse;
132 
136 inline double varianceError(double const variance, int const n) {
137  return 2 * (n - 1) * variance * variance / static_cast<double>(n * n);
138 }
139 
142 
143 /*
144  * Functions which convert the booleans into calls to the proper templated types, one type per
145  * recursion level
146  */
156 template <typename IsFinite, typename HasValueLtMin, typename HasValueGtMax, typename InClipRange,
157  bool useWeights, typename ImageT, typename MaskT, typename VarianceT, typename WeightT>
158 StandardReturn processPixels(ImageT const &img, MaskT const &msk, VarianceT const &var,
159  WeightT const &weights, int const, int const nCrude, int const stride,
160  double const meanCrude, double const cliplimit,
161  bool const weightsAreMultiplicative, int const andMask,
162  bool const calcErrorFromInputVariance,
163  std::vector<double> const &maskPropagationThresholds) {
164  int n = 0;
165  double sumw = 0.0; // sum(weight) (N.b. weight will be 1.0 if !useWeights)
166  double sumw2 = 0.0; // sum(weight^2)
167  double sumx = 0; // sum(data*weight)
168  double sumx2 = 0; // sum(data*weight^2)
169 #if 1
170  double sumvw2 = 0.0; // sum(variance*weight^2)
171 #endif
172  double min = (nCrude) ? meanCrude : MAX_DOUBLE;
173  double max = (nCrude) ? meanCrude : -MAX_DOUBLE;
174 
175  image::MaskPixel allPixelOrMask = 0x0;
176 
177  std::vector<double> rejectedWeightsByBit(maskPropagationThresholds.size(), 0.0);
178 
179  for (int iY = 0; iY < img.getHeight(); iY += stride) {
180  typename MaskT::x_iterator mptr = msk.row_begin(iY);
181  typename VarianceT::x_iterator vptr = var.row_begin(iY);
182  typename WeightT::x_iterator wptr = weights.row_begin(iY);
183 
184  for (typename ImageT::x_iterator ptr = img.row_begin(iY), end = ptr + img.getWidth(); ptr != end;
185  ++ptr, ++mptr, ++vptr, ++wptr) {
186  if (IsFinite()(*ptr) && !(*mptr & andMask) &&
187  InClipRange()(*ptr, meanCrude, cliplimit)) { // clip
188 
189  double const delta = (*ptr - meanCrude);
190 
191  if (useWeights) {
192  double weight = *wptr;
193  if (weightsAreMultiplicative) {
194  ;
195  } else {
196  if (*wptr <= 0) {
197  continue;
198  }
199  weight = 1 / weight;
200  }
201 
202  sumw += weight;
203  sumw2 += weight * weight;
204  sumx += weight * delta;
205  sumx2 += weight * delta * delta;
206 
207  if (calcErrorFromInputVariance) {
208  double const var = *vptr;
209  sumvw2 += var * weight * weight;
210  }
211  } else {
212  sumx += delta;
213  sumx2 += delta * delta;
214 
215  if (calcErrorFromInputVariance) {
216  double const var = *vptr;
217  sumvw2 += var;
218  }
219  }
220 
221  allPixelOrMask |= *mptr;
222 
223  if (HasValueLtMin()(*ptr, min)) {
224  min = *ptr;
225  }
226  if (HasValueGtMax()(*ptr, max)) {
227  max = *ptr;
228  }
229  n++;
230  } else { // pixel has been clipped, rejected, etc.
231  for (int bit = 0, nBits = maskPropagationThresholds.size(); bit < nBits; ++bit) {
232  image::MaskPixel mask = 1 << bit;
233  if (*mptr & mask) {
234  double weight = 1.0;
235  if (useWeights) {
236  weight = *wptr;
237  if (!weightsAreMultiplicative) {
238  if (*wptr <= 0) {
239  continue;
240  }
241  weight = 1.0 / weight;
242  }
243  }
244  rejectedWeightsByBit[bit] += weight;
245  }
246  }
247  }
248  }
249  }
250  if (n == 0) {
251  min = NaN;
252  max = NaN;
253  }
254 
255  // estimate of population mean and variance.
256  double mean, variance;
257  if (!useWeights) {
258  sumw = sumw2 = n;
259  }
260 
261  for (int bit = 0, nBits = maskPropagationThresholds.size(); bit < nBits; ++bit) {
262  double hypotheticalTotalWeight = sumw + rejectedWeightsByBit[bit];
263  rejectedWeightsByBit[bit] /= hypotheticalTotalWeight;
264  if (rejectedWeightsByBit[bit] > maskPropagationThresholds[bit]) {
265  allPixelOrMask |= (1 << bit);
266  }
267  }
268 
269  // N.b. if sumw == 0 or sumw*sumw == sumw2 (e.g. n == 1) we'll get NaNs
270  // N.b. the estimator of the variance assumes that the sample points all have the same variance;
271  // otherwise, what is it that we're estimating?
272  mean = sumx / sumw;
273  variance = sumx2 / sumw - ::pow(mean, 2); // biased estimator
274  variance *= sumw * sumw / (sumw * sumw - sumw2); // debias
275 
276  double meanVar; // (standard error of mean)^2
277  if (calcErrorFromInputVariance) {
278  meanVar = sumvw2 / (sumw * sumw);
279  } else {
280  meanVar = variance * sumw2 / (sumw * sumw);
281  }
282 
283  double varVar = varianceError(variance, n); // error in variance; incorrect if useWeights is true
284 
285  sumx += sumw * meanCrude;
286  mean += meanCrude;
287 
288  return StandardReturn(n, sumx, Statistics::Value(mean, meanVar), Statistics::Value(variance, varVar), min,
289  max, allPixelOrMask);
290 }
291 
292 template <typename IsFinite, typename HasValueLtMin, typename HasValueGtMax, typename InClipRange,
293  bool useWeights, typename ImageT, typename MaskT, typename VarianceT, typename WeightT>
294 StandardReturn processPixels(ImageT const &img, MaskT const &msk, VarianceT const &var,
295  WeightT const &weights, int const flags, int const nCrude, int const stride,
296  double const meanCrude, double const cliplimit,
297  bool const weightsAreMultiplicative, int const andMask,
298  bool const calcErrorFromInputVariance, bool doGetWeighted,
299  std::vector<double> const &maskPropagationThresholds) {
300  if (doGetWeighted) {
301  return processPixels<IsFinite, HasValueLtMin, HasValueGtMax, InClipRange, true>(
302  img, msk, var, weights, flags, nCrude, 1, meanCrude, cliplimit, weightsAreMultiplicative,
303  andMask, calcErrorFromInputVariance, maskPropagationThresholds);
304  } else {
305  return processPixels<IsFinite, HasValueLtMin, HasValueGtMax, InClipRange, false>(
306  img, msk, var, weights, flags, nCrude, 1, meanCrude, cliplimit, weightsAreMultiplicative,
307  andMask, calcErrorFromInputVariance, maskPropagationThresholds);
308  }
309 }
310 
311 template <typename IsFinite, typename HasValueLtMin, typename HasValueGtMax, typename InClipRange,
312  bool useWeights, typename ImageT, typename MaskT, typename VarianceT, typename WeightT>
313 StandardReturn processPixels(ImageT const &img, MaskT const &msk, VarianceT const &var,
314  WeightT const &weights, int const flags, int const nCrude, int const stride,
315  double const meanCrude, double const cliplimit,
316  bool const weightsAreMultiplicative, int const andMask,
317  bool const calcErrorFromInputVariance, bool doCheckFinite, bool doGetWeighted,
318  std::vector<double> const &maskPropagationThresholds) {
319  if (doCheckFinite) {
320  return processPixels<CheckFinite, HasValueLtMin, HasValueGtMax, InClipRange, useWeights>(
321  img, msk, var, weights, flags, nCrude, 1, meanCrude, cliplimit, weightsAreMultiplicative,
322  andMask, calcErrorFromInputVariance, doGetWeighted, maskPropagationThresholds);
323  } else {
324  return processPixels<AlwaysTrue, HasValueLtMin, HasValueGtMax, InClipRange, useWeights>(
325  img, msk, var, weights, flags, nCrude, 1, meanCrude, cliplimit, weightsAreMultiplicative,
326  andMask, calcErrorFromInputVariance, doGetWeighted, maskPropagationThresholds);
327  }
328 }
329 
347 template <typename ImageT, typename MaskT, typename VarianceT, typename WeightT>
348 StandardReturn getStandard(ImageT const &img, MaskT const &msk, VarianceT const &var, WeightT const &weights,
349  int const flags, bool const weightsAreMultiplicative, int const andMask,
350  bool const calcErrorFromInputVariance, bool doCheckFinite, bool doGetWeighted,
351  std::vector<double> const &maskPropagationThresholds) {
352  // =====================================================
353  // a crude estimate of the mean, used for numerical stability of variance
354  int nCrude = 0;
355  double meanCrude = 0.0;
356 
357  // for small numbers of values, use a small stride
358  int const nPix = img.getWidth() * img.getHeight();
359  int strideCrude;
360  if (nPix < 100) {
361  strideCrude = 2;
362  } else {
363  strideCrude = 10;
364  }
365 
366  double cliplimit = -1; // unused
367  StandardReturn values = processPixels<ChkFin, AlwaysF, AlwaysF, AlwaysT, true>(
368  img, msk, var, weights, flags, nCrude, strideCrude, meanCrude, cliplimit,
369  weightsAreMultiplicative, andMask, calcErrorFromInputVariance, doCheckFinite, doGetWeighted,
370  maskPropagationThresholds);
371  nCrude = std::get<0>(values);
372  double sumCrude = std::get<1>(values);
373 
374  meanCrude = 0.0;
375  if (nCrude > 0) {
376  meanCrude = sumCrude / nCrude;
377  }
378 
379  // =======================================================
380  // Estimate the full precision variance using that crude mean
381  // - get the min and max as well
382 
383  if (flags & (MIN | MAX)) {
384  return processPixels<ChkFin, ChkMin, ChkMax, AlwaysT, true>(
385  img, msk, var, weights, flags, nCrude, 1, meanCrude, cliplimit, weightsAreMultiplicative,
386  andMask, calcErrorFromInputVariance, true, doGetWeighted, maskPropagationThresholds);
387  } else {
388  return processPixels<ChkFin, AlwaysF, AlwaysF, AlwaysT, true>(
389  img, msk, var, weights, flags, nCrude, 1, meanCrude, cliplimit, weightsAreMultiplicative,
390  andMask, calcErrorFromInputVariance, doCheckFinite, doGetWeighted, maskPropagationThresholds);
391  }
392 }
393 
412 template <typename ImageT, typename MaskT, typename VarianceT, typename WeightT>
413 StandardReturn getStandard(ImageT const &img, MaskT const &msk, VarianceT const &var, WeightT const &weights,
414  int const flags, std::pair<double, double> const clipinfo,
415 
416  bool const weightsAreMultiplicative, int const andMask,
417  bool const calcErrorFromInputVariance, bool doCheckFinite, bool doGetWeighted,
418  std::vector<double> const &maskPropagationThresholds) {
419  double const center = clipinfo.first;
420  double const cliplimit = clipinfo.second;
421 
422  if (std::isnan(center) || std::isnan(cliplimit)) {
423  return StandardReturn(0, NaN, Statistics::Value(NaN, NaN), Statistics::Value(NaN, NaN), NaN, NaN,
424  ~0x0);
425  }
426 
427  // =======================================================
428  // Estimate the full precision variance using that crude mean
429  int const stride = 1;
430  int nCrude = 0;
431 
432  if (flags & (MIN | MAX)) {
433  return processPixels<ChkFin, ChkMin, ChkMax, ChkClip, true>(
434  img, msk, var, weights, flags, nCrude, stride, center, cliplimit, weightsAreMultiplicative,
435  andMask, calcErrorFromInputVariance, true, doGetWeighted, maskPropagationThresholds);
436  } else { // fast loop ... just the mean & variance
437  return processPixels<ChkFin, AlwaysF, AlwaysF, ChkClip, true>(
438  img, msk, var, weights, flags, nCrude, stride, center, cliplimit, weightsAreMultiplicative,
439  andMask, calcErrorFromInputVariance, doCheckFinite, doGetWeighted, maskPropagationThresholds);
440  }
441 }
442 
451 template <typename Pixel>
453 percentile(std::vector<Pixel> &img, double const fraction) {
454  assert(fraction >= 0.0 && fraction <= 1.0);
455 
456  int const n = img.size();
457 
458  if (n > 1) {
459  double const idx = fraction * (n - 1);
460 
461  // interpolate linearly between the adjacent values
462  // For efficiency:
463  // - if we're asked for a fraction > 0.5,
464  // we'll do the second partial sort on shorter (upper) portion
465  // - otherwise, the shorter portion will be the lower one, we'll partial-sort that.
466 
467  int const q1 = static_cast<int>(idx);
468  int const q2 = q1 + 1;
469 
470  auto mid1 = img.begin() + q1;
471  auto mid2 = img.begin() + q2;
472  if (fraction > 0.5) {
473  std::nth_element(img.begin(), mid1, img.end());
474  std::nth_element(mid1, mid2, img.end());
475  } else {
476  std::nth_element(img.begin(), mid2, img.end());
477  std::nth_element(img.begin(), mid1, mid2);
478  }
479 
480  double val1 = static_cast<double>(*mid1);
481  double val2 = static_cast<double>(*mid2);
482  double w1 = (static_cast<double>(q2) - idx);
483  double w2 = (idx - static_cast<double>(q1));
484  return w1 * val1 + w2 * val2;
485 
486  } else if (n == 1) {
487  return img[0];
488  } else {
489  return NaN;
490  }
491 }
492 
493 namespace {
494  //
495  // Helper function to estimate a floating-point quantile from integer data
496  //
497  // The data has been partially sorted, using nth_element
498  //
499  // begin: iterator to a point which we know to be below our median
500  // end: iterator to a point which we know to be beyond our median
501  // naive: the integer value of the desired quantile
502  // target: the number of points that should be to the left of the quantile.
503  // N.b. if begin isn't the start of the data, this may not be the
504  // desired number of points. Caveat Callor
505  template<typename T>
506  double
507  computeQuantile(typename std::vector<T>::const_iterator begin,
509  T const naive,
510  double const target
511  )
512  {
513  // investigate the cumulative histogram near naive
514  std::size_t left = 0; // number of values less than naive
515  std::size_t middle = 0; // number of values equal to naive
516 
517  for (auto ptr = begin; ptr != end; ++ptr) {
518  auto const val = *ptr;
519  if (val < naive) {
520  ++left;
521  } else if (val == naive) {
522  ++middle;
523  }
524  }
525 
526  return naive - 0.5 + (target - left)/middle;
527  }
528 
537  template <typename Pixel>
539  percentile(std::vector<Pixel> &img, double const fraction) {
540  assert(fraction >= 0.0 && fraction <= 1.0);
541 
542  auto const n = img.size();
543 
544  if (n == 0) {
545  return NaN;
546  } else if (n == 1) {
547  return img[0];
548  } else {
549  // We need to handle ties. The proper way to do this is to analyse the cumulative curve after
550  // building the histograms (which is faster than a generic partitioning algorithm), but it's a
551  // nuisance as we don't know the range of values
552  //
553  // This code looks clean enough, but actually the call to nth_element is expensive
554  // and we *still* have to go through the array a second time
555 
556  double const idx = fraction*(n - 1);
557 
558  auto midP = img.begin() + static_cast<int>(idx);
559  std::nth_element(img.begin(), midP, img.end());
560  auto const naiveP = *midP; // value of desired element
561 
562  return computeQuantile(img.begin(), img.end(), naiveP, fraction*n);
563  }
564  }
565 }
566 
567 using MedianQuartileReturn = std::tuple<double, double, double>;
568 namespace {
576  template <typename Pixel>
577  typename enable_if<!is_integral<Pixel>::value, MedianQuartileReturn>::type
578  medianAndQuartiles(std::vector<Pixel> &img) {
579  int const n = img.size();
580 
581  if (n > 1) {
582  double const idx50 = 0.50 * (n - 1);
583  double const idx25 = 0.25 * (n - 1);
584  double const idx75 = 0.75 * (n - 1);
585 
586  // For efficiency:
587  // - partition at 50th, then partition the two half further to get 25th and 75th
588  // - to get the adjacent points (for interpolation), partition between 25/50, 50/75, 75/end
589  // these should be much smaller partitions
590 
591  int const q50a = static_cast<int>(idx50);
592  int const q50b = q50a + 1;
593  int const q25a = static_cast<int>(idx25);
594  int const q25b = q25a + 1;
595  int const q75a = static_cast<int>(idx75);
596  int const q75b = q75a + 1;
597 
598  auto mid50a = img.begin() + q50a;
599  auto mid50b = img.begin() + q50b;
600  auto mid25a = img.begin() + q25a;
601  auto mid25b = img.begin() + q25b;
602  auto mid75a = img.begin() + q75a;
603  auto mid75b = img.begin() + q75b;
604 
605  // get the 50th percentile, then get the 25th and 75th on the smaller partitions
606  std::nth_element(img.begin(), mid50a, img.end());
607  std::nth_element(mid50a, mid75a, img.end());
608  std::nth_element(img.begin(), mid25a, mid50a);
609 
610  // and the adjacent points for each ... use the smallest segments available.
611  std::nth_element(mid50a, mid50b, mid75a);
612  std::nth_element(mid25a, mid25b, mid50a);
613  std::nth_element(mid75a, mid75b, img.end());
614 
615  // interpolate linearly between the adjacent values
616  double val50a = static_cast<double>(*mid50a);
617  double val50b = static_cast<double>(*mid50b);
618  double w50a = (static_cast<double>(q50b) - idx50);
619  double w50b = (idx50 - static_cast<double>(q50a));
620  double median = w50a * val50a + w50b * val50b;
621 
622  double val25a = static_cast<double>(*mid25a);
623  double val25b = static_cast<double>(*mid25b);
624  double w25a = (static_cast<double>(q25b) - idx25);
625  double w25b = (idx25 - static_cast<double>(q25a));
626  double q1 = w25a * val25a + w25b * val25b;
627 
628  double val75a = static_cast<double>(*mid75a);
629  double val75b = static_cast<double>(*mid75b);
630  double w75a = (static_cast<double>(q75b) - idx75);
631  double w75b = (idx75 - static_cast<double>(q75a));
632  double q3 = w75a * val75a + w75b * val75b;
633 
634  return MedianQuartileReturn(median, q1, q3);
635  } else if (n == 1) {
636  return MedianQuartileReturn(img[0], img[0], img[0]);
637  } else {
638  return MedianQuartileReturn(NaN, NaN, NaN);
639  }
640  }
641 
649  template <typename Pixel>
650  typename enable_if<is_integral<Pixel>::value, MedianQuartileReturn>::type
651  medianAndQuartiles(std::vector<Pixel> &img) {
652  auto const n = img.size();
653 
654  if (n == 0) {
655  return MedianQuartileReturn(NaN, NaN, NaN);
656  } else if (n == 1) {
657  return MedianQuartileReturn(img[0], img[0], img[0]);
658  } else {
659  // We need to handle ties. The proper way to do this is to analyse the cumulative curve after
660  // building the histograms (which is faster than a generic partitioning algorithm), but it's a
661  // nuisance as we don't know the range of values
662  //
663  // This code looks clean enough, but actually the call to nth_element is expensive
664  // and we *still* have to go through the array a second time.
665 
666  // For efficiency:
667  // - partition at 50th, then partition the two halves further to get 25th and 75th
668 
669  auto mid25 = img.begin() + static_cast<int>(0.25*(n - 1));
670  auto mid50 = img.begin() + static_cast<int>(0.50*(n - 1));
671  auto mid75 = img.begin() + static_cast<int>(0.75*(n - 1));
672 
673  // get the 50th percentile, then get the 25th and 75th on the smaller partitions
674  std::nth_element(img.begin(), mid50, img.end());
675  std::nth_element(img.begin(), mid25, mid50);
676  std::nth_element(mid50, mid75, img.end());
677 
678  double const q1 = computeQuantile(img.begin(), mid50, *mid25,
679  0.25*n);
680  double const median = computeQuantile(mid25, mid75, *mid50,
681  0.50*n - (mid25 - img.begin()));
682  double const q3 = computeQuantile(mid50, img.end(), *mid75,
683  0.75*n - (mid50 - img.begin()));
684 
685  return MedianQuartileReturn(median, q1, q3);
686  }
687  }
688 }
689 
697 template <typename IsFinite, typename ImageT, typename MaskT, typename VarianceT>
698 std::shared_ptr<std::vector<typename ImageT::Pixel> > makeVectorCopy(ImageT const &img, MaskT const &msk,
699  VarianceT const &, int const andMask) {
700  // Note need to keep track of allPixelOrMask here ... processPixels() does that
701  // and it always gets called
703 
704  for (int i_y = 0; i_y < img.getHeight(); ++i_y) {
705  typename MaskT::x_iterator mptr = msk.row_begin(i_y);
706  for (typename ImageT::x_iterator ptr = img.row_begin(i_y), end = img.row_end(i_y); ptr != end;
707  ++ptr) {
708  if (IsFinite()(*ptr) && !(*mptr & andMask)) {
709  imgcp->push_back(*ptr);
710  }
711  ++mptr;
712  }
713  }
714 
715  return imgcp;
716 }
717 } // namespace
718 
719 double StatisticsControl::getMaskPropagationThreshold(int bit) const {
720  int oldSize = _maskPropagationThresholds.size();
721  if (oldSize <= bit) {
722  return 1.0;
723  }
724  return _maskPropagationThresholds[bit];
725 }
726 
727 void StatisticsControl::setMaskPropagationThreshold(int bit, double threshold) {
728  int oldSize = _maskPropagationThresholds.size();
729  if (oldSize <= bit) {
730  int newSize = bit + 1;
731  _maskPropagationThresholds.resize(newSize);
732  for (int i = oldSize; i < bit; ++i) {
733  _maskPropagationThresholds[i] = 1.0;
734  }
735  }
736  _maskPropagationThresholds[bit] = threshold;
737 }
738 
740  static std::map<std::string, Property> statisticsProperty;
741  if (statisticsProperty.size() == 0) {
742  statisticsProperty["NOTHING"] = NOTHING;
743  statisticsProperty["ERRORS"] = ERRORS;
744  statisticsProperty["NPOINT"] = NPOINT;
745  statisticsProperty["MEAN"] = MEAN;
746  statisticsProperty["STDEV"] = STDEV;
747  statisticsProperty["VARIANCE"] = VARIANCE;
748  statisticsProperty["MEDIAN"] = MEDIAN;
749  statisticsProperty["IQRANGE"] = IQRANGE;
750  statisticsProperty["MEANCLIP"] = MEANCLIP;
751  statisticsProperty["STDEVCLIP"] = STDEVCLIP;
752  statisticsProperty["VARIANCECLIP"] = VARIANCECLIP;
753  statisticsProperty["MIN"] = MIN;
754  statisticsProperty["MAX"] = MAX;
755  statisticsProperty["SUM"] = SUM;
756  statisticsProperty["MEANSQUARE"] = MEANSQUARE;
757  statisticsProperty["ORMASK"] = ORMASK;
758  statisticsProperty["NCLIPPED"] = NCLIPPED;
759  statisticsProperty["NMASKED"] = NMASKED;
760  }
761  return statisticsProperty[property];
762 }
763 
764 template <typename ImageT, typename MaskT, typename VarianceT>
765 Statistics::Statistics(ImageT const &img, MaskT const &msk, VarianceT const &var, int const flags,
766  StatisticsControl const &sctrl)
767  : _flags(flags),
768  _mean(NaN, NaN),
769  _variance(NaN, NaN),
770  _min(NaN),
771  _max(NaN),
772  _sum(NaN),
773  _meanclip(NaN, NaN),
774  _varianceclip(NaN, NaN),
775  _median(NaN, NaN),
776  _nClipped(0),
777  _nMasked(0),
778  _iqrange(NaN),
779  _sctrl(sctrl),
780  _weightsAreMultiplicative(false) {
781  doStatistics(img, msk, var, var, _flags, _sctrl);
782 }
783 
784 namespace {
785 template <typename T>
786 bool isEmpty(T const &t) {
787  return t.empty();
788 }
789 
790 template <typename T>
791 bool isEmpty(image::Image<T> const &im) {
792  return (im.getWidth() == 0 && im.getHeight() == 0);
793 }
794 
795 // Asserts that image dimensions are equal
796 template <typename ImageT1, typename ImageT2>
797 void checkDimensions(ImageT1 const &image1, ImageT2 const &image2) {
798  if (image1.getDimensions() != image2.getDimensions()) {
800  (boost::format("Image sizes don't match: %s vs %s") % image1.getDimensions() %
801  image2.getDimensions())
802  .str());
803  }
804 }
805 
806 // Overloads for MaskImposter (which doesn't have a size)
807 template <typename ImageT, typename PixelT>
808 void checkDimensions(ImageT const &image1, MaskImposter<PixelT> const &image2) {}
809 template <typename ImageT, typename PixelT>
810 void checkDimensions(MaskImposter<PixelT> const &image1, ImageT const &image2) {}
811 } // namespace
812 
813 template <typename ImageT, typename MaskT, typename VarianceT, typename WeightT>
814 Statistics::Statistics(ImageT const &img, MaskT const &msk, VarianceT const &var, WeightT const &weights,
815  int const flags, StatisticsControl const &sctrl)
816  : _flags(flags),
817  _mean(NaN, NaN),
818  _variance(NaN, NaN),
819  _min(NaN),
820  _max(NaN),
821  _sum(NaN),
822  _meanclip(NaN, NaN),
823  _varianceclip(NaN, NaN),
824  _median(NaN, NaN),
825  _nClipped(0),
826  _nMasked(0),
827  _iqrange(NaN),
828  _sctrl(sctrl),
829  _weightsAreMultiplicative(true) {
830  if (!isEmpty(weights)) {
831  if (_sctrl.getWeightedIsSet() && !_sctrl.getWeighted()) {
833  "You must use the weights if you provide them");
834  }
835 
836  _sctrl.setWeighted(true);
837  }
838  doStatistics(img, msk, var, weights, _flags, _sctrl);
839 }
840 
841 template <typename ImageT, typename MaskT, typename VarianceT, typename WeightT>
842 void Statistics::doStatistics(ImageT const &img, MaskT const &msk, VarianceT const &var,
843  WeightT const &weights, int const flags, StatisticsControl const &sctrl) {
844  int const num = img.getWidth() * img.getHeight();
845  _n = num;
846  if (_n == 0) {
847  throw LSST_EXCEPT(pexExceptions::InvalidParameterError, "Image contains no pixels");
848  }
849  checkDimensions(img, msk);
850  checkDimensions(img, var);
851  if (sctrl.getWeighted()) {
852  checkDimensions(img, weights);
853  }
854 
855  // Check that an int's large enough to hold the number of pixels
856  assert(img.getWidth() * static_cast<double>(img.getHeight()) < std::numeric_limits<int>::max());
857 
858  // get the standard statistics
859  StandardReturn standard =
860  getStandard(img, msk, var, weights, flags, _weightsAreMultiplicative, _sctrl.getAndMask(),
861  _sctrl.getCalcErrorFromInputVariance(), _sctrl.getNanSafe(), _sctrl.getWeighted(),
862  _sctrl._maskPropagationThresholds);
863 
864  _n = std::get<0>(standard);
865  _sum = std::get<1>(standard);
866  _mean = std::get<2>(standard);
867  _variance = std::get<3>(standard);
868  _min = std::get<4>(standard);
869  _max = std::get<5>(standard);
870  _allPixelOrMask = std::get<6>(standard);
871 
872  // ==========================================================
873  // now only calculate it if it's specifically requested - these all cost more!
874 
875  if (flags & NMASKED) {
876  _nMasked = num - _n;
877  }
878 
879  // copy the image for any routines that will use median or quantiles
880  if (flags & (MEDIAN | IQRANGE | MEANCLIP | STDEVCLIP | VARIANCECLIP)) {
881  // make a vector copy of the image to get the median and quartiles (will move values)
883  if (_sctrl.getNanSafe()) {
884  imgcp = makeVectorCopy<ChkFin>(img, msk, var, _sctrl.getAndMask());
885  } else {
886  imgcp = makeVectorCopy<AlwaysT>(img, msk, var, _sctrl.getAndMask());
887  }
888 
889  // if we *only* want the median, just use percentile(), otherwise use medianAndQuartiles()
890  if ((flags & (MEDIAN)) && !(flags & (IQRANGE | MEANCLIP | STDEVCLIP | VARIANCECLIP))) {
891  _median = Value(percentile(*imgcp, 0.5), NaN);
892  } else {
893  MedianQuartileReturn mq = medianAndQuartiles(*imgcp);
894  _median = Value(std::get<0>(mq), NaN);
895  _iqrange = std::get<2>(mq) - std::get<1>(mq);
896  }
897 
898  if (flags & (MEANCLIP | STDEVCLIP | VARIANCECLIP)) {
899  for (int i_i = 0; i_i < _sctrl.getNumIter(); ++i_i) {
900  double const center = ((i_i > 0) ? _meanclip : _median).first;
901  double const hwidth = (i_i > 0 && _n > 1)
902  ? _sctrl.getNumSigmaClip() * std::sqrt(_varianceclip.first)
903  : _sctrl.getNumSigmaClip() * IQ_TO_STDEV * _iqrange;
904  std::pair<double, double> const clipinfo(center, hwidth);
905 
906  StandardReturn clipped = getStandard(
907  img, msk, var, weights, flags, clipinfo, _weightsAreMultiplicative,
908  _sctrl.getAndMask(), _sctrl.getCalcErrorFromInputVariance(), _sctrl.getNanSafe(),
909  _sctrl.getWeighted(), _sctrl._maskPropagationThresholds);
910 
911  int const nClip = std::get<0>(clipped); // number after clipping
912  _nClipped = _n - nClip; // number clipped
913  _meanclip = std::get<2>(clipped); // clipped mean
914  double const varClip = std::get<3>(clipped).first; // clipped variance
915 
916  _varianceclip = Value(varClip, varianceError(varClip, nClip));
917  // ... ignore other values
918  }
919  }
920  }
921 }
922 
924  // if iProp == NOTHING try to return their heart's delight, as specified in the constructor
925  Property const prop = static_cast<Property>(((iProp == NOTHING) ? _flags : iProp) & ~ERRORS);
926 
927  if (!(prop & _flags)) { // we didn't calculate it
929  (boost::format("You didn't ask me to calculate %d") % prop).str());
930  }
931 
932  Value ret(NaN, NaN);
933  switch (prop) {
934  case NPOINT:
935  ret.first = static_cast<double>(_n);
936  if (_flags & ERRORS) {
937  ret.second = 0;
938  }
939  break;
940 
941  case NCLIPPED:
942  ret.first = static_cast<double>(_nClipped);
943  if (_flags & ERRORS) {
944  ret.second = 0;
945  }
946  break;
947 
948  case NMASKED:
949  ret.first = static_cast<double>(_nMasked);
950  if (_flags & ERRORS) {
951  ret.second = 0;
952  }
953  break;
954 
955  case SUM:
956  ret.first = static_cast<double>(_sum);
957  if (_flags & ERRORS) {
958  ret.second = 0;
959  }
960  break;
961 
962  // == means ==
963  case MEAN:
964  ret.first = _mean.first;
965  if (_flags & ERRORS) {
966  ret.second = ::sqrt(_mean.second);
967  }
968  break;
969  case MEANCLIP:
970  ret.first = _meanclip.first;
971  if (_flags & ERRORS) {
972  ret.second = ::sqrt(_meanclip.second);
973  }
974  break;
975 
976  // == stdevs & variances ==
977  case VARIANCE:
978  ret.first = _variance.first;
979  if (_flags & ERRORS) {
980  ret.second = ::sqrt(_variance.second);
981  }
982  break;
983  case STDEV:
984  ret.first = sqrt(_variance.first);
985  if (_flags & ERRORS) {
986  ret.second = 0.5 * ::sqrt(_variance.second) / ret.first;
987  }
988  break;
989  case VARIANCECLIP:
990  ret.first = _varianceclip.first;
991  if (_flags & ERRORS) {
992  ret.second = ret.second;
993  }
994  break;
995  case STDEVCLIP:
996  ret.first = sqrt(_varianceclip.first);
997  if (_flags & ERRORS) {
998  ret.second = 0.5 * ::sqrt(_varianceclip.second) / ret.first;
999  }
1000  break;
1001 
1002  case MEANSQUARE:
1003  ret.first = (_n - 1) / static_cast<double>(_n) * _variance.first + ::pow(_mean.first, 2);
1004  if (_flags & ERRORS) {
1005  ret.second = ::sqrt(2 * ::pow(ret.first / _n, 2)); // assumes Gaussian
1006  }
1007  break;
1008 
1009  // == other stats ==
1010  case MIN:
1011  ret.first = _min;
1012  if (_flags & ERRORS) {
1013  ret.second = 0;
1014  }
1015  break;
1016  case MAX:
1017  ret.first = _max;
1018  if (_flags & ERRORS) {
1019  ret.second = 0;
1020  }
1021  break;
1022  case MEDIAN:
1023  ret.first = _median.first;
1024  if (_flags & ERRORS) {
1025  ret.second = sqrt(geom::HALFPI * _variance.first / _n); // assumes Gaussian
1026  }
1027  break;
1028  case IQRANGE:
1029  ret.first = _iqrange;
1030  if (_flags & ERRORS) {
1031  ret.second = NaN; // we're not estimating this properly
1032  }
1033  break;
1034 
1035  // no-op to satisfy the compiler
1036  case ERRORS:
1037  break;
1038  // default: redundant as 'ret' is initialized to NaN, NaN
1039  default: // we must have set prop to _flags
1040  assert(iProp == 0);
1042  "getValue() may only be called without a parameter"
1043  " if you asked for only one statistic");
1044  }
1045  return ret;
1046 }
1047 
1048 double Statistics::getValue(Property const prop) const { return getResult(prop).first; }
1049 
1050 double Statistics::getError(Property const prop) const { return getResult(prop).second; }
1051 
1061 template <>
1063  image::Mask<image::MaskPixel> const &var, int const flags,
1064  StatisticsControl const &sctrl)
1065  : _flags(flags),
1066  _mean(NaN, NaN),
1067  _variance(NaN, NaN),
1068  _min(NaN),
1069  _max(NaN),
1070  _meanclip(NaN, NaN),
1071  _varianceclip(NaN, NaN),
1072  _median(NaN, NaN),
1073  _nClipped(0),
1074  _iqrange(NaN),
1075  _sctrl(sctrl) {
1076  if ((flags & ~(NPOINT | SUM)) != 0x0) {
1078  "Statistics<Mask> only supports NPOINT and SUM");
1079  }
1080 
1081  using Mask = image::Mask<>;
1082 
1083  _n = msk.getWidth() * msk.getHeight();
1084  if (_n == 0) {
1085  throw LSST_EXCEPT(pexExceptions::InvalidParameterError, "Image contains no pixels");
1086  }
1087 
1088  // Check that an int's large enough to hold the number of pixels
1089  assert(msk.getWidth() * static_cast<double>(msk.getHeight()) < std::numeric_limits<int>::max());
1090 
1091  image::MaskPixel sum = 0x0;
1092  for (int y = 0; y != msk.getHeight(); ++y) {
1093  for (Mask::x_iterator ptr = msk.row_begin(y), end = msk.row_end(y); ptr != end; ++ptr) {
1094  sum |= (*ptr)[0];
1095  }
1096  }
1097  _sum = sum;
1098 }
1099 
1100 /*
1101  * Although short, the definition can't be in the header as it must
1102  * follow the specialization definition
1103  * (g++ complained when this was in the header.)
1104  */
1106  StatisticsControl const &sctrl) {
1107  return Statistics(msk, msk, msk, flags, sctrl);
1108 }
1109 
1110 /*
1111  * Explicit instantiations
1112  *
1113  * explicit Statistics(MaskedImage const& img, int const flags,
1114  * StatisticsControl const& sctrl=StatisticsControl());
1115  */
1117 //
1118 #define STAT Statistics
1119 
1120 using VPixel = image::VariancePixel;
1121 
1122 #define INSTANTIATE_MASKEDIMAGE_STATISTICS(TYPE) \
1123  template STAT::Statistics(image::Image<TYPE> const &img, image::Mask<image::MaskPixel> const &msk, \
1124  image::Image<VPixel> const &var, int const flags, \
1125  StatisticsControl const &sctrl); \
1126  template STAT::Statistics(image::Image<TYPE> const &img, image::Mask<image::MaskPixel> const &msk, \
1127  image::Image<VPixel> const &var, image::Image<VPixel> const &weights, \
1128  int const flags, StatisticsControl const &sctrl); \
1129  template STAT::Statistics(image::Image<TYPE> const &img, image::Mask<image::MaskPixel> const &msk, \
1130  image::Image<VPixel> const &var, ImageImposter<VPixel> const &weights, \
1131  int const flags, StatisticsControl const &sctrl)
1132 
1133 #define INSTANTIATE_MASKEDIMAGE_STATISTICS_NO_MASK(TYPE) \
1134  template STAT::Statistics(image::Image<TYPE> const &img, MaskImposter<image::MaskPixel> const &msk, \
1135  image::Image<VPixel> const &var, int const flags, \
1136  StatisticsControl const &sctrl); \
1137  template STAT::Statistics(image::Image<TYPE> const &img, MaskImposter<image::MaskPixel> const &msk, \
1138  image::Image<VPixel> const &var, image::Image<VPixel> const &weights, \
1139  int const flags, StatisticsControl const &sctrl)
1140 
1141 #define INSTANTIATE_MASKEDIMAGE_STATISTICS_NO_VAR(TYPE) \
1142  template STAT::Statistics(image::Image<TYPE> const &img, image::Mask<image::MaskPixel> const &msk, \
1143  MaskImposter<VPixel> const &var, int const flags, \
1144  StatisticsControl const &sctrl); \
1145  template STAT::Statistics(image::Image<TYPE> const &img, image::Mask<image::MaskPixel> const &msk, \
1146  MaskImposter<VPixel> const &var, image::Image<VPixel> const &weights, \
1147  int const flags, StatisticsControl const &sctrl); \
1148  template STAT::Statistics(image::Image<TYPE> const &img, image::Mask<image::MaskPixel> const &msk, \
1149  MaskImposter<VPixel> const &var, ImageImposter<VPixel> const &weights, \
1150  int const flags, StatisticsControl const &sctrl)
1151 
1152 #define INSTANTIATE_REGULARIMAGE_STATISTICS(TYPE) \
1153  template STAT::Statistics(image::Image<TYPE> const &img, MaskImposter<image::MaskPixel> const &msk, \
1154  MaskImposter<VPixel> const &var, int const flags, \
1155  StatisticsControl const &sctrl)
1156 
1157 #define INSTANTIATE_VECTOR_STATISTICS(TYPE) \
1158  template STAT::Statistics(ImageImposter<TYPE> const &img, MaskImposter<image::MaskPixel> const &msk, \
1159  MaskImposter<VPixel> const &var, int const flags, \
1160  StatisticsControl const &sctrl); \
1161  template STAT::Statistics(ImageImposter<TYPE> const &img, MaskImposter<image::MaskPixel> const &msk, \
1162  MaskImposter<VPixel> const &var, ImageImposter<VPixel> const &weights, \
1163  int const flags, StatisticsControl const &sctrl)
1164 
1165 #define INSTANTIATE_IMAGE_STATISTICS(T) \
1166  INSTANTIATE_MASKEDIMAGE_STATISTICS(T); \
1167  INSTANTIATE_MASKEDIMAGE_STATISTICS_NO_VAR(T); \
1168  INSTANTIATE_MASKEDIMAGE_STATISTICS_NO_MASK(T); \
1169  INSTANTIATE_REGULARIMAGE_STATISTICS(T); \
1170  INSTANTIATE_VECTOR_STATISTICS(T)
1171 
1172 INSTANTIATE_IMAGE_STATISTICS(double);
1173 INSTANTIATE_IMAGE_STATISTICS(float);
1174 INSTANTIATE_IMAGE_STATISTICS(int);
1175 INSTANTIATE_IMAGE_STATISTICS(std::uint16_t);
1176 INSTANTIATE_IMAGE_STATISTICS(std::uint64_t);
1177 
1179 } // namespace math
1180 } // namespace afw
1181 } // namespace lsst
Key< Flag > const & target
int end
table::Key< int > type
Definition: Detector.cc:163
#define LSST_EXCEPT(type,...)
afw::table::Key< afw::table::Array< MaskPixelT > > mask
afw::table::Key< afw::table::Array< VariancePixelT > > variance
int y
Definition: SpanSet.cc:49
T begin(T... args)
int getWidth() const
Return the number of columns in the image.
Definition: ImageBase.h:294
int getHeight() const
Return the number of rows in the image.
Definition: ImageBase.h:296
x_iterator row_begin(int y) const
Return an x_iterator to the start of the y'th row.
Definition: ImageBase.h:401
x_iterator row_end(int y) const
Return an x_iterator to the end of the y'th row.
Definition: ImageBase.h:404
A class to represent a 2-dimensional array of pixels.
Definition: Image.h:58
Represent a 2-dimensional array of bitmask pixels.
Definition: Mask.h:77
Pass parameters to a Statistics object.
Definition: Statistics.h:93
bool getCalcErrorFromInputVariance() const noexcept
Definition: Statistics.h:140
int getAndMask() const noexcept
Definition: Statistics.h:135
bool getWeightedIsSet() const noexcept
Definition: Statistics.h:139
void setWeighted(bool useWeights) noexcept
Definition: Statistics.h:159
double getNumSigmaClip() const noexcept
Definition: Statistics.h:133
bool getWeighted() const noexcept
Definition: Statistics.h:138
int getNumIter() const noexcept
Definition: Statistics.h:134
bool getNanSafe() const noexcept
Definition: Statistics.h:137
A class to evaluate image statistics.
Definition: Statistics.h:221
Value getResult(Property const prop=NOTHING) const
Return the value and error in the specified statistic (e.g.
Definition: Statistics.cc:923
double getError(Property const prop=NOTHING) const
Return the error in the desired property (if specified in the constructor)
Definition: Statistics.cc:1050
Statistics(ImageT const &img, MaskT const &msk, VarianceT const &var, int const flags, StatisticsControl const &sctrl=StatisticsControl())
Constructor for Statistics object.
Definition: Statistics.cc:765
double getValue(Property const prop=NOTHING) const
Return the value of the desired property (if specified in the constructor)
Definition: Statistics.cc:1048
std::pair< double, double > Value
The type used to report (value, error) for desired statistics.
Definition: Statistics.h:224
T end(T... args)
T fabs(T... args)
T isfinite(T... args)
T isnan(T... args)
T left(T... args)
T max(T... args)
T min(T... args)
float VariancePixel
default type for MaskedImage variance images
Statistics makeStatistics(lsst::afw::image::Image< Pixel > const &img, lsst::afw::image::Mask< image::MaskPixel > const &msk, int const flags, StatisticsControl const &sctrl=StatisticsControl())
Handle a watered-down front-end to the constructor (no variance)
Definition: Statistics.h:360
Property
control what is calculated
Definition: Statistics.h:63
@ ORMASK
get the or-mask of all pixels used.
Definition: Statistics.h:80
@ ERRORS
Include errors of requested quantities.
Definition: Statistics.h:65
@ VARIANCECLIP
estimate sample N-sigma clipped variance (N set in StatisticsControl, default=3)
Definition: Statistics.h:74
@ MEANSQUARE
find mean value of square of pixel values
Definition: Statistics.h:79
@ MIN
estimate sample minimum
Definition: Statistics.h:76
@ NCLIPPED
number of clipped points
Definition: Statistics.h:81
@ NOTHING
We don't want anything.
Definition: Statistics.h:64
@ STDEV
estimate sample standard deviation
Definition: Statistics.h:68
@ NMASKED
number of masked points
Definition: Statistics.h:82
@ STDEVCLIP
estimate sample N-sigma clipped stdev (N set in StatisticsControl, default=3)
Definition: Statistics.h:73
@ VARIANCE
estimate sample variance
Definition: Statistics.h:69
@ MEDIAN
estimate sample median
Definition: Statistics.h:70
@ MAX
estimate sample maximum
Definition: Statistics.h:77
@ SUM
find sum of pixels in the image
Definition: Statistics.h:78
@ IQRANGE
estimate sample inter-quartile range
Definition: Statistics.h:71
@ MEAN
estimate sample mean
Definition: Statistics.h:67
@ MEANCLIP
estimate sample N-sigma clipped mean (N set in StatisticsControl, default=3)
Definition: Statistics.h:72
@ NPOINT
number of sample points
Definition: Statistics.h:66
Property stringToStatisticsProperty(std::string const property)
Conversion function to switch a string to a Property (see Statistics.h)
Definition: Statistics.cc:739
constexpr double HALFPI
A base class for image defects.
STL namespace.
T nth_element(T... args)
T pow(T... args)
T quiet_NaN(T... args)
T size(T... args)
T sqrt(T... args)