You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by ah...@apache.org on 2022/01/29 08:58:10 UTC

[commons-statistics] branch master updated: Update f distribution pdf function

This is an automated email from the ASF dual-hosted git repository.

aherbert pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-statistics.git


The following commit(s) were added to refs/heads/master by this push:
     new b85d233  Update f distribution pdf function
b85d233 is described below

commit b85d2339ddc21080e3aeed9d9cb3f1be3b710868
Author: Alex Herbert <ah...@apache.org>
AuthorDate: Sat Jan 29 08:58:07 2022 +0000

    Update f distribution pdf function
    
    Greater accuracy is obtained using log(pdf) than using the log
    computation. However the regular pdf may have intermediate overflow with
    extreme parameters when the result is finite.
    
    Extract the pdf and logpdf computation to a single method. Try the
    regular pdf and switch to the log computation when the density is
    sub-normal or overflows.
---
 .../statistics/distribution/FDistribution.java     |  56 +++--
 .../statistics/distribution/FDistributionTest.java |  84 +++++++
 .../statistics/distribution/test.f.6.properties    | 265 ++++++++++-----------
 3 files changed, 241 insertions(+), 164 deletions(-)

diff --git a/commons-statistics-distribution/src/main/java/org/apache/commons/statistics/distribution/FDistribution.java b/commons-statistics-distribution/src/main/java/org/apache/commons/statistics/distribution/FDistribution.java
index 1a9ca94..5ca3555 100644
--- a/commons-statistics-distribution/src/main/java/org/apache/commons/statistics/distribution/FDistribution.java
+++ b/commons-statistics-distribution/src/main/java/org/apache/commons/statistics/distribution/FDistribution.java
@@ -152,21 +152,7 @@ public final class FDistribution extends AbstractContinuousDistribution {
             }
             return 0;
         }
-
-        // Keep the z argument to the regularized beta well away from 1 to avoid rounding error.
-        // See: https://www.boost.org/doc/libs/1_78_0/libs/math/doc/html/math_toolkit/dist_ref/dists/f_dist.html
-
-        final double n = numeratorDegreesOfFreedom;
-        final double m = denominatorDegreesOfFreedom;
-        final double nx = n * x;
-        final double z = m + nx;
-        final double y = n * m / (z * z);
-        if (nx > m) {
-            return y * RegularizedBeta.derivative(m / z,
-                                                  m / 2, n / 2);
-        }
-        return y * RegularizedBeta.derivative(nx / z,
-                                              n / 2, m / 2);
+        return computeDensity(x, false);
     }
 
     /** {@inheritDoc}
@@ -192,13 +178,45 @@ public final class FDistribution extends AbstractContinuousDistribution {
             }
             return Double.NEGATIVE_INFINITY;
         }
+        return computeDensity(x, true);
+    }
+
+    /**
+     * Compute the density at point x. Assumes x is within the support bound.
+     *
+     * @param x Value
+     * @param log true to compute the log density
+     * @return pdf(x) or logpdf(x)
+     */
+    private double computeDensity(double x, boolean log) {
+        // The log computation may suffer cancellation effects due to summation of large
+        // opposing terms. Use it when the standard PDF result is not normal.
+
+        // Keep the z argument to the regularized beta well away from 1 to avoid rounding error.
+        // See: https://www.boost.org/doc/libs/1_78_0/libs/math/doc/html/math_toolkit/dist_ref/dists/f_dist.html
 
         final double n = numeratorDegreesOfFreedom;
         final double m = denominatorDegreesOfFreedom;
-
-        // This may suffer cancellation effects due to summation of opposing terms.
-        return nHalfLogNmHalfLogM + (n / 2 - 1) * Math.log(x) - logBetaNhalfMhalf -
-            ((n + m) / 2) * Math.log(n * x + m);
+        final double nx = n * x;
+        final double z = m + nx;
+        final double y = n * m / (z * z);
+        double p;
+        if (nx > m) {
+            p = y * RegularizedBeta.derivative(m / z,
+                                               m / 2, n / 2);
+        } else {
+            p = y * RegularizedBeta.derivative(nx / z,
+                                               n / 2, m / 2);
+        }
+        // RegularizedBeta.derivative can have intermediate overflow before normalisation
+        // with small y. Check the result for a normal finite number.
+        if (p <= Double.MAX_VALUE && p >= Double.MIN_NORMAL) {
+            return log ? Math.log(p) : p;
+        }
+        // Fall back to the log computation
+        p = nHalfLogNmHalfLogM + (n / 2 - 1) * Math.log(x) - logBetaNhalfMhalf -
+                ((n + m) / 2) * Math.log(z);
+        return log ? p : Math.exp(p);
     }
 
     /**
diff --git a/commons-statistics-distribution/src/test/java/org/apache/commons/statistics/distribution/FDistributionTest.java b/commons-statistics-distribution/src/test/java/org/apache/commons/statistics/distribution/FDistributionTest.java
index f095114..0681dbf 100644
--- a/commons-statistics-distribution/src/test/java/org/apache/commons/statistics/distribution/FDistributionTest.java
+++ b/commons-statistics-distribution/src/test/java/org/apache/commons/statistics/distribution/FDistributionTest.java
@@ -101,4 +101,88 @@ class FDistributionTest extends BaseContinuousDistributionTest {
         final double result = f.inverseCumulativeProbability(prob);
         Assertions.assertTrue(result < 1.0, "Failing to calculate inverse cumulative probability");
     }
+
+    @Test
+    void test() {
+        FDistribution.of(1e-100, 1).logDensity(1e-300);
+    }
+
+    @Test
+    void testAdditionalLogDensity() {
+        // Computed using Boost multiprecision to 100 digits (output 25 digits).
+
+        // Edge cases when the standard density is sub-normal or zero.
+        final double[] x = new double[] {1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9};
+        testLogDensity(FDistribution.of(100, 100), x,
+                       new double[] {-56.96014024624318913110565,
+                                     -165.8559950938238412964495,
+                                     -282.3927517845117162132265,
+                                     -399.7346409939330236149964,
+                                     -517.157481231596055999464,
+                                     -634.5884209792423525846316,
+                                     -752.0201707219881824362492,
+                                     -869.4520014646850073211334,
+                                     -986.883840307381342156051},
+                       DoubleTolerances.relative(1e-15));
+
+        testLogDensity(FDistribution.of(952, 912), x,
+                       new double[] {-509.5128641158461391223255,
+                                     -1485.417858108384337659572,
+                                     -2529.705750311339816652123,
+                                     -3581.184004620825529681231,
+                                     -4633.385040722443349533971,
+                                     -5685.658392700035382988623,
+                                     -6737.938976642435125691553,
+                                     -7790.220283785087985050541,
+                                     -8842.501663247803879775318},
+                       DoubleTolerances.relative(1e-15));
+
+        // This causes intermediate overflow of the density function
+        testLogDensity(FDistribution.of(1e-100, 1),
+                       new double[] {1e-200, 1e-250, 1e-300},
+                       new double[] {229.5653621188446231302736,
+                                     344.6946167685469072592738,
+                                     459.8238714182491914891139},
+                       DoubleTolerances.relative(1e-15));
+    }
+
+    @Test
+    void testAdditionalDensity() {
+        // Edge cases when the standard density is sub-normal.
+        testDensity(FDistribution.of(100, 100),
+                    new double[] {1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 3e6, 4e6, 4.5e6, 5e6, 1e7},
+                    new double[] {1.830313161302986740491046e-25,
+                                  9.325165326363852979476269e-73,
+                                  2.282370632180103030176872e-123,
+                                  2.49718772086196154389661e-174,
+                                  2.51976260334572601372639e-225,
+                                  2.522031398014840106819471e-276,
+                                  1.171103964711921105069224e-300,
+                                  4.97420298008526384736197e-307,
+                                  1.224464123468993962344698e-309,
+                                  5.679564178845752345371413e-312,
+                                  0},
+                   DoubleTolerances.relative(3e-13));
+
+        testDensity(FDistribution.of(952, 912),
+                    new double[] {10, 11, 12, 13, 14, 15, 16, 17, 18},
+                    new double[] {5.264712450643104177155291e-222,
+                                  1.083049754753448067375765e-237,
+                                  2.996024821196787172008532e-252,
+                                  7.919262482129153149257417e-266,
+                                  1.511696585130734458293958e-278,
+                                  1.652611434344889324846565e-290,
+                                  8.522337060963566999523664e-302,
+                                  1.760000675560273604454495e-312,
+                                  1.266172656954210816606837e-322},
+                    DoubleTolerances.relative(2e-13));
+
+        // This causes intermediate overflow of the density function
+        testDensity(FDistribution.of(1e-100, 1),
+                    new double[] {1e-200, 1e-250, 1e-300},
+                    new double[] {5.000000000000000189458187e+99,
+                                  4.999999999999999829961813e+149,
+                                  4.99999999999999997466404e+199},
+                    DoubleTolerances.relative(5e-14));
+    }
 }
diff --git a/commons-statistics-distribution/src/test/resources/org/apache/commons/statistics/distribution/test.f.6.properties b/commons-statistics-distribution/src/test/resources/org/apache/commons/statistics/distribution/test.f.6.properties
index 693fa98..985c5bb 100644
--- a/commons-statistics-distribution/src/test/resources/org/apache/commons/statistics/distribution/test.f.6.properties
+++ b/commons-statistics-distribution/src/test/resources/org/apache/commons/statistics/distribution/test.f.6.properties
@@ -15,7 +15,7 @@
 
 parameters = 100.0 100.0
 # Limited by pdf values
-tolerance.relative = 3e-13
+tolerance.relative = 2e-14
 
 # Computed using scipy.stats f
 mean = 1.0204081632653061
@@ -24,154 +24,129 @@ lower = 0
 cdf.points = \
   0.0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.1, 0.25, 0.5, 0.75, 1.0, 1.25, 1.5, 1.75,  2.0, 2.25, 2.5, 2.75, 3.0, 3.25, 3.5, 3.75, 4.0, 4.25, 4.5, 4.75, 5.0
 
-cdf.values = \
-  0.00000000000000000000e+00,   1.90195008441456752357e-72,\
-  8.15317293024812868281e-58,   1.99842701059155630730e-49,\
-  1.36935327808126602372e-43,   3.75769320391289185847e-39,\
-  4.45273934155116804551e-26,   1.32861996318604157343e-11,\
-  3.09136862607055286790e-04,   7.60200343602339401183e-02,\
-  5.00000000000000000000e-01,   8.66884397722028854361e-01,\
-  9.78069557869914785186e-01,   9.97224952918882134867e-01,\
-  9.99690863137392971005e-01,   9.99967131656259833328e-01,\
-  9.99996502464976400049e-01,   9.99999617026797626984e-01,\
-  9.99999956151529589832e-01,   9.99999994703667183238e-01,\
-  9.99999999322085608711e-01,   9.99999999907889236539e-01,\
-  9.99999999986713850042e-01,   9.99999999997967625731e-01,\
-  9.99999999999670818873e-01,   9.99999999999943711693e-01,\
-  9.99999999999989785948e-01
-
-# TODO:
-# PDF values are inconsistent
-# Reference values from Mathematica, Matlab, Scipy only agree to approximately ~1e-13.
-# scipy is computed using exp(logpdf). The method of Mathematica and Matlab are unknown.
+# Note: Reference PDF values are inconsistent.
+# Values from Mathematica, Matlab, Scipy and R only agree to approximately ~1e-13.
+# scipy is computed using exp(logpdf). The other methods are unknown.
 # Note that the PDF approaches 1 for x=0.75 and x=1.25. This will be computed with
-# cancellation effects using logs (as the log pdf -> 0 with positive and negative terms in
-# a summation)
-
-# Wolfram Mathematica - pass at 8e-13 (fails on the logpdf, no log pdf is available)
-# These match closely with the current implementation.
-# Note these floating-point numbers are inexact and the full decimal representation
-# was obtained from e.g. new BigDecimal(0.1).toString().
-#  0.0 \
-#  9.3251653263638622991e-69 \
-#  1.9599569387331353308e-54 \
-#  3.1406021899708862397e-46 \
-#  1.5827035031843255814e-40 \
-#  3.4071733011342913426e-36 \
-#  1.8303131613029907954e-23 \
-#  1.621261398867281e-09 \
-#  0.0110204220067404 \
-#  0.946217456596435 \
-#  1.9897309346794690375, \
-#  0.8553281090792538 \
-#  0.172291854230817 \
-#  0.02384453848700535 \
-#  0.0027551055016848700217 \
-#  0.000295393685948072 \
-#  0.00003119248021908925 \
-#  3.357137609156691e-6 \
-#  3.7560923959053191116e-7 \
-#  4.417591985387405e-8 \
-#  5.493848943473314e-9 \
-#  7.243777042409837e-10 \
-#  1.0132883742919304427e-10 \
-#  1.502869083243862e-11 \
-#  2.360142057328525e-12 \
-#  3.91738558683421e-13 \
-#  6.8580254516416926559e-14
+# cancellation effects using logs (as the log pdf -> 0 with large positive and
+# negative terms in a summation).
 
-# Matlab - pass at 7e-13 (fails on the logpdf, no log pdf is available from Matlab)
-#  0 \
-#  9.3251653263634157e-69 \
-#  1.959956938733149e-54 \
-#  3.1406021899708079e-46 \
-#  1.5827035031842939e-40 \
-#  3.4071733011341871e-36 \
-#  1.8303131613029334e-23 \
-#  1.6212613988670735e-09 \
-#  0.0110204220067393 \
-#  0.94621745659646195 \
-#  1.9897309346794265 \
-#  0.85532810907907153 \
-#  0.17229185423079005 \
-#  0.023844538487001965 \
-#  0.0027551055016848541 \
-#  0.00029539368594804681 \
-#  3.1192480219083929e-05 \
-#  3.357137609155976e-06 \
-#  3.7560923959052268e-07 \
-#  4.4175919853867784e-08 \
-#  5.4938489434729238e-09 \
-#  7.2437770424093207e-10 \
-#  1.0132883742919134e-10 \
-#  1.5028690832436914e-11 \
-#  2.3601420573281563e-12 \
-#  3.9173855868335976e-13 \
-#  6.8580254516415638e-14
+# Computed using Boost multiprecision to 100 digits (output 25 digits).
+# boost::math::fisher_f_distribution<boost::multiprecision::cpp_bin_float_100> f(n, m);
 
-# scipy.stats - pass at 2e-12
-# 0.00000000000000000000e+00,   9.32516532636367921761e-69,\
-# 1.95995693873326028191e-54,   3.14060218997071882346e-46,\
-# 1.58270350318433878314e-40,   3.40717330113428396147e-36,\
-# 1.83031316130290750821e-23,   1.62126139886700438187e-09,\
-# 1.10204220067397704041e-02,   9.46217456596381234490e-01,\
-# 1.98973093467951134272e+00,   8.55328109079107945512e-01,\
-# 1.72291854230807173209e-01,   2.38445384870012849765e-02,\
-# 2.75510550168477563390e-03,   2.95393685948038410387e-04,\
-# 3.11924802190892479052e-05,   3.35713760915630972549e-06,\
-# 3.75609239590597380614e-07,   4.41759198538690344890e-08,\
-# 5.49384894347331426951e-09,   7.24377704240983564821e-10,\
-# 1.01328837429192775725e-10,   1.50286908324386235308e-11,\
-# 2.36014205732825684207e-12,   3.91738558683376469201e-13,\
-# 6.85802545164195382838e-14
-
-# R - Pass at 1e-12
-# 0.0000000000000000000e+00 9.3251653263639139462e-69 \
-# 1.9599569387331759554e-54 3.1406021899709451856e-46 \
-# 1.5827035031843542807e-40 3.4071733011343073482e-36 \
-# 1.8303131613029665768e-23 1.6212613988670885475e-09 \
-# 1.1020422006739544890e-02 9.4621745659648370808e-01 \
-# 1.9897309346794691542e+00 8.5532810907908718434e-01 \
-# 1.7229185423079382278e-01 2.3844538487002037846e-02 \
-# 2.7551055016848810184e-03 2.9539368594805125818e-04 \
-# 3.1192480219084748466e-05 3.3571376091560772149e-06 \
-# 3.7560923959053067677e-07 4.4175919853868194073e-08 \
-# 5.4938489434729833973e-09 7.2437770424094313637e-10 \
-# 1.0132883742919338319e-10 1.5028690832437327830e-11 \
-# 2.3601420573281809095e-12 3.9173855868336505912e-13 \
-# 6.8580254516417531422e-14
+cdf.values = \
+  0 \
+  1.901950084414567407982158e-72 \
+  8.153172930248139977433232e-58 \
+  1.99842701059154721870235e-49 \
+  1.369353278081261030190558e-43 \
+  3.757693203912921334647162e-39 \
+  4.452739341551176774249983e-26 \
+  1.328619963186033594255562e-11 \
+  0.0003091368626070543984378357 \
+  0.07602003436023445574153653 \
+  0.5 \
+  0.8668843977220287562944396 \
+  0.9780695578699148036264808 \
+  0.9972249529188820889055435 \
+  0.9996908631373929456015622 \
+  0.9999671316562598509602863 \
+  0.9999965024649763969899356 \
+  0.9999996170267975869211967 \
+  0.9999999561515295406653993 \
+  0.999999994703667204204685 \
+  0.999999999322085559743031 \
+  0.9999999999078892287240481 \
+  0.9999999999867138003681397 \
+  0.999999999997967598617426 \
+  0.9999999999996708261097527 \
+  0.9999999999999436648900561 \
+  0.9999999999999898347310846
 
-# This is a hybrid set of data using Mathematica and the previous values
-# from a different version of scipy.stats. This data allows the lowest test
-# tolerance of 3e-13 to be used.
 pdf.values = \
-  0.0,                          9.3251653263638622991e-69,\
-  1.9599569387331353308e-54,    3.1406021899708862397e-46,\
-  1.5827035031843255814e-40,    3.4071733011342913426e-36,\
-  1.8303131613029907954e-23,    1.621261398867281e-09,\
-  1.10204220067403966393e-02,   9.46217456596435080307e-01,\
-  1.98973093467962458547e+00,   8.55328109079156573280e-01,\
-  1.72291854230816970928e-01,   2.38445384870026415303e-02,\
-  2.75510550168493219270e-03,   2.95393685948055161310e-04,\
-  3.11924802190892479052e-05,   3.35713760915630972549e-06,\
-  3.75609239590597433554e-07,   4.41759198538690344890e-08,\
-  5.49384894347331426951e-09,   7.24377704240983564821e-10,\
-  1.01328837429192775725e-10,   1.50286908324386235308e-11,\
-  2.36014205732825684207e-12,   3.91738558683376469201e-13,\
-  6.85802545164195382838e-14
+  0 \
+  9.325165326363862299110167e-69 \
+  1.959956938733135330795681e-54 \
+  3.140602189970877870069027e-46 \
+  1.582703503184325581370323e-40 \
+  3.407173301134291342578344e-36 \
+  1.830313161302990795366057e-23 \
+  1.621261398867088708375361e-09 \
+  0.01102042200673948008677748 \
+  0.9462174565964842468753402 \
+  1.989730934679469037453176 \
+  0.8553281090790869882551671 \
+  0.1722918542307936225928836 \
+  0.0238445384870020472886454 \
+  0.002755105501684870021694369 \
+  0.0002953936859480514730238029 \
+  3.119248021908464015005759e-05 \
+  3.35713760915607390762951e-06 \
+  3.756092395905319111553199e-07 \
+  4.417591985386864422836571e-08 \
+  5.493848943473037745776927e-09 \
+  7.243777042409484776357334e-10 \
+  1.013288374291930442734601e-10 \
+  1.502869083243724014262325e-11 \
+  2.360142057328199693420267e-12 \
+  3.917385586833650381327999e-13 \
+  6.858025451641692655938516e-14
+
+# High precision log of pdf result
+logpdf.values = \
+  -inf \
+  -156.6456547218476575609707 \
+  -123.6666725187115004375142 \
+  -104.7744997159479173760266 \
+  -91.64426925698433863963414 \
+  -81.66718034420312977094868 \
+  -52.35497006025509554767009 \
+  -20.24006134940998914131113 \
+  -4.508005181368131601038561 \
+  -0.05528286679390731340345701 \
+  0.6879994208910958164839122 \
+  -0.1562701303509716048470393 \
+  -1.758565413229825043222954 \
+  -3.736200082126654137267273 \
+  -5.894299542488022219873025 \
+  -8.12720156267887552984991 \
+  -10.3753335108175746166121 \
+  -12.60442184910080162180042 \
+  -14.79471649036606024687228 \
+  -16.93508599600625963956778 \
+  -19.0196367444687487904008 \
+  -21.04570816863570444833372 \
+  -23.01265007164977037898006 \
+  -24.9210600195876633990678 \
+  -26.77229930491946711097436 \
+  -28.56818171975933822146814 \
+  -30.3107717366489549676034
 
 sf.values = \
-  1.00000000000000000000e+00,   9.99999999999999888978e-01,\
-  9.99999999999999888978e-01,   9.99999999999999888978e-01,\
-  9.99999999999999888978e-01,   9.99999999999999888978e-01,\
-  9.99999999999999888978e-01,   9.99999999986713850042e-01,\
-  9.99690863137392971005e-01,   9.23979965639765365992e-01,\
-  5.00000000000000000000e-01,   1.33115602277971173395e-01,\
-  2.19304421300852252219e-02,   2.77504708111791500644e-03,\
-  3.09136862607055286790e-04,   3.28683437401490946344e-05,\
-  3.49753502360300536433e-06,   3.82973202413080513972e-07,\
-  4.38484704593346391169e-08,   5.29633279579529713456e-09,\
-  6.77914440256968018369e-10,   9.21107712759518643957e-11,\
-  1.32861996318604157343e-11,   2.03240138257400182105e-12,\
-  3.29173890247300592167e-13,   5.63351099439457020525e-14,\
-  1.01652689153928951395e-14
+  1 \
+  1 \
+  1 \
+  1 \
+  1 \
+  1 \
+  1 \
+  0.9999999999867138003681397 \
+  0.9996908631373929456015622 \
+  0.9239799656397655442584635 \
+  0.5 \
+  0.1331156022779712437055604 \
+  0.02193044213008519637351916 \
+  0.002775047081117911094456535 \
+  0.0003091368626070543984378357 \
+  3.286834374014903971373219e-05 \
+  3.497535023603010064379036e-06 \
+  3.829732024130788032679052e-07 \
+  4.384847045933460068233507e-08 \
+  5.296332795795314996396501e-09 \
+  6.779144402569690229234681e-10 \
+  9.21107712759519496970036e-11 \
+  1.328619963186033594255562e-11 \
+  2.032401382574005274047957e-12 \
+  3.291738902473012191522577e-13 \
+  5.633510994394574646201736e-14 \
+  1.016526891539289621285067e-14