You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by al...@apache.org on 2020/09/11 23:05:27 UTC
[incubator-datasketches-postgresql] branch aod_sketch updated:
students t test using boost
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch aod_sketch
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-postgresql.git
The following commit(s) were added to refs/heads/aod_sketch by this push:
new d2fa89e students t test using boost
d2fa89e is described below
commit d2fa89eafcdcbc241f7d455f5b38ea1f2f2a507a
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Fri Sep 11 16:05:01 2020 -0700
students t test using boost
---
Makefile | 7 +++--
sql/datasketches_aod_sketch.sql | 4 +++
src/aod_sketch_c_adapter.cpp | 58 +++++++++++++++++++++++++++++++++++++++++
src/aod_sketch_c_adapter.h | 2 ++
src/aod_sketch_pg_functions.c | 35 +++++++++++++++++++++++++
5 files changed, 104 insertions(+), 2 deletions(-)
diff --git a/Makefile b/Makefile
index fe2c471..10d85b0 100644
--- a/Makefile
+++ b/Makefile
@@ -38,10 +38,13 @@ OBJS = src/global_hooks.o src/base64.o src/common.o \
src/hll_sketch_pg_functions.o src/hll_sketch_c_adapter.o \
src/aod_sketch_pg_functions.o src/aod_sketch_c_adapter.o
-# assume a copy or link datasketches-cpp in the current dir
+# assume a dir or link named "datasketches-cpp" in the current dir
CORE = datasketches-cpp
-PG_CPPFLAGS = -fPIC -I/usr/local/include -I$(CORE)/kll/include -I$(CORE)/common/include -I$(CORE)/cpc/include -I$(CORE)/theta/include -I$(CORE)/fi/include -I$(CORE)/hll/include -I$(CORE)/tuple/include
+# assume a dir or link named "boost" in the current dir
+BOOST = boost
+
+PG_CPPFLAGS = -fPIC -I/usr/local/include -I$(CORE)/kll/include -I$(CORE)/common/include -I$(CORE)/cpc/include -I$(CORE)/theta/include -I$(CORE)/fi/include -I$(CORE)/hll/include -I$(CORE)/tuple/include -I$(BOOST)
PG_CXXFLAGS = -std=c++11
SHLIB_LINK = -lstdc++ -L/usr/local/lib
diff --git a/sql/datasketches_aod_sketch.sql b/sql/datasketches_aod_sketch.sql
index 49da709..bda7560 100644
--- a/sql/datasketches_aod_sketch.sql
+++ b/sql/datasketches_aod_sketch.sql
@@ -177,3 +177,7 @@ CREATE OR REPLACE FUNCTION aod_sketch_to_kll_float_sketch(aod_sketch, int) RETUR
CREATE OR REPLACE FUNCTION aod_sketch_to_kll_float_sketch(aod_sketch, int, int) RETURNS kll_float_sketch
AS '$libdir/datasketches', 'pg_aod_sketch_to_kll_float_sketch'
LANGUAGE C STRICT IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION aod_sketch_students_t_test(aod_sketch, aod_sketch) RETURNS double precision[]
+ AS '$libdir/datasketches', 'pg_aod_sketch_students_t_test'
+ LANGUAGE C STRICT IMMUTABLE;
diff --git a/src/aod_sketch_c_adapter.cpp b/src/aod_sketch_c_adapter.cpp
index fbd28b7..a7f268e 100644
--- a/src/aod_sketch_c_adapter.cpp
+++ b/src/aod_sketch_c_adapter.cpp
@@ -27,6 +27,12 @@
#include <array_of_doubles_intersection.hpp>
#include <array_of_doubles_a_not_b.hpp>
+#include <boost/accumulators/accumulators.hpp>
+#include <boost/accumulators/statistics/stats.hpp>
+#include <boost/accumulators/statistics/mean.hpp>
+#include <boost/accumulators/statistics/variance.hpp>
+#include <boost/math/distributions/students_t.hpp>
+
using vector_double = std::vector<double, palloc_allocator<double>>;
using update_aod_sketch_pg = datasketches::update_array_of_doubles_sketch_alloc<palloc_allocator<double>>;
@@ -283,3 +289,55 @@ void* aod_sketch_to_kll_float_sketch(const void* sketchptr, unsigned column_inde
}
pg_unreachable();
}
+
+double t_test_unequal_sd(double m1, double v1, uint64_t n1, double m2, double v2, uint64_t n2) {
+ double degrees_of_freedom = v1 / n1 + v2 / n2;
+ degrees_of_freedom *= degrees_of_freedom;
+ double t1 = v1 / n1;
+ t1 *= t1;
+ t1 /= (n1 - 1);
+ double t2 = v2 / n2;
+ t2 *= t2;
+ t2 /= (n2 - 1);
+ degrees_of_freedom /= (t1 + t2);
+ double t_stat = (m1 - m2) / sqrt(v1 / n1 + v2 / n2);
+ using boost::math::students_t;
+ students_t distribution(degrees_of_freedom);
+ return 2 * cdf(complement(distribution, fabs(t_stat))); // double to match 2-sided test in Java (commons-math3)
+}
+
+Datum* aod_sketch_students_t_test(const void* sketchptr1, const void* sketchptr2, unsigned* arr_len_out) {
+ try {
+ const auto& sketch1 = *static_cast<const compact_aod_sketch_pg*>(sketchptr1);
+ const auto& sketch2 = *static_cast<const compact_aod_sketch_pg*>(sketchptr2);
+ if (sketch1.get_num_values() != sketch2.get_num_values()) pg_error("aod_sketch_students_t_test: number of values mismatch");
+ unsigned num_values = sketch1.get_num_values();
+ Datum* p_values = (Datum*) palloc(sizeof(Datum) * num_values);
+ *arr_len_out = num_values;
+
+ using namespace boost::accumulators;
+ using Accum = accumulator_set<double, stats<tag::mean, tag::variance>>;
+
+ std::vector<Accum, palloc_allocator<Accum>> stats1(num_values);
+ for (const auto& entry: sketch1) {
+ for (unsigned i = 0; i < num_values; ++i) stats1[i](entry.second[i]);
+ }
+
+ std::vector<Accum, palloc_allocator<Accum>> stats2(num_values);
+ for (const auto& entry: sketch2) {
+ for (unsigned i = 0; i < num_values; ++i) stats2[i](entry.second[i]);
+ }
+
+ for (unsigned i = 0; i < num_values; ++i) {
+ p_values[i] = pg_float8_get_datum(t_test_unequal_sd(
+ mean(stats1[i]), variance(stats1[i]), sketch1.get_num_retained(),
+ mean(stats2[i]), variance(stats2[i]), sketch2.get_num_retained()
+ ));
+ }
+
+ return p_values;
+ } catch (std::exception& e) {
+ pg_error(e.what());
+ }
+ pg_unreachable();
+}
diff --git a/src/aod_sketch_c_adapter.h b/src/aod_sketch_c_adapter.h
index ebf92c7..5bfa3b0 100644
--- a/src/aod_sketch_c_adapter.h
+++ b/src/aod_sketch_c_adapter.h
@@ -58,6 +58,8 @@ void* aod_a_not_b(const void* sketchptr1, const void* sketchptr2);
void* aod_sketch_to_kll_float_sketch(const void* sketchptr, unsigned column_index, unsigned k);
+void** aod_sketch_students_t_test(const void* sketch1ptr, const void* sketch2ptr, unsigned* arr_len_out);
+
#ifdef __cplusplus
}
#endif
diff --git a/src/aod_sketch_pg_functions.c b/src/aod_sketch_pg_functions.c
index 825b1b2..93254a4 100644
--- a/src/aod_sketch_pg_functions.c
+++ b/src/aod_sketch_pg_functions.c
@@ -43,6 +43,7 @@ PG_FUNCTION_INFO_V1(pg_aod_sketch_union);
PG_FUNCTION_INFO_V1(pg_aod_sketch_intersection);
PG_FUNCTION_INFO_V1(pg_aod_sketch_a_not_b);
PG_FUNCTION_INFO_V1(pg_aod_sketch_to_kll_float_sketch);
+PG_FUNCTION_INFO_V1(pg_aod_sketch_students_t_test);
/* function declarations */
Datum pg_aod_sketch_recv(PG_FUNCTION_ARGS);
@@ -61,6 +62,7 @@ Datum pg_aod_sketch_union(PG_FUNCTION_ARGS);
Datum pg_aod_sketch_intersection(PG_FUNCTION_ARGS);
Datum pg_aod_sketch_a_not_b(PG_FUNCTION_ARGS);
Datum pg_aod_sketch_to_kll_float_sketch(PG_FUNCTION_ARGS);
+Datum pg_aod_sketch_students_t_test(PG_FUNCTION_ARGS);
Datum pg_aod_sketch_add_item(PG_FUNCTION_ARGS) {
void* sketchptr;
@@ -488,3 +490,36 @@ Datum pg_aod_sketch_to_kll_float_sketch(PG_FUNCTION_ARGS) {
SET_VARSIZE(bytes_out.ptr, bytes_out.size);
PG_RETURN_BYTEA_P(bytes_out.ptr);
}
+
+Datum pg_aod_sketch_students_t_test(PG_FUNCTION_ARGS) {
+ const bytea* bytes_in1;
+ const bytea* bytes_in2;
+ void* sketchptr1;
+ void* sketchptr2;
+
+ // output array of p-values
+ Datum* p_values;
+ ArrayType* arr_out;
+ int16 elmlen_out;
+ bool elmbyval_out;
+ char elmalign_out;
+ unsigned arr_len_out;
+
+ if (PG_ARGISNULL(0) || PG_ARGISNULL(1)) {
+ elog(ERROR, "aod_a_not_b expects two valid aod sketches");
+ }
+
+ bytes_in1 = PG_GETARG_BYTEA_P(0);
+ sketchptr1 = aod_sketch_deserialize(VARDATA(bytes_in1), VARSIZE(bytes_in1) - VARHDRSZ);
+ bytes_in2 = PG_GETARG_BYTEA_P(1);
+ sketchptr2 = aod_sketch_deserialize(VARDATA(bytes_in2), VARSIZE(bytes_in2) - VARHDRSZ);
+ p_values = (Datum*) aod_sketch_students_t_test(sketchptr1, sketchptr2, &arr_len_out);
+ compact_aod_sketch_delete(sketchptr1);
+ compact_aod_sketch_delete(sketchptr2);
+
+ // construct output array of p-values
+ get_typlenbyvalalign(FLOAT8OID, &elmlen_out, &elmbyval_out, &elmalign_out);
+ arr_out = construct_array(p_values, arr_len_out, FLOAT8OID, elmlen_out, elmbyval_out, elmalign_out);
+
+ PG_RETURN_ARRAYTYPE_P(arr_out);
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org