You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by al...@apache.org on 2024/09/26 22:19:31 UTC
(datasketches-bigquery) branch kll_no_base64 created (now 90d9d9f)
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a change to branch kll_no_base64
in repository https://gitbox.apache.org/repos/asf/datasketches-bigquery.git
at 90d9d9f removed base64 decoding from KLL
This branch includes the following new commits:
new 90d9d9f removed base64 decoding from KLL
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
(datasketches-bigquery) 01/01: removed base64 decoding from KLL
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch kll_no_base64
in repository https://gitbox.apache.org/repos/asf/datasketches-bigquery.git
commit 90d9d9faab7b85410943fe848b93611702378d38
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Thu Sep 26 15:19:07 2024 -0700
removed base64 decoding from KLL
---
kll/kll_sketch.cpp | 16 ++++------------
kll/sqlx/kll_sketch_float_get_cdf.sqlx | 6 ++++--
kll/sqlx/kll_sketch_float_get_n.sqlx | 4 +++-
kll/sqlx/kll_sketch_float_get_pmf.sqlx | 4 +++-
kll/sqlx/kll_sketch_float_get_quantile.sqlx | 4 +++-
kll/sqlx/kll_sketch_float_get_rank.sqlx | 4 +++-
kll/sqlx/kll_sketch_float_kolmogorov_smirnov.sqlx | 2 ++
kll/sqlx/kll_sketch_float_to_string.sqlx | 4 +++-
8 files changed, 25 insertions(+), 19 deletions(-)
diff --git a/kll/kll_sketch.cpp b/kll/kll_sketch.cpp
index 01ffa6a..95addfa 100644
--- a/kll/kll_sketch.cpp
+++ b/kll/kll_sketch.cpp
@@ -22,8 +22,6 @@
#include <kll_sketch.hpp>
#include <kolmogorov_smirnov.hpp>
-#include "../base64.hpp"
-
using kll_sketch_float = datasketches::kll_sketch<float>;
const emscripten::val Uint8Array = emscripten::val::global("Uint8Array");
@@ -56,9 +54,7 @@ EMSCRIPTEN_BINDINGS(kll_sketch_float) {
auto bytes = self.serialize();
return Uint8Array.new_(emscripten::typed_memory_view(bytes.size(), bytes.data()));
}))
- .class_function("deserializeFromB64", emscripten::optional_override([](const std::string& b64) {
- std::vector<char> bytes(b64_dec_len(b64.data(), b64.size()));
- b64_decode(b64.data(), b64.size(), bytes.data());
+ .class_function("deserialize", emscripten::optional_override([](const std::string& bytes) {
return new kll_sketch_float(kll_sketch_float::deserialize(bytes.data(), bytes.size()));
}), emscripten::allow_raw_pointers())
.function("getN", &kll_sketch_float::get_n)
@@ -75,14 +71,10 @@ EMSCRIPTEN_BINDINGS(kll_sketch_float) {
}))
;
- emscripten::function("kolmogorovSmirnovTest", emscripten::optional_override([](const std::string& sketch1_b64, const std::string& sketch2_b64, double pvalue) {
- std::vector<char> bytes1(b64_dec_len(sketch1_b64.data(), sketch1_b64.size()));
- b64_decode(sketch1_b64.data(), sketch1_b64.size(), bytes1.data());
- std::vector<char> bytes2(b64_dec_len(sketch2_b64.data(), sketch2_b64.size()));
- b64_decode(sketch2_b64.data(), sketch2_b64.size(), bytes2.data());
+ emscripten::function("kolmogorovSmirnovTest", emscripten::optional_override([](const std::string& sketch_bytes1, const std::string& sketch_bytes2, double pvalue) {
return datasketches::kolmogorov_smirnov::test(
- kll_sketch_float::deserialize(bytes1.data(), bytes1.size()),
- kll_sketch_float::deserialize(bytes2.data(), bytes2.size()),
+ kll_sketch_float::deserialize(sketch_bytes1.data(), sketch_bytes1.size()),
+ kll_sketch_float::deserialize(sketch_bytes2.data(), sketch_bytes2.size()),
pvalue
);
}));
diff --git a/kll/sqlx/kll_sketch_float_get_cdf.sqlx b/kll/sqlx/kll_sketch_float_get_cdf.sqlx
index 2bdb764..e631cfe 100644
--- a/kll/sqlx/kll_sketch_float_get_cdf.sqlx
+++ b/kll/sqlx/kll_sketch_float_get_cdf.sqlx
@@ -24,6 +24,7 @@ RETURNS ARRAY<FLOAT64>
LANGUAGE js
OPTIONS (
library=["gs://$GCS_BUCKET/kll_sketch.js"],
+ js_parameter_encoding_mode='STANDARD',
description = '''Returns an approximation to the Cumulative Distribution Function (CDF)
of the input stream as an array of cumulative probabilities defined by the given split_points.
@@ -55,11 +56,11 @@ For more information:
try {
var sketchObject = null;
try {
- sketchObject = Module.kll_sketch_float.deserializeFromB64(sketch);
+ sketchObject = Module.kll_sketch_float.deserialize(sketch);
var vectorFloat = new Module.VectorFloat();
split_points.forEach(value => vectorFloat.push_back(value));
if (sketchObject.isEmpty()) return null;
- var cdf = sketchObject.getCDF(vectorFloat, inclusive);
+ const cdf = sketchObject.getCDF(vectorFloat, inclusive);
var array = [];
for (let i = 0; i < cdf.size(); i++) array.push(cdf.get(i));
return array;
@@ -67,6 +68,7 @@ try {
if (sketchObject != null) sketchObject.delete();
}
} catch (e) {
+ if (e.message != null) throw e;
throw new Error(Module.getExceptionMessage(e));
}
""";
diff --git a/kll/sqlx/kll_sketch_float_get_n.sqlx b/kll/sqlx/kll_sketch_float_get_n.sqlx
index 63a32fa..346c886 100644
--- a/kll/sqlx/kll_sketch_float_get_n.sqlx
+++ b/kll/sqlx/kll_sketch_float_get_n.sqlx
@@ -24,6 +24,7 @@ RETURNS INT64
LANGUAGE js
OPTIONS (
library=["gs://$GCS_BUCKET/kll_sketch.js"],
+ js_parameter_encoding_mode='STANDARD',
description = '''Returns the length of the input stream.
Param sketch: the given sketch as BYTES.
@@ -36,12 +37,13 @@ For more information:
try {
var sketchObject = null;
try {
- sketchObject = Module.kll_sketch_float.deserializeFromB64(sketch);
+ sketchObject = Module.kll_sketch_float.deserialize(sketch);
return sketchObject.getN();
} finally {
if (sketchObject != null) sketchObject.delete();
}
} catch (e) {
+ if (e.message != null) throw e;
throw new Error(Module.getExceptionMessage(e));
}
""";
diff --git a/kll/sqlx/kll_sketch_float_get_pmf.sqlx b/kll/sqlx/kll_sketch_float_get_pmf.sqlx
index 945e90f..9f53f16 100644
--- a/kll/sqlx/kll_sketch_float_get_pmf.sqlx
+++ b/kll/sqlx/kll_sketch_float_get_pmf.sqlx
@@ -24,6 +24,7 @@ RETURNS ARRAY<FLOAT64>
LANGUAGE js
OPTIONS (
library=["gs://$GCS_BUCKET/kll_sketch.js"],
+ js_parameter_encoding_mode='STANDARD',
description = '''Returns an approximation to the Probability Mass Function (PMF)
of the input stream as an array of probability masses defined by the given split_points.
@@ -57,7 +58,7 @@ For more information:
try {
var sketchObject = null;
try {
- sketchObject = Module.kll_sketch_float.deserializeFromB64(sketch);
+ sketchObject = Module.kll_sketch_float.deserialize(sketch);
var vectorFloat = new Module.VectorFloat();
split_points.forEach(value => vectorFloat.push_back(value));
if (sketchObject.isEmpty()) return null;
@@ -69,6 +70,7 @@ try {
if (sketchObject != null) sketchObject.delete();
}
} catch (e) {
+ if (e.message != null) throw e;
throw new Error(Module.getExceptionMessage(e));
}
""";
diff --git a/kll/sqlx/kll_sketch_float_get_quantile.sqlx b/kll/sqlx/kll_sketch_float_get_quantile.sqlx
index 79b76c2..008db3d 100644
--- a/kll/sqlx/kll_sketch_float_get_quantile.sqlx
+++ b/kll/sqlx/kll_sketch_float_get_quantile.sqlx
@@ -24,6 +24,7 @@ RETURNS FLOAT64
LANGUAGE js
OPTIONS (
library=["gs://$GCS_BUCKET/kll_sketch.js"],
+ js_parameter_encoding_mode='STANDARD',
description = '''Returns a value from the sketch that is the best approximation to a value from the original stream with the given rank.
Param sketch: the given sketch in serialized form.
@@ -38,13 +39,14 @@ For more information:
try {
var sketchObject = null;
try {
- sketchObject = Module.kll_sketch_float.deserializeFromB64(sketch);
+ sketchObject = Module.kll_sketch_float.deserialize(sketch);
if (sketchObject.isEmpty()) return null;
return sketchObject.getQuantile(rank, inclusive);
} finally {
if (sketchObject != null) sketchObject.delete();
}
} catch (e) {
+ if (e.message != null) throw e;
throw new Error(Module.getExceptionMessage(e));
}
""";
diff --git a/kll/sqlx/kll_sketch_float_get_rank.sqlx b/kll/sqlx/kll_sketch_float_get_rank.sqlx
index 5fa70c1..ae16a1e 100644
--- a/kll/sqlx/kll_sketch_float_get_rank.sqlx
+++ b/kll/sqlx/kll_sketch_float_get_rank.sqlx
@@ -24,6 +24,7 @@ RETURNS FLOAT64
LANGUAGE js
OPTIONS (
library=["gs://$GCS_BUCKET/kll_sketch.js"],
+ js_parameter_encoding_mode='STANDARD',
description = '''Returns an approximation to the normalized rank, on the interval [0.0, 1.0], of the given value.
Param sketch: the given sketch in serialized form.
@@ -38,13 +39,14 @@ For more information:
try {
var sketchObject = null;
try {
- sketchObject = Module.kll_sketch_float.deserializeFromB64(sketch);
+ sketchObject = Module.kll_sketch_float.deserialize(sketch);
if (sketchObject.isEmpty()) return null;
return sketchObject.getRank(value, inclusive);
} finally {
if (sketchObject != null) sketchObject.delete();
}
} catch (e) {
+ if (e.message != null) throw e;
throw new Error(Module.getExceptionMessage(e));
}
""";
diff --git a/kll/sqlx/kll_sketch_float_kolmogorov_smirnov.sqlx b/kll/sqlx/kll_sketch_float_kolmogorov_smirnov.sqlx
index 7e552d7..4c0798e 100644
--- a/kll/sqlx/kll_sketch_float_kolmogorov_smirnov.sqlx
+++ b/kll/sqlx/kll_sketch_float_kolmogorov_smirnov.sqlx
@@ -24,6 +24,7 @@ RETURNS BOOL
LANGUAGE js
OPTIONS (
library=["gs://$GCS_BUCKET/kll_sketch.js"],
+ js_parameter_encoding_mode='STANDARD',
description = '''Performs the Kolmogorov-Smirnov Test between two KLL sketches of type FLOAT64.
If the given sketches have insufficient data or if the sketch sizes are too small, this will return false.
@@ -40,6 +41,7 @@ For more information:
try {
return Module.kolmogorovSmirnovTest(sketchA, sketchB, pvalue);
} catch (e) {
+ if (e.message != null) throw e;
throw new Error(Module.getExceptionMessage(e));
}
""";
diff --git a/kll/sqlx/kll_sketch_float_to_string.sqlx b/kll/sqlx/kll_sketch_float_to_string.sqlx
index 8e795c1..05ad279 100644
--- a/kll/sqlx/kll_sketch_float_to_string.sqlx
+++ b/kll/sqlx/kll_sketch_float_to_string.sqlx
@@ -24,6 +24,7 @@ RETURNS STRING
LANGUAGE js
OPTIONS (
library=["gs://$GCS_BUCKET/kll_sketch.js"],
+ js_parameter_encoding_mode='STANDARD',
description = '''Returns a summary string that represents the state of the given sketch.
Param sketch: the given sketch as sketch encoded bytes.
@@ -36,12 +37,13 @@ For more information:
try {
var sketchObject = null;
try {
- sketchObject = Module.kll_sketch_float.deserializeFromB64(sketch);
+ sketchObject = Module.kll_sketch_float.deserialize(sketch);
return sketchObject.toString();
} finally {
if (sketchObject != null) sketchObject.delete();
}
} catch (e) {
+ if (e.message != null) throw e;
throw new Error(Module.getExceptionMessage(e));
}
""";
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org