You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by al...@apache.org on 2024/09/25 22:40:58 UTC
(datasketches-bigquery) branch hll_no_base64 created (now 95a5621)
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a change to branch hll_no_base64
in repository https://gitbox.apache.org/repos/asf/datasketches-bigquery.git
at 95a5621 removed base64 encoding-decoding from HLL
This branch includes the following new commits:
new 95a5621 removed base64 encoding-decoding from HLL
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
(datasketches-bigquery) 01/01: removed base64 encoding-decoding from HLL
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch hll_no_base64
in repository https://gitbox.apache.org/repos/asf/datasketches-bigquery.git
commit 95a5621d43f302f8b6ad62da5abba5c415bc9858
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Wed Sep 25 15:40:36 2024 -0700
removed base64 encoding-decoding from HLL
---
hll/hll_sketch.cpp | 54 +++++++++---------------
hll/sqlx/hll_sketch_get_estimate.sqlx | 10 ++---
hll/sqlx/hll_sketch_get_estimate_and_bounds.sqlx | 19 ++++-----
hll/sqlx/hll_sketch_to_string.sqlx | 10 ++---
hll/sqlx/hll_sketch_union_lgk_type.sqlx | 14 ++----
5 files changed, 37 insertions(+), 70 deletions(-)
diff --git a/hll/hll_sketch.cpp b/hll/hll_sketch.cpp
index 9e4c800..35b7f92 100644
--- a/hll/hll_sketch.cpp
+++ b/hll/hll_sketch.cpp
@@ -22,8 +22,6 @@
#include <hll.hpp>
-#include "../base64.hpp"
-
datasketches::target_hll_type convert_tgt_type(const std::string& tgt_type_str) {
if (tgt_type_str == "" || tgt_type_str == "HLL_4") return datasketches::HLL_4;
if (tgt_type_str == "HLL_6") return datasketches::HLL_6;
@@ -34,6 +32,7 @@ datasketches::target_hll_type convert_tgt_type(const std::string& tgt_type_str)
const emscripten::val Uint8Array = emscripten::val::global("Uint8Array");
EMSCRIPTEN_BINDINGS(hll_sketch) {
+ emscripten::register_vector<double>("VectorDouble");
emscripten::function("getExceptionMessage", emscripten::optional_override([](intptr_t ptr) {
return std::string(reinterpret_cast<std::exception*>(ptr)->what());
@@ -48,21 +47,16 @@ EMSCRIPTEN_BINDINGS(hll_sketch) {
auto bytes = self.serialize_compact();
return Uint8Array.new_(emscripten::typed_memory_view(bytes.size(), bytes.data()));
}))
- .class_function("deserializeFromB64", emscripten::optional_override([](const std::string& b64) {
- std::vector<char> bytes(b64_dec_len(b64.data(), b64.size()));
- b64_decode(b64.data(), b64.size(), bytes.data());
- return new datasketches::hll_sketch(datasketches::hll_sketch::deserialize(bytes.data(), bytes.size()));
- }), emscripten::allow_raw_pointers())
- .class_function("deserializeFromBytes", emscripten::optional_override([](const std::string& bytes) {
- return new datasketches::hll_sketch(datasketches::hll_sketch::deserialize(bytes.data(), bytes.size()));
- }), emscripten::allow_raw_pointers())
- .function("getEstimate", &datasketches::hll_sketch::get_estimate)
- .function("getLowerBound", &datasketches::hll_sketch::get_lower_bound)
- .function("getUpperBound", &datasketches::hll_sketch::get_upper_bound)
- .function("toString", emscripten::optional_override([](const datasketches::hll_sketch& self) {
- return self.to_string();
+ .class_function("getEstimate", emscripten::optional_override([](const std::string& bytes) {
+ return datasketches::hll_sketch::deserialize(bytes.data(), bytes.size()).get_estimate();
+ }))
+ .class_function("getEstimateAndBounds", emscripten::optional_override([](const std::string& bytes, uint8_t num_std_devs) {
+ const auto sketch = datasketches::hll_sketch::deserialize(bytes.data(), bytes.size());
+ return std::vector<double>{sketch.get_estimate(), sketch.get_lower_bound(num_std_devs), sketch.get_upper_bound(num_std_devs)};
+ }))
+ .class_function("toString", emscripten::optional_override([](const std::string& bytes) {
+ return datasketches::hll_sketch::deserialize(bytes.data(), bytes.size()).to_string();
}))
- .class_function("getMaxSerializedSizeBytes", &datasketches::hll_sketch::get_max_updatable_serialization_bytes)
;
emscripten::class_<datasketches::hll_union>("hll_union")
@@ -75,29 +69,19 @@ EMSCRIPTEN_BINDINGS(hll_sketch) {
.function("updateWithBytes", emscripten::optional_override([](datasketches::hll_union& self, const std::string& bytes) {
self.update(datasketches::hll_sketch::deserialize(bytes.data(), bytes.size()));
}), emscripten::allow_raw_pointers())
- .function("updateWithB64", emscripten::optional_override([](datasketches::hll_union& self, const std::string& b64) {
- std::vector<char> bytes(b64_dec_len(b64.data(), b64.size()));
- b64_decode(b64.data(), b64.size(), bytes.data());
- self.update(datasketches::hll_sketch::deserialize(bytes.data(), bytes.size()));
- }), emscripten::allow_raw_pointers())
- .function("updateWithBuffer", emscripten::optional_override([](datasketches::hll_union& self, intptr_t bytes, size_t size) {
- self.update(datasketches::hll_sketch::deserialize(reinterpret_cast<void*>(bytes), size));
- }))
-// .function("getResultStream", emscripten::optional_override([](datasketches::hll_union& self, intptr_t bytes, size_t size, datasketches::target_hll_type tgt_type) {
-// std::strstream stream(reinterpret_cast<char*>(bytes), size);
-// self.get_result(tgt_type).serialize_compact(stream);
-// return (int) stream.tellp();
-// }))
.function("getResultAsUint8Array", emscripten::optional_override([](datasketches::hll_union& self, const std::string& tgt_type_str) {
auto bytes = self.get_result(convert_tgt_type(tgt_type_str)).serialize_compact();
return Uint8Array.new_(emscripten::typed_memory_view(bytes.size(), bytes.data()));
}))
- .function("getResultB64", emscripten::optional_override([](datasketches::hll_union& self, const std::string& tgt_type_str) {
- auto bytes = self.get_result(convert_tgt_type(tgt_type_str)).serialize_compact();
- std::vector<char> b64(b64_enc_len(bytes.size()));
- b64_encode((const char*) bytes.data(), bytes.size(), b64.data());
- return std::string(b64.data(), b64.size());
- }))
;
+ emscripten::function("hllUnion", emscripten::optional_override([](
+ const std::string& bytes1, const std::string& bytes2, uint8_t lg_k, const std::string& tgt_type_str
+ ) {
+ datasketches::hll_union u(lg_k);
+ u.update(datasketches::hll_sketch::deserialize(bytes1.data(), bytes1.size()));
+ u.update(datasketches::hll_sketch::deserialize(bytes2.data(), bytes2.size()));
+ const auto bytes = u.get_result(convert_tgt_type(tgt_type_str)).serialize_compact();
+ return Uint8Array.new_(emscripten::typed_memory_view(bytes.size(), bytes.data()));
+ }));
}
diff --git a/hll/sqlx/hll_sketch_get_estimate.sqlx b/hll/sqlx/hll_sketch_get_estimate.sqlx
index 2900485..6605806 100644
--- a/hll/sqlx/hll_sketch_get_estimate.sqlx
+++ b/hll/sqlx/hll_sketch_get_estimate.sqlx
@@ -24,6 +24,7 @@ RETURNS FLOAT64
LANGUAGE js
OPTIONS (
library=["gs://$GCS_BUCKET/hll_sketch.js"],
+ js_parameter_encoding_mode='STANDARD',
description = '''Returns a summary string that represents the state of the given sketch.
Param sketch: the given sketch as BYTES.
@@ -34,14 +35,9 @@ For more information:
'''
) AS R"""
try {
- var sketchObject = null;
- try {
- sketchObject = Module.hll_sketch.deserializeFromB64(sketch);
- return sketchObject.getEstimate();
- } finally {
- if (sketchObject != null) sketchObject.delete();
- }
+ return Module.hll_sketch.getEstimate(sketch);
} catch (e) {
+ if (e.message != null) throw e;
throw new Error(Module.getExceptionMessage(e));
}
""";
diff --git a/hll/sqlx/hll_sketch_get_estimate_and_bounds.sqlx b/hll/sqlx/hll_sketch_get_estimate_and_bounds.sqlx
index 4ba7e3a..cb69563 100644
--- a/hll/sqlx/hll_sketch_get_estimate_and_bounds.sqlx
+++ b/hll/sqlx/hll_sketch_get_estimate_and_bounds.sqlx
@@ -24,6 +24,7 @@ RETURNS STRUCT<estimate FLOAT64, lower_bound FLOAT64, upper_bound FLOAT64>
LANGUAGE js
OPTIONS (
library=["gs://$GCS_BUCKET/hll_sketch.js"],
+ js_parameter_encoding_mode='STANDARD',
description = '''Gets cardinality estimate and bounds from given sketch.
Param sketch: The given sketch to query as BYTES.
@@ -37,18 +38,14 @@ For more information:
'''
) AS R"""
try {
- var sketchObject = null;
- try {
- sketchObject = Module.hll_sketch.deserializeFromB64(sketch);
- return {
- estimate: sketchObject.getEstimate(),
- lower_bound: sketchObject.getLowerBound(num_std_devs),
- upper_bound: sketchObject.getUpperBound(num_std_devs)
- };
- } finally {
- if (sketchObject != null) sketchObject.delete();
- }
+ const result = Module.hll_sketch.getEstimateAndBounds(sketch, Number(num_std_devs));
+ return {
+ estimate: result.get(0),
+ lower_bound: result.get(1),
+ upper_bound: result.get(2)
+ };
} catch (e) {
+ if (e.message != null) throw e;
throw new Error(Module.getExceptionMessage(e));
}
""";
diff --git a/hll/sqlx/hll_sketch_to_string.sqlx b/hll/sqlx/hll_sketch_to_string.sqlx
index dfc7c72..cd83b2a 100644
--- a/hll/sqlx/hll_sketch_to_string.sqlx
+++ b/hll/sqlx/hll_sketch_to_string.sqlx
@@ -24,6 +24,7 @@ RETURNS STRING
LANGUAGE js
OPTIONS (
library=["gs://$GCS_BUCKET/hll_sketch.js"],
+ js_parameter_encoding_mode='STANDARD',
description = '''Returns a summary string that represents the state of the given sketch.
Param sketch: the given sketch as BYTES.
@@ -34,14 +35,9 @@ For more information:
'''
) AS R"""
try {
- var sketchObject = null;
- try {
- sketchObject = Module.hll_sketch.deserializeFromB64(sketch);
- return sketchObject.toString();
- } finally {
- if (sketchObject != null) sketchObject.delete();
- }
+ return Module.hll_sketch.toString(sketch);
} catch (e) {
+ if (e.message != null) throw e;
throw new Error(Module.getExceptionMessage(e));
}
""";
diff --git a/hll/sqlx/hll_sketch_union_lgk_type.sqlx b/hll/sqlx/hll_sketch_union_lgk_type.sqlx
index ba8c25f..5760630 100644
--- a/hll/sqlx/hll_sketch_union_lgk_type.sqlx
+++ b/hll/sqlx/hll_sketch_union_lgk_type.sqlx
@@ -24,6 +24,7 @@ RETURNS BYTES
LANGUAGE js
OPTIONS (
library=["gs://$GCS_BUCKET/hll_sketch.js"],
+ js_parameter_encoding_mode='STANDARD',
description = '''Computes a sketch that represents the union of the two given sketches.
Param sketchA: the first sketch as bytes.
Param sketchB: the second sketch as bytes.
@@ -35,18 +36,11 @@ For more information:
- https://datasketches.apache.org/docs/HLL/HllSketches.html
'''
) AS R"""
-const default_lg_k = 12;
+const default_lg_k = Number(12);
try {
- var union = null;
- try {
- union = new Module.hll_union(lg_k ? lg_k : default_lg_k);
- union.updateWithB64(sketchA)
- union.updateWithB64(sketchB)
- return union.getResultB64(tgt_type ? tgt_type : "");
- } finally {
- if (union != null) union.delete();
- }
+ return Module.hllUnion(sketchA, sketchB, lg_k ? Number(lg_k) : default_lg_k, tgt_type ? tgt_type : "");
} catch (e) {
+ if (e.message != null) throw e;
throw new Error(Module.getExceptionMessage(e));
}
""";
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org