You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by al...@apache.org on 2024/09/18 03:09:35 UTC
(datasketches-bigquery) 01/01: CPC sketch functions with defaults
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch cpc_defaults
in repository https://gitbox.apache.org/repos/asf/datasketches-bigquery.git
commit 0a9a5ad7427b7084521d649f32562c4f6ab52f13
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Tue Sep 17 20:08:51 2024 -0700
CPC sketch functions with defaults
---
cpc/Makefile | 7 +-
cpc/sqlx/cpc_sketch_agg_string.sqlx | 113 +--------------------
...ng.sqlx => cpc_sketch_agg_string_lgk_seed.sqlx} | 0
cpc/sqlx/cpc_sketch_agg_union.sqlx | 99 +-----------------
...ion.sqlx => cpc_sketch_agg_union_lgk_seed.sqlx} | 0
cpc/sqlx/cpc_sketch_get_estimate.sqlx | 22 +---
cpc/sqlx/cpc_sketch_get_estimate_and_bounds.sqlx | 26 +----
...> cpc_sketch_get_estimate_and_bounds_seed.sqlx} | 0
...mate.sqlx => cpc_sketch_get_estimate_seed.sqlx} | 0
cpc/sqlx/cpc_sketch_to_string.sqlx | 23 +----
..._string.sqlx => cpc_sketch_to_string_seed.sqlx} | 0
...tch_scalar_union.sqlx => cpc_sketch_union.sqlx} | 27 +----
...r_union.sqlx => cpc_sketch_union_lgk_seed.sqlx} | 0
13 files changed, 36 insertions(+), 281 deletions(-)
diff --git a/cpc/Makefile b/cpc/Makefile
index d90ba59..0f05994 100644
--- a/cpc/Makefile
+++ b/cpc/Makefile
@@ -48,8 +48,13 @@ upload: all
gcloud storage cp $$file gs://$(GCS_BUCKET)/ ; \
done
+# some functions (with shorter name) can call other functions (with suffix)
+# so reversing the order should create full-signature functions first
+reverse = $(if $(wordlist 2,2,$(1)),$(call reverse,$(wordlist 2,$(words $(1)),$(1))) $(firstword $(1)),$(1))
+
create:
- @for file in $(wildcard sqlx/*.sqlx); do \
+ @for file in $(call reverse,$(wildcard sqlx/*.sqlx)); do \
+ echo creating $$file; \
../substitute_and_run.sh $$file ; \
done
diff --git a/cpc/sqlx/cpc_sketch_agg_string.sqlx b/cpc/sqlx/cpc_sketch_agg_string.sqlx
index 0131cf0..6c4f864 100644
--- a/cpc/sqlx/cpc_sketch_agg_string.sqlx
+++ b/cpc/sqlx/cpc_sketch_agg_string.sqlx
@@ -19,122 +19,19 @@
config { hasOutput: true }
-CREATE OR REPLACE AGGREGATE FUNCTION ${self()}(str STRING, params STRUCT<lg_k BYTEINT, seed INT64> NOT AGGREGATE)
+CREATE OR REPLACE AGGREGATE FUNCTION ${self()}(str STRING)
RETURNS BYTES
-LANGUAGE js
OPTIONS (
- library=["gs://$GCS_BUCKET/cpc_sketch.mjs"],
description = '''Creates a sketch that represents the cardinality of the given STRING column.
Param str: the STRING column of identifiers.
-Param lg_k: the sketch accuracy/size parameter as an integer in the range [4, 26].
-Param seed: the seed to be used by the underlying hash function.
+Default lg_k and seed.
Returns: a Compact, Compressed CPC Sketch, as BYTES
For more information:
- https://datasketches.apache.org/docs/CPC/CPC.html
- https://datasketches.apache.org/docs/CPC/CpcPerformance.html
'''
-) AS R"""
-import ModuleFactory from "gs://$GCS_BUCKET/cpc_sketch.mjs";
-var Module = await ModuleFactory();
-const default_lg_k = Number(Module.DEFAULT_LG_K);
-const default_seed = BigInt(Module.DEFAULT_SEED);
-
-function destroyState(state) {
- if (state.sketch) {
- state.sketch.delete();
- state.sketch = null;
- }
- if (state.union) {
- state.union.delete();
- state.union = null;
- }
- state.serialized = null;
-}
-
-// UDAF interface
-export function initialState(params) {
- var state = {
- lg_k: params.lg_k == null ? default_lg_k : Number(params.lg_k),
- seed: params.seed == null ? default_seed : BigInt(params.seed),
- sketch: null,
- union: null,
- serialized: null
- };
- state.sketch = new Module.cpc_sketch(state.lg_k, state.seed);
- return state;
-}
-
-export function aggregate(state, str) {
- if (state.sketch == null) {
- state.sketch = new Module.cpc_sketch(state.lg_k, state.seed);
- }
- state.sketch.updateString(str);
-}
-
-export function serialize(state) {
- try {
- if (state.sketch != null && state.serialized != null) {
- // merge aggregated and serialized state
- var u = new Module.cpc_union(state.lg_k, state.seed);
- try {
- u.updateWithSketch(state.sketch);
- u.updateWithBytes(state.serialized, state.seed);
- state.serialized = u.getResultAsUint8Array();
- } finally {
- u.delete();
- }
- } else if (state.sketch != null) {
- state.serialized = state.sketch.serializeAsUint8Array();
- } else if (state.union != null) {
- state.serialized = state.union.getResultAsUint8Array();
- } else if (state.serialized == null) {
- throw new Error("Unexpected state in serialization " + JSON.stringify(state));
- }
- return {
- lg_k: state.lg_k,
- seed: state.seed,
- bytes: state.serialized
- };
- } finally {
- destroyState(state);
- }
-}
-
-export function deserialize(serialized) {
- return {
- sketch: null,
- union: null,
- serialized: serialized.bytes,
- lg_k: serialized.lg_k,
- seed: serialized.seed
- };
-}
-
-export function merge(state, other_state) {
- if (!state.union) {
- state.union = new Module.cpc_union(state.lg_k, state.seed);
- }
- if (state.sketch || other_state.sketch) {
- throw new Error("sketch is not expected in merge");
- }
- if (other_state.union) {
- throw new Error("other_state should not have union in merge");
- }
- if (state.serialized) {
- state.union.updateWithBytes(state.serialized, state.seed);
- state.serialized = null;
- }
- if (other_state.serialized) {
- state.union.updateWithBytes(other_state.serialized, other_state.seed);
- other_state.serialized = null;
- } else {
- throw new Error("other_state should have serialized sketch in merge");
- }
-}
-
-export function finalize(state) {
- return serialize(state).bytes
-}
-""";
+) AS (
+ $BQ_DATASET.cpc_sketch_agg_string_lgk_seed(str, STRUCT<BYTEINT, INT64>(NULL, NULL))
+);
diff --git a/cpc/sqlx/cpc_sketch_agg_string.sqlx b/cpc/sqlx/cpc_sketch_agg_string_lgk_seed.sqlx
similarity index 100%
copy from cpc/sqlx/cpc_sketch_agg_string.sqlx
copy to cpc/sqlx/cpc_sketch_agg_string_lgk_seed.sqlx
diff --git a/cpc/sqlx/cpc_sketch_agg_union.sqlx b/cpc/sqlx/cpc_sketch_agg_union.sqlx
index 23f05c0..f5b6223 100644
--- a/cpc/sqlx/cpc_sketch_agg_union.sqlx
+++ b/cpc/sqlx/cpc_sketch_agg_union.sqlx
@@ -19,108 +19,19 @@
config { hasOutput: true }
-CREATE OR REPLACE AGGREGATE FUNCTION ${self()}(sketch BYTES, params STRUCT<lg_k BYTEINT, seed INT64> NOT AGGREGATE)
+CREATE OR REPLACE AGGREGATE FUNCTION ${self()}(sketch BYTES)
RETURNS BYTES
-LANGUAGE js
OPTIONS (
- library=["gs://$GCS_BUCKET/cpc_sketch.mjs"],
description = '''Creates a sketch that represents the union of the given column of sketches.
Param sketch: the column of sketches. Each as BYTES.
-Param lg_k: the sketch accuracy/size parameter as an integer in the range [4, 26].
-Param seed: This is used to confirm that the given sketches were configured with the correct seed.
+Default lg_k and seed.
Returns: a Compact, Compressed CPC Sketch, as BYTES.
For more information:
- https://datasketches.apache.org/docs/CPC/CPC.html
- https://datasketches.apache.org/docs/CPC/CpcPerformance.html
'''
-) AS R"""
-import ModuleFactory from "gs://$GCS_BUCKET/cpc_sketch.mjs";
-var Module = await ModuleFactory();
-const default_lg_k = Number(Module.DEFAULT_LG_K);
-const default_seed = BigInt(Module.DEFAULT_SEED);
-
-// ensures we have a union
-// if there is a serialized sketch, add it to the union and destroy it
-function ensureUnion(state) {
- try {
- if (state.union == null) {
- state.union = new Module.cpc_union(state.lg_k, state.seed);
- }
- if (state.serialized != null) {
- state.union.updateWithBytes(state.serialized, state.seed);
- state.serialized = null;
- }
- } catch (e) {
- throw new Error(Module.getExceptionMessage(e));
- }
-}
-
-export function initialState(params) {
- var state = {
- lg_k: params.lg_k == null ? default_lg_k : Number(params.lg_k),
- seed: params.seed == null ? default_seed : BigInt(params.seed),
- sketch: null,
- union: null,
- serialized: null
- };
- return state;
-}
-
-export function aggregate(state, sketch) {
- if (sketch != null) {
- ensureUnion(state);
- try {
- state.union.updateWithBytes(sketch, state.seed);
- } catch (e) {
- throw new Error(Module.getExceptionMessage(e));
- }
- }
-}
-
-export function serialize(state) {
- try {
- ensureUnion(state);
- return {
- lg_k: state.lg_k,
- seed: state.seed,
- bytes: state.union.getResultAsUint8Array()
- };
- } catch (e) {
- throw new Error(Module.getExceptionMessage(e));
- } finally {
- state.union.delete();
- state.union = null;
- }
-}
-
-export function deserialize(serialized) {
- return {
- lg_k: serialized.lg_k,
- seed: serialized.seed,
- union: null,
- serialized: serialized.bytes
- };
-}
-
-export function merge(state, other_state) {
- if (other_state.union) {
- throw new Error("Did not expect union in other state");
- }
- ensureUnion(state);
- if (other_state.serialized != null) {
- try {
- state.union.updateWithBytes(other_state.serialized, other_state.seed);
- other_state.serialized = null;
- } catch (e) {
- throw new Error(Module.getExceptionMessage(e));
- }
- } else {
- throw new Error("Expected serialized sketch in other_state");
- }
-}
-export function finalize(state) {
- return serialize(state).bytes;
-}
-""";
+) AS (
+ $BQ_DATASET.cpc_sketch_agg_union_lgk_seed(sketch, STRUCT<BYTEINT, INT64>(NULL, NULL))
+);
diff --git a/cpc/sqlx/cpc_sketch_agg_union.sqlx b/cpc/sqlx/cpc_sketch_agg_union_lgk_seed.sqlx
similarity index 100%
copy from cpc/sqlx/cpc_sketch_agg_union.sqlx
copy to cpc/sqlx/cpc_sketch_agg_union_lgk_seed.sqlx
diff --git a/cpc/sqlx/cpc_sketch_get_estimate.sqlx b/cpc/sqlx/cpc_sketch_get_estimate.sqlx
index ab89e05..321922a 100644
--- a/cpc/sqlx/cpc_sketch_get_estimate.sqlx
+++ b/cpc/sqlx/cpc_sketch_get_estimate.sqlx
@@ -19,31 +19,19 @@
config { hasOutput: true }
-CREATE OR REPLACE FUNCTION ${self()}(sketch BYTES, seed INT64)
+CREATE OR REPLACE FUNCTION ${self()}(sketch BYTES)
RETURNS FLOAT64
-LANGUAGE js
OPTIONS (
- library=["gs://$GCS_BUCKET/cpc_sketch.js"],
description = '''Gets cardinality estimate and bounds from given sketch.
Param sketch: The given sketch to query as BYTES.
-Param seed: This is used to confirm that the given sketch was configured with the correct seed.
+Default seed.
Returns: a FLOAT64 value as the cardinality estimate.
For more information:
- https://datasketches.apache.org/docs/CPC/CPC.html
- https://datasketches.apache.org/docs/CPC/CpcPerformance.html
'''
-) AS R"""
-try {
- var sketchObject = null;
- try {
- sketchObject = Module.cpc_sketch.deserializeFromB64(sketch, seed ? BigInt(seed) : BigInt(Module.DEFAULT_SEED));
- return sketchObject.getEstimate();
- } finally {
- if (sketchObject != null) sketchObject.delete();
- }
-} catch (e) {
- throw new Error(Module.getExceptionMessage(e));
-}
-""";
+) AS (
+ $BQ_DATASET.cpc_sketch_get_estimate_seed(sketch, NULL)
+);
diff --git a/cpc/sqlx/cpc_sketch_get_estimate_and_bounds.sqlx b/cpc/sqlx/cpc_sketch_get_estimate_and_bounds.sqlx
index 8d573be..537082f 100644
--- a/cpc/sqlx/cpc_sketch_get_estimate_and_bounds.sqlx
+++ b/cpc/sqlx/cpc_sketch_get_estimate_and_bounds.sqlx
@@ -19,38 +19,22 @@
config { hasOutput: true }
-CREATE OR REPLACE FUNCTION ${self()}(sketch BYTES, num_std_devs BYTEINT, seed INT64)
+CREATE OR REPLACE FUNCTION ${self()}(sketch BYTES, num_std_devs BYTEINT)
RETURNS STRUCT<estimate FLOAT64, lower_bound FLOAT64, upper_bound FLOAT64>
-LANGUAGE js
OPTIONS (
- library=["gs://$GCS_BUCKET/cpc_sketch.js"],
description = '''Gets cardinality estimate and bounds from given sketch.
Param sketch: The given sketch to query as bytes.
Param num_std_devs: The returned bounds will be based on the statistical confidence interval determined by the given number of standard deviations
from the returned estimate. This number may be one of {1,2,3}, where 1 represents 68% confidence, 2 represents 95% confidence and 3 represents 99.7% confidence.
For example, if the given num_std_devs = 2 and the returned values are {1000, 990, 1010} that means that with 95% confidence, the true value lies within the range [990, 1010].
-Param seed: This is used to confirm that the given sketch was configured with the correct seed.
+Default seed.
Returns: a STRUCT with 3 FLOAT64 values as {estimate, lower_bound, upper_bound}.
For more information:
- https://datasketches.apache.org/docs/CPC/CPC.html
- https://datasketches.apache.org/docs/CPC/CpcPerformance.html
'''
-) AS R"""
-try {
- var sketchObject = null;
- try {
- sketchObject = Module.cpc_sketch.deserializeFromB64(sketch, seed ? BigInt(seed) : BigInt(Module.DEFAULT_SEED));
- return {
- estimate: sketchObject.getEstimate(),
- lower_bound: sketchObject.getLowerBound(num_std_devs),
- upper_bound: sketchObject.getUpperBound(num_std_devs)
- };
- } finally {
- if (sketchObject != null) sketchObject.delete();
- }
-} catch (e) {
- throw new Error(Module.getExceptionMessage(e));
-}
-""";
+) AS (
+ $BQ_DATASET.cpc_sketch_get_estimate_and_bounds_seed(sketch, num_std_devs, NULL)
+);
diff --git a/cpc/sqlx/cpc_sketch_get_estimate_and_bounds.sqlx b/cpc/sqlx/cpc_sketch_get_estimate_and_bounds_seed.sqlx
similarity index 100%
copy from cpc/sqlx/cpc_sketch_get_estimate_and_bounds.sqlx
copy to cpc/sqlx/cpc_sketch_get_estimate_and_bounds_seed.sqlx
diff --git a/cpc/sqlx/cpc_sketch_get_estimate.sqlx b/cpc/sqlx/cpc_sketch_get_estimate_seed.sqlx
similarity index 100%
copy from cpc/sqlx/cpc_sketch_get_estimate.sqlx
copy to cpc/sqlx/cpc_sketch_get_estimate_seed.sqlx
diff --git a/cpc/sqlx/cpc_sketch_to_string.sqlx b/cpc/sqlx/cpc_sketch_to_string.sqlx
index 270684c..6301a41 100644
--- a/cpc/sqlx/cpc_sketch_to_string.sqlx
+++ b/cpc/sqlx/cpc_sketch_to_string.sqlx
@@ -19,31 +19,18 @@
config { hasOutput: true }
-CREATE OR REPLACE FUNCTION ${self()}(sketch BYTES, seed INT64)
+CREATE OR REPLACE FUNCTION ${self()}(sketch BYTES)
RETURNS STRING
-LANGUAGE js
OPTIONS (
- library=["gs://$GCS_BUCKET/cpc_sketch.js"],
description = '''Returns a summary string that represents the state of the given sketch.
Param sketch the given sketch as BYTES.
-Param seed: This is used to confirm that the given sketch was configured with the correct seed.
+Default seed.
Returns: a STRING that represents the state of the given sketch.
For more information:
- https://datasketches.apache.org/docs/CPC/CPC.html
'''
-) AS R"""
-const default_seed = BigInt(Module.DEFAULT_SEED);
-try {
- var sketchObject = null;
- try {
- sketchObject = Module.cpc_sketch.deserializeFromB64(sketch, seed ? BigInt(seed) : default_seed);
- return sketchObject.toString();
- } finally {
- if (sketchObject != null) sketchObject.delete();
- }
-} catch (e) {
- throw new Error(Module.getExceptionMessage(e));
-}
-""";
+) AS (
+ $BQ_DATASET.cpc_sketch_to_string_seed(sketch, NULL)
+);
diff --git a/cpc/sqlx/cpc_sketch_to_string.sqlx b/cpc/sqlx/cpc_sketch_to_string_seed.sqlx
similarity index 100%
copy from cpc/sqlx/cpc_sketch_to_string.sqlx
copy to cpc/sqlx/cpc_sketch_to_string_seed.sqlx
diff --git a/cpc/sqlx/cpc_sketch_scalar_union.sqlx b/cpc/sqlx/cpc_sketch_union.sqlx
similarity index 62%
copy from cpc/sqlx/cpc_sketch_scalar_union.sqlx
copy to cpc/sqlx/cpc_sketch_union.sqlx
index 98f530b..9f5c4d6 100644
--- a/cpc/sqlx/cpc_sketch_scalar_union.sqlx
+++ b/cpc/sqlx/cpc_sketch_union.sqlx
@@ -19,37 +19,20 @@
config { hasOutput: true }
-CREATE OR REPLACE FUNCTION ${self()}(sketchA BYTES, sketchB BYTES, lg_k BYTEINT, seed INT64)
+CREATE OR REPLACE FUNCTION ${self()}(sketchA BYTES, sketchB BYTES)
RETURNS BYTES
-LANGUAGE js
OPTIONS (
- library=["gs://$GCS_BUCKET/cpc_sketch.js"],
description = '''Computes a sketch that represents the scalar union of the two given sketches.
Param sketchA: the first sketch as BYTES.
Param sketchB: the second sketch as BYTES.
-Param lg_k: the sketch accuracy/size parameter as an integer in the range [4, 26].
-Param seed: This is used to confirm that the given sketches were configured with the correct seed.
+Default lg_k and seed.
Returns: a CPC Sketch, as BYTES.
For more information:
- https://datasketches.apache.org/docs/CPC/CPC.html
- https://datasketches.apache.org/docs/CPC/CpcPerformance.html
'''
-) AS R"""
-const default_lg_k = 12;
-const default_seed = BigInt(Module.DEFAULT_SEED);
-try {
- var union = null;
- try {
- union = new Module.cpc_union(lg_k ? lg_k : default_lg_k, seed ? BigInt(seed) : default_seed);
- union.updateWithB64(sketchA, seed ? BigInt(seed) : default_seed)
- union.updateWithB64(sketchB, seed ? BigInt(seed) : default_seed)
- return union.getResultB64();
- } finally {
- if (union != null) union.delete();
- }
-} catch (e) {
- throw new Error(Module.getExceptionMessage(e));
-}
-""";
+) AS (
+ $BQ_DATASET.cpc_sketch_union_lgk_seed(sketchA, sketchB, NULL, NULL)
+);
diff --git a/cpc/sqlx/cpc_sketch_scalar_union.sqlx b/cpc/sqlx/cpc_sketch_union_lgk_seed.sqlx
similarity index 100%
rename from cpc/sqlx/cpc_sketch_scalar_union.sqlx
rename to cpc/sqlx/cpc_sketch_union_lgk_seed.sqlx
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org