You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by al...@apache.org on 2024/09/17 19:55:59 UTC
(datasketches-bigquery) branch theta_defaults created (now 7af53e4)
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a change to branch theta_defaults
in repository https://gitbox.apache.org/repos/asf/datasketches-bigquery.git
at 7af53e4 functions with defaults
This branch includes the following new commits:
new 7af53e4 functions with defaults
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
(datasketches-bigquery) 01/01: functions with defaults
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch theta_defaults
in repository https://gitbox.apache.org/repos/asf/datasketches-bigquery.git
commit 7af53e42354719106a727928cec806a9ce453df5
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Tue Sep 17 12:55:32 2024 -0700
functions with defaults
---
theta/sqlx/theta_sketch_a_not_b.sqlx | 23 +---
...a_not_b.sqlx => theta_sketch_a_not_b_seed.sqlx} | 0
theta/sqlx/theta_sketch_agg_union.sqlx | 134 +--------------------
...n.sqlx => theta_sketch_agg_union_lgk_seed.sqlx} | 0
theta/sqlx/theta_sketch_get_estimate.sqlx | 23 +---
.../sqlx/theta_sketch_get_estimate_and_bounds.sqlx | 27 +----
...theta_sketch_get_estimate_and_bounds_seed.sqlx} | 0
...te.sqlx => theta_sketch_get_estimate_seed.sqlx} | 0
...section.sqlx => theta_sketch_intersection.sqlx} | 24 +---
...on.sqlx => theta_sketch_intersection_seed.sqlx} | 0
theta/sqlx/theta_sketch_jaccard_similarity.sqlx | 22 +---
...x => theta_sketch_jaccard_similarity_seed.sqlx} | 0
theta/sqlx/theta_sketch_to_string.sqlx | 23 +---
...tring.sqlx => theta_sketch_to_string_seed.sqlx} | 0
...h_scalar_union.sqlx => theta_sketch_union.sqlx} | 26 +---
...union.sqlx => theta_sketch_union_lgk_seed.sqlx} | 0
theta/test/theta_sketch_test.sql | 102 ++++++++++++----
17 files changed, 121 insertions(+), 283 deletions(-)
diff --git a/theta/sqlx/theta_sketch_a_not_b.sqlx b/theta/sqlx/theta_sketch_a_not_b.sqlx
index 9b36bcf..1e5961a 100644
--- a/theta/sqlx/theta_sketch_a_not_b.sqlx
+++ b/theta/sqlx/theta_sketch_a_not_b.sqlx
@@ -19,16 +19,14 @@
config { hasOutput: true }
-CREATE OR REPLACE FUNCTION ${self()}(sketchA BYTES, sketchB BYTES, seed INT64)
+CREATE OR REPLACE FUNCTION ${self()}(sketchA BYTES, sketchB BYTES)
RETURNS BYTES
-LANGUAGE js
OPTIONS (
- library=["gs://$GCS_BUCKET/theta_sketch.js"],
description = '''Computes a sketch that represents the scalar set difference: sketchA and not sketchB.
Param sketchA: the first sketch "A" as bytes.
Param sketchB: the second sketch "B" as bytes.
-Param seed: This is used to confirm that the given sketches were configured with the correct seed.
+Default seed.
Returns: a Compact, Compressed Theta Sketch, as BYTES.
For more information:
@@ -40,17 +38,6 @@ For more information:
- https://datasketches.apache.org/docs/Theta/ThetaSize.html
- https://datasketches.apache.org/docs/Theta/ThetaSketchFramework.html
'''
-) AS R"""
-const default_seed = BigInt(Module.DEFAULT_SEED);
-try {
- var a_not_b = null;
- try {
- a_not_b = new Module.theta_a_not_b(seed ? BigInt(seed) : default_seed);
- return a_not_b.computeWithB64ReturnB64Compressed(sketchA, sketchB, seed ? BigInt(seed) : default_seed);
- } finally {
- if (a_not_b != null) a_not_b.delete();
- }
-} catch (e) {
- throw new Error(Module.getExceptionMessage(e));
-}
-""";
+) AS (
+ $BQ_DATASET.theta_sketch_a_not_b_seed(sketchA, sketchB, NULL)
+);
diff --git a/theta/sqlx/theta_sketch_a_not_b.sqlx b/theta/sqlx/theta_sketch_a_not_b_seed.sqlx
similarity index 100%
copy from theta/sqlx/theta_sketch_a_not_b.sqlx
copy to theta/sqlx/theta_sketch_a_not_b_seed.sqlx
diff --git a/theta/sqlx/theta_sketch_agg_union.sqlx b/theta/sqlx/theta_sketch_agg_union.sqlx
index 49c3633..bcfc87e 100644
--- a/theta/sqlx/theta_sketch_agg_union.sqlx
+++ b/theta/sqlx/theta_sketch_agg_union.sqlx
@@ -19,16 +19,13 @@
config { hasOutput: true }
-CREATE OR REPLACE AGGREGATE FUNCTION ${self()}(sketch BYTES, params STRUCT<lg_k BYTEINT, seed INT64> NOT AGGREGATE)
+CREATE OR REPLACE AGGREGATE FUNCTION ${self()}(sketch BYTES)
RETURNS BYTES
-LANGUAGE js
OPTIONS (
- library=["gs://$GCS_BUCKET/theta_sketch.mjs"],
description = '''Creates a sketch that represents the union of the given column of sketches.
Param sketch: the column of sketches. Each as BYTES.
-Param lg_k: the sketch accuracy/size parameter as a BYTEINT in the range [4, 26].
-Param seed: This is used to confirm that the given sketches were configured with the correct seed.
+Default lg_k and seed.
Returns: a Compact, Compressed Theta Sketch, as BYTES.
For more information:
@@ -38,127 +35,6 @@ For more information:
- https://datasketches.apache.org/docs/Theta/ThetaSize.html
- https://datasketches.apache.org/docs/Theta/ThetaSketchFramework.html
'''
-) AS R"""
-import ModuleFactory from "gs://$GCS_BUCKET/theta_sketch.mjs";
-var Module = await ModuleFactory();
-const default_lg_k = Number(Module.DEFAULT_LG_K);
-const default_seed = BigInt(Module.DEFAULT_SEED);
-
-var buffer = {ptr: 0, size: 0};
-
-function reserveBuffer(size) {
- if (buffer.size < size) {
- if (buffer.ptr != 0) {
- Module._free(buffer.ptr);
- }
- buffer.ptr = Module._malloc(size);
- buffer.size = size;
- }
-}
-
-// ensures we have a theta_union
-// if there is a compact_theta_sketch, add it to the union and destroy it
-function ensureUnion(state) {
- try {
- if (state.union == null) {
- state.union = new Module.theta_union(state.lg_k, state.seed);
- }
- if (state.serialized != null) {
- reserveBuffer(state.serialized.length);
- Module.HEAPU8.subarray(buffer.ptr, buffer.ptr + state.serialized.length).set(state.serialized);
- state.union.updateWithBuffer(buffer.ptr, state.serialized.length, state.seed);
- state.serialized = null;
- }
- } catch (e) {
- throw new Error(Module.getExceptionMessage(e));
- }
-}
-
-export function initialState(params) {
- var state = {
- lg_k: params.lg_k == null ? default_lg_k : Number(params.lg_k),
- seed: params.seed == null ? default_seed : BigInt(params.seed),
- union: null,
- serialized: null
- };
- reserveBuffer(Module.compact_theta_sketch.getMaxSerializedSizeBytes(state.lg_k));
- return state;
-}
-
-export function aggregate(state, sketch) {
- if (sketch != null) {
- ensureUnion(state);
- try {
- reserveBuffer(sketch.length);
- Module.HEAPU8.subarray(buffer.ptr, buffer.ptr + sketch.length).set(sketch);
- state.union.updateWithBuffer(buffer.ptr, sketch.length, state.seed);
- } catch (e) {
- throw new Error(Module.getExceptionMessage(e));
- }
- }
-}
-
-export function serialize(state) {
- try {
- ensureUnion(state);
- reserveBuffer(Module.compact_theta_sketch.getMaxSerializedSizeBytes(state.lg_k));
- var size = state.union.getResultStreamCompressed(buffer.ptr, buffer.size);
- return {
- lg_k: state.lg_k,
- seed: state.seed,
- bytes: Module.HEAPU8.slice(buffer.ptr, buffer.ptr + size)
- };
- } catch (e) {
- throw new Error(Module.getExceptionMessage(e));
- } finally {
- if (state.union != null) {
- state.union.delete();
- }
- state.union = null;
- }
-}
-
-export function deserialize(serialized) {
- return {
- lg_k: serialized.lg_k,
- seed: serialized.seed,
- union: null,
- serialized: serialized.bytes
- };
-}
-
-export function merge(state, other_state) {
- reserveBuffer(Module.compact_theta_sketch.getMaxSerializedSizeBytes(state.lg_k));
- if (other_state.union) {
- throw new Error("Did not expect union in other state");
- }
- if (state.union == null) {
- state.union = new Module.theta_union(state.lg_k, state.seed);
- }
- if (state.serialized != null) {
- try {
- reserveBuffer(state.serialized.length);
- Module.HEAPU8.subarray(buffer.ptr, buffer.ptr + state.serialized.length).set(state.serialized);
- state.union.updateWithBuffer(buffer.ptr, state.serialized.length, state.seed);
- state.serialized = null;
- } catch (e) {
- throw new Error(Module.getExceptionMessage(e));
- }
- }
- if (other_state.serialized != null) {
- try {
- reserveBuffer(other_state.serialized.length);
- Module.HEAPU8.subarray(buffer.ptr, buffer.ptr + other_state.serialized.length).set(other_state.serialized);
- state.union.updateWithBuffer(buffer.ptr, other_state.serialized.length, other_state.seed);
- other_state.serialized = null;
- } catch (e) {
- throw new Error(Module.getExceptionMessage(e));
- }
- } else {
- throw new Error("Expected serialized sketch in other_state");
- }
-}
-export function finalize(state) {
- return serialize(state).bytes;
-}
-""";
+) AS (
+ $BQ_DATASET.theta_sketch_agg_union_lgk_seed(sketch, STRUCT<BYTEINT, INT64>(NULL, NULL))
+);
diff --git a/theta/sqlx/theta_sketch_agg_union.sqlx b/theta/sqlx/theta_sketch_agg_union_lgk_seed.sqlx
similarity index 100%
copy from theta/sqlx/theta_sketch_agg_union.sqlx
copy to theta/sqlx/theta_sketch_agg_union_lgk_seed.sqlx
diff --git a/theta/sqlx/theta_sketch_get_estimate.sqlx b/theta/sqlx/theta_sketch_get_estimate.sqlx
index f51c50f..bb0c855 100644
--- a/theta/sqlx/theta_sketch_get_estimate.sqlx
+++ b/theta/sqlx/theta_sketch_get_estimate.sqlx
@@ -19,15 +19,13 @@
config { hasOutput: true }
-CREATE OR REPLACE FUNCTION ${self()}(sketch BYTES, seed INT64)
+CREATE OR REPLACE FUNCTION ${self()}(sketch BYTES)
RETURNS FLOAT64
-LANGUAGE js
OPTIONS (
- library=["gs://$GCS_BUCKET/theta_sketch.js"],
description = '''Gets cardinality estimate and bounds from given sketch.
Param sketch: The given sketch to query as BYTES.
-Param seed: This is used to confirm that the given sketch was configured with the correct seed.
+Default seed.
Returns: a FLOAT64 value as the cardinality estimate.
For more information:
@@ -36,17 +34,6 @@ For more information:
- https://datasketches.apache.org/docs/Theta/ThetaErrorTable.html
- https://datasketches.apache.org/docs/Theta/ThetaSketchFramework.html
'''
-) AS R"""
-const default_seed = BigInt(Module.DEFAULT_SEED);
-try {
- var sketchObject = null;
- try {
- sketchObject = Module.compact_theta_sketch.deserializeFromB64(sketch, seed ? BigInt(seed) : default_seed);
- return sketchObject.getEstimate();
- } finally {
- if (sketchObject != null) sketchObject.delete();
- }
-} catch (e) {
- throw new Error(Module.getExceptionMessage(e));
-}
-""";
+) AS (
+ $BQ_DATASET.theta_sketch_get_estimate_seed(sketch, NULL)
+);
diff --git a/theta/sqlx/theta_sketch_get_estimate_and_bounds.sqlx b/theta/sqlx/theta_sketch_get_estimate_and_bounds.sqlx
index c0d9ec6..e249f1f 100644
--- a/theta/sqlx/theta_sketch_get_estimate_and_bounds.sqlx
+++ b/theta/sqlx/theta_sketch_get_estimate_and_bounds.sqlx
@@ -19,11 +19,9 @@
config { hasOutput: true }
-CREATE OR REPLACE FUNCTION ${self()}(sketch BYTES, num_std_devs BYTEINT, seed INT64)
+CREATE OR REPLACE FUNCTION ${self()}(sketch BYTES, num_std_devs BYTEINT)
RETURNS STRUCT<estimate FLOAT64, lower_bound FLOAT64, upper_bound FLOAT64>
-LANGUAGE js
OPTIONS (
- library=["gs://$GCS_BUCKET/theta_sketch.js"],
description = '''Gets cardinality estimate and bounds from given sketch.
Param sketch: The given sketch to query as BYTES.
@@ -33,7 +31,7 @@ Param num_std_devs: The returned bounds will be based on the statistical confide
2 represents 95% confidence and 3 represents 99.7% confidence.
For example, if the given num_std_devs = 2 and the returned values are {1000, 990, 1010}
that means that with 95% confidence, the true value lies within the range [990, 1010].
-Param seed: This is used to confirm that the given sketch was configured with the correct seed.
+Default seed.
Returns: a STRUCT with three FLOAT64 values as {estimate, lower_bound, upper_bound}.
For more information:
@@ -42,21 +40,6 @@ For more information:
- https://datasketches.apache.org/docs/Theta/ThetaErrorTable.html
- https://datasketches.apache.org/docs/Theta/ThetaSketchFramework.html
'''
-) AS R"""
-const default_seed = BigInt(Module.DEFAULT_SEED);
-try {
- var sketchObject = null;
- try {
- sketchObject = Module.compact_theta_sketch.deserializeFromB64(sketch, seed ? BigInt(seed) : default_seed);
- return {
- estimate: sketchObject.getEstimate(),
- lower_bound: sketchObject.getLowerBound(num_std_devs),
- upper_bound: sketchObject.getUpperBound(num_std_devs)
- };
- } finally {
- if (sketchObject != null) sketchObject.delete();
- }
-} catch (e) {
- throw new Error(Module.getExceptionMessage(e));
-}
-""";
+) AS (
+ $BQ_DATASET.theta_sketch_get_estimate_and_bounds_seed(sketch, num_std_devs, NULL)
+);
diff --git a/theta/sqlx/theta_sketch_get_estimate_and_bounds.sqlx b/theta/sqlx/theta_sketch_get_estimate_and_bounds_seed.sqlx
similarity index 100%
copy from theta/sqlx/theta_sketch_get_estimate_and_bounds.sqlx
copy to theta/sqlx/theta_sketch_get_estimate_and_bounds_seed.sqlx
diff --git a/theta/sqlx/theta_sketch_get_estimate.sqlx b/theta/sqlx/theta_sketch_get_estimate_seed.sqlx
similarity index 100%
copy from theta/sqlx/theta_sketch_get_estimate.sqlx
copy to theta/sqlx/theta_sketch_get_estimate_seed.sqlx
diff --git a/theta/sqlx/theta_sketch_scalar_intersection.sqlx b/theta/sqlx/theta_sketch_intersection.sqlx
similarity index 71%
copy from theta/sqlx/theta_sketch_scalar_intersection.sqlx
copy to theta/sqlx/theta_sketch_intersection.sqlx
index 24a5621..eaf3016 100644
--- a/theta/sqlx/theta_sketch_scalar_intersection.sqlx
+++ b/theta/sqlx/theta_sketch_intersection.sqlx
@@ -19,16 +19,14 @@
config { hasOutput: true }
-CREATE OR REPLACE FUNCTION ${self()}(sketchA BYTES, sketchB BYTES, seed INT64)
+CREATE OR REPLACE FUNCTION ${self()}(sketchA BYTES, sketchB BYTES)
RETURNS BYTES
-LANGUAGE js
OPTIONS (
- library=["gs://$GCS_BUCKET/theta_sketch.js"],
description = '''Computes a sketch that represents the scalar intersection of the two given sketches.
Param sketchA: the first sketch as BYTES.
Param sketchB: the second sketch as BYTES.
-Param seed: This is used to confirm that the given sketches were configured with the correct seed.
+Default seed.
Returns: a Compact, Compressed Theta Sketch, as BYTES.
For more information:
@@ -40,18 +38,6 @@ For more information:
- https://datasketches.apache.org/docs/Theta/ThetaSize.html
- https://datasketches.apache.org/docs/Theta/ThetaSketchFramework.html
'''
-) AS R"""
-const default_seed = BigInt(Module.DEFAULT_SEED);
-try {
- var intersection = new Module.theta_intersection(seed ? BigInt(seed) : default_seed);
- try {
- intersection.updateWithB64(sketchA, seed ? BigInt(seed) : default_seed);
- intersection.updateWithB64(sketchB, seed ? BigInt(seed) : default_seed);
- return intersection.getResultB64Compressed();
- } finally {
- intersection.delete();
- }
-} catch (e) {
- throw new Error(Module.getExceptionMessage(e));
-}
-""";
+) AS (
+ $BQ_DATASET.theta_sketch_intersection_seed(sketchA, sketchB, NULL)
+);
diff --git a/theta/sqlx/theta_sketch_scalar_intersection.sqlx b/theta/sqlx/theta_sketch_intersection_seed.sqlx
similarity index 100%
rename from theta/sqlx/theta_sketch_scalar_intersection.sqlx
rename to theta/sqlx/theta_sketch_intersection_seed.sqlx
diff --git a/theta/sqlx/theta_sketch_jaccard_similarity.sqlx b/theta/sqlx/theta_sketch_jaccard_similarity.sqlx
index a48fbcc..2265e29 100644
--- a/theta/sqlx/theta_sketch_jaccard_similarity.sqlx
+++ b/theta/sqlx/theta_sketch_jaccard_similarity.sqlx
@@ -19,11 +19,9 @@
config { hasOutput: true }
-CREATE OR REPLACE FUNCTION ${self()}(sketchA BYTES, sketchB BYTES, seed INT64)
+CREATE OR REPLACE FUNCTION ${self()}(sketchA BYTES, sketchB BYTES)
RETURNS STRUCT<lower_bound FLOAT64, estimate FLOAT64, upper_bound FLOAT64>
-LANGUAGE js
OPTIONS (
- library=["gs://$GCS_BUCKET/theta_sketch.js"],
description = '''Computes the Jaccard similarity index with upper and lower bounds.
The Jaccard similarity index J(A,B) = (A ^ B)/(A U B) is used to measure how similar the two sketches are to each other.
If J = 1.0, the sketches are considered equal. If J = 0, the two sketches are disjoint.
@@ -31,22 +29,12 @@ A Jaccard of .95 means the overlap between the two sets is 95% of the union of t
Param sketchA: the first sketch as bytes.
Param sketchB: the second sketch as bytes.
-Param seed: This is used to confirm that the given sketches were configured with the correct seed.
+Default seed.
Returns: a STRUCT with three FLOAT64 values {lower_bound, estimate, upper_bound} of the Jaccard index.
For more information:
- https://datasketches.apache.org/docs/Theta/ThetaSketchFramework.html
'''
-) AS R"""
-const default_seed = BigInt(Module.DEFAULT_SEED);
-try {
- const jaccard = Module.thetaJaccardSimilarity(sketchA, sketchB, seed == null ? default_seed : BigInt(seed));
- return {
- lower_bound: jaccard.get(0),
- estimate: jaccard.get(1),
- upper_bound: jaccard.get(2)
- };
-} catch (e) {
- throw new Error(Module.getExceptionMessage(e));
-}
-""";
+) AS (
+ $BQ_DATASET.theta_sketch_jaccard_similarity_seed(sketchA, sketchB, NULL)
+);
diff --git a/theta/sqlx/theta_sketch_jaccard_similarity.sqlx b/theta/sqlx/theta_sketch_jaccard_similarity_seed.sqlx
similarity index 100%
copy from theta/sqlx/theta_sketch_jaccard_similarity.sqlx
copy to theta/sqlx/theta_sketch_jaccard_similarity_seed.sqlx
diff --git a/theta/sqlx/theta_sketch_to_string.sqlx b/theta/sqlx/theta_sketch_to_string.sqlx
index d59b301..ee2f79d 100644
--- a/theta/sqlx/theta_sketch_to_string.sqlx
+++ b/theta/sqlx/theta_sketch_to_string.sqlx
@@ -19,31 +19,18 @@
config { hasOutput: true }
-CREATE OR REPLACE FUNCTION ${self()}(sketch BYTES, seed INT64)
+CREATE OR REPLACE FUNCTION ${self()}(sketch BYTES)
RETURNS STRING
-LANGUAGE js
OPTIONS (
- library=["gs://$GCS_BUCKET/theta_sketch.js"],
description = '''Returns a summary string that represents the state of the given sketch.
Param sketch: the given sketch as BYTES.
-Param seed: This is used to confirm that the given sketch was configured with the correct seed.
+Default seed.
Returns: a STRING that represents the state of the given sketch.
For more information:
- https://datasketches.apache.org/docs/Theta/ThetaSketchFramework.html
'''
-) AS R"""
-const default_seed = BigInt(Module.DEFAULT_SEED);
-try {
- var sketchObject = null;
- try {
- sketchObject = Module.compact_theta_sketch.deserializeFromB64(sketch, seed ? BigInt(seed) : default_seed);
- return sketchObject.toString();
- } finally {
- if (sketchObject != null) sketchObject.delete();
- }
-} catch (e) {
- throw new Error(Module.getExceptionMessage(e));
-}
-""";
+) AS (
+ $BQ_DATASET.theta_sketch_to_string_seed(sketch, NULL)
+);
diff --git a/theta/sqlx/theta_sketch_to_string.sqlx b/theta/sqlx/theta_sketch_to_string_seed.sqlx
similarity index 100%
copy from theta/sqlx/theta_sketch_to_string.sqlx
copy to theta/sqlx/theta_sketch_to_string_seed.sqlx
diff --git a/theta/sqlx/theta_sketch_scalar_union.sqlx b/theta/sqlx/theta_sketch_union.sqlx
similarity index 65%
copy from theta/sqlx/theta_sketch_scalar_union.sqlx
copy to theta/sqlx/theta_sketch_union.sqlx
index d3fd346..5b6efe1 100644
--- a/theta/sqlx/theta_sketch_scalar_union.sqlx
+++ b/theta/sqlx/theta_sketch_union.sqlx
@@ -19,17 +19,14 @@
config { hasOutput: true }
-CREATE OR REPLACE FUNCTION ${self()}(sketchA BYTES, sketchB BYTES, lg_k BYTEINT, seed INT64)
+CREATE OR REPLACE FUNCTION ${self()}(sketchA BYTES, sketchB BYTES)
RETURNS BYTES
-LANGUAGE js
OPTIONS (
- library=["gs://$GCS_BUCKET/theta_sketch.js"],
description = '''Computes a sketch that represents the scalar union of the two given sketches.
Param sketchA: the first sketch as BYTES.
Param sketchB: the second sketch as BYTES.
-Param lg_k: the sketch accuracy/size parameter as an integer in the range [4, 26].
-Param seed: This is used to confirm that the given sketches were configured with the correct seed.
+Default lg_k and seed.
Returns: a Compact, Compressed Theta Sketch, as BYTES.
For more information:
@@ -39,19 +36,6 @@ For more information:
- https://datasketches.apache.org/docs/Theta/ThetaSize.html
- https://datasketches.apache.org/docs/Theta/ThetaSketchFramework.html
'''
-) AS R"""
-const default_lg_k = Number(Module.DEFAULT_LG_K);
-const default_seed = BigInt(Module.DEFAULT_SEED);
-try {
- var union = new Module.theta_union(lg_k ? lg_k : default_lg_k, seed ? BigInt(seed) : default_seed);
- try {
- union.updateWithB64(sketchA, seed ? BigInt(seed) : default_seed)
- union.updateWithB64(sketchB, seed ? BigInt(seed) : default_seed)
- return union.getResultB64Compressed();
- } finally {
- union.delete();
- }
-} catch (e) {
- throw new Error(Module.getExceptionMessage(e));
-}
-""";
+) AS (
+ $BQ_DATASET.theta_sketch_union_lgk_seed(sketchA, sketchB, NULL, NULL)
+);
diff --git a/theta/sqlx/theta_sketch_scalar_union.sqlx b/theta/sqlx/theta_sketch_union_lgk_seed.sqlx
similarity index 100%
rename from theta/sqlx/theta_sketch_scalar_union.sqlx
rename to theta/sqlx/theta_sketch_union_lgk_seed.sqlx
diff --git a/theta/test/theta_sketch_test.sql b/theta/test/theta_sketch_test.sql
index bbb1d20..bbce179 100644
--- a/theta/test/theta_sketch_test.sql
+++ b/theta/test/theta_sketch_test.sql
@@ -17,62 +17,122 @@
* under the License.
*/
+# using defaults
create or replace table $BQ_DATASET.theta_sketch(sketch bytes);
insert into $BQ_DATASET.theta_sketch
-(select $BQ_DATASET.theta_sketch_agg_string_lgk_seed_p(cast(value as string), struct<int, int, float64>(14, null, 0.8)) from unnest(GENERATE_ARRAY(1, 10000, 1)) as value);
+(select $BQ_DATASET.theta_sketch_agg_string(cast(value as string)) from unnest(GENERATE_ARRAY(1, 10000, 1)) as value);
insert into $BQ_DATASET.theta_sketch
-(select $BQ_DATASET.theta_sketch_agg_string_lgk_seed_p(cast(value as string), struct<int, int, float64>(14, null, 0.8)) from unnest(GENERATE_ARRAY(100000, 110000, 1)) as value);
+(select $BQ_DATASET.theta_sketch_agg_string(cast(value as string)) from unnest(GENERATE_ARRAY(100000, 110000, 1)) as value);
# expected about 20000
select $BQ_DATASET.theta_sketch_get_estimate_and_bounds(
- $BQ_DATASET.theta_sketch_agg_union(sketch, struct<int, int>(10, null)),
- 2,
- null
+ $BQ_DATASET.theta_sketch_agg_union(sketch),
+ 2
) from $BQ_DATASET.theta_sketch;
# expected estimate about 20000
select $BQ_DATASET.theta_sketch_to_string(
- $BQ_DATASET.theta_sketch_agg_union(sketch, struct<int, int>(null, null)),
- null
+ $BQ_DATASET.theta_sketch_agg_union(sketch)
) from $BQ_DATASET.theta_sketch;
drop table $BQ_DATASET.theta_sketch;
+# using full signatures
+create or replace table $BQ_DATASET.theta_sketch(sketch bytes);
+
+insert into $BQ_DATASET.theta_sketch
+(select $BQ_DATASET.theta_sketch_agg_string_lgk_seed_p(cast(value as string), struct<int, int, float64>(14, 111, 0.9)) from unnest(GENERATE_ARRAY(1, 10000, 1)) as value);
+insert into $BQ_DATASET.theta_sketch
+(select $BQ_DATASET.theta_sketch_agg_string_lgk_seed_p(cast(value as string), struct<int, int, float64>(14, 111, 0.9)) from unnest(GENERATE_ARRAY(100000, 110000, 1)) as value);
+
+# expected about 20000
+select $BQ_DATASET.theta_sketch_get_estimate_and_bounds_seed(
+ $BQ_DATASET.theta_sketch_agg_union_lgk_seed(sketch, struct<int, int>(10, 111)),
+ 2,
+ 111
+) from $BQ_DATASET.theta_sketch;
+
+# expected estimate about 20000
+select $BQ_DATASET.theta_sketch_to_string_seed(
+ $BQ_DATASET.theta_sketch_agg_union_lgk_seed(sketch, struct<int, int>(10, 111)),
+ 111
+) from $BQ_DATASET.theta_sketch;
+
+drop table $BQ_DATASET.theta_sketch;
+
+# using defaults
# expected 5
select $BQ_DATASET.theta_sketch_get_estimate(
- $BQ_DATASET.theta_sketch_scalar_union(
+ $BQ_DATASET.theta_sketch_union(
(select $BQ_DATASET.theta_sketch_agg_string(str) from unnest(["a", "b", "c"]) as str),
- (select $BQ_DATASET.theta_sketch_agg_string(str) from unnest(["c", "d", "e"]) as str),
- null,
- null
+ (select $BQ_DATASET.theta_sketch_agg_string(str) from unnest(["c", "d", "e"]) as str)
+ )
+);
+
+# full signatures
+# expected 5
+select $BQ_DATASET.theta_sketch_get_estimate_seed(
+ $BQ_DATASET.theta_sketch_union_lgk_seed(
+ (select $BQ_DATASET.theta_sketch_agg_string_lgk_seed_p(str, struct<int, int, float64>(10, 111, 0.999)) from unnest(["a", "b", "c"]) as str),
+ (select $BQ_DATASET.theta_sketch_agg_string_lgk_seed_p(str, struct<int, int, float64>(10, 111, 0.999)) from unnest(["c", "d", "e"]) as str),
+ 10,
+ 111
),
- null
+ 111
);
+# using defaults
# expected 1
select $BQ_DATASET.theta_sketch_get_estimate(
- $BQ_DATASET.theta_sketch_scalar_intersection(
+ $BQ_DATASET.theta_sketch_intersection(
(select $BQ_DATASET.theta_sketch_agg_string(str) from unnest(["a", "b", "c"]) as str),
- (select $BQ_DATASET.theta_sketch_agg_string(str) from unnest(["c", "d", "e"]) as str),
- null
+ (select $BQ_DATASET.theta_sketch_agg_string(str) from unnest(["c", "d", "e"]) as str)
+ )
+);
+
+# full signatures
+# expected 1
+select $BQ_DATASET.theta_sketch_get_estimate_seed(
+ $BQ_DATASET.theta_sketch_intersection_seed(
+ (select $BQ_DATASET.theta_sketch_agg_string_lgk_seed_p(str, struct<int, int, float64>(10, 111, 0.999)) from unnest(["a", "b", "c"]) as str),
+ (select $BQ_DATASET.theta_sketch_agg_string_lgk_seed_p(str, struct<int, int, float64>(10, 111, 0.999)) from unnest(["c", "d", "e"]) as str),
+ 111
),
- null
+ 111
);
+# using defaults
# expected 2
select $BQ_DATASET.theta_sketch_get_estimate(
$BQ_DATASET.theta_sketch_a_not_b(
(select $BQ_DATASET.theta_sketch_agg_string(str) from unnest(["a", "b", "c"]) as str),
- (select $BQ_DATASET.theta_sketch_agg_string(str) from unnest(["c", "d", "e"]) as str),
- null
+ (select $BQ_DATASET.theta_sketch_agg_string(str) from unnest(["c", "d", "e"]) as str)
+ )
+);
+
+# full signatures
+# expected 2
+select $BQ_DATASET.theta_sketch_get_estimate_seed(
+ $BQ_DATASET.theta_sketch_a_not_b_seed(
+ (select $BQ_DATASET.theta_sketch_agg_string_lgk_seed_p(str, struct<int, int, float64>(10, 111, 0.999)) from unnest(["a", "b", "c"]) as str),
+ (select $BQ_DATASET.theta_sketch_agg_string_lgk_seed_p(str, struct<int, int, float64>(10, 111, 0.999)) from unnest(["c", "d", "e"]) as str),
+ 111
),
- null
+ 111
);
+# using defaults
# expected 0.2
select $BQ_DATASET.theta_sketch_jaccard_similarity(
(select $BQ_DATASET.theta_sketch_agg_string(str) from unnest(["a", "b", "c"]) as str),
- (select $BQ_DATASET.theta_sketch_agg_string(str) from unnest(["c", "d", "e"]) as str),
- null
+ (select $BQ_DATASET.theta_sketch_agg_string(str) from unnest(["c", "d", "e"]) as str)
+);
+
+# using full signatures
+# expected 0.2
+select $BQ_DATASET.theta_sketch_jaccard_similarity_seed(
+ (select $BQ_DATASET.theta_sketch_agg_string_lgk_seed_p(str, struct<int, int, float64>(10, 111, 0.999)) from unnest(["a", "b", "c"]) as str),
+ (select $BQ_DATASET.theta_sketch_agg_string_lgk_seed_p(str, struct<int, int, float64>(10, 111, 0.999)) from unnest(["c", "d", "e"]) as str),
+ 111
);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org