You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by al...@apache.org on 2024/11/06 23:17:50 UTC
(datasketches-bigquery) 01/01: hll sketch tests
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch hll_sketch_test
in repository https://gitbox.apache.org/repos/asf/datasketches-bigquery.git
commit 1e044a9ca65dff7f55ecc45bc3b9cffb17b870be
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Wed Nov 6 15:17:23 2024 -0800
hll sketch tests
---
tests/hll_sketch_test.js | 168 +++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 168 insertions(+)
diff --git a/tests/hll_sketch_test.js b/tests/hll_sketch_test.js
new file mode 100644
index 0000000..85875bd
--- /dev/null
+++ b/tests/hll_sketch_test.js
@@ -0,0 +1,168 @@
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+const { generate_udf_test, generate_udaf_test } = require('./unit_test_utils');
+
+// using defaults
+
+const hll_1 = `FROM_BASE64('AgEHDAMIAwAvgjsECv+ABG8Z3AY=')`;
+
+generate_udaf_test("hll_sketch_agg_string", {
+ input_columns: [`str`],
+ input_rows: `SELECT * FROM UNNEST(['a', 'b', 'c']) AS str`,
+ expected_output: hll_1
+});
+
+const hll_2 = `FROM_BASE64('AgEHDAMIAwBvGdwGwL9qEp160As=')`;
+
+generate_udaf_test("hll_sketch_agg_string", {
+ input_columns: [`str`],
+ input_rows: `SELECT * FROM UNNEST(['c', 'd', 'e']) AS str`,
+ expected_output: hll_2
+});
+
+const hll_union_1 = `FROM_BASE64('AgEHDAMIBQAvgjsECv+ABG8Z3AbAv2oSnXrQCw==')`;
+
+generate_udf_test("hll_sketch_union", [{
+ inputs: [ hll_1, hll_2 ],
+ expected_output: hll_union_1
+}]);
+
+generate_udf_test("hll_sketch_get_estimate", [{
+ inputs: [ hll_union_1 ],
+ expected_output: 5.000000049670538
+}]);
+
+generate_udf_test("hll_sketch_to_string", [{
+ inputs: [ hll_union_1 ],
+ expected_output: `'''### HLL sketch summary:
+ Log Config K : 12
+ Hll Target : HLL_4
+ Current Mode : LIST
+ LB : 5
+ Estimate : 5
+ UB : 5.00025
+ OutOfOrder flag: false
+ Coupon count : 5
+### End HLL sketch summary
+'''`
+}]);
+
+const hll_3 = `FROM_BASE64('CgEHDAAIAAI/Uk0InnjDQAAAAPBfcpJAAAAAAAAAAAByAQAAAAAAAFQjESCDUyBCAzNRElExO2MhQjAyIiQyNRUyMxFRIEJDQjY4JDIkIghjIFIjIxIUQCEjMBFGIyIhUTQhJBIkEyQlA0MkIxNTEkQDQSIDFyEDEyElI1MTFjJiVTEmISAxESYTNCkRYUMXESQwY0ACQUMHFTchEzEgEzZDQlE0QkMCUxNkMkMyAiETMhMiJkIUEVJTERQzcCEmkTN0ZiAWQWMxAiEzMRIEMkBKEzUzEyISIUQTIhNQMwJyESQCQDYBMiIQIgYSJBZAQlRiIgcTEWQxQjAhdhFEMhAhFjYkITQUQyJDM1UAAhNBMiEhUhIEERRBVDMhZEMTQiFDBhQ1UkMhNRAiNDQTI1IDM0NEMEI1EAETAxEggkYjATMFMTAgMiJUEiEVIRYTMxQzNBEjEQMkJAFA [...]
+
+generate_udaf_test("hll_sketch_agg_int64", {
+ input_columns: [`value`],
+ input_rows: `SELECT * FROM UNNEST(GENERATE_ARRAY(1, 10000, 1)) AS value`,
+ expected_output: hll_3
+});
+
+const hll_4 = `FROM_BASE64('CgEHDAUIAAJUyNXwJZbDQAAAADjDZZJAAAAAAAAAAABiAQAAAQAAABMREiJEIkMzI0cjICM0IQIAAUIhAEIUMVI0E1IUJARCEhFCIjNDAAFWEgUyIzEkggIzIzNRBEJ1I0IUMmYgJEQlFRIiIREjMTVABBEDEUBGMgE2MyIxEVEFIkJFMTJEIkIhgxUgJEMHFSE1ETI1KhIBVUIwQzQQNRMSIjQRI1FhISElMxEyQwNBBBEhElFjQzMSYkIQIgVEQyJEIkISUyMhIgNmFTFCOjMDEiAzMRNFIiQRFhMzVjQCETAiGEIUMTFCFUFUQEVAYmJDQkAQFCEzMTFUIhEwBAMjIRFlEzMYQjeEITExOUISMDFRJRIxJCIgUyJBASEhIyBBNBJCIRYyMiQWMxYkA0QxISSTYQMRQhUyIAEwJzRmIBISMHRTMiREEVEzEjEjJTESZBMzMSYjQQUj [...]
+
+generate_udaf_test("hll_sketch_agg_int64", {
+ input_columns: [`value`],
+ input_rows: `SELECT * FROM UNNEST(GENERATE_ARRAY(100000, 110000, 1)) AS value`,
+ expected_output: hll_4
+});
+
+const hll_union_2 = `FROM_BASE64('CgEHDAUYAAIAAAAAAAAAAAAAAFD2qYJAAAAAAAAAAAAhAAAAAQAAAFQjEiKEU0NDI0dTIlM0O2MhQkIyIkQ0NVU0M1JUJERDQjZIJDNEIghmIlUzIzIkgiIzMzNWJEJ1U0QkNGYkJEQlFUMkIxNTMkVDRCIDF0FGMyE2M1MzFlJlVUJGMTJEIkYjhCkhZEMXFSQ1Y0I1SkMHVUcxQzQgNTZDQlQ0Q1NiUyNlM0MyQyNDNBMiJlJkQ1NTYkQzciVGkzN0ZkIWU2MxIiNmNTJEOkNKEzUzMyNFIkQTJhNTVjRyETQiSEYUMjJCJUZUREZAYmRjQkcTFGQzQjFUdhFENBMjJjZlIzQYQzeEM1UxOUNCMjFRVRI0JCRBVDNBZEMjQyFDNhRFUkYyNSQmNDYkI1QzM0SUYUM1QhUzIxEwh0ZmITMVMXRTMiRUElE1IjYjNTQzZBMzMS [...]
+
+generate_udaf_test("hll_sketch_agg_union", {
+ input_columns: [`sketch`],
+ input_rows: `SELECT * FROM UNNEST([${hll_3}, ${hll_4}]) AS sketch`,
+ expected_output: hll_union_2
+});
+
+generate_udf_test("hll_sketch_get_estimate_and_bounds", [{
+ inputs: [ hll_union_2, 3 ],
+ expected_output: `STRUCT(20250.985334743167 AS estimate, 19292.57752380849 AS lower_bound, 21249.1304948276 AS upper_bound)`
+}]);
+
+
+// using full signatures
+
+const hll_8_hll6_1 = `FROM_BASE64('AgEHCAMIAwQvgjsECv+ABG8Z3AY=')`;
+
+generate_udaf_test("hll_sketch_agg_string_lgk_type", {
+ input_columns: [`str`, 'STRUCT(8 AS lgk, "HLL_6" AS type) NOT AGGREGATE'],
+ input_rows: `SELECT * FROM UNNEST(['a', 'b', 'c']) AS str`,
+ expected_output: hll_8_hll6_1
+});
+
+const hll_8_hll6_2 = `FROM_BASE64('AgEHCAMIAwRvGdwGwL9qEp160As=')`;
+
+generate_udaf_test("hll_sketch_agg_string_lgk_type", {
+ input_columns: [`str`, 'STRUCT(8 AS lgk, "HLL_6" AS type) NOT AGGREGATE'],
+ input_rows: `SELECT * FROM UNNEST(['c', 'd', 'e']) AS str`,
+ expected_output: hll_8_hll6_2
+});
+
+const hll_union_8_hll6_1 = `FROM_BASE64('AgEHCAMIBQQvgjsECv+ABG8Z3AbAv2oSnXrQCw==')`;
+
+generate_udf_test("hll_sketch_union_lgk_type", [{
+ inputs: [ hll_8_hll6_1, hll_8_hll6_2, 8, `"HLL_6"` ],
+ expected_output: hll_union_8_hll6_1
+}]);
+
+generate_udf_test("hll_sketch_get_estimate", [{
+ inputs: [ hll_union_8_hll6_1 ],
+ expected_output: 5.000000049670538
+}]);
+
+generate_udf_test("hll_sketch_to_string", [{
+ inputs: [ hll_union_8_hll6_1 ],
+ expected_output: `'''### HLL sketch summary:
+ Log Config K : 8
+ Hll Target : HLL_6
+ Current Mode : LIST
+ LB : 5
+ Estimate : 5
+ UB : 5.00025
+ OutOfOrder flag: false
+ Coupon count : 5
+### End HLL sketch summary
+'''`
+}]);
+
+const hll_8_hll6_3 = `FROM_BASE64('CgEHCAAIAAam0v3qmrTDQAAAAADwBxJAAAAAAAAAAAAAAAAAAAAAAEhRIIxRIAhiHMVhGIeBIEmSGAaCFMtRGMqRFAlBNAZhIARiHEeCGMVhIAhyHEVhGISRGAhRFMdhIMqBIMmhIEhxHMVhHIaBGIlxJMRBHEdRGIZRGMhRGIRhEMdiHEWBFApxFINjGAqyGEyxHIWBFIZBIAdyGMZhGMZhEAiRHApSFIZRKIRxGEZhGEZiGMdhHAdTHAWSGIVSGARxGAZRJEdiHIeBGIZxHEdhHIdBJIlhGEVhFAphFIWRHIdREEVRGAA=')`;
+
+generate_udaf_test("hll_sketch_agg_int64_lgk_type", {
+ input_columns: [`value`, 'STRUCT(8 AS lgk, "HLL_6" AS type) NOT AGGREGATE'],
+ input_rows: `SELECT * FROM UNNEST(GENERATE_ARRAY(1, 10000, 1)) AS value`,
+ expected_output: hll_8_hll6_3
+});
+
+const hll_8_hll6_4 = `FROM_BASE64('CgEHCAAIAAYaGqx/Y6bCQAAAAAA4NRNAAAAAAAAAAAAAAAAAAAAAAAdCFAZiGEZhFIhhHEZxEAZyGEZiGIlBIIRRFIeBGISBGAeBHIdBGMhTHARCHIVxFMaRFEhhGAZiEEiiGEZRHIaRGISBHAZhMIphGEtxEE5BGEdRHEdhGMdRFAhyNIhxHIVxHEVxHEVRGMVBIEeSGAVSKAZSLIZhHAVRFEZhHAVxIIZhFIdBFEZiFIZRFIWBHIVxJAhyFMdhEElxLEiBFEVhFEaBIEphJEVSHEWBFElxEEeSHMVxGAhxFEZxGApjGAA=')`;
+
+generate_udaf_test("hll_sketch_agg_int64_lgk_type", {
+ input_columns: [`value`, 'STRUCT(8 AS lgk, "HLL_6" AS type) NOT AGGREGATE'],
+ input_rows: `SELECT * FROM UNNEST(GENERATE_ARRAY(100000, 110000, 1)) AS value`,
+ expected_output: hll_8_hll6_4
+});
+
+const hll_union_8_hll6_2 = `FROM_BASE64('CgEHCAAYAAYAAAAAAAAAAAAAAABQSgJAAAAAAAAAAAAAAAAAAAAAAAhSIAxiIAhiHMhhHIeBIEmSGEaCGMtRIMqRFImBNIaBIAeCHEeCGMhjIAhyHIVxGMaRGEhhGAdiIEqiIMmhIIiRHMWBHIaBMIpxJMtxHE5RGIdRHMhhGMdhFMhyNIiBHIpxHIVzHAqyGMyxIEeSGAZSKAdyLMZhHMZhFEiRHApyIIZhKIdxGEZiGEZiGMeBHAdzJAiSGIdiGElxLEiBJEdiHIeBIIpxJEdiHIeBJIlxGEeSHMpxGIiRHIdxGApjGAA=')`;
+
+generate_udaf_test("hll_sketch_agg_union_lgk_type", {
+ input_columns: [`sketch`, 'STRUCT(8 AS lgk, "HLL_6" AS type) NOT AGGREGATE'],
+ input_rows: `SELECT * FROM UNNEST([${hll_8_hll6_3}, ${hll_8_hll6_4}]) AS sketch`,
+ expected_output: hll_union_8_hll6_2
+});
+
+generate_udf_test("hll_sketch_get_estimate_and_bounds", [{
+ inputs: [ hll_union_8_hll6_2, 3 ],
+ expected_output: `STRUCT(20589.9367655959 AS estimate, 16893.540168045525 AS lower_bound, 24930.66787891017 AS upper_bound)`
+}]);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org