You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by ti...@apache.org on 2026/08/01 06:37:40 UTC
(datasketches-rust) branch codex/go-snapshot-compatibility created (now 1bf9cdf)
This is an automated email from the ASF dual-hosted git repository.
tisonkun pushed a change to branch codex/go-snapshot-compatibility
in repository https://gitbox.apache.org/repos/asf/datasketches-rust.git
at 1bf9cdf test: cover Go serialization snapshots
This branch includes the following new commits:
new 1bf9cdf test: cover Go serialization snapshots
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
(datasketches-rust) 01/01: test: cover Go serialization snapshots
Posted by ti...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tisonkun pushed a commit to branch codex/go-snapshot-compatibility
in repository https://gitbox.apache.org/repos/asf/datasketches-rust.git
commit 1bf9cdff86aca263cd567ad6535bc8060e1b0d21
Author: tison <wa...@gmail.com>
AuthorDate: Sat Aug 1 14:37:29 2026 +0800
test: cover Go serialization snapshots
---
CONTRIBUTING.md | 6 +-
datasketches/tests/serde_tests.rs | 2 +-
datasketches/tests/serde_tests/.gitignore | 1 +
datasketches/tests/serde_tests/bloom.rs | 16 ++++++
datasketches/tests/serde_tests/cpc.rs | 12 ++++
datasketches/tests/serde_tests/frequencies.rs | 79 +++++++++++++++++++++++++++
datasketches/tests/serde_tests/hll.rs | 34 ++++++++++++
datasketches/tests/serde_tests/tdigest.rs | 15 +++++
datasketches/tests/serde_tests/theta.rs | 24 +++++++-
datasketches/tests/serde_tests/tuple.rs | 13 ++++-
xtask/src/main.rs | 7 ++-
11 files changed, 201 insertions(+), 8 deletions(-)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index b280268..4942e6e 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -118,14 +118,16 @@ Serialization compatibility tests use snapshots from a pinned revision of [`apac
The `cargo x prepare-testdata` command downloads the TCK archive and synchronizes its snapshots into:
-- `datasketches/tests/serde_tests/java_generated_files`
- `datasketches/tests/serde_tests/cpp_generated_files`
+- `datasketches/tests/serde_tests/go_generated_files`
+- `datasketches/tests/serde_tests/java_generated_files`
You can synchronize them separately:
```shell
-cargo x prepare-testdata java
cargo x prepare-testdata cpp
+cargo x prepare-testdata go
+cargo x prepare-testdata java
```
If no language is specified, all languages are prepared. These directories are not stored in Git. Run the command before the first test run and again whenever the pinned TCK revision changes. It requires network access and replaces the selected generated directories.
diff --git a/datasketches/tests/serde_tests.rs b/datasketches/tests/serde_tests.rs
index 7348b0e..5f31c2d 100644
--- a/datasketches/tests/serde_tests.rs
+++ b/datasketches/tests/serde_tests.rs
@@ -33,7 +33,7 @@ pub fn serialization_test_data(sub_dir: &str, name: &str) -> PathBuf {
run the following commands from the project root to prepare the test data files
if they are missing:
- $ cargo x prepare-testdata --all
+ $ cargo x prepare-testdata
"#,
path.display(),
);
diff --git a/datasketches/tests/serde_tests/.gitignore b/datasketches/tests/serde_tests/.gitignore
index 84c031c..d642f7a 100644
--- a/datasketches/tests/serde_tests/.gitignore
+++ b/datasketches/tests/serde_tests/.gitignore
@@ -1,2 +1,3 @@
cpp_generated_files
+go_generated_files
java_generated_files
diff --git a/datasketches/tests/serde_tests/bloom.rs b/datasketches/tests/serde_tests/bloom.rs
index 5875b1f..39075c5 100644
--- a/datasketches/tests/serde_tests/bloom.rs
+++ b/datasketches/tests/serde_tests/bloom.rs
@@ -219,3 +219,19 @@ fn test_cpp_bloom_n30000000_h5() {
let path = serialization_test_data("cpp_generated_files", "bf_n30000000_h5_cpp.sk");
test_bloom_filter_file(path, 30000000, 5);
}
+
+#[test]
+fn test_go_compatibility() {
+ for (n, num_hashes) in [
+ (0, 3),
+ (0, 5),
+ (10_000, 3),
+ (10_000, 5),
+ (2_000_000, 3),
+ (2_000_000, 5),
+ ] {
+ let filename = format!("bf_n{n}_h{num_hashes}_go.sk");
+ let path = serialization_test_data("go_generated_files", &filename);
+ test_bloom_filter_file(path, n, num_hashes);
+ }
+}
diff --git a/datasketches/tests/serde_tests/cpc.rs b/datasketches/tests/serde_tests/cpc.rs
index a99ca6c..cf27650 100644
--- a/datasketches/tests/serde_tests/cpc.rs
+++ b/datasketches/tests/serde_tests/cpc.rs
@@ -107,3 +107,15 @@ fn test_cpp_compatibility() {
test_sketch_replay(&path, sketch, 1..=n);
}
}
+
+#[test]
+fn test_go_compatibility() {
+ let test_cases = [0, 100, 200, 2000, 20000];
+
+ for n in test_cases {
+ let filename = format!("cpc_n{n}_go.sk");
+ let path = serialization_test_data("go_generated_files", &filename);
+ let sketch = test_sketch_file(&path, n);
+ test_sketch_replay(&path, sketch, 0..n);
+ }
+}
diff --git a/datasketches/tests/serde_tests/frequencies.rs b/datasketches/tests/serde_tests/frequencies.rs
index 07051ba..31a80f8 100644
--- a/datasketches/tests/serde_tests/frequencies.rs
+++ b/datasketches/tests/serde_tests/frequencies.rs
@@ -264,3 +264,82 @@ fn test_cpp_frequent_strings_utf8() {
assert_eq!(sketch.estimate(&"шщъыь".to_string()), 6);
assert_eq!(sketch.estimate(&"эюя".to_string()), 7);
}
+
+#[test]
+fn test_go_frequent_longs_compatibility() {
+ let test_cases = [0, 1, 10, 100, 1000, 10000, 100000, 1000000];
+ for n in test_cases {
+ let filename = format!("frequent_long_n{n}_go.sk");
+ let path = serialization_test_data("go_generated_files", &filename);
+ let bytes = fs::read(&path).unwrap();
+ let sketch = FrequentItemsSketch::<i64>::deserialize(&bytes).unwrap();
+ assert_eq!(sketch.is_empty(), n == 0);
+ if n > 10 {
+ assert!(sketch.maximum_error() > 0);
+ } else {
+ assert_eq!(sketch.maximum_error(), 0);
+ }
+ assert_eq!(sketch.total_weight(), n);
+ }
+}
+
+#[test]
+fn test_go_frequent_strings_compatibility() {
+ let test_cases = [0, 1, 10, 100, 1000, 10000, 100000, 1000000];
+ for n in test_cases {
+ let filename = format!("frequent_string_n{n}_go.sk");
+ let path = serialization_test_data("go_generated_files", &filename);
+ let bytes = fs::read(&path).unwrap();
+ let sketch = FrequentItemsSketch::<String>::deserialize(&bytes).unwrap();
+ assert_eq!(sketch.is_empty(), n == 0);
+ if n > 10 {
+ assert!(sketch.maximum_error() > 0);
+ } else {
+ assert_eq!(sketch.maximum_error(), 0);
+ }
+ assert_eq!(sketch.total_weight(), n);
+ }
+}
+
+#[test]
+fn test_go_frequent_strings_ascii() {
+ let path = serialization_test_data("go_generated_files", "frequent_string_ascii_go.sk");
+ let bytes = fs::read(&path).unwrap();
+ let sketch = FrequentItemsSketch::<String>::deserialize(&bytes).unwrap();
+ assert!(!sketch.is_empty());
+ assert_eq!(sketch.maximum_error(), 0);
+ assert_eq!(sketch.total_weight(), 10);
+ assert_eq!(
+ sketch.estimate(&"aaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_string()),
+ 1
+ );
+ assert_eq!(
+ sketch.estimate(&"bbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string()),
+ 2
+ );
+ assert_eq!(
+ sketch.estimate(&"ccccccccccccccccccccccccccccc".to_string()),
+ 3
+ );
+ assert_eq!(
+ sketch.estimate(&"ddddddddddddddddddddddddddddd".to_string()),
+ 4
+ );
+}
+
+#[test]
+fn test_go_frequent_strings_utf8() {
+ let path = serialization_test_data("go_generated_files", "frequent_string_utf8_go.sk");
+ let bytes = fs::read(&path).unwrap();
+ let sketch = FrequentItemsSketch::<String>::deserialize(&bytes).unwrap();
+ assert!(!sketch.is_empty());
+ assert_eq!(sketch.maximum_error(), 0);
+ assert_eq!(sketch.total_weight(), 28);
+ assert_eq!(sketch.estimate(&"абвгд".to_string()), 1);
+ assert_eq!(sketch.estimate(&"еёжзи".to_string()), 2);
+ assert_eq!(sketch.estimate(&"йклмн".to_string()), 3);
+ assert_eq!(sketch.estimate(&"опрст".to_string()), 4);
+ assert_eq!(sketch.estimate(&"уфхцч".to_string()), 5);
+ assert_eq!(sketch.estimate(&"шщъыь".to_string()), 6);
+ assert_eq!(sketch.estimate(&"эюя".to_string()), 7);
+}
diff --git a/datasketches/tests/serde_tests/hll.rs b/datasketches/tests/serde_tests/hll.rs
index fb1e4fb..3447c78 100644
--- a/datasketches/tests/serde_tests/hll.rs
+++ b/datasketches/tests/serde_tests/hll.rs
@@ -164,6 +164,7 @@ fn test_serialized_bytes_match_reference_files_for_coupon_modes() {
for (dir, suffix) in [
("java_generated_files", "java"),
("cpp_generated_files", "cpp"),
+ ("go_generated_files", "go"),
] {
let filename = format!("{type_name}_n{n}_{suffix}.sk");
let path = serialization_test_data(dir, &filename);
@@ -251,6 +252,39 @@ fn test_cpp_hll8_compatibility() {
}
}
+#[test]
+fn test_go_hll4_compatibility() {
+ let test_cases = [0, 1, 10, 100, 1000, 10000, 100000, 1000000];
+
+ for n in test_cases {
+ let filename = format!("hll4_n{n}_go.sk");
+ let path = serialization_test_data("go_generated_files", &filename);
+ test_sketch_file(path, n, 12);
+ }
+}
+
+#[test]
+fn test_go_hll6_compatibility() {
+ let test_cases = [0, 1, 10, 100, 1000, 10000, 100000, 1000000];
+
+ for n in test_cases {
+ let filename = format!("hll6_n{n}_go.sk");
+ let path = serialization_test_data("go_generated_files", &filename);
+ test_sketch_file(path, n, 12);
+ }
+}
+
+#[test]
+fn test_go_hll8_compatibility() {
+ let test_cases = [0, 1, 10, 100, 1000, 10000, 100000, 1000000];
+
+ for n in test_cases {
+ let filename = format!("hll8_n{n}_go.sk");
+ let path = serialization_test_data("go_generated_files", &filename);
+ test_sketch_file(path, n, 12);
+ }
+}
+
#[test]
fn test_estimate_accuracy() {
// This test verifies and prints actual estimates to show accuracy
diff --git a/datasketches/tests/serde_tests/tdigest.rs b/datasketches/tests/serde_tests/tdigest.rs
index a6251ab..7c860d7 100644
--- a/datasketches/tests/serde_tests/tdigest.rs
+++ b/datasketches/tests/serde_tests/tdigest.rs
@@ -131,6 +131,21 @@ fn test_deserialize_from_java_snapshots() {
}
}
+#[test]
+fn test_deserialize_from_go_snapshots() {
+ let ns = [0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000];
+ for n in ns {
+ let filename = format!("tdigest_double_n{n}_go.sk");
+ let path = serialization_test_data("go_generated_files", &filename);
+ test_sketch_file(path, n, false, false);
+ }
+ for n in ns {
+ let filename = format!("tdigest_double_buf_n{n}_go.sk");
+ let path = serialization_test_data("go_generated_files", &filename);
+ test_sketch_file(path, n, true, false);
+ }
+}
+
#[test]
fn test_empty() {
let mut td = TDigestMut::new(100);
diff --git a/datasketches/tests/serde_tests/theta.rs b/datasketches/tests/serde_tests/theta.rs
index b74cd44..1f0f09b 100644
--- a/datasketches/tests/serde_tests/theta.rs
+++ b/datasketches/tests/serde_tests/theta.rs
@@ -48,7 +48,7 @@ fn test_sketch_file(path: PathBuf, expected_cardinality: usize, use_compressed_r
)
});
- // Theta snapshots from Java/C++ are not required to match byte-for-byte output
+ // Theta snapshots from other implementations are not required to match byte-for-byte output
// from this implementation. Verify our own serialization is stable instead.
let serialized_bytes2 = if use_compressed_round_trip {
sketch2.serialize_compressed()
@@ -116,6 +116,28 @@ fn test_cpp_compatibility() {
test_sketch_file(path, 0, false);
}
+#[test]
+fn test_go_compatibility() {
+ let test_cases = [0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000];
+
+ for n in test_cases {
+ let filename = format!("theta_n{n}_go.sk");
+ let path = serialization_test_data("go_generated_files", &filename);
+ test_sketch_file(path, n, false);
+ }
+
+ let compressed_test_cases = [10, 100, 1000, 10_000, 100_000, 1_000_000];
+
+ for n in compressed_test_cases {
+ let filename = format!("theta_compressed_n{n}_go.sk");
+ let path = serialization_test_data("go_generated_files", &filename);
+ test_sketch_file(path, n, true);
+ }
+
+ let path = serialization_test_data("go_generated_files", "theta_non_empty_no_entries_go.sk");
+ test_sketch_file(path, 0, false);
+}
+
#[test]
fn malformed_input_is_rejected() {
let mut sketch = ThetaSketchBuilder::default().lg_k(5).build();
diff --git a/datasketches/tests/serde_tests/tuple.rs b/datasketches/tests/serde_tests/tuple.rs
index 03b0d7c..3dadf98 100644
--- a/datasketches/tests/serde_tests/tuple.rs
+++ b/datasketches/tests/serde_tests/tuple.rs
@@ -44,7 +44,7 @@ fn test_sketch_file(path: PathBuf, expected_cardinality: usize) {
let estimate1 = sketch1.estimate();
assert_that!(estimate1, near(expected, expected * 0.03));
- // Snapshots from Java/C++ are not required to match byte-for-byte output from this
+ // Snapshots from other implementations are not required to match byte-for-byte output from this
// implementation. Verify our own serialization is stable across a round-trip instead.
let serialized_bytes = sketch1.serialize();
let sketch2 = CompactTupleSketch::<i32>::deserialize(&serialized_bytes).unwrap_or_else(|err| {
@@ -94,6 +94,17 @@ fn test_cpp_compatibility() {
}
}
+#[test]
+fn test_go_compatibility() {
+ let test_cases = [0, 1, 10, 100, 1000, 10_000, 100_000, 1_000_000];
+
+ for n in test_cases {
+ let filename = format!("tuple_int_n{n}_go.sk");
+ let path = serialization_test_data("go_generated_files", &filename);
+ test_sketch_file(path, n);
+ }
+}
+
#[test]
fn round_trip_preserves_summaries() {
let mut sketch = TupleSketchBuilder::new(DefaultUpdatePolicy::<u64>::default()).build();
diff --git a/xtask/src/main.rs b/xtask/src/main.rs
index a251ff6..50768fa 100644
--- a/xtask/src/main.rs
+++ b/xtask/src/main.rs
@@ -283,7 +283,7 @@ fn make_taplo_cmd(fix: bool) -> StdCommand {
struct CommandPrepareTestData {
#[arg(
value_name = "LANG",
- value_parser = ["java", "cpp", "c++"],
+ value_parser = ["java", "cpp", "c++", "go", "golang"],
help = "Languages to prepare (all by default)."
)]
langs: Vec<String>,
@@ -298,7 +298,7 @@ impl CommandPrepareTestData {
}
fn prepare(self) -> Result<()> {
- const REVISION: &str = "0016a517cc87e13339298550afe8e6a7e961bf46";
+ const REVISION: &str = "d363b12d293b395d90abb42677f9ea63178dbc0d";
let serde_tests =
Path::new(env!("CARGO_WORKSPACE_DIR")).join("datasketches/tests/serde_tests");
let archive_url =
@@ -372,7 +372,7 @@ impl CommandPrepareTestData {
fn languages(&self) -> Vec<&'static str> {
if self.langs.is_empty() {
- return vec!["cpp", "java"];
+ return vec!["cpp", "go", "java"];
}
let mut languages = vec![];
@@ -380,6 +380,7 @@ impl CommandPrepareTestData {
let language = match language.as_str() {
"java" => "java",
"cpp" | "c++" => "cpp",
+ "go" | "golang" => "go",
_ => unreachable!("language is validated by clap"),
};
if !languages.contains(&language) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org