You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by le...@apache.org on 2023/08/15 23:52:18 UTC
[datasketches-java] branch kll_cross_lang_IT created (now b446b1f1)
This is an automated email from the ASF dual-hosted git repository.
leerho pushed a change to branch kll_cross_lang_IT
in repository https://gitbox.apache.org/repos/asf/datasketches-java.git
at b446b1f1 Interim 1 Cross lang testing
This branch includes the following new commits:
new b446b1f1 Interim 1 Cross lang testing
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[datasketches-java] 01/01: Interim 1 Cross lang testing
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leerho pushed a commit to branch kll_cross_lang_IT
in repository https://gitbox.apache.org/repos/asf/datasketches-java.git
commit b446b1f1817990647207038613d4f7e828409d02
Author: Lee Rhodes <le...@users.noreply.github.com>
AuthorDate: Tue Aug 15 16:52:05 2023 -0700
Interim 1 Cross lang testing
Fixed KllItems generation starting with 1
---
.../datasketches/kll/KllCrossLanguageIT.java | 125 +++++++++++++++++++++
.../kll/KllDoublesSketchSerDeTest.java | 41 -------
.../datasketches/kll/KllFloatsSketchSerDeTest.java | 41 -------
.../datasketches/kll/KllItemsSketchSerDeTest.java | 18 +--
4 files changed, 126 insertions(+), 99 deletions(-)
diff --git a/src/test/java/org/apache/datasketches/kll/KllCrossLanguageIT.java b/src/test/java/org/apache/datasketches/kll/KllCrossLanguageIT.java
new file mode 100644
index 00000000..6f6e3a57
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/kll/KllCrossLanguageIT.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.kll;
+
+import static org.apache.datasketches.common.Util.getResourceBytes;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.util.Comparator;
+
+import org.apache.datasketches.common.ArrayOfStringsSerDe;
+import org.apache.datasketches.common.Util;
+import org.apache.datasketches.memory.MapHandle;
+import org.apache.datasketches.memory.Memory;
+import org.testng.annotations.Test;
+
+/**
+ * Methods for cross language integration testing
+ */
+public class KllCrossLanguageIT {
+ private ArrayOfStringsSerDe serDe = new ArrayOfStringsSerDe();
+
+ @Test(groups = {"generate"})
+ public void generateKllDoublesSketchBinaries() throws Exception {
+ final int[] nArr = {0, 1, 10, 100, 1_000, 10_000, 100_000, 1_000_000};
+ for (int n: nArr) {
+ final KllDoublesSketch sk = KllDoublesSketch.newHeapInstance();
+ for (int i = 1; i <= n; i++) sk.update(i);
+ try (final FileOutputStream file = new FileOutputStream("kll_double_n" + n + "_java.sk")) {
+ file.write(sk.toByteArray());
+ }
+ }
+ }
+
+ @Test(groups = {"generate"})
+ public void generateKllFloatsSketchBinaries() throws Exception {
+ final int[] nArr = {0, 1, 10, 100, 1_000, 10_000, 100_000, 1_000_000};
+ for (int n: nArr) {
+ final KllFloatsSketch sk = KllFloatsSketch.newHeapInstance();
+ for (int i = 1; i <= n; i++) sk.update(i);
+ try (final FileOutputStream file = new FileOutputStream("kll_float_n" + n + "_java.sk")) {
+ file.write(sk.toByteArray());
+ }
+ }
+ }
+
+ @Test(groups = {"generate"})
+ public void generateKllItemsSketchBinaries() throws Exception {
+ final int[] nArr = {0, 1, 10, 100, 1_000, 10_000, 100_000, 1_000_000};
+ for (int n: nArr) {
+ final int digits = Util.numDigits(n);
+ final KllItemsSketch<String> sketch = KllItemsSketch.newHeapInstance(Comparator.naturalOrder(), serDe);
+ for (int i = 1; i <= n; i++) sketch.update(Util.intToFixedLengthString(i, digits));
+ try (final FileOutputStream file = new FileOutputStream("kll_string_n" + n + "_java.sk")) {
+ file.write(sketch.toByteArray());
+ }
+ }
+ }
+
+ @Test(groups = {"cross_language_check"})
+ public void checkCppKllDoublesSketchEstimationMode() throws Exception {
+ final File file = Util.getResourceFile("kll_double_estimation_cpp.sk");
+ try (MapHandle mh = Memory.map(file)) {
+ final KllDoublesSketch sk = KllDoublesSketch.heapify(mh.get());
+ assertEquals(sk.getMinItem(), 0);
+ assertEquals(sk.getMaxItem(), 999);
+ assertEquals(sk.getN(), 1000);
+ }
+ }
+
+ @Test(groups = {"cross_language_check"})
+ public void checkCppKllDoublesSketchOneItemVersion1() throws Exception {
+ final byte[] bytes = getResourceBytes("kll_sketch_double_one_item_v1.sk");
+ final KllDoublesSketch sk = KllDoublesSketch.heapify(Memory.wrap(bytes));
+ assertFalse(sk.isEmpty());
+ assertFalse(sk.isEstimationMode());
+ assertEquals(sk.getN(), 1);
+ assertEquals(sk.getNumRetained(), 1);
+ assertEquals(sk.getMinItem(), 1.0);
+ assertEquals(sk.getMaxItem(), 1.0);
+ }
+
+ @Test(groups = {"cross_language_check"})
+ public void checkCppKllFloatsSketchEstimationMode() throws Exception {
+ final File file = Util.getResourceFile("kll_float_estimation_cpp.sk");
+ try (MapHandle mh = Memory.map(file)) {
+ final KllFloatsSketch sk = KllFloatsSketch.heapify(mh.get());
+ assertEquals(sk.getMinItem(), 0);
+ assertEquals(sk.getMaxItem(), 999);
+ assertEquals(sk.getN(), 1000);
+ }
+ }
+
+ @Test(groups = {"cross_language_check"})
+ public void checkCppKllFloatsSketchOneItemVersion1() throws Exception {
+ final byte[] bytes = getResourceBytes("kll_sketch_float_one_item_v1.sk");
+ final KllFloatsSketch sk = KllFloatsSketch.heapify(Memory.wrap(bytes));
+ assertFalse(sk.isEmpty());
+ assertFalse(sk.isEstimationMode());
+ assertEquals(sk.getN(), 1);
+ assertEquals(sk.getNumRetained(), 1);
+ assertEquals(sk.getMinItem(), 1.0F);
+ assertEquals(sk.getMaxItem(), 1.0F);
+ }
+
+}
diff --git a/src/test/java/org/apache/datasketches/kll/KllDoublesSketchSerDeTest.java b/src/test/java/org/apache/datasketches/kll/KllDoublesSketchSerDeTest.java
index 71780037..31a58b4a 100644
--- a/src/test/java/org/apache/datasketches/kll/KllDoublesSketchSerDeTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllDoublesSketchSerDeTest.java
@@ -19,18 +19,12 @@
package org.apache.datasketches.kll;
-import static org.apache.datasketches.common.Util.getResourceBytes;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
-import java.io.File;
-import java.io.FileOutputStream;
-
import org.apache.datasketches.common.SketchesArgumentException;
-import org.apache.datasketches.common.Util;
-import org.apache.datasketches.memory.MapHandle;
import org.apache.datasketches.memory.Memory;
import org.testng.annotations.Test;
@@ -127,39 +121,4 @@ public class KllDoublesSketchSerDeTest {
assertEquals(bytes, bytes2);
}
- @Test
- public void compatibilityWithCppEstimationMode() throws Exception {
- final File file = Util.getResourceFile("kll_double_estimation_cpp.sk");
- try (MapHandle mh = Memory.map(file)) {
- final KllDoublesSketch sk = KllDoublesSketch.heapify(mh.get());
- assertEquals(sk.getMinItem(), 0);
- assertEquals(sk.getMaxItem(), 999);
- assertEquals(sk.getN(), 1000);
- }
- }
-
- @Test
- public void deserializeOneValueVersion1() throws Exception {
- final byte[] bytes = getResourceBytes("kll_sketch_double_one_item_v1.sk");
- final KllDoublesSketch sk = KllDoublesSketch.heapify(Memory.wrap(bytes));
- assertFalse(sk.isEmpty());
- assertFalse(sk.isEstimationMode());
- assertEquals(sk.getN(), 1);
- assertEquals(sk.getNumRetained(), 1);
- assertEquals(sk.getMinItem(), 1.0);
- assertEquals(sk.getMaxItem(), 1.0);
- }
-
- @Test(groups = {"generate"})
- public void generateBinariesForCompatibilityTesting() throws Exception {
- final int[] nArr = {0, 1, 10, 100, 1_000, 10_000, 100_000, 1_000_000};
- for (int n: nArr) {
- final KllDoublesSketch sk = KllDoublesSketch.newHeapInstance();
- for (int i = 1; i <= n; i++) sk.update(i);
- try (final FileOutputStream file = new FileOutputStream("kll_double_n" + n + ".sk")) {
- file.write(sk.toByteArray());
- }
- }
- }
-
}
diff --git a/src/test/java/org/apache/datasketches/kll/KllFloatsSketchSerDeTest.java b/src/test/java/org/apache/datasketches/kll/KllFloatsSketchSerDeTest.java
index 183a063c..3bbb44b1 100644
--- a/src/test/java/org/apache/datasketches/kll/KllFloatsSketchSerDeTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllFloatsSketchSerDeTest.java
@@ -19,18 +19,12 @@
package org.apache.datasketches.kll;
-import static org.apache.datasketches.common.Util.getResourceBytes;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
-import java.io.File;
-import java.io.FileOutputStream;
-
import org.apache.datasketches.common.SketchesArgumentException;
-import org.apache.datasketches.common.Util;
-import org.apache.datasketches.memory.MapHandle;
import org.apache.datasketches.memory.Memory;
import org.testng.annotations.Test;
@@ -127,39 +121,4 @@ public class KllFloatsSketchSerDeTest {
assertEquals(bytes, bytes2);
}
- @Test
- public void compatibilityWithCppEstimationMode() throws Exception {
- final File file = Util.getResourceFile("kll_float_estimation_cpp.sk");
- try (MapHandle mh = Memory.map(file)) {
- final KllFloatsSketch sk = KllFloatsSketch.heapify(mh.get());
- assertEquals(sk.getMinItem(), 0);
- assertEquals(sk.getMaxItem(), 999);
- assertEquals(sk.getN(), 1000);
- }
- }
-
- @Test
- public void deserializeOneValueVersion1() throws Exception {
- final byte[] bytes = getResourceBytes("kll_sketch_float_one_item_v1.sk");
- final KllFloatsSketch sk = KllFloatsSketch.heapify(Memory.wrap(bytes));
- assertFalse(sk.isEmpty());
- assertFalse(sk.isEstimationMode());
- assertEquals(sk.getN(), 1);
- assertEquals(sk.getNumRetained(), 1);
- assertEquals(sk.getMinItem(), 1.0F);
- assertEquals(sk.getMaxItem(), 1.0F);
- }
-
- @Test(groups = {"generate"})
- public void generateBinariesForCompatibilityTesting() throws Exception {
- final int[] nArr = {0, 1, 10, 100, 1_000, 10_000, 100_000, 1_000_000};
- for (int n: nArr) {
- final KllFloatsSketch sk = KllFloatsSketch.newHeapInstance();
- for (int i = 1; i <= n; i++) sk.update(i);
- try (final FileOutputStream file = new FileOutputStream("kll_float_n" + n + ".sk")) {
- file.write(sk.toByteArray());
- }
- }
- }
-
}
diff --git a/src/test/java/org/apache/datasketches/kll/KllItemsSketchSerDeTest.java b/src/test/java/org/apache/datasketches/kll/KllItemsSketchSerDeTest.java
index 35ffb591..951811cc 100644
--- a/src/test/java/org/apache/datasketches/kll/KllItemsSketchSerDeTest.java
+++ b/src/test/java/org/apache/datasketches/kll/KllItemsSketchSerDeTest.java
@@ -24,7 +24,6 @@ import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
-import java.io.FileOutputStream;
import java.util.Comparator;
import org.apache.datasketches.common.ArrayOfStringsSerDe;
@@ -34,7 +33,7 @@ import org.apache.datasketches.memory.Memory;
import org.testng.annotations.Test;
public class KllItemsSketchSerDeTest {
- public ArrayOfStringsSerDe serDe = new ArrayOfStringsSerDe();
+ private ArrayOfStringsSerDe serDe = new ArrayOfStringsSerDe();
@Test
public void serializeDeserializeEmpty() {
@@ -127,19 +126,4 @@ public class KllItemsSketchSerDeTest {
assertEquals(bytes, bytes2);
}
- //no cross language tests yet
-
- //@Test(groups = {"generate"})
- public void generateBinariesForCompatibilityTesting() throws Exception {
- final int[] nArr = {0, 1, 10, 100, 1_000, 10_000, 100_000, 1_000_000};
- for (int n: nArr) {
- final int digits = Util.numDigits(n);
- final KllItemsSketch<String> sketch = KllItemsSketch.newHeapInstance(Comparator.naturalOrder(), serDe);
- for (int i = 0; i < n; i++) sketch.update(Util.intToFixedLengthString(i, digits));
- try (final FileOutputStream file = new FileOutputStream("kll_string_n" + n + ".sk")) {
- file.write(sketch.toByteArray());
- }
- }
- }
-
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org