You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by al...@apache.org on 2023/08/17 22:21:16 UTC
[datasketches-java] 01/01: added tests, added _java suffix
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch serde_tests
in repository https://gitbox.apache.org/repos/asf/datasketches-java.git
commit 7635243dbb9a15d779d229aa120e544e33eb1e8b
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Thu Aug 17 15:21:04 2023 -0700
added tests, added _java suffix
---
.../apache/datasketches/cpc/CpcCBinariesTest.java | 2 +-
.../frequencies/SerDeCompatibilityTest.java | 8 ++---
.../datasketches/hll/HllSketchSerDeTest.java | 6 ++--
.../quantiles/DoublesSketchSerDeTest.java | 2 +-
.../quantiles/ItemsSketchSerDeTest.java | 2 +-
.../datasketches/req/ReqSketchSerDeTest.java | 2 +-
.../sampling/VarOptSketchSerDeTest.java | 29 +++++++++++++++++-
...tchSerDeTest.java => VarOptUnionSerDeTest.java} | 35 ++++++++++++++++------
.../datasketches/theta/ThetaSketchSerDeTest.java | 4 +--
.../ArrayOfDoublesSketchSerDeTest.java | 6 ++--
10 files changed, 70 insertions(+), 26 deletions(-)
diff --git a/src/test/java/org/apache/datasketches/cpc/CpcCBinariesTest.java b/src/test/java/org/apache/datasketches/cpc/CpcCBinariesTest.java
index 1b3f0472..a7144dc0 100644
--- a/src/test/java/org/apache/datasketches/cpc/CpcCBinariesTest.java
+++ b/src/test/java/org/apache/datasketches/cpc/CpcCBinariesTest.java
@@ -292,7 +292,7 @@ public class CpcCBinariesTest {
final CpcSketch sketch = new CpcSketch(11);
for (int i = 0; i < n; i++) sketch.update(i);
assertEquals(sketch.getFlavor(), flavorArr[flavorIdx++]);
- try (final FileOutputStream file = new FileOutputStream("cpc_n" + n + ".sk")) {
+ try (final FileOutputStream file = new FileOutputStream("cpc_n" + n + "_java.sk")) {
file.write(sketch.toByteArray());
}
}
diff --git a/src/test/java/org/apache/datasketches/frequencies/SerDeCompatibilityTest.java b/src/test/java/org/apache/datasketches/frequencies/SerDeCompatibilityTest.java
index a492b4c1..cceff360 100644
--- a/src/test/java/org/apache/datasketches/frequencies/SerDeCompatibilityTest.java
+++ b/src/test/java/org/apache/datasketches/frequencies/SerDeCompatibilityTest.java
@@ -93,7 +93,7 @@ public class SerDeCompatibilityTest {
} else {
assertEquals(sketch.getMaximumError(), 0);
}
- try (final FileOutputStream file = new FileOutputStream("frequent_long_n" + n + ".sk")) {
+ try (final FileOutputStream file = new FileOutputStream("frequent_long_n" + n + "_java.sk")) {
file.write(sketch.toByteArray());
}
}
@@ -111,7 +111,7 @@ public class SerDeCompatibilityTest {
} else {
assertEquals(sketch.getMaximumError(), 0);
}
- try (final FileOutputStream file = new FileOutputStream("frequent_string_n" + n + ".sk")) {
+ try (final FileOutputStream file = new FileOutputStream("frequent_string_n" + n + "_java.sk")) {
file.write(sketch.toByteArray(new ArrayOfStringsSerDe()));
}
}
@@ -124,7 +124,7 @@ public class SerDeCompatibilityTest {
sketch.update("bbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 2);
sketch.update("ccccccccccccccccccccccccccccc", 3);
sketch.update("ddddddddddddddddddddddddddddd", 4);
- try (final FileOutputStream file = new FileOutputStream("frequent_string_ascii.sk")) {
+ try (final FileOutputStream file = new FileOutputStream("frequent_string_ascii_java.sk")) {
file.write(sketch.toByteArray(new ArrayOfStringsSerDe()));
}
}
@@ -139,7 +139,7 @@ public class SerDeCompatibilityTest {
sketch.update("уфхцч", 5);
sketch.update("шщъыь", 6);
sketch.update("эюя", 7);
- try (final FileOutputStream file = new FileOutputStream("frequent_string_utf8.sk")) {
+ try (final FileOutputStream file = new FileOutputStream("frequent_string_utf8_java.sk")) {
file.write(sketch.toByteArray(new ArrayOfStringsSerDe()));
}
}
diff --git a/src/test/java/org/apache/datasketches/hll/HllSketchSerDeTest.java b/src/test/java/org/apache/datasketches/hll/HllSketchSerDeTest.java
index 55862437..dcb17b23 100644
--- a/src/test/java/org/apache/datasketches/hll/HllSketchSerDeTest.java
+++ b/src/test/java/org/apache/datasketches/hll/HllSketchSerDeTest.java
@@ -39,13 +39,13 @@ public class HllSketchSerDeTest {
for (int i = 0; i < n; i++) hll4.update(i);
for (int i = 0; i < n; i++) hll6.update(i);
for (int i = 0; i < n; i++) hll8.update(i);
- try (final FileOutputStream file = new FileOutputStream("hll4_n" + n + ".sk")) {
+ try (final FileOutputStream file = new FileOutputStream("hll4_n" + n + "_java.sk")) {
file.write(hll4.toCompactByteArray());
}
- try (final FileOutputStream file = new FileOutputStream("hll6_n" + n + ".sk")) {
+ try (final FileOutputStream file = new FileOutputStream("hll6_n" + n + "_java.sk")) {
file.write(hll6.toCompactByteArray());
}
- try (final FileOutputStream file = new FileOutputStream("hll8_n" + n + ".sk")) {
+ try (final FileOutputStream file = new FileOutputStream("hll8_n" + n + "_java.sk")) {
file.write(hll8.toCompactByteArray());
}
}
diff --git a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchSerDeTest.java b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchSerDeTest.java
index f3746c74..90d4477d 100644
--- a/src/test/java/org/apache/datasketches/quantiles/DoublesSketchSerDeTest.java
+++ b/src/test/java/org/apache/datasketches/quantiles/DoublesSketchSerDeTest.java
@@ -31,7 +31,7 @@ public class DoublesSketchSerDeTest {
for (final int n: nArr) {
final UpdateDoublesSketch sketch = DoublesSketch.builder().build();
for (int i = 1; i <= n; i++) sketch.update(i);
- try (final FileOutputStream file = new FileOutputStream("quantiles_double_n" + n + ".sk")) {
+ try (final FileOutputStream file = new FileOutputStream("quantiles_double_n" + n + "_java.sk")) {
file.write(sketch.toByteArray(true));
}
}
diff --git a/src/test/java/org/apache/datasketches/quantiles/ItemsSketchSerDeTest.java b/src/test/java/org/apache/datasketches/quantiles/ItemsSketchSerDeTest.java
index 71d18f48..1d568090 100644
--- a/src/test/java/org/apache/datasketches/quantiles/ItemsSketchSerDeTest.java
+++ b/src/test/java/org/apache/datasketches/quantiles/ItemsSketchSerDeTest.java
@@ -46,7 +46,7 @@ public class ItemsSketchSerDeTest {
assertEquals(sketch.getMinItem(), "1");
assertEquals(sketch.getMaxItem(), Integer.toString(n));
}
- try (final FileOutputStream file = new FileOutputStream("quantiles_string_n" + n + ".sk")) {
+ try (final FileOutputStream file = new FileOutputStream("quantiles_string_n" + n + "_java.sk")) {
file.write(sketch.toByteArray(new ArrayOfStringsSerDe()));
}
}
diff --git a/src/test/java/org/apache/datasketches/req/ReqSketchSerDeTest.java b/src/test/java/org/apache/datasketches/req/ReqSketchSerDeTest.java
index c241c07e..7be874d4 100644
--- a/src/test/java/org/apache/datasketches/req/ReqSketchSerDeTest.java
+++ b/src/test/java/org/apache/datasketches/req/ReqSketchSerDeTest.java
@@ -31,7 +31,7 @@ public class ReqSketchSerDeTest {
for (int n: nArr) {
final ReqSketch sketch = ReqSketch.builder().build();
for (int i = 1; i <= n; i++) sketch.update(i);
- try (final FileOutputStream file = new FileOutputStream("req_float_n" + n + ".sk")) {
+ try (final FileOutputStream file = new FileOutputStream("req_float_n" + n + "_java.sk")) {
file.write(sketch.toByteArray());
}
}
diff --git a/src/test/java/org/apache/datasketches/sampling/VarOptSketchSerDeTest.java b/src/test/java/org/apache/datasketches/sampling/VarOptSketchSerDeTest.java
index c0b574c2..1862b601 100644
--- a/src/test/java/org/apache/datasketches/sampling/VarOptSketchSerDeTest.java
+++ b/src/test/java/org/apache/datasketches/sampling/VarOptSketchSerDeTest.java
@@ -22,6 +22,7 @@ package org.apache.datasketches.sampling;
import java.io.FileOutputStream;
import org.apache.datasketches.common.ArrayOfLongsSerDe;
+import org.apache.datasketches.common.ArrayOfStringsSerDe;
import org.testng.annotations.Test;
public class VarOptSketchSerDeTest {
@@ -32,10 +33,36 @@ public class VarOptSketchSerDeTest {
for (int n: nArr) {
final VarOptItemsSketch<Long> sketch = VarOptItemsSketch.newInstance(32);
for (int i = 1; i <= n; i++) sketch.update(Long.valueOf(i), 1.0);
- try (final FileOutputStream file = new FileOutputStream("varopt_long_n" + n + ".sk")) {
+ try (final FileOutputStream file = new FileOutputStream("varopt_sketch_long_n" + n + "_java.sk")) {
file.write(sketch.toByteArray(new ArrayOfLongsSerDe()));
}
}
}
+ @Test(groups = {"generate"})
+ public void generateBinariesForCompatibilityTestingStringExact() throws Exception {
+ final VarOptItemsSketch<String> sketch = VarOptItemsSketch.newInstance(1024);
+ for (int i = 1; i <= 200; ++i) {
+ sketch.update(Integer.toString(i), 1000.0 / i);
+ }
+ try (final FileOutputStream file = new FileOutputStream("varopt_sketch_string_exact_java.sk")) {
+ file.write(sketch.toByteArray(new ArrayOfStringsSerDe()));
+ }
+ }
+
+ @Test(groups = {"generate"})
+ public void generateBinariesForCompatibilityTestingLongSampling() throws Exception {
+ final VarOptItemsSketch<Long> sketch = VarOptItemsSketch.newInstance(1024);
+ for (long i = 0; i < 2000; ++i) {
+ sketch.update(i, 1.0);
+ }
+ // heavy items have negative weights to allow a simple predicate to filter
+ sketch.update(-1L, 100000.0);
+ sketch.update(-2L, 110000.0);
+ sketch.update(-3L, 120000.0);
+ try (final FileOutputStream file = new FileOutputStream("varopt_sketch_long_sampling_java.sk")) {
+ file.write(sketch.toByteArray(new ArrayOfLongsSerDe()));
+ }
+ }
+
}
diff --git a/src/test/java/org/apache/datasketches/sampling/VarOptSketchSerDeTest.java b/src/test/java/org/apache/datasketches/sampling/VarOptUnionSerDeTest.java
similarity index 51%
copy from src/test/java/org/apache/datasketches/sampling/VarOptSketchSerDeTest.java
copy to src/test/java/org/apache/datasketches/sampling/VarOptUnionSerDeTest.java
index c0b574c2..44347ce0 100644
--- a/src/test/java/org/apache/datasketches/sampling/VarOptSketchSerDeTest.java
+++ b/src/test/java/org/apache/datasketches/sampling/VarOptUnionSerDeTest.java
@@ -21,20 +21,37 @@ package org.apache.datasketches.sampling;
import java.io.FileOutputStream;
-import org.apache.datasketches.common.ArrayOfLongsSerDe;
+import org.apache.datasketches.common.ArrayOfDoublesSerDe;
import org.testng.annotations.Test;
-public class VarOptSketchSerDeTest {
+public class VarOptUnionSerDeTest {
@Test(groups = {"generate"})
public void generateBinariesForCompatibilityTesting() throws Exception {
- final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
- for (int n: nArr) {
- final VarOptItemsSketch<Long> sketch = VarOptItemsSketch.newInstance(32);
- for (int i = 1; i <= n; i++) sketch.update(Long.valueOf(i), 1.0);
- try (final FileOutputStream file = new FileOutputStream("varopt_long_n" + n + ".sk")) {
- file.write(sketch.toByteArray(new ArrayOfLongsSerDe()));
- }
+ final int kSmall = 16;
+ final int n1 = 32;
+ final int n2 = 64;
+ final int kMax = 128;
+
+ // small k sketch, but sampling
+ VarOptItemsSketch<Double> sketch = VarOptItemsSketch.newInstance(kSmall);
+ for (int i = 0; i < n1; ++i) {
+ sketch.update(1.0 * i, 1.0);
+ }
+ sketch.update(-1.0, n1 * n1); // heavy items have negative weights to allow a simple predicate to filter
+
+
+ final VarOptItemsUnion<Double> union = VarOptItemsUnion.newInstance(kMax);
+ union.update(sketch);
+
+ // another one, but different n to get a different per-item weight
+ sketch = VarOptItemsSketch.newInstance(kSmall);
+ for (int i = 0; i < n2; ++i) {
+ sketch.update(1.0 * i, 1.0);
+ }
+ union.update(sketch);
+ try (final FileOutputStream file = new FileOutputStream("varopt_union_double_sampling_java.sk")) {
+ file.write(union.toByteArray(new ArrayOfDoublesSerDe()));
}
}
diff --git a/src/test/java/org/apache/datasketches/theta/ThetaSketchSerDeTest.java b/src/test/java/org/apache/datasketches/theta/ThetaSketchSerDeTest.java
index 6642370a..eb952ceb 100644
--- a/src/test/java/org/apache/datasketches/theta/ThetaSketchSerDeTest.java
+++ b/src/test/java/org/apache/datasketches/theta/ThetaSketchSerDeTest.java
@@ -34,7 +34,7 @@ public class ThetaSketchSerDeTest {
for (int n: nArr) {
final UpdateSketch sketch = UpdateSketch.builder().build();
for (int i = 0; i < n; i++) sketch.update(i);
- try (final FileOutputStream file = new FileOutputStream("theta_n" + n + ".sk")) {
+ try (final FileOutputStream file = new FileOutputStream("theta_n" + n + "_java.sk")) {
file.write(sketch.compact().toByteArray());
}
}
@@ -46,7 +46,7 @@ public class ThetaSketchSerDeTest {
sketch.update(1);
assertFalse(sketch.isEmpty());
assertEquals(sketch.getRetainedEntries(), 0);
- try (final FileOutputStream file = new FileOutputStream("theta_non_empty_no_entries.sk")) {
+ try (final FileOutputStream file = new FileOutputStream("theta_non_empty_no_entries_java.sk")) {
file.write(sketch.compact().toByteArray());
}
}
diff --git a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSketchSerDeTest.java b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSketchSerDeTest.java
index 0920473d..fce13f5a 100644
--- a/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSketchSerDeTest.java
+++ b/src/test/java/org/apache/datasketches/tuple/arrayofdoubles/ArrayOfDoublesSketchSerDeTest.java
@@ -34,7 +34,7 @@ public class ArrayOfDoublesSketchSerDeTest {
for (int n: nArr) {
final ArrayOfDoublesUpdatableSketch sketch = new ArrayOfDoublesUpdatableSketchBuilder().build();
for (int i = 0; i < n; i++) sketch.update(i, new double[] {i});
- try (final FileOutputStream file = new FileOutputStream("aod_1_n" + n + ".sk")) {
+ try (final FileOutputStream file = new FileOutputStream("aod_1_n" + n + "_java.sk")) {
file.write(sketch.compact().toByteArray());
}
}
@@ -46,7 +46,7 @@ public class ArrayOfDoublesSketchSerDeTest {
for (int n: nArr) {
final ArrayOfDoublesUpdatableSketch sketch = new ArrayOfDoublesUpdatableSketchBuilder().setNumberOfValues(3).build();
for (int i = 0; i < n; i++) sketch.update(i, new double[] {i, i, i});
- try (final FileOutputStream file = new FileOutputStream("aod_3_n" + n + ".sk")) {
+ try (final FileOutputStream file = new FileOutputStream("aod_3_n" + n + "_java.sk")) {
file.write(sketch.compact().toByteArray());
}
}
@@ -58,7 +58,7 @@ public class ArrayOfDoublesSketchSerDeTest {
sketch.update(1, new double[] {1});
assertFalse(sketch.isEmpty());
assertEquals(sketch.getRetainedEntries(), 0);
- try (final FileOutputStream file = new FileOutputStream("aod_1_non_empty_no_entries.sk")) {
+ try (final FileOutputStream file = new FileOutputStream("aod_1_non_empty_no_entries_java.sk")) {
file.write(sketch.compact().toByteArray());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org