You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by al...@apache.org on 2023/08/07 19:32:30 UTC
[datasketches-java] branch cpp_serde_compat_testing created (now f7e5e06f)
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a change to branch cpp_serde_compat_testing
in repository https://gitbox.apache.org/repos/asf/datasketches-java.git
at f7e5e06f generate varopt sketches for compatibility test
This branch includes the following new commits:
new afc54a61 serde ID is not used anymore
new 3147807d generate frequent items sketches for compatibility test
new f7e5e06f generate varopt sketches for compatibility test
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[datasketches-java] 03/03: generate varopt sketches for compatibility test
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch cpp_serde_compat_testing
in repository https://gitbox.apache.org/repos/asf/datasketches-java.git
commit f7e5e06fb88b084dbf7c692fe827112ee4824b96
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Mon Aug 7 12:32:13 2023 -0700
generate varopt sketches for compatibility test
---
.../sampling/VarOptSketchSerDeTest.java | 41 ++++++++++++++++++++++
1 file changed, 41 insertions(+)
diff --git a/src/test/java/org/apache/datasketches/sampling/VarOptSketchSerDeTest.java b/src/test/java/org/apache/datasketches/sampling/VarOptSketchSerDeTest.java
new file mode 100644
index 00000000..c0b574c2
--- /dev/null
+++ b/src/test/java/org/apache/datasketches/sampling/VarOptSketchSerDeTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.sampling;
+
+import java.io.FileOutputStream;
+
+import org.apache.datasketches.common.ArrayOfLongsSerDe;
+import org.testng.annotations.Test;
+
+public class VarOptSketchSerDeTest {
+
+ @Test(groups = {"generate"})
+ public void generateBinariesForCompatibilityTesting() throws Exception {
+ final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+ for (int n: nArr) {
+ final VarOptItemsSketch<Long> sketch = VarOptItemsSketch.newInstance(32);
+ for (int i = 1; i <= n; i++) sketch.update(Long.valueOf(i), 1.0);
+ try (final FileOutputStream file = new FileOutputStream("varopt_long_n" + n + ".sk")) {
+ file.write(sketch.toByteArray(new ArrayOfLongsSerDe()));
+ }
+ }
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[datasketches-java] 01/03: serde ID is not used anymore
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch cpp_serde_compat_testing
in repository https://gitbox.apache.org/repos/asf/datasketches-java.git
commit afc54a6175c687fbfd0363c4d79eb030cc1aec58
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Mon Aug 7 12:26:37 2023 -0700
serde ID is not used anymore
---
.../apache/datasketches/frequencies/PreambleUtil.java | 16 +++++-----------
1 file changed, 5 insertions(+), 11 deletions(-)
diff --git a/src/main/java/org/apache/datasketches/frequencies/PreambleUtil.java b/src/main/java/org/apache/datasketches/frequencies/PreambleUtil.java
index ac2137ca..785c211f 100644
--- a/src/main/java/org/apache/datasketches/frequencies/PreambleUtil.java
+++ b/src/main/java/org/apache/datasketches/frequencies/PreambleUtil.java
@@ -52,7 +52,7 @@ import org.apache.datasketches.memory.Memory;
* * Long || Start Byte Adr:
* Adr:
* || 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 |
- * 0 ||-------SerDeId-----|-Flags--|-LgCur--| LgMax | FamID | SerVer | PreambleLongs |
+ * 0 ||------ unused -----|-Flags--|-LgCur--| LgMax | FamID | SerVer | PreambleLongs |
* || 15 | 14 | 13 | 12 | 11 | 10 | 9 | 8 |
* 1 ||------------(unused)-----------------|--------ActiveItems------------------------|
* || 23 | 22 | 21 | 20 | 19 | 18 | 17 | 16 |
@@ -83,7 +83,9 @@ final class PreambleUtil {
static final int OFFSET_LONG = 24; // to 31 : pre3
// flag bit masks
- static final int EMPTY_FLAG_MASK = 4;
+ // due to a mistake different bits were used in C++ and Java to indicate empty sketch
+ // therefore both are set and checked for compatibility with historical binary format
+ static final int EMPTY_FLAG_MASK = 5;
// Specific values for this implementation
static final int SER_VER = 1;
@@ -105,7 +107,6 @@ final class PreambleUtil {
final int lgMaxMapSize = extractLgMaxMapSize(pre0); //byte 3
final int lgCurMapSize = extractLgCurMapSize(pre0); //byte 4
final int flags = extractFlags(pre0); //byte 5
- final int type = extractSerDeId(pre0); //byte 6
final String flagsStr = zeroPad(Integer.toBinaryString(flags), 8) + ", " + (flags);
final boolean empty = (flags & EMPTY_FLAG_MASK) > 0;
@@ -138,8 +139,7 @@ final class PreambleUtil {
.append("Byte 3: MaxMapSize : ").append(maxMapSize).append(LS)
.append("Byte 4: CurMapSize : ").append(curMapSize).append(LS)
.append("Byte 5: Flags Field : ").append(flagsStr).append(LS)
- .append(" EMPTY : ").append(empty).append(LS)
- .append("Byte 6: Freq Sketch Type : ").append(type).append(LS);
+ .append(" EMPTY : ").append(empty).append(LS);
if (preLongs == 1) {
sb.append(" --ABSENT, ASSUMED:").append(LS);
@@ -193,12 +193,6 @@ final class PreambleUtil {
return (int) ((pre0 >>> shift) & mask);
}
- static short extractSerDeId(final long pre0) { //Byte 6,7
- final int shift = SER_DE_ID_SHORT << 3;
- final long mask = 0XFFFFL;
- return (short) ((pre0 >>> shift) & mask);
- }
-
static int extractActiveItems(final long pre1) { //Bytes 8 to 11
final long mask = 0XFFFFFFFFL;
return (int) (pre1 & mask) ;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[datasketches-java] 02/03: generate frequent items sketches for compatibility test
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch cpp_serde_compat_testing
in repository https://gitbox.apache.org/repos/asf/datasketches-java.git
commit 3147807d53627c2b7c214d34bc3a02984810f439
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Mon Aug 7 12:31:40 2023 -0700
generate frequent items sketches for compatibility test
---
.../frequencies/SerDeCompatibilityTest.java | 55 +++++++++++++++++++---
1 file changed, 49 insertions(+), 6 deletions(-)
diff --git a/src/test/java/org/apache/datasketches/frequencies/SerDeCompatibilityTest.java b/src/test/java/org/apache/datasketches/frequencies/SerDeCompatibilityTest.java
index a32f95c4..025cfb98 100644
--- a/src/test/java/org/apache/datasketches/frequencies/SerDeCompatibilityTest.java
+++ b/src/test/java/org/apache/datasketches/frequencies/SerDeCompatibilityTest.java
@@ -19,8 +19,15 @@
package org.apache.datasketches.frequencies;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
+import java.io.FileOutputStream;
+
import org.apache.datasketches.common.ArrayOfItemsSerDe;
import org.apache.datasketches.common.ArrayOfLongsSerDe;
+import org.apache.datasketches.common.ArrayOfStringsSerDe;
import org.apache.datasketches.memory.WritableMemory;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -31,14 +38,14 @@ public class SerDeCompatibilityTest {
@Test
public void itemsToLongs() {
- ItemsSketch<Long> sketch1 = new ItemsSketch<>(8);
+ final ItemsSketch<Long> sketch1 = new ItemsSketch<>(8);
sketch1.update(1L);
sketch1.update(2L);
sketch1.update(3L);
sketch1.update(4L);
- byte[] bytes = sketch1.toByteArray(serDe);
- LongsSketch sketch2 = LongsSketch.getInstance(WritableMemory.writableWrap(bytes));
+ final byte[] bytes = sketch1.toByteArray(serDe);
+ final LongsSketch sketch2 = LongsSketch.getInstance(WritableMemory.writableWrap(bytes));
sketch2.update(2L);
sketch2.update(3L);
sketch2.update(2L);
@@ -54,14 +61,14 @@ public class SerDeCompatibilityTest {
@Test
public void longsToItems() {
- LongsSketch sketch1 = new LongsSketch(8);
+ final LongsSketch sketch1 = new LongsSketch(8);
sketch1.update(1L);
sketch1.update(2L);
sketch1.update(3L);
sketch1.update(4L);
- byte[] bytes = sketch1.toByteArray();
- ItemsSketch<Long> sketch2 = ItemsSketch.getInstance(WritableMemory.writableWrap(bytes), serDe);
+ final byte[] bytes = sketch1.toByteArray();
+ final ItemsSketch<Long> sketch2 = ItemsSketch.getInstance(WritableMemory.writableWrap(bytes), serDe);
sketch2.update(2L);
sketch2.update(3L);
sketch2.update(2L);
@@ -75,4 +82,40 @@ public class SerDeCompatibilityTest {
Assert.assertEquals(sketch2.getEstimate(4L), 1);
}
+ @Test(groups = {"generate"})
+ public void generateBinariesForCompatibilityTestingLongsSketch() throws Exception {
+ final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+ for (int n: nArr) {
+ LongsSketch sketch = new LongsSketch(64);
+ for (int i = 1; i <= n; i++) sketch.update(i);
+ assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty());
+ if (n > 10) {
+ assertTrue(sketch.getMaximumError() > 0);
+ } else {
+ assertEquals(sketch.getMaximumError(), 0);
+ }
+ try (final FileOutputStream file = new FileOutputStream("frequent_long_n" + n + ".sk")) {
+ file.write(sketch.toByteArray());
+ }
+ }
+ }
+
+ @Test(groups = {"generate"})
+ public void generateBinariesForCompatibilityTestingStringsSketch() throws Exception {
+ final int[] nArr = {0, 1, 10, 100, 1000, 10000, 100000, 1000000};
+ for (int n: nArr) {
+ ItemsSketch<String> sketch = new ItemsSketch<>(64);
+ for (int i = 1; i <= n; i++) sketch.update(Integer.toString(i));
+ assertTrue(n == 0 ? sketch.isEmpty() : !sketch.isEmpty());
+ if (n > 10) {
+ assertTrue(sketch.getMaximumError() > 0);
+ } else {
+ assertEquals(sketch.getMaximumError(), 0);
+ }
+ try (final FileOutputStream file = new FileOutputStream("frequent_string_n" + n + ".sk")) {
+ file.write(sketch.toByteArray(new ArrayOfStringsSerDe()));
+ }
+ }
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org