You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by jm...@apache.org on 2024/01/26 00:27:42 UTC
(datasketches-java) branch ebpps updated: test deserialize of corrupted images
This is an automated email from the ASF dual-hosted git repository.
jmalkin pushed a commit to branch ebpps
in repository https://gitbox.apache.org/repos/asf/datasketches-java.git
The following commit(s) were added to refs/heads/ebpps by this push:
new 30fab5a5 test deserialize of corrupted images
30fab5a5 is described below
commit 30fab5a5c9a0ed120928055cf4a899ed11b68a71
Author: jmalkin <78...@users.noreply.github.com>
AuthorDate: Thu Jan 25 16:27:34 2024 -0800
test deserialize of corrupted images
---
.../datasketches/sampling/EbppsItemsSample.java | 3 -
.../datasketches/sampling/EbppsItemsSketch.java | 27 +++---
.../datasketches/sampling/EbppsSketchTest.java | 97 ++++++++++++++++++++++
3 files changed, 112 insertions(+), 15 deletions(-)
diff --git a/src/main/java/org/apache/datasketches/sampling/EbppsItemsSample.java b/src/main/java/org/apache/datasketches/sampling/EbppsItemsSample.java
index 0f1a6cf4..99e672f5 100644
--- a/src/main/java/org/apache/datasketches/sampling/EbppsItemsSample.java
+++ b/src/main/java/org/apache/datasketches/sampling/EbppsItemsSample.java
@@ -26,9 +26,6 @@ import java.util.ArrayList;
import java.util.Random;
import java.util.concurrent.ThreadLocalRandom;
-import org.apache.datasketches.common.ArrayOfItemsSerDe;
-import org.apache.datasketches.memory.Memory;
-
// this is a supporting class used to hold the raw data sample
class EbppsItemsSample<T> {
diff --git a/src/main/java/org/apache/datasketches/sampling/EbppsItemsSketch.java b/src/main/java/org/apache/datasketches/sampling/EbppsItemsSketch.java
index bcb5403d..e0d4f1f7 100644
--- a/src/main/java/org/apache/datasketches/sampling/EbppsItemsSketch.java
+++ b/src/main/java/org/apache/datasketches/sampling/EbppsItemsSketch.java
@@ -144,24 +144,24 @@ public class EbppsItemsSketch<T> {
}
final double cumWt = PreambleUtil.extractEbppsCumulativeWeight(srcMem);
- if (cumWt < 0) {
- throw new SketchesArgumentException("Possible Corruption: cumWt cannot be negative: " + n);
+ if (cumWt < 0 || Double.isNaN(cumWt) || Double.isInfinite(cumWt)) {
+ throw new SketchesArgumentException("Possible Corruption: cumWt must be nonnegative and finite: " + n);
}
final double maxWt = PreambleUtil.extractEbppsMaxWeight(srcMem);
- if (maxWt < 0) {
- throw new SketchesArgumentException("Possible Corruption: maxWt cannot be negative: " + n);
+ if (maxWt < 0 || Double.isNaN(maxWt) || Double.isInfinite(maxWt)) {
+ throw new SketchesArgumentException("Possible Corruption: maxWt must be nonnegative and finite: " + n);
}
final double rho = PreambleUtil.extractEbppsRho(srcMem);
- if (rho < 0) {
- throw new SketchesArgumentException("Possible Corruption: rho cannot be negative: " + n);
+ if (rho < 0 || Double.isNaN(rho) || Double.isInfinite(rho)) {
+ throw new SketchesArgumentException("Possible Corruption: rho must be nonnegative and finite: " + n);
}
// extract C (part of sample_, not the preamble)
final double c = srcMem.getDouble(EBPPS_C_DOUBLE);
- if (c < 0) {
- throw new SketchesArgumentException("Possible Corruption: c cannot be negative: " + n);
+ if (c < 0 || Double.isNaN(c) || Double.isInfinite(c)) {
+ throw new SketchesArgumentException("Possible Corruption: c must be nonnegative and finite: " + n);
}
// extract items
@@ -398,7 +398,9 @@ public class EbppsItemsSketch<T> {
public int getSerializedSizeBytes(final ArrayOfItemsSerDe<? super T> serDe) {
if (isEmpty())
return Family.EBPPS.getMinPreLongs() << 3;
- else
+ else if (sample_.getC() < 1.0)
+ return getSerializedSizeBytes(serDe, sample_.getPartialItem().getClass());
+ else
return getSerializedSizeBytes(serDe, sample_.getSample().get(0).getClass());
}
@@ -427,12 +429,13 @@ public class EbppsItemsSketch<T> {
* @return a byte array representation of this sketch
*/
public byte[] toByteArray(final ArrayOfItemsSerDe<? super T> serDe) {
- if (n_ == 0) {
+ if (n_ == 0)
// null class is ok since empty -- no need to call serDe
return toByteArray(serDe, null);
- } else {
+ else if (sample_.getC() < 1.0)
+ return toByteArray(serDe, sample_.getPartialItem().getClass());
+ else
return toByteArray(serDe, sample_.getSample().get(0).getClass());
- }
}
/*
diff --git a/src/test/java/org/apache/datasketches/sampling/EbppsSketchTest.java b/src/test/java/org/apache/datasketches/sampling/EbppsSketchTest.java
index 48b3c9a1..4f725801 100644
--- a/src/test/java/org/apache/datasketches/sampling/EbppsSketchTest.java
+++ b/src/test/java/org/apache/datasketches/sampling/EbppsSketchTest.java
@@ -25,6 +25,7 @@ import static org.testng.Assert.assertTrue;
import java.util.ArrayList;
+import org.apache.datasketches.common.ArrayOfLongsSerDe;
import org.apache.datasketches.common.ArrayOfStringsSerDe;
import org.apache.datasketches.common.SketchesArgumentException;
import org.apache.datasketches.memory.Memory;
@@ -208,4 +209,100 @@ public class EbppsSketchTest {
sk_heapify = EbppsItemsSketch.heapify(mem, new ArrayOfStringsSerDe());
checkIfEqual(sk, sk_heapify);
}
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void deserializeZeroK() {
+ EbppsItemsSketch<String> sk = new EbppsItemsSketch<>(5);
+ final byte[] bytes = sk.toByteArray(new ArrayOfStringsSerDe());
+ final WritableMemory mem = WritableMemory.writableWrap(bytes);
+ PreambleUtil.insertK(mem, 0);
+ EbppsItemsSketch.heapify(mem, new ArrayOfStringsSerDe());
+ }
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void deserializeTooLargeK() {
+ EbppsItemsSketch<String> sk = new EbppsItemsSketch<>(5);
+ final byte[] bytes = sk.toByteArray(new ArrayOfStringsSerDe());
+ final WritableMemory mem = WritableMemory.writableWrap(bytes);
+ PreambleUtil.insertK(mem, Integer.MAX_VALUE);
+ EbppsItemsSketch.heapify(mem, new ArrayOfStringsSerDe());
+ }
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void deserializeBadSerVer() {
+ EbppsItemsSketch<String> sk = new EbppsItemsSketch<>(5);
+ final byte[] bytes = sk.toByteArray(new ArrayOfStringsSerDe());
+ final WritableMemory mem = WritableMemory.writableWrap(bytes);
+ PreambleUtil.insertSerVer(mem, -1);
+ EbppsItemsSketch.heapify(mem, new ArrayOfStringsSerDe());
+ }
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void deserializeBadFamily() {
+ EbppsItemsSketch<String> sk = new EbppsItemsSketch<>(5);
+ final byte[] bytes = sk.toByteArray(new ArrayOfStringsSerDe());
+ final WritableMemory mem = WritableMemory.writableWrap(bytes);
+ PreambleUtil.insertFamilyID(mem, 0);
+ EbppsItemsSketch.heapify(mem, new ArrayOfStringsSerDe());
+ }
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void deserializeNegativeN() {
+ EbppsItemsSketch<String> sk = new EbppsItemsSketch<>(5);
+ for (int i = 0; i < 10; ++i) sk.update(Integer.toString(i));
+ final byte[] bytes = sk.toByteArray(new ArrayOfStringsSerDe());
+ final WritableMemory mem = WritableMemory.writableWrap(bytes);
+ PreambleUtil.insertN(mem, -1000);
+ EbppsItemsSketch.heapify(mem, new ArrayOfStringsSerDe());
+ }
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void deserializeNaNCumulativeWeight() {
+ EbppsItemsSketch<String> sk = new EbppsItemsSketch<>(5);
+ for (int i = 0; i < 10; ++i) sk.update(Integer.toString(i));
+ final byte[] bytes = sk.toByteArray(new ArrayOfStringsSerDe());
+ final WritableMemory mem = WritableMemory.writableWrap(bytes);
+ PreambleUtil.insertEbppsCumulativeWeight(mem, Double.NaN);
+ EbppsItemsSketch.heapify(mem, new ArrayOfStringsSerDe());
+ }
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void deserializeInfiniteMaxWeight() {
+ EbppsItemsSketch<String> sk = new EbppsItemsSketch<>(5);
+ for (int i = 0; i < 10; ++i) sk.update(Integer.toString(i));
+ final byte[] bytes = sk.toByteArray(new ArrayOfStringsSerDe());
+ final WritableMemory mem = WritableMemory.writableWrap(bytes);
+ PreambleUtil.insertEbppsMaxWeight(mem, Double.POSITIVE_INFINITY);
+ EbppsItemsSketch.heapify(mem, new ArrayOfStringsSerDe());
+ }
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void deserializeNegativeRho() {
+ EbppsItemsSketch<String> sk = new EbppsItemsSketch<>(5);
+ for (int i = 0; i < 10; ++i) sk.update(Integer.toString(i));
+ final byte[] bytes = sk.toByteArray(new ArrayOfStringsSerDe());
+ final WritableMemory mem = WritableMemory.writableWrap(bytes);
+ PreambleUtil.insertEbppsRho(mem, -0.1);
+ EbppsItemsSketch.heapify(mem, new ArrayOfStringsSerDe());
+ }
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void deserializeNegativeC() {
+ EbppsItemsSketch<String> sk = new EbppsItemsSketch<>(5);
+ for (int i = 0; i < 10; ++i) sk.update(Integer.toString(i));
+ final byte[] bytes = sk.toByteArray(new ArrayOfStringsSerDe());
+ final WritableMemory mem = WritableMemory.writableWrap(bytes);
+ mem.putDouble(40, -2.0); // from the defined spec
+ EbppsItemsSketch.heapify(mem, new ArrayOfStringsSerDe());
+ }
+
+ @Test(expectedExceptions = SketchesArgumentException.class)
+ public void deserializeTooShort() {
+ EbppsItemsSketch<Long> sk = new EbppsItemsSketch<>(5);
+ for (long i = 0; i < 10; ++i) sk.update(i);
+ final byte[] bytes = sk.toByteArray(new ArrayOfLongsSerDe());
+ final WritableMemory mem = WritableMemory.writableWrap(bytes);
+ final Memory shortMem = mem.region(0, mem.getCapacity() - 1);
+ EbppsItemsSketch.heapify(shortMem, new ArrayOfStringsSerDe());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org