You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by le...@apache.org on 2022/10/24 23:33:01 UTC
[datasketches-java] branch checkBounds created (now d6ce7153)
This is an automated email from the ASF dual-hosted git repository.
leerho pushed a change to branch checkBounds
in repository https://gitbox.apache.org/repos/asf/datasketches-java.git
at d6ce7153 Added checkBounds on all the sketch entry points where I could find them.
This branch includes the following new commits:
new d6ce7153 Added checkBounds on all the sketch entry points where I could find them.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org
[datasketches-java] 01/01: Added checkBounds on all the sketch entry points where I could find them.
Posted by le...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leerho pushed a commit to branch checkBounds
in repository https://gitbox.apache.org/repos/asf/datasketches-java.git
commit d6ce715346e8ea1277ba6e00a5442c2c3ea59144
Author: Lee Rhodes <le...@users.noreply.github.com>
AuthorDate: Mon Oct 24 16:30:08 2022 -0700
Added checkBounds on all the sketch entry points where I could find
them.
---
.../java/org/apache/datasketches/common/Util.java | 4 ++--
.../org/apache/datasketches/cpc/PreambleUtil.java | 4 ++++
.../org/apache/datasketches/fdt/PostProcessor.java | 3 +++
.../datasketches/frequencies/ItemsSketch.java | 11 +++++++++-
.../datasketches/frequencies/LongsSketch.java | 12 +++++++++--
.../org/apache/datasketches/hll/HllSketch.java | 20 +++++++++++++-----
.../java/org/apache/datasketches/hll/HllUtil.java | 3 +++
.../apache/datasketches/theta/HeapAlphaSketch.java | 22 ++++++++++++++++++++
.../datasketches/theta/HeapUpdateSketch.java | 18 ++++++++--------
.../apache/datasketches/theta/PreambleUtil.java | 5 +++--
.../apache/datasketches/theta/UpdateSketch.java | 13 +++++++++++-
.../datasketches/tuple/QuickSelectSketch.java | 24 ++++++++++++++--------
.../org/apache/datasketches/tuple/Sketches.java | 7 +++++--
13 files changed, 113 insertions(+), 33 deletions(-)
diff --git a/src/main/java/org/apache/datasketches/common/Util.java b/src/main/java/org/apache/datasketches/common/Util.java
index baf10225..44c02b6d 100644
--- a/src/main/java/org/apache/datasketches/common/Util.java
+++ b/src/main/java/org/apache/datasketches/common/Util.java
@@ -673,8 +673,8 @@ public final class Util {
*/
public static void checkBounds(final long reqOff, final long reqLen, final long allocSize) {
if ((reqOff | reqLen | (reqOff + reqLen) | (allocSize - (reqOff + reqLen))) < 0) {
- throw new SketchesArgumentException(
- "reqOffset: " + reqOff + ", reqLength: " + reqLen
+ throw new SketchesArgumentException("Bounds Violation: "
+ + "reqOffset: " + reqOff + ", reqLength: " + reqLen
+ ", (reqOff + reqLen): " + (reqOff + reqLen) + ", allocSize: " + allocSize);
}
}
diff --git a/src/main/java/org/apache/datasketches/cpc/PreambleUtil.java b/src/main/java/org/apache/datasketches/cpc/PreambleUtil.java
index 5a8c0802..3c63bd7f 100644
--- a/src/main/java/org/apache/datasketches/cpc/PreambleUtil.java
+++ b/src/main/java/org/apache/datasketches/cpc/PreambleUtil.java
@@ -19,11 +19,13 @@
package org.apache.datasketches.cpc;
+import static org.apache.datasketches.common.Util.checkBounds;
import static org.apache.datasketches.common.Util.zeroPad;
import static org.apache.datasketches.cpc.RuntimeAsserts.rtAssert;
import static org.apache.datasketches.cpc.RuntimeAsserts.rtAssertEquals;
import java.nio.ByteOrder;
+import java.util.Objects;
import org.apache.datasketches.common.Family;
import org.apache.datasketches.common.SketchesArgumentException;
@@ -807,6 +809,8 @@ final class PreambleUtil {
//basic checks of SerVer, Format, preInts, Family, fiCol, lgK.
static void checkLoPreamble(final Memory mem) {
+ Objects.requireNonNull(mem, "Source Memory must not be null");
+ checkBounds(0, 8, mem.getCapacity()); //need min 8 bytes
rtAssertEquals(getSerVer(mem), SER_VER & 0XFF);
final Format fmt = getFormat(mem);
final int preIntsDef = getDefinedPreInts(fmt) & 0XFF;
diff --git a/src/main/java/org/apache/datasketches/fdt/PostProcessor.java b/src/main/java/org/apache/datasketches/fdt/PostProcessor.java
index 5b903e6f..3260c327 100644
--- a/src/main/java/org/apache/datasketches/fdt/PostProcessor.java
+++ b/src/main/java/org/apache/datasketches/fdt/PostProcessor.java
@@ -26,6 +26,7 @@ import static org.apache.datasketches.tuple.Util.stringHash;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Objects;
import org.apache.datasketches.tuple.TupleSketchIterator;
import org.apache.datasketches.tuple.strings.ArrayOfStringsSummary;
@@ -58,6 +59,8 @@ public class PostProcessor {
* @param sep the separator character
*/
public PostProcessor(final FdtSketch sketch, final Group group, final char sep) {
+ Objects.requireNonNull(sketch, "sketch must me non-null");
+ Objects.requireNonNull(group, "group must be non-null");
this.sketch = sketch.copy();
this.sep = sep;
final int numEntries = sketch.getRetainedEntries();
diff --git a/src/main/java/org/apache/datasketches/frequencies/ItemsSketch.java b/src/main/java/org/apache/datasketches/frequencies/ItemsSketch.java
index 39bf1272..22ad66f7 100644
--- a/src/main/java/org/apache/datasketches/frequencies/ItemsSketch.java
+++ b/src/main/java/org/apache/datasketches/frequencies/ItemsSketch.java
@@ -20,6 +20,7 @@
package org.apache.datasketches.frequencies;
import static org.apache.datasketches.common.Util.LS;
+import static org.apache.datasketches.common.Util.checkBounds;
import static org.apache.datasketches.common.Util.exactLog2OfInt;
import static org.apache.datasketches.common.Util.isIntPowerOf2;
import static org.apache.datasketches.frequencies.PreambleUtil.EMPTY_FLAG_MASK;
@@ -44,6 +45,7 @@ import static org.apache.datasketches.frequencies.Util.SAMPLE_SIZE;
import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.Comparator;
+import java.util.Objects;
import org.apache.datasketches.common.ArrayOfItemsSerDe;
import org.apache.datasketches.common.Family;
@@ -226,6 +228,9 @@ public class ItemsSketch<T> {
*/
public static <T> ItemsSketch<T> getInstance(final Memory srcMem,
final ArrayOfItemsSerDe<T> serDe) {
+ Objects.requireNonNull(srcMem, "srcMem must not be null.");
+ Objects.requireNonNull(serDe, "serDe must not be null.");
+
final long pre0 = PreambleUtil.checkPreambleSize(srcMem); //make sure preamble will fit
final int maxPreLongs = Family.FREQUENCY.getMaxPreLongs();
@@ -270,11 +275,15 @@ public class ItemsSketch<T> {
final int preBytes = preLongs << 3;
final int activeItems = extractActiveItems(preArr[1]);
+
//Get countArray
final long[] countArray = new long[activeItems];
+ int reqBytes = preBytes + activeItems * Long.BYTES; //count Arr only
+ checkBounds(0, reqBytes, srcMem.getCapacity()); //check Memory capacity
srcMem.getLongArray(preBytes, countArray, 0, activeItems);
+
//Get itemArray
- final int itemsOffset = preBytes + (8 * activeItems);
+ final int itemsOffset = preBytes + (Long.BYTES * activeItems);
final T[] itemArray = serDe.deserializeFromMemory(
srcMem.region(itemsOffset, srcMem.getCapacity() - itemsOffset), activeItems);
//update the sketch
diff --git a/src/main/java/org/apache/datasketches/frequencies/LongsSketch.java b/src/main/java/org/apache/datasketches/frequencies/LongsSketch.java
index 7398f498..069440b6 100644
--- a/src/main/java/org/apache/datasketches/frequencies/LongsSketch.java
+++ b/src/main/java/org/apache/datasketches/frequencies/LongsSketch.java
@@ -20,6 +20,7 @@
package org.apache.datasketches.frequencies;
import static org.apache.datasketches.common.Util.LS;
+import static org.apache.datasketches.common.Util.checkBounds;
import static org.apache.datasketches.common.Util.exactLog2OfInt;
import static org.apache.datasketches.common.Util.isIntPowerOf2;
import static org.apache.datasketches.frequencies.PreambleUtil.EMPTY_FLAG_MASK;
@@ -43,6 +44,7 @@ import static org.apache.datasketches.frequencies.Util.SAMPLE_SIZE;
import java.util.ArrayList;
import java.util.Comparator;
+import java.util.Objects;
import org.apache.datasketches.common.Family;
import org.apache.datasketches.common.SketchesArgumentException;
@@ -220,7 +222,8 @@ public class LongsSketch {
* @return a sketch instance of this class.
*/
public static LongsSketch getInstance(final Memory srcMem) {
- final long pre0 = PreambleUtil.checkPreambleSize(srcMem); //make sure preamble will fit
+ Objects.requireNonNull(srcMem, "Source Memory must not be null.");
+ final long pre0 = PreambleUtil.checkPreambleSize(srcMem); //check Memory capacity
final int maxPreLongs = Family.FREQUENCY.getMaxPreLongs();
final int preLongs = extractPreLongs(pre0); //Byte 0
@@ -264,11 +267,15 @@ public class LongsSketch {
final int preBytes = preLongs << 3;
final int activeItems = extractActiveItems(preArr[1]);
+
//Get countArray
final long[] countArray = new long[activeItems];
+ final int reqBytes = preBytes + 2 * activeItems * Long.BYTES; //count Arr + Items Arr
+ checkBounds(0, reqBytes, srcMem.getCapacity()); //check Memory capacity
srcMem.getLongArray(preBytes, countArray, 0, activeItems);
+
//Get itemArray
- final int itemsOffset = preBytes + (8 * activeItems);
+ final int itemsOffset = preBytes + (Long.BYTES * activeItems);
final long[] itemArray = new long[activeItems];
srcMem.getLongArray(itemsOffset, itemArray, 0, activeItems);
//update the sketch
@@ -287,6 +294,7 @@ public class LongsSketch {
* @return a sketch instance of this class.
*/
public static LongsSketch getInstance(final String string) {
+ Objects.requireNonNull(string, "string must not be null.");
final String[] tokens = string.split(",");
if (tokens.length < (STR_PREAMBLE_TOKENS + 2)) {
throw new SketchesArgumentException(
diff --git a/src/main/java/org/apache/datasketches/hll/HllSketch.java b/src/main/java/org/apache/datasketches/hll/HllSketch.java
index 58050f80..1ec74df9 100644
--- a/src/main/java/org/apache/datasketches/hll/HllSketch.java
+++ b/src/main/java/org/apache/datasketches/hll/HllSketch.java
@@ -19,6 +19,7 @@
package org.apache.datasketches.hll;
+import static org.apache.datasketches.common.Util.checkBounds;
import static org.apache.datasketches.hll.HllUtil.EMPTY;
import static org.apache.datasketches.hll.HllUtil.KEY_BITS_26;
import static org.apache.datasketches.hll.HllUtil.LG_AUX_ARR_INTS;
@@ -28,6 +29,8 @@ import static org.apache.datasketches.hll.PreambleUtil.extractCompactFlag;
import static org.apache.datasketches.hll.PreambleUtil.extractLgK;
import static org.apache.datasketches.hll.PreambleUtil.extractTgtHllType;
+import java.util.Objects;
+
import org.apache.datasketches.common.SketchesArgumentException;
import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.memory.WritableMemory;
@@ -119,6 +122,7 @@ public class HllSketch extends BaseHllSketch {
* @param dstMem the destination memory for the sketch.
*/
public HllSketch(final int lgConfigK, final TgtHllType tgtHllType, final WritableMemory dstMem) {
+ Objects.requireNonNull(dstMem, "Destination Memory must not be null");
final long minBytes = getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType);
final long capBytes = dstMem.getCapacity();
HllUtil.checkMemSize(minBytes, capBytes);
@@ -163,6 +167,8 @@ public class HllSketch extends BaseHllSketch {
//used by union and above
static final HllSketch heapify(final Memory srcMem, final boolean checkRebuild) {
+ Objects.requireNonNull(srcMem, "Source Memory must not be null");
+ checkBounds(0, 8, srcMem.getCapacity()); //need min 8 bytes
final CurMode curMode = checkPreamble(srcMem);
final HllSketch heapSketch;
if (curMode == CurMode.HLL) {
@@ -197,16 +203,18 @@ public class HllSketch extends BaseHllSketch {
* @return an HllSketch where the sketch data is in the given dstMem.
*/
public static final HllSketch writableWrap(final WritableMemory srcWmem) {
- if (extractCompactFlag(srcWmem)) {
- throw new SketchesArgumentException(
- "Cannot perform a writableWrap of a writable sketch image that is in compact form. "
- + "Compact sketches are by definition immutable.");
- }
return writableWrap(srcWmem, true);
}
//used by union and above
static final HllSketch writableWrap( final WritableMemory srcWmem, final boolean checkRebuild) {
+ Objects.requireNonNull(srcWmem, "Source Memory must not be null");
+ checkBounds(0, 8, srcWmem.getCapacity()); //need min 8 bytes
+ if (extractCompactFlag(srcWmem)) {
+ throw new SketchesArgumentException(
+ "Cannot perform a writableWrap of a writable sketch image that is in compact form. "
+ + "Compact sketches are by definition immutable.");
+ }
final int lgConfigK = extractLgK(srcWmem);
final TgtHllType tgtHllType = extractTgtHllType(srcWmem);
final long minBytes = getMaxUpdatableSerializationBytes(lgConfigK, tgtHllType);
@@ -244,6 +252,8 @@ public class HllSketch extends BaseHllSketch {
*
*/
public static final HllSketch wrap(final Memory srcMem) {
+ Objects.requireNonNull(srcMem, "Source Memory must not be null");
+ checkBounds(0, 8, srcMem.getCapacity()); //need min 8 bytes
final int lgConfigK = extractLgK(srcMem);
final TgtHllType tgtHllType = extractTgtHllType(srcMem);
diff --git a/src/main/java/org/apache/datasketches/hll/HllUtil.java b/src/main/java/org/apache/datasketches/hll/HllUtil.java
index 48be77b8..0a1ef518 100644
--- a/src/main/java/org/apache/datasketches/hll/HllUtil.java
+++ b/src/main/java/org/apache/datasketches/hll/HllUtil.java
@@ -21,6 +21,7 @@ package org.apache.datasketches.hll;
import static java.lang.Math.log;
import static java.lang.Math.sqrt;
+import static org.apache.datasketches.common.Util.checkBounds;
import static org.apache.datasketches.hll.PreambleUtil.HASH_SET_PREINTS;
import static org.apache.datasketches.hll.PreambleUtil.HLL_PREINTS;
import static org.apache.datasketches.hll.PreambleUtil.LIST_PREINTS;
@@ -94,7 +95,9 @@ final class HllUtil {
}
static CurMode checkPreamble(final Memory mem) {
+ checkBounds(0, 8, mem.getCapacity()); //need min 8 bytes
final int preInts = extractPreInts(mem);
+ checkBounds(0, preInts * Integer.BYTES, mem.getCapacity());
final int serVer = extractSerVer(mem);
final int famId = extractFamilyId(mem);
final CurMode curMode = extractCurMode(mem);
diff --git a/src/main/java/org/apache/datasketches/theta/HeapAlphaSketch.java b/src/main/java/org/apache/datasketches/theta/HeapAlphaSketch.java
index 7b6652bb..23d0027a 100644
--- a/src/main/java/org/apache/datasketches/theta/HeapAlphaSketch.java
+++ b/src/main/java/org/apache/datasketches/theta/HeapAlphaSketch.java
@@ -23,6 +23,7 @@ import static java.lang.Math.max;
import static java.lang.Math.min;
import static java.lang.Math.sqrt;
import static org.apache.datasketches.common.Util.LONG_MAX_VALUE_AS_DOUBLE;
+import static org.apache.datasketches.common.Util.checkBounds;
import static org.apache.datasketches.theta.PreambleUtil.extractCurCount;
import static org.apache.datasketches.theta.PreambleUtil.extractFamilyID;
import static org.apache.datasketches.theta.PreambleUtil.extractLgArrLongs;
@@ -37,6 +38,8 @@ import static org.apache.datasketches.theta.UpdateReturnState.RejectedDuplicate;
import static org.apache.datasketches.theta.UpdateReturnState.RejectedOverTheta;
import static org.apache.datasketches.thetacommon.HashOperations.STRIDE_MASK;
+import java.util.Objects;
+
import org.apache.datasketches.common.Family;
import org.apache.datasketches.common.ResizeFactor;
import org.apache.datasketches.common.SketchesArgumentException;
@@ -113,11 +116,14 @@ final class HeapAlphaSketch extends HeapUpdateSketch {
* Heapify a sketch from a Memory object containing sketch data.
* @param srcMem The source Memory object.
* <a href="{@docRoot}/resources/dictionary.html#mem">See Memory</a>
+ * It must have a size of at least 24 bytes.
* @param expectedSeed the seed used to validate the given Memory image.
* <a href="{@docRoot}/resources/dictionary.html#seed">See seed</a>
* @return instance of this sketch
*/
static HeapAlphaSketch heapifyInstance(final Memory srcMem, final long expectedSeed) {
+ Objects.requireNonNull(srcMem, "Source Memory must not be null");
+ checkBounds(0, 24, srcMem.getCapacity());
final int preambleLongs = extractPreLongs(srcMem); //byte 0
final int lgNomLongs = extractLgNomLongs(srcMem); //byte 3
final int lgArrLongs = extractLgArrLongs(srcMem); //byte 4
@@ -225,6 +231,22 @@ final class HeapAlphaSketch extends HeapUpdateSketch {
return empty_;
}
+ /*
+ * Alpha Sketch Preamble Layout ( same as Theta UpdateSketch )
+ * <pre>
+ * Long || Start Byte Adr:
+ * Adr:
+ * || 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 |
+ * 0 || Seed Hash | Flags | LgArr | LgNom | FamID | SerVer | lgRF | PreLongs=3 |
+ *
+ * || 15 | 14 | 13 | 12 | 11 | 10 | 9 | 8 |
+ * 1 ||-----------------p-----------------|----------Retained Entries Count---------------|
+ *
+ * || 23 | 22 | 21 | 20 | 19 | 18 | 17 | 16 |
+ * 2 ||---------------------------------Theta---------------------------------------------|
+ * </pre>
+ */
+
@Override
public byte[] toByteArray() {
return toByteArray(Family.ALPHA.getMinPreLongs(), (byte) Family.ALPHA.getID());
diff --git a/src/main/java/org/apache/datasketches/theta/HeapUpdateSketch.java b/src/main/java/org/apache/datasketches/theta/HeapUpdateSketch.java
index f8ff9271..e8e2a1e1 100644
--- a/src/main/java/org/apache/datasketches/theta/HeapUpdateSketch.java
+++ b/src/main/java/org/apache/datasketches/theta/HeapUpdateSketch.java
@@ -105,24 +105,24 @@ abstract class HeapUpdateSketch extends UpdateSketch {
return ThetaUtil.computeSeedHash(getSeed());
}
- //Used by HeapAlphaSketch and HeapQuickSelectSketch
+ //Used by HeapAlphaSketch and HeapQuickSelectSketch / Theta UpdateSketch
byte[] toByteArray(final int preLongs, final byte familyID) {
if (isDirty()) { rebuild(); }
checkIllegalCurCountAndEmpty(isEmpty(), getRetainedEntries(true));
- final int preBytes = (preLongs << 3) & 0X3F;
+ final int preBytes = (preLongs << 3) & 0X3F; //24 bytes
final int dataBytes = getCurrentDataLongs() << 3;
final byte[] byteArrOut = new byte[preBytes + dataBytes];
final WritableMemory memOut = WritableMemory.writableWrap(byteArrOut);
//preamble first 8 bytes. Note: only compact can be reduced to 8 bytes.
final int lgRf = getResizeFactor().lg() & 0x3;
- insertPreLongs(memOut, preLongs);
- insertLgResizeFactor(memOut, lgRf);
- insertSerVer(memOut, SER_VER);
- insertFamilyID(memOut, familyID);
- insertLgNomLongs(memOut, getLgNomLongs());
- insertLgArrLongs(memOut, getLgArrLongs());
- insertSeedHash(memOut, getSeedHash());
+ insertPreLongs(memOut, preLongs); //byte 0 low 6 bits
+ insertLgResizeFactor(memOut, lgRf); //byte 0 high 2 bits
+ insertSerVer(memOut, SER_VER); //byte 1
+ insertFamilyID(memOut, familyID); //byte 2
+ insertLgNomLongs(memOut, getLgNomLongs()); //byte 3
+ insertLgArrLongs(memOut, getLgArrLongs()); //byte 4
+ insertSeedHash(memOut, getSeedHash()); //bytes 6 & 7
insertCurCount(memOut, this.getRetainedEntries(true));
insertP(memOut, getP());
diff --git a/src/main/java/org/apache/datasketches/theta/PreambleUtil.java b/src/main/java/org/apache/datasketches/theta/PreambleUtil.java
index 12d27246..50a70558 100644
--- a/src/main/java/org/apache/datasketches/theta/PreambleUtil.java
+++ b/src/main/java/org/apache/datasketches/theta/PreambleUtil.java
@@ -106,9 +106,10 @@ import org.apache.datasketches.thetacommon.ThetaUtil;
* 3 ||----------------------Start of Compact Long Array----------------------------------|
* </pre>
*
- * <p>An UpdateSketch requires 24 bytes of preamble plus a non-compact array of longs representing a
- * hash table.</p>
+ * <p>The UpdateSketch and AlphaSketch require 24 bytes of preamble followed by a non-compact
+ * array of longs representing a hash table.</p>
*
+ * The following table applies to both the Theta UpdateSketch and the Alpha Sketch
* <pre>
* Long || Start Byte Adr:
* Adr:
diff --git a/src/main/java/org/apache/datasketches/theta/UpdateSketch.java b/src/main/java/org/apache/datasketches/theta/UpdateSketch.java
index 0c0d6657..f351b955 100644
--- a/src/main/java/org/apache/datasketches/theta/UpdateSketch.java
+++ b/src/main/java/org/apache/datasketches/theta/UpdateSketch.java
@@ -21,6 +21,7 @@ package org.apache.datasketches.theta;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.datasketches.common.Util.LONG_MAX_VALUE_AS_DOUBLE;
+import static org.apache.datasketches.common.Util.checkBounds;
import static org.apache.datasketches.hash.MurmurHash3.hash;
import static org.apache.datasketches.theta.CompactOperations.componentsToCompact;
import static org.apache.datasketches.theta.PreambleUtil.BIG_ENDIAN_FLAG_MASK;
@@ -41,6 +42,8 @@ import static org.apache.datasketches.theta.PreambleUtil.extractThetaLong;
import static org.apache.datasketches.theta.PreambleUtil.getMemBytes;
import static org.apache.datasketches.theta.UpdateReturnState.RejectedNullOrEmpty;
+import java.util.Objects;
+
import org.apache.datasketches.common.Family;
import org.apache.datasketches.common.ResizeFactor;
import org.apache.datasketches.common.SketchesArgumentException;
@@ -66,6 +69,7 @@ public abstract class UpdateSketch extends Sketch {
* {@link org.apache.datasketches.thetacommon.ThetaUtil#DEFAULT_UPDATE_SEED}.
* <a href="{@docRoot}/resources/dictionary.html#defaultUpdateSeed">Default Update Seed</a>.
* @param srcMem an image of a Sketch where the image seed hash matches the default seed hash.
+ * It must have a size of at least 24 bytes.
* <a href="{@docRoot}/resources/dictionary.html#mem">See Memory</a>
* @return a Sketch backed by the given Memory
*/
@@ -80,6 +84,7 @@ public abstract class UpdateSketch extends Sketch {
* An attempt to "wrap" earlier version sketches will result in a "heapified", normal
* Java Heap version of the sketch where all data will be copied to the heap.
* @param srcMem an image of a Sketch where the image seed hash matches the given seed hash.
+ * It must have a size of at least 24 bytes.
* <a href="{@docRoot}/resources/dictionary.html#mem">See Memory</a>
* @param expectedSeed the seed used to validate the given Memory image.
* <a href="{@docRoot}/resources/dictionary.html#seed">See Update Hash Seed</a>.
@@ -87,6 +92,8 @@ public abstract class UpdateSketch extends Sketch {
* @return a UpdateSketch backed by the given Memory
*/
public static UpdateSketch wrap(final WritableMemory srcMem, final long expectedSeed) {
+ Objects.requireNonNull(srcMem, "Source Memory must not be null");
+ checkBounds(0, 24, srcMem.getCapacity()); //need min 24 bytes
final int preLongs = srcMem.getByte(PREAMBLE_LONGS_BYTE) & 0X3F;
final int serVer = srcMem.getByte(SER_VER_BYTE) & 0XFF;
final int familyID = srcMem.getByte(FAMILY_BYTE) & 0XFF;
@@ -99,7 +106,7 @@ public abstract class UpdateSketch extends Sketch {
return DirectQuickSelectSketch.writableWrap(srcMem, expectedSeed);
} else {
throw new SketchesArgumentException(
- "Corrupted: An UpdateSketch image: must have SerVer = 3 and preLongs = 3");
+ "Corrupted: An UpdateSketch image must have SerVer = 3 and preLongs = 3");
}
}
@@ -107,6 +114,7 @@ public abstract class UpdateSketch extends Sketch {
* Instantiates an on-heap UpdateSketch from Memory. This method assumes the
* {@link org.apache.datasketches.thetacommon.ThetaUtil#DEFAULT_UPDATE_SEED}.
* @param srcMem <a href="{@docRoot}/resources/dictionary.html#mem">See Memory</a>
+ * It must have a size of at least 24 bytes.
* @return an UpdateSketch
*/
public static UpdateSketch heapify(final Memory srcMem) {
@@ -116,11 +124,14 @@ public abstract class UpdateSketch extends Sketch {
/**
* Instantiates an on-heap UpdateSketch from Memory.
* @param srcMem <a href="{@docRoot}/resources/dictionary.html#mem">See Memory</a>
+ * It must have a size of at least 24 bytes.
* @param expectedSeed the seed used to validate the given Memory image.
* <a href="{@docRoot}/resources/dictionary.html#seed">See Update Hash Seed</a>.
* @return an UpdateSketch
*/
public static UpdateSketch heapify(final Memory srcMem, final long expectedSeed) {
+ Objects.requireNonNull(srcMem, "Source Memory must not be null");
+ checkBounds(0, 24, srcMem.getCapacity()); //need min 24 bytes
final Family family = Family.idToFamily(srcMem.getByte(FAMILY_BYTE));
if (family.equals(Family.ALPHA)) {
return HeapAlphaSketch.heapifyInstance(srcMem, expectedSeed);
diff --git a/src/main/java/org/apache/datasketches/tuple/QuickSelectSketch.java b/src/main/java/org/apache/datasketches/tuple/QuickSelectSketch.java
index 85cfc14e..6e6f1eaf 100644
--- a/src/main/java/org/apache/datasketches/tuple/QuickSelectSketch.java
+++ b/src/main/java/org/apache/datasketches/tuple/QuickSelectSketch.java
@@ -20,11 +20,13 @@
package org.apache.datasketches.tuple;
import static org.apache.datasketches.common.Util.ceilingIntPowerOf2;
+import static org.apache.datasketches.common.Util.checkBounds;
import static org.apache.datasketches.common.Util.exactLog2OfLong;
import static org.apache.datasketches.thetacommon.HashOperations.count;
import java.lang.reflect.Array;
import java.nio.ByteOrder;
+import java.util.Objects;
import org.apache.datasketches.common.ByteArrayUtil;
import org.apache.datasketches.common.Family;
@@ -165,30 +167,34 @@ class QuickSelectSketch<S extends Summary> extends Sketch<S> {
final Memory mem,
final SummaryDeserializer<S> deserializer,
final SummaryFactory<S> summaryFactory) {
+ Objects.requireNonNull(mem, "SourceMemory must not be null.");
+ Objects.requireNonNull(deserializer, "Deserializer must not be null.");
+ checkBounds(0, 8, mem.getCapacity());
summaryFactory_ = summaryFactory;
int offset = 0;
- final byte preambleLongs = mem.getByte(offset++);
- final byte version = mem.getByte(offset++);
- final byte familyId = mem.getByte(offset++);
+ final byte preambleLongs = mem.getByte(offset++); //byte 0 PreLongs
+ final byte version = mem.getByte(offset++); //byte 1 SerVer
+ final byte familyId = mem.getByte(offset++); //byte 2 FamID
SerializerDeserializer.validateFamily(familyId, preambleLongs);
if (version > serialVersionUID) {
throw new SketchesArgumentException(
"Unsupported serial version. Expected: " + serialVersionUID + " or lower, actual: "
+ version);
}
- SerializerDeserializer.validateType(mem.getByte(offset++),
+ SerializerDeserializer.validateType(mem.getByte(offset++), //byte 3
SerializerDeserializer.SketchType.QuickSelectSketch);
- final byte flags = mem.getByte(offset++);
+ final byte flags = mem.getByte(offset++); //byte 4
final boolean isBigEndian = (flags & 1 << Flags.IS_BIG_ENDIAN.ordinal()) > 0;
if (isBigEndian ^ ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) {
throw new SketchesArgumentException("Endian byte order mismatch");
}
- nomEntries_ = 1 << mem.getByte(offset++);
- lgCurrentCapacity_ = mem.getByte(offset++);
- lgResizeFactor_ = mem.getByte(offset++);
+ nomEntries_ = 1 << mem.getByte(offset++); //byte 5
+ lgCurrentCapacity_ = mem.getByte(offset++); //byte 6
+ lgResizeFactor_ = mem.getByte(offset++); //byte 7
+ checkBounds(0, preambleLongs * 8, mem.getCapacity());
final boolean isInSamplingMode = (flags & 1 << Flags.IS_IN_SAMPLING_MODE.ordinal()) > 0;
- samplingProbability_ = isInSamplingMode ? mem.getFloat(offset) : 1f;
+ samplingProbability_ = isInSamplingMode ? mem.getFloat(offset) : 1f; //bytes 8 - 11
if (isInSamplingMode) {
offset += Float.BYTES;
}
diff --git a/src/main/java/org/apache/datasketches/tuple/Sketches.java b/src/main/java/org/apache/datasketches/tuple/Sketches.java
index 0630daa2..7a573518 100644
--- a/src/main/java/org/apache/datasketches/tuple/Sketches.java
+++ b/src/main/java/org/apache/datasketches/tuple/Sketches.java
@@ -46,7 +46,8 @@ public final class Sketches {
* @return Sketch created from its Memory representation
*/
@SuppressWarnings("deprecation")
- public static <S extends Summary> Sketch<S> heapifySketch(final Memory mem,
+ public static <S extends Summary> Sketch<S> heapifySketch(
+ final Memory mem,
final SummaryDeserializer<S> deserializer) {
final SerializerDeserializer.SketchType sketchType = SerializerDeserializer.getSketchType(mem);
if (sketchType == SerializerDeserializer.SketchType.QuickSelectSketch) {
@@ -69,7 +70,9 @@ public final class Sketches {
*/
@Deprecated
public static <U, S extends UpdatableSummary<U>> UpdatableSketch<U, S> heapifyUpdatableSketch(
- final Memory mem, final SummaryDeserializer<S> deserializer, final SummaryFactory<S> summaryFactory) {
+ final Memory mem,
+ final SummaryDeserializer<S> deserializer,
+ final SummaryFactory<S> summaryFactory) {
return new UpdatableSketch<U, S>(mem, deserializer, summaryFactory);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org