You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by jm...@apache.org on 2020/06/01 23:17:31 UTC
[incubator-datasketches-hive] 01/01: Replace calls to
BytesWritable.getBytes() to ensure we don't read beyond the valid buffer
This is an automated email from the ASF dual-hosted git repository.
jmalkin pushed a commit to branch byteswritable_length_fix
in repository https://gitbox.apache.org/repos/asf/incubator-datasketches-hive.git
commit 3337f01a765ad79003e4ff8181b0bf67f4a63ba4
Author: Jon Malkin <jm...@users.noreply.github.com>
AuthorDate: Mon Jun 1 16:16:56 2020 -0700
Replace calls to BytesWritable.getBytes() to ensure we don't read beyond the valid buffer
---
.../BytesWritableHelper.java} | 33 +++++++++-------------
.../hive/cpc/GetEstimateAndErrorBoundsUDF.java | 4 +--
.../datasketches/hive/cpc/GetEstimateUDF.java | 4 +--
.../datasketches/hive/cpc/SketchEvaluator.java | 4 +--
.../apache/datasketches/hive/cpc/SketchState.java | 2 +-
.../datasketches/hive/cpc/SketchToStringUDF.java | 4 +--
.../datasketches/hive/cpc/UnionSketchUDF.java | 6 ++--
.../GetFrequentItemsFromStringsSketchUDTF.java | 4 +--
.../hive/frequencies/ItemsEvaluator.java | 8 ++++--
.../datasketches/hive/frequencies/ItemsState.java | 4 +--
.../datasketches/hive/hll/SketchEvaluator.java | 4 +--
.../apache/datasketches/hive/hll/SketchState.java | 2 +-
.../hll/SketchToEstimateAndErrorBoundsUDF.java | 4 +--
.../datasketches/hive/hll/SketchToEstimateUDF.java | 4 +--
.../datasketches/hive/hll/SketchToStringUDF.java | 4 +--
.../datasketches/hive/hll/UnionSketchUDF.java | 6 ++--
.../apache/datasketches/hive/hll/UnionState.java | 2 +-
.../apache/datasketches/hive/kll/GetCdfUDF.java | 4 +--
.../org/apache/datasketches/hive/kll/GetNUDF.java | 4 +--
.../apache/datasketches/hive/kll/GetPmfUDF.java | 4 +--
.../datasketches/hive/kll/GetQuantileUDF.java | 4 +--
.../datasketches/hive/kll/GetQuantilesUDF.java | 4 +--
.../apache/datasketches/hive/kll/GetRankUDF.java | 4 +--
.../datasketches/hive/kll/SketchEvaluator.java | 7 +++--
.../datasketches/hive/kll/SketchToStringUDF.java | 4 +--
.../hive/quantiles/DoublesEvaluator.java | 8 ++++--
.../hive/quantiles/DoublesSketchToStringUDF.java | 4 +--
.../hive/quantiles/DoublesUnionState.java | 4 +--
.../hive/quantiles/GetCdfFromDoublesSketchUDF.java | 4 +--
.../hive/quantiles/GetCdfFromStringsSketchUDF.java | 4 +--
.../hive/quantiles/GetKFromDoublesSketchUDF.java | 4 +--
.../hive/quantiles/GetKFromStringsSketchUDF.java | 4 +--
.../hive/quantiles/GetNFromDoublesSketchUDF.java | 4 +--
.../hive/quantiles/GetNFromStringsSketchUDF.java | 4 +--
.../hive/quantiles/GetPmfFromDoublesSketchUDF.java | 4 +--
.../hive/quantiles/GetPmfFromStringsSketchUDF.java | 4 +--
.../quantiles/GetQuantileFromDoublesSketchUDF.java | 4 +--
.../quantiles/GetQuantileFromStringsSketchUDF.java | 4 +--
.../GetQuantilesFromDoublesSketchUDF.java | 6 ++--
.../GetQuantilesFromStringsSketchUDF.java | 6 ++--
.../hive/quantiles/ItemsEvaluator.java | 8 ++++--
.../hive/quantiles/ItemsUnionState.java | 4 +--
.../hive/quantiles/StringsSketchToStringUDF.java | 4 +--
.../datasketches/hive/theta/EstimateSketchUDF.java | 9 ++++--
.../datasketches/hive/theta/ExcludeSketchUDF.java | 6 ++--
.../hive/theta/IntersectSketchUDAF.java | 13 +++++----
.../hive/theta/IntersectSketchUDF.java | 6 ++--
.../datasketches/hive/theta/UnionEvaluator.java | 7 +++--
.../datasketches/hive/theta/UnionSketchUDF.java | 6 ++--
.../apache/datasketches/hive/theta/UnionState.java | 3 +-
.../hive/tuple/ArrayOfDoublesSketchEvaluator.java | 7 +++--
.../hive/tuple/ArrayOfDoublesSketchState.java | 2 +-
...OfDoublesSketchToEstimateAndErrorBoundsUDF.java | 4 +--
.../tuple/ArrayOfDoublesSketchToEstimatesUDF.java | 4 +--
.../hive/tuple/ArrayOfDoublesSketchToMeansUDF.java | 4 +--
...fDoublesSketchToNumberOfRetainedEntriesUDF.java | 4 +--
.../ArrayOfDoublesSketchToQuantilesSketchUDF.java | 4 +--
.../tuple/ArrayOfDoublesSketchToValuesUDTF.java | 4 +--
.../tuple/ArrayOfDoublesSketchToVariancesUDF.java | 4 +--
.../hive/tuple/ArrayOfDoublesSketchesTTestUDF.java | 6 ++--
.../tuple/DoubleSummarySketchToEstimatesUDF.java | 4 +--
.../tuple/DoubleSummarySketchToPercentileUDF.java | 4 +--
.../datasketches/hive/tuple/SketchEvaluator.java | 4 +--
.../datasketches/hive/tuple/SketchState.java | 2 +-
.../hive/cpc/DataToSketchUDAFTest.java | 16 +++++------
.../datasketches/hive/cpc/UnionSketchUDAFTest.java | 14 ++++-----
.../datasketches/hive/cpc/UnionSketchUDFTest.java | 10 +++----
.../frequencies/DataToStringsSketchUDAFTest.java | 8 +++---
.../frequencies/UnionStringsSketchUDAFTest.java | 6 ++--
.../hive/hll/DataToSketchUDAFTest.java | 16 +++++------
.../datasketches/hive/hll/UnionSketchUDAFTest.java | 14 ++++-----
.../datasketches/hive/hll/UnionSketchUDFTest.java | 8 +++---
.../hive/kll/DataToSketchUDAFTest.java | 14 ++++-----
.../datasketches/hive/kll/UnionSketchUDAFTest.java | 12 ++++----
.../quantiles/DataToDoublesSketchUDAFTest.java | 14 ++++-----
.../quantiles/DataToStringsSketchUDAFTest.java | 14 ++++-----
.../hive/quantiles/UnionDoublesSketchUDAFTest.java | 12 ++++----
.../hive/quantiles/UnionStringsSketchUDAFTest.java | 12 ++++----
.../hive/theta/DataToSketchUDAFTest.java | 14 ++++-----
.../hive/theta/ExcludeSketchUDFTest.java | 11 ++++----
.../hive/theta/IntersectSketchUDAFTest.java | 12 ++++----
.../hive/theta/IntersectSketchUDFTest.java | 9 +++---
.../datasketches/hive/theta/SampleSketchUDF.java | 7 +++--
.../hive/theta/UnionSketchUDAFTest.java | 14 ++++-----
.../hive/theta/UnionSketchUDFTest.java | 9 +++---
...rayOfDoublesSketchToQuantilesSketchUDFTest.java | 8 +++---
.../tuple/DataToArrayOfDoublesSketchUDAFTest.java | 16 +++++------
.../tuple/DataToDoubleSummarySketchUDAFTest.java | 16 +++++------
.../DataToDoubleSummaryWithModeSketchUDAFTest.java | 15 +++++-----
.../tuple/UnionArrayOfDoublesSketchUDAFTest.java | 14 ++++-----
.../tuple/UnionDoubleSummarySketchUDAFTest.java | 14 ++++-----
.../UnionDoubleSummaryWithModeSketchUDAFTest.java | 16 +++++------
92 files changed, 342 insertions(+), 328 deletions(-)
diff --git a/src/main/java/org/apache/datasketches/hive/quantiles/GetKFromDoublesSketchUDF.java b/src/main/java/org/apache/datasketches/hive/common/BytesWritableHelper.java
similarity index 53%
copy from src/main/java/org/apache/datasketches/hive/quantiles/GetKFromDoublesSketchUDF.java
copy to src/main/java/org/apache/datasketches/hive/common/BytesWritableHelper.java
index 9ba75e9..839c2ed 100644
--- a/src/main/java/org/apache/datasketches/hive/quantiles/GetKFromDoublesSketchUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/common/BytesWritableHelper.java
@@ -17,28 +17,23 @@
* under the License.
*/
-package org.apache.datasketches.hive.quantiles;
+package org.apache.datasketches.hive.common;
import org.apache.datasketches.memory.Memory;
-import org.apache.datasketches.quantiles.DoublesSketch;
-import org.apache.hadoop.hive.ql.exec.Description;
-import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
-@Description(name = "GetK", value = "_FUNC_(sketch)",
-extended = " Returns parameter K from a given DoublesSketch sketch.")
-@SuppressWarnings("javadoc")
-public class GetKFromDoublesSketchUDF extends UDF {
-
- /**
- * Returns parameter K from a given sketch
- * @param serializedSketch serialized sketch
- * @return parameter K
- */
- public Integer evaluate(final BytesWritable serializedSketch) {
- if (serializedSketch == null) { return null; }
- final DoublesSketch sketch = DoublesSketch.wrap(Memory.wrap(serializedSketch.getBytes()));
- return sketch.getK();
- }
+import java.nio.ByteOrder;
+/**
+ * Provides a helper class to simplify frequent operations on BytesWritable.
+ */
+public class BytesWritableHelper {
+ /**
+ * Wraps BytesWritable with a read-only Memory interface, without copying the underlying data.
+ * @param bw Input BytesWritable object
+ * @return Read-only Memory wrapping the input BytesWritable
+ */
+ public static Memory wrapAsMemory(BytesWritable bw) {
+ return Memory.wrap(bw.getBytes(), 0, bw.getLength(), ByteOrder.nativeOrder());
+ }
}
diff --git a/src/main/java/org/apache/datasketches/hive/cpc/GetEstimateAndErrorBoundsUDF.java b/src/main/java/org/apache/datasketches/hive/cpc/GetEstimateAndErrorBoundsUDF.java
index 68113ad..b93a639 100644
--- a/src/main/java/org/apache/datasketches/hive/cpc/GetEstimateAndErrorBoundsUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/cpc/GetEstimateAndErrorBoundsUDF.java
@@ -25,7 +25,7 @@ import java.util.Arrays;
import java.util.List;
import org.apache.datasketches.cpc.CpcSketch;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
@@ -70,7 +70,7 @@ public class GetEstimateAndErrorBoundsUDF extends UDF {
*/
public List<Double> evaluate(final BytesWritable serializedSketch, final int kappa, final long seed) {
if (serializedSketch == null) { return null; }
- final CpcSketch sketch = CpcSketch.heapify(Memory.wrap(serializedSketch.getBytes()), seed);
+ final CpcSketch sketch = CpcSketch.heapify(BytesWritableHelper.wrapAsMemory(serializedSketch), seed);
return Arrays.asList(sketch.getEstimate(), sketch.getLowerBound(kappa), sketch.getUpperBound(kappa));
}
diff --git a/src/main/java/org/apache/datasketches/hive/cpc/GetEstimateUDF.java b/src/main/java/org/apache/datasketches/hive/cpc/GetEstimateUDF.java
index 6b53711..9041f81 100644
--- a/src/main/java/org/apache/datasketches/hive/cpc/GetEstimateUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/cpc/GetEstimateUDF.java
@@ -22,7 +22,7 @@ package org.apache.datasketches.hive.cpc;
import static org.apache.datasketches.Util.DEFAULT_UPDATE_SEED;
import org.apache.datasketches.cpc.CpcSketch;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
@@ -43,7 +43,7 @@ public class GetEstimateUDF extends UDF {
*/
public Double evaluate(final BytesWritable serializedSketch, final long seed) {
if (serializedSketch == null) { return null; }
- final CpcSketch sketch = CpcSketch.heapify(Memory.wrap(serializedSketch.getBytes()), seed);
+ final CpcSketch sketch = CpcSketch.heapify(BytesWritableHelper.wrapAsMemory(serializedSketch), seed);
return sketch.getEstimate();
}
diff --git a/src/main/java/org/apache/datasketches/hive/cpc/SketchEvaluator.java b/src/main/java/org/apache/datasketches/hive/cpc/SketchEvaluator.java
index f7ce24f..ca2e0a9 100644
--- a/src/main/java/org/apache/datasketches/hive/cpc/SketchEvaluator.java
+++ b/src/main/java/org/apache/datasketches/hive/cpc/SketchEvaluator.java
@@ -22,7 +22,7 @@ package org.apache.datasketches.hive.cpc;
import java.util.Arrays;
import org.apache.datasketches.cpc.CpcSketch;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
@@ -70,7 +70,7 @@ abstract class SketchEvaluator extends GenericUDAFEvaluator {
}
final BytesWritable serializedSketch = (BytesWritable) intermediateInspector_.getStructFieldData(
data, intermediateInspector_.getStructFieldRef(SKETCH_FIELD));
- state.update(CpcSketch.heapify(Memory.wrap(serializedSketch.getBytes()), state.getSeed()));
+ state.update(CpcSketch.heapify(BytesWritableHelper.wrapAsMemory(serializedSketch), state.getSeed()));
}
private void initializeState(final UnionState state, final Object data) {
diff --git a/src/main/java/org/apache/datasketches/hive/cpc/SketchState.java b/src/main/java/org/apache/datasketches/hive/cpc/SketchState.java
index 932880c..5206fe7 100644
--- a/src/main/java/org/apache/datasketches/hive/cpc/SketchState.java
+++ b/src/main/java/org/apache/datasketches/hive/cpc/SketchState.java
@@ -42,7 +42,7 @@ class SketchState extends State {
switch (objectInspector.getPrimitiveCategory()) {
case BINARY:
sketch_.update(PrimitiveObjectInspectorUtils.getBinary(data, objectInspector)
- .getBytes());
+ .copyBytes());
return;
case BYTE:
sketch_.update(PrimitiveObjectInspectorUtils.getByte(data, objectInspector));
diff --git a/src/main/java/org/apache/datasketches/hive/cpc/SketchToStringUDF.java b/src/main/java/org/apache/datasketches/hive/cpc/SketchToStringUDF.java
index e058c65..7a6d4cd 100644
--- a/src/main/java/org/apache/datasketches/hive/cpc/SketchToStringUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/cpc/SketchToStringUDF.java
@@ -22,7 +22,7 @@ package org.apache.datasketches.hive.cpc;
import static org.apache.datasketches.Util.DEFAULT_UPDATE_SEED;
import org.apache.datasketches.cpc.CpcSketch;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
@@ -42,7 +42,7 @@ public class SketchToStringUDF extends UDF {
*/
public String evaluate(final BytesWritable serializedSketch, final long seed) {
if (serializedSketch == null) { return null; }
- final CpcSketch sketch = CpcSketch.heapify(Memory.wrap(serializedSketch.getBytes()), seed);
+ final CpcSketch sketch = CpcSketch.heapify(BytesWritableHelper.wrapAsMemory(serializedSketch), seed);
return sketch.toString();
}
diff --git a/src/main/java/org/apache/datasketches/hive/cpc/UnionSketchUDF.java b/src/main/java/org/apache/datasketches/hive/cpc/UnionSketchUDF.java
index 2fb990c..cd9f9cd 100644
--- a/src/main/java/org/apache/datasketches/hive/cpc/UnionSketchUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/cpc/UnionSketchUDF.java
@@ -23,7 +23,7 @@ import static org.apache.datasketches.Util.DEFAULT_UPDATE_SEED;
import org.apache.datasketches.cpc.CpcSketch;
import org.apache.datasketches.cpc.CpcUnion;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
@@ -52,11 +52,11 @@ public class UnionSketchUDF extends UDF {
final CpcUnion union = new CpcUnion(lgK, seed);
if (firstSketch != null) {
- union.update(CpcSketch.heapify(Memory.wrap(firstSketch.getBytes()), seed));
+ union.update(CpcSketch.heapify(BytesWritableHelper.wrapAsMemory(firstSketch), seed));
}
if (secondSketch != null) {
- union.update(CpcSketch.heapify(Memory.wrap(secondSketch.getBytes()), seed));
+ union.update(CpcSketch.heapify(BytesWritableHelper.wrapAsMemory(secondSketch), seed));
}
return new BytesWritable(union.getResult().toByteArray());
diff --git a/src/main/java/org/apache/datasketches/hive/frequencies/GetFrequentItemsFromStringsSketchUDTF.java b/src/main/java/org/apache/datasketches/hive/frequencies/GetFrequentItemsFromStringsSketchUDTF.java
index 45cf09a..1247112 100644
--- a/src/main/java/org/apache/datasketches/hive/frequencies/GetFrequentItemsFromStringsSketchUDTF.java
+++ b/src/main/java/org/apache/datasketches/hive/frequencies/GetFrequentItemsFromStringsSketchUDTF.java
@@ -24,7 +24,7 @@ import java.util.Arrays;
import org.apache.datasketches.ArrayOfStringsSerDe;
import org.apache.datasketches.frequencies.ErrorType;
import org.apache.datasketches.frequencies.ItemsSketch;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
@@ -93,7 +93,7 @@ public class GetFrequentItemsFromStringsSketchUDTF extends GenericUDTF {
final BytesWritable serializedSketch =
(BytesWritable) inputObjectInspector.getPrimitiveWritableObject(data[0]);
final ItemsSketch<String> sketch = ItemsSketch.getInstance(
- Memory.wrap(serializedSketch.getBytes()), new ArrayOfStringsSerDe());
+ BytesWritableHelper.wrapAsMemory(serializedSketch), new ArrayOfStringsSerDe());
ErrorType errorType = ErrorType.NO_FALSE_POSITIVES;
if (data.length > 1) {
errorType = ErrorType.valueOf((String) errorTypeObjectInspector.getPrimitiveJavaObject(data[1]));
diff --git a/src/main/java/org/apache/datasketches/hive/frequencies/ItemsEvaluator.java b/src/main/java/org/apache/datasketches/hive/frequencies/ItemsEvaluator.java
index 0ca7ee9..4d5a629 100644
--- a/src/main/java/org/apache/datasketches/hive/frequencies/ItemsEvaluator.java
+++ b/src/main/java/org/apache/datasketches/hive/frequencies/ItemsEvaluator.java
@@ -21,6 +21,8 @@ package org.apache.datasketches.hive.frequencies;
import org.apache.datasketches.ArrayOfItemsSerDe;
import org.apache.datasketches.frequencies.ItemsSketch;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
+import org.apache.datasketches.memory.Memory;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -65,9 +67,9 @@ abstract class ItemsEvaluator<T> extends GenericUDAFEvaluator {
if (data == null) { return; }
@SuppressWarnings("unchecked")
final ItemsState<T> state = (ItemsState<T>) buf;
- final BytesWritable serializedSketch =
- (BytesWritable) inputObjectInspector.getPrimitiveWritableObject(data);
- state.update(serializedSketch.getBytes());
+ final Memory serializedSketch = BytesWritableHelper.wrapAsMemory(
+ (BytesWritable) inputObjectInspector.getPrimitiveWritableObject(data));
+ state.update(serializedSketch);
}
@SuppressWarnings("deprecation")
diff --git a/src/main/java/org/apache/datasketches/hive/frequencies/ItemsState.java b/src/main/java/org/apache/datasketches/hive/frequencies/ItemsState.java
index 965e928..09478c8 100644
--- a/src/main/java/org/apache/datasketches/hive/frequencies/ItemsState.java
+++ b/src/main/java/org/apache/datasketches/hive/frequencies/ItemsState.java
@@ -49,8 +49,8 @@ class ItemsState<T> extends AbstractAggregationBuffer {
sketch.update(value);
}
- void update(final byte[] serializedSketch) {
- final ItemsSketch<T> incomingSketch = ItemsSketch.getInstance(Memory.wrap(serializedSketch), serDe_);
+ void update(final Memory serializedSketch) {
+ final ItemsSketch<T> incomingSketch = ItemsSketch.getInstance(serializedSketch, serDe_);
if (sketch == null) {
sketch = incomingSketch;
} else {
diff --git a/src/main/java/org/apache/datasketches/hive/hll/SketchEvaluator.java b/src/main/java/org/apache/datasketches/hive/hll/SketchEvaluator.java
index c41265a..083f9ac 100644
--- a/src/main/java/org/apache/datasketches/hive/hll/SketchEvaluator.java
+++ b/src/main/java/org/apache/datasketches/hive/hll/SketchEvaluator.java
@@ -21,9 +21,9 @@ package org.apache.datasketches.hive.hll;
import java.util.Arrays;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.hll.HllSketch;
import org.apache.datasketches.hll.TgtHllType;
-import org.apache.datasketches.memory.Memory;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
@@ -72,7 +72,7 @@ abstract class SketchEvaluator extends GenericUDAFEvaluator {
}
final BytesWritable serializedSketch = (BytesWritable) intermediateInspector_.getStructFieldData(
data, intermediateInspector_.getStructFieldRef(SKETCH_FIELD));
- state.update(HllSketch.wrap(Memory.wrap(serializedSketch.getBytes())));
+ state.update(HllSketch.wrap(BytesWritableHelper.wrapAsMemory(serializedSketch)));
}
private void initializeState(final UnionState state, final Object data) {
diff --git a/src/main/java/org/apache/datasketches/hive/hll/SketchState.java b/src/main/java/org/apache/datasketches/hive/hll/SketchState.java
index 5da2f49..c002a47 100644
--- a/src/main/java/org/apache/datasketches/hive/hll/SketchState.java
+++ b/src/main/java/org/apache/datasketches/hive/hll/SketchState.java
@@ -44,7 +44,7 @@ class SketchState extends State {
switch (objectInspector.getPrimitiveCategory()) {
case BINARY:
sketch_.update(PrimitiveObjectInspectorUtils.getBinary(data, objectInspector)
- .getBytes());
+ .copyBytes());
return;
case BYTE:
sketch_.update(PrimitiveObjectInspectorUtils.getByte(data, objectInspector));
diff --git a/src/main/java/org/apache/datasketches/hive/hll/SketchToEstimateAndErrorBoundsUDF.java b/src/main/java/org/apache/datasketches/hive/hll/SketchToEstimateAndErrorBoundsUDF.java
index 905aaa9..fd32ee7 100644
--- a/src/main/java/org/apache/datasketches/hive/hll/SketchToEstimateAndErrorBoundsUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/hll/SketchToEstimateAndErrorBoundsUDF.java
@@ -22,8 +22,8 @@ package org.apache.datasketches.hive.hll;
import java.util.Arrays;
import java.util.List;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.hll.HllSketch;
-import org.apache.datasketches.memory.Memory;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
@@ -56,7 +56,7 @@ public class SketchToEstimateAndErrorBoundsUDF extends UDF {
*/
public List<Double> evaluate(final BytesWritable serializedSketch, final int kappa) {
if (serializedSketch == null) { return null; }
- final HllSketch sketch = HllSketch.wrap(Memory.wrap(serializedSketch.getBytes()));
+ final HllSketch sketch = HllSketch.wrap(BytesWritableHelper.wrapAsMemory(serializedSketch));
return Arrays.asList(sketch.getEstimate(), sketch.getLowerBound(kappa), sketch.getUpperBound(kappa));
}
diff --git a/src/main/java/org/apache/datasketches/hive/hll/SketchToEstimateUDF.java b/src/main/java/org/apache/datasketches/hive/hll/SketchToEstimateUDF.java
index 03a5d6d..899948f 100644
--- a/src/main/java/org/apache/datasketches/hive/hll/SketchToEstimateUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/hll/SketchToEstimateUDF.java
@@ -19,8 +19,8 @@
package org.apache.datasketches.hive.hll;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.hll.HllSketch;
-import org.apache.datasketches.memory.Memory;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
@@ -40,7 +40,7 @@ public class SketchToEstimateUDF extends UDF {
*/
public Double evaluate(final BytesWritable serializedSketch) {
if (serializedSketch == null) { return null; }
- final HllSketch sketch = HllSketch.wrap(Memory.wrap(serializedSketch.getBytes()));
+ final HllSketch sketch = HllSketch.wrap(BytesWritableHelper.wrapAsMemory(serializedSketch));
return sketch.getEstimate();
}
diff --git a/src/main/java/org/apache/datasketches/hive/hll/SketchToStringUDF.java b/src/main/java/org/apache/datasketches/hive/hll/SketchToStringUDF.java
index 33b3ad2..1c9a0f5 100644
--- a/src/main/java/org/apache/datasketches/hive/hll/SketchToStringUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/hll/SketchToStringUDF.java
@@ -19,8 +19,8 @@
package org.apache.datasketches.hive.hll;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.hll.HllSketch;
-import org.apache.datasketches.memory.Memory;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
@@ -39,7 +39,7 @@ public class SketchToStringUDF extends UDF {
*/
public String evaluate(final BytesWritable serializedSketch) {
if (serializedSketch == null) { return null; }
- final HllSketch sketch = HllSketch.wrap(Memory.wrap(serializedSketch.getBytes()));
+ final HllSketch sketch = HllSketch.wrap(BytesWritableHelper.wrapAsMemory(serializedSketch));
return sketch.toString();
}
diff --git a/src/main/java/org/apache/datasketches/hive/hll/UnionSketchUDF.java b/src/main/java/org/apache/datasketches/hive/hll/UnionSketchUDF.java
index 25cf218..4d0e1b7 100644
--- a/src/main/java/org/apache/datasketches/hive/hll/UnionSketchUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/hll/UnionSketchUDF.java
@@ -19,10 +19,10 @@
package org.apache.datasketches.hive.hll;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.hll.HllSketch;
import org.apache.datasketches.hll.TgtHllType;
import org.apache.datasketches.hll.Union;
-import org.apache.datasketches.memory.Memory;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
@@ -52,11 +52,11 @@ public class UnionSketchUDF extends UDF {
final Union union = new Union(lgK);
if (firstSketch != null) {
- union.update(HllSketch.wrap(Memory.wrap(firstSketch.getBytes())));
+ union.update(HllSketch.wrap(BytesWritableHelper.wrapAsMemory(firstSketch)));
}
if (secondSketch != null) {
- union.update(HllSketch.wrap(Memory.wrap(secondSketch.getBytes())));
+ union.update(HllSketch.wrap(BytesWritableHelper.wrapAsMemory(secondSketch)));
}
return new BytesWritable(union.getResult(hllType).toCompactByteArray());
diff --git a/src/main/java/org/apache/datasketches/hive/hll/UnionState.java b/src/main/java/org/apache/datasketches/hive/hll/UnionState.java
index 8cd82e4..89d8e06 100644
--- a/src/main/java/org/apache/datasketches/hive/hll/UnionState.java
+++ b/src/main/java/org/apache/datasketches/hive/hll/UnionState.java
@@ -45,7 +45,7 @@ class UnionState extends State {
switch (objectInspector.getPrimitiveCategory()) {
case BINARY:
union_.update(PrimitiveObjectInspectorUtils.getBinary(data, objectInspector)
- .getBytes());
+ .copyBytes());
return;
case BYTE:
union_.update(PrimitiveObjectInspectorUtils.getByte(data, objectInspector));
diff --git a/src/main/java/org/apache/datasketches/hive/kll/GetCdfUDF.java b/src/main/java/org/apache/datasketches/hive/kll/GetCdfUDF.java
index b6131ff..12fa3fc 100644
--- a/src/main/java/org/apache/datasketches/hive/kll/GetCdfUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/kll/GetCdfUDF.java
@@ -21,8 +21,8 @@ package org.apache.datasketches.hive.kll;
import java.util.List;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.kll.KllFloatsSketch;
-import org.apache.datasketches.memory.Memory;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
@@ -49,7 +49,7 @@ public class GetCdfUDF extends UDF {
*/
public List<Double> evaluate(final BytesWritable serializedSketch, final Float... splitPoints) {
if (serializedSketch == null) { return null; }
- final KllFloatsSketch sketch = KllFloatsSketch.heapify(Memory.wrap(serializedSketch.getBytes()));
+ final KllFloatsSketch sketch = KllFloatsSketch.heapify(BytesWritableHelper.wrapAsMemory(serializedSketch));
final double[] cdf = sketch.getCDF(Util.objectsToPrimitives(splitPoints));
if (cdf == null) { return null; }
return Util.primitivesToList(cdf);
diff --git a/src/main/java/org/apache/datasketches/hive/kll/GetNUDF.java b/src/main/java/org/apache/datasketches/hive/kll/GetNUDF.java
index 800d055..e129c18 100644
--- a/src/main/java/org/apache/datasketches/hive/kll/GetNUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/kll/GetNUDF.java
@@ -19,8 +19,8 @@
package org.apache.datasketches.hive.kll;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.kll.KllFloatsSketch;
-import org.apache.datasketches.memory.Memory;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
@@ -37,7 +37,7 @@ public class GetNUDF extends UDF {
*/
public Long evaluate(final BytesWritable serializedSketch) {
if (serializedSketch == null) { return null; }
- final KllFloatsSketch sketch = KllFloatsSketch.heapify(Memory.wrap(serializedSketch.getBytes()));
+ final KllFloatsSketch sketch = KllFloatsSketch.heapify(BytesWritableHelper.wrapAsMemory(serializedSketch));
return sketch.getN();
}
diff --git a/src/main/java/org/apache/datasketches/hive/kll/GetPmfUDF.java b/src/main/java/org/apache/datasketches/hive/kll/GetPmfUDF.java
index 490e2b6..b30ec31 100644
--- a/src/main/java/org/apache/datasketches/hive/kll/GetPmfUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/kll/GetPmfUDF.java
@@ -21,8 +21,8 @@ package org.apache.datasketches.hive.kll;
import java.util.List;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.kll.KllFloatsSketch;
-import org.apache.datasketches.memory.Memory;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
@@ -49,7 +49,7 @@ public class GetPmfUDF extends UDF {
*/
public List<Double> evaluate(final BytesWritable serializedSketch, final Float... splitPoints) {
if (serializedSketch == null) { return null; }
- final KllFloatsSketch sketch = KllFloatsSketch.heapify(Memory.wrap(serializedSketch.getBytes()));
+ final KllFloatsSketch sketch = KllFloatsSketch.heapify(BytesWritableHelper.wrapAsMemory(serializedSketch));
final double[] pmf = sketch.getPMF(Util.objectsToPrimitives(splitPoints));
if (pmf == null) { return null; }
return Util.primitivesToList(pmf);
diff --git a/src/main/java/org/apache/datasketches/hive/kll/GetQuantileUDF.java b/src/main/java/org/apache/datasketches/hive/kll/GetQuantileUDF.java
index ee20976..966936e 100644
--- a/src/main/java/org/apache/datasketches/hive/kll/GetQuantileUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/kll/GetQuantileUDF.java
@@ -19,8 +19,8 @@
package org.apache.datasketches.hive.kll;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.kll.KllFloatsSketch;
-import org.apache.datasketches.memory.Memory;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
@@ -43,7 +43,7 @@ public class GetQuantileUDF extends UDF {
*/
public Float evaluate(final BytesWritable serializedSketch, final double fraction) {
if (serializedSketch == null) { return null; }
- final KllFloatsSketch sketch = KllFloatsSketch.heapify(Memory.wrap(serializedSketch.getBytes()));
+ final KllFloatsSketch sketch = KllFloatsSketch.heapify(BytesWritableHelper.wrapAsMemory(serializedSketch));
return sketch.getQuantile(fraction);
}
diff --git a/src/main/java/org/apache/datasketches/hive/kll/GetQuantilesUDF.java b/src/main/java/org/apache/datasketches/hive/kll/GetQuantilesUDF.java
index 951e777..599bcd0 100644
--- a/src/main/java/org/apache/datasketches/hive/kll/GetQuantilesUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/kll/GetQuantilesUDF.java
@@ -21,8 +21,8 @@ package org.apache.datasketches.hive.kll;
import java.util.List;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.kll.KllFloatsSketch;
-import org.apache.datasketches.memory.Memory;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
@@ -46,7 +46,7 @@ public class GetQuantilesUDF extends UDF {
*/
public List<Float> evaluate(final BytesWritable serializedSketch, final Double... fractions) {
if (serializedSketch == null) { return null; }
- final KllFloatsSketch sketch = KllFloatsSketch.heapify(Memory.wrap(serializedSketch.getBytes()));
+ final KllFloatsSketch sketch = KllFloatsSketch.heapify(BytesWritableHelper.wrapAsMemory(serializedSketch));
return Util.primitivesToList(sketch.getQuantiles(Util.objectsToPrimitives(fractions)));
}
diff --git a/src/main/java/org/apache/datasketches/hive/kll/GetRankUDF.java b/src/main/java/org/apache/datasketches/hive/kll/GetRankUDF.java
index 2181ac6..3b9ef18 100644
--- a/src/main/java/org/apache/datasketches/hive/kll/GetRankUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/kll/GetRankUDF.java
@@ -19,8 +19,8 @@
package org.apache.datasketches.hive.kll;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.kll.KllFloatsSketch;
-import org.apache.datasketches.memory.Memory;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
@@ -40,7 +40,7 @@ public class GetRankUDF extends UDF {
*/
public Double evaluate(final BytesWritable serializedSketch, final float value) {
if (serializedSketch == null) { return null; }
- final KllFloatsSketch sketch = KllFloatsSketch.heapify(Memory.wrap(serializedSketch.getBytes()));
+ final KllFloatsSketch sketch = KllFloatsSketch.heapify(BytesWritableHelper.wrapAsMemory(serializedSketch));
return sketch.getRank(value);
}
diff --git a/src/main/java/org/apache/datasketches/hive/kll/SketchEvaluator.java b/src/main/java/org/apache/datasketches/hive/kll/SketchEvaluator.java
index bef19fc..d5ec827 100644
--- a/src/main/java/org/apache/datasketches/hive/kll/SketchEvaluator.java
+++ b/src/main/java/org/apache/datasketches/hive/kll/SketchEvaluator.java
@@ -19,6 +19,7 @@
package org.apache.datasketches.hive.kll;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.kll.KllFloatsSketch;
import org.apache.datasketches.memory.Memory;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -69,9 +70,9 @@ abstract class SketchEvaluator extends GenericUDAFEvaluator {
public void merge(final AggregationBuffer buf, final Object data) throws HiveException {
if (data == null) { return; }
final SketchState state = (SketchState) buf;
- final BytesWritable serializedSketch =
- (BytesWritable) inputInspector_.getPrimitiveWritableObject(data);
- state.update(KllFloatsSketch.heapify(Memory.wrap(serializedSketch.getBytes())));
+ final Memory serializedSketch = BytesWritableHelper.wrapAsMemory(
+ (BytesWritable) inputInspector_.getPrimitiveWritableObject(data));
+ state.update(KllFloatsSketch.heapify(serializedSketch));
}
@SuppressWarnings("deprecation")
diff --git a/src/main/java/org/apache/datasketches/hive/kll/SketchToStringUDF.java b/src/main/java/org/apache/datasketches/hive/kll/SketchToStringUDF.java
index e66882e..80151b2 100644
--- a/src/main/java/org/apache/datasketches/hive/kll/SketchToStringUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/kll/SketchToStringUDF.java
@@ -19,8 +19,8 @@
package org.apache.datasketches.hive.kll;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.kll.KllFloatsSketch;
-import org.apache.datasketches.memory.Memory;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
@@ -37,7 +37,7 @@ public class SketchToStringUDF extends UDF {
*/
public String evaluate(final BytesWritable serializedSketch) {
if (serializedSketch == null) { return null; }
- final KllFloatsSketch sketch = KllFloatsSketch.heapify(Memory.wrap(serializedSketch.getBytes()));
+ final KllFloatsSketch sketch = KllFloatsSketch.heapify(BytesWritableHelper.wrapAsMemory(serializedSketch));
return sketch.toString();
}
diff --git a/src/main/java/org/apache/datasketches/hive/quantiles/DoublesEvaluator.java b/src/main/java/org/apache/datasketches/hive/quantiles/DoublesEvaluator.java
index afdea0a..62b7947 100644
--- a/src/main/java/org/apache/datasketches/hive/quantiles/DoublesEvaluator.java
+++ b/src/main/java/org/apache/datasketches/hive/quantiles/DoublesEvaluator.java
@@ -19,6 +19,8 @@
package org.apache.datasketches.hive.quantiles;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
+import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.quantiles.DoublesSketch;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
@@ -68,9 +70,9 @@ abstract class DoublesEvaluator extends GenericUDAFEvaluator {
public void merge(final AggregationBuffer buf, final Object data) throws HiveException {
if (data == null) { return; }
final DoublesUnionState state = (DoublesUnionState) buf;
- final BytesWritable serializedSketch =
- (BytesWritable) inputObjectInspector.getPrimitiveWritableObject(data);
- state.update(serializedSketch.getBytes());
+ final Memory serializedSketch = BytesWritableHelper.wrapAsMemory(
+ (BytesWritable) inputObjectInspector.getPrimitiveWritableObject(data));
+ state.update(serializedSketch);
}
@SuppressWarnings("deprecation")
diff --git a/src/main/java/org/apache/datasketches/hive/quantiles/DoublesSketchToStringUDF.java b/src/main/java/org/apache/datasketches/hive/quantiles/DoublesSketchToStringUDF.java
index 11c497a..2e4af0c 100644
--- a/src/main/java/org/apache/datasketches/hive/quantiles/DoublesSketchToStringUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/quantiles/DoublesSketchToStringUDF.java
@@ -19,7 +19,7 @@
package org.apache.datasketches.hive.quantiles;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.quantiles.DoublesSketch;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
@@ -37,7 +37,7 @@ public class DoublesSketchToStringUDF extends UDF {
*/
public String evaluate(final BytesWritable serializedSketch) {
if (serializedSketch == null) { return null; }
- final DoublesSketch sketch = DoublesSketch.wrap(Memory.wrap(serializedSketch.getBytes()));
+ final DoublesSketch sketch = DoublesSketch.wrap(BytesWritableHelper.wrapAsMemory(serializedSketch));
return sketch.toString();
}
diff --git a/src/main/java/org/apache/datasketches/hive/quantiles/DoublesUnionState.java b/src/main/java/org/apache/datasketches/hive/quantiles/DoublesUnionState.java
index 2b2fe8b..653f2ce 100644
--- a/src/main/java/org/apache/datasketches/hive/quantiles/DoublesUnionState.java
+++ b/src/main/java/org/apache/datasketches/hive/quantiles/DoublesUnionState.java
@@ -47,8 +47,8 @@ class DoublesUnionState extends AbstractAggregationBuffer {
union.update(value);
}
- void update(final byte[] serializedSketch) {
- final DoublesSketch incomingSketch = DoublesSketch.wrap(Memory.wrap(serializedSketch));
+ void update(final Memory serializedSketch) {
+ final DoublesSketch incomingSketch = DoublesSketch.wrap(serializedSketch);
if (union == null) {
union = DoublesUnion.heapify(incomingSketch);
} else {
diff --git a/src/main/java/org/apache/datasketches/hive/quantiles/GetCdfFromDoublesSketchUDF.java b/src/main/java/org/apache/datasketches/hive/quantiles/GetCdfFromDoublesSketchUDF.java
index e747a8d..2e0fabb 100644
--- a/src/main/java/org/apache/datasketches/hive/quantiles/GetCdfFromDoublesSketchUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/quantiles/GetCdfFromDoublesSketchUDF.java
@@ -21,7 +21,7 @@ package org.apache.datasketches.hive.quantiles;
import java.util.List;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.quantiles.DoublesSketch;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
@@ -49,7 +49,7 @@ public class GetCdfFromDoublesSketchUDF extends UDF {
*/
public List<Double> evaluate(final BytesWritable serializedSketch, final Double... splitPoints) {
if (serializedSketch == null) { return null; }
- final DoublesSketch sketch = DoublesSketch.wrap(Memory.wrap(serializedSketch.getBytes()));
+ final DoublesSketch sketch = DoublesSketch.wrap(BytesWritableHelper.wrapAsMemory(serializedSketch));
final double[] cdf = sketch.getCDF(Util.objectsToPrimitives(splitPoints));
if (cdf == null) { return null; }
return Util.primitivesToList(cdf);
diff --git a/src/main/java/org/apache/datasketches/hive/quantiles/GetCdfFromStringsSketchUDF.java b/src/main/java/org/apache/datasketches/hive/quantiles/GetCdfFromStringsSketchUDF.java
index 3722cd0..6f2485f 100644
--- a/src/main/java/org/apache/datasketches/hive/quantiles/GetCdfFromStringsSketchUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/quantiles/GetCdfFromStringsSketchUDF.java
@@ -23,7 +23,7 @@ import java.util.Comparator;
import java.util.List;
import org.apache.datasketches.ArrayOfStringsSerDe;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.quantiles.ItemsSketch;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
@@ -52,7 +52,7 @@ public class GetCdfFromStringsSketchUDF extends UDF {
public List<Double> evaluate(final BytesWritable serializedSketch, final String... splitPoints) {
if (serializedSketch == null) { return null; }
final ItemsSketch<String> sketch = ItemsSketch.getInstance(
- Memory.wrap(serializedSketch.getBytes()),
+ BytesWritableHelper.wrapAsMemory(serializedSketch),
Comparator.naturalOrder(),
new ArrayOfStringsSerDe()
);
diff --git a/src/main/java/org/apache/datasketches/hive/quantiles/GetKFromDoublesSketchUDF.java b/src/main/java/org/apache/datasketches/hive/quantiles/GetKFromDoublesSketchUDF.java
index 9ba75e9..740b383 100644
--- a/src/main/java/org/apache/datasketches/hive/quantiles/GetKFromDoublesSketchUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/quantiles/GetKFromDoublesSketchUDF.java
@@ -19,7 +19,7 @@
package org.apache.datasketches.hive.quantiles;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.quantiles.DoublesSketch;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
@@ -37,7 +37,7 @@ public class GetKFromDoublesSketchUDF extends UDF {
*/
public Integer evaluate(final BytesWritable serializedSketch) {
if (serializedSketch == null) { return null; }
- final DoublesSketch sketch = DoublesSketch.wrap(Memory.wrap(serializedSketch.getBytes()));
+ final DoublesSketch sketch = DoublesSketch.wrap(BytesWritableHelper.wrapAsMemory(serializedSketch));
return sketch.getK();
}
diff --git a/src/main/java/org/apache/datasketches/hive/quantiles/GetKFromStringsSketchUDF.java b/src/main/java/org/apache/datasketches/hive/quantiles/GetKFromStringsSketchUDF.java
index e0366bf..74c4fb6 100644
--- a/src/main/java/org/apache/datasketches/hive/quantiles/GetKFromStringsSketchUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/quantiles/GetKFromStringsSketchUDF.java
@@ -22,7 +22,7 @@ package org.apache.datasketches.hive.quantiles;
import java.util.Comparator;
import org.apache.datasketches.ArrayOfStringsSerDe;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.quantiles.ItemsSketch;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
@@ -41,7 +41,7 @@ public class GetKFromStringsSketchUDF extends UDF {
public Integer evaluate(final BytesWritable serializedSketch) {
if (serializedSketch == null) { return null; }
final ItemsSketch<String> sketch = ItemsSketch.getInstance(
- Memory.wrap(serializedSketch.getBytes()),
+ BytesWritableHelper.wrapAsMemory(serializedSketch),
Comparator.naturalOrder(),
new ArrayOfStringsSerDe()
);
diff --git a/src/main/java/org/apache/datasketches/hive/quantiles/GetNFromDoublesSketchUDF.java b/src/main/java/org/apache/datasketches/hive/quantiles/GetNFromDoublesSketchUDF.java
index fce02b5..62a953d 100644
--- a/src/main/java/org/apache/datasketches/hive/quantiles/GetNFromDoublesSketchUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/quantiles/GetNFromDoublesSketchUDF.java
@@ -19,7 +19,7 @@
package org.apache.datasketches.hive.quantiles;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.quantiles.DoublesSketch;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
@@ -37,7 +37,7 @@ public class GetNFromDoublesSketchUDF extends UDF {
*/
public Long evaluate(final BytesWritable serializedSketch) {
if (serializedSketch == null) { return null; }
- final DoublesSketch sketch = DoublesSketch.wrap(Memory.wrap(serializedSketch.getBytes()));
+ final DoublesSketch sketch = DoublesSketch.wrap(BytesWritableHelper.wrapAsMemory(serializedSketch));
return sketch.getN();
}
diff --git a/src/main/java/org/apache/datasketches/hive/quantiles/GetNFromStringsSketchUDF.java b/src/main/java/org/apache/datasketches/hive/quantiles/GetNFromStringsSketchUDF.java
index 860e3b7..3dc3b23 100644
--- a/src/main/java/org/apache/datasketches/hive/quantiles/GetNFromStringsSketchUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/quantiles/GetNFromStringsSketchUDF.java
@@ -22,7 +22,7 @@ package org.apache.datasketches.hive.quantiles;
import java.util.Comparator;
import org.apache.datasketches.ArrayOfStringsSerDe;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.quantiles.ItemsSketch;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
@@ -42,7 +42,7 @@ public class GetNFromStringsSketchUDF extends UDF {
public Long evaluate(final BytesWritable serializedSketch) {
if (serializedSketch == null) { return null; }
final ItemsSketch<String> sketch = ItemsSketch.getInstance(
- Memory.wrap(serializedSketch.getBytes()),
+ BytesWritableHelper.wrapAsMemory(serializedSketch),
Comparator.naturalOrder(),
new ArrayOfStringsSerDe()
);
diff --git a/src/main/java/org/apache/datasketches/hive/quantiles/GetPmfFromDoublesSketchUDF.java b/src/main/java/org/apache/datasketches/hive/quantiles/GetPmfFromDoublesSketchUDF.java
index c54a572..f37ee7e 100644
--- a/src/main/java/org/apache/datasketches/hive/quantiles/GetPmfFromDoublesSketchUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/quantiles/GetPmfFromDoublesSketchUDF.java
@@ -21,7 +21,7 @@ package org.apache.datasketches.hive.quantiles;
import java.util.List;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.quantiles.DoublesSketch;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
@@ -49,7 +49,7 @@ public class GetPmfFromDoublesSketchUDF extends UDF {
*/
public List<Double> evaluate(final BytesWritable serializedSketch, final Double... splitPoints) {
if (serializedSketch == null) { return null; }
- final DoublesSketch sketch = DoublesSketch.wrap(Memory.wrap(serializedSketch.getBytes()));
+ final DoublesSketch sketch = DoublesSketch.wrap(BytesWritableHelper.wrapAsMemory(serializedSketch));
final double[] pmf = sketch.getPMF(Util.objectsToPrimitives(splitPoints));
if (pmf == null) { return null; }
return Util.primitivesToList(pmf);
diff --git a/src/main/java/org/apache/datasketches/hive/quantiles/GetPmfFromStringsSketchUDF.java b/src/main/java/org/apache/datasketches/hive/quantiles/GetPmfFromStringsSketchUDF.java
index 1fb1305..aa05962 100644
--- a/src/main/java/org/apache/datasketches/hive/quantiles/GetPmfFromStringsSketchUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/quantiles/GetPmfFromStringsSketchUDF.java
@@ -23,7 +23,7 @@ import java.util.Comparator;
import java.util.List;
import org.apache.datasketches.ArrayOfStringsSerDe;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.quantiles.ItemsSketch;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
@@ -52,7 +52,7 @@ public class GetPmfFromStringsSketchUDF extends UDF {
public List<Double> evaluate(final BytesWritable serializedSketch, final String... splitPoints) {
if (serializedSketch == null) { return null; }
final ItemsSketch<String> sketch = ItemsSketch.getInstance(
- Memory.wrap(serializedSketch.getBytes()),
+ BytesWritableHelper.wrapAsMemory(serializedSketch),
Comparator.naturalOrder(),
new ArrayOfStringsSerDe()
);
diff --git a/src/main/java/org/apache/datasketches/hive/quantiles/GetQuantileFromDoublesSketchUDF.java b/src/main/java/org/apache/datasketches/hive/quantiles/GetQuantileFromDoublesSketchUDF.java
index 5339309..502d1ba 100644
--- a/src/main/java/org/apache/datasketches/hive/quantiles/GetQuantileFromDoublesSketchUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/quantiles/GetQuantileFromDoublesSketchUDF.java
@@ -19,7 +19,7 @@
package org.apache.datasketches.hive.quantiles;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.quantiles.DoublesSketch;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
@@ -43,7 +43,7 @@ public class GetQuantileFromDoublesSketchUDF extends UDF {
*/
public Double evaluate(final BytesWritable serializedSketch, final double fraction) {
if (serializedSketch == null) { return null; }
- final DoublesSketch sketch = DoublesSketch.wrap(Memory.wrap(serializedSketch.getBytes()));
+ final DoublesSketch sketch = DoublesSketch.wrap(BytesWritableHelper.wrapAsMemory(serializedSketch));
return sketch.getQuantile(fraction);
}
diff --git a/src/main/java/org/apache/datasketches/hive/quantiles/GetQuantileFromStringsSketchUDF.java b/src/main/java/org/apache/datasketches/hive/quantiles/GetQuantileFromStringsSketchUDF.java
index 768bdaf..d9054e3 100644
--- a/src/main/java/org/apache/datasketches/hive/quantiles/GetQuantileFromStringsSketchUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/quantiles/GetQuantileFromStringsSketchUDF.java
@@ -22,7 +22,7 @@ package org.apache.datasketches.hive.quantiles;
import java.util.Comparator;
import org.apache.datasketches.ArrayOfStringsSerDe;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.quantiles.ItemsSketch;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
@@ -47,7 +47,7 @@ public class GetQuantileFromStringsSketchUDF extends UDF {
public String evaluate(final BytesWritable serializedSketch, final double fraction) {
if (serializedSketch == null) { return null; }
final ItemsSketch<String> sketch = ItemsSketch.getInstance(
- Memory.wrap(serializedSketch.getBytes()),
+ BytesWritableHelper.wrapAsMemory(serializedSketch),
Comparator.naturalOrder(),
new ArrayOfStringsSerDe()
);
diff --git a/src/main/java/org/apache/datasketches/hive/quantiles/GetQuantilesFromDoublesSketchUDF.java b/src/main/java/org/apache/datasketches/hive/quantiles/GetQuantilesFromDoublesSketchUDF.java
index 5c10be2..c06be09 100644
--- a/src/main/java/org/apache/datasketches/hive/quantiles/GetQuantilesFromDoublesSketchUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/quantiles/GetQuantilesFromDoublesSketchUDF.java
@@ -21,7 +21,7 @@ package org.apache.datasketches.hive.quantiles;
import java.util.List;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.quantiles.DoublesSketch;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
@@ -51,7 +51,7 @@ public class GetQuantilesFromDoublesSketchUDF extends UDF {
*/
public List<Double> evaluate(final BytesWritable serializedSketch, final Double... fractions) {
if (serializedSketch == null) { return null; }
- final DoublesSketch sketch = DoublesSketch.wrap(Memory.wrap(serializedSketch.getBytes()));
+ final DoublesSketch sketch = DoublesSketch.wrap(BytesWritableHelper.wrapAsMemory(serializedSketch));
return Util.primitivesToList(sketch.getQuantiles(Util.objectsToPrimitives(fractions)));
}
@@ -63,7 +63,7 @@ public class GetQuantilesFromDoublesSketchUDF extends UDF {
*/
public List<Double> evaluate(final BytesWritable serializedSketch, final int number) {
if (serializedSketch == null) { return null; }
- final DoublesSketch sketch = DoublesSketch.wrap(Memory.wrap(serializedSketch.getBytes()));
+ final DoublesSketch sketch = DoublesSketch.wrap(BytesWritableHelper.wrapAsMemory(serializedSketch));
final double[] quantiles = sketch.getQuantiles(number);
if (quantiles == null) { return null; }
return Util.primitivesToList(quantiles);
diff --git a/src/main/java/org/apache/datasketches/hive/quantiles/GetQuantilesFromStringsSketchUDF.java b/src/main/java/org/apache/datasketches/hive/quantiles/GetQuantilesFromStringsSketchUDF.java
index c079126..161f019 100644
--- a/src/main/java/org/apache/datasketches/hive/quantiles/GetQuantilesFromStringsSketchUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/quantiles/GetQuantilesFromStringsSketchUDF.java
@@ -24,7 +24,7 @@ import java.util.Comparator;
import java.util.List;
import org.apache.datasketches.ArrayOfStringsSerDe;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.quantiles.ItemsSketch;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
@@ -55,7 +55,7 @@ public class GetQuantilesFromStringsSketchUDF extends UDF {
public List<String> evaluate(final BytesWritable serializedSketch, final Double... fractions) {
if (serializedSketch == null) { return null; }
final ItemsSketch<String> sketch = ItemsSketch.getInstance(
- Memory.wrap(serializedSketch.getBytes()),
+ BytesWritableHelper.wrapAsMemory(serializedSketch),
Comparator.naturalOrder(),
new ArrayOfStringsSerDe()
);
@@ -71,7 +71,7 @@ public class GetQuantilesFromStringsSketchUDF extends UDF {
public List<String> evaluate(final BytesWritable serializedSketch, final int number) {
if (serializedSketch == null) { return null; }
final ItemsSketch<String> sketch = ItemsSketch.getInstance(
- Memory.wrap(serializedSketch.getBytes()),
+ BytesWritableHelper.wrapAsMemory(serializedSketch),
Comparator.naturalOrder(),
new ArrayOfStringsSerDe()
);
diff --git a/src/main/java/org/apache/datasketches/hive/quantiles/ItemsEvaluator.java b/src/main/java/org/apache/datasketches/hive/quantiles/ItemsEvaluator.java
index 954a3e7..b2343ef 100644
--- a/src/main/java/org/apache/datasketches/hive/quantiles/ItemsEvaluator.java
+++ b/src/main/java/org/apache/datasketches/hive/quantiles/ItemsEvaluator.java
@@ -22,6 +22,8 @@ package org.apache.datasketches.hive.quantiles;
import java.util.Comparator;
import org.apache.datasketches.ArrayOfItemsSerDe;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
+import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.quantiles.ItemsSketch;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
@@ -80,9 +82,9 @@ abstract class ItemsEvaluator<T> extends GenericUDAFEvaluator {
if (data == null) { return; }
@SuppressWarnings("unchecked")
final ItemsUnionState<T> state = (ItemsUnionState<T>) buf;
- final BytesWritable serializedSketch =
- (BytesWritable) inputObjectInspector.getPrimitiveWritableObject(data);
- state.update(serializedSketch.getBytes());
+ final Memory serializedSketch = BytesWritableHelper.wrapAsMemory(
+ (BytesWritable) inputObjectInspector.getPrimitiveWritableObject(data));
+ state.update(serializedSketch);
}
@SuppressWarnings("deprecation")
diff --git a/src/main/java/org/apache/datasketches/hive/quantiles/ItemsUnionState.java b/src/main/java/org/apache/datasketches/hive/quantiles/ItemsUnionState.java
index e98e48b..9e7ee31 100644
--- a/src/main/java/org/apache/datasketches/hive/quantiles/ItemsUnionState.java
+++ b/src/main/java/org/apache/datasketches/hive/quantiles/ItemsUnionState.java
@@ -58,9 +58,9 @@ class ItemsUnionState<T> extends AbstractAggregationBuffer {
union.update(value);
}
- void update(final byte[] serializedSketch) {
+ void update(final Memory serializedSketch) {
final ItemsSketch<T> incomingSketch =
- ItemsSketch.getInstance(Memory.wrap(serializedSketch), comparator_, serDe_);
+ ItemsSketch.getInstance(serializedSketch, comparator_, serDe_);
if (union == null) {
union = ItemsUnion.getInstance(incomingSketch);
} else {
diff --git a/src/main/java/org/apache/datasketches/hive/quantiles/StringsSketchToStringUDF.java b/src/main/java/org/apache/datasketches/hive/quantiles/StringsSketchToStringUDF.java
index 551e06d..3d5211a 100644
--- a/src/main/java/org/apache/datasketches/hive/quantiles/StringsSketchToStringUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/quantiles/StringsSketchToStringUDF.java
@@ -22,7 +22,7 @@ package org.apache.datasketches.hive.quantiles;
import java.util.Comparator;
import org.apache.datasketches.ArrayOfStringsSerDe;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.quantiles.ItemsSketch;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
@@ -41,7 +41,7 @@ public class StringsSketchToStringUDF extends UDF {
public String evaluate(final BytesWritable serializedSketch) {
if (serializedSketch == null) { return null; }
final ItemsSketch<String> sketch = ItemsSketch.getInstance(
- Memory.wrap(serializedSketch.getBytes()),
+ BytesWritableHelper.wrapAsMemory(serializedSketch),
Comparator.naturalOrder(),
new ArrayOfStringsSerDe()
);
diff --git a/src/main/java/org/apache/datasketches/hive/theta/EstimateSketchUDF.java b/src/main/java/org/apache/datasketches/hive/theta/EstimateSketchUDF.java
index dd4c1a7..ce357b6 100644
--- a/src/main/java/org/apache/datasketches/hive/theta/EstimateSketchUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/theta/EstimateSketchUDF.java
@@ -21,11 +21,14 @@ package org.apache.datasketches.hive.theta;
import static org.apache.datasketches.Util.DEFAULT_UPDATE_SEED;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.theta.Sketch;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
+import java.nio.ByteOrder;
+
/**
* Hive estimate sketch udf. V4
*
@@ -56,13 +59,13 @@ public class EstimateSketchUDF extends UDF {
return 0.0;
}
- final byte[] serializedSketch = binarySketch.getBytes();
+ final Memory serializedSketch = BytesWritableHelper.wrapAsMemory(binarySketch);
- if (serializedSketch.length <= EMPTY_SKETCH_SIZE_BYTES) {
+ if (serializedSketch.getCapacity() <= EMPTY_SKETCH_SIZE_BYTES) {
return 0.0;
}
- return Sketch.wrap(Memory.wrap(serializedSketch), seed).getEstimate();
+ return Sketch.wrap(serializedSketch, seed).getEstimate();
}
}
diff --git a/src/main/java/org/apache/datasketches/hive/theta/ExcludeSketchUDF.java b/src/main/java/org/apache/datasketches/hive/theta/ExcludeSketchUDF.java
index 778e5fe..2b5b2d0 100644
--- a/src/main/java/org/apache/datasketches/hive/theta/ExcludeSketchUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/theta/ExcludeSketchUDF.java
@@ -21,7 +21,7 @@ package org.apache.datasketches.hive.theta;
import static org.apache.datasketches.Util.DEFAULT_UPDATE_SEED;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.theta.AnotB;
import org.apache.datasketches.theta.SetOperation;
import org.apache.datasketches.theta.Sketch;
@@ -52,12 +52,12 @@ public class ExcludeSketchUDF extends UDF {
Sketch firstSketch = null;
if (firstSketchBytes != null && firstSketchBytes.getLength() > 0) {
- firstSketch = Sketch.wrap(Memory.wrap(firstSketchBytes.getBytes()), hashSeed);
+ firstSketch = Sketch.wrap(BytesWritableHelper.wrapAsMemory(firstSketchBytes), hashSeed);
}
Sketch secondSketch = null;
if (secondSketchBytes != null && secondSketchBytes.getLength() > 0) {
- secondSketch = Sketch.wrap(Memory.wrap(secondSketchBytes.getBytes()), hashSeed);
+ secondSketch = Sketch.wrap(BytesWritableHelper.wrapAsMemory(secondSketchBytes), hashSeed);
}
final AnotB anotb = SetOperation.builder().setSeed(hashSeed).buildANotB();
diff --git a/src/main/java/org/apache/datasketches/hive/theta/IntersectSketchUDAF.java b/src/main/java/org/apache/datasketches/hive/theta/IntersectSketchUDAF.java
index 8a1f626..cda0f75 100644
--- a/src/main/java/org/apache/datasketches/hive/theta/IntersectSketchUDAF.java
+++ b/src/main/java/org/apache/datasketches/hive/theta/IntersectSketchUDAF.java
@@ -23,6 +23,7 @@ import static org.apache.datasketches.Util.DEFAULT_UPDATE_SEED;
import java.util.Arrays;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.theta.Intersection;
import org.apache.datasketches.theta.SetOperation;
@@ -131,7 +132,7 @@ public class IntersectSketchUDAF extends AbstractGenericUDAFResolver {
}
final byte[] serializedSketch = (byte[]) inputObjectInspector.getPrimitiveJavaObject(data[0]);
if (serializedSketch == null) { return; }
- state.update(serializedSketch);
+ state.update(Memory.wrap(serializedSketch));
}
@Override
@@ -158,10 +159,10 @@ public class IntersectSketchUDAF extends AbstractGenericUDAFResolver {
state.init(seed);
}
- final BytesWritable serializedSketch =
+ final Memory serializedSketch = BytesWritableHelper.wrapAsMemory(
(BytesWritable) intermediateObjectInspector.getStructFieldData(
- data, intermediateObjectInspector.getStructFieldRef(SKETCH_FIELD));
- state.update(serializedSketch.getBytes());
+ data, intermediateObjectInspector.getStructFieldRef(SKETCH_FIELD)));
+ state.update(serializedSketch);
}
@Override
@@ -203,8 +204,8 @@ public class IntersectSketchUDAF extends AbstractGenericUDAFResolver {
return seed_;
}
- void update(final byte[] serializedSketch) {
- intersection_.update(Sketches.wrapSketch(Memory.wrap(serializedSketch), seed_));
+ void update(final Memory serializedSketch) {
+ intersection_.update(Sketches.wrapSketch(serializedSketch, seed_));
}
Sketch getResult() {
diff --git a/src/main/java/org/apache/datasketches/hive/theta/IntersectSketchUDF.java b/src/main/java/org/apache/datasketches/hive/theta/IntersectSketchUDF.java
index 07be37c..52ba553 100644
--- a/src/main/java/org/apache/datasketches/hive/theta/IntersectSketchUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/theta/IntersectSketchUDF.java
@@ -21,7 +21,7 @@ package org.apache.datasketches.hive.theta;
import static org.apache.datasketches.Util.DEFAULT_UPDATE_SEED;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.theta.Intersection;
import org.apache.datasketches.theta.SetOperation;
import org.apache.datasketches.theta.Sketch;
@@ -50,12 +50,12 @@ public class IntersectSketchUDF extends UDF {
final BytesWritable secondSketchBytes, final long hashSeed) {
Sketch firstSketch = null;
if (firstSketchBytes != null && firstSketchBytes.getLength() > 0) {
- firstSketch = Sketch.wrap(Memory.wrap(firstSketchBytes.getBytes()), hashSeed);
+ firstSketch = Sketch.wrap(BytesWritableHelper.wrapAsMemory(firstSketchBytes), hashSeed);
}
Sketch secondSketch = null;
if (secondSketchBytes != null && secondSketchBytes.getLength() > 0) {
- secondSketch = Sketch.wrap(Memory.wrap(secondSketchBytes.getBytes()), hashSeed);
+ secondSketch = Sketch.wrap(BytesWritableHelper.wrapAsMemory(secondSketchBytes), hashSeed);
}
final Intersection intersect = SetOperation.builder().setSeed(hashSeed).buildIntersection();
diff --git a/src/main/java/org/apache/datasketches/hive/theta/UnionEvaluator.java b/src/main/java/org/apache/datasketches/hive/theta/UnionEvaluator.java
index 14b821c..ac86461 100644
--- a/src/main/java/org/apache/datasketches/hive/theta/UnionEvaluator.java
+++ b/src/main/java/org/apache/datasketches/hive/theta/UnionEvaluator.java
@@ -21,6 +21,7 @@ package org.apache.datasketches.hive.theta;
import java.util.Arrays;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.theta.Sketch;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -88,10 +89,10 @@ public abstract class UnionEvaluator extends GenericUDAFEvaluator {
if (!state.isInitialized()) {
initializeState(state, partial);
}
- final BytesWritable serializedSketch =
+ final Memory serializedSketch = BytesWritableHelper.wrapAsMemory(
(BytesWritable) intermediateObjectInspector.getStructFieldData(
- partial, intermediateObjectInspector.getStructFieldRef(SKETCH_FIELD));
- state.update(Memory.wrap(serializedSketch.getBytes()));
+ partial, intermediateObjectInspector.getStructFieldRef(SKETCH_FIELD)));
+ state.update(serializedSketch);
}
private void initializeState(final UnionState state, final Object partial) {
diff --git a/src/main/java/org/apache/datasketches/hive/theta/UnionSketchUDF.java b/src/main/java/org/apache/datasketches/hive/theta/UnionSketchUDF.java
index 5cb0358..2076ca6 100644
--- a/src/main/java/org/apache/datasketches/hive/theta/UnionSketchUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/theta/UnionSketchUDF.java
@@ -22,7 +22,7 @@ package org.apache.datasketches.hive.theta;
import static org.apache.datasketches.Util.DEFAULT_NOMINAL_ENTRIES;
import static org.apache.datasketches.Util.DEFAULT_UPDATE_SEED;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.theta.SetOperation;
import org.apache.datasketches.theta.Union;
import org.apache.hadoop.hive.ql.exec.UDF;
@@ -55,11 +55,11 @@ public class UnionSketchUDF extends UDF {
final Union union = SetOperation.builder().setSeed(seed).setNominalEntries(sketchSize).buildUnion();
if ((firstSketch != null) && (firstSketch.getLength() >= EMPTY_SKETCH_SIZE_BYTES)) {
- union.update(Memory.wrap(firstSketch.getBytes()));
+ union.update(BytesWritableHelper.wrapAsMemory(firstSketch));
}
if ((secondSketch != null) && (secondSketch.getLength() >= EMPTY_SKETCH_SIZE_BYTES)) {
- union.update(Memory.wrap(secondSketch.getBytes()));
+ union.update(BytesWritableHelper.wrapAsMemory(secondSketch));
}
return new BytesWritable(union.getResult().toByteArray());
diff --git a/src/main/java/org/apache/datasketches/hive/theta/UnionState.java b/src/main/java/org/apache/datasketches/hive/theta/UnionState.java
index a7e3b20..0285721 100644
--- a/src/main/java/org/apache/datasketches/hive/theta/UnionState.java
+++ b/src/main/java/org/apache/datasketches/hive/theta/UnionState.java
@@ -19,6 +19,7 @@
package org.apache.datasketches.hive.theta;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.theta.SetOperation;
import org.apache.datasketches.theta.Sketch;
@@ -66,7 +67,7 @@ class UnionState extends AbstractAggregationBuffer {
public void update(final Object value, final PrimitiveObjectInspector objectInspector) {
switch (objectInspector.getPrimitiveCategory()) {
case BINARY:
- union_.update(PrimitiveObjectInspectorUtils.getBinary(value, objectInspector).getBytes());
+ union_.update(BytesWritableHelper.wrapAsMemory(PrimitiveObjectInspectorUtils.getBinary(value, objectInspector)));
return;
case BYTE:
union_.update(PrimitiveObjectInspectorUtils.getByte(value, objectInspector));
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchEvaluator.java b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchEvaluator.java
index 7a50cd4..6f00015 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchEvaluator.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchEvaluator.java
@@ -21,6 +21,7 @@ package org.apache.datasketches.hive.tuple;
import java.util.Arrays;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.tuple.ArrayOfDoublesSketch;
import org.apache.datasketches.tuple.ArrayOfDoublesSketches;
@@ -62,10 +63,10 @@ abstract class ArrayOfDoublesSketchEvaluator extends GenericUDAFEvaluator {
if (!state.isInitialized()) {
initializeState(state, data);
}
- final BytesWritable serializedSketch =
+ final Memory serializedSketch = BytesWritableHelper.wrapAsMemory(
(BytesWritable) intermediateInspector_.getStructFieldData(
- data, intermediateInspector_.getStructFieldRef(SKETCH_FIELD));
- state.update(ArrayOfDoublesSketches.wrapSketch(Memory.wrap(serializedSketch.getBytes())));
+ data, intermediateInspector_.getStructFieldRef(SKETCH_FIELD)));
+ state.update(ArrayOfDoublesSketches.wrapSketch(serializedSketch));
}
private void initializeState(final ArrayOfDoublesUnionState state, final Object data) {
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchState.java b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchState.java
index e0c38aa..bd1930f 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchState.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchState.java
@@ -47,7 +47,7 @@ class ArrayOfDoublesSketchState extends ArrayOfDoublesState {
}
switch (keyInspector.getPrimitiveCategory()) {
case BINARY:
- sketch_.update(PrimitiveObjectInspectorUtils.getBinary(data[0], keyInspector).getBytes(), values);
+ sketch_.update(PrimitiveObjectInspectorUtils.getBinary(data[0], keyInspector).copyBytes(), values);
return;
case BYTE:
sketch_.update(PrimitiveObjectInspectorUtils.getByte(data[0], keyInspector), values);
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToEstimateAndErrorBoundsUDF.java b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToEstimateAndErrorBoundsUDF.java
index 158b3b0..1dc805f 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToEstimateAndErrorBoundsUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToEstimateAndErrorBoundsUDF.java
@@ -22,7 +22,7 @@ package org.apache.datasketches.hive.tuple;
import java.util.Arrays;
import java.util.List;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.tuple.ArrayOfDoublesSketch;
import org.apache.datasketches.tuple.ArrayOfDoublesSketches;
import org.apache.hadoop.hive.ql.exec.Description;
@@ -47,7 +47,7 @@ public class ArrayOfDoublesSketchToEstimateAndErrorBoundsUDF extends UDF {
public List<Double> evaluate(final BytesWritable serializedSketch) {
if (serializedSketch == null) { return null; }
final ArrayOfDoublesSketch sketch = ArrayOfDoublesSketches.wrapSketch(
- Memory.wrap(serializedSketch.getBytes()));
+ BytesWritableHelper.wrapAsMemory(serializedSketch));
return Arrays.asList(sketch.getEstimate(), sketch.getLowerBound(2), sketch.getUpperBound(2));
}
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToEstimatesUDF.java b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToEstimatesUDF.java
index 8813bcc..89da31a 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToEstimatesUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToEstimatesUDF.java
@@ -22,7 +22,7 @@ package org.apache.datasketches.hive.tuple;
import java.util.ArrayList;
import java.util.List;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.tuple.ArrayOfDoublesSketch;
import org.apache.datasketches.tuple.ArrayOfDoublesSketchIterator;
import org.apache.datasketches.tuple.ArrayOfDoublesSketches;
@@ -49,7 +49,7 @@ public class ArrayOfDoublesSketchToEstimatesUDF extends UDF {
public List<Double> evaluate(final BytesWritable serializedSketch) {
if (serializedSketch == null) { return null; }
final ArrayOfDoublesSketch sketch = ArrayOfDoublesSketches.wrapSketch(
- Memory.wrap(serializedSketch.getBytes()));
+ BytesWritableHelper.wrapAsMemory(serializedSketch));
final double[] sums = new double[sketch.getNumValues()];
final ArrayOfDoublesSketchIterator it = sketch.iterator();
while (it.next()) {
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToMeansUDF.java b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToMeansUDF.java
index c4e499d..c5daae6 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToMeansUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToMeansUDF.java
@@ -23,7 +23,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.tuple.ArrayOfDoublesSketch;
import org.apache.datasketches.tuple.ArrayOfDoublesSketches;
import org.apache.hadoop.hive.ql.exec.Description;
@@ -47,7 +47,7 @@ public class ArrayOfDoublesSketchToMeansUDF extends UDF {
public List<Double> evaluate(final BytesWritable serializedSketch) {
if (serializedSketch == null) { return null; }
final ArrayOfDoublesSketch sketch = ArrayOfDoublesSketches.wrapSketch(
- Memory.wrap(serializedSketch.getBytes()));
+ BytesWritableHelper.wrapAsMemory(serializedSketch));
if (sketch.getRetainedEntries() < 1) {
return null;
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToNumberOfRetainedEntriesUDF.java b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToNumberOfRetainedEntriesUDF.java
index ff17a8c..9b42b24 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToNumberOfRetainedEntriesUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToNumberOfRetainedEntriesUDF.java
@@ -19,7 +19,7 @@
package org.apache.datasketches.hive.tuple;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.tuple.ArrayOfDoublesSketch;
import org.apache.datasketches.tuple.ArrayOfDoublesSketches;
import org.apache.hadoop.hive.ql.exec.Description;
@@ -42,7 +42,7 @@ public class ArrayOfDoublesSketchToNumberOfRetainedEntriesUDF extends UDF {
public Integer evaluate(final BytesWritable serializedSketch) {
if (serializedSketch == null) { return null; }
final ArrayOfDoublesSketch sketch = ArrayOfDoublesSketches.wrapSketch(
- Memory.wrap(serializedSketch.getBytes()));
+ BytesWritableHelper.wrapAsMemory(serializedSketch));
return sketch.getRetainedEntries();
}
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToQuantilesSketchUDF.java b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToQuantilesSketchUDF.java
index 398db48..cfee6b9 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToQuantilesSketchUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToQuantilesSketchUDF.java
@@ -19,7 +19,7 @@
package org.apache.datasketches.hive.tuple;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.quantiles.DoublesSketch;
import org.apache.datasketches.quantiles.DoublesSketchBuilder;
import org.apache.datasketches.quantiles.UpdateDoublesSketch;
@@ -74,7 +74,7 @@ public class ArrayOfDoublesSketchToQuantilesSketchUDF extends UDF {
final int k) {
if (serializedSketch == null) { return null; }
final ArrayOfDoublesSketch sketch = ArrayOfDoublesSketches.wrapSketch(
- Memory.wrap(serializedSketch.getBytes()));
+ BytesWritableHelper.wrapAsMemory(serializedSketch));
if (column < 1) {
throw new IllegalArgumentException("Column number must be greater than zero. Received: "
+ column);
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToValuesUDTF.java b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToValuesUDTF.java
index ebd905c..dabd1fc 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToValuesUDTF.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToValuesUDTF.java
@@ -23,7 +23,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.tuple.ArrayOfDoublesSketch;
import org.apache.datasketches.tuple.ArrayOfDoublesSketchIterator;
import org.apache.datasketches.tuple.ArrayOfDoublesSketches;
@@ -74,7 +74,7 @@ public class ArrayOfDoublesSketchToValuesUDTF extends GenericUDTF {
final BytesWritable serializedSketch =
(BytesWritable) inputObjectInspector.getPrimitiveWritableObject(data[0]);
final ArrayOfDoublesSketch sketch = ArrayOfDoublesSketches.wrapSketch(
- Memory.wrap(serializedSketch.getBytes()));
+ BytesWritableHelper.wrapAsMemory(serializedSketch));
final ArrayOfDoublesSketchIterator it = sketch.iterator();
while (it.next()) {
forward(new Object[] { primitivesToList(it.getValues()) });
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToVariancesUDF.java b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToVariancesUDF.java
index d632a88..f2ba17f 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToVariancesUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToVariancesUDF.java
@@ -23,7 +23,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.tuple.ArrayOfDoublesSketch;
import org.apache.datasketches.tuple.ArrayOfDoublesSketches;
import org.apache.hadoop.hive.ql.exec.Description;
@@ -47,7 +47,7 @@ public class ArrayOfDoublesSketchToVariancesUDF extends UDF {
public List<Double> evaluate(final BytesWritable serializedSketch) {
if (serializedSketch == null) { return null; }
final ArrayOfDoublesSketch sketch = ArrayOfDoublesSketches.wrapSketch(
- Memory.wrap(serializedSketch.getBytes()));
+ BytesWritableHelper.wrapAsMemory(serializedSketch));
if (sketch.getRetainedEntries() < 1) {
return null;
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchesTTestUDF.java b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchesTTestUDF.java
index 73f75ed..4ed64dc 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchesTTestUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchesTTestUDF.java
@@ -24,7 +24,7 @@ import java.util.List;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.apache.commons.math3.stat.inference.TTest;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.tuple.ArrayOfDoublesSketch;
import org.apache.datasketches.tuple.ArrayOfDoublesSketches;
import org.apache.hadoop.hive.ql.exec.Description;
@@ -50,9 +50,9 @@ public class ArrayOfDoublesSketchesTTestUDF extends UDF {
public List<Double> evaluate(final BytesWritable serializedSketchA, final BytesWritable serializedSketchB) {
if ((serializedSketchA == null) || (serializedSketchB == null)) { return null; }
final ArrayOfDoublesSketch sketchA =
- ArrayOfDoublesSketches.wrapSketch(Memory.wrap(serializedSketchA.getBytes()));
+ ArrayOfDoublesSketches.wrapSketch(BytesWritableHelper.wrapAsMemory(serializedSketchA));
final ArrayOfDoublesSketch sketchB =
- ArrayOfDoublesSketches.wrapSketch(Memory.wrap(serializedSketchB.getBytes()));
+ ArrayOfDoublesSketches.wrapSketch(BytesWritableHelper.wrapAsMemory(serializedSketchB));
if (sketchA.getNumValues() != sketchB.getNumValues()) {
throw new IllegalArgumentException("Both sketches must have the same number of values");
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/DoubleSummarySketchToEstimatesUDF.java b/src/main/java/org/apache/datasketches/hive/tuple/DoubleSummarySketchToEstimatesUDF.java
index 54819bf..80512a2 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/DoubleSummarySketchToEstimatesUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/DoubleSummarySketchToEstimatesUDF.java
@@ -22,7 +22,7 @@ package org.apache.datasketches.hive.tuple;
import java.util.Arrays;
import java.util.List;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.tuple.Sketch;
import org.apache.datasketches.tuple.SketchIterator;
import org.apache.datasketches.tuple.Sketches;
@@ -56,7 +56,7 @@ public class DoubleSummarySketchToEstimatesUDF extends UDF {
public List<Double> evaluate(final BytesWritable serializedSketch) {
if (serializedSketch == null) { return null; }
final Sketch<DoubleSummary> sketch =
- Sketches.heapifySketch(Memory.wrap(serializedSketch.getBytes()), SUMMARY_DESERIALIZER);
+ Sketches.heapifySketch(BytesWritableHelper.wrapAsMemory(serializedSketch), SUMMARY_DESERIALIZER);
double sum = 0;
final SketchIterator<DoubleSummary> it = sketch.iterator();
while (it.next()) {
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/DoubleSummarySketchToPercentileUDF.java b/src/main/java/org/apache/datasketches/hive/tuple/DoubleSummarySketchToPercentileUDF.java
index 484d002..9497212 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/DoubleSummarySketchToPercentileUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/DoubleSummarySketchToPercentileUDF.java
@@ -19,7 +19,7 @@
package org.apache.datasketches.hive.tuple;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.quantiles.DoublesSketch;
import org.apache.datasketches.quantiles.UpdateDoublesSketch;
import org.apache.datasketches.tuple.Sketch;
@@ -60,7 +60,7 @@ public class DoubleSummarySketchToPercentileUDF extends UDF {
throw new IllegalArgumentException("percentile must be between 0 and 100");
}
final Sketch<DoubleSummary> sketch =
- Sketches.heapifySketch(Memory.wrap(serializedSketch.getBytes()), SUMMARY_DESERIALIZER);
+ Sketches.heapifySketch(BytesWritableHelper.wrapAsMemory(serializedSketch), SUMMARY_DESERIALIZER);
final UpdateDoublesSketch qs = DoublesSketch.builder().setK(QUANTILES_SKETCH_K).build();
final SketchIterator<DoubleSummary> it = sketch.iterator();
while (it.next()) {
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/SketchEvaluator.java b/src/main/java/org/apache/datasketches/hive/tuple/SketchEvaluator.java
index add93de..a15b613 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/SketchEvaluator.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/SketchEvaluator.java
@@ -21,7 +21,7 @@ package org.apache.datasketches.hive.tuple;
import java.util.Arrays;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.tuple.Sketch;
import org.apache.datasketches.tuple.Sketches;
import org.apache.datasketches.tuple.Summary;
@@ -93,7 +93,7 @@ abstract class SketchEvaluator<S extends Summary> extends GenericUDAFEvaluator {
final BytesWritable serializedSketch =
(BytesWritable) intermediateInspector_.getStructFieldData(
data, intermediateInspector_.getStructFieldRef(SKETCH_FIELD));
- state.update(Sketches.heapifySketch(Memory.wrap(serializedSketch.getBytes()), getSummaryDeserializer()));
+ state.update(Sketches.heapifySketch(BytesWritableHelper.wrapAsMemory(serializedSketch), getSummaryDeserializer()));
}
protected void initializeState(final UnionState<S> state, final Object data) {
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/SketchState.java b/src/main/java/org/apache/datasketches/hive/tuple/SketchState.java
index 1556e01..14c17f8 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/SketchState.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/SketchState.java
@@ -45,7 +45,7 @@ class SketchState<U, S extends UpdatableSummary<U>> extends State<S> {
void update(final Object data, final PrimitiveObjectInspector keyObjectInspector, final U value) {
switch (keyObjectInspector.getPrimitiveCategory()) {
case BINARY:
- sketch_.update(PrimitiveObjectInspectorUtils.getBinary(data, keyObjectInspector).getBytes(), value);
+ sketch_.update(PrimitiveObjectInspectorUtils.getBinary(data, keyObjectInspector).copyBytes(), value);
return;
case BYTE:
sketch_.update(PrimitiveObjectInspectorUtils.getByte(data, keyObjectInspector), value);
diff --git a/src/test/java/org/apache/datasketches/hive/cpc/DataToSketchUDAFTest.java b/src/test/java/org/apache/datasketches/hive/cpc/DataToSketchUDAFTest.java
index efd24c5..6d026f0 100644
--- a/src/test/java/org/apache/datasketches/hive/cpc/DataToSketchUDAFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/cpc/DataToSketchUDAFTest.java
@@ -25,7 +25,7 @@ import java.util.Arrays;
import java.util.List;
import org.apache.datasketches.cpc.CpcSketch;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
@@ -161,7 +161,7 @@ public class DataToSketchUDAFTest {
Assert.assertEquals(r.size(), 3);
Assert.assertEquals(((IntWritable) r.get(0)).get(), SketchEvaluator.DEFAULT_LG_K);
Assert.assertEquals(((LongWritable) r.get(1)).get(), DEFAULT_UPDATE_SEED);
- CpcSketch resultSketch = CpcSketch.heapify(Memory.wrap(((BytesWritable) r.get(2)).getBytes()));
+ CpcSketch resultSketch = CpcSketch.heapify(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)));
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.01);
}
}
@@ -188,7 +188,7 @@ public class DataToSketchUDAFTest {
Assert.assertEquals(r.size(), 3);
Assert.assertEquals(((IntWritable) r.get(0)).get(), lgK);
Assert.assertEquals(((LongWritable) r.get(1)).get(), seed);
- CpcSketch resultSketch = CpcSketch.heapify(Memory.wrap(((BytesWritable) r.get(2)).getBytes()), seed);
+ CpcSketch resultSketch = CpcSketch.heapify(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)), seed);
Assert.assertEquals(resultSketch.getLgK(), lgK);
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.01);
}
@@ -228,7 +228,7 @@ public class DataToSketchUDAFTest {
Assert.assertEquals(r.size(), 3);
Assert.assertEquals(((IntWritable) r.get(0)).get(), SketchEvaluator.DEFAULT_LG_K);
Assert.assertEquals(((LongWritable) r.get(1)).get(), DEFAULT_UPDATE_SEED);
- CpcSketch resultSketch = CpcSketch.heapify(Memory.wrap(((BytesWritable) r.get(2)).getBytes()));
+ CpcSketch resultSketch = CpcSketch.heapify(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)));
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.01);
eval.reset(state);
@@ -267,7 +267,7 @@ public class DataToSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- CpcSketch resultSketch = CpcSketch.heapify(Memory.wrap(((BytesWritable) result).getBytes()));
+ CpcSketch resultSketch = CpcSketch.heapify(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.01);
}
}
@@ -302,7 +302,7 @@ public class DataToSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- CpcSketch resultSketch = CpcSketch.heapify(Memory.wrap(((BytesWritable) result).getBytes()), seed);
+ CpcSketch resultSketch = CpcSketch.heapify(BytesWritableHelper.wrapAsMemory((BytesWritable) result), seed);
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.01);
}
}
@@ -323,7 +323,7 @@ public class DataToSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- CpcSketch resultSketch = CpcSketch.heapify(Memory.wrap(((BytesWritable) result).getBytes()));
+ CpcSketch resultSketch = CpcSketch.heapify(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.01);
eval.reset(state);
@@ -351,7 +351,7 @@ public class DataToSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- CpcSketch resultSketch = CpcSketch.heapify(Memory.wrap(((BytesWritable) result).getBytes()), seed);
+ CpcSketch resultSketch = CpcSketch.heapify(BytesWritableHelper.wrapAsMemory((BytesWritable) result), seed);
Assert.assertEquals(resultSketch.getLgK(), lgK);
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.03);
diff --git a/src/test/java/org/apache/datasketches/hive/cpc/UnionSketchUDAFTest.java b/src/test/java/org/apache/datasketches/hive/cpc/UnionSketchUDAFTest.java
index cab5b7a..3882259 100644
--- a/src/test/java/org/apache/datasketches/hive/cpc/UnionSketchUDAFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/cpc/UnionSketchUDAFTest.java
@@ -25,7 +25,7 @@ import java.util.Arrays;
import java.util.List;
import org.apache.datasketches.cpc.CpcSketch;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
@@ -169,7 +169,7 @@ public class UnionSketchUDAFTest {
Assert.assertEquals(r.size(), 3);
Assert.assertEquals(((IntWritable) r.get(0)).get(), SketchEvaluator.DEFAULT_LG_K);
Assert.assertEquals(((LongWritable) r.get(1)).get(), DEFAULT_UPDATE_SEED);
- CpcSketch resultSketch = CpcSketch.heapify(Memory.wrap(((BytesWritable) r.get(2)).getBytes()));
+ CpcSketch resultSketch = CpcSketch.heapify(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)));
Assert.assertEquals(resultSketch.getLgK(), SketchEvaluator.DEFAULT_LG_K);
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.01);
}
@@ -204,7 +204,7 @@ public class UnionSketchUDAFTest {
Assert.assertEquals(r.size(), 3);
Assert.assertEquals(((IntWritable) r.get(0)).get(), lgK);
Assert.assertEquals(((LongWritable) r.get(1)).get(), seed);
- CpcSketch resultSketch = CpcSketch.heapify(Memory.wrap(((BytesWritable) r.get(2)).getBytes()), seed);
+ CpcSketch resultSketch = CpcSketch.heapify(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)), seed);
Assert.assertEquals(resultSketch.getLgK(), lgK);
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.01);
}
@@ -244,7 +244,7 @@ public class UnionSketchUDAFTest {
Assert.assertEquals(r.size(), 3);
Assert.assertEquals(((IntWritable) r.get(0)).get(), SketchEvaluator.DEFAULT_LG_K);
Assert.assertEquals(((LongWritable) r.get(1)).get(), DEFAULT_UPDATE_SEED);
- CpcSketch resultSketch = CpcSketch.heapify(Memory.wrap(((BytesWritable) r.get(2)).getBytes()));
+ CpcSketch resultSketch = CpcSketch.heapify(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)));
Assert.assertEquals(resultSketch.getLgK(), SketchEvaluator.DEFAULT_LG_K);
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.01);
@@ -284,7 +284,7 @@ public class UnionSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- CpcSketch resultSketch = CpcSketch.heapify(Memory.wrap(((BytesWritable) result).getBytes()));
+ CpcSketch resultSketch = CpcSketch.heapify(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.01);
}
}
@@ -311,7 +311,7 @@ public class UnionSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- CpcSketch resultSketch = CpcSketch.heapify(Memory.wrap(((BytesWritable) result).getBytes()));
+ CpcSketch resultSketch = CpcSketch.heapify(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.01);
eval.reset(state);
@@ -345,7 +345,7 @@ public class UnionSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- CpcSketch resultSketch = CpcSketch.heapify(Memory.wrap(((BytesWritable) result).getBytes()), seed);
+ CpcSketch resultSketch = CpcSketch.heapify(BytesWritableHelper.wrapAsMemory((BytesWritable) result), seed);
Assert.assertEquals(resultSketch.getLgK(), lgK);
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.03);
diff --git a/src/test/java/org/apache/datasketches/hive/cpc/UnionSketchUDFTest.java b/src/test/java/org/apache/datasketches/hive/cpc/UnionSketchUDFTest.java
index 56bff15..f9607b5 100644
--- a/src/test/java/org/apache/datasketches/hive/cpc/UnionSketchUDFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/cpc/UnionSketchUDFTest.java
@@ -19,11 +19,11 @@
package org.apache.datasketches.hive.cpc;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.hadoop.io.BytesWritable;
import org.testng.Assert;
import org.testng.annotations.Test;
-import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.cpc.CpcSketch;
@SuppressWarnings("javadoc")
@@ -33,7 +33,7 @@ public class UnionSketchUDFTest {
public void nullInputs() {
UnionSketchUDF udf = new UnionSketchUDF();
BytesWritable result = udf.evaluate(null, null);
- CpcSketch resultSketch = CpcSketch.heapify(Memory.wrap(result.getBytes()));
+ CpcSketch resultSketch = CpcSketch.heapify(BytesWritableHelper.wrapAsMemory(result));
Assert.assertTrue(resultSketch.isEmpty());
Assert.assertEquals(resultSketch.getEstimate(), 0.0);
}
@@ -57,7 +57,7 @@ public class UnionSketchUDFTest {
BytesWritable result = udf.evaluate(input1, input2);
- CpcSketch resultSketch = CpcSketch.heapify(Memory.wrap(result.getBytes()));
+ CpcSketch resultSketch = CpcSketch.heapify(BytesWritableHelper.wrapAsMemory(result));
Assert.assertEquals(resultSketch.getEstimate(), 256.0, 256 * 0.02);
}
@@ -83,7 +83,7 @@ public class UnionSketchUDFTest {
BytesWritable result = udf.evaluate(input1, input2, lgK);
- CpcSketch resultSketch = CpcSketch.heapify(Memory.wrap(result.getBytes()));
+ CpcSketch resultSketch = CpcSketch.heapify(BytesWritableHelper.wrapAsMemory(result));
Assert.assertEquals(resultSketch.getLgK(), lgK);
Assert.assertEquals(resultSketch.getEstimate(), 256.0, 256 * 0.02);
@@ -111,7 +111,7 @@ public class UnionSketchUDFTest {
BytesWritable result = udf.evaluate(input1, input2, lgK, seed);
- CpcSketch resultSketch = CpcSketch.heapify(Memory.wrap(result.getBytes()), seed);
+ CpcSketch resultSketch = CpcSketch.heapify(BytesWritableHelper.wrapAsMemory(result), seed);
Assert.assertEquals(resultSketch.getLgK(), lgK);
Assert.assertEquals(resultSketch.getEstimate(), 256.0, 256 * 0.02);
diff --git a/src/test/java/org/apache/datasketches/hive/frequencies/DataToStringsSketchUDAFTest.java b/src/test/java/org/apache/datasketches/hive/frequencies/DataToStringsSketchUDAFTest.java
index 3855b8d..99fbdd2 100644
--- a/src/test/java/org/apache/datasketches/hive/frequencies/DataToStringsSketchUDAFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/frequencies/DataToStringsSketchUDAFTest.java
@@ -24,7 +24,7 @@ import java.util.Arrays;
import org.apache.datasketches.ArrayOfItemsSerDe;
import org.apache.datasketches.ArrayOfStringsSerDe;
import org.apache.datasketches.frequencies.ItemsSketch;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
@@ -108,7 +108,7 @@ public class DataToStringsSketchUDAFTest {
eval.iterate(state, new Object[] { new org.apache.hadoop.io.Text("b"), new IntWritable(256) });
BytesWritable bytes = (BytesWritable) eval.terminatePartial(state);
- ItemsSketch<String> resultSketch = ItemsSketch.getInstance(Memory.wrap(bytes.getBytes()), serDe);
+ ItemsSketch<String> resultSketch = ItemsSketch.getInstance(BytesWritableHelper.wrapAsMemory(bytes), serDe);
Assert.assertEquals(resultSketch.getStreamLength(), 2);
Assert.assertEquals(resultSketch.getNumActiveItems(), 2);
Assert.assertEquals(resultSketch.getEstimate("a"), 1);
@@ -136,7 +136,7 @@ public class DataToStringsSketchUDAFTest {
eval.merge(state, new BytesWritable(sketch2.toByteArray(serDe)));
BytesWritable bytes = (BytesWritable) eval.terminate(state);
- ItemsSketch<String> resultSketch = ItemsSketch.getInstance(Memory.wrap(bytes.getBytes()), serDe);
+ ItemsSketch<String> resultSketch = ItemsSketch.getInstance(BytesWritableHelper.wrapAsMemory(bytes), serDe);
Assert.assertEquals(resultSketch.getStreamLength(), 2);
Assert.assertEquals(resultSketch.getNumActiveItems(), 2);
Assert.assertEquals(resultSketch.getEstimate("a"), 1);
@@ -163,7 +163,7 @@ public class DataToStringsSketchUDAFTest {
eval.merge(state, new BytesWritable(sketch.toByteArray(serDe)));
BytesWritable bytes = (BytesWritable) eval.terminate(state);
- ItemsSketch<String> resultSketch = ItemsSketch.getInstance(Memory.wrap(bytes.getBytes()), serDe);
+ ItemsSketch<String> resultSketch = ItemsSketch.getInstance(BytesWritableHelper.wrapAsMemory(bytes), serDe);
Assert.assertEquals(resultSketch.getStreamLength(), 2);
Assert.assertEquals(resultSketch.getNumActiveItems(), 2);
Assert.assertEquals(resultSketch.getEstimate("a"), 1);
diff --git a/src/test/java/org/apache/datasketches/hive/frequencies/UnionStringsSketchUDAFTest.java b/src/test/java/org/apache/datasketches/hive/frequencies/UnionStringsSketchUDAFTest.java
index a3f43fc..47e39d7 100644
--- a/src/test/java/org/apache/datasketches/hive/frequencies/UnionStringsSketchUDAFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/frequencies/UnionStringsSketchUDAFTest.java
@@ -24,7 +24,7 @@ import java.util.Arrays;
import org.apache.datasketches.ArrayOfItemsSerDe;
import org.apache.datasketches.ArrayOfStringsSerDe;
import org.apache.datasketches.frequencies.ItemsSketch;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
@@ -102,7 +102,7 @@ public class UnionStringsSketchUDAFTest {
eval.iterate(state, new Object[] { new BytesWritable(sketch.toByteArray(serDe)) });
BytesWritable bytes = (BytesWritable) eval.terminatePartial(state);
- ItemsSketch<String> resultSketch = ItemsSketch.getInstance(Memory.wrap(bytes.getBytes()), serDe);
+ ItemsSketch<String> resultSketch = ItemsSketch.getInstance(BytesWritableHelper.wrapAsMemory(bytes), serDe);
Assert.assertEquals(resultSketch.getStreamLength(), 2);
Assert.assertEquals(resultSketch.getNumActiveItems(), 2);
Assert.assertEquals(resultSketch.getEstimate("a"), 1);
@@ -129,7 +129,7 @@ public class UnionStringsSketchUDAFTest {
eval.merge(state, new BytesWritable(sketch.toByteArray(serDe)));
BytesWritable bytes = (BytesWritable) eval.terminate(state);
- ItemsSketch<String> resultSketch = ItemsSketch.getInstance(Memory.wrap(bytes.getBytes()), serDe);
+ ItemsSketch<String> resultSketch = ItemsSketch.getInstance(BytesWritableHelper.wrapAsMemory(bytes), serDe);
Assert.assertEquals(resultSketch.getStreamLength(), 2);
Assert.assertEquals(resultSketch.getNumActiveItems(), 2);
Assert.assertEquals(resultSketch.getEstimate("a"), 1);
diff --git a/src/test/java/org/apache/datasketches/hive/hll/DataToSketchUDAFTest.java b/src/test/java/org/apache/datasketches/hive/hll/DataToSketchUDAFTest.java
index f224a66..090e2c3 100644
--- a/src/test/java/org/apache/datasketches/hive/hll/DataToSketchUDAFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/hll/DataToSketchUDAFTest.java
@@ -22,9 +22,9 @@ package org.apache.datasketches.hive.hll;
import java.util.Arrays;
import java.util.List;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.hll.HllSketch;
import org.apache.datasketches.hll.TgtHllType;
-import org.apache.datasketches.memory.Memory;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
@@ -156,7 +156,7 @@ public class DataToSketchUDAFTest {
Assert.assertEquals(r.size(), 3);
Assert.assertEquals(((IntWritable) r.get(0)).get(), SketchEvaluator.DEFAULT_LG_K);
Assert.assertEquals(((Text) r.get(1)).toString(), SketchEvaluator.DEFAULT_HLL_TYPE.toString());
- HllSketch resultSketch = HllSketch.heapify(Memory.wrap(((BytesWritable) r.get(2)).getBytes()));
+ HllSketch resultSketch = HllSketch.heapify(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)));
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.01);
}
}
@@ -183,7 +183,7 @@ public class DataToSketchUDAFTest {
Assert.assertEquals(r.size(), 3);
Assert.assertEquals(((IntWritable) r.get(0)).get(), lgK);
Assert.assertEquals(((Text) r.get(1)).toString(), hllType.toString());
- HllSketch resultSketch = HllSketch.heapify(Memory.wrap(((BytesWritable) r.get(2)).getBytes()));
+ HllSketch resultSketch = HllSketch.heapify(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)));
Assert.assertEquals(resultSketch.getLgConfigK(), lgK);
Assert.assertEquals(resultSketch.getTgtHllType(), hllType);
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.01);
@@ -210,7 +210,7 @@ public class DataToSketchUDAFTest {
Assert.assertEquals(r.size(), 3);
Assert.assertEquals(((IntWritable) r.get(0)).get(), SketchEvaluator.DEFAULT_LG_K);
Assert.assertEquals(((Text) r.get(1)).toString(), SketchEvaluator.DEFAULT_HLL_TYPE.toString());
- HllSketch resultSketch = HllSketch.wrap(Memory.wrap(((BytesWritable) r.get(2)).getBytes()));
+ HllSketch resultSketch = HllSketch.wrap(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)));
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.01);
}
}
@@ -249,7 +249,7 @@ public class DataToSketchUDAFTest {
Assert.assertEquals(r.size(), 3);
Assert.assertEquals(((IntWritable) r.get(0)).get(), SketchEvaluator.DEFAULT_LG_K);
Assert.assertEquals(((Text) r.get(1)).toString(), SketchEvaluator.DEFAULT_HLL_TYPE.toString());
- HllSketch resultSketch = HllSketch.heapify(Memory.wrap(((BytesWritable) r.get(2)).getBytes()));
+ HllSketch resultSketch = HllSketch.heapify(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)));
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.01);
eval.reset(state);
@@ -288,7 +288,7 @@ public class DataToSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- HllSketch resultSketch = HllSketch.heapify(Memory.wrap(((BytesWritable) result).getBytes()));
+ HllSketch resultSketch = HllSketch.heapify(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.01);
}
}
@@ -309,7 +309,7 @@ public class DataToSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- HllSketch resultSketch = HllSketch.heapify(Memory.wrap(((BytesWritable) result).getBytes()));
+ HllSketch resultSketch = HllSketch.heapify(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.01);
eval.reset(state);
@@ -337,7 +337,7 @@ public class DataToSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- HllSketch resultSketch = HllSketch.heapify(Memory.wrap(((BytesWritable) result).getBytes()));
+ HllSketch resultSketch = HllSketch.heapify(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
Assert.assertEquals(resultSketch.getLgConfigK(), lgK);
Assert.assertEquals(resultSketch.getTgtHllType(), hllType);
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.01);
diff --git a/src/test/java/org/apache/datasketches/hive/hll/UnionSketchUDAFTest.java b/src/test/java/org/apache/datasketches/hive/hll/UnionSketchUDAFTest.java
index 8062fb6..9fc34c4 100644
--- a/src/test/java/org/apache/datasketches/hive/hll/UnionSketchUDAFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/hll/UnionSketchUDAFTest.java
@@ -22,9 +22,9 @@ package org.apache.datasketches.hive.hll;
import java.util.Arrays;
import java.util.List;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.hll.HllSketch;
import org.apache.datasketches.hll.TgtHllType;
-import org.apache.datasketches.memory.Memory;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
@@ -162,7 +162,7 @@ public class UnionSketchUDAFTest {
Assert.assertEquals(r.size(), 3);
Assert.assertEquals(((IntWritable) r.get(0)).get(), SketchEvaluator.DEFAULT_LG_K);
Assert.assertEquals(((Text) r.get(1)).toString(), SketchEvaluator.DEFAULT_HLL_TYPE.toString());
- HllSketch resultSketch = HllSketch.heapify(Memory.wrap(((BytesWritable) r.get(2)).getBytes()));
+ HllSketch resultSketch = HllSketch.heapify(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)));
Assert.assertEquals(resultSketch.getLgConfigK(), SketchEvaluator.DEFAULT_LG_K);
Assert.assertEquals(resultSketch.getTgtHllType(), SketchEvaluator.DEFAULT_HLL_TYPE);
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.01);
@@ -198,7 +198,7 @@ public class UnionSketchUDAFTest {
Assert.assertEquals(r.size(), 3);
Assert.assertEquals(((IntWritable) r.get(0)).get(), lgK);
Assert.assertEquals(((Text) r.get(1)).toString(), hllType.toString());
- HllSketch resultSketch = HllSketch.heapify(Memory.wrap(((BytesWritable) r.get(2)).getBytes()));
+ HllSketch resultSketch = HllSketch.heapify(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)));
Assert.assertEquals(resultSketch.getLgConfigK(), lgK);
Assert.assertEquals(resultSketch.getTgtHllType(), hllType);
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.01);
@@ -239,7 +239,7 @@ public class UnionSketchUDAFTest {
Assert.assertEquals(r.size(), 3);
Assert.assertEquals(((IntWritable) r.get(0)).get(), SketchEvaluator.DEFAULT_LG_K);
Assert.assertEquals(((Text) r.get(1)).toString(), SketchEvaluator.DEFAULT_HLL_TYPE.toString());
- HllSketch resultSketch = HllSketch.heapify(Memory.wrap(((BytesWritable) r.get(2)).getBytes()));
+ HllSketch resultSketch = HllSketch.heapify(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)));
Assert.assertEquals(resultSketch.getLgConfigK(), SketchEvaluator.DEFAULT_LG_K);
Assert.assertEquals(resultSketch.getTgtHllType(), SketchEvaluator.DEFAULT_HLL_TYPE);
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.01);
@@ -280,7 +280,7 @@ public class UnionSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- HllSketch resultSketch = HllSketch.heapify(Memory.wrap(((BytesWritable) result).getBytes()));
+ HllSketch resultSketch = HllSketch.heapify(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.01);
}
}
@@ -307,7 +307,7 @@ public class UnionSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- HllSketch resultSketch = HllSketch.heapify(Memory.wrap(((BytesWritable) result).getBytes()));
+ HllSketch resultSketch = HllSketch.heapify(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.01);
eval.reset(state);
@@ -341,7 +341,7 @@ public class UnionSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- HllSketch resultSketch = HllSketch.heapify(Memory.wrap(((BytesWritable) result).getBytes()));
+ HllSketch resultSketch = HllSketch.heapify(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
Assert.assertEquals(resultSketch.getLgConfigK(), lgK);
Assert.assertEquals(resultSketch.getTgtHllType(), hllType);
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.01);
diff --git a/src/test/java/org/apache/datasketches/hive/hll/UnionSketchUDFTest.java b/src/test/java/org/apache/datasketches/hive/hll/UnionSketchUDFTest.java
index ddb0b1c..de50157 100644
--- a/src/test/java/org/apache/datasketches/hive/hll/UnionSketchUDFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/hll/UnionSketchUDFTest.java
@@ -19,11 +19,11 @@
package org.apache.datasketches.hive.hll;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.hadoop.io.BytesWritable;
import org.testng.Assert;
import org.testng.annotations.Test;
-import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.hll.HllSketch;
import org.apache.datasketches.hll.TgtHllType;
@@ -34,7 +34,7 @@ public class UnionSketchUDFTest {
public void nullInputs() {
UnionSketchUDF udf = new UnionSketchUDF();
BytesWritable result = udf.evaluate(null, null);
- HllSketch resultSketch = HllSketch.heapify(Memory.wrap(result.getBytes()));
+ HllSketch resultSketch = HllSketch.heapify(BytesWritableHelper.wrapAsMemory(result));
Assert.assertTrue(resultSketch.isEmpty());
Assert.assertEquals(resultSketch.getEstimate(), 0.0);
}
@@ -58,7 +58,7 @@ public class UnionSketchUDFTest {
BytesWritable result = udf.evaluate(input1, input2);
- HllSketch resultSketch = HllSketch.heapify(Memory.wrap(result.getBytes()));
+ HllSketch resultSketch = HllSketch.heapify(BytesWritableHelper.wrapAsMemory(result));
Assert.assertEquals(resultSketch.getEstimate(), 256.0, 256 * 0.01);
}
@@ -85,7 +85,7 @@ public class UnionSketchUDFTest {
BytesWritable result = udf.evaluate(input1, input2, lgK, type.toString());
- HllSketch resultSketch = HllSketch.heapify(Memory.wrap(result.getBytes()));
+ HllSketch resultSketch = HllSketch.heapify(BytesWritableHelper.wrapAsMemory(result));
Assert.assertEquals(resultSketch.getLgConfigK(), lgK);
Assert.assertEquals(resultSketch.getTgtHllType(), type);
diff --git a/src/test/java/org/apache/datasketches/hive/kll/DataToSketchUDAFTest.java b/src/test/java/org/apache/datasketches/hive/kll/DataToSketchUDAFTest.java
index 161fc4e..d4a0c8c 100644
--- a/src/test/java/org/apache/datasketches/hive/kll/DataToSketchUDAFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/kll/DataToSketchUDAFTest.java
@@ -21,8 +21,8 @@ package org.apache.datasketches.hive.kll;
import java.util.Arrays;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.kll.KllFloatsSketch;
-import org.apache.datasketches.memory.Memory;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
@@ -112,7 +112,7 @@ public class DataToSketchUDAFTest {
eval.iterate(state, new Object[] { new FloatWritable(2) });
BytesWritable bytes = (BytesWritable) eval.terminatePartial(state);
- KllFloatsSketch resultSketch = KllFloatsSketch.heapify(Memory.wrap(bytes.getBytes()));
+ KllFloatsSketch resultSketch = KllFloatsSketch.heapify(BytesWritableHelper.wrapAsMemory(bytes));
Assert.assertEquals(resultSketch.getNormalizedRankError(false), KllFloatsSketch.getNormalizedRankError(200, false));
Assert.assertEquals(resultSketch.getNumRetained(), 2);
Assert.assertEquals(resultSketch.getMinValue(), 1f);
@@ -133,7 +133,7 @@ public class DataToSketchUDAFTest {
eval.iterate(state, new Object[] { new FloatWritable(2), new IntWritable(400) });
BytesWritable bytes = (BytesWritable) eval.terminatePartial(state);
- KllFloatsSketch resultSketch = KllFloatsSketch.heapify(Memory.wrap(bytes.getBytes()));
+ KllFloatsSketch resultSketch = KllFloatsSketch.heapify(BytesWritableHelper.wrapAsMemory(bytes));
Assert.assertEquals(resultSketch.getNormalizedRankError(false), KllFloatsSketch.getNormalizedRankError(400, false));
Assert.assertEquals(resultSketch.getNumRetained(), 2);
Assert.assertEquals(resultSketch.getMinValue(), 1f);
@@ -160,7 +160,7 @@ public class DataToSketchUDAFTest {
eval.merge(state, new BytesWritable(sketch2.toByteArray()));
BytesWritable bytes = (BytesWritable) eval.terminate(state);
- KllFloatsSketch resultSketch = KllFloatsSketch.heapify(Memory.wrap(bytes.getBytes()));
+ KllFloatsSketch resultSketch = KllFloatsSketch.heapify(BytesWritableHelper.wrapAsMemory(bytes));
Assert.assertEquals(resultSketch.getNumRetained(), 2);
Assert.assertEquals(resultSketch.getMinValue(), 1f);
Assert.assertEquals(resultSketch.getMaxValue(), 2f);
@@ -186,7 +186,7 @@ public class DataToSketchUDAFTest {
eval.merge(state, new BytesWritable(sketch2.toByteArray()));
BytesWritable bytes = (BytesWritable) eval.terminate(state);
- KllFloatsSketch resultSketch = KllFloatsSketch.heapify(Memory.wrap(bytes.getBytes()));
+ KllFloatsSketch resultSketch = KllFloatsSketch.heapify(BytesWritableHelper.wrapAsMemory(bytes));
Assert.assertEquals(resultSketch.getNormalizedRankError(false), KllFloatsSketch.getNormalizedRankError(400, false));
Assert.assertEquals(resultSketch.getNumRetained(), 2);
Assert.assertEquals(resultSketch.getMinValue(), 1f);
@@ -208,7 +208,7 @@ public class DataToSketchUDAFTest {
eval.iterate(state, new Object[] { new FloatWritable(2) });
BytesWritable bytes = (BytesWritable) eval.terminate(state);
- KllFloatsSketch resultSketch = KllFloatsSketch.heapify(Memory.wrap(bytes.getBytes()));
+ KllFloatsSketch resultSketch = KllFloatsSketch.heapify(BytesWritableHelper.wrapAsMemory(bytes));
Assert.assertEquals(resultSketch.getNormalizedRankError(false), KllFloatsSketch.getNormalizedRankError(200, false));
Assert.assertEquals(resultSketch.getNumRetained(), 2);
Assert.assertEquals(resultSketch.getMinValue(), 1f);
@@ -229,7 +229,7 @@ public class DataToSketchUDAFTest {
eval.iterate(state, new Object[] { new FloatWritable(2), new IntWritable(400) });
BytesWritable bytes = (BytesWritable) eval.terminate(state);
- KllFloatsSketch resultSketch = KllFloatsSketch.heapify(Memory.wrap(bytes.getBytes()));
+ KllFloatsSketch resultSketch = KllFloatsSketch.heapify(BytesWritableHelper.wrapAsMemory(bytes));
Assert.assertEquals(resultSketch.getNormalizedRankError(false), KllFloatsSketch.getNormalizedRankError(400, false));
Assert.assertEquals(resultSketch.getNumRetained(), 2);
Assert.assertEquals(resultSketch.getMinValue(), 1f);
diff --git a/src/test/java/org/apache/datasketches/hive/kll/UnionSketchUDAFTest.java b/src/test/java/org/apache/datasketches/hive/kll/UnionSketchUDAFTest.java
index 1bef416..16ba82c 100644
--- a/src/test/java/org/apache/datasketches/hive/kll/UnionSketchUDAFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/kll/UnionSketchUDAFTest.java
@@ -21,8 +21,8 @@ package org.apache.datasketches.hive.kll;
import java.util.Arrays;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.kll.KllFloatsSketch;
-import org.apache.datasketches.memory.Memory;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
@@ -114,7 +114,7 @@ public class UnionSketchUDAFTest {
eval.iterate(state, new Object[] { new BytesWritable(sketch2.toByteArray()) });
BytesWritable bytes = (BytesWritable) eval.terminatePartial(state);
- KllFloatsSketch resultSketch = KllFloatsSketch.heapify(Memory.wrap(bytes.getBytes()));
+ KllFloatsSketch resultSketch = KllFloatsSketch.heapify(BytesWritableHelper.wrapAsMemory(bytes));
Assert.assertEquals(resultSketch.getNormalizedRankError(false), KllFloatsSketch.getNormalizedRankError(200, false));
Assert.assertEquals(resultSketch.getNumRetained(), 2);
Assert.assertEquals(resultSketch.getMinValue(), 1f);
@@ -141,7 +141,7 @@ public class UnionSketchUDAFTest {
eval.iterate(state, new Object[] { new BytesWritable(sketch2.toByteArray()), new IntWritable(400) });
BytesWritable bytes = (BytesWritable) eval.terminatePartial(state);
- KllFloatsSketch resultSketch = KllFloatsSketch.heapify(Memory.wrap(bytes.getBytes()));
+ KllFloatsSketch resultSketch = KllFloatsSketch.heapify(BytesWritableHelper.wrapAsMemory(bytes));
Assert.assertEquals(resultSketch.getNormalizedRankError(false), KllFloatsSketch.getNormalizedRankError(400, false));
Assert.assertEquals(resultSketch.getNumRetained(), 2);
Assert.assertEquals(resultSketch.getMinValue(), 1f);
@@ -169,7 +169,7 @@ public class UnionSketchUDAFTest {
eval.merge(state, new BytesWritable(sketch2.toByteArray()));
BytesWritable bytes = (BytesWritable) eval.terminatePartial(state);
- KllFloatsSketch resultSketch = KllFloatsSketch.heapify(Memory.wrap(bytes.getBytes()));
+ KllFloatsSketch resultSketch = KllFloatsSketch.heapify(BytesWritableHelper.wrapAsMemory(bytes));
Assert.assertEquals(resultSketch.getNormalizedRankError(false), KllFloatsSketch.getNormalizedRankError(400, false));
Assert.assertEquals(resultSketch.getNumRetained(), 2);
Assert.assertEquals(resultSketch.getMinValue(), 1f);
@@ -197,7 +197,7 @@ public class UnionSketchUDAFTest {
eval.merge(state, new BytesWritable(sketch2.toByteArray()));
BytesWritable bytes = (BytesWritable) eval.terminate(state);
- KllFloatsSketch resultSketch = KllFloatsSketch.heapify(Memory.wrap(bytes.getBytes()));
+ KllFloatsSketch resultSketch = KllFloatsSketch.heapify(BytesWritableHelper.wrapAsMemory(bytes));
Assert.assertEquals(resultSketch.getNormalizedRankError(false), KllFloatsSketch.getNormalizedRankError(400, false));
Assert.assertEquals(resultSketch.getNumRetained(), 2);
Assert.assertEquals(resultSketch.getMinValue(), 1f);
@@ -225,7 +225,7 @@ public class UnionSketchUDAFTest {
eval.iterate(state, new Object[] { new BytesWritable(sketch2.toByteArray()) });
BytesWritable bytes = (BytesWritable) eval.terminatePartial(state);
- KllFloatsSketch resultSketch = KllFloatsSketch.heapify(Memory.wrap(bytes.getBytes()));
+ KllFloatsSketch resultSketch = KllFloatsSketch.heapify(BytesWritableHelper.wrapAsMemory(bytes));
Assert.assertEquals(resultSketch.getNormalizedRankError(false), KllFloatsSketch.getNormalizedRankError(200, false));
Assert.assertEquals(resultSketch.getNumRetained(), 2);
Assert.assertEquals(resultSketch.getMinValue(), 1f);
diff --git a/src/test/java/org/apache/datasketches/hive/quantiles/DataToDoublesSketchUDAFTest.java b/src/test/java/org/apache/datasketches/hive/quantiles/DataToDoublesSketchUDAFTest.java
index bf51761..6368ec7 100644
--- a/src/test/java/org/apache/datasketches/hive/quantiles/DataToDoublesSketchUDAFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/quantiles/DataToDoublesSketchUDAFTest.java
@@ -21,7 +21,7 @@ package org.apache.datasketches.hive.quantiles;
import java.util.Arrays;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.quantiles.DoublesSketch;
import org.apache.datasketches.quantiles.UpdateDoublesSketch;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
@@ -113,7 +113,7 @@ public class DataToDoublesSketchUDAFTest {
eval.iterate(state, new Object[] { new DoubleWritable(2.0) });
BytesWritable bytes = (BytesWritable) eval.terminatePartial(state);
- DoublesSketch resultSketch = DoublesSketch.wrap(Memory.wrap(bytes.getBytes()));
+ DoublesSketch resultSketch = DoublesSketch.wrap(BytesWritableHelper.wrapAsMemory(bytes));
Assert.assertEquals(resultSketch.getK(), 128);
Assert.assertEquals(resultSketch.getRetainedItems(), 2);
Assert.assertEquals(resultSketch.getMinValue(), 1.0);
@@ -134,7 +134,7 @@ public class DataToDoublesSketchUDAFTest {
eval.iterate(state, new Object[] { new DoubleWritable(2.0), new IntWritable(256) });
BytesWritable bytes = (BytesWritable) eval.terminatePartial(state);
- DoublesSketch resultSketch = DoublesSketch.wrap(Memory.wrap(bytes.getBytes()));
+ DoublesSketch resultSketch = DoublesSketch.wrap(BytesWritableHelper.wrapAsMemory(bytes));
Assert.assertEquals(resultSketch.getK(), 256);
Assert.assertEquals(resultSketch.getRetainedItems(), 2);
Assert.assertEquals(resultSketch.getMinValue(), 1.0);
@@ -161,7 +161,7 @@ public class DataToDoublesSketchUDAFTest {
eval.merge(state, new BytesWritable(sketch2.toByteArray()));
BytesWritable bytes = (BytesWritable) eval.terminate(state);
- DoublesSketch resultSketch = DoublesSketch.wrap(Memory.wrap(bytes.getBytes()));
+ DoublesSketch resultSketch = DoublesSketch.wrap(BytesWritableHelper.wrapAsMemory(bytes));
Assert.assertEquals(resultSketch.getRetainedItems(), 2);
Assert.assertEquals(resultSketch.getMinValue(), 1.0);
Assert.assertEquals(resultSketch.getMaxValue(), 2.0);
@@ -187,7 +187,7 @@ public class DataToDoublesSketchUDAFTest {
eval.merge(state, new BytesWritable(sketch2.toByteArray()));
BytesWritable bytes = (BytesWritable) eval.terminate(state);
- DoublesSketch resultSketch = DoublesSketch.wrap(Memory.wrap(bytes.getBytes()));
+ DoublesSketch resultSketch = DoublesSketch.wrap(BytesWritableHelper.wrapAsMemory(bytes));
Assert.assertEquals(resultSketch.getK(), 256);
Assert.assertEquals(resultSketch.getRetainedItems(), 2);
Assert.assertEquals(resultSketch.getMinValue(), 1.0);
@@ -209,7 +209,7 @@ public class DataToDoublesSketchUDAFTest {
eval.iterate(state, new Object[] { new DoubleWritable(2.0) });
BytesWritable bytes = (BytesWritable) eval.terminate(state);
- DoublesSketch resultSketch = DoublesSketch.wrap(Memory.wrap(bytes.getBytes()));
+ DoublesSketch resultSketch = DoublesSketch.wrap(BytesWritableHelper.wrapAsMemory(bytes));
Assert.assertEquals(resultSketch.getK(), 128);
Assert.assertEquals(resultSketch.getRetainedItems(), 2);
Assert.assertEquals(resultSketch.getMinValue(), 1.0);
@@ -230,7 +230,7 @@ public class DataToDoublesSketchUDAFTest {
eval.iterate(state, new Object[] { new DoubleWritable(2.0), new IntWritable(256) });
BytesWritable bytes = (BytesWritable) eval.terminate(state);
- DoublesSketch resultSketch = DoublesSketch.wrap(Memory.wrap(bytes.getBytes()));
+ DoublesSketch resultSketch = DoublesSketch.wrap(BytesWritableHelper.wrapAsMemory(bytes));
Assert.assertEquals(resultSketch.getK(), 256);
Assert.assertEquals(resultSketch.getRetainedItems(), 2);
Assert.assertEquals(resultSketch.getMinValue(), 1.0);
diff --git a/src/test/java/org/apache/datasketches/hive/quantiles/DataToStringsSketchUDAFTest.java b/src/test/java/org/apache/datasketches/hive/quantiles/DataToStringsSketchUDAFTest.java
index c09d104..4276832 100644
--- a/src/test/java/org/apache/datasketches/hive/quantiles/DataToStringsSketchUDAFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/quantiles/DataToStringsSketchUDAFTest.java
@@ -24,7 +24,7 @@ import java.util.Comparator;
import org.apache.datasketches.ArrayOfItemsSerDe;
import org.apache.datasketches.ArrayOfStringsSerDe;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.quantiles.ItemsSketch;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
@@ -110,7 +110,7 @@ public class DataToStringsSketchUDAFTest {
eval.iterate(state, new Object[] { new org.apache.hadoop.io.Text("b") });
BytesWritable bytes = (BytesWritable) eval.terminatePartial(state);
- ItemsSketch<String> resultSketch = ItemsSketch.getInstance(Memory.wrap(bytes.getBytes()), comparator, serDe);
+ ItemsSketch<String> resultSketch = ItemsSketch.getInstance(BytesWritableHelper.wrapAsMemory(bytes), comparator, serDe);
Assert.assertEquals(resultSketch.getK(), 128);
Assert.assertEquals(resultSketch.getRetainedItems(), 2);
Assert.assertEquals(resultSketch.getMinValue(), "a");
@@ -132,7 +132,7 @@ public class DataToStringsSketchUDAFTest {
eval.iterate(state, new Object[] { new org.apache.hadoop.io.Text("b"), new IntWritable(256) });
BytesWritable bytes = (BytesWritable) eval.terminatePartial(state);
- ItemsSketch<String> resultSketch = ItemsSketch.getInstance(Memory.wrap(bytes.getBytes()), comparator, serDe);
+ ItemsSketch<String> resultSketch = ItemsSketch.getInstance(BytesWritableHelper.wrapAsMemory(bytes), comparator, serDe);
Assert.assertEquals(resultSketch.getK(), 256);
Assert.assertEquals(resultSketch.getRetainedItems(), 2);
Assert.assertEquals(resultSketch.getMinValue(), "a");
@@ -161,7 +161,7 @@ public class DataToStringsSketchUDAFTest {
eval.merge(state, new BytesWritable(sketch2.toByteArray(serDe)));
BytesWritable bytes = (BytesWritable) eval.terminatePartial(state);
- ItemsSketch<String> resultSketch = ItemsSketch.getInstance(Memory.wrap(bytes.getBytes()), comparator, serDe);
+ ItemsSketch<String> resultSketch = ItemsSketch.getInstance(BytesWritableHelper.wrapAsMemory(bytes), comparator, serDe);
Assert.assertEquals(resultSketch.getK(), 256);
Assert.assertEquals(resultSketch.getRetainedItems(), 2);
Assert.assertEquals(resultSketch.getMinValue(), "a");
@@ -190,7 +190,7 @@ public class DataToStringsSketchUDAFTest {
eval.merge(state, new BytesWritable(sketch2.toByteArray(serDe)));
BytesWritable bytes = (BytesWritable) eval.terminate(state);
- ItemsSketch<String> resultSketch = ItemsSketch.getInstance(Memory.wrap(bytes.getBytes()), comparator, serDe);
+ ItemsSketch<String> resultSketch = ItemsSketch.getInstance(BytesWritableHelper.wrapAsMemory(bytes), comparator, serDe);
Assert.assertEquals(resultSketch.getK(), 128);
Assert.assertEquals(resultSketch.getRetainedItems(), 2);
Assert.assertEquals(resultSketch.getMinValue(), "a");
@@ -213,7 +213,7 @@ public class DataToStringsSketchUDAFTest {
eval.iterate(state, new Object[] { new org.apache.hadoop.io.Text("b") });
BytesWritable bytes = (BytesWritable) eval.terminate(state);
- ItemsSketch<String> resultSketch = ItemsSketch.getInstance(Memory.wrap(bytes.getBytes()), comparator, serDe);
+ ItemsSketch<String> resultSketch = ItemsSketch.getInstance(BytesWritableHelper.wrapAsMemory(bytes), comparator, serDe);
Assert.assertEquals(resultSketch.getK(), 128);
Assert.assertEquals(resultSketch.getRetainedItems(), 2);
Assert.assertEquals(resultSketch.getMinValue(), "a");
@@ -235,7 +235,7 @@ public class DataToStringsSketchUDAFTest {
eval.iterate(state, new Object[] { new org.apache.hadoop.io.Text("b"), new IntWritable(256) });
BytesWritable bytes = (BytesWritable) eval.terminate(state);
- ItemsSketch<String> resultSketch = ItemsSketch.getInstance(Memory.wrap(bytes.getBytes()), comparator, serDe);
+ ItemsSketch<String> resultSketch = ItemsSketch.getInstance(BytesWritableHelper.wrapAsMemory(bytes), comparator, serDe);
Assert.assertEquals(resultSketch.getK(), 256);
Assert.assertEquals(resultSketch.getRetainedItems(), 2);
Assert.assertEquals(resultSketch.getMinValue(), "a");
diff --git a/src/test/java/org/apache/datasketches/hive/quantiles/UnionDoublesSketchUDAFTest.java b/src/test/java/org/apache/datasketches/hive/quantiles/UnionDoublesSketchUDAFTest.java
index 39b09b7..b818a19 100644
--- a/src/test/java/org/apache/datasketches/hive/quantiles/UnionDoublesSketchUDAFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/quantiles/UnionDoublesSketchUDAFTest.java
@@ -21,7 +21,7 @@ package org.apache.datasketches.hive.quantiles;
import java.util.Arrays;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.quantiles.DoublesSketch;
import org.apache.datasketches.quantiles.UpdateDoublesSketch;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
@@ -115,7 +115,7 @@ public class UnionDoublesSketchUDAFTest {
eval.iterate(state, new Object[] { new BytesWritable(sketch2.toByteArray()) });
BytesWritable bytes = (BytesWritable) eval.terminatePartial(state);
- DoublesSketch resultSketch = DoublesSketch.wrap(Memory.wrap(bytes.getBytes()));
+ DoublesSketch resultSketch = DoublesSketch.wrap(BytesWritableHelper.wrapAsMemory(bytes));
Assert.assertEquals(resultSketch.getK(), 128);
Assert.assertEquals(resultSketch.getRetainedItems(), 2);
Assert.assertEquals(resultSketch.getMinValue(), 1.0);
@@ -142,7 +142,7 @@ public class UnionDoublesSketchUDAFTest {
eval.iterate(state, new Object[] { new BytesWritable(sketch2.toByteArray()), new IntWritable(256) });
BytesWritable bytes = (BytesWritable) eval.terminatePartial(state);
- DoublesSketch resultSketch = DoublesSketch.wrap(Memory.wrap(bytes.getBytes()));
+ DoublesSketch resultSketch = DoublesSketch.wrap(BytesWritableHelper.wrapAsMemory(bytes));
Assert.assertEquals(resultSketch.getK(), 256);
Assert.assertEquals(resultSketch.getRetainedItems(), 2);
Assert.assertEquals(resultSketch.getMinValue(), 1.0);
@@ -170,7 +170,7 @@ public class UnionDoublesSketchUDAFTest {
eval.merge(state, new BytesWritable(sketch2.toByteArray()));
BytesWritable bytes = (BytesWritable) eval.terminatePartial(state);
- DoublesSketch resultSketch = DoublesSketch.wrap(Memory.wrap(bytes.getBytes()));
+ DoublesSketch resultSketch = DoublesSketch.wrap(BytesWritableHelper.wrapAsMemory(bytes));
Assert.assertEquals(resultSketch.getK(), 256);
Assert.assertEquals(resultSketch.getRetainedItems(), 2);
Assert.assertEquals(resultSketch.getMinValue(), 1.0);
@@ -198,7 +198,7 @@ public class UnionDoublesSketchUDAFTest {
eval.merge(state, new BytesWritable(sketch2.toByteArray()));
BytesWritable bytes = (BytesWritable) eval.terminate(state);
- DoublesSketch resultSketch = DoublesSketch.wrap(Memory.wrap(bytes.getBytes()));
+ DoublesSketch resultSketch = DoublesSketch.wrap(BytesWritableHelper.wrapAsMemory(bytes));
Assert.assertEquals(resultSketch.getK(), 256);
Assert.assertEquals(resultSketch.getRetainedItems(), 2);
Assert.assertEquals(resultSketch.getMinValue(), 1.0);
@@ -226,7 +226,7 @@ public class UnionDoublesSketchUDAFTest {
eval.iterate(state, new Object[] { new BytesWritable(sketch2.toByteArray()) });
BytesWritable bytes = (BytesWritable) eval.terminatePartial(state);
- DoublesSketch resultSketch = DoublesSketch.wrap(Memory.wrap(bytes.getBytes()));
+ DoublesSketch resultSketch = DoublesSketch.wrap(BytesWritableHelper.wrapAsMemory(bytes));
Assert.assertEquals(resultSketch.getK(), 128);
Assert.assertEquals(resultSketch.getRetainedItems(), 2);
Assert.assertEquals(resultSketch.getMinValue(), 1.0);
diff --git a/src/test/java/org/apache/datasketches/hive/quantiles/UnionStringsSketchUDAFTest.java b/src/test/java/org/apache/datasketches/hive/quantiles/UnionStringsSketchUDAFTest.java
index ed8fc36..6df21f1 100644
--- a/src/test/java/org/apache/datasketches/hive/quantiles/UnionStringsSketchUDAFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/quantiles/UnionStringsSketchUDAFTest.java
@@ -24,7 +24,7 @@ import java.util.Comparator;
import org.apache.datasketches.ArrayOfItemsSerDe;
import org.apache.datasketches.ArrayOfStringsSerDe;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.quantiles.ItemsSketch;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
@@ -121,7 +121,7 @@ public class UnionStringsSketchUDAFTest {
eval.iterate(state, new Object[] { new BytesWritable(sketch2.toByteArray(serDe)) });
BytesWritable bytes = (BytesWritable) eval.terminatePartial(state);
- ItemsSketch<String> resultSketch = ItemsSketch.getInstance(Memory.wrap(bytes.getBytes()), comparator, serDe);
+ ItemsSketch<String> resultSketch = ItemsSketch.getInstance(BytesWritableHelper.wrapAsMemory(bytes), comparator, serDe);
Assert.assertEquals(resultSketch.getK(), 128);
Assert.assertEquals(resultSketch.getRetainedItems(), 2);
Assert.assertEquals(resultSketch.getMinValue(), "a");
@@ -149,7 +149,7 @@ public class UnionStringsSketchUDAFTest {
eval.iterate(state, new Object[] { new BytesWritable(sketch2.toByteArray(serDe)), new IntWritable(256) });
BytesWritable bytes = (BytesWritable) eval.terminatePartial(state);
- ItemsSketch<String> resultSketch = ItemsSketch.getInstance(Memory.wrap(bytes.getBytes()), comparator, serDe);
+ ItemsSketch<String> resultSketch = ItemsSketch.getInstance(BytesWritableHelper.wrapAsMemory(bytes), comparator, serDe);
Assert.assertEquals(resultSketch.getK(), 256);
Assert.assertEquals(resultSketch.getRetainedItems(), 2);
Assert.assertEquals(resultSketch.getMinValue(), "a");
@@ -178,7 +178,7 @@ public class UnionStringsSketchUDAFTest {
eval.merge(state, new BytesWritable(sketch2.toByteArray(serDe)));
BytesWritable bytes = (BytesWritable) eval.terminatePartial(state);
- ItemsSketch<String> resultSketch = ItemsSketch.getInstance(Memory.wrap(bytes.getBytes()), comparator, serDe);
+ ItemsSketch<String> resultSketch = ItemsSketch.getInstance(BytesWritableHelper.wrapAsMemory(bytes), comparator, serDe);
Assert.assertEquals(resultSketch.getK(), 256);
Assert.assertEquals(resultSketch.getRetainedItems(), 2);
Assert.assertEquals(resultSketch.getMinValue(), "a");
@@ -207,7 +207,7 @@ public class UnionStringsSketchUDAFTest {
eval.merge(state, new BytesWritable(sketch2.toByteArray(serDe)));
BytesWritable bytes = (BytesWritable) eval.terminate(state);
- ItemsSketch<String> resultSketch = ItemsSketch.getInstance(Memory.wrap(bytes.getBytes()), comparator, serDe);
+ ItemsSketch<String> resultSketch = ItemsSketch.getInstance(BytesWritableHelper.wrapAsMemory(bytes), comparator, serDe);
Assert.assertEquals(resultSketch.getK(), 256);
Assert.assertEquals(resultSketch.getRetainedItems(), 2);
Assert.assertEquals(resultSketch.getMinValue(), "a");
@@ -236,7 +236,7 @@ public class UnionStringsSketchUDAFTest {
eval.iterate(state, new Object[] { new BytesWritable(sketch2.toByteArray(serDe)) });
BytesWritable bytes = (BytesWritable) eval.terminate(state);
- ItemsSketch<String> resultSketch = ItemsSketch.getInstance(Memory.wrap(bytes.getBytes()), comparator, serDe);
+ ItemsSketch<String> resultSketch = ItemsSketch.getInstance(BytesWritableHelper.wrapAsMemory(bytes), comparator, serDe);
Assert.assertEquals(resultSketch.getK(), 128);
Assert.assertEquals(resultSketch.getRetainedItems(), 2);
Assert.assertEquals(resultSketch.getMinValue(), "a");
diff --git a/src/test/java/org/apache/datasketches/hive/theta/DataToSketchUDAFTest.java b/src/test/java/org/apache/datasketches/hive/theta/DataToSketchUDAFTest.java
index b2b8e5e..ed4afe6 100644
--- a/src/test/java/org/apache/datasketches/hive/theta/DataToSketchUDAFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/theta/DataToSketchUDAFTest.java
@@ -25,7 +25,7 @@ import static org.apache.datasketches.Util.DEFAULT_UPDATE_SEED;
import java.util.Arrays;
import java.util.List;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.theta.SetOperation;
import org.apache.datasketches.theta.Sketch;
import org.apache.datasketches.theta.Sketches;
@@ -180,7 +180,7 @@ public class DataToSketchUDAFTest {
Assert.assertEquals(r.size(), 3);
Assert.assertEquals(((IntWritable) r.get(0)).get(), DEFAULT_NOMINAL_ENTRIES);
Assert.assertEquals(((LongWritable) r.get(1)).get(), DEFAULT_UPDATE_SEED);
- Sketch resultSketch = Sketches.wrapSketch(Memory.wrap(((BytesWritable) r.get(2)).getBytes()));
+ Sketch resultSketch = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)));
Assert.assertFalse(resultSketch.isEstimationMode());
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
}
@@ -206,7 +206,7 @@ public class DataToSketchUDAFTest {
Assert.assertEquals(r.size(), 3);
Assert.assertEquals(((IntWritable) r.get(0)).get(), 16);
Assert.assertEquals(((LongWritable) r.get(1)).get(), seed);
- Sketch resultSketch = Sketches.wrapSketch(Memory.wrap(((BytesWritable) r.get(2)).getBytes()), seed);
+ Sketch resultSketch = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)), seed);
// because of sampling probability < 1
Assert.assertTrue(resultSketch.isEstimationMode());
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.05);
@@ -252,7 +252,7 @@ public class DataToSketchUDAFTest {
Assert.assertEquals(r.size(), 3);
Assert.assertEquals(((IntWritable) r.get(0)).get(), DEFAULT_NOMINAL_ENTRIES);
Assert.assertEquals(((LongWritable) r.get(1)).get(), DEFAULT_UPDATE_SEED);
- Sketch resultSketch = Sketches.wrapSketch(Memory.wrap(((BytesWritable) r.get(2)).getBytes()));
+ Sketch resultSketch = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)));
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
}
}
@@ -287,7 +287,7 @@ public class DataToSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- Sketch resultSketch = Sketches.wrapSketch(Memory.wrap(((BytesWritable) result).getBytes()));
+ Sketch resultSketch = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
}
}
@@ -308,7 +308,7 @@ public class DataToSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- Sketch resultSketch = Sketches.wrapSketch(Memory.wrap(((BytesWritable) result).getBytes()));
+ Sketch resultSketch = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
eval.reset(state);
@@ -333,7 +333,7 @@ public class DataToSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- Sketch resultSketch = Sketches.wrapSketch(Memory.wrap(((BytesWritable) result).getBytes()), seed);
+ Sketch resultSketch = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) result), seed);
// because of sampling probability < 1
Assert.assertTrue(resultSketch.isEstimationMode());
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.05);
diff --git a/src/test/java/org/apache/datasketches/hive/theta/ExcludeSketchUDFTest.java b/src/test/java/org/apache/datasketches/hive/theta/ExcludeSketchUDFTest.java
index 0bf1425..225f18a 100644
--- a/src/test/java/org/apache/datasketches/hive/theta/ExcludeSketchUDFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/theta/ExcludeSketchUDFTest.java
@@ -22,6 +22,7 @@ package org.apache.datasketches.hive.theta;
import static org.apache.datasketches.Util.DEFAULT_UPDATE_SEED;
import static org.testng.AssertJUnit.assertEquals;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.hadoop.io.BytesWritable;
import org.testng.annotations.Test;
@@ -39,7 +40,7 @@ public class ExcludeSketchUDFTest {
BytesWritable intermResult = testObject.evaluate(null, null);
- Memory mem = Memory.wrap(intermResult.getBytes());
+ Memory mem = BytesWritableHelper.wrapAsMemory(intermResult);
Sketch testResult = Sketches.wrapSketch(mem);
@@ -52,7 +53,7 @@ public class ExcludeSketchUDFTest {
BytesWritable intermResult = testObject.evaluate(new BytesWritable(), new BytesWritable());
- Memory mem = Memory.wrap(intermResult.getBytes());
+ Memory mem = BytesWritableHelper.wrapAsMemory(intermResult);
Sketch testResult = Sketches.wrapSketch(mem);
@@ -78,7 +79,7 @@ public class ExcludeSketchUDFTest {
BytesWritable output = testObject.evaluate(input1, input2);
- Sketch result = Sketches.wrapSketch(Memory.wrap(output.getBytes()));
+ Sketch result = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory(output));
assertEquals(100.0, result.getEstimate());
}
@@ -102,7 +103,7 @@ public class ExcludeSketchUDFTest {
BytesWritable output = testObject.evaluate(input1, input2, DEFAULT_UPDATE_SEED);
- Sketch result = Sketches.wrapSketch(Memory.wrap(output.getBytes()));
+ Sketch result = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory(output));
assertEquals(100.0, result.getEstimate());
}
@@ -127,7 +128,7 @@ public class ExcludeSketchUDFTest {
BytesWritable output = testObject.evaluate(input1, input2, seed);
- Sketch result = Sketches.wrapSketch(Memory.wrap(output.getBytes()), seed);
+ Sketch result = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory(output), seed);
assertEquals(100.0, result.getEstimate());
}
diff --git a/src/test/java/org/apache/datasketches/hive/theta/IntersectSketchUDAFTest.java b/src/test/java/org/apache/datasketches/hive/theta/IntersectSketchUDAFTest.java
index b84cfc1..7432557 100644
--- a/src/test/java/org/apache/datasketches/hive/theta/IntersectSketchUDAFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/theta/IntersectSketchUDAFTest.java
@@ -24,7 +24,7 @@ import static org.apache.datasketches.Util.DEFAULT_UPDATE_SEED;
import java.util.Arrays;
import java.util.List;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.theta.Sketch;
import org.apache.datasketches.theta.Sketches;
import org.apache.datasketches.theta.UpdateSketch;
@@ -134,7 +134,7 @@ public class IntersectSketchUDAFTest {
List<?> r = (List<?>) result;
Assert.assertEquals(r.size(), 2);
Assert.assertEquals(((LongWritable) r.get(0)).get(), DEFAULT_UPDATE_SEED);
- Sketch resultSketch = Sketches.wrapSketch(Memory.wrap(((BytesWritable) r.get(1)).getBytes()));
+ Sketch resultSketch = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(1)));
Assert.assertEquals(resultSketch.getRetainedEntries(true), 2);
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
}
@@ -170,7 +170,7 @@ public class IntersectSketchUDAFTest {
List<?> r = (List<?>) result;
Assert.assertEquals(r.size(), 2);
Assert.assertEquals(((LongWritable) r.get(0)).get(), seed);
- Sketch resultSketch = Sketches.wrapSketch(Memory.wrap(((BytesWritable) r.get(1)).getBytes()), seed);
+ Sketch resultSketch = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(1)), seed);
Assert.assertEquals(resultSketch.getRetainedEntries(true), 2);
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
}
@@ -212,7 +212,7 @@ public class IntersectSketchUDAFTest {
List<?> r = (List<?>) result;
Assert.assertEquals(r.size(), 2);
Assert.assertEquals(((LongWritable) r.get(0)).get(), DEFAULT_UPDATE_SEED);
- Sketch resultSketch = Sketches.wrapSketch(Memory.wrap(((BytesWritable) r.get(1)).getBytes()));
+ Sketch resultSketch = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(1)));
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
}
}
@@ -248,7 +248,7 @@ public class IntersectSketchUDAFTest {
));
BytesWritable bytes = (BytesWritable) eval.terminate(state);
- Sketch resultSketch = Sketches.wrapSketch(Memory.wrap(bytes.getBytes()));
+ Sketch resultSketch = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory(bytes));
Assert.assertEquals(resultSketch.getRetainedEntries(true), 2);
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
}
@@ -281,7 +281,7 @@ public class IntersectSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- Sketch resultSketch = Sketches.wrapSketch(Memory.wrap(((BytesWritable) result).getBytes()));
+ Sketch resultSketch = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
eval.reset(state);
diff --git a/src/test/java/org/apache/datasketches/hive/theta/IntersectSketchUDFTest.java b/src/test/java/org/apache/datasketches/hive/theta/IntersectSketchUDFTest.java
index 761874c..9792c90 100644
--- a/src/test/java/org/apache/datasketches/hive/theta/IntersectSketchUDFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/theta/IntersectSketchUDFTest.java
@@ -21,6 +21,7 @@ package org.apache.datasketches.hive.theta;
import static org.testng.AssertJUnit.assertEquals;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.hadoop.io.BytesWritable;
import org.testng.annotations.Test;
@@ -36,7 +37,7 @@ public class IntersectSketchUDFTest {
public void evaluateNull() {
IntersectSketchUDF testObject = new IntersectSketchUDF();
BytesWritable intermResult = testObject.evaluate(null, null);
- Memory mem = Memory.wrap(intermResult.getBytes());
+ Memory mem = BytesWritableHelper.wrapAsMemory(intermResult);
Sketch testResult = Sketches.wrapSketch(mem);
assertEquals(0.0, testResult.getEstimate());
}
@@ -45,7 +46,7 @@ public class IntersectSketchUDFTest {
public void evaluateEmpty() {
IntersectSketchUDF testObject = new IntersectSketchUDF();
BytesWritable intermResult = testObject.evaluate(new BytesWritable(), new BytesWritable());
- Memory mem = Memory.wrap(intermResult.getBytes());
+ Memory mem = BytesWritableHelper.wrapAsMemory(intermResult);
Sketch testResult = Sketches.wrapSketch(mem);
assertEquals(0.0, testResult.getEstimate());
}
@@ -69,7 +70,7 @@ public class IntersectSketchUDFTest {
BytesWritable output = testObject.evaluate(input1, input2);
- Sketch result = Sketches.wrapSketch(Memory.wrap(output.getBytes()));
+ Sketch result = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory(output));
assertEquals(28.0, result.getEstimate());
}
@@ -94,7 +95,7 @@ public class IntersectSketchUDFTest {
BytesWritable output = testObject.evaluate(input1, input2, seed);
- Sketch result = Sketches.wrapSketch(Memory.wrap(output.getBytes()), seed);
+ Sketch result = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory(output), seed);
assertEquals(28.0, result.getEstimate());
}
diff --git a/src/test/java/org/apache/datasketches/hive/theta/SampleSketchUDF.java b/src/test/java/org/apache/datasketches/hive/theta/SampleSketchUDF.java
index 9c49586..07b14d9 100644
--- a/src/test/java/org/apache/datasketches/hive/theta/SampleSketchUDF.java
+++ b/src/test/java/org/apache/datasketches/hive/theta/SampleSketchUDF.java
@@ -19,6 +19,7 @@
package org.apache.datasketches.hive.theta;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
@@ -55,16 +56,16 @@ public class SampleSketchUDF extends UDF {
return null;
}
- byte[] serializedSketch = binarySketch.getBytes();
+ Memory serializedSketch = BytesWritableHelper.wrapAsMemory(binarySketch);
- if (serializedSketch.length <= 8) {
+ if (serializedSketch.getCapacity() <= 8) {
return null;
}
// The builder will catch errors with improper sketchSize or probability
Union union = SetOperation.builder().setP(probability).setNominalEntries(sketchSize).buildUnion();
- union.update(Memory.wrap(serializedSketch)); //Union can accept Memory object directly
+ union.update(serializedSketch); //Union can accept Memory object directly
Sketch intermediateSketch = union.getResult(false, null); //to CompactSketch(unordered, on-heap)
byte[] resultSketch = intermediateSketch.toByteArray();
diff --git a/src/test/java/org/apache/datasketches/hive/theta/UnionSketchUDAFTest.java b/src/test/java/org/apache/datasketches/hive/theta/UnionSketchUDAFTest.java
index 978a7ea..c74f7a8 100644
--- a/src/test/java/org/apache/datasketches/hive/theta/UnionSketchUDAFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/theta/UnionSketchUDAFTest.java
@@ -25,7 +25,7 @@ import static org.apache.datasketches.Util.DEFAULT_UPDATE_SEED;
import java.util.Arrays;
import java.util.List;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.theta.Sketch;
import org.apache.datasketches.theta.Sketches;
import org.apache.datasketches.theta.UpdateSketch;
@@ -155,7 +155,7 @@ public class UnionSketchUDAFTest {
Assert.assertEquals(r.size(), 3);
Assert.assertEquals(((IntWritable) r.get(0)).get(), DEFAULT_NOMINAL_ENTRIES);
Assert.assertEquals(((LongWritable) r.get(1)).get(), DEFAULT_UPDATE_SEED);
- Sketch resultSketch = Sketches.wrapSketch(Memory.wrap(((BytesWritable) r.get(2)).getBytes()));
+ Sketch resultSketch = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)));
Assert.assertFalse(resultSketch.isEstimationMode());
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
}
@@ -188,7 +188,7 @@ public class UnionSketchUDAFTest {
Assert.assertEquals(r.size(), 3);
Assert.assertEquals(((IntWritable) r.get(0)).get(), nomEntries);
Assert.assertEquals(((LongWritable) r.get(1)).get(), seed);
- Sketch resultSketch = Sketches.wrapSketch(Memory.wrap(((BytesWritable) r.get(2)).getBytes()), seed);
+ Sketch resultSketch = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)), seed);
Assert.assertFalse(resultSketch.isEstimationMode());
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
}
@@ -228,7 +228,7 @@ public class UnionSketchUDAFTest {
Assert.assertEquals(r.size(), 3);
Assert.assertEquals(((IntWritable) r.get(0)).get(), DEFAULT_NOMINAL_ENTRIES);
Assert.assertEquals(((LongWritable) r.get(1)).get(), DEFAULT_UPDATE_SEED);
- Sketch resultSketch = Sketches.wrapSketch(Memory.wrap(((BytesWritable) r.get(2)).getBytes()));
+ Sketch resultSketch = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)));
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
}
}
@@ -263,7 +263,7 @@ public class UnionSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- Sketch resultSketch = Sketches.wrapSketch(Memory.wrap(((BytesWritable) result).getBytes()));
+ Sketch resultSketch = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
}
}
@@ -290,7 +290,7 @@ public class UnionSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- Sketch resultSketch = Sketches.wrapSketch(Memory.wrap(((BytesWritable) result).getBytes()));
+ Sketch resultSketch = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
eval.reset(state);
@@ -322,7 +322,7 @@ public class UnionSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- Sketch resultSketch = Sketches.wrapSketch(Memory.wrap(((BytesWritable) result).getBytes()), seed);
+ Sketch resultSketch = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) result), seed);
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
}
}
diff --git a/src/test/java/org/apache/datasketches/hive/theta/UnionSketchUDFTest.java b/src/test/java/org/apache/datasketches/hive/theta/UnionSketchUDFTest.java
index 15f9f1b..e66179f 100644
--- a/src/test/java/org/apache/datasketches/hive/theta/UnionSketchUDFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/theta/UnionSketchUDFTest.java
@@ -19,6 +19,7 @@
package org.apache.datasketches.hive.theta;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.hadoop.io.BytesWritable;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -35,7 +36,7 @@ public class UnionSketchUDFTest {
public void evaluateNull() {
UnionSketchUDF testObject = new UnionSketchUDF();
BytesWritable intermResult = testObject.evaluate(null, null);
- Memory mem = Memory.wrap(intermResult.getBytes());
+ Memory mem = BytesWritableHelper.wrapAsMemory(intermResult);
Sketch testResult = Sketches.wrapSketch(mem);
Assert.assertEquals(testResult.getEstimate(), 0.0);
}
@@ -44,7 +45,7 @@ public class UnionSketchUDFTest {
public void testEvaluateEmpty() {
UnionSketchUDF testObject = new UnionSketchUDF();
BytesWritable intermResult = testObject.evaluate(new BytesWritable(), new BytesWritable());
- Memory mem = Memory.wrap(intermResult.getBytes());
+ Memory mem = BytesWritableHelper.wrapAsMemory(intermResult);
Sketch testResult = Sketches.wrapSketch(mem);
Assert.assertEquals(testResult.getEstimate(), 0.0);
}
@@ -68,7 +69,7 @@ public class UnionSketchUDFTest {
BytesWritable output = testObject.evaluate(input1, input2);
- Sketch result = Sketches.wrapSketch(Memory.wrap(output.getBytes()));
+ Sketch result = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory(output));
Assert.assertEquals(result.getEstimate(), 256.0);
}
@@ -93,7 +94,7 @@ public class UnionSketchUDFTest {
BytesWritable output = testObject.evaluate(input1, input2, 128, seed);
- Sketch result = Sketches.wrapSketch(Memory.wrap(output.getBytes()), seed);
+ Sketch result = Sketches.wrapSketch(BytesWritableHelper.wrapAsMemory(output), seed);
Assert.assertEquals(result.getEstimate(), 256.0, 256 * 0.02);
Assert.assertTrue(result.getRetainedEntries(true) <= 128.0);
diff --git a/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToQuantilesSketchUDFTest.java b/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToQuantilesSketchUDFTest.java
index bec9614..2b4e45d 100644
--- a/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToQuantilesSketchUDFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToQuantilesSketchUDFTest.java
@@ -19,11 +19,11 @@
package org.apache.datasketches.hive.tuple;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.hadoop.io.BytesWritable;
import org.testng.Assert;
import org.testng.annotations.Test;
-import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.quantiles.DoublesSketch;
import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketch;
import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketchBuilder;
@@ -60,7 +60,7 @@ public class ArrayOfDoublesSketchToQuantilesSketchUDFTest {
ArrayOfDoublesUpdatableSketch sketch = new ArrayOfDoublesUpdatableSketchBuilder().build();
BytesWritable result = new ArrayOfDoublesSketchToQuantilesSketchUDF().evaluate(new BytesWritable(sketch.compact().toByteArray()));
Assert.assertNotNull(result);
- DoublesSketch qs = DoublesSketch.wrap(Memory.wrap(result.getBytes()));
+ DoublesSketch qs = DoublesSketch.wrap(BytesWritableHelper.wrapAsMemory(result));
Assert.assertTrue(qs.isEmpty());
}
@@ -74,7 +74,7 @@ public class ArrayOfDoublesSketchToQuantilesSketchUDFTest {
1
);
Assert.assertNotNull(result);
- DoublesSketch qs = DoublesSketch.wrap(Memory.wrap(result.getBytes()));
+ DoublesSketch qs = DoublesSketch.wrap(BytesWritableHelper.wrapAsMemory(result));
Assert.assertFalse(qs.isEmpty());
Assert.assertEquals(qs.getMinValue(), 1.0);
Assert.assertEquals(qs.getMaxValue(), 10.0);
@@ -92,7 +92,7 @@ public class ArrayOfDoublesSketchToQuantilesSketchUDFTest {
k
);
Assert.assertNotNull(result);
- DoublesSketch qs = DoublesSketch.wrap(Memory.wrap(result.getBytes()));
+ DoublesSketch qs = DoublesSketch.wrap(BytesWritableHelper.wrapAsMemory(result));
Assert.assertFalse(qs.isEmpty());
Assert.assertEquals(qs.getK(), k);
Assert.assertEquals(qs.getMinValue(), 2.0);
diff --git a/src/test/java/org/apache/datasketches/hive/tuple/DataToArrayOfDoublesSketchUDAFTest.java b/src/test/java/org/apache/datasketches/hive/tuple/DataToArrayOfDoublesSketchUDAFTest.java
index 0beca3b..3db4a28 100644
--- a/src/test/java/org/apache/datasketches/hive/tuple/DataToArrayOfDoublesSketchUDAFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/tuple/DataToArrayOfDoublesSketchUDAFTest.java
@@ -24,7 +24,7 @@ import static org.apache.datasketches.Util.DEFAULT_NOMINAL_ENTRIES;
import java.util.Arrays;
import java.util.List;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.tuple.ArrayOfDoublesSketch;
import org.apache.datasketches.tuple.ArrayOfDoublesSketches;
import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketch;
@@ -171,7 +171,7 @@ public class DataToArrayOfDoublesSketchUDAFTest {
Assert.assertEquals(r.size(), 3);
Assert.assertEquals(((IntWritable) r.get(0)).get(), DEFAULT_NOMINAL_ENTRIES);
Assert.assertEquals(((IntWritable) r.get(1)).get(), 1);
- ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(Memory.wrap(((BytesWritable) r.get(2)).getBytes()));
+ ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)));
Assert.assertFalse(resultSketch.isEstimationMode());
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
}
@@ -196,7 +196,7 @@ public class DataToArrayOfDoublesSketchUDAFTest {
Assert.assertEquals(r.size(), 3);
Assert.assertEquals(((IntWritable) r.get(0)).get(), 32);
Assert.assertEquals(((IntWritable) r.get(1)).get(), 2);
- ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(Memory.wrap(((BytesWritable) r.get(2)).getBytes()));
+ ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)));
// because of sampling probability < 1
Assert.assertTrue(resultSketch.isEstimationMode());
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.05);
@@ -237,7 +237,7 @@ public class DataToArrayOfDoublesSketchUDAFTest {
Assert.assertEquals(r.size(), 3);
Assert.assertEquals(((IntWritable) r.get(0)).get(), DEFAULT_NOMINAL_ENTRIES);
Assert.assertEquals(((IntWritable) r.get(1)).get(), 1);
- ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(Memory.wrap(((BytesWritable) r.get(2)).getBytes()));
+ ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)));
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
eval.reset(state);
@@ -276,7 +276,7 @@ public class DataToArrayOfDoublesSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(Memory.wrap(((BytesWritable) result).getBytes()));
+ ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
}
}
@@ -297,7 +297,7 @@ public class DataToArrayOfDoublesSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(Memory.wrap(((BytesWritable) result).getBytes()));
+ ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
eval.reset(state);
@@ -321,7 +321,7 @@ public class DataToArrayOfDoublesSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(Memory.wrap(((BytesWritable) result).getBytes()));
+ ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
// because of sampling probability < 1
Assert.assertTrue(resultSketch.isEstimationMode());
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.05);
@@ -348,7 +348,7 @@ public class DataToArrayOfDoublesSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(Memory.wrap(((BytesWritable) result).getBytes()));
+ ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
Assert.assertEquals(resultSketch.getEstimate(), 10000.0, 10000 * 0.03);
Assert.assertTrue(resultSketch.getRetainedEntries() <= 4096, "retained entries: " + resultSketch.getRetainedEntries());
diff --git a/src/test/java/org/apache/datasketches/hive/tuple/DataToDoubleSummarySketchUDAFTest.java b/src/test/java/org/apache/datasketches/hive/tuple/DataToDoubleSummarySketchUDAFTest.java
index 0158220..2d7a725 100644
--- a/src/test/java/org/apache/datasketches/hive/tuple/DataToDoubleSummarySketchUDAFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/tuple/DataToDoubleSummarySketchUDAFTest.java
@@ -24,7 +24,7 @@ import static org.apache.datasketches.Util.DEFAULT_NOMINAL_ENTRIES;
import java.util.Arrays;
import java.util.List;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.tuple.Sketch;
import org.apache.datasketches.tuple.Sketches;
import org.apache.datasketches.tuple.UpdatableSketch;
@@ -161,7 +161,7 @@ public class DataToDoubleSummarySketchUDAFTest {
Assert.assertEquals(r.size(), 2);
Assert.assertEquals(((IntWritable) r.get(0)).get(), DEFAULT_NOMINAL_ENTRIES);
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) r.get(1)).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(1)), new DoubleSummaryDeserializer());
Assert.assertFalse(resultSketch.isEstimationMode());
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
}
@@ -187,7 +187,7 @@ public class DataToDoubleSummarySketchUDAFTest {
Assert.assertEquals(r.size(), 2);
Assert.assertEquals(((IntWritable) r.get(0)).get(), 32);
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) r.get(1)).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(1)), new DoubleSummaryDeserializer());
// because of sampling probability < 1
Assert.assertTrue(resultSketch.isEstimationMode());
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.05);
@@ -227,7 +227,7 @@ public class DataToDoubleSummarySketchUDAFTest {
Assert.assertEquals(r.size(), 2);
Assert.assertEquals(((IntWritable) r.get(0)).get(), DEFAULT_NOMINAL_ENTRIES);
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) r.get(1)).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(1)), new DoubleSummaryDeserializer());
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
eval.reset(state);
@@ -266,7 +266,7 @@ public class DataToDoubleSummarySketchUDAFTest {
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) result).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) result), new DoubleSummaryDeserializer());
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
}
}
@@ -289,7 +289,7 @@ public class DataToDoubleSummarySketchUDAFTest {
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) result).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) result), new DoubleSummaryDeserializer());
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
eval.reset(state);
@@ -316,7 +316,7 @@ public class DataToDoubleSummarySketchUDAFTest {
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) result).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) result), new DoubleSummaryDeserializer());
// because of sampling probability < 1
Assert.assertTrue(resultSketch.isEstimationMode());
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.05);
@@ -345,7 +345,7 @@ public class DataToDoubleSummarySketchUDAFTest {
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) result).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) result), new DoubleSummaryDeserializer());
Assert.assertEquals(resultSketch.getEstimate(), 10000.0, 10000 * 0.03);
Assert.assertTrue(resultSketch.getRetainedEntries() <= 4096, "retained entries: " + resultSketch.getRetainedEntries());
diff --git a/src/test/java/org/apache/datasketches/hive/tuple/DataToDoubleSummaryWithModeSketchUDAFTest.java b/src/test/java/org/apache/datasketches/hive/tuple/DataToDoubleSummaryWithModeSketchUDAFTest.java
index 1fc8be2..1af7ef9 100644
--- a/src/test/java/org/apache/datasketches/hive/tuple/DataToDoubleSummaryWithModeSketchUDAFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/tuple/DataToDoubleSummaryWithModeSketchUDAFTest.java
@@ -24,6 +24,7 @@ import static org.apache.datasketches.Util.DEFAULT_NOMINAL_ENTRIES;
import java.util.Arrays;
import java.util.List;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.tuple.Sketch;
import org.apache.datasketches.tuple.SketchIterator;
@@ -165,7 +166,7 @@ public class DataToDoubleSummaryWithModeSketchUDAFTest {
Assert.assertEquals(((IntWritable) r.get(0)).get(), DEFAULT_NOMINAL_ENTRIES);
Assert.assertEquals(((Text) r.get(1)).toString(), DoubleSummary.Mode.Sum.toString());
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) r.get(2)).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)), new DoubleSummaryDeserializer());
Assert.assertFalse(resultSketch.isEstimationMode());
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
SketchIterator<DoubleSummary> it = resultSketch.iterator();
@@ -198,7 +199,7 @@ public class DataToDoubleSummaryWithModeSketchUDAFTest {
Assert.assertEquals(((IntWritable) r.get(0)).get(), 32);
Assert.assertEquals(((Text) r.get(1)).toString(), DoubleSummary.Mode.Min.toString());
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) r.get(2)).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)), new DoubleSummaryDeserializer());
// because of sampling probability < 1
Assert.assertTrue(resultSketch.isEstimationMode());
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.05);
@@ -247,7 +248,7 @@ public class DataToDoubleSummaryWithModeSketchUDAFTest {
Assert.assertEquals(((IntWritable) r.get(0)).get(), DEFAULT_NOMINAL_ENTRIES);
Assert.assertEquals(((Text) r.get(1)).toString(), DoubleSummary.Mode.Sum.toString());
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) r.get(2)).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)), new DoubleSummaryDeserializer());
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
SketchIterator<DoubleSummary> it = resultSketch.iterator();
while (it.next()) {
@@ -294,7 +295,7 @@ public class DataToDoubleSummaryWithModeSketchUDAFTest {
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) result).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) result), new DoubleSummaryDeserializer());
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
SketchIterator<DoubleSummary> it = resultSketch.iterator();
while (it.next()) {
@@ -323,7 +324,7 @@ public class DataToDoubleSummaryWithModeSketchUDAFTest {
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) result).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) result), new DoubleSummaryDeserializer());
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
SketchIterator<DoubleSummary> it = resultSketch.iterator();
while (it.next()) {
@@ -356,7 +357,7 @@ public class DataToDoubleSummaryWithModeSketchUDAFTest {
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) result).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) result), new DoubleSummaryDeserializer());
// because of sampling probability < 1
Assert.assertTrue(resultSketch.isEstimationMode());
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.05);
@@ -389,7 +390,7 @@ public class DataToDoubleSummaryWithModeSketchUDAFTest {
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) result).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) result), new DoubleSummaryDeserializer());
Assert.assertEquals(resultSketch.getEstimate(), 10000.0, 10000 * 0.03);
Assert.assertTrue(resultSketch.getRetainedEntries() <= 4096, "retained entries: " + resultSketch.getRetainedEntries());
diff --git a/src/test/java/org/apache/datasketches/hive/tuple/UnionArrayOfDoublesSketchUDAFTest.java b/src/test/java/org/apache/datasketches/hive/tuple/UnionArrayOfDoublesSketchUDAFTest.java
index ac00988..87d0a83 100644
--- a/src/test/java/org/apache/datasketches/hive/tuple/UnionArrayOfDoublesSketchUDAFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/tuple/UnionArrayOfDoublesSketchUDAFTest.java
@@ -24,7 +24,7 @@ import static org.apache.datasketches.Util.DEFAULT_NOMINAL_ENTRIES;
import java.util.Arrays;
import java.util.List;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.tuple.ArrayOfDoublesSketch;
import org.apache.datasketches.tuple.ArrayOfDoublesSketches;
import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketch;
@@ -144,7 +144,7 @@ public class UnionArrayOfDoublesSketchUDAFTest {
Assert.assertEquals(r.size(), 3);
Assert.assertEquals(((IntWritable) r.get(0)).get(), DEFAULT_NOMINAL_ENTRIES);
Assert.assertEquals(((IntWritable) r.get(1)).get(), 1);
- ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(Memory.wrap(((BytesWritable) r.get(2)).getBytes()));
+ ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)));
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
}
}
@@ -180,7 +180,7 @@ public class UnionArrayOfDoublesSketchUDAFTest {
Assert.assertEquals(r.size(), 3);
Assert.assertEquals(((IntWritable) r.get(0)).get(), nomNumEntries);
Assert.assertEquals(((IntWritable) r.get(1)).get(), numValues);
- ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(Memory.wrap(((BytesWritable) r.get(2)).getBytes()));
+ ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)));
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
}
}
@@ -219,7 +219,7 @@ public class UnionArrayOfDoublesSketchUDAFTest {
Assert.assertEquals(r.size(), 3);
Assert.assertEquals(((IntWritable) r.get(0)).get(), DEFAULT_NOMINAL_ENTRIES);
Assert.assertEquals(((IntWritable) r.get(1)).get(), 1);
- ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(Memory.wrap(((BytesWritable) r.get(2)).getBytes()));
+ ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)));
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
eval.reset(state);
@@ -258,7 +258,7 @@ public class UnionArrayOfDoublesSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(Memory.wrap(((BytesWritable) result).getBytes()));
+ ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
}
}
@@ -285,7 +285,7 @@ public class UnionArrayOfDoublesSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(Memory.wrap(((BytesWritable) result).getBytes()));
+ ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
eval.reset(state);
@@ -321,7 +321,7 @@ public class UnionArrayOfDoublesSketchUDAFTest {
Object result = eval.terminate(state);
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
- ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(Memory.wrap(((BytesWritable) result).getBytes()));
+ ArrayOfDoublesSketch resultSketch = ArrayOfDoublesSketches.wrapSketch(BytesWritableHelper.wrapAsMemory((BytesWritable) result));
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
eval.reset(state);
diff --git a/src/test/java/org/apache/datasketches/hive/tuple/UnionDoubleSummarySketchUDAFTest.java b/src/test/java/org/apache/datasketches/hive/tuple/UnionDoubleSummarySketchUDAFTest.java
index 132601c..5d441c5 100644
--- a/src/test/java/org/apache/datasketches/hive/tuple/UnionDoubleSummarySketchUDAFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/tuple/UnionDoubleSummarySketchUDAFTest.java
@@ -24,7 +24,7 @@ import static org.apache.datasketches.Util.DEFAULT_NOMINAL_ENTRIES;
import java.util.Arrays;
import java.util.List;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.tuple.Sketch;
import org.apache.datasketches.tuple.Sketches;
import org.apache.datasketches.tuple.UpdatableSketch;
@@ -134,7 +134,7 @@ public class UnionDoubleSummarySketchUDAFTest {
Assert.assertEquals(r.size(), 2);
Assert.assertEquals(((IntWritable) r.get(0)).get(), DEFAULT_NOMINAL_ENTRIES);
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) r.get(1)).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(1)), new DoubleSummaryDeserializer());
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
}
}
@@ -170,7 +170,7 @@ public class UnionDoubleSummarySketchUDAFTest {
Assert.assertEquals(r.size(), 2);
Assert.assertEquals(((IntWritable) r.get(0)).get(), nomNumEntries);
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) r.get(1)).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(1)), new DoubleSummaryDeserializer());
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
}
}
@@ -208,7 +208,7 @@ public class UnionDoubleSummarySketchUDAFTest {
Assert.assertEquals(r.size(), 2);
Assert.assertEquals(((IntWritable) r.get(0)).get(), DEFAULT_NOMINAL_ENTRIES);
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) r.get(1)).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(1)), new DoubleSummaryDeserializer());
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
eval.reset(state);
@@ -247,7 +247,7 @@ public class UnionDoubleSummarySketchUDAFTest {
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) result).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) result), new DoubleSummaryDeserializer());
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
}
}
@@ -276,7 +276,7 @@ public class UnionDoubleSummarySketchUDAFTest {
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) result).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) result), new DoubleSummaryDeserializer());
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
eval.reset(state);
@@ -313,7 +313,7 @@ public class UnionDoubleSummarySketchUDAFTest {
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) result).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) result), new DoubleSummaryDeserializer());
Assert.assertEquals(resultSketch.getEstimate(), 2.0, 0.05);
eval.reset(state);
diff --git a/src/test/java/org/apache/datasketches/hive/tuple/UnionDoubleSummaryWithModeSketchUDAFTest.java b/src/test/java/org/apache/datasketches/hive/tuple/UnionDoubleSummaryWithModeSketchUDAFTest.java
index 99cd9d8..67a5309 100644
--- a/src/test/java/org/apache/datasketches/hive/tuple/UnionDoubleSummaryWithModeSketchUDAFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/tuple/UnionDoubleSummaryWithModeSketchUDAFTest.java
@@ -24,7 +24,7 @@ import static org.apache.datasketches.Util.DEFAULT_NOMINAL_ENTRIES;
import java.util.Arrays;
import java.util.List;
-import org.apache.datasketches.memory.Memory;
+import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.tuple.Sketch;
import org.apache.datasketches.tuple.SketchIterator;
import org.apache.datasketches.tuple.Sketches;
@@ -159,7 +159,7 @@ public class UnionDoubleSummaryWithModeSketchUDAFTest {
Assert.assertEquals(((IntWritable) r.get(0)).get(), DEFAULT_NOMINAL_ENTRIES);
Assert.assertEquals(((Text) r.get(1)).toString(), DoubleSummary.Mode.Sum.toString());
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) r.get(2)).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)), new DoubleSummaryDeserializer());
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
SketchIterator<DoubleSummary> it = resultSketch.iterator();
while (it.next()) {
@@ -202,7 +202,7 @@ public class UnionDoubleSummaryWithModeSketchUDAFTest {
Assert.assertEquals(((IntWritable) r.get(0)).get(), nomNumEntries);
Assert.assertEquals(((Text) r.get(1)).toString(), DoubleSummary.Mode.Max.toString());
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) r.get(2)).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)), new DoubleSummaryDeserializer());
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
SketchIterator<DoubleSummary> it = resultSketch.iterator();
while (it.next()) {
@@ -249,7 +249,7 @@ public class UnionDoubleSummaryWithModeSketchUDAFTest {
Assert.assertEquals(((IntWritable) r.get(0)).get(), DEFAULT_NOMINAL_ENTRIES);
Assert.assertEquals(((Text) r.get(1)).toString(), DoubleSummary.Mode.Sum.toString());
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) r.get(2)).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(2)), new DoubleSummaryDeserializer());
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
SketchIterator<DoubleSummary> it = resultSketch.iterator();
while (it.next()) {
@@ -296,7 +296,7 @@ public class UnionDoubleSummaryWithModeSketchUDAFTest {
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) result).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) result), new DoubleSummaryDeserializer());
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
SketchIterator<DoubleSummary> it = resultSketch.iterator();
while (it.next()) {
@@ -331,7 +331,7 @@ public class UnionDoubleSummaryWithModeSketchUDAFTest {
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) result).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) result), new DoubleSummaryDeserializer());
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
SketchIterator<DoubleSummary> it = resultSketch.iterator();
while (it.next()) {
@@ -374,7 +374,7 @@ public class UnionDoubleSummaryWithModeSketchUDAFTest {
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) result).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) result), new DoubleSummaryDeserializer());
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
SketchIterator<DoubleSummary> it = resultSketch.iterator();
while (it.next()) {
@@ -417,7 +417,7 @@ public class UnionDoubleSummaryWithModeSketchUDAFTest {
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BytesWritable);
Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
- Memory.wrap(((BytesWritable) result).getBytes()), new DoubleSummaryDeserializer());
+ BytesWritableHelper.wrapAsMemory((BytesWritable) result), new DoubleSummaryDeserializer());
Assert.assertEquals(resultSketch.getEstimate(), 2.0);
SketchIterator<DoubleSummary> it = resultSketch.iterator();
while (it.next()) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org