You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datasketches.apache.org by le...@apache.org on 2021/03/02 00:43:40 UTC
[datasketches-hive] 01/01: Updates required to match java core
2.0.0.
This is an automated email from the ASF dual-hosted git repository.
leerho pushed a commit to branch Update_to_core2.0.0
in repository https://gitbox.apache.org/repos/asf/datasketches-hive.git
commit e70423a071037370c9095188746b2c93e1b753c4
Author: Lee Rhodes <le...@users.noreply.github.com>
AuthorDate: Mon Mar 1 16:43:08 2021 -0800
Updates required to match java core 2.0.0.
---
pom.xml | 55 +++++++++++++++++-----
.../datasketches/hive/cpc/DataToSketchUDAF.java | 15 ++++--
.../datasketches/hive/hll/DataToSketchUDAF.java | 5 ++
.../datasketches/hive/theta/DataToSketchUDAF.java | 10 +++-
.../datasketches/hive/theta/ExcludeSketchUDF.java | 3 +-
.../hive/theta/IntersectSketchUDAF.java | 6 +--
.../hive/theta/IntersectSketchUDF.java | 4 +-
.../datasketches/hive/theta/UnionSketchUDF.java | 8 ++--
.../apache/datasketches/hive/theta/UnionState.java | 2 +-
.../hive/tuple/ArrayOfDoublesSketchEvaluator.java | 4 +-
.../hive/tuple/ArrayOfDoublesSketchState.java | 6 +--
.../hive/tuple/ArrayOfDoublesSketchStats.java | 4 +-
...OfDoublesSketchToEstimateAndErrorBoundsUDF.java | 4 +-
.../tuple/ArrayOfDoublesSketchToEstimatesUDF.java | 6 +--
.../hive/tuple/ArrayOfDoublesSketchToMeansUDF.java | 4 +-
...fDoublesSketchToNumberOfRetainedEntriesUDF.java | 4 +-
.../ArrayOfDoublesSketchToQuantilesSketchUDF.java | 6 +--
.../tuple/ArrayOfDoublesSketchToValuesUDTF.java | 6 +--
.../tuple/ArrayOfDoublesSketchToVariancesUDF.java | 4 +-
.../hive/tuple/ArrayOfDoublesSketchesTTestUDF.java | 8 ++--
.../hive/tuple/ArrayOfDoublesState.java | 2 +-
.../hive/tuple/ArrayOfDoublesUnionState.java | 8 ++--
.../datasketches/hive/tuple/DataToSketchUDAF.java | 16 +++++--
.../hive/tuple/UnionArrayOfDoublesSketchUDAF.java | 6 +--
.../apache/datasketches/hive/tuple/UnionState.java | 4 +-
.../hive/theta/DataToSketchUDAFTest.java | 2 +-
.../hive/theta/ExcludeSketchUDFTest.java | 36 ++++++--------
.../hive/theta/IntersectSketchUDFTest.java | 25 +++++-----
.../datasketches/hive/theta/SampleSketchUDF.java | 7 ++-
...ublesSketchToEstimateAndErrorBoundsUDFTest.java | 4 +-
.../ArrayOfDoublesSketchToEstimatesUDFTest.java | 4 +-
.../tuple/ArrayOfDoublesSketchToMeansUDFTest.java | 4 +-
...blesSketchToNumberOfRetainedEntriesUDFTest.java | 5 +-
...rayOfDoublesSketchToQuantilesSketchUDFTest.java | 7 ++-
.../ArrayOfDoublesSketchToValuesUDTFTest.java | 11 ++---
.../ArrayOfDoublesSketchToVariancesUDFTest.java | 4 +-
.../tuple/ArrayOfDoublesSketchesTTestUDFTest.java | 5 +-
.../tuple/DataToArrayOfDoublesSketchUDAFTest.java | 8 ++--
.../tuple/UnionArrayOfDoublesSketchUDAFTest.java | 8 ++--
39 files changed, 183 insertions(+), 147 deletions(-)
diff --git a/pom.xml b/pom.xml
index 2ca8c7d..54e3c1f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -21,6 +21,7 @@ under the License.
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
<modelVersion>4.0.0</modelVersion>
<parent>
@@ -30,17 +31,14 @@ under the License.
</parent>
<groupId>org.apache.datasketches</groupId>
-
- <!-- UNIQUE FOR THIS JAVA COMPONENT -->
<artifactId>datasketches-hive</artifactId>
<version>1.2.0-SNAPSHOT</version>
+ <packaging>jar</packaging>
+
+ <name>${project.artifactId}</name>
<description>Apache Hive adaptors for the DataSketches library.</description>
- <!-- END: UNIQUE FOR THIS JAVA COMPONENT -->
-
<url>https://datasketches.apache.org/</url>
- <name>${project.artifactId}</name>
<inceptionYear>2015</inceptionYear>
- <packaging>jar</packaging> <!-- jar is the default -->
<mailingLists>
<mailingList>
@@ -85,7 +83,7 @@ under the License.
<properties>
<!-- UNIQUE FOR THIS JAVA COMPONENT -->
- <datasketches-java.version>1.3.0-incubating</datasketches-java.version>
+ <datasketches-java.version>2.0.0</datasketches-java.version>
<hive-exec.version>2.3.4</hive-exec.version>
<hadoop-common.version>2.8.5</hadoop-common.version>
<slf4j-simple.version>1.7.30</slf4j-simple.version>
@@ -95,7 +93,7 @@ under the License.
<testng.version>7.1.0</testng.version>
<!-- System-wide properties -->
- <maven.version>3.0.0</maven.version>
+ <maven.version>3.5.0</maven.version>
<java.version>1.8</java.version>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
@@ -105,8 +103,11 @@ under the License.
<project.build.resourceEncoding>${charset.encoding}</project.build.resourceEncoding>
<project.reporting.outputEncoding>${charset.encoding}</project.reporting.outputEncoding>
- <!-- org.codehaus.plexus used for strict profile testing-->
- <plexus-compiler-javac-errorprone.version>2.8.8</plexus-compiler-javac-errorprone.version>
+ <!-- org.codehaus plugins -->
+ <!-- used for strict profile testing-->
+ <plexus-compiler-javac-errorprone.version>2.8.5</plexus-compiler-javac-errorprone.version>
+ <versions-maven-plugin.version>2.8.1</versions-maven-plugin.version>
+
<!-- Maven Plugins -->
<maven-assembly-plugin.version>3.3.0</maven-assembly-plugin.version> <!-- overrides parent -->
<maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version> <!-- overrides parent -->
@@ -124,6 +125,7 @@ under the License.
<!-- org.jacoco Maven Plugins -->
<jacoco-maven-plugin.version>0.8.6</jacoco-maven-plugin.version>
<!-- org.eluder Maven Plugins -->
+ <coveralls-repo-token></coveralls-repo-token>
<coveralls-maven-plugin.version>4.3.0</coveralls-maven-plugin.version>
<!-- other -->
<lifecycle-mapping.version>1.0.0</lifecycle-mapping.version>
@@ -209,6 +211,13 @@ under the License.
<build>
<pluginManagement>
<plugins>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>versions-maven-plugin</artifactId>
+ <version>${versions-maven-plugin.version}</version>
+ </plugin>
+
<plugin>
<!-- We want to deploy the artifacts to a staging location for perusal -->
<!-- Apache Parent pom: apache-release profile -->
@@ -220,12 +229,14 @@ under the License.
<!-- see maven-install-plugin -->
</configuration>
</plugin>
+
<plugin>
<!-- Apache Parent pom, pluginManagement-->
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-release-plugin</artifactId>
<version>${maven-release-plugin.version}</version>
</plugin>
+
<plugin>
<!-- Extends Apache Parent pom, pluginManagement-->
<groupId>org.apache.maven.plugins</groupId>
@@ -234,18 +245,21 @@ under the License.
<executions>
<execution>
<id>default-jar</id>
+ <phase>package</phase>
<goals>
<goal>jar</goal>
</goals>
</execution>
<execution>
<id>default-test-jar</id>
+ <phase>package</phase>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
+
<plugin>
<!-- Extends Apache Parent pom, apache-release profile -->
<groupId>org.apache.maven.plugins</groupId>
@@ -263,6 +277,7 @@ under the License.
</execution>
</executions>
</plugin>
+
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
@@ -293,6 +308,7 @@ under the License.
</execution>
</executions>
</plugin>
+
<plugin>
<!-- Apache Parent pom, pluginManagement-->
<groupId>org.apache.rat</groupId>
@@ -312,13 +328,16 @@ under the License.
<useDefaultExcludes>true</useDefaultExcludes>
<excludes>
<!-- rat uses .gitignore for excludes by default -->
+ <exclude>**/*.yaml</exclude>
+ <exclude>**/*.yml</exclude>
+ <exclude>**/.*</exclude>
<exclude>**/test/resources/**/*.txt</exclude>
- <exclude>.asf.yaml</exclude>
<exclude>LICENSE</exclude>
<exclude>NOTICE</exclude>
</excludes>
</configuration>
</plugin>
+
<plugin>
<!-- Extends Apache Parent pom, apache-release profile -->
<groupId>org.apache.maven.plugins</groupId>
@@ -341,6 +360,7 @@ under the License.
</execution>
</executions>
</plugin>
+
<plugin>
<!-- Apache Parent pom, pluginManagement-->
<groupId>org.apache.maven.plugins</groupId>
@@ -352,6 +372,7 @@ under the License.
<redirectTestOutputToFile>true</redirectTestOutputToFile>
</configuration>
</plugin>
+
<plugin>
<!-- Generates code coverage report from website. -->
<groupId>org.jacoco</groupId>
@@ -359,22 +380,30 @@ under the License.
<version>${jacoco-maven-plugin.version}</version>
<executions>
<execution>
- <id>prepare-agent</id>
+ <id>default-prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
+ <execution>
+ <id>default-report</id>
+ <goals>
+ <goal>report</goal>
+ </goals>
+ </execution>
</executions>
</plugin>
+
<plugin>
<!-- Submit code coverage report to Coveralls.io. -->
<groupId>org.eluder.coveralls</groupId>
<artifactId>coveralls-maven-plugin</artifactId>
<version>${coveralls-maven-plugin.version}</version>
<configuration>
- <!-- Since we use Travis CI we do not have to put a Coveralls token here. -->
+ <repoToken>${coveralls-repo-token}</repoToken>
</configuration>
</plugin>
+
</plugins>
</pluginManagement>
<plugins>
diff --git a/src/main/java/org/apache/datasketches/hive/cpc/DataToSketchUDAF.java b/src/main/java/org/apache/datasketches/hive/cpc/DataToSketchUDAF.java
index 732a5b8..400d2bb 100644
--- a/src/main/java/org/apache/datasketches/hive/cpc/DataToSketchUDAF.java
+++ b/src/main/java/org/apache/datasketches/hive/cpc/DataToSketchUDAF.java
@@ -41,7 +41,12 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectIn
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
/**
- * Hive UDAF to create an HllSketch from raw data.
+ * Hive UDAF to create an CPCSketch from raw data.
+ *
+ * <p><b>Note</b> Strings as raw data values are encoded as a UTF-16 VARCHAR
+ * prior to being submitted to the sketch. If the user requires a different
+ * encoding for cross-platform compatibility, it is recommended that these values be encoded prior
+ * to being submitted and then typed as a BINARY byte[].</p>
*/
@Description(
name = "dataToSketch",
@@ -51,7 +56,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectIn
+ "> SELECT dataToSketch(val, 12) FROM src;\n"
+ "The return value is a binary blob that can be operated on by other sketch related functions."
+ " The lgK parameter controls the sketch size and rlative error expected from the sketch."
- + " It is optional an must be from 4 to 26. The default is 11, which is expected to yield errors"
+ + " It is optional and must be from 4 to 26. The default is 11, which is expected to yield errors"
+ " of roughly +-1.5% in the estimation of uniques with 95% confidence."
+ " The seed parameter is optional")
public class DataToSketchUDAF extends AbstractGenericUDAFResolver {
@@ -119,7 +124,7 @@ public class DataToSketchUDAF extends AbstractGenericUDAFResolver {
public AggregationBuffer getNewAggregationBuffer() throws HiveException {
// Different State is used for the iterate phase and the merge phase.
// SketchState is more space-efficient, so let's use SketchState if possible.
- if ((mode_ == Mode.PARTIAL1) || (mode_ == Mode.COMPLETE)) { // iterate() will be used
+ if (mode_ == Mode.PARTIAL1 || mode_ == Mode.COMPLETE) { // iterate() will be used
return new SketchState();
}
return new UnionState();
@@ -137,7 +142,7 @@ public class DataToSketchUDAF extends AbstractGenericUDAFResolver {
public ObjectInspector init(final Mode mode, final ObjectInspector[] parameters) throws HiveException {
super.init(mode, parameters);
mode_ = mode;
- if ((mode == Mode.PARTIAL1) || (mode == Mode.COMPLETE)) {
+ if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {
// input is original data
inputInspector_ = (PrimitiveObjectInspector) parameters[0];
if (parameters.length > 1) {
@@ -151,7 +156,7 @@ public class DataToSketchUDAF extends AbstractGenericUDAFResolver {
intermediateInspector_ = (StructObjectInspector) parameters[0];
}
- if ((mode == Mode.PARTIAL1) || (mode == Mode.PARTIAL2)) {
+ if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {
// intermediate results need to include the lgK and the target HLL type
return ObjectInspectorFactory.getStandardStructObjectInspector(
Arrays.asList(LG_K_FIELD, SEED_FIELD, SKETCH_FIELD),
diff --git a/src/main/java/org/apache/datasketches/hive/hll/DataToSketchUDAF.java b/src/main/java/org/apache/datasketches/hive/hll/DataToSketchUDAF.java
index 6a4da67..0e05f93 100644
--- a/src/main/java/org/apache/datasketches/hive/hll/DataToSketchUDAF.java
+++ b/src/main/java/org/apache/datasketches/hive/hll/DataToSketchUDAF.java
@@ -41,6 +41,11 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectIn
/**
* Hive UDAF to create an HllSketch from raw data.
+ *
+ * <p><b>Note</b> Strings as raw data values are encoded as a UTF-16 VARCHAR
+ * prior to being submitted to the sketch. If the user requires a different
+ * encoding for cross-platform compatibility, it is recommended that these values be encoded prior
+ * to being submitted and then typed as a BINARY byte[].</p>
*/
@Description(
name = "dataToSketch",
diff --git a/src/main/java/org/apache/datasketches/hive/theta/DataToSketchUDAF.java b/src/main/java/org/apache/datasketches/hive/theta/DataToSketchUDAF.java
index 581f5a6..9bcad81 100644
--- a/src/main/java/org/apache/datasketches/hive/theta/DataToSketchUDAF.java
+++ b/src/main/java/org/apache/datasketches/hive/theta/DataToSketchUDAF.java
@@ -41,6 +41,12 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+/**
+ * <p><b>Note</b> Strings as raw data values are encoded as a UTF-16 VARCHAR
+ * prior to being submitted to the sketch. If the user requires a different
+ * encoding for cross-platform compatibility, it is recommended that these values be encoded prior
+ * to being submitted and then typed as a BINARY byte[].</p>
+ */
@Description(
name = "dataToSketch",
value = "_FUNC_(expr, size, prob, seed) - "
@@ -141,7 +147,7 @@ public class DataToSketchUDAF extends AbstractGenericUDAFResolver {
public ObjectInspector init(final Mode mode, final ObjectInspector[] parameters) throws HiveException {
super.init(mode, parameters);
- if ((mode == Mode.PARTIAL1) || (mode == Mode.COMPLETE)) {
+ if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {
// input is original data
inputObjectInspector = (PrimitiveObjectInspector) parameters[0];
if (parameters.length > 1) {
@@ -158,7 +164,7 @@ public class DataToSketchUDAF extends AbstractGenericUDAFResolver {
intermediateObjectInspector = (StructObjectInspector) parameters[0];
}
- if ((mode == Mode.PARTIAL1) || (mode == Mode.PARTIAL2)) {
+ if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {
// intermediate results need to include the the nominal number of entries and the seed
return ObjectInspectorFactory.getStandardStructObjectInspector(
Arrays.asList(NOMINAL_ENTRIES_FIELD, SEED_FIELD, SKETCH_FIELD),
diff --git a/src/main/java/org/apache/datasketches/hive/theta/ExcludeSketchUDF.java b/src/main/java/org/apache/datasketches/hive/theta/ExcludeSketchUDF.java
index 2b5b2d0..8eb4755 100644
--- a/src/main/java/org/apache/datasketches/hive/theta/ExcludeSketchUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/theta/ExcludeSketchUDF.java
@@ -61,8 +61,7 @@ public class ExcludeSketchUDF extends UDF {
}
final AnotB anotb = SetOperation.builder().setSeed(hashSeed).buildANotB();
- anotb.update(firstSketch, secondSketch);
- final byte[] excludeSketchBytes = anotb.getResult().toByteArray();
+ final byte[] excludeSketchBytes = anotb.aNotB(firstSketch, secondSketch).toByteArray();
final BytesWritable result = new BytesWritable();
result.set(excludeSketchBytes, 0, excludeSketchBytes.length);
return result;
diff --git a/src/main/java/org/apache/datasketches/hive/theta/IntersectSketchUDAF.java b/src/main/java/org/apache/datasketches/hive/theta/IntersectSketchUDAF.java
index cda0f75..9403cde 100644
--- a/src/main/java/org/apache/datasketches/hive/theta/IntersectSketchUDAF.java
+++ b/src/main/java/org/apache/datasketches/hive/theta/IntersectSketchUDAF.java
@@ -93,7 +93,7 @@ public class IntersectSketchUDAF extends AbstractGenericUDAFResolver {
@Override
public ObjectInspector init(final Mode mode, final ObjectInspector[] parameters) throws HiveException {
super.init(mode, parameters);
- if ((mode == Mode.PARTIAL1) || (mode == Mode.COMPLETE)) {
+ if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {
inputObjectInspector = (PrimitiveObjectInspector) parameters[0];
if (parameters.length > 1) {
seedObjectInspector = (PrimitiveObjectInspector) parameters[1];
@@ -102,7 +102,7 @@ public class IntersectSketchUDAF extends AbstractGenericUDAFResolver {
intermediateObjectInspector = (StandardStructObjectInspector) parameters[0];
}
- if ((mode == Mode.PARTIAL1) || (mode == Mode.PARTIAL2)) {
+ if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {
// intermediate results need to include the seed
return ObjectInspectorFactory.getStandardStructObjectInspector(
Arrays.asList(SEED_FIELD, SKETCH_FIELD),
@@ -205,7 +205,7 @@ public class IntersectSketchUDAF extends AbstractGenericUDAFResolver {
}
void update(final Memory serializedSketch) {
- intersection_.update(Sketches.wrapSketch(serializedSketch, seed_));
+ intersection_.intersect(Sketches.wrapSketch(serializedSketch, seed_));
}
Sketch getResult() {
diff --git a/src/main/java/org/apache/datasketches/hive/theta/IntersectSketchUDF.java b/src/main/java/org/apache/datasketches/hive/theta/IntersectSketchUDF.java
index 52ba553..6c4c234 100644
--- a/src/main/java/org/apache/datasketches/hive/theta/IntersectSketchUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/theta/IntersectSketchUDF.java
@@ -59,9 +59,7 @@ public class IntersectSketchUDF extends UDF {
}
final Intersection intersect = SetOperation.builder().setSeed(hashSeed).buildIntersection();
- intersect.update(firstSketch);
- intersect.update(secondSketch);
- return new BytesWritable(intersect.getResult().toByteArray());
+ return new BytesWritable(intersect.intersect(firstSketch, secondSketch).toByteArray());
}
/**
diff --git a/src/main/java/org/apache/datasketches/hive/theta/UnionSketchUDF.java b/src/main/java/org/apache/datasketches/hive/theta/UnionSketchUDF.java
index 2076ca6..41256d7 100644
--- a/src/main/java/org/apache/datasketches/hive/theta/UnionSketchUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/theta/UnionSketchUDF.java
@@ -54,12 +54,12 @@ public class UnionSketchUDF extends UDF {
final Union union = SetOperation.builder().setSeed(seed).setNominalEntries(sketchSize).buildUnion();
- if ((firstSketch != null) && (firstSketch.getLength() >= EMPTY_SKETCH_SIZE_BYTES)) {
- union.update(BytesWritableHelper.wrapAsMemory(firstSketch));
+ if (firstSketch != null && firstSketch.getLength() >= EMPTY_SKETCH_SIZE_BYTES) {
+ union.union(BytesWritableHelper.wrapAsMemory(firstSketch));
}
- if ((secondSketch != null) && (secondSketch.getLength() >= EMPTY_SKETCH_SIZE_BYTES)) {
- union.update(BytesWritableHelper.wrapAsMemory(secondSketch));
+ if (secondSketch != null && secondSketch.getLength() >= EMPTY_SKETCH_SIZE_BYTES) {
+ union.union(BytesWritableHelper.wrapAsMemory(secondSketch));
}
return new BytesWritable(union.getResult().toByteArray());
diff --git a/src/main/java/org/apache/datasketches/hive/theta/UnionState.java b/src/main/java/org/apache/datasketches/hive/theta/UnionState.java
index 177da22..5a2a4c3 100644
--- a/src/main/java/org/apache/datasketches/hive/theta/UnionState.java
+++ b/src/main/java/org/apache/datasketches/hive/theta/UnionState.java
@@ -60,7 +60,7 @@ class UnionState extends AbstractAggregationBuffer {
}
public void update(final Memory mem) {
- union_.update(mem);
+ union_.union(mem);
}
public void update(final Object value, final PrimitiveObjectInspector objectInspector) {
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchEvaluator.java b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchEvaluator.java
index 6f00015..4aca967 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchEvaluator.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchEvaluator.java
@@ -23,8 +23,8 @@ import java.util.Arrays;
import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.memory.Memory;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketches;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketches;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchState.java b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchState.java
index bd1930f..ea8adb9 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchState.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchState.java
@@ -19,9 +19,9 @@
package org.apache.datasketches.hive.tuple;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketchBuilder;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketchBuilder;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchStats.java b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchStats.java
index 06890fd..00e1014 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchStats.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchStats.java
@@ -20,8 +20,8 @@
package org.apache.datasketches.hive.tuple;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketchIterator;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketchIterator;
class ArrayOfDoublesSketchStats {
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToEstimateAndErrorBoundsUDF.java b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToEstimateAndErrorBoundsUDF.java
index 1dc805f..ccfc731 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToEstimateAndErrorBoundsUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToEstimateAndErrorBoundsUDF.java
@@ -23,8 +23,8 @@ import java.util.Arrays;
import java.util.List;
import org.apache.datasketches.hive.common.BytesWritableHelper;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketches;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketches;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToEstimatesUDF.java b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToEstimatesUDF.java
index 89da31a..e988c07 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToEstimatesUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToEstimatesUDF.java
@@ -23,9 +23,9 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.datasketches.hive.common.BytesWritableHelper;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketchIterator;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketches;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketchIterator;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketches;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToMeansUDF.java b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToMeansUDF.java
index c5daae6..7f57752 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToMeansUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToMeansUDF.java
@@ -24,8 +24,8 @@ import java.util.List;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.apache.datasketches.hive.common.BytesWritableHelper;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketches;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketches;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToNumberOfRetainedEntriesUDF.java b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToNumberOfRetainedEntriesUDF.java
index 9b42b24..914fd2b 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToNumberOfRetainedEntriesUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToNumberOfRetainedEntriesUDF.java
@@ -20,8 +20,8 @@
package org.apache.datasketches.hive.tuple;
import org.apache.datasketches.hive.common.BytesWritableHelper;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketches;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketches;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToQuantilesSketchUDF.java b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToQuantilesSketchUDF.java
index cfee6b9..f556640 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToQuantilesSketchUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToQuantilesSketchUDF.java
@@ -23,9 +23,9 @@ import org.apache.datasketches.hive.common.BytesWritableHelper;
import org.apache.datasketches.quantiles.DoublesSketch;
import org.apache.datasketches.quantiles.DoublesSketchBuilder;
import org.apache.datasketches.quantiles.UpdateDoublesSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketchIterator;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketches;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketchIterator;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketches;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToValuesUDTF.java b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToValuesUDTF.java
index ee9bda3..3532de0 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToValuesUDTF.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToValuesUDTF.java
@@ -24,9 +24,9 @@ import java.util.Arrays;
import java.util.List;
import org.apache.datasketches.hive.common.BytesWritableHelper;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketchIterator;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketches;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketchIterator;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketches;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToVariancesUDF.java b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToVariancesUDF.java
index f2ba17f..8612917 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToVariancesUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToVariancesUDF.java
@@ -24,8 +24,8 @@ import java.util.List;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.apache.datasketches.hive.common.BytesWritableHelper;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketches;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketches;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchesTTestUDF.java b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchesTTestUDF.java
index 4ed64dc..aafee45 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchesTTestUDF.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchesTTestUDF.java
@@ -25,8 +25,8 @@ import java.util.List;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.apache.commons.math3.stat.inference.TTest;
import org.apache.datasketches.hive.common.BytesWritableHelper;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketches;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketches;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
@@ -48,7 +48,7 @@ public class ArrayOfDoublesSketchesTTestUDF extends UDF {
* @return list of p-values
*/
public List<Double> evaluate(final BytesWritable serializedSketchA, final BytesWritable serializedSketchB) {
- if ((serializedSketchA == null) || (serializedSketchB == null)) { return null; }
+ if (serializedSketchA == null || serializedSketchB == null) { return null; }
final ArrayOfDoublesSketch sketchA =
ArrayOfDoublesSketches.wrapSketch(BytesWritableHelper.wrapAsMemory(serializedSketchA));
final ArrayOfDoublesSketch sketchB =
@@ -59,7 +59,7 @@ public class ArrayOfDoublesSketchesTTestUDF extends UDF {
}
// If the sketches contain fewer than 2 values, the p-value can't be calculated
- if ((sketchA.getRetainedEntries() < 2) || (sketchB.getRetainedEntries() < 2)) {
+ if (sketchA.getRetainedEntries() < 2 || sketchB.getRetainedEntries() < 2) {
return null;
}
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesState.java b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesState.java
index 59743a0..ef93bb6 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesState.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesState.java
@@ -19,7 +19,7 @@
package org.apache.datasketches.hive.tuple;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketch;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AbstractAggregationBuffer;
abstract class ArrayOfDoublesState extends AbstractAggregationBuffer {
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesUnionState.java b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesUnionState.java
index 2610f7f..4d26653 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesUnionState.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesUnionState.java
@@ -19,9 +19,9 @@
package org.apache.datasketches.hive.tuple;
-import org.apache.datasketches.tuple.ArrayOfDoublesSetOperationBuilder;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesUnion;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSetOperationBuilder;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUnion;
class ArrayOfDoublesUnionState extends ArrayOfDoublesState {
@@ -39,7 +39,7 @@ class ArrayOfDoublesUnionState extends ArrayOfDoublesState {
}
void update(final ArrayOfDoublesSketch sketch) {
- union_.update(sketch);
+ union_.union(sketch);
}
@Override
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/DataToSketchUDAF.java b/src/main/java/org/apache/datasketches/hive/tuple/DataToSketchUDAF.java
index b236329..8b31f53 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/DataToSketchUDAF.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/DataToSketchUDAF.java
@@ -39,6 +39,12 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+/**
+ * <p><b>Note</b> Strings as raw data values are encoded as a UTF-16 VARCHAR
+ * prior to being submitted to the sketch. If the user requires a different
+ * encoding for cross-platform compatibility, it is recommended that these values be encoded prior
+ * to being submitted and then typed as a BINARY byte[].</p>
+ */
@SuppressWarnings("javadoc")
public abstract class DataToSketchUDAF extends AbstractGenericUDAFResolver {
@@ -63,8 +69,8 @@ public abstract class DataToSketchUDAF extends AbstractGenericUDAFResolver {
if (inspectors.length > 3) {
ObjectInspectorValidator.validateCategoryPrimitive(inspectors[3], 3);
final PrimitiveObjectInspector primitiveInspector = (PrimitiveObjectInspector) inspectors[3];
- if ((primitiveInspector.getPrimitiveCategory() != PrimitiveCategory.FLOAT)
- && (primitiveInspector.getPrimitiveCategory() != PrimitiveCategory.DOUBLE)) {
+ if (primitiveInspector.getPrimitiveCategory() != PrimitiveCategory.FLOAT
+ && primitiveInspector.getPrimitiveCategory() != PrimitiveCategory.DOUBLE) {
throw new UDFArgumentTypeException(3, "float or double value expected as parameter 4 but "
+ primitiveInspector.getPrimitiveCategory().name() + " was received");
}
@@ -107,7 +113,7 @@ public abstract class DataToSketchUDAF extends AbstractGenericUDAFResolver {
public ObjectInspector init(final Mode mode, final ObjectInspector[] inspectors) throws HiveException {
super.init(mode, inspectors);
mode_ = mode;
- if ((mode == Mode.PARTIAL1) || (mode == Mode.COMPLETE)) {
+ if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {
// input is original data
keyInspector_ = (PrimitiveObjectInspector) inspectors[0];
valueInspector_ = (PrimitiveObjectInspector) inspectors[1];
@@ -122,7 +128,7 @@ public abstract class DataToSketchUDAF extends AbstractGenericUDAFResolver {
intermediateInspector_ = (StructObjectInspector) inspectors[0];
}
- if ((mode == Mode.PARTIAL1) || (mode == Mode.PARTIAL2)) {
+ if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {
// intermediate results need to include the nominal number of entries
return ObjectInspectorFactory.getStandardStructObjectInspector(
Arrays.asList(NOMINAL_NUM_ENTRIES_FIELD, SKETCH_FIELD),
@@ -164,7 +170,7 @@ public abstract class DataToSketchUDAF extends AbstractGenericUDAFResolver {
@SuppressWarnings("deprecation")
@Override
public AggregationBuffer getNewAggregationBuffer() throws HiveException {
- if ((mode_ == Mode.PARTIAL1) || (mode_ == Mode.COMPLETE)) {
+ if (mode_ == Mode.PARTIAL1 || mode_ == Mode.COMPLETE) {
return new SketchState<U, S>();
}
return new UnionState<S>();
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/UnionArrayOfDoublesSketchUDAF.java b/src/main/java/org/apache/datasketches/hive/tuple/UnionArrayOfDoublesSketchUDAF.java
index 324ef06..3f62fb4 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/UnionArrayOfDoublesSketchUDAF.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/UnionArrayOfDoublesSketchUDAF.java
@@ -24,7 +24,7 @@ import static org.apache.datasketches.Util.DEFAULT_NOMINAL_ENTRIES;
import java.util.Arrays;
import org.apache.datasketches.memory.Memory;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketches;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketches;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
@@ -91,7 +91,7 @@ public class UnionArrayOfDoublesSketchUDAF extends AbstractGenericUDAFResolver {
@Override
public ObjectInspector init(final Mode mode, final ObjectInspector[] inspectors) throws HiveException {
super.init(mode, inspectors);
- if ((mode == Mode.PARTIAL1) || (mode == Mode.COMPLETE)) {
+ if (mode == Mode.PARTIAL1 || mode == Mode.COMPLETE) {
// input is original data
sketchInspector_ = (PrimitiveObjectInspector) inspectors[0];
if (inspectors.length > 1) {
@@ -105,7 +105,7 @@ public class UnionArrayOfDoublesSketchUDAF extends AbstractGenericUDAFResolver {
intermediateInspector_ = (StructObjectInspector) inspectors[0];
}
- if ((mode == Mode.PARTIAL1) || (mode == Mode.PARTIAL2)) {
+ if (mode == Mode.PARTIAL1 || mode == Mode.PARTIAL2) {
// intermediate results need to include the the nominal number of entries and number of values
return ObjectInspectorFactory.getStandardStructObjectInspector(
Arrays.asList(NOMINAL_NUM_ENTRIES_FIELD, NUM_VALUES_FIELD, SKETCH_FIELD),
diff --git a/src/main/java/org/apache/datasketches/hive/tuple/UnionState.java b/src/main/java/org/apache/datasketches/hive/tuple/UnionState.java
index 90b9383..4b19907 100644
--- a/src/main/java/org/apache/datasketches/hive/tuple/UnionState.java
+++ b/src/main/java/org/apache/datasketches/hive/tuple/UnionState.java
@@ -34,11 +34,11 @@ class UnionState<S extends Summary> extends State<S> {
void init(final int nominalNumEntries, final SummarySetOperations<S> summarySetOps) {
super.init(nominalNumEntries);
- union_ = new Union<S>(nominalNumEntries, summarySetOps);
+ union_ = new Union<>(nominalNumEntries, summarySetOps);
}
void update(final Sketch<S> sketch) {
- union_.update(sketch);
+ union_.union(sketch);
}
@Override
diff --git a/src/test/java/org/apache/datasketches/hive/theta/DataToSketchUDAFTest.java b/src/test/java/org/apache/datasketches/hive/theta/DataToSketchUDAFTest.java
index ed4afe6..3d9c0ca 100644
--- a/src/test/java/org/apache/datasketches/hive/theta/DataToSketchUDAFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/theta/DataToSketchUDAFTest.java
@@ -214,7 +214,7 @@ public class DataToSketchUDAFTest {
// check if seed is correct in the result
Union union = SetOperation.builder().setSeed(seed).buildUnion();
// this must fail if the seed is incompatible
- union.update(resultSketch);
+ union.union(resultSketch);
}
}
diff --git a/src/test/java/org/apache/datasketches/hive/theta/ExcludeSketchUDFTest.java b/src/test/java/org/apache/datasketches/hive/theta/ExcludeSketchUDFTest.java
index 225f18a..5cfd07f 100644
--- a/src/test/java/org/apache/datasketches/hive/theta/ExcludeSketchUDFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/theta/ExcludeSketchUDFTest.java
@@ -20,44 +20,38 @@
package org.apache.datasketches.hive.theta;
import static org.apache.datasketches.Util.DEFAULT_UPDATE_SEED;
+import static org.testng.Assert.fail;
import static org.testng.AssertJUnit.assertEquals;
+import org.apache.datasketches.SketchesArgumentException;
import org.apache.datasketches.hive.common.BytesWritableHelper;
-import org.apache.hadoop.io.BytesWritable;
-import org.testng.annotations.Test;
-
-import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.theta.Sketch;
import org.apache.datasketches.theta.Sketches;
import org.apache.datasketches.theta.UpdateSketch;
+import org.apache.hadoop.io.BytesWritable;
+import org.testng.annotations.Test;
@SuppressWarnings("javadoc")
public class ExcludeSketchUDFTest {
+ @SuppressWarnings("unused")
@Test
public void evaluateNull() {
ExcludeSketchUDF testObject = new ExcludeSketchUDF();
-
- BytesWritable intermResult = testObject.evaluate(null, null);
-
- Memory mem = BytesWritableHelper.wrapAsMemory(intermResult);
-
- Sketch testResult = Sketches.wrapSketch(mem);
-
- assertEquals(0.0, testResult.getEstimate());
+ try {
+ BytesWritable intermResult = testObject.evaluate(null, null);
+ fail();
+ } catch (SketchesArgumentException e) {}
}
+ @SuppressWarnings("unused")
@Test
- public void evaluateEmpty() {
+ public void evaluateEmpty() { //Current impl is more restrictive than it needs to be.
ExcludeSketchUDF testObject = new ExcludeSketchUDF();
-
- BytesWritable intermResult = testObject.evaluate(new BytesWritable(), new BytesWritable());
-
- Memory mem = BytesWritableHelper.wrapAsMemory(intermResult);
-
- Sketch testResult = Sketches.wrapSketch(mem);
-
- assertEquals(0.0, testResult.getEstimate());
+ try {
+ BytesWritable intermResult = testObject.evaluate(new BytesWritable(), new BytesWritable());
+ fail();
+ } catch (SketchesArgumentException e) {}
}
@Test
diff --git a/src/test/java/org/apache/datasketches/hive/theta/IntersectSketchUDFTest.java b/src/test/java/org/apache/datasketches/hive/theta/IntersectSketchUDFTest.java
index 9792c90..49ad0db 100644
--- a/src/test/java/org/apache/datasketches/hive/theta/IntersectSketchUDFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/theta/IntersectSketchUDFTest.java
@@ -21,34 +21,33 @@ package org.apache.datasketches.hive.theta;
import static org.testng.AssertJUnit.assertEquals;
+import org.apache.datasketches.SketchesArgumentException;
import org.apache.datasketches.hive.common.BytesWritableHelper;
-import org.apache.hadoop.io.BytesWritable;
-import org.testng.annotations.Test;
-
-import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.theta.Sketch;
import org.apache.datasketches.theta.Sketches;
import org.apache.datasketches.theta.UpdateSketch;
+import org.apache.hadoop.io.BytesWritable;
+import org.testng.annotations.Test;
@SuppressWarnings("javadoc")
public class IntersectSketchUDFTest {
+ @SuppressWarnings("unused")
@Test
public void evaluateNull() {
IntersectSketchUDF testObject = new IntersectSketchUDF();
- BytesWritable intermResult = testObject.evaluate(null, null);
- Memory mem = BytesWritableHelper.wrapAsMemory(intermResult);
- Sketch testResult = Sketches.wrapSketch(mem);
- assertEquals(0.0, testResult.getEstimate());
+ try {
+ BytesWritable intermResult = testObject.evaluate(null, null);
+ } catch (SketchesArgumentException e) {}
}
+ @SuppressWarnings("unused")
@Test
- public void evaluateEmpty() {
+ public void evaluateEmpty() { //Current impl is more restrictive than it needs to be.
IntersectSketchUDF testObject = new IntersectSketchUDF();
- BytesWritable intermResult = testObject.evaluate(new BytesWritable(), new BytesWritable());
- Memory mem = BytesWritableHelper.wrapAsMemory(intermResult);
- Sketch testResult = Sketches.wrapSketch(mem);
- assertEquals(0.0, testResult.getEstimate());
+ try {
+ BytesWritable intermResult = testObject.evaluate(new BytesWritable(), new BytesWritable());
+ } catch (SketchesArgumentException e) {}
}
@Test
diff --git a/src/test/java/org/apache/datasketches/hive/theta/SampleSketchUDF.java b/src/test/java/org/apache/datasketches/hive/theta/SampleSketchUDF.java
index 07b14d9..23c06cd 100644
--- a/src/test/java/org/apache/datasketches/hive/theta/SampleSketchUDF.java
+++ b/src/test/java/org/apache/datasketches/hive/theta/SampleSketchUDF.java
@@ -20,13 +20,12 @@
package org.apache.datasketches.hive.theta;
import org.apache.datasketches.hive.common.BytesWritableHelper;
-import org.apache.hadoop.hive.ql.exec.UDF;
-import org.apache.hadoop.io.BytesWritable;
-
import org.apache.datasketches.memory.Memory;
import org.apache.datasketches.theta.SetOperation;
import org.apache.datasketches.theta.Sketch;
import org.apache.datasketches.theta.Union;
+import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.io.BytesWritable;
/**
* Hive estimate sketch UDF.
@@ -65,7 +64,7 @@ public class SampleSketchUDF extends UDF {
// The builder will catch errors with improper sketchSize or probability
Union union = SetOperation.builder().setP(probability).setNominalEntries(sketchSize).buildUnion();
- union.update(serializedSketch); //Union can accept Memory object directly
+ union.union(serializedSketch); //Union can accept Memory object directly
Sketch intermediateSketch = union.getResult(false, null); //to CompactSketch(unordered, on-heap)
byte[] resultSketch = intermediateSketch.toByteArray();
diff --git a/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToEstimateAndErrorBoundsUDFTest.java b/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToEstimateAndErrorBoundsUDFTest.java
index 652e8f3..a474097 100644
--- a/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToEstimateAndErrorBoundsUDFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToEstimateAndErrorBoundsUDFTest.java
@@ -21,8 +21,8 @@ package org.apache.datasketches.hive.tuple;
import java.util.List;
-import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketchBuilder;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketchBuilder;
import org.apache.hadoop.io.BytesWritable;
import org.testng.Assert;
import org.testng.annotations.Test;
diff --git a/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToEstimatesUDFTest.java b/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToEstimatesUDFTest.java
index 19f4ff3..4fbec88 100644
--- a/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToEstimatesUDFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToEstimatesUDFTest.java
@@ -21,8 +21,8 @@ package org.apache.datasketches.hive.tuple;
import java.util.List;
-import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketchBuilder;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketchBuilder;
import org.apache.hadoop.io.BytesWritable;
import org.testng.Assert;
import org.testng.annotations.Test;
diff --git a/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToMeansUDFTest.java b/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToMeansUDFTest.java
index 5997d87..3e5bda8 100644
--- a/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToMeansUDFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToMeansUDFTest.java
@@ -22,8 +22,8 @@ package org.apache.datasketches.hive.tuple;
import java.util.List;
import java.util.Random;
-import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketchBuilder;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketchBuilder;
import org.apache.hadoop.io.BytesWritable;
import org.testng.Assert;
import org.testng.annotations.Test;
diff --git a/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToNumberOfRetainedEntriesUDFTest.java b/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToNumberOfRetainedEntriesUDFTest.java
index d7c40a5..254e4a5 100644
--- a/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToNumberOfRetainedEntriesUDFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToNumberOfRetainedEntriesUDFTest.java
@@ -19,13 +19,12 @@
package org.apache.datasketches.hive.tuple;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketchBuilder;
import org.apache.hadoop.io.BytesWritable;
import org.testng.Assert;
import org.testng.annotations.Test;
-import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketchBuilder;
-
@SuppressWarnings("javadoc")
public class ArrayOfDoublesSketchToNumberOfRetainedEntriesUDFTest {
diff --git a/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToQuantilesSketchUDFTest.java b/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToQuantilesSketchUDFTest.java
index 2b4e45d..fa3b88e 100644
--- a/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToQuantilesSketchUDFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToQuantilesSketchUDFTest.java
@@ -20,14 +20,13 @@
package org.apache.datasketches.hive.tuple;
import org.apache.datasketches.hive.common.BytesWritableHelper;
+import org.apache.datasketches.quantiles.DoublesSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketchBuilder;
import org.apache.hadoop.io.BytesWritable;
import org.testng.Assert;
import org.testng.annotations.Test;
-import org.apache.datasketches.quantiles.DoublesSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketchBuilder;
-
@SuppressWarnings("javadoc")
public class ArrayOfDoublesSketchToQuantilesSketchUDFTest {
diff --git a/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToValuesUDTFTest.java b/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToValuesUDTFTest.java
index a7c3367..66db4a7 100644
--- a/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToValuesUDTFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToValuesUDTFTest.java
@@ -23,6 +23,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketchBuilder;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -32,17 +34,14 @@ import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.io.BytesWritable;
import org.testng.Assert;
import org.testng.annotations.Test;
-import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketchBuilder;
-
@SuppressWarnings("javadoc")
public class ArrayOfDoublesSketchToValuesUDTFTest {
@@ -104,8 +103,8 @@ public class ArrayOfDoublesSketchToValuesUDTFTest {
func.process(new Object[] {new BytesWritable(sketch.toByteArray())});
Assert.assertEquals(collector.list.size(), 2);
Assert.assertEquals(((Object[]) collector.list.get(0)).length, 1);
- Assert.assertEquals(((List<Double>) ((Object[]) collector.list.get(0))[0]), Arrays.asList(1.0, 2.0));
- Assert.assertEquals(((List<Double>) ((Object[]) collector.list.get(1))[0]), Arrays.asList(1.0, 2.0));
+ Assert.assertEquals((List<Double>) ((Object[]) collector.list.get(0))[0], Arrays.asList(1.0, 2.0));
+ Assert.assertEquals((List<Double>) ((Object[]) collector.list.get(1))[0], Arrays.asList(1.0, 2.0));
}
private static void checkResultInspector(ObjectInspector resultInspector) {
diff --git a/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToVariancesUDFTest.java b/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToVariancesUDFTest.java
index e019d83..8c53b43 100644
--- a/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToVariancesUDFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchToVariancesUDFTest.java
@@ -22,8 +22,8 @@ package org.apache.datasketches.hive.tuple;
import java.util.List;
import java.util.Random;
-import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketchBuilder;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketchBuilder;
import org.apache.hadoop.io.BytesWritable;
import org.testng.Assert;
import org.testng.annotations.Test;
diff --git a/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchesTTestUDFTest.java b/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchesTTestUDFTest.java
index 26b7c99..13d0560 100644
--- a/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchesTTestUDFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/tuple/ArrayOfDoublesSketchesTTestUDFTest.java
@@ -21,13 +21,12 @@ package org.apache.datasketches.hive.tuple;
import java.util.List;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketchBuilder;
import org.apache.hadoop.io.BytesWritable;
import org.testng.Assert;
import org.testng.annotations.Test;
-import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketchBuilder;
-
@SuppressWarnings("javadoc")
public class ArrayOfDoublesSketchesTTestUDFTest {
diff --git a/src/test/java/org/apache/datasketches/hive/tuple/DataToArrayOfDoublesSketchUDAFTest.java b/src/test/java/org/apache/datasketches/hive/tuple/DataToArrayOfDoublesSketchUDAFTest.java
index 3db4a28..9a2e17a 100644
--- a/src/test/java/org/apache/datasketches/hive/tuple/DataToArrayOfDoublesSketchUDAFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/tuple/DataToArrayOfDoublesSketchUDAFTest.java
@@ -25,10 +25,10 @@ import java.util.Arrays;
import java.util.List;
import org.apache.datasketches.hive.common.BytesWritableHelper;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketches;
-import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketchBuilder;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketches;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketchBuilder;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
diff --git a/src/test/java/org/apache/datasketches/hive/tuple/UnionArrayOfDoublesSketchUDAFTest.java b/src/test/java/org/apache/datasketches/hive/tuple/UnionArrayOfDoublesSketchUDAFTest.java
index 87d0a83..5c94869 100644
--- a/src/test/java/org/apache/datasketches/hive/tuple/UnionArrayOfDoublesSketchUDAFTest.java
+++ b/src/test/java/org/apache/datasketches/hive/tuple/UnionArrayOfDoublesSketchUDAFTest.java
@@ -25,10 +25,10 @@ import java.util.Arrays;
import java.util.List;
import org.apache.datasketches.hive.common.BytesWritableHelper;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesSketches;
-import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketch;
-import org.apache.datasketches.tuple.ArrayOfDoublesUpdatableSketchBuilder;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesSketches;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketch;
+import org.apache.datasketches.tuple.arrayofdoubles.ArrayOfDoublesUpdatableSketchBuilder;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datasketches.apache.org
For additional commands, e-mail: commits-help@datasketches.apache.org