You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2014/10/14 21:07:05 UTC
svn commit: r1631841 [2/42] - in /hive/branches/llap: ./ accumulo-handler/
accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/columns/
accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/
accumulo-handler/src/java/org/apache/hadoop/hive...
Propchange: hive/branches/llap/
------------------------------------------------------------------------------
Merged /hive/branches/cbo:r1605012-1627125
Merged /hive/trunk:r1625337-1631832
Modified: hive/branches/llap/accumulo-handler/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/llap/accumulo-handler/pom.xml?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/accumulo-handler/pom.xml (original)
+++ hive/branches/llap/accumulo-handler/pom.xml Tue Oct 14 19:06:45 2014
@@ -112,6 +112,12 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${hadoop-20S.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<version>${hadoop-20S.version}</version>
<optional>true</optional>
@@ -123,6 +129,12 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${hadoop-23.version}</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop-23.version}</version>
<optional>true</optional>
Modified: hive/branches/llap/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/columns/ColumnMapper.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/columns/ColumnMapper.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/columns/ColumnMapper.java (original)
+++ hive/branches/llap/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/columns/ColumnMapper.java Tue Oct 14 19:06:45 2014
@@ -20,11 +20,11 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.accumulo.AccumuloHiveConstants;
import org.apache.hadoop.hive.accumulo.serde.TooManyAccumuloColumnsException;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.util.StringUtils;
import org.apache.log4j.Logger;
import com.google.common.base.Preconditions;
Modified: hive/branches/llap/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/HiveAccumuloTableOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/HiveAccumuloTableOutputFormat.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/HiveAccumuloTableOutputFormat.java (original)
+++ hive/branches/llap/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/HiveAccumuloTableOutputFormat.java Tue Oct 14 19:06:45 2014
@@ -22,10 +22,14 @@ import org.apache.accumulo.core.client.A
import org.apache.accumulo.core.client.mapred.AccumuloOutputFormat;
import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.Mutation;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hive.accumulo.AccumuloConnectionParameters;
import org.apache.hadoop.hive.accumulo.serde.AccumuloSerDeParameters;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordWriter;
+import org.apache.hadoop.util.Progressable;
import com.google.common.base.Preconditions;
@@ -41,6 +45,13 @@ public class HiveAccumuloTableOutputForm
super.checkOutputSpecs(ignored, job);
}
+ @Override
+ public RecordWriter<Text,Mutation> getRecordWriter(FileSystem ignored, JobConf job, String name, Progressable progress) throws IOException {
+ configureAccumuloOutputFormat(job);
+
+ return super.getRecordWriter(ignored, job, name, progress);
+ }
+
protected void configureAccumuloOutputFormat(JobConf job) throws IOException {
AccumuloConnectionParameters cnxnParams = new AccumuloConnectionParameters(job);
@@ -76,16 +87,32 @@ public class HiveAccumuloTableOutputForm
protected void setAccumuloConnectorInfo(JobConf conf, String username, AuthenticationToken token)
throws AccumuloSecurityException {
- AccumuloOutputFormat.setConnectorInfo(conf, username, token);
+ try {
+ AccumuloOutputFormat.setConnectorInfo(conf, username, token);
+ } catch (IllegalStateException e) {
+ // AccumuloOutputFormat complains if you re-set an already set value. We just don't care.
+ log.debug("Ignoring exception setting Accumulo Connector instance for user " + username, e);
+ }
}
@SuppressWarnings("deprecation")
protected void setAccumuloZooKeeperInstance(JobConf conf, String instanceName, String zookeepers) {
- AccumuloOutputFormat.setZooKeeperInstance(conf, instanceName, zookeepers);
+ try {
+ AccumuloOutputFormat.setZooKeeperInstance(conf, instanceName, zookeepers);
+ } catch (IllegalStateException ise) {
+ // AccumuloOutputFormat complains if you re-set an already set value. We just don't care.
+ log.debug("Ignoring exception setting ZooKeeper instance of " + instanceName + " at "
+ + zookeepers, ise);
+ }
}
protected void setAccumuloMockInstance(JobConf conf, String instanceName) {
- AccumuloOutputFormat.setMockInstance(conf, instanceName);
+ try {
+ AccumuloOutputFormat.setMockInstance(conf, instanceName);
+ } catch (IllegalStateException e) {
+ // AccumuloOutputFormat complains if you re-set an already set value. We just don't care.
+ log.debug("Ignoring exception setting mock instance of " + instanceName, e);
+ }
}
protected void setDefaultAccumuloTableName(JobConf conf, String tableName) {
Modified: hive/branches/llap/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/AccumuloRowSerializer.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/AccumuloRowSerializer.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/AccumuloRowSerializer.java (original)
+++ hive/branches/llap/accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/AccumuloRowSerializer.java Tue Oct 14 19:06:45 2014
@@ -99,9 +99,6 @@ public class AccumuloRowSerializer {
// The ObjectInspector for the row ID
ObjectInspector fieldObjectInspector = field.getFieldObjectInspector();
- log.info("Serializing rowId with " + value + " in " + field + " using "
- + rowIdFactory.getClass());
-
// Serialize the row component using the RowIdFactory. In the normal case, this will just
// delegate back to the "local" serializeRowId method
byte[] data = rowIdFactory.serializeRowId(value, field, output);
Modified: hive/branches/llap/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/columns/TestColumnMapper.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/columns/TestColumnMapper.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/columns/TestColumnMapper.java (original)
+++ hive/branches/llap/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/columns/TestColumnMapper.java Tue Oct 14 19:06:45 2014
@@ -20,12 +20,12 @@ import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.accumulo.AccumuloHiveConstants;
import org.apache.hadoop.hive.accumulo.serde.TooManyAccumuloColumnsException;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.hadoop.util.StringUtils;
import org.junit.Assert;
import org.junit.Test;
Modified: hive/branches/llap/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/FirstCharAccumuloCompositeRowId.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/FirstCharAccumuloCompositeRowId.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/FirstCharAccumuloCompositeRowId.java (original)
+++ hive/branches/llap/accumulo-handler/src/test/org/apache/hadoop/hive/accumulo/serde/FirstCharAccumuloCompositeRowId.java Tue Oct 14 19:06:45 2014
@@ -19,10 +19,10 @@ package org.apache.hadoop.hive.accumulo.
import java.util.Arrays;
import java.util.Properties;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector;
-import org.apache.hadoop.util.StringUtils;
import org.apache.log4j.Logger;
/**
Modified: hive/branches/llap/beeline/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/llap/beeline/pom.xml?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/beeline/pom.xml (original)
+++ hive/branches/llap/beeline/pom.xml Tue Oct 14 19:06:45 2014
@@ -49,6 +49,11 @@
<artifactId>hive-shims</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-jdbc</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<!-- inter-project -->
<dependency>
<groupId>commons-cli</groupId>
@@ -88,12 +93,6 @@
<!-- test intra-project -->
<dependency>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-jdbc</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
<artifactId>hive-exec</artifactId>
<version>${project.version}</version>
<classifier>tests</classifier>
Modified: hive/branches/llap/beeline/src/java/org/apache/hive/beeline/BeeLine.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/beeline/src/java/org/apache/hive/beeline/BeeLine.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/beeline/src/java/org/apache/hive/beeline/BeeLine.java (original)
+++ hive/branches/llap/beeline/src/java/org/apache/hive/beeline/BeeLine.java Tue Oct 14 19:06:45 2014
@@ -692,10 +692,6 @@ public class BeeLine implements Closeabl
int code = 0;
if (!commands.isEmpty()) {
- // for single command execute, disable color
- getOpts().setColor(false);
- getOpts().setHeaderInterval(-1);
-
for (Iterator<String> i = commands.iterator(); i.hasNext();) {
String command = i.next().toString();
debug(loc("executing-command", command));
Modified: hive/branches/llap/beeline/src/java/org/apache/hive/beeline/Commands.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/beeline/src/java/org/apache/hive/beeline/Commands.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/beeline/src/java/org/apache/hive/beeline/Commands.java (original)
+++ hive/branches/llap/beeline/src/java/org/apache/hive/beeline/Commands.java Tue Oct 14 19:06:45 2014
@@ -38,6 +38,7 @@ import java.sql.Driver;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import java.sql.SQLWarning;
import java.util.Arrays;
import java.util.Iterator;
import java.util.LinkedList;
@@ -47,10 +48,13 @@ import java.util.Set;
import java.util.TreeSet;
import org.apache.hadoop.hive.common.cli.ShellCmdExecutor;
+import org.apache.hive.jdbc.HiveStatement;
public class Commands {
private final BeeLine beeLine;
+ private static final int DEFAULT_QUERY_PROGRESS_INTERVAL = 1000;
+ private static final int DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT = 10 * 1000;
/**
* @param beeLine
@@ -728,7 +732,7 @@ public class Commands {
beeLine.handleException(e);
}
-
+ line = line.trim();
if (line.endsWith(";")) {
line = line.substring(0, line.length() - 1);
}
@@ -758,6 +762,7 @@ public class Commands {
try {
Statement stmnt = null;
boolean hasResults;
+ Thread logThread = null;
try {
long start = System.currentTimeMillis();
@@ -767,7 +772,15 @@ public class Commands {
hasResults = ((CallableStatement) stmnt).execute();
} else {
stmnt = beeLine.createStatement();
- hasResults = stmnt.execute(sql);
+ if (beeLine.getOpts().isSilent()) {
+ hasResults = stmnt.execute(sql);
+ } else {
+ logThread = new Thread(createLogRunnable(stmnt));
+ logThread.setDaemon(true);
+ logThread.start();
+ hasResults = stmnt.execute(sql);
+ logThread.interrupt();
+ }
}
beeLine.showWarnings();
@@ -782,6 +795,11 @@ public class Commands {
beeLine.info(beeLine.loc("rows-selected", count) + " "
+ beeLine.locElapsedTime(end - start));
} finally {
+ if (logThread != null) {
+ logThread.join(DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT);
+ showRemainingLogsIfAny(stmnt);
+ logThread = null;
+ }
rs.close();
}
} while (BeeLine.getMoreResults(stmnt));
@@ -792,6 +810,13 @@ public class Commands {
+ " " + beeLine.locElapsedTime(end - start));
}
} finally {
+ if (logThread != null) {
+ if (!logThread.isInterrupted()) {
+ logThread.interrupt();
+ }
+ logThread.join(DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT);
+ showRemainingLogsIfAny(stmnt);
+ }
if (stmnt != null) {
stmnt.close();
}
@@ -803,6 +828,61 @@ public class Commands {
return true;
}
+ private Runnable createLogRunnable(Statement statement) {
+ if (statement instanceof HiveStatement) {
+ final HiveStatement hiveStatement = (HiveStatement) statement;
+
+ Runnable runnable = new Runnable() {
+ @Override
+ public void run() {
+ while (hiveStatement.hasMoreLogs()) {
+ try {
+ // fetch the log periodically and output to beeline console
+ for (String log : hiveStatement.getQueryLog()) {
+ beeLine.info(log);
+ }
+ Thread.sleep(DEFAULT_QUERY_PROGRESS_INTERVAL);
+ } catch (SQLException e) {
+ beeLine.error(new SQLWarning(e));
+ return;
+ } catch (InterruptedException e) {
+ beeLine.debug("Getting log thread is interrupted, since query is done!");
+ return;
+ }
+ }
+ }
+ };
+ return runnable;
+ } else {
+ beeLine.debug("The statement instance is not HiveStatement type: " + statement.getClass());
+ return new Runnable() {
+ @Override
+ public void run() {
+ // do nothing.
+ }
+ };
+ }
+ }
+
+ private void showRemainingLogsIfAny(Statement statement) {
+ if (statement instanceof HiveStatement) {
+ HiveStatement hiveStatement = (HiveStatement) statement;
+ List<String> logs;
+ do {
+ try {
+ logs = hiveStatement.getQueryLog();
+ } catch (SQLException e) {
+ beeLine.error(new SQLWarning(e));
+ return;
+ }
+ for (String log : logs) {
+ beeLine.info(log);
+ }
+ } while (logs.size() > 0);
+ } else {
+ beeLine.debug("The statement instance is not HiveStatement type: " + statement.getClass());
+ }
+ }
public boolean quit(String line) {
beeLine.setExit(true);
Modified: hive/branches/llap/bin/beeline.cmd
URL: http://svn.apache.org/viewvc/hive/branches/llap/bin/beeline.cmd?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/bin/beeline.cmd (original)
+++ hive/branches/llap/bin/beeline.cmd Tue Oct 14 19:06:45 2014
@@ -43,7 +43,22 @@ if not exist %HADOOP_HOME%\libexec\hadoo
@rem supress the HADOOP_HOME warnings in 1.x.x
set HADOOP_HOME_WARN_SUPPRESS=true
call %HADOOP_HOME%\libexec\hadoop-config.cmd
-set CLASSPATH=%CLASSPATH%;%HIVE_HOME%\lib\*;
+
+@rem include only the beeline client jar and its dependencies
+pushd %HIVE_HOME%\lib
+for /f %%a IN ('dir /b hive-beeline-**.jar') do (
+ set CLASSPATH=%CLASSPATH%;%HIVE_HOME%\lib\%%a
+)
+for /f %%a IN ('dir /b super-csv-**.jar') do (
+ set CLASSPATH=%CLASSPATH%;%HIVE_HOME%\lib\%%a
+)
+for /f %%a IN ('dir /b jline-**.jar') do (
+ set CLASSPATH=%CLASSPATH%;%HIVE_HOME%\lib\%%a
+)
+for /f %%a IN ('dir /b hive-jdbc-**-standalone.jar') do (
+ set CLASSPATH=%CLASSPATH%;%HIVE_HOME%\lib\%%a
+)
+popd
call %JAVA_HOME%\bin\java %JAVA_HEAP_MAX% %HADOOP_OPTS% -classpath %CLASSPATH% org.apache.hive.beeline.BeeLine %*
Modified: hive/branches/llap/bin/ext/beeline.sh
URL: http://svn.apache.org/viewvc/hive/branches/llap/bin/ext/beeline.sh?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/bin/ext/beeline.sh (original)
+++ hive/branches/llap/bin/ext/beeline.sh Tue Oct 14 19:06:45 2014
@@ -19,11 +19,17 @@ export SERVICE_LIST="${SERVICE_LIST}${TH
beeline () {
CLASS=org.apache.hive.beeline.BeeLine;
- execHiveCmd $CLASS "$@"
+
+ # include only the beeline client jar and its dependencies
+ beelineJarPath=`ls ${HIVE_LIB}/hive-beeline-*.jar`
+ superCsvJarPath=`ls ${HIVE_LIB}/super-csv-*.jar`
+ jlineJarPath=`ls ${HIVE_LIB}/jline-*.jar`
+ jdbcStandaloneJarPath=`ls ${HIVE_LIB}/hive-jdbc-*-standalone.jar`
+ export HADOOP_CLASSPATH=${beelineJarPath}:${superCsvJarPath}:${jlineJarPath}:${jdbcStandaloneJarPath}
+
+ exec $HADOOP jar ${beelineJarPath} $CLASS $HIVE_OPTS "$@"
}
beeline_help () {
- CLASS=org.apache.hive.beeline.BeeLine;
- execHiveCmd $CLASS "--help"
+ beeline "--help"
}
-
Modified: hive/branches/llap/bin/ext/hiveserver2.cmd
URL: http://svn.apache.org/viewvc/hive/branches/llap/bin/ext/hiveserver2.cmd?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/bin/ext/hiveserver2.cmd (original)
+++ hive/branches/llap/bin/ext/hiveserver2.cmd Tue Oct 14 19:06:45 2014
@@ -59,10 +59,7 @@ if [%1]==[hiveserver2_help] goto :hivese
if [%1]==[hiveserver2_catservice] goto :hiveserver2_catservice
-if [%1]==[hiveserver2_catcmd] goto :hiveserver2_catcmd
-
:hiveserver2
- echo "Starting Hive Thrift Server"
@rem hadoop 20 or newer - skip the aux_jars option and hiveconf
call %HIVE_BIN_PATH%\ext\util\execHiveCmd.cmd %CLASS%
@@ -78,21 +75,11 @@ goto :EOF
@echo ^<id^>HiveServer2^</id^>
@echo ^<name^>HiveServer2^</name^>
@echo ^<description^>Hadoop HiveServer2 Service^</description^>
-@echo ^<executable^>%SystemRoot%\system32\cmd.exe^</executable^>
-@echo ^<arguments^>/c %HIVE_BIN_PATH%\ext\hs2service.cmd ^</arguments^>
+@echo ^<executable^>%JAVA_HOME%\bin\java^</executable^>
+@echo ^<arguments^>%JAVA_HEAP_MAX% %HADOOP_OPTS% -classpath %CLASSPATH%;%HIVE_HBASE_PATH% %CLASS% -hiveconf hive.hadoop.classpath=%HIVE_LIB%\* -hiveconf hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory -hiveconf hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator -hiveconf hive.metastore.uris=" " %HIVE_OPTS%^</arguments^>
@echo ^</service^>
goto :EOF
-
-:hiveserver2_catcmd
-if not defined HADOOP_CLASSPATH (
- @echo set HADOOP_CLASSPATH=%HIVE_LIB%\*
- ) else (
- @echo set HADOOP_CLASSPATH=%HADOOP_CLASSPATH%;%HIVE_LIB%\*
- )
-@echo %JAVA_HOME%\bin\java %JAVA_HEAP_MAX% %HADOOP_OPTS% -classpath %CLASSPATH%;%HIVE_HBASE_PATH% %CLASS% -hiveconf hive.metastore.uris=" " -hiveconf hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory -hiveconf hive.security.authenticator.manager=org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator %HIVE_OPTS%
-goto :EOF
-
:AddToHiveHbasePath
if not defined HIVE_HBASE_PATH (
set HIVE_HBASE_PATH=%1
Modified: hive/branches/llap/bin/ext/hiveserver2.sh
URL: http://svn.apache.org/viewvc/hive/branches/llap/bin/ext/hiveserver2.sh?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/bin/ext/hiveserver2.sh (original)
+++ hive/branches/llap/bin/ext/hiveserver2.sh Tue Oct 14 19:06:45 2014
@@ -17,7 +17,6 @@ THISSERVICE=hiveserver2
export SERVICE_LIST="${SERVICE_LIST}${THISSERVICE} "
hiveserver2() {
- echo "Starting HiveServer2"
CLASS=org.apache.hive.service.server.HiveServer2
if $cygwin; then
HIVE_LIB=`cygpath -w "$HIVE_LIB"`
Modified: hive/branches/llap/bin/hive.cmd
URL: http://svn.apache.org/viewvc/hive/branches/llap/bin/hive.cmd?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/bin/hive.cmd (original)
+++ hive/branches/llap/bin/hive.cmd Tue Oct 14 19:06:45 2014
@@ -284,9 +284,6 @@ if defined CATSERVICE (
) else (
call %HADOOP_HOME%\libexec\hadoop-config.cmd
)
- if %TORUN% == hiveserver2 (
- call %HIVE_BIN_PATH%\ext\hiveserver2.cmd hiveserver2_catcmd > %HIVE_BIN_PATH%\ext\hs2service.cmd
- )
call %HIVE_BIN_PATH%\ext\%TORUN%.cmd %TORUN%%CATSERVICE% %*
goto :EOF
)
Modified: hive/branches/llap/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java (original)
+++ hive/branches/llap/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java Tue Oct 14 19:06:45 2014
@@ -60,6 +60,7 @@ import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.exec.mr.HadoopJobExecHelper;
+import org.apache.hadoop.hive.ql.exec.tez.TezJobExecHelper;
import org.apache.hadoop.hive.ql.parse.HiveParser;
import org.apache.hadoop.hive.ql.parse.VariableSubstitution;
import org.apache.hadoop.hive.ql.processors.CommandProcessor;
@@ -384,6 +385,7 @@ public class CliDriver {
// First, kill any running MR jobs
HadoopJobExecHelper.killRunningJobs();
+ TezJobExecHelper.killRunningJobs();
HiveInterruptUtils.interrupt();
}
});
Modified: hive/branches/llap/common/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/llap/common/pom.xml?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/common/pom.xml (original)
+++ hive/branches/llap/common/pom.xml Tue Oct 14 19:06:45 2014
@@ -72,6 +72,12 @@
</dependency>
<!-- test inter-project -->
<dependency>
+ <groupId>com.google.code.tempus-fugit</groupId>
+ <artifactId>tempus-fugit</artifactId>
+ <version>${tempus-fugit.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
Modified: hive/branches/llap/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/branches/llap/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Tue Oct 14 19:06:45 2014
@@ -205,8 +205,8 @@ public class HiveConf extends Configurat
PLAN_SERIALIZATION("hive.plan.serialization.format", "kryo",
"Query plan format serialization between client and task nodes. \n" +
"Two supported values are : kryo and javaXML. Kryo is default."),
- SCRATCHDIR("hive.exec.scratchdir", "/tmp/hive",
- "HDFS root scratch dir for Hive jobs which gets created with 777 permission. " +
+ SCRATCHDIR("hive.exec.scratchdir", "/tmp/hive",
+ "HDFS root scratch dir for Hive jobs which gets created with write all (733) permission. " +
"For each connecting user, an HDFS scratch dir: ${hive.exec.scratchdir}/<username> is created, " +
"with ${hive.scratch.dir.permission}."),
LOCALSCRATCHDIR("hive.exec.local.scratchdir",
@@ -215,7 +215,7 @@ public class HiveConf extends Configurat
DOWNLOADED_RESOURCES_DIR("hive.downloaded.resources.dir",
"${system:java.io.tmpdir}" + File.separator + "${hive.session.id}_resources",
"Temporary local directory for added resources in the remote file system."),
- SCRATCHDIRPERMISSION("hive.scratch.dir.permission", "700",
+ SCRATCHDIRPERMISSION("hive.scratch.dir.permission", "700",
"The permission for the user specific scratch directories that get created."),
SUBMITVIACHILD("hive.exec.submitviachild", false, ""),
SUBMITLOCALTASKVIACHILD("hive.exec.submit.local.task.via.child", true,
@@ -240,9 +240,9 @@ public class HiveConf extends Configurat
"The compression codec and other options are determined from Hadoop config variables mapred.output.compress*"),
COMPRESSINTERMEDIATECODEC("hive.intermediate.compression.codec", "", ""),
COMPRESSINTERMEDIATETYPE("hive.intermediate.compression.type", "", ""),
- BYTESPERREDUCER("hive.exec.reducers.bytes.per.reducer", (long) (1000 * 1000 * 1000),
- "size per reducer.The default is 1G, i.e if the input size is 10G, it will use 10 reducers."),
- MAXREDUCERS("hive.exec.reducers.max", 999,
+ BYTESPERREDUCER("hive.exec.reducers.bytes.per.reducer", (long) (256 * 1000 * 1000),
+ "size per reducer.The default is 256Mb, i.e if the input size is 1G, it will use 4 reducers."),
+ MAXREDUCERS("hive.exec.reducers.max", 1009,
"max number of reducers will be used. If the one specified in the configuration parameter mapred.reduce.tasks is\n" +
"negative, Hive will use this one as the max number of reducers when automatically determine number of reducers."),
PREEXECHOOKS("hive.exec.pre.hooks", "",
@@ -640,6 +640,9 @@ public class HiveConf extends Configurat
HIVEJOINCACHESIZE("hive.join.cache.size", 25000,
"How many rows in the joining tables (except the streaming table) should be cached in memory."),
+ // CBO related
+ HIVE_CBO_ENABLED("hive.cbo.enable", false, "Flag to control enabling Cost Based Optimizations using Calcite framework."),
+
// hive.mapjoin.bucket.cache.size has been replaced by hive.smbjoin.cache.row,
// need to remove by hive .13. Also, do not change default (see SMB operator)
HIVEMAPJOINBUCKETCACHESIZE("hive.mapjoin.bucket.cache.size", 100, ""),
@@ -1041,7 +1044,7 @@ public class HiveConf extends Configurat
"That means if reducer-num of the child RS is fixed (order by or forced bucketing) and small, it can make very slow, single MR.\n" +
"The optimization will be automatically disabled if number of reducers would be less than specified value."),
- HIVEOPTSORTDYNAMICPARTITION("hive.optimize.sort.dynamic.partition", true,
+ HIVEOPTSORTDYNAMICPARTITION("hive.optimize.sort.dynamic.partition", false,
"When enabled dynamic partitioning column will be globally sorted.\n" +
"This way we can keep only one record writer open for each partition value\n" +
"in the reducer thereby reducing the memory pressure on reducers."),
@@ -1186,13 +1189,6 @@ public class HiveConf extends Configurat
"Average row size is computed from average column size of all columns in the row. In the absence\n" +
"of column statistics and for variable length complex columns like map, the average number of\n" +
"entries/values can be specified using this config."),
- // to accurately compute statistics for GROUPBY map side parallelism needs to be known
- HIVE_STATS_MAP_SIDE_PARALLELISM("hive.stats.map.parallelism", 1,
- "Hive/Tez optimizer estimates the data size flowing through each of the operators.\n" +
- "For GROUPBY operator, to accurately compute the data size map-side parallelism needs to\n" +
- "be known. By default, this value is set to 1 since optimizer is not aware of the number of\n" +
- "mappers during compile-time. This Hive config can be used to specify the number of mappers\n" +
- "to be used for data size computation of GROUPBY operator."),
// statistics annotation fetches stats for each partition, which can be expensive. turning
// this off will result in basic sizes being fetched from namenode instead
HIVE_STATS_FETCH_PARTITION_STATS("hive.stats.fetch.partition.stats", true,
@@ -1243,10 +1239,16 @@ public class HiveConf extends Configurat
"This param is to control whether or not only do lock on queries\n" +
"that need to execute at least one mapred job."),
+ // Zookeeper related configs
HIVE_ZOOKEEPER_QUORUM("hive.zookeeper.quorum", "",
- "The list of ZooKeeper servers to talk to. This is only needed for read/write locks."),
+ "List of ZooKeeper servers to talk to. This is needed for: " +
+ "1. Read/write locks - when hive.lock.manager is set to " +
+ "org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager, " +
+ "2. When HiveServer2 supports service discovery via Zookeeper."),
HIVE_ZOOKEEPER_CLIENT_PORT("hive.zookeeper.client.port", "2181",
- "The port of ZooKeeper servers to talk to. This is only needed for read/write locks."),
+ "The port of ZooKeeper servers to talk to. " +
+ "If the list of Zookeeper servers specified in hive.zookeeper.quorum," +
+ "does not contain port numbers, this value is used."),
HIVE_ZOOKEEPER_SESSION_TIMEOUT("hive.zookeeper.session.timeout", 600*1000,
"ZooKeeper client's session timeout. The client is disconnected, and as a result, all locks released, \n" +
"if a heartbeat is not sent in the timeout."),
@@ -1294,6 +1296,9 @@ public class HiveConf extends Configurat
"Number of aborted transactions involving a particular table or partition before major\n" +
"compaction is initiated."),
+ HIVE_COMPACTOR_CLEANER_RUN_INTERVAL("hive.compactor.cleaner.run.interval", "5000ms",
+ new TimeValidator(TimeUnit.MILLISECONDS), "Time between runs of the cleaner thread"),
+
// For HBase storage handler
HIVE_HBASE_WAL_ENABLED("hive.hbase.wal.enabled", true,
"Whether writes to HBase should be forced to the write-ahead log. \n" +
@@ -1368,6 +1373,8 @@ public class HiveConf extends Configurat
"authorization manager class name to be used in the metastore for authorization.\n" +
"The user defined authorization class should implement interface \n" +
"org.apache.hadoop.hive.ql.security.authorization.HiveMetastoreAuthorizationProvider. "),
+ HIVE_METASTORE_AUTHORIZATION_AUTH_READS("hive.security.metastore.authorization.auth.reads", true,
+ "If this is true, metastore authorizer authorizes read actions on database, table"),
HIVE_METASTORE_AUTHENTICATOR_MANAGER("hive.security.metastore.authenticator.manager",
"org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator",
"authenticator manager class name to be used in the metastore for authentication. \n" +
@@ -1446,11 +1453,6 @@ public class HiveConf extends Configurat
"If the property is set, the value must be a valid URI (java.net.URI, e.g. \"file:///tmp/my-logging.properties\"), \n" +
"which you can then extract a URL from and pass to PropertyConfigurator.configure(URL)."),
- // Hive global init file location
- HIVE_GLOBAL_INIT_FILE_LOCATION("hive.server2.global.init.file.location", "${env:HIVE_CONF_DIR}",
- "The location of HS2 global init file (.hiverc).\n" +
- "If the property is reset, the value must be a valid path where the init file is located."),
-
// prefix used to auto generated column aliases (this should be started with '_')
HIVE_AUTOGEN_COLUMNALIAS_PREFIX_LABEL("hive.autogen.columnalias.prefix.label", "_c",
"String used as a prefix when auto generating column alias.\n" +
@@ -1468,10 +1470,10 @@ public class HiveConf extends Configurat
HIVE_INSERT_INTO_MULTILEVEL_DIRS("hive.insert.into.multilevel.dirs", false,
"Where to insert into multilevel directories like\n" +
"\"insert directory '/HIVEFT25686/chinna/' from table\""),
- HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS("hive.warehouse.subdir.inherit.perms", false,
- "Set this to true if the the table directories should inherit the\n" +
- "permission of the warehouse or database directory instead of being created\n" +
- "with the permissions derived from dfs umask"),
+ HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS("hive.warehouse.subdir.inherit.perms", true,
+ "Set this to false if the table directories should be created\n" +
+ "with the permissions derived from dfs umask instead of\n" +
+ "inheriting the permission of the warehouse or database directory."),
HIVE_INSERT_INTO_EXTERNAL_TABLES("hive.insert.into.external.tables", true,
"whether insert into external tables is allowed"),
@@ -1489,16 +1491,29 @@ public class HiveConf extends Configurat
"table. From 0.12 onwards, they are displayed separately. This flag will let you\n" +
"get old behavior, if desired. See, test-case in patch for HIVE-6689."),
+ // HiveServer2 specific configs
HIVE_SERVER2_MAX_START_ATTEMPTS("hive.server2.max.start.attempts", 30L, new RangeValidator(0L, null),
- "This number of times HiveServer2 will attempt to start before exiting, sleeping 60 seconds between retries. \n" +
- "The default of 30 will keep trying for 30 minutes."),
-
+ "Number of times HiveServer2 will attempt to start before exiting, sleeping 60 seconds " +
+ "between retries. \n The default of 30 will keep trying for 30 minutes."),
+ HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY("hive.server2.support.dynamic.service.discovery", false,
+ "Whether HiveServer2 supports dynamic service discovery for its clients. " +
+ "To support this, each instance of HiveServer2 currently uses ZooKeeper to register itself, " +
+ "when it is brought up. JDBC/ODBC clients should use the ZooKeeper ensemble: " +
+ "hive.zookeeper.quorum in their connection string."),
+ HIVE_SERVER2_ZOOKEEPER_NAMESPACE("hive.server2.zookeeper.namespace", "hiveserver2",
+ "The parent node in ZooKeeper used by HiveServer2 when supporting dynamic service discovery."),
+ // HiveServer2 global init file location
+ HIVE_SERVER2_GLOBAL_INIT_FILE_LOCATION("hive.server2.global.init.file.location", "${env:HIVE_CONF_DIR}",
+ "Either the location of a HS2 global init file or a directory containing a .hiverc file. If the \n" +
+ "property is set, the value must be a valid path to an init file or directory where the init file is located."),
HIVE_SERVER2_TRANSPORT_MODE("hive.server2.transport.mode", "binary", new StringSet("binary", "http"),
"Transport mode of HiveServer2."),
+ HIVE_SERVER2_THRIFT_BIND_HOST("hive.server2.thrift.bind.host", "",
+ "Bind host on which to run the HiveServer2 Thrift service."),
// http (over thrift) transport settings
HIVE_SERVER2_THRIFT_HTTP_PORT("hive.server2.thrift.http.port", 10001,
- "Port number when in HTTP mode."),
+ "Port number of HiveServer2 Thrift interface when hive.server2.transport.mode is 'http'."),
HIVE_SERVER2_THRIFT_HTTP_PATH("hive.server2.thrift.http.path", "cliservice",
"Path component of URL endpoint when in HTTP mode."),
HIVE_SERVER2_THRIFT_HTTP_MIN_WORKER_THREADS("hive.server2.thrift.http.min.worker.threads", 5,
@@ -1515,11 +1530,7 @@ public class HiveConf extends Configurat
// binary transport settings
HIVE_SERVER2_THRIFT_PORT("hive.server2.thrift.port", 10000,
- "Port number of HiveServer2 Thrift interface.\n" +
- "Can be overridden by setting $HIVE_SERVER2_THRIFT_PORT"),
- HIVE_SERVER2_THRIFT_BIND_HOST("hive.server2.thrift.bind.host", "",
- "Bind host on which to run the HiveServer2 Thrift interface.\n" +
- "Can be overridden by setting $HIVE_SERVER2_THRIFT_BIND_HOST"),
+ "Port number of HiveServer2 Thrift interface when hive.server2.transport.mode is 'binary'."),
// hadoop.rpc.protection being set to a higher level than HiveServer2
// does not make sense in most situations.
// HiveServer2 ignores hadoop.rpc.protection in favor of hive.server2.thrift.sasl.qop.
@@ -1702,6 +1713,9 @@ public class HiveConf extends Configurat
HIVE_VECTORIZATION_REDUCE_ENABLED("hive.vectorized.execution.reduce.enabled", true,
"This flag should be set to true to enable vectorized mode of the reduce-side of query execution.\n" +
"The default value is true."),
+ HIVE_VECTORIZATION_REDUCE_GROUPBY_ENABLED("hive.vectorized.execution.reduce.groupby.enabled", true,
+ "This flag should be set to true to enable vectorized mode of the reduce-side GROUP BY query execution.\n" +
+ "The default value is true."),
HIVE_VECTORIZATION_GROUPBY_CHECKINTERVAL("hive.vectorized.groupby.checkinterval", 100000,
"Number of entries added to the group by aggregation hash before a recomputation of average entry size is performed."),
HIVE_VECTORIZATION_GROUPBY_MAXENTRIES("hive.vectorized.groupby.maxentries", 1000000,
@@ -1710,8 +1724,10 @@ public class HiveConf extends Configurat
HIVE_VECTORIZATION_GROUPBY_FLUSH_PERCENT("hive.vectorized.groupby.flush.percent", (float) 0.1,
"Percent of entries in the group by aggregation hash flushed when the memory threshold is exceeded."),
-
HIVE_TYPE_CHECK_ON_INSERT("hive.typecheck.on.insert", true, ""),
+ HIVE_HADOOP_CLASSPATH("hive.hadoop.classpath", null,
+ "For Windows OS, we need to pass HIVE_HADOOP_CLASSPATH Java parameter while starting HiveServer2 \n" +
+ "using \"-hiveconf hive.hadoop.classpath=%HIVE_LIB%\"."),
HIVE_RPC_QUERY_PLAN("hive.rpc.query.plan", false,
"Whether to send the query plan via local resource or RPC"),
Modified: hive/branches/llap/common/src/test/org/apache/hadoop/hive/common/type/TestHiveChar.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/common/src/test/org/apache/hadoop/hive/common/type/TestHiveChar.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/common/src/test/org/apache/hadoop/hive/common/type/TestHiveChar.java (original)
+++ hive/branches/llap/common/src/test/org/apache/hadoop/hive/common/type/TestHiveChar.java Tue Oct 14 19:06:45 2014
@@ -18,10 +18,19 @@
package org.apache.hadoop.hive.common.type;
-import junit.framework.TestCase;
-
-public class TestHiveChar extends TestCase {
-
+import com.google.code.tempusfugit.concurrency.annotations.*;
+import com.google.code.tempusfugit.concurrency.*;
+import org.junit.*;
+import static org.junit.Assert.*;
+
+public class TestHiveChar {
+
+ @Rule public ConcurrentRule concurrentRule = new ConcurrentRule();
+ @Rule public RepeatingRule repeatingRule = new RepeatingRule();
+
+ @Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testBasic() {
HiveChar hc = new HiveChar("abc", 10);
assertEquals("abc ", hc.toString());
@@ -47,6 +56,9 @@ public class TestHiveChar extends TestCa
assertEquals(3, hc.getCharacterLength());
}
+ @Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testStringLength() {
HiveChar hc = new HiveChar();
@@ -60,6 +72,9 @@ public class TestHiveChar extends TestCa
assertEquals("0123456789 ", hc.toString());
}
+ @Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testComparison() {
HiveChar hc1 = new HiveChar();
HiveChar hc2 = new HiveChar();
Modified: hive/branches/llap/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java (original)
+++ hive/branches/llap/common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java Tue Oct 14 19:06:45 2014
@@ -20,12 +20,19 @@ package org.apache.hadoop.hive.common.ty
import java.math.BigDecimal;
import java.math.BigInteger;
-import org.junit.Assert;
-import org.junit.Test;
+import com.google.code.tempusfugit.concurrency.annotations.*;
+import com.google.code.tempusfugit.concurrency.*;
+import org.junit.*;
+import static org.junit.Assert.*;
public class TestHiveDecimal {
+ @Rule public ConcurrentRule concurrentRule = new ConcurrentRule();
+ @Rule public RepeatingRule repeatingRule = new RepeatingRule();
+
@Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testPrecisionScaleEnforcement() {
String decStr = "1786135888657847525803324040144343378.09799306448796128931113691624";
HiveDecimal dec = HiveDecimal.create(decStr);
@@ -82,6 +89,8 @@ public class TestHiveDecimal {
}
@Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testMultiply() {
HiveDecimal dec1 = HiveDecimal.create("0.00001786135888657847525803");
HiveDecimal dec2 = HiveDecimal.create("3.0000123456789");
@@ -105,6 +114,8 @@ public class TestHiveDecimal {
}
@Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testPow() {
HiveDecimal dec = HiveDecimal.create("3.00001415926");
Assert.assertEquals(dec.pow(2), dec.multiply(dec));
@@ -118,6 +129,8 @@ public class TestHiveDecimal {
}
@Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testDivide() {
HiveDecimal dec1 = HiveDecimal.create("3.14");
HiveDecimal dec2 = HiveDecimal.create("3");
@@ -133,6 +146,8 @@ public class TestHiveDecimal {
}
@Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testPlus() {
HiveDecimal dec1 = HiveDecimal.create("99999999999999999999999999999999999");
HiveDecimal dec2 = HiveDecimal.create("1");
@@ -145,6 +160,8 @@ public class TestHiveDecimal {
@Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testSubtract() {
HiveDecimal dec1 = HiveDecimal.create("3.140");
HiveDecimal dec2 = HiveDecimal.create("1.00");
@@ -152,6 +169,8 @@ public class TestHiveDecimal {
}
@Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testPosMod() {
HiveDecimal hd1 = HiveDecimal.create("-100.91");
HiveDecimal hd2 = HiveDecimal.create("9.8");
@@ -160,12 +179,16 @@ public class TestHiveDecimal {
}
@Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testHashCode() {
Assert.assertEquals(HiveDecimal.create("9").hashCode(), HiveDecimal.create("9.00").hashCode());
Assert.assertEquals(HiveDecimal.create("0").hashCode(), HiveDecimal.create("0.00").hashCode());
}
@Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testException() {
HiveDecimal dec = HiveDecimal.create("3.1415.926");
Assert.assertNull(dec);
@@ -174,6 +197,8 @@ public class TestHiveDecimal {
}
@Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testBinaryConversion() {
testBinaryConversion("0.00");
testBinaryConversion("-12.25");
Modified: hive/branches/llap/common/src/test/org/apache/hadoop/hive/common/type/TestHiveVarchar.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/common/src/test/org/apache/hadoop/hive/common/type/TestHiveVarchar.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/common/src/test/org/apache/hadoop/hive/common/type/TestHiveVarchar.java (original)
+++ hive/branches/llap/common/src/test/org/apache/hadoop/hive/common/type/TestHiveVarchar.java Tue Oct 14 19:06:45 2014
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hive.common.type;
-import junit.framework.TestCase;
import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.common.LogUtils;
@@ -28,8 +27,15 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Random;
+import com.google.code.tempusfugit.concurrency.annotations.*;
+import com.google.code.tempusfugit.concurrency.*;
+import org.junit.*;
+import static org.junit.Assert.*;
+
+public class TestHiveVarchar {
+ @Rule public ConcurrentRule concurrentRule = new ConcurrentRule();
+ @Rule public RepeatingRule repeatingRule = new RepeatingRule();
-public class TestHiveVarchar extends TestCase {
public TestHiveVarchar() {
super();
}
@@ -65,6 +71,9 @@ public class TestHiveVarchar extends Tes
}
}
+ @Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testStringLength() throws Exception {
int strLen = 20;
int[] lengths = { 15, 20, 25 };
@@ -124,6 +133,9 @@ public class TestHiveVarchar extends Tes
assertEquals(5, vc1.getCharacterLength());
}
+ @Test
+ @Concurrent(count=4)
+ @Repeating(repetition=100)
public void testComparison() throws Exception {
HiveVarchar hc1 = new HiveVarchar("abcd", 20);
HiveVarchar hc2 = new HiveVarchar("abcd", 20);
Modified: hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleAvg.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleAvg.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleAvg.java (original)
+++ hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleAvg.java Tue Oct 14 19:06:45 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.contrib.udaf.example;
+import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDAF;
import org.apache.hadoop.hive.ql.exec.UDAFEvaluator;
@@ -32,6 +33,8 @@ import org.apache.hadoop.hive.ql.exec.UD
* more efficient.
*
*/
+@Description(name = "example_avg",
+value = "_FUNC_(col) - Example UDAF to compute average")
public final class UDAFExampleAvg extends UDAF {
/**
Modified: hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleGroupConcat.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleGroupConcat.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleGroupConcat.java (original)
+++ hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleGroupConcat.java Tue Oct 14 19:06:45 2014
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.contrib.u
import java.util.ArrayList;
import java.util.Collections;
+import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDAF;
import org.apache.hadoop.hive.ql.exec.UDAFEvaluator;
@@ -35,6 +36,8 @@ import org.apache.hadoop.hive.ql.exec.UD
* implement built-in aggregation functions, which are harder to program but
* more efficient.
*/
+@Description(name = "example_group_concat",
+value = "_FUNC_(col) - Example UDAF that concatenates all arguments from different rows into a single string")
public class UDAFExampleGroupConcat extends UDAF {
/**
Modified: hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMaxN.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMaxN.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMaxN.java (original)
+++ hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMaxN.java Tue Oct 14 19:06:45 2014
@@ -19,11 +19,13 @@
package org.apache.hadoop.hive.contrib.udaf.example;
+import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDAF;
/**
* Returns the max N double values.
*/
+@Description(name = "example_max_n", value = "_FUNC_(expr) - Example UDAF that returns the max N double values")
public class UDAFExampleMaxN extends UDAF {
/**
Modified: hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMinN.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMinN.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMinN.java (original)
+++ hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udaf/example/UDAFExampleMinN.java Tue Oct 14 19:06:45 2014
@@ -19,11 +19,13 @@
package org.apache.hadoop.hive.contrib.udaf.example;
+import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDAF;
/**
* Returns the min N double values.
*/
+@Description(name = "example_min_n", value = "_FUNC_(expr) - Example UDAF that returns the min N double values")
public class UDAFExampleMinN extends UDAF{
/**
Modified: hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleAdd.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleAdd.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleAdd.java (original)
+++ hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleAdd.java Tue Oct 14 19:06:45 2014
@@ -17,12 +17,14 @@
*/
package org.apache.hadoop.hive.contrib.udf.example;
+import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
/**
* UDFExampleAdd.
*
*/
+@Description(name = "example_add", value = "_FUNC_(expr) - Example UDAF that returns the sum")
public class UDFExampleAdd extends UDF {
public Integer evaluate(Integer... a) {
Modified: hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleArraySum.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleArraySum.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleArraySum.java (original)
+++ hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleArraySum.java Tue Oct 14 19:06:45 2014
@@ -19,12 +19,14 @@ package org.apache.hadoop.hive.contrib.u
import java.util.List;
+import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
/**
* UDFExampleArraySum.
*
*/
+@Description(name = "example_arraysum", value = "_FUNC_(expr) - Example UDAF that returns the sum")
public class UDFExampleArraySum extends UDF {
public Double evaluate(List<Double> a) {
Modified: hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleFormat.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleFormat.java (original)
+++ hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleFormat.java Tue Oct 14 19:06:45 2014
@@ -17,12 +17,14 @@
*/
package org.apache.hadoop.hive.contrib.udf.example;
+import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
/**
* UDFExampleFormat.
*
*/
+@Description(name = "example_format", value = "_FUNC_(expr) - Example UDAF that returns formated String")
public class UDFExampleFormat extends UDF {
public String evaluate(String format, Object... args) {
Modified: hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleMapConcat.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleMapConcat.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleMapConcat.java (original)
+++ hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleMapConcat.java Tue Oct 14 19:06:45 2014
@@ -21,12 +21,15 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Map;
+import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
/**
* UDFExampleMapConcat.
*
*/
+@Description(name = "example_mapconcat",
+value = "_FUNC_(expr) - Example UDAF that returns contents of Map as a formated String")
public class UDFExampleMapConcat extends UDF {
public String evaluate(Map<String, String> a) {
Modified: hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleStructPrint.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleStructPrint.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleStructPrint.java (original)
+++ hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleStructPrint.java Tue Oct 14 19:06:45 2014
@@ -19,12 +19,15 @@ package org.apache.hadoop.hive.contrib.u
import java.util.List;
+import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
/**
* UDFExampleStructPrint.
*
*/
+@Description(name = "example_structprint",
+value = "_FUNC_(obj) - Example UDAF that returns contents of an object")
public class UDFExampleStructPrint extends UDF {
public String evaluate(Object a) {
Modified: hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFCount2.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFCount2.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFCount2.java (original)
+++ hive/branches/llap/contrib/src/java/org/apache/hadoop/hive/contrib/udtf/example/GenericUDTFCount2.java Tue Oct 14 19:06:45 2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.contrib.u
import java.util.ArrayList;
+import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
@@ -34,6 +35,8 @@ import org.apache.hadoop.hive.serde2.obj
* to test outputting of rows on close with lateral view.
*
*/
+@Description(name = "udtfCount2",
+value = "_FUNC_(col) - UDF outputs the number of rows seen, twice.")
public class GenericUDTFCount2 extends GenericUDTF {
private transient Integer count = Integer.valueOf(0);
Modified: hive/branches/llap/contrib/src/test/results/clientpositive/udf_example_add.q.out
URL: http://svn.apache.org/viewvc/hive/branches/llap/contrib/src/test/results/clientpositive/udf_example_add.q.out?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/contrib/src/test/results/clientpositive/udf_example_add.q.out (original)
+++ hive/branches/llap/contrib/src/test/results/clientpositive/udf_example_add.q.out Tue Oct 14 19:06:45 2014
@@ -25,36 +25,24 @@ SELECT example_add(1, 2),
FROM src LIMIT 1
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
+ Stage-0 is a root stage
STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: src
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: 3 (type: int), 6 (type: int), 10 (type: int), 3.3000000000000003 (type: double), 6.6 (type: double), 11.0 (type: double), 10.4 (type: double)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 500 Data size: 22000 Basic stats: COMPLETE Column stats: COMPLETE
- Limit
- Number of rows: 1
- Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
Stage: Stage-0
Fetch Operator
limit: 1
Processor Tree:
- ListSink
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: 3 (type: int), 6 (type: int), 10 (type: int), 3.3000000000000003 (type: double), 6.6 (type: double), 11.0 (type: double), 10.4 (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 500 Data size: 22000 Basic stats: COMPLETE Column stats: COMPLETE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 44 Basic stats: COMPLETE Column stats: COMPLETE
+ ListSink
PREHOOK: query: SELECT example_add(1, 2),
example_add(1, 2, 3),
Modified: hive/branches/llap/contrib/src/test/results/clientpositive/udf_example_format.q.out
URL: http://svn.apache.org/viewvc/hive/branches/llap/contrib/src/test/results/clientpositive/udf_example_format.q.out?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/contrib/src/test/results/clientpositive/udf_example_format.q.out (original)
+++ hive/branches/llap/contrib/src/test/results/clientpositive/udf_example_format.q.out Tue Oct 14 19:06:45 2014
@@ -19,36 +19,24 @@ SELECT example_format("abc"),
FROM src LIMIT 1
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
- Stage-1 is a root stage
- Stage-0 depends on stages: Stage-1
+ Stage-0 is a root stage
STAGE PLANS:
- Stage: Stage-1
- Map Reduce
- Map Operator Tree:
- TableScan
- alias: src
- Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: 'abc' (type: string), '1.1' (type: string), '1.1 1.200000e+00' (type: string), 'a 12 10' (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 500 Data size: 182500 Basic stats: COMPLETE Column stats: COMPLETE
- Limit
- Number of rows: 1
- Statistics: Num rows: 1 Data size: 365 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 1 Data size: 365 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-
Stage: Stage-0
Fetch Operator
limit: 1
Processor Tree:
- ListSink
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: 'abc' (type: string), '1.1' (type: string), '1.1 1.200000e+00' (type: string), 'a 12 10' (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 500 Data size: 182500 Basic stats: COMPLETE Column stats: COMPLETE
+ Limit
+ Number of rows: 1
+ Statistics: Num rows: 1 Data size: 365 Basic stats: COMPLETE Column stats: COMPLETE
+ ListSink
PREHOOK: query: SELECT example_format("abc"),
example_format("%1$s", 1.1),
Modified: hive/branches/llap/data/files/parquet_types.txt
URL: http://svn.apache.org/viewvc/hive/branches/llap/data/files/parquet_types.txt?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/data/files/parquet_types.txt (original)
+++ hive/branches/llap/data/files/parquet_types.txt Tue Oct 14 19:06:45 2014
@@ -1,21 +1,21 @@
-100|1|1|1.0|0.0|abc|2011-01-01 01:01:01.111111111|a |a
-101|2|2|1.1|0.3|def|2012-02-02 02:02:02.222222222|ab |ab
-102|3|3|1.2|0.6|ghi|2013-03-03 03:03:03.333333333|abc|abc
-103|1|4|1.3|0.9|jkl|2014-04-04 04:04:04.444444444|abcd|abcd
-104|2|5|1.4|1.2|mno|2015-05-05 05:05:05.555555555|abcde|abcde
-105|3|1|1.0|1.5|pqr|2016-06-06 06:06:06.666666666|abcdef|abcdef
-106|1|2|1.1|1.8|stu|2017-07-07 07:07:07.777777777|abcdefg|abcdefg
-107|2|3|1.2|2.1|vwx|2018-08-08 08:08:08.888888888|bcdefg|abcdefgh
-108|3|4|1.3|2.4|yza|2019-09-09 09:09:09.999999999|cdefg|abcdefghijklmnop
-109|1|5|1.4|2.7|bcd|2020-10-10 10:10:10.101010101|klmno|abcdedef
-110|2|1|1.0|3.0|efg|2021-11-11 11:11:11.111111111|pqrst|abcdede
-111|3|2|1.1|3.3|hij|2022-12-12 12:12:12.121212121|nopqr|abcded
-112|1|3|1.2|3.6|klm|2023-01-02 13:13:13.131313131|opqrs|abcdd
-113|2|4|1.3|3.9|nop|2024-02-02 14:14:14.141414141|pqrst|abc
-114|3|5|1.4|4.2|qrs|2025-03-03 15:15:15.151515151|qrstu|b
-115|1|1|1.0|4.5|tuv|2026-04-04 16:16:16.161616161|rstuv|abcded
-116|2|2|1.1|4.8|wxy|2027-05-05 17:17:17.171717171|stuvw|abcded
-117|3|3|1.2|5.1|zab|2028-06-06 18:18:18.181818181|tuvwx|abcded
-118|1|4|1.3|5.4|cde|2029-07-07 19:19:19.191919191|uvwzy|abcdede
-119|2|5|1.4|5.7|fgh|2030-08-08 20:20:20.202020202|vwxyz|abcdede
-120|3|1|1.0|6.0|ijk|2031-09-09 21:21:21.212121212|wxyza|abcde
+100|1|1|1.0|0.0|abc|2011-01-01 01:01:01.111111111|a |a |k1:v1|101,200|10,abc
+101|2|2|1.1|0.3|def|2012-02-02 02:02:02.222222222|ab |ab |k2:v2|102,200|10,def
+102|3|3|1.2|0.6|ghi|2013-03-03 03:03:03.333333333|abc|abc|k3:v3|103,200|10,ghi
+103|1|4|1.3|0.9|jkl|2014-04-04 04:04:04.444444444|abcd|abcd|k4:v4|104,200|10,jkl
+104|2|5|1.4|1.2|mno|2015-05-05 05:05:05.555555555|abcde|abcde|k5:v5|105,200|10,mno
+105|3|1|1.0|1.5|pqr|2016-06-06 06:06:06.666666666|abcdef|abcdef|k6:v6|106,200|10,pqr
+106|1|2|1.1|1.8|stu|2017-07-07 07:07:07.777777777|abcdefg|abcdefg|k7:v7|107,200|10,stu
+107|2|3|1.2|2.1|vwx|2018-08-08 08:08:08.888888888|bcdefg|abcdefgh|k8:v8|108,200|10,vwx
+108|3|4|1.3|2.4|yza|2019-09-09 09:09:09.999999999|cdefg|abcdefghijklmnop|k9:v9|109,200|10,yza
+109|1|5|1.4|2.7|bcd|2020-10-10 10:10:10.101010101|klmno|abcdedef|k10:v10|110,200|10,bcd
+110|2|1|1.0|3.0|efg|2021-11-11 11:11:11.111111111|pqrst|abcdede|k11:v11|111,200|10,efg
+111|3|2|1.1|3.3|hij|2022-12-12 12:12:12.121212121|nopqr|abcded|k12:v12|112,200|10,hij
+112|1|3|1.2|3.6|klm|2023-01-02 13:13:13.131313131|opqrs|abcdd|k13:v13|113,200|10,klm
+113|2|4|1.3|3.9|nop|2024-02-02 14:14:14.141414141|pqrst|abc|k14:v14|114,200|10,nop
+114|3|5|1.4|4.2|qrs|2025-03-03 15:15:15.151515151|qrstu|b|k15:v15|115,200|10,qrs
+115|1|1|1.0|4.5|qrs|2026-04-04 16:16:16.161616161|rstuv|abcded|k16:v16|116,200|10,qrs
+116|2|2|1.1|4.8|wxy|2027-05-05 17:17:17.171717171|stuvw|abcded|k17:v17|117,200|10,wxy
+117|3|3|1.2|5.1|zab|2028-06-06 18:18:18.181818181|tuvwx|abcded|k18:v18|118,200|10,zab
+118|1|4|1.3|5.4|cde|2029-07-07 19:19:19.191919191|uvwzy|abcdede|k19:v19|119,200|10,cde
+119|2|5|1.4|5.7|fgh|2030-08-08 20:20:20.202020202|vwxyz|abcdede|k20:v20|120,200|10,fgh
+120|3|1|1.0|6.0|ijk|2031-09-09 21:21:21.212121212|wxyza|abcde|k21:v21|121,200|10,ijk
Propchange: hive/branches/llap/hbase-handler/pom.xml
------------------------------------------------------------------------------
Merged /hive/branches/cbo/hbase-handler/pom.xml:r1605012-1627125
Merged /hive/trunk/hbase-handler/pom.xml:r1625337-1631832
Modified: hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/DefaultHBaseKeyFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/DefaultHBaseKeyFactory.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/DefaultHBaseKeyFactory.java (original)
+++ hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/DefaultHBaseKeyFactory.java Tue Oct 14 19:06:45 2014
@@ -18,6 +18,9 @@
package org.apache.hadoop.hive.hbase;
+import java.io.IOException;
+import java.util.Properties;
+
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase;
@@ -26,9 +29,6 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import java.io.IOException;
-import java.util.Properties;
-
public class DefaultHBaseKeyFactory extends AbstractHBaseKeyFactory implements HBaseKeyFactory {
protected LazySimpleSerDe.SerDeParameters serdeParams;
Modified: hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java (original)
+++ hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java Tue Oct 14 19:06:45 2014
@@ -53,6 +53,7 @@ public class HBaseSerDe extends Abstract
public static final String HBASE_COMPOSITE_KEY_CLASS = "hbase.composite.key.class";
public static final String HBASE_COMPOSITE_KEY_TYPES = "hbase.composite.key.types";
public static final String HBASE_COMPOSITE_KEY_FACTORY = "hbase.composite.key.factory";
+ public static final String HBASE_STRUCT_SERIALIZER_CLASS = "hbase.struct.serialization.class";
public static final String HBASE_SCAN_CACHE = "hbase.scan.cache";
public static final String HBASE_SCAN_CACHEBLOCKS = "hbase.scan.cacheblock";
public static final String HBASE_SCAN_BATCH = "hbase.scan.batch";
@@ -98,7 +99,7 @@ public class HBaseSerDe extends Abstract
cachedHBaseRow = new LazyHBaseRow(
(LazySimpleStructObjectInspector) cachedObjectInspector,
- serdeParams.getKeyIndex(), serdeParams.getKeyFactory());
+ serdeParams.getKeyIndex(), serdeParams.getKeyFactory(), serdeParams.getValueFactories());
serializer = new HBaseRowSerializer(serdeParams);
Modified: hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeHelper.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeHelper.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeHelper.java (original)
+++ hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeHelper.java Tue Oct 14 19:06:45 2014
@@ -41,6 +41,10 @@ import org.apache.hadoop.hive.serde.serd
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.avro.AvroObjectInspectorGenerator;
import org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils;
+import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
+import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.util.StringUtils;
@@ -371,6 +375,19 @@ public class HBaseSerDeHelper {
}
/**
+ * Create the {@link LazyObjectBase lazy field}
+ * */
+ public static LazyObjectBase createLazyField(ColumnMapping[] columnMappings, int fieldID,
+ ObjectInspector inspector) {
+ ColumnMapping colMap = columnMappings[fieldID];
+ if (colMap.getQualifierName() == null && !colMap.isHbaseRowKey()) {
+ // a column family
+ return new LazyHBaseCellMap((LazyMapObjectInspector) inspector);
+ }
+ return LazyFactory.createLazyObject(inspector, colMap.getBinaryStorage().get(0));
+ }
+
+ /**
* Auto-generates the key struct for composite keys
*
* @param compositeKeyParts map of composite key part name to its type. Usually this would be
Modified: hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeParameters.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeParameters.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeParameters.java (original)
+++ hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDeParameters.java Tue Oct 14 19:06:45 2014
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.hbase.Colu
import org.apache.hadoop.hive.hbase.struct.AvroHBaseValueFactory;
import org.apache.hadoop.hive.hbase.struct.DefaultHBaseValueFactory;
import org.apache.hadoop.hive.hbase.struct.HBaseValueFactory;
+import org.apache.hadoop.hive.hbase.struct.StructHBaseValueFactory;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils;
@@ -204,11 +205,21 @@ public class HBaseSerDeParameters {
for (int i = 0; i < columnMappings.size(); i++) {
String serType = getSerializationType(conf, tbl, columnMappings.getColumnsMapping()[i]);
- if (serType != null && serType.equals(AVRO_SERIALIZATION_TYPE)) {
+ if (AVRO_SERIALIZATION_TYPE.equals(serType)) {
Schema schema = getSchema(conf, tbl, columnMappings.getColumnsMapping()[i]);
- valueFactories.add(new AvroHBaseValueFactory(schema));
+ valueFactories.add(new AvroHBaseValueFactory(i, schema));
+ } else if (STRUCT_SERIALIZATION_TYPE.equals(serType)) {
+ String structValueClassName = tbl.getProperty(HBaseSerDe.HBASE_STRUCT_SERIALIZER_CLASS);
+
+ if (structValueClassName == null) {
+ throw new IllegalArgumentException(HBaseSerDe.HBASE_STRUCT_SERIALIZER_CLASS
+ + " must be set for hbase columns of type [" + STRUCT_SERIALIZATION_TYPE + "]");
+ }
+
+ Class<?> structValueClass = job.getClassByName(structValueClassName);
+ valueFactories.add(new StructHBaseValueFactory(i, structValueClass));
} else {
- valueFactories.add(new DefaultHBaseValueFactory());
+ valueFactories.add(new DefaultHBaseValueFactory(i));
}
}
} catch (Exception e) {
Modified: hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java (original)
+++ hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java Tue Oct 14 19:06:45 2014
@@ -20,15 +20,15 @@ package org.apache.hadoop.hive.hbase;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hive.hbase.ColumnMappings.ColumnMapping;
+import org.apache.hadoop.hive.hbase.struct.HBaseValueFactory;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
-import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase;
import org.apache.hadoop.hive.serde2.lazy.LazyStruct;
-import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector;
import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
@@ -47,18 +47,21 @@ public class LazyHBaseRow extends LazySt
private final int iKey;
private final HBaseKeyFactory keyFactory;
+ private final List<HBaseValueFactory> valueFactories;
public LazyHBaseRow(LazySimpleStructObjectInspector oi) {
- this(oi, -1, null);
+ this(oi, -1, null, null);
}
/**
* Construct a LazyHBaseRow object with the ObjectInspector.
*/
- public LazyHBaseRow(LazySimpleStructObjectInspector oi, int iKey, HBaseKeyFactory keyFactory) {
+ public LazyHBaseRow(LazySimpleStructObjectInspector oi, int iKey, HBaseKeyFactory keyFactory,
+ List<HBaseValueFactory> valueFactories) {
super(oi);
this.iKey = iKey;
this.keyFactory = keyFactory;
+ this.valueFactories = valueFactories;
}
/**
@@ -76,13 +79,14 @@ public class LazyHBaseRow extends LazySt
if (fieldID == iKey) {
return keyFactory.createKey(fieldRef.getFieldObjectInspector());
}
- ColumnMapping colMap = columnsMapping[fieldID];
- if (colMap.qualifierName == null && !colMap.hbaseRowKey) {
- // a column family
- return new LazyHBaseCellMap((LazyMapObjectInspector) fieldRef.getFieldObjectInspector());
+
+ if (valueFactories != null) {
+ return valueFactories.get(fieldID).createValueObject(fieldRef.getFieldObjectInspector());
}
- return LazyFactory.createLazyObject(fieldRef.getFieldObjectInspector(),
- colMap.binaryStorage.get(0));
+
+ // fallback to default
+ return HBaseSerDeHelper.createLazyField(columnsMapping, fieldID,
+ fieldRef.getFieldObjectInspector());
}
/**
Modified: hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/AvroHBaseValueFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/AvroHBaseValueFactory.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/AvroHBaseValueFactory.java (original)
+++ hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/AvroHBaseValueFactory.java Tue Oct 14 19:06:45 2014
@@ -48,7 +48,8 @@ public class AvroHBaseValueFactory exten
*
* @param schema the associated {@link Schema schema}
* */
- public AvroHBaseValueFactory(Schema schema) {
+ public AvroHBaseValueFactory(int fieldID, Schema schema) {
+ super(fieldID);
this.schema = schema;
}
Modified: hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/DefaultHBaseValueFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/DefaultHBaseValueFactory.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/DefaultHBaseValueFactory.java (original)
+++ hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/DefaultHBaseValueFactory.java Tue Oct 14 19:06:45 2014
@@ -21,9 +21,12 @@ import java.io.IOException;
import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.hbase.ColumnMappings;
+import org.apache.hadoop.hive.hbase.HBaseSerDeHelper;
import org.apache.hadoop.hive.hbase.HBaseSerDeParameters;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
+import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase;
import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
@@ -32,33 +35,46 @@ import org.apache.hadoop.hive.serde2.typ
/**
* Default implementation of the {@link HBaseValueFactory}
* */
-public class DefaultHBaseValueFactory implements HBaseValueFactory{
+public class DefaultHBaseValueFactory implements HBaseValueFactory {
protected LazySimpleSerDe.SerDeParameters serdeParams;
+ protected ColumnMappings columnMappings;
protected HBaseSerDeParameters hbaseParams;
protected Properties properties;
protected Configuration conf;
- @Override
+ private int fieldID;
+
+ public DefaultHBaseValueFactory(int fieldID) {
+ this.fieldID = fieldID;
+ }
+
+ @Override
public void init(HBaseSerDeParameters hbaseParams, Configuration conf, Properties properties)
- throws SerDeException {
+ throws SerDeException {
this.hbaseParams = hbaseParams;
this.serdeParams = hbaseParams.getSerdeParams();
+ this.columnMappings = hbaseParams.getColumnMappings();
this.properties = properties;
this.conf = conf;
- }
+ }
- @Override
- public ObjectInspector createValueObjectInspector(TypeInfo type)
- throws SerDeException {
+ @Override
+ public ObjectInspector createValueObjectInspector(TypeInfo type)
+ throws SerDeException {
return LazyFactory.createLazyObjectInspector(type, serdeParams.getSeparators(),
1, serdeParams.getNullSequence(), serdeParams.isEscaped(), serdeParams.getEscapeChar());
- }
+ }
- @Override
- public byte[] serializeValue(Object object, StructField field)
- throws IOException {
+ @Override
+ public LazyObjectBase createValueObject(ObjectInspector inspector) throws SerDeException {
+ return HBaseSerDeHelper.createLazyField(columnMappings.getColumnsMapping(), fieldID, inspector);
+ }
+
+ @Override
+ public byte[] serializeValue(Object object, StructField field)
+ throws IOException {
// TODO Add support for serialization of values here
- return null;
- }
-}
\ No newline at end of file
+ return null;
+ }
+}
Modified: hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/HBaseValueFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/HBaseValueFactory.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/HBaseValueFactory.java (original)
+++ hive/branches/llap/hbase-handler/src/java/org/apache/hadoop/hive/hbase/struct/HBaseValueFactory.java Tue Oct 14 19:06:45 2014
@@ -22,8 +22,10 @@ import java.io.IOException;
import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.hbase.HBaseKeyFactory;
import org.apache.hadoop.hive.hbase.HBaseSerDeParameters;
import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
@@ -53,6 +55,13 @@ public interface HBaseValueFactory {
ObjectInspector createValueObjectInspector(TypeInfo type) throws SerDeException;
/**
+ * create custom object for hbase value
+ *
+ * @param inspector OI create by {@link HBaseKeyFactory#createKeyObjectInspector}
+ */
+ LazyObjectBase createValueObject(ObjectInspector inspector) throws SerDeException;
+
+ /**
* Serialize the given hive object
*
* @param object the object to be serialized