You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by th...@apache.org on 2013/12/05 02:32:30 UTC
svn commit: r1547986 [5/5] - in /hive/branches/tez: ./
ant/src/org/apache/hadoop/hive/ant/ beeline/
beeline/src/java/org/apache/hive/beeline/ beeline/src/main/resources/
beeline/src/test/org/apache/hive/beeline/src/test/ cli/ common/
common/src/java/or...
Modified: hive/branches/tez/ql/src/test/templates/TestCliDriver.vm
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/templates/TestCliDriver.vm?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/templates/TestCliDriver.vm (original)
+++ hive/branches/tez/ql/src/test/templates/TestCliDriver.vm Thu Dec 5 01:32:27 2013
@@ -114,8 +114,8 @@ public class $className extends TestCase
return suite;
}
- static String debugHint = "\nSee build/ql/tmp/hive.log, "
- + "or try \"ant test ... -Dtest.silent=false\" to get more logs.";
+ static String debugHint = "\nSee ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, "
+ + "or check ./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/ for specific test cases logs.";
#foreach ($qf in $qfiles)
#set ($fname = $qf.getName())
Modified: hive/branches/tez/ql/src/test/templates/TestNegativeCliDriver.vm
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/templates/TestNegativeCliDriver.vm?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/templates/TestNegativeCliDriver.vm (original)
+++ hive/branches/tez/ql/src/test/templates/TestNegativeCliDriver.vm Thu Dec 5 01:32:27 2013
@@ -112,8 +112,8 @@ public class $className extends TestCase
System.err.println ("Cleaning up " + "$className");
}
- static String debugHint = "\nSee build/ql/tmp/hive.log, "
- + "or try \"ant test ... -Dtest.silent=false\" to get more logs.";
+ static String debugHint = "\nSee ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, "
+ + "or check ./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/ for specific test cases logs.";
#foreach ($qf in $qfiles)
#set ($fname = $qf.getName())
Modified: hive/branches/tez/ql/src/test/templates/TestParse.vm
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/templates/TestParse.vm?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/templates/TestParse.vm (original)
+++ hive/branches/tez/ql/src/test/templates/TestParse.vm Thu Dec 5 01:32:27 2013
@@ -100,8 +100,8 @@ public class $className extends TestCase
System.err.println ("Cleaning up " + "$className");
}
- static String debugHint = "\nSee build/ql/tmp/hive.log, "
- + "or try \"ant test ... -Dtest.silent=false\" to get more logs.";
+ static String debugHint = "\nSee ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, "
+ + "or check ./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/ for specific test cases logs.";
#foreach ($qf in $qfiles)
#set ($fname = $qf.getName())
Modified: hive/branches/tez/ql/src/test/templates/TestParseNegative.vm
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/templates/TestParseNegative.vm?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/templates/TestParseNegative.vm (original)
+++ hive/branches/tez/ql/src/test/templates/TestParseNegative.vm Thu Dec 5 01:32:27 2013
@@ -99,8 +99,8 @@ public class $className extends TestCase
return suite;
}
- static String debugHint = "\nSee build/ql/tmp/hive.log, "
- + "or try \"ant test ... -Dtest.silent=false\" to get more logs.";
+ static String debugHint = "\nSee ./ql/target/tmp/log/hive.log or ./itests/qtest/target/tmp/log/hive.log, "
+ + "or check ./ql/target/surefire-reports or ./itests/qtest/target/surefire-reports/ for specific test cases logs.";
#foreach ($qf in $qfiles)
#set ($fname = $qf.getName())
Modified: hive/branches/tez/serde/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/serde/pom.xml?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/serde/pom.xml (original)
+++ hive/branches/tez/serde/pom.xml Thu Dec 5 01:32:27 2013
@@ -88,9 +88,6 @@
<profiles>
<profile>
<id>hadoop-1</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -117,6 +114,13 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop-23.version}</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>${hadoop-23.version}</version>
<optional>true</optional>
Modified: hive/branches/tez/serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java (original)
+++ hive/branches/tez/serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java Thu Dec 5 01:32:27 2013
@@ -37,6 +37,34 @@ public final class ColumnProjectionUtils
private static final boolean READ_ALL_COLUMNS_DEFAULT = true;
/**
+ * @deprecated for backwards compatibility with <= 0.12, use setReadAllColumns
+ */
+ @Deprecated
+ public static void setFullyReadColumns(Configuration conf) {
+ setReadAllColumns(conf);
+ }
+
+ /**
+ * @deprecated for backwards compatibility with <= 0.12, use setReadAllColumns
+ * and appendReadColumns
+ */
+ @Deprecated
+ public static void setReadColumnIDs(Configuration conf, List<Integer> ids) {
+ setReadColumnIDConf(conf, READ_COLUMN_IDS_CONF_STR_DEFAULT);
+ appendReadColumns(conf, ids);
+ }
+
+ /**
+ * @deprecated for backwards compatibility with <= 0.12, use appendReadColumns
+ */
+ @Deprecated
+ public static void appendReadColumnID(Configuration conf, List<Integer> ids) {
+ appendReadColumns(conf, ids);
+ }
+
+
+
+ /**
* Sets the <em>READ_ALL_COLUMNS</em> flag and removes any previously
* set column ids.
*/
Modified: hive/branches/tez/serde/src/java/org/apache/hadoop/hive/serde2/RegexSerDe.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/serde/src/java/org/apache/hadoop/hive/serde2/RegexSerDe.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/serde/src/java/org/apache/hadoop/hive/serde2/RegexSerDe.java (original)
+++ hive/branches/tez/serde/src/java/org/apache/hadoop/hive/serde2/RegexSerDe.java Thu Dec 5 01:32:27 2013
@@ -29,6 +29,7 @@ import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.serde.serdeConstants;
@@ -37,6 +38,7 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
@@ -191,53 +193,72 @@ public class RegexSerDe extends Abstract
try {
String t = m.group(c+1);
TypeInfo typeInfo = columnTypes.get(c);
- String typeName = typeInfo.getTypeName();
// Convert the column to the correct type when needed and set in row obj
- if (typeName.equals(serdeConstants.STRING_TYPE_NAME)) {
+ PrimitiveTypeInfo pti = (PrimitiveTypeInfo) typeInfo;
+ switch (pti.getPrimitiveCategory()) {
+ case STRING:
row.set(c, t);
- } else if (typeName.equals(serdeConstants.TINYINT_TYPE_NAME)) {
+ break;
+ case BYTE:
Byte b;
b = Byte.valueOf(t);
row.set(c,b);
- } else if (typeName.equals(serdeConstants.SMALLINT_TYPE_NAME)) {
+ break;
+ case SHORT:
Short s;
s = Short.valueOf(t);
row.set(c,s);
- } else if (typeName.equals(serdeConstants.INT_TYPE_NAME)) {
+ break;
+ case INT:
Integer i;
i = Integer.valueOf(t);
row.set(c, i);
- } else if (typeName.equals(serdeConstants.BIGINT_TYPE_NAME)) {
+ break;
+ case LONG:
Long l;
l = Long.valueOf(t);
row.set(c, l);
- } else if (typeName.equals(serdeConstants.FLOAT_TYPE_NAME)) {
+ break;
+ case FLOAT:
Float f;
f = Float.valueOf(t);
row.set(c,f);
- } else if (typeName.equals(serdeConstants.DOUBLE_TYPE_NAME)) {
+ break;
+ case DOUBLE:
Double d;
d = Double.valueOf(t);
row.set(c,d);
- } else if (typeName.equals(serdeConstants.BOOLEAN_TYPE_NAME)) {
- Boolean b;
- b = Boolean.valueOf(t);
- row.set(c, b);
- } else if (typeName.equals(serdeConstants.TIMESTAMP_TYPE_NAME)) {
+ break;
+ case BOOLEAN:
+ Boolean bool;
+ bool = Boolean.valueOf(t);
+ row.set(c, bool);
+ break;
+ case TIMESTAMP:
Timestamp ts;
ts = Timestamp.valueOf(t);
row.set(c, ts);
- } else if (typeName.equals(serdeConstants.DATE_TYPE_NAME)) {
- Date d;
- d = Date.valueOf(t);
- row.set(c, d);
- } else if (typeInfo instanceof DecimalTypeInfo) {
+ break;
+ case DATE:
+ Date date;
+ date = Date.valueOf(t);
+ row.set(c, date);
+ break;
+ case DECIMAL:
HiveDecimal bd = HiveDecimal.create(t);
row.set(c, bd);
- } else if (typeInfo instanceof VarcharTypeInfo) {
+ break;
+ case CHAR:
+ HiveChar hc = new HiveChar(t, ((CharTypeInfo) typeInfo).getLength());
+ row.set(c, hc);
+ break;
+ case VARCHAR:
HiveVarchar hv = new HiveVarchar(t, ((VarcharTypeInfo)typeInfo).getLength());
row.set(c, hv);
+ break;
+ default:
+ throw new SerDeException("Unsupported type " + typeInfo);
}
} catch (RuntimeException e) {
partialMatchedRowsCount++;
Modified: hive/branches/tez/serde/src/test/org/apache/hadoop/hive/serde2/TestColumnProjectionUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/serde/src/test/org/apache/hadoop/hive/serde2/TestColumnProjectionUtils.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/serde/src/test/org/apache/hadoop/hive/serde2/TestColumnProjectionUtils.java (original)
+++ hive/branches/tez/serde/src/test/org/apache/hadoop/hive/serde2/TestColumnProjectionUtils.java Thu Dec 5 01:32:27 2013
@@ -83,4 +83,35 @@ public class TestColumnProjectionUtils {
assertEquals(columnIds, actual);
assertFalse(ColumnProjectionUtils.isReadAllColumns(conf));
}
+
+ @Test
+ public void testDeprecatedMethods() {
+ List<Integer> columnIds = new ArrayList<Integer>();
+ List<Integer> actual;
+
+ assertEquals(Collections.EMPTY_LIST, ColumnProjectionUtils.getReadColumnIDs(conf));
+
+ // test that read columns are initially an empty list
+ actual = ColumnProjectionUtils.getReadColumnIDs(conf);
+ assertEquals(Collections.EMPTY_LIST, actual);
+ // setting empty list results in reading none
+ ColumnProjectionUtils.setReadColumnIDs(conf, columnIds);
+ actual = ColumnProjectionUtils.getReadColumnIDs(conf);
+ assertEquals(Collections.EMPTY_LIST, actual);
+ // test set and append methods
+ columnIds.add(1);
+ ColumnProjectionUtils.setReadColumnIDs(conf, Collections.singletonList(1));
+ actual = ColumnProjectionUtils.getReadColumnIDs(conf);
+ Collections.sort(actual);
+ assertEquals(columnIds, actual);
+ columnIds.add(2);
+ ColumnProjectionUtils.appendReadColumnID(conf, Collections.singletonList(2));
+ actual = ColumnProjectionUtils.getReadColumnIDs(conf);
+ Collections.sort(actual);
+ assertEquals(columnIds, actual);
+ // test that setting read column ids set read all columns to false
+ assertFalse(ColumnProjectionUtils.isReadAllColumns(conf));
+ ColumnProjectionUtils.setFullyReadColumns(conf);
+ assertTrue(ColumnProjectionUtils.isReadAllColumns(conf));
+ }
}
Modified: hive/branches/tez/service/pom.xml
URL: http://svn.apache.org/viewvc/hive/branches/tez/service/pom.xml?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/service/pom.xml (original)
+++ hive/branches/tez/service/pom.xml Thu Dec 5 01:32:27 2013
@@ -106,9 +106,6 @@
<profiles>
<profile>
<id>hadoop-1</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- </activation>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
Modified: hive/branches/tez/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java (original)
+++ hive/branches/tez/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java Thu Dec 5 01:32:27 2013
@@ -76,19 +76,13 @@ public class SQLOperation extends Execut
this.runAsync = runInBackground;
}
-
- public void prepare() throws HiveSQLException {
- }
-
- private void runInternal(HiveConf sqlOperationConf) throws HiveSQLException {
+ /***
+ * Compile the query and extract metadata
+ * @param sqlOperationConf
+ * @throws HiveSQLException
+ */
+ public void prepare(HiveConf sqlOperationConf) throws HiveSQLException {
setState(OperationState.RUNNING);
- String statement_trimmed = statement.trim();
- String[] tokens = statement_trimmed.split("\\s");
- String cmd_1 = statement_trimmed.substring(tokens[0].length()).trim();
-
- int ret = 0;
- String errorMessage = "";
- String SQLState = null;
try {
driver = new Driver(sqlOperationConf, getParentSession().getUserName());
@@ -98,10 +92,9 @@ public class SQLOperation extends Execut
driver.setTryCount(Integer.MAX_VALUE);
String subStatement = new VariableSubstitution().substitute(sqlOperationConf, statement);
-
- response = driver.run(subStatement);
+ response = driver.compileAndRespond(subStatement);
if (0 != response.getResponseCode()) {
- throw new HiveSQLException("Error while processing statement: "
+ throw new HiveSQLException("Error while compiling statement: "
+ response.getErrorMessage(), response.getSQLState(), response.getResponseCode());
}
@@ -112,7 +105,7 @@ public class SQLOperation extends Execut
if(driver.getPlan().getFetchTask() != null) {
//Schema has to be set
if (mResultSchema == null || !mResultSchema.isSetFieldSchemas()) {
- throw new HiveSQLException("Error running query: Schema and FieldSchema " +
+ throw new HiveSQLException("Error compiling query: Schema and FieldSchema " +
"should be set when query plan has a FetchTask");
}
resultSchema = new TableSchema(mResultSchema);
@@ -136,12 +129,35 @@ public class SQLOperation extends Execut
setState(OperationState.ERROR);
throw new HiveSQLException("Error running query: " + e.toString(), e);
}
+ }
+
+ private void runInternal(HiveConf sqlOperationConf) throws HiveSQLException {
+ try {
+ // In Hive server mode, we are not able to retry in the FetchTask
+ // case, when calling fetch queries since execute() has returned.
+ // For now, we disable the test attempts.
+ driver.setTryCount(Integer.MAX_VALUE);
+
+ response = driver.run();
+ if (0 != response.getResponseCode()) {
+ throw new HiveSQLException("Error while processing statement: "
+ + response.getErrorMessage(), response.getSQLState(), response.getResponseCode());
+ }
+
+ } catch (HiveSQLException e) {
+ setState(OperationState.ERROR);
+ throw e;
+ } catch (Exception e) {
+ setState(OperationState.ERROR);
+ throw new HiveSQLException("Error running query: " + e.toString(), e);
+ }
setState(OperationState.FINISHED);
}
@Override
public void run() throws HiveSQLException {
setState(OperationState.PENDING);
+ prepare(getConfigForOperation());
if (!shouldRunAsync()) {
runInternal(getConfigForOperation());
} else {
Modified: hive/branches/tez/service/src/test/org/apache/hive/service/cli/CLIServiceTest.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/service/src/test/org/apache/hive/service/cli/CLIServiceTest.java?rev=1547986&r1=1547985&r2=1547986&view=diff
==============================================================================
--- hive/branches/tez/service/src/test/org/apache/hive/service/cli/CLIServiceTest.java (original)
+++ hive/branches/tez/service/src/test/org/apache/hive/service/cli/CLIServiceTest.java Thu Dec 5 01:32:27 2013
@@ -21,6 +21,7 @@ package org.apache.hive.service.cli;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;
+import static org.junit.Assert.assertTrue;
import java.util.Collections;
import java.util.HashMap;
@@ -155,7 +156,7 @@ public abstract class CLIServiceTest {
long pollTimeout = System.currentTimeMillis() + 100000;
assertNotNull(sessionHandle);
OperationState state = null;
- OperationHandle ophandle;
+ OperationHandle ophandle = null;
// Change lock manager, otherwise unit-test doesn't go through
String queryString = "SET hive.lock.manager=" +
@@ -171,8 +172,16 @@ public abstract class CLIServiceTest {
client.executeStatement(sessionHandle, queryString, confOverlay);
// Test async execution response when query is malformed
- String wrongQueryString = "SELECT NAME FROM TEST_EXEC";
- ophandle = client.executeStatementAsync(sessionHandle, wrongQueryString, confOverlay);
+ String wrongQuery = "SELECT NAME FROM TEST_EXEC";
+ try {
+ ophandle = client.executeStatementAsync(sessionHandle, wrongQuery, confOverlay);
+ fail("Async syntax excution should fail");
+ } catch (HiveSQLException e) {
+ // expected error
+ }
+
+ wrongQuery = "CREATE TABLE NON_EXISTING_TAB (ID STRING) location 'hdfs://fooNN:10000/a/b/c'";
+ ophandle = client.executeStatementAsync(sessionHandle, wrongQuery, confOverlay);
int count = 0;
while (true) {
@@ -199,6 +208,7 @@ public abstract class CLIServiceTest {
ophandle =
client.executeStatementAsync(sessionHandle, queryString, confOverlay);
+ assertTrue(ophandle.hasResultSet());
count = 0;
while (true) {
// Break if polling times out