You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by we...@apache.org on 2017/05/05 17:32:20 UTC
[33/51] [partial] hive git commit: HIVE-14671 : merge master into
hive-14535 (Wei Zheng)
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java
index 0a53259..d227275 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java
@@ -17,12 +17,12 @@
*/
package org.apache.hive.jdbc;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.File;
-import java.net.URLEncoder;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
@@ -35,7 +35,10 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+
import org.apache.hive.jdbc.miniHS2.MiniHS2;
+import org.hadoop.hive.jdbc.SSLTestUtils;
+
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -44,10 +47,13 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
public class TestSSL {
+
private static final Logger LOG = LoggerFactory.getLogger(TestSSL.class);
private static final String LOCALHOST_KEY_STORE_NAME = "keystore.jks";
private static final String EXAMPLEDOTCOM_KEY_STORE_NAME = "keystore_exampledotcom.jks";
@@ -55,19 +61,12 @@ public class TestSSL {
private static final String KEY_STORE_TRUST_STORE_PASSWORD = "HiveJdbc";
private static final String JAVA_TRUST_STORE_PROP = "javax.net.ssl.trustStore";
private static final String JAVA_TRUST_STORE_PASS_PROP = "javax.net.ssl.trustStorePassword";
- private static final String HS2_BINARY_MODE = "binary";
- private static final String HS2_HTTP_MODE = "http";
- private static final String HS2_HTTP_ENDPOINT = "cliservice";
- private static final String HS2_BINARY_AUTH_MODE = "NONE";
private MiniHS2 miniHS2 = null;
private static HiveConf conf = new HiveConf();
private Connection hs2Conn = null;
- private String dataFileDir = conf.get("test.data.files");
+ private String dataFileDir = SSLTestUtils.getDataFileDir();
private Map<String, String> confOverlay;
- private final String SSL_CONN_PARAMS = "ssl=true;sslTrustStore="
- + URLEncoder.encode(dataFileDir + File.separator + TRUST_STORE_NAME) + ";trustStorePassword="
- + KEY_STORE_TRUST_STORE_PASSWORD;
@BeforeClass
public static void beforeTest() throws Exception {
@@ -83,10 +82,6 @@ public class TestSSL {
@Before
public void setUp() throws Exception {
DriverManager.setLoginTimeout(0);
- if (!System.getProperty("test.data.files", "").isEmpty()) {
- dataFileDir = System.getProperty("test.data.files");
- }
- dataFileDir = dataFileDir.replace('\\', '/').replace("c:", "");
miniHS2 = new MiniHS2.Builder().withConf(conf).cleanupLocalDirOnStartup(false).build();
confOverlay = new HashMap<String, String>();
}
@@ -127,9 +122,9 @@ public class TestSSL {
// we depend on linux openssl exit codes
Assume.assumeTrue(System.getProperty("os.name").toLowerCase().contains("linux"));
- setSslConfOverlay(confOverlay);
+ SSLTestUtils.setSslConfOverlay(confOverlay);
// Test in binary mode
- setBinaryConfOverlay(confOverlay);
+ SSLTestUtils.setBinaryConfOverlay(confOverlay);
// Start HS2 with SSL
miniHS2.start(confOverlay);
@@ -146,7 +141,7 @@ public class TestSSL {
miniHS2.stop();
// Test in http mode
- setHttpConfOverlay(confOverlay);
+ SSLTestUtils.setHttpConfOverlay(confOverlay);
miniHS2.start(confOverlay);
// make SSL connection
try {
@@ -175,13 +170,13 @@ public class TestSSL {
*/
@Test
public void testInvalidConfig() throws Exception {
- clearSslConfOverlay(confOverlay);
+ SSLTestUtils.clearSslConfOverlay(confOverlay);
// Test in binary mode
- setBinaryConfOverlay(confOverlay);
+ SSLTestUtils.setBinaryConfOverlay(confOverlay);
miniHS2.start(confOverlay);
DriverManager.setLoginTimeout(4);
try {
- hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSL_CONN_PARAMS),
+ hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSLTestUtils.SSL_CONN_PARAMS),
System.getProperty("user.name"), "bar");
fail("SSL connection should fail with NON-SSL server");
} catch (SQLException e) {
@@ -204,10 +199,10 @@ public class TestSSL {
// Test in http mode with ssl properties specified in url
System.clearProperty(JAVA_TRUST_STORE_PROP);
System.clearProperty(JAVA_TRUST_STORE_PASS_PROP);
- setHttpConfOverlay(confOverlay);
+ SSLTestUtils.setHttpConfOverlay(confOverlay);
miniHS2.start(confOverlay);
try {
- hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSL_CONN_PARAMS),
+ hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSLTestUtils.SSL_CONN_PARAMS),
System.getProperty("user.name"), "bar");
fail("SSL connection should fail with NON-SSL server");
} catch (SQLException e) {
@@ -223,9 +218,9 @@ public class TestSSL {
*/
@Test
public void testConnectionMismatch() throws Exception {
- setSslConfOverlay(confOverlay);
+ SSLTestUtils.setSslConfOverlay(confOverlay);
// Test in binary mode
- setBinaryConfOverlay(confOverlay);
+ SSLTestUtils.setBinaryConfOverlay(confOverlay);
miniHS2.start(confOverlay);
// Start HS2 with SSL
try {
@@ -247,7 +242,7 @@ public class TestSSL {
miniHS2.stop();
// Test in http mode
- setHttpConfOverlay(confOverlay);
+ SSLTestUtils.setHttpConfOverlay(confOverlay);
miniHS2.start(confOverlay);
try {
hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", ";ssl=false"),
@@ -266,23 +261,23 @@ public class TestSSL {
*/
@Test
public void testSSLConnectionWithURL() throws Exception {
- setSslConfOverlay(confOverlay);
+ SSLTestUtils.setSslConfOverlay(confOverlay);
// Test in binary mode
- setBinaryConfOverlay(confOverlay);
+ SSLTestUtils.setBinaryConfOverlay(confOverlay);
// Start HS2 with SSL
miniHS2.start(confOverlay);
// make SSL connection
- hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSL_CONN_PARAMS),
+ hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSLTestUtils.SSL_CONN_PARAMS),
System.getProperty("user.name"), "bar");
hs2Conn.close();
miniHS2.stop();
// Test in http mode
- setHttpConfOverlay(confOverlay);
+ SSLTestUtils.setHttpConfOverlay(confOverlay);
miniHS2.start(confOverlay);
// make SSL connection
- hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSL_CONN_PARAMS),
+ hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSLTestUtils.SSL_CONN_PARAMS),
System.getProperty("user.name"), "bar");
hs2Conn.close();
}
@@ -293,9 +288,9 @@ public class TestSSL {
*/
@Test
public void testSSLConnectionWithProperty() throws Exception {
- setSslConfOverlay(confOverlay);
+ SSLTestUtils.setSslConfOverlay(confOverlay);
// Test in binary mode
- setBinaryConfOverlay(confOverlay);
+ SSLTestUtils.setBinaryConfOverlay(confOverlay);
// Start HS2 with SSL
miniHS2.start(confOverlay);
@@ -308,10 +303,10 @@ public class TestSSL {
miniHS2.stop();
// Test in http mode
- setHttpConfOverlay(confOverlay);
+ SSLTestUtils.setHttpConfOverlay(confOverlay);
miniHS2.start(confOverlay);
// make SSL connection
- hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSL_CONN_PARAMS),
+ hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default",SSLTestUtils.SSL_CONN_PARAMS),
System.getProperty("user.name"), "bar");
hs2Conn.close();
}
@@ -322,9 +317,9 @@ public class TestSSL {
*/
@Test
public void testSSLFetch() throws Exception {
- setSslConfOverlay(confOverlay);
+ SSLTestUtils.setSslConfOverlay(confOverlay);
// Test in binary mode
- setBinaryConfOverlay(confOverlay);
+ SSLTestUtils.setBinaryConfOverlay(confOverlay);
// Start HS2 with SSL
miniHS2.start(confOverlay);
@@ -332,11 +327,11 @@ public class TestSSL {
Path dataFilePath = new Path(dataFileDir, "kv1.txt");
// make SSL connection
- hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSL_CONN_PARAMS),
+ hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSLTestUtils.SSL_CONN_PARAMS),
System.getProperty("user.name"), "bar");
// Set up test data
- setupTestTableWithData(tableName, dataFilePath, hs2Conn);
+ SSLTestUtils.setupTestTableWithData(tableName, dataFilePath, hs2Conn);
Statement stmt = hs2Conn.createStatement();
ResultSet res = stmt.executeQuery("SELECT * FROM " + tableName);
@@ -357,20 +352,20 @@ public class TestSSL {
*/
@Test
public void testSSLFetchHttp() throws Exception {
- setSslConfOverlay(confOverlay);
+ SSLTestUtils.setSslConfOverlay(confOverlay);
// Test in http mode
- setHttpConfOverlay(confOverlay);
+ SSLTestUtils.setHttpConfOverlay(confOverlay);
miniHS2.start(confOverlay);
String tableName = "sslTab";
Path dataFilePath = new Path(dataFileDir, "kv1.txt");
// make SSL connection
- hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSL_CONN_PARAMS),
+ hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSLTestUtils.SSL_CONN_PARAMS),
System.getProperty("user.name"), "bar");
// Set up test data
- setupTestTableWithData(tableName, dataFilePath, hs2Conn);
+ SSLTestUtils.setupTestTableWithData(tableName, dataFilePath, hs2Conn);
Statement stmt = hs2Conn.createStatement();
ResultSet res = stmt.executeQuery("SELECT * FROM " + tableName);
int rowCount = 0;
@@ -393,16 +388,16 @@ public class TestSSL {
@Test
public void testConnectionWrongCertCN() throws Exception {
// This call sets the default ssl params including the correct keystore in the server config
- setSslConfOverlay(confOverlay);
+ SSLTestUtils.setSslConfOverlay(confOverlay);
// Replace default keystore with keystore for www.example.com
confOverlay.put(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname, dataFileDir + File.separator
+ EXAMPLEDOTCOM_KEY_STORE_NAME);
// Binary (TCP) mode
- setBinaryConfOverlay(confOverlay);
+ SSLTestUtils.setBinaryConfOverlay(confOverlay);
miniHS2.start(confOverlay);
try {
hs2Conn =
- DriverManager.getConnection(miniHS2.getJdbcURL("default", SSL_CONN_PARAMS),
+ DriverManager.getConnection(miniHS2.getJdbcURL("default", SSLTestUtils.SSL_CONN_PARAMS),
System.getProperty("user.name"), "bar");
fail("SSL connection, with the server providing wrong certifcate (with CN www.example.com, "
+ "instead of localhost), should fail");
@@ -415,11 +410,11 @@ public class TestSSL {
miniHS2.stop();
// Http mode
- setHttpConfOverlay(confOverlay);
+ SSLTestUtils.setHttpConfOverlay(confOverlay);
miniHS2.start(confOverlay);
try {
hs2Conn =
- DriverManager.getConnection(miniHS2.getJdbcURL("default", SSL_CONN_PARAMS),
+ DriverManager.getConnection(miniHS2.getJdbcURL("default", SSLTestUtils.SSL_CONN_PARAMS),
System.getProperty("user.name"), "bar");
fail("SSL connection, with the server providing wrong certifcate (with CN www.example.com, "
+ "instead of localhost), should fail");
@@ -439,10 +434,10 @@ public class TestSSL {
*/
@Test
public void testMetastoreWithSSL() throws Exception {
- setMetastoreSslConf(conf);
- setSslConfOverlay(confOverlay);
+ SSLTestUtils.setMetastoreSslConf(conf);
+ SSLTestUtils.setSslConfOverlay(confOverlay);
// Test in http mode
- setHttpConfOverlay(confOverlay);
+ SSLTestUtils.setHttpConfOverlay(confOverlay);
miniHS2 = new MiniHS2.Builder().withRemoteMetastore().withConf(conf).cleanupLocalDirOnStartup(false).build();
miniHS2.start(confOverlay);
@@ -450,11 +445,11 @@ public class TestSSL {
Path dataFilePath = new Path(dataFileDir, "kv1.txt");
// make SSL connection
- hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSL_CONN_PARAMS),
+ hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSLTestUtils.SSL_CONN_PARAMS),
System.getProperty("user.name"), "bar");
// Set up test data
- setupTestTableWithData(tableName, dataFilePath, hs2Conn);
+ SSLTestUtils.setupTestTableWithData(tableName, dataFilePath, hs2Conn);
Statement stmt = hs2Conn.createStatement();
ResultSet res = stmt.executeQuery("SELECT * FROM " + tableName);
int rowCount = 0;
@@ -474,7 +469,7 @@ public class TestSSL {
*/
@Test
public void testMetastoreConnectionWrongCertCN() throws Exception {
- setMetastoreSslConf(conf);
+ SSLTestUtils.setMetastoreSslConf(conf);
conf.setVar(ConfVars.HIVE_METASTORE_SSL_KEYSTORE_PATH,
dataFileDir + File.separator + EXAMPLEDOTCOM_KEY_STORE_NAME);
miniHS2 = new MiniHS2.Builder().withRemoteMetastore().withConf(conf).cleanupLocalDirOnStartup(false).build();
@@ -486,55 +481,4 @@ public class TestSSL {
miniHS2.stop();
}
-
- private void setupTestTableWithData(String tableName, Path dataFilePath,
- Connection hs2Conn) throws Exception {
- Statement stmt = hs2Conn.createStatement();
- stmt.execute("set hive.support.concurrency = false");
-
- stmt.execute("drop table if exists " + tableName);
- stmt.execute("create table " + tableName
- + " (under_col int comment 'the under column', value string)");
-
- // load data
- stmt.execute("load data local inpath '"
- + dataFilePath.toString() + "' into table " + tableName);
- stmt.close();
- }
-
- private void setSslConfOverlay(Map<String, String> confOverlay) {
- confOverlay.put(ConfVars.HIVE_SERVER2_USE_SSL.varname, "true");
- confOverlay.put(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname,
- dataFileDir + File.separator + LOCALHOST_KEY_STORE_NAME);
- confOverlay.put(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname,
- KEY_STORE_TRUST_STORE_PASSWORD);
- }
-
- private void setMetastoreSslConf(HiveConf conf) {
- conf.setBoolVar(ConfVars.HIVE_METASTORE_USE_SSL, true);
- conf.setVar(ConfVars.HIVE_METASTORE_SSL_KEYSTORE_PATH,
- dataFileDir + File.separator + LOCALHOST_KEY_STORE_NAME);
- conf.setVar(ConfVars.HIVE_METASTORE_SSL_KEYSTORE_PASSWORD,
- KEY_STORE_TRUST_STORE_PASSWORD);
- conf.setVar(ConfVars.HIVE_METASTORE_SSL_TRUSTSTORE_PATH,
- dataFileDir + File.separator + TRUST_STORE_NAME);
- conf.setVar(ConfVars.HIVE_METASTORE_SSL_TRUSTSTORE_PASSWORD,
- KEY_STORE_TRUST_STORE_PASSWORD);
- }
-
- private void clearSslConfOverlay(Map<String, String> confOverlay) {
- confOverlay.put(ConfVars.HIVE_SERVER2_USE_SSL.varname, "false");
- }
-
- private void setHttpConfOverlay(Map<String, String> confOverlay) {
- confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, HS2_HTTP_MODE);
- confOverlay.put(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH.varname, HS2_HTTP_ENDPOINT);
- confOverlay.put(ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "true");
- }
-
- private void setBinaryConfOverlay(Map<String, String> confOverlay) {
- confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, HS2_BINARY_MODE);
- confOverlay.put(ConfVars.HIVE_SERVER2_AUTHENTICATION.varname, HS2_BINARY_AUTH_MODE);
- confOverlay.put(ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "true");
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
index e344e0f..1a8337f 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
@@ -100,22 +100,6 @@ public class TestOperationLoggingLayout {
String row = iter.next()[0].toString();
Assert.assertEquals(true, row.matches("^.*(FATAL|ERROR|WARN|INFO|DEBUG|TRACE).*$"));
}
-
- String queryString = "set hive.server2.logging.operation.level=verbose";
- client.executeStatement(sessionHandle, queryString, null);
- operationHandle = client.executeStatement(sessionHandle, sqlCntStar, null);
- // just check for first few lines, some log lines are multi-line strings which can break format
- // checks below
- rowSetLog = client.fetchResults(operationHandle, FetchOrientation.FETCH_FIRST, 10,
- FetchType.LOG);
- iter = rowSetLog.iterator();
- // verbose pattern is "%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n"
- while (iter.hasNext()) {
- String row = iter.next()[0].toString();
- // just check if the log line starts with date
- Assert.assertEquals(true,
- row.matches("^\\d{2}[/](0[1-9]|1[012])[/](0[1-9]|[12][0-9]|3[01]).*$"));
- }
}
private SessionHandle setupSession() throws Exception {
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/pom.xml
----------------------------------------------------------------------
diff --git a/itests/pom.xml b/itests/pom.xml
index e5b54bf..400075b 100644
--- a/itests/pom.xml
+++ b/itests/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hive</groupId>
<artifactId>hive</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>3.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/qtest-accumulo/pom.xml
----------------------------------------------------------------------
diff --git a/itests/qtest-accumulo/pom.xml b/itests/qtest-accumulo/pom.xml
index 31cee36..b7ce283 100644
--- a/itests/qtest-accumulo/pom.xml
+++ b/itests/qtest-accumulo/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.hive</groupId>
<artifactId>hive-it</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>3.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/qtest-spark/pom.xml
----------------------------------------------------------------------
diff --git a/itests/qtest-spark/pom.xml b/itests/qtest-spark/pom.xml
index f301504..a506f7f 100644
--- a/itests/qtest-spark/pom.xml
+++ b/itests/qtest-spark/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.hive</groupId>
<artifactId>hive-it</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>3.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/qtest/pom.xml
----------------------------------------------------------------------
diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml
index 1c3b601..02664f3 100644
--- a/itests/qtest/pom.xml
+++ b/itests/qtest/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.hive</groupId>
<artifactId>hive-it</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>3.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java
----------------------------------------------------------------------
diff --git a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java
index 24eeb9d..4dd17c1e 100644
--- a/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java
+++ b/itests/qtest/src/test/java/org/apache/hadoop/hive/cli/TestBeeLineDriver.java
@@ -22,15 +22,15 @@ import java.util.List;
import org.apache.hadoop.hive.cli.control.CliAdapter;
import org.apache.hadoop.hive.cli.control.CliConfigs;
+import org.apache.hive.beeline.Parallelized;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestRule;
import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-@RunWith(Parameterized.class)
+@RunWith(Parallelized.class)
public class TestBeeLineDriver {
static CliAdapter adapter = new CliConfigs.BeeLineConfig().getCliAdapter();
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index dc02fd4..df947bb 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -71,6 +71,7 @@ minillap.shared.query.files=insert_into1.q,\
orc_merge3.q,\
orc_merge4.q,\
orc_merge_diff_fs.q,\
+ parallel_colstats.q,\
unionDistinct_1.q,\
union_type_chk.q,\
cte_2.q,\
@@ -352,6 +353,8 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\
vector_reduce_groupby_decimal.q,\
vector_string_concat.q,\
vector_struct_in.q,\
+ vector_udf_character_length.q,\
+ vector_udf_octet_length.q,\
vector_varchar_4.q,\
vector_varchar_mapjoin1.q,\
vector_varchar_simple.q,\
@@ -405,6 +408,7 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\
minillap.query.files=acid_bucket_pruning.q,\
bucket5.q,\
bucket6.q,\
+ dynamic_semijoin_user_level.q,\
except_distinct.q,\
explainuser_2.q,\
empty_dir_in_table.q,\
@@ -499,6 +503,7 @@ minillaplocal.query.files=acid_globallimit.q,\
groupby2.q,\
hybridgrace_hashjoin_1.q,\
hybridgrace_hashjoin_2.q,\
+ is_distinct_from.q,\
infer_bucket_sort_bucketed_table.q,\
input16_cc.q,\
insert_dir_distcp.q,\
@@ -510,12 +515,14 @@ minillaplocal.query.files=acid_globallimit.q,\
join_max_hashtable.q,\
join_nulls.q,\
join_nullsafe.q,\
+ join_is_not_distinct_from.q,\
leftsemijoin_mr.q,\
limit_join_transpose.q,\
lineage2.q,\
lineage3.q,\
list_bucket_dml_10.q,\
llap_partitioned.q,\
+ llap_vector_nohybridgrace.q,\
load_dyn_part5.q,\
lvj_mapjoin.q,\
mapjoin_decimal.q,\
@@ -567,6 +574,7 @@ minillaplocal.query.files=acid_globallimit.q,\
schema_evol_text_vecrow_table.q,\
selectDistinctStar.q,\
semijoin.q,\
+ semijoin_hint.q,\
smb_cache.q,\
special_character_in_tabnames_1.q,\
sqlmerge.q,\
@@ -611,11 +619,26 @@ minillaplocal.query.files=acid_globallimit.q,\
vector_auto_smb_mapjoin_14.q,\
vector_decimal_2.q,\
vector_decimal_udf.q,\
+ vector_groupby_cube1.q,\
+ vector_groupby_grouping_id1.q,\
+ vector_groupby_grouping_id2.q,\
+ vector_groupby_grouping_id3.q,\
+ vector_groupby_grouping_sets1.q,\
+ vector_groupby_grouping_sets2.q,\
+ vector_groupby_grouping_sets3.q,\
+ vector_groupby_grouping_sets4.q,\
+ vector_groupby_grouping_sets5.q,\
+ vector_groupby_grouping_sets6.q,\
+ vector_groupby_grouping_sets_grouping.q,\
+ vector_groupby_grouping_sets_limit.q,\
+ vector_groupby_grouping_window.q,\
+ vector_groupby_rollup1.q,\
vector_join30.q,\
vector_join_filters.q,\
vector_leftsemi_mapjoin.q,\
vector_number_compare_projection.q,\
vector_partitioned_date_time.q,\
+ vector_ptf_part_simple.q,\
vector_udf1.q,\
vectorization_short_regress.q,\
vectorized_dynamic_partition_pruning.q,\
@@ -725,10 +748,22 @@ encrypted.query.files=encryption_join_unencrypted_tbl.q,\
encryption_drop_view.q \
encryption_drop_partition.q \
encryption_with_trash.q \
- encryption_ctas.q
+ encryption_ctas.q \
+ encryption_auto_purge_tables.q \
+ encryption_drop_table_in_encrypted_db.q
beeline.positive.include=drop_with_concurrency.q,\
- escape_comments.q
+ escape_comments.q,\
+ smb_mapjoin_1.q,\
+ smb_mapjoin_10.q,\
+ smb_mapjoin_11.q,\
+ smb_mapjoin_12.q,\
+ smb_mapjoin_13.q,\
+ smb_mapjoin_16.q,\
+ smb_mapjoin_2.q,\
+ smb_mapjoin_3.q,\
+ smb_mapjoin_7.q,\
+ select_dummy_source.q
minimr.query.negative.files=cluster_tasklog_retrieval.q,\
file_with_header_footer_negative.q,\
@@ -1336,6 +1371,7 @@ spark.query.files=add_part_multiple.q, \
spark.only.query.files=spark_combine_equivalent_work.q,\
spark_dynamic_partition_pruning.q,\
spark_dynamic_partition_pruning_2.q,\
+ spark_explainuser_1.q,\
spark_vectorized_dynamic_partition_pruning.q,\
spark_use_file_size_for_mapjoin.q,\
spark_use_op_stats.q
@@ -1410,4 +1446,6 @@ spark.query.negative.files=groupby2_map_skew_multi_distinct.q,\
groupby2_multi_distinct.q,\
groupby3_map_skew_multi_distinct.q,\
groupby3_multi_distinct.q,\
- groupby_grouping_sets7.q
+ groupby_grouping_sets7.q,\
+ spark_job_max_tasks.q
+
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/test-serde/pom.xml
----------------------------------------------------------------------
diff --git a/itests/test-serde/pom.xml b/itests/test-serde/pom.xml
index df0ce8a..bf5f5d2 100644
--- a/itests/test-serde/pom.xml
+++ b/itests/test-serde/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hive</groupId>
<artifactId>hive-it</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>3.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/pom.xml
----------------------------------------------------------------------
diff --git a/itests/util/pom.xml b/itests/util/pom.xml
index 3740af8..a44a55c 100644
--- a/itests/util/pom.xml
+++ b/itests/util/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hive</groupId>
<artifactId>hive-it</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>3.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
index 9c97c31..02abe53 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.cli.control.AbstractCliConfig.MetastoreType;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveVariableSource;
import org.apache.hadoop.hive.conf.VariableSubstitution;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.junit.After;
@@ -139,12 +140,14 @@ public abstract class AbstractCoreBlobstoreCliDriver extends CliAdapter {
if ((ecode == 0) ^ expectSuccess) {
qt.failed(ecode, fname, debugHint);
}
- ecode = qt.checkCliDriverResults(fname);
- if (ecode != 0) {
- qt.failedDiff(ecode, fname, debugHint);
+ QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+ if (result.getReturnCode() != 0) {
+ String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
+ debugHint : "\r\n" + result.getCapturedOutput();
+ qt.failedDiff(result.getReturnCode(), fname, message);
}
}
- catch (Throwable e) {
+ catch (Exception e) {
qt.failed(e, fname, debugHint);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
index 67064b8..1457db0 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CliConfigs.java
@@ -287,6 +287,7 @@ public class CliConfigs {
excludesFrom(testConfigProps, "minimr.query.negative.files");
excludeQuery("authorization_uri_import.q");
+ excludeQuery("spark_job_max_tasks.q");
setResultsDir("ql/src/test/results/clientnegative");
setLogDir("itests/qtest/target/qfile-results/clientnegative");
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
index 3e4b373..73e5632 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreAccumuloCliDriver.java
@@ -18,8 +18,10 @@
package org.apache.hadoop.hive.cli.control;
import static org.junit.Assert.assertTrue;
+
import org.apache.hadoop.hive.accumulo.AccumuloQTestUtil;
import org.apache.hadoop.hive.accumulo.AccumuloTestSetup;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.junit.After;
import org.junit.AfterClass;
@@ -92,13 +94,13 @@ public class CoreAccumuloCliDriver extends CliAdapter {
qt.failed(ecode, fname, null);
}
- ecode = qt.checkCliDriverResults(fname);
- if (ecode != 0) {
- qt.failedDiff(ecode, fname, null);
+ QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+ if (result.getReturnCode() != 0) {
+ qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput());
}
qt.clearPostTestEffects();
- } catch (Throwable e) {
+ } catch (Exception e) {
qt.failed(e, fname, null);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java
index acc02eb..8c7057c 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreBeeLineDriver.java
@@ -17,21 +17,23 @@
*/
package org.apache.hadoop.hive.cli.control;
-import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import com.google.common.base.Strings;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.hooks.PreExecutePrinter;
-import org.apache.hive.beeline.qfile.QFile;
-import org.apache.hive.beeline.qfile.QFile.QFileBuilder;
-import org.apache.hive.beeline.qfile.QFileBeeLineClient;
-import org.apache.hive.beeline.qfile.QFileBeeLineClient.QFileClientBuilder;
+import org.apache.hive.beeline.QFile;
+import org.apache.hive.beeline.QFile.QFileBuilder;
+import org.apache.hive.beeline.QFileBeeLineClient;
+import org.apache.hive.beeline.QFileBeeLineClient.QFileClientBuilder;
import org.apache.hive.jdbc.miniHS2.MiniHS2;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import java.io.File;
import java.io.IOException;
+import java.sql.SQLException;
import java.util.HashMap;
public class CoreBeeLineDriver extends CliAdapter {
@@ -44,6 +46,7 @@ public class CoreBeeLineDriver extends CliAdapter {
private final File testDataDirectory;
private final File testScriptDirectory;
private boolean overwrite = false;
+ private boolean rewriteSourceTables = true;
private MiniHS2 miniHS2;
private QFileClientBuilder clientBuilder;
private QFileBuilder fileBuilder;
@@ -68,6 +71,10 @@ public class CoreBeeLineDriver extends CliAdapter {
if (testOutputOverwrite != null && "true".equalsIgnoreCase(testOutputOverwrite)) {
overwrite = true;
}
+ String testRewriteSourceTables = System.getProperty("test.rewrite.source.tables");
+ if (testRewriteSourceTables != null && "false".equalsIgnoreCase(testRewriteSourceTables)) {
+ rewriteSourceTables = false;
+ }
HiveConf hiveConf = new HiveConf();
// We do not need Zookeeper at the moment
@@ -92,19 +99,17 @@ public class CoreBeeLineDriver extends CliAdapter {
.setPassword("password");
fileBuilder = new QFileBuilder()
- .setHiveRootDirectory(hiveRootDirectory)
.setLogDirectory(logDirectory)
.setQueryDirectory(queryDirectory)
.setResultsDirectory(resultsDirectory)
- .setScratchDirectoryString(hiveConf.getVar(HiveConf.ConfVars.SCRATCHDIR))
- .setWarehouseDirectoryString(hiveConf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE));
+ .setRewriteSourceTables(rewriteSourceTables);
runInfraScript(initScript, new File(logDirectory, "init.beeline"),
new File(logDirectory, "init.raw"));
}
protected void runInfraScript(File script, File beeLineOutput, File log)
- throws IOException {
+ throws IOException, SQLException {
try (QFileBeeLineClient beeLineClient = clientBuilder.getClient(beeLineOutput)) {
beeLineClient.execute(
new String[]{
@@ -114,6 +119,9 @@ public class CoreBeeLineDriver extends CliAdapter {
"!run " + script,
},
log);
+ } catch (Exception e) {
+ throw new SQLException("Error running infra script: " + script
+ + "\nCheck the following logs for details:\n - " + beeLineOutput + "\n - " + log, e);
}
}
@@ -134,28 +142,41 @@ public class CoreBeeLineDriver extends CliAdapter {
try (QFileBeeLineClient beeLineClient = clientBuilder.getClient(qFile.getLogFile())) {
long startTime = System.currentTimeMillis();
System.err.println(">>> STARTED " + qFile.getName());
- assertTrue("QFile execution failed, see logs for details", beeLineClient.execute(qFile));
- long endTime = System.currentTimeMillis();
- System.err.println(">>> EXECUTED " + qFile.getName() + ":" + (endTime - startTime) / 1000
- + "s");
+ beeLineClient.execute(qFile);
+
+ long queryEndTime = System.currentTimeMillis();
+ System.err.println(">>> EXECUTED " + qFile.getName() + ": " + (queryEndTime - startTime)
+ + "ms");
qFile.filterOutput();
long filterEndTime = System.currentTimeMillis();
- System.err.println(">>> FILTERED " + qFile.getName() + ":" + (filterEndTime - endTime) / 1000
- + "s");
+ System.err.println(">>> FILTERED " + qFile.getName() + ": " + (filterEndTime - queryEndTime)
+ + "ms");
if (!overwrite) {
- if (qFile.compareResults()) {
+ QTestProcessExecResult result = qFile.compareResults();
+
+ long compareEndTime = System.currentTimeMillis();
+ System.err.println(">>> COMPARED " + qFile.getName() + ": "
+ + (compareEndTime - filterEndTime) + "ms");
+ if (result.getReturnCode() == 0) {
System.err.println(">>> PASSED " + qFile.getName());
} else {
System.err.println(">>> FAILED " + qFile.getName());
- fail("Failed diff");
+ String messageText = "Client result comparison failed with error code = "
+ + result.getReturnCode() + " while executing fname=" + qFile.getName() + "\n";
+ String messageBody = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
+ qFile.getDebugHint() : result.getCapturedOutput();
+ fail(messageText + messageBody);
}
} else {
qFile.overwriteResults();
System.err.println(">>> PASSED " + qFile.getName());
}
+ } catch (Exception e) {
+ throw new Exception("Exception running or analyzing the results of the query file: " + qFile
+ + "\n" + qFile.getDebugHint(), e);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
index a735346..d59b650 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java
@@ -23,7 +23,9 @@ import static org.junit.Assert.fail;
import java.util.concurrent.TimeUnit;
import com.google.common.base.Stopwatch;
+import com.google.common.base.Strings;
import org.apache.hadoop.hive.cli.control.AbstractCliConfig.MetastoreType;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.apache.hadoop.hive.util.ElapsedTimeLoggingWrapper;
@@ -175,13 +177,15 @@ public class CoreCliDriver extends CliAdapter {
failed = true;
qt.failed(ecode, fname, debugHint);
}
- ecode = qt.checkCliDriverResults(fname);
- if (ecode != 0) {
+ QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+ if (result.getReturnCode() != 0) {
failed = true;
- qt.failedDiff(ecode, fname, debugHint);
+ String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
+ debugHint : "\r\n" + result.getCapturedOutput();
+ qt.failedDiff(result.getReturnCode(), fname, message);
}
}
- catch (Throwable e) {
+ catch (Exception e) {
failed = true;
qt.failed(e, fname, debugHint);
} finally {
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
index 71a02bc..bff81dd 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCompareCliDriver.java
@@ -25,6 +25,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import com.google.common.base.Strings;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.junit.After;
@@ -143,12 +145,14 @@ public class CoreCompareCliDriver extends CliAdapter{
}
}
- ecode = qt.checkCompareCliDriverResults(fname, outputs);
- if (ecode != 0) {
- qt.failedDiff(ecode, fname, debugHint);
+ QTestProcessExecResult result = qt.checkCompareCliDriverResults(fname, outputs);
+ if (result.getReturnCode() != 0) {
+ String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
+ debugHint : "\r\n" + result.getCapturedOutput();
+ qt.failedDiff(result.getReturnCode(), fname, message);
}
}
- catch (Throwable e) {
+ catch (Exception e) {
qt.failed(e, fname, debugHint);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
index 956a42d..aa0b071 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.fail;
import org.apache.hadoop.hive.hbase.HBaseQTestUtil;
import org.apache.hadoop.hive.hbase.HBaseTestSetup;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.junit.After;
import org.junit.AfterClass;
@@ -120,12 +121,12 @@ public class CoreHBaseCliDriver extends CliAdapter {
qt.failed(ecode, fname, null);
}
- ecode = qt.checkCliDriverResults(fname);
- if (ecode != 0) {
- qt.failedDiff(ecode, fname, null);
+ QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+ if (result.getReturnCode() != 0) {
+ qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput());
}
- } catch (Throwable e) {
+ } catch (Exception e) {
qt.failed(e, fname, null);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
index 6225180..8320a80 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseNegativeCliDriver.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.fail;
import org.apache.hadoop.hive.hbase.HBaseQTestUtil;
import org.apache.hadoop.hive.hbase.HBaseTestSetup;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.junit.After;
import org.junit.AfterClass;
@@ -101,13 +102,13 @@ public class CoreHBaseNegativeCliDriver extends CliAdapter {
qt.failed(fname, null);
}
- ecode = qt.checkCliDriverResults(fname);
- if (ecode != 0) {
- qt.failedDiff(ecode, fname, null);
+ QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+ if (result.getReturnCode() != 0) {
+ qt.failedDiff(result.getReturnCode(), fname, result.getCapturedOutput());
}
qt.clearPostTestEffects();
- } catch (Throwable e) {
+ } catch (Exception e) {
qt.failed(e, fname, null);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
index 65b2ce7..438a61e 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.hive.cli.control;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import com.google.common.base.Strings;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.junit.After;
@@ -123,12 +125,14 @@ public class CoreNegativeCliDriver extends CliAdapter{
qt.failed(fname, debugHint);
}
- ecode = qt.checkCliDriverResults(fname);
- if (ecode != 0) {
- qt.failedDiff(ecode, fname, debugHint);
+ QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+ if (result.getReturnCode() != 0) {
+ String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
+ debugHint : "\r\n" + result.getCapturedOutput();
+ qt.failedDiff(result.getReturnCode(), fname, message);
}
}
- catch (Throwable e) {
+ catch (Exception e) {
qt.failed(e, fname, debugHint);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
index 8620cde..26b9ce1 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java
@@ -22,6 +22,8 @@ package org.apache.hadoop.hive.cli.control;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import com.google.common.base.Strings;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.junit.After;
@@ -121,11 +123,13 @@ public class CorePerfCliDriver extends CliAdapter{
if (ecode != 0) {
qt.failed(ecode, fname, debugHint);
}
- ecode = qt.checkCliDriverResults(fname);
- if (ecode != 0) {
- qt.failedDiff(ecode, fname, debugHint);
+ QTestProcessExecResult result = qt.checkCliDriverResults(fname);
+ if (result.getReturnCode() != 0) {
+ String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
+ debugHint : "\r\n" + result.getCapturedOutput();
+ qt.failedDiff(result.getReturnCode(), fname, message);
}
- } catch (Throwable e) {
+ } catch (Exception e) {
qt.failed(e, fname, debugHint);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java
index 6acc7a0..75cce14 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java
@@ -18,12 +18,8 @@
package org.apache.hadoop.hive.ql;
-
-import com.google.common.base.Strings;
-import org.apache.commons.lang3.StringUtils;
-
/**
- * Standard output and return code of a process executed during the qtests
+ * Standard output and return code of a process executed during the qtests.
*/
public class QTestProcessExecResult {
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index e32ee74..b897ffa 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -39,6 +39,10 @@ import java.io.PrintStream;
import java.io.Serializable;
import java.io.StringWriter;
import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.StandardOpenOption;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
@@ -51,6 +55,7 @@ import java.util.Collection;
import java.util.Comparator;
import java.util.Deque;
import java.util.EnumSet;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
@@ -59,17 +64,22 @@ import java.util.Properties;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import java.util.stream.Stream;
import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileStatus;
@@ -118,7 +128,6 @@ import org.apache.hadoop.hive.ql.processors.HiveCommand;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.util.Shell;
import org.apache.hive.common.util.StreamPrinter;
import org.apache.logging.log4j.util.Strings;
import org.apache.tools.ant.BuildException;
@@ -167,7 +176,7 @@ public class QTestUtil {
private final Set<String> qNoSessionReuseQuerySet;
private final Set<String> qJavaVersionSpecificOutput;
private static final String SORT_SUFFIX = ".sorted";
- private final HashSet<String> srcTables;
+ private final Set<String> srcTables;
private final Set<String> srcUDFs;
private final MiniClusterType clusterType;
private final FsType fsType;
@@ -200,7 +209,7 @@ public class QTestUtil {
}
private HBaseTestingUtility utility;
- HashSet<String> getSrcTables() {
+ public static Set<String> getSrcTables() {
HashSet<String> srcTables = new HashSet<String>();
// FIXME: moved default value to here...for now
// i think this features is never really used from the command line
@@ -1395,7 +1404,8 @@ public class QTestUtil {
int rc = response.getResponseCode();
if (rc != 0) {
- SessionState.get().out.println(response);
+ SessionState.getConsole().printError(response.toString(), response.getException() != null ?
+ Throwables.getStackTraceAsString(response.getException()) : "");
}
return rc;
@@ -1403,7 +1413,7 @@ public class QTestUtil {
throw new RuntimeException("Could not get CommandProcessor for command: " + commandName);
}
} catch (Exception e) {
- throw new RuntimeException("Could not execute test command: " + e.getMessage());
+ throw new RuntimeException("Could not execute test command", e);
}
}
@@ -1466,10 +1476,6 @@ public class QTestUtil {
// Create an instance of hive in order to create the tables
testWarehouse = conf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE);
db = Hive.get(conf);
- // Create dest4 to replace dest4_sequencefile
- LinkedList<String> cols = new LinkedList<String>();
- cols.add("key");
- cols.add("value");
// Move all data from dest4_sequencefile to dest4
drv
@@ -1480,7 +1486,7 @@ public class QTestUtil {
true, true);
}
- public int checkNegativeResults(String tname, Exception e) throws Exception {
+ public QTestProcessExecResult checkNegativeResults(String tname, Exception e) throws Exception {
String outFileExtension = getOutFileExtension(tname);
@@ -1503,16 +1509,17 @@ public class QTestUtil {
outfd.write(e.getMessage());
outfd.close();
- int exitVal = executeDiffCommand(outf.getPath(), expf, false,
+ QTestProcessExecResult result = executeDiffCommand(outf.getPath(), expf, false,
qSortSet.contains(qf.getName()));
- if (exitVal != 0 && overWrite) {
- exitVal = overwriteResults(outf.getPath(), expf);
+ if (overWrite) {
+ overwriteResults(outf.getPath(), expf);
+ return QTestProcessExecResult.createWithoutOutput(0);
}
- return exitVal;
+ return result;
}
- public int checkParseResults(String tname, ASTNode tree) throws Exception {
+ public QTestProcessExecResult checkParseResults(String tname, ASTNode tree) throws Exception {
if (tree != null) {
String outFileExtension = getOutFileExtension(tname);
@@ -1528,10 +1535,11 @@ public class QTestUtil {
outfd.write(tree.toStringTree());
outfd.close();
- int exitVal = executeDiffCommand(outf.getPath(), expf, false, false);
+ QTestProcessExecResult exitVal = executeDiffCommand(outf.getPath(), expf, false, false);
- if (exitVal != 0 && overWrite) {
- exitVal = overwriteResults(outf.getPath(), expf);
+ if (overWrite) {
+ overwriteResults(outf.getPath(), expf);
+ return QTestProcessExecResult.createWithoutOutput(0);
}
return exitVal;
@@ -1700,7 +1708,7 @@ public class QTestUtil {
});
/* This list may be modified by specific cli drivers to mask strings that change on every test */
- private List<Pair<Pattern, String>> patternsWithMaskComments = new ArrayList<Pair<Pattern, String>>() {{
+ private final List<Pair<Pattern, String>> patternsWithMaskComments = new ArrayList<Pair<Pattern, String>>() {{
add(toPatternPair("(pblob|s3.?|swift|wasb.?).*hive-staging.*","### BLOBSTORE_STAGING_PATH ###"));
}};
@@ -1712,7 +1720,7 @@ public class QTestUtil {
patternsWithMaskComments.add(toPatternPair(patternStr, maskComment));
}
- public int checkCliDriverResults(String tname) throws Exception {
+ public QTestProcessExecResult checkCliDriverResults(String tname) throws Exception {
assert(qMap.containsKey(tname));
String outFileExtension = getOutFileExtension(tname);
@@ -1721,69 +1729,71 @@ public class QTestUtil {
File f = new File(logDir, tname + outFileExtension);
maskPatterns(planMask, f.getPath());
- int exitVal = executeDiffCommand(f.getPath(),
+ QTestProcessExecResult exitVal = executeDiffCommand(f.getPath(),
outFileName, false,
qSortSet.contains(tname));
- if (exitVal != 0 && overWrite) {
- exitVal = overwriteResults(f.getPath(), outFileName);
+ if (overWrite) {
+ overwriteResults(f.getPath(), outFileName);
+ return QTestProcessExecResult.createWithoutOutput(0);
}
return exitVal;
}
- public int checkCompareCliDriverResults(String tname, List<String> outputs) throws Exception {
+ public QTestProcessExecResult checkCompareCliDriverResults(String tname, List<String> outputs)
+ throws Exception {
assert outputs.size() > 1;
maskPatterns(planMask, outputs.get(0));
for (int i = 1; i < outputs.size(); ++i) {
maskPatterns(planMask, outputs.get(i));
- int ecode = executeDiffCommand(
+ QTestProcessExecResult result = executeDiffCommand(
outputs.get(i - 1), outputs.get(i), false, qSortSet.contains(tname));
- if (ecode != 0) {
+ if (result.getReturnCode() != 0) {
System.out.println("Files don't match: " + outputs.get(i - 1) + " and " + outputs.get(i));
- return ecode;
+ return result;
}
}
- return 0;
+ return QTestProcessExecResult.createWithoutOutput(0);
}
- private static int overwriteResults(String inFileName, String outFileName) throws Exception {
+ private static void overwriteResults(String inFileName, String outFileName) throws Exception {
// This method can be replaced with Files.copy(source, target, REPLACE_EXISTING)
// once Hive uses JAVA 7.
System.out.println("Overwriting results " + inFileName + " to " + outFileName);
- return executeCmd(new String[] {
+ int result = executeCmd(new String[]{
"cp",
getQuotedString(inFileName),
getQuotedString(outFileName)
- });
+ }).getReturnCode();
+ if (result != 0)
+ throw new IllegalStateException("Unexpected error while overwriting " +
+ inFileName + " with " + outFileName);
}
- private static int executeDiffCommand(String inFileName,
+ private static QTestProcessExecResult executeDiffCommand(String inFileName,
String outFileName,
boolean ignoreWhiteSpace,
boolean sortResults
) throws Exception {
- int result = 0;
+ QTestProcessExecResult result;
if (sortResults) {
// sort will try to open the output file in write mode on windows. We need to
// close it first.
SessionState ss = SessionState.get();
if (ss != null && ss.out != null && ss.out != System.out) {
- ss.out.close();
+ ss.out.close();
}
String inSorted = inFileName + SORT_SUFFIX;
String outSorted = outFileName + SORT_SUFFIX;
- result = sortFiles(inFileName, inSorted);
- result |= sortFiles(outFileName, outSorted);
- if (result != 0) {
- System.err.println("ERROR: Could not sort files before comparing");
- return result;
- }
+ sortFiles(inFileName, inSorted);
+ sortFiles(outFileName, outSorted);
+
inFileName = inSorted;
outFileName = outSorted;
}
@@ -1813,40 +1823,47 @@ public class QTestUtil {
return result;
}
- private static int sortFiles(String in, String out) throws Exception {
- return executeCmd(new String[] {
+ private static void sortFiles(String in, String out) throws Exception {
+ int result = executeCmd(new String[]{
"sort",
getQuotedString(in),
- }, out, null);
+ }, out, null).getReturnCode();
+ if (result != 0)
+ throw new IllegalStateException("Unexpected error while sorting " + in);
}
- private static int executeCmd(Collection<String> args) throws Exception {
+ private static QTestProcessExecResult executeCmd(Collection<String> args) throws Exception {
return executeCmd(args, null, null);
}
- private static int executeCmd(String[] args) throws Exception {
+ private static QTestProcessExecResult executeCmd(String[] args) throws Exception {
return executeCmd(args, null, null);
}
- private static int executeCmd(Collection<String> args, String outFile, String errFile) throws Exception {
+ private static QTestProcessExecResult executeCmd(Collection<String> args, String outFile,
+ String errFile) throws Exception {
String[] cmdArray = args.toArray(new String[args.size()]);
return executeCmd(cmdArray, outFile, errFile);
}
- private static int executeCmd(String[] args, String outFile, String errFile) throws Exception {
+ private static QTestProcessExecResult executeCmd(String[] args, String outFile,
+ String errFile) throws Exception {
System.out.println("Running: " + org.apache.commons.lang.StringUtils.join(args, ' '));
PrintStream out = outFile == null ?
SessionState.getConsole().getChildOutStream() :
- new PrintStream(new FileOutputStream(outFile), true);
+ new PrintStream(new FileOutputStream(outFile), true, "UTF-8");
PrintStream err = errFile == null ?
SessionState.getConsole().getChildErrStream() :
- new PrintStream(new FileOutputStream(errFile), true);
+ new PrintStream(new FileOutputStream(errFile), true, "UTF-8");
Process executor = Runtime.getRuntime().exec(args);
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintStream str = new PrintStream(bos, true, "UTF-8");
+
StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, err);
- StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, out);
+ StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, out, str);
outPrinter.start();
errPrinter.start();
@@ -1864,7 +1881,8 @@ public class QTestUtil {
err.close();
}
- return result;
+ return QTestProcessExecResult.
+ create(result, new String(bos.toByteArray(), StandardCharsets.UTF_8));
}
private static String getQuotedString(String str){
@@ -2040,11 +2058,18 @@ public class QTestUtil {
qt[i].clearTestSideEffects();
qt[i].cliInit(qfiles[i].getName(), false);
qt[i].executeClient(qfiles[i].getName());
- int ecode = qt[i].checkCliDriverResults(qfiles[i].getName());
- if (ecode != 0) {
+ QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName());
+ if (result.getReturnCode() != 0) {
failed = true;
- System.err.println("Test " + qfiles[i].getName()
- + " results check failed with error code " + ecode);
+ StringBuilder builder = new StringBuilder();
+ builder.append("Test ")
+ .append(qfiles[i].getName())
+ .append(" results check failed with error code ")
+ .append(result.getReturnCode());
+ if (Strings.isNotEmpty(result.getCapturedOutput())) {
+ builder.append(" and diff value ").append(result.getCapturedOutput());
+ }
+ System.err.println(builder.toString());
outputTestFailureHelpMessage();
}
qt[i].clearPostTestEffects();
@@ -2091,11 +2116,18 @@ public class QTestUtil {
for (int i = 0; i < qfiles.length; i++) {
qtThread[i].join();
- int ecode = qt[i].checkCliDriverResults(qfiles[i].getName());
- if (ecode != 0) {
+ QTestProcessExecResult result = qt[i].checkCliDriverResults(qfiles[i].getName());
+ if (result.getReturnCode() != 0) {
failed = true;
- System.err.println("Test " + qfiles[i].getName()
- + " results check failed with error code " + ecode);
+ StringBuilder builder = new StringBuilder();
+ builder.append("Test ")
+ .append(qfiles[i].getName())
+ .append(" results check failed with error code ")
+ .append(result.getReturnCode());
+ if (Strings.isNotEmpty(result.getCapturedOutput())) {
+ builder.append(" and diff value ").append(result.getCapturedOutput());
+ }
+ System.err.println(builder.toString());
outputTestFailureHelpMessage();
}
}
@@ -2193,16 +2225,15 @@ public class QTestUtil {
public void failedDiff(int ecode, String fname, String debugHint) {
String message =
- "Client Execution results failed with error code = " + ecode + " while executing fname=" +
+ "Client Execution succeeded but contained differences " +
+ "(error code = " + ecode + ") after executing " +
fname + (debugHint != null ? (" " + debugHint) : "");
LOG.error(message);
Assert.fail(message);
}
- public void failed(Throwable e, String fname, String debugHint) {
+ public void failed(Exception e, String fname, String debugHint) {
String command = SessionState.get() != null ? SessionState.get().getLastCommand() : null;
- System.err.println("Exception: " + e.getMessage());
- e.printStackTrace();
System.err.println("Failed query: " + fname);
System.err.flush();
Assert.fail("Unexpected exception " +
@@ -2242,9 +2273,6 @@ public class QTestUtil {
}
br.close();
} catch (Exception e) {
- System.err.println("Exception: " + e.getMessage());
- e.printStackTrace();
- System.err.flush();
Assert.fail("Unexpected exception " + org.apache.hadoop.util.StringUtils.stringifyException(e));
}
}
@@ -2268,7 +2296,9 @@ public class QTestUtil {
String mdbPath = AbstractCliConfig.HIVE_ROOT + "/data/files/tpcds-perf/metastore_export/";
// Setup the table column stats
- BufferedReader br = new BufferedReader(new FileReader(new File(AbstractCliConfig.HIVE_ROOT + "/metastore/scripts/upgrade/derby/022-HIVE-11107.derby.sql")));
+ BufferedReader br = new BufferedReader(
+ new FileReader(
+ new File(AbstractCliConfig.HIVE_ROOT + "/metastore/scripts/upgrade/derby/022-HIVE-11107.derby.sql")));
String command;
s.execute("DROP TABLE APP.TABLE_PARAMS");
@@ -2294,15 +2324,13 @@ public class QTestUtil {
}
br.close();
- File tabColStatsCsv = new File(mdbPath+"csv/TAB_COL_STATS.txt");
- File tabParamsCsv = new File(mdbPath+"csv/TABLE_PARAMS.txt");
+ java.nio.file.Path tabColStatsCsv = FileSystems.getDefault().getPath(mdbPath, "csv" ,"TAB_COL_STATS.txt.bz2");
+ java.nio.file.Path tabParamsCsv = FileSystems.getDefault().getPath(mdbPath, "csv", "TABLE_PARAMS.txt.bz2");
// Set up the foreign key constraints properly in the TAB_COL_STATS data
String tmpBaseDir = System.getProperty(TEST_TMP_DIR_PROPERTY);
- File tmpFileLoc1 = new File(tmpBaseDir+"/TAB_COL_STATS.txt");
- File tmpFileLoc2 = new File(tmpBaseDir+"/TABLE_PARAMS.txt");
- FileUtils.copyFile(tabColStatsCsv, tmpFileLoc1);
- FileUtils.copyFile(tabParamsCsv, tmpFileLoc2);
+ java.nio.file.Path tmpFileLoc1 = FileSystems.getDefault().getPath(tmpBaseDir, "TAB_COL_STATS.txt");
+ java.nio.file.Path tmpFileLoc2 = FileSystems.getDefault().getPath(tmpBaseDir, "TABLE_PARAMS.txt");
class MyComp implements Comparator<String> {
@Override
@@ -2314,7 +2342,7 @@ public class QTestUtil {
}
}
- SortedMap<String, Integer> tableNameToID = new TreeMap<String, Integer>(new MyComp());
+ final SortedMap<String, Integer> tableNameToID = new TreeMap<String, Integer>(new MyComp());
rs = s.executeQuery("SELECT * FROM APP.TBLS");
while(rs.next()) {
@@ -2326,29 +2354,73 @@ public class QTestUtil {
LOG.debug("Resultset : " + tblName + " | " + tblId);
}
}
- for (Map.Entry<String, Integer> entry : tableNameToID.entrySet()) {
- String toReplace1 = ",_" + entry.getKey() + "_" ;
- String replacementString1 = ","+entry.getValue();
- String toReplace2 = "_" + entry.getKey() + "_@" ;
- String replacementString2 = ""+entry.getValue()+"@";
- try {
- String content1 = FileUtils.readFileToString(tmpFileLoc1, "UTF-8");
- content1 = content1.replaceAll(toReplace1, replacementString1);
- FileUtils.writeStringToFile(tmpFileLoc1, content1, "UTF-8");
- String content2 = FileUtils.readFileToString(tmpFileLoc2, "UTF-8");
- content2 = content2.replaceAll(toReplace2, replacementString2);
- FileUtils.writeStringToFile(tmpFileLoc2, content2, "UTF-8");
- } catch (IOException e) {
- LOG.info("Generating file failed", e);
+
+ final Map<String, Map<String, String>> data = new HashMap<>();
+ rs = s.executeQuery("select TBLS.TBL_NAME, a.COLUMN_NAME, a.TYPE_NAME from "
+ + "(select COLUMN_NAME, TYPE_NAME, SDS.SD_ID from APP.COLUMNS_V2 join APP.SDS on SDS.CD_ID = COLUMNS_V2.CD_ID) a"
+ + " join APP.TBLS on TBLS.SD_ID = a.SD_ID");
+ while (rs.next()) {
+ String tblName = rs.getString(1);
+ String colName = rs.getString(2);
+ String typeName = rs.getString(3);
+ Map<String, String> cols = data.get(tblName);
+ if (null == cols) {
+ cols = new HashMap<>();
}
+ cols.put(colName, typeName);
+ data.put(tblName, cols);
}
+ BufferedReader reader = new BufferedReader(new InputStreamReader(
+ new BZip2CompressorInputStream(Files.newInputStream(tabColStatsCsv, StandardOpenOption.READ))));
+
+ Stream<String> replaced = reader.lines().parallel().map(str-> {
+ String[] splits = str.split(",");
+ String tblName = splits[0];
+ String colName = splits[1];
+ Integer tblID = tableNameToID.get(tblName);
+ StringBuilder sb = new StringBuilder("default@"+tblName + "@" + colName + "@" + data.get(tblName).get(colName)+"@");
+ for (int i = 2; i < splits.length; i++) {
+ sb.append(splits[i]+"@");
+ }
+ return sb.append(tblID).toString();
+ });
+
+ Files.write(tmpFileLoc1, (Iterable<String>)replaced::iterator);
+ replaced.close();
+ reader.close();
+
+ BufferedReader reader2 = new BufferedReader(new InputStreamReader(
+ new BZip2CompressorInputStream(Files.newInputStream(tabParamsCsv, StandardOpenOption.READ))));
+ final Map<String,String> colStats = new ConcurrentHashMap<>();
+ Stream<String> replacedStream = reader2.lines().parallel().map(str-> {
+ String[] splits = str.split("_@");
+ String tblName = splits[0];
+ Integer tblId = tableNameToID.get(tblName);
+ Map<String,String> cols = data.get(tblName);
+ StringBuilder sb = new StringBuilder();
+ sb.append("{\"COLUMN_STATS\":{");
+ for (String colName : cols.keySet()) {
+ sb.append("\""+colName+"\":\"true\",");
+ }
+ sb.append("},\"BASIC_STATS\":\"true\"}");
+ colStats.put(tblId.toString(), sb.toString());
+
+ return tblId.toString() + "@" + splits[1];
+ });
+
+ Files.write(tmpFileLoc2, (Iterable<String>)replacedStream::iterator);
+ Files.write(tmpFileLoc2, (Iterable<String>)colStats.entrySet().stream()
+ .map(map->map.getKey()+"@COLUMN_STATS_ACCURATE@"+map.getValue())::iterator, StandardOpenOption.APPEND);
+
+ replacedStream.close();
+ reader2.close();
// Load the column stats and table params with 30 TB scale
- String importStatement1 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE_LOBS_FROM_EXTFILE(null, '" + "TAB_COL_STATS" +
- "', '" + tmpFileLoc1.getAbsolutePath() +
- "', ',', null, 'UTF-8', 1)";
- String importStatement2 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE_LOBS_FROM_EXTFILE(null, '" + "TABLE_PARAMS" +
- "', '" + tmpFileLoc2.getAbsolutePath() +
+ String importStatement1 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE(null, '" + "TAB_COL_STATS" +
+ "', '" + tmpFileLoc1.toAbsolutePath().toString() +
+ "', '@', null, 'UTF-8', 1)";
+ String importStatement2 = "CALL SYSCS_UTIL.SYSCS_IMPORT_TABLE(null, '" + "TABLE_PARAMS" +
+ "', '" + tmpFileLoc2.toAbsolutePath().toString() +
"', '@', null, 'UTF-8', 1)";
try {
PreparedStatement psImport1 = conn.prepareStatement(importStatement1);
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
index 8dba0bb..31f69a3 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/parse/CoreParseNegative.java
@@ -21,9 +21,12 @@ import static org.junit.Assert.fail;
import java.io.Serializable;
import java.util.List;
+
+import com.google.common.base.Strings;
import org.apache.hadoop.hive.cli.control.AbstractCliConfig;
import org.apache.hadoop.hive.cli.control.CliAdapter;
import org.apache.hadoop.hive.cli.control.CliConfigs;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
import org.apache.hadoop.hive.ql.QTestUtil;
import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType;
import org.apache.hadoop.hive.ql.exec.Task;
@@ -106,18 +109,20 @@ public class CoreParseNegative extends CliAdapter{
fail("Unexpected success for query: " + fname + debugHint);
}
catch (ParseException pe) {
- int ecode = qt.checkNegativeResults(fname, pe);
- if (ecode != 0) {
- qt.failed(ecode, fname, debugHint);
+ QTestProcessExecResult result = qt.checkNegativeResults(fname, pe);
+ if (result.getReturnCode() != 0) {
+ qt.failed(result.getReturnCode(), fname, result.getCapturedOutput() + "\r\n" + debugHint);
}
}
catch (SemanticException se) {
- int ecode = qt.checkNegativeResults(fname, se);
- if (ecode != 0) {
- qt.failedDiff(ecode, fname, debugHint);
+ QTestProcessExecResult result = qt.checkNegativeResults(fname, se);
+ if (result.getReturnCode() != 0) {
+ String message = Strings.isNullOrEmpty(result.getCapturedOutput()) ?
+ debugHint : "\r\n" + result.getCapturedOutput();
+ qt.failedDiff(result.getReturnCode(), fname, message);
}
}
- catch (Throwable e) {
+ catch (Exception e) {
qt.failed(e, fname, debugHint);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hive/beeline/Parallelized.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hive/beeline/Parallelized.java b/itests/util/src/main/java/org/apache/hive/beeline/Parallelized.java
new file mode 100644
index 0000000..a819e26
--- /dev/null
+++ b/itests/util/src/main/java/org/apache/hive/beeline/Parallelized.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive.beeline;
+
+import org.junit.runners.Parameterized;
+import org.junit.runners.model.RunnerScheduler;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Class to run Parameterized test in parallel.
+ * Source: http://hwellmann.blogspot.hu/2009/12/running-parameterized-junit-tests-in.html
+ */
+public class Parallelized extends Parameterized {
+ private static class ThreadPoolScheduler implements RunnerScheduler {
+ private ExecutorService executor;
+
+ public ThreadPoolScheduler() {
+ String threads = System.getProperty("junit.parallel.threads");
+ int numThreads = Runtime.getRuntime().availableProcessors();
+ if (threads != null) {
+ numThreads = Integer.parseInt(threads);
+ }
+ executor = Executors.newFixedThreadPool(numThreads);
+ }
+
+ @Override
+ public void finished() {
+ executor.shutdown();
+ try {
+ executor.awaitTermination(10, TimeUnit.MINUTES);
+ } catch (InterruptedException exc) {
+ throw new RuntimeException(exc);
+ }
+ }
+
+ @Override
+ public void schedule(Runnable childStatement) {
+ executor.submit(childStatement);
+ }
+ }
+
+ public Parallelized(Class klass) throws Throwable {
+ super(klass);
+ setScheduler(new ThreadPoolScheduler());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/187eb760/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hive/beeline/QFile.java b/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
new file mode 100644
index 0000000..0bde529
--- /dev/null
+++ b/itests/util/src/main/java/org/apache/hive/beeline/QFile.java
@@ -0,0 +1,333 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.beeline;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hive.ql.QTestProcessExecResult;
+import org.apache.hadoop.hive.ql.QTestUtil;
+import org.apache.hadoop.util.Shell;
+import org.apache.hive.common.util.StreamPrinter;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+/**
+ * Class for representing a Query and the connected files. It provides accessors for the specific
+ * input and output files, and provides methods for filtering the output of the runs.
+ */
+public final class QFile {
+ private static final Set<String> srcTables = QTestUtil.getSrcTables();
+ private static final String DEBUG_HINT =
+ "The following files can help you identifying the problem:%n"
+ + " - Query file: %1s%n"
+ + " - Raw output file: %2s%n"
+ + " - Filtered output file: %3s%n"
+ + " - Expected output file: %4s%n"
+ + " - Client log file: %5s%n"
+ + " - Client log files before the test: %6s%n"
+ + " - Client log files after the test: %7s%n"
+ + " - Hiveserver2 log file: %8s%n";
+ private static final String USE_COMMAND_WARNING =
+ "The query file %1s contains \"%2s\" command.%n"
+ + "The source table name rewrite is turned on, so this might cause problems when the used "
+ + "database contains tables named any of the following: " + srcTables + "%n"
+ + "To turn off the table name rewrite use -Dtest.rewrite.source.tables=false%n";
+
+ private static final Pattern USE_PATTERN =
+ Pattern.compile("^\\s*use\\s.*", Pattern.CASE_INSENSITIVE);
+
+ private static final String MASK_PATTERN = "#### A masked pattern was here ####\n";
+
+ private String name;
+ private File inputFile;
+ private File rawOutputFile;
+ private File outputFile;
+ private File expectedOutputFile;
+ private File logFile;
+ private File beforeExecuteLogFile;
+ private File afterExecuteLogFile;
+ private static RegexFilterSet staticFilterSet = getStaticFilterSet();
+ private RegexFilterSet specificFilterSet;
+ private boolean rewriteSourceTables;
+
+ private QFile() {}
+
+ public String getName() {
+ return name;
+ }
+
+ public File getInputFile() {
+ return inputFile;
+ }
+
+ public File getRawOutputFile() {
+ return rawOutputFile;
+ }
+
+ public File getOutputFile() {
+ return outputFile;
+ }
+
+ public File getExpectedOutputFile() {
+ return expectedOutputFile;
+ }
+
+ public File getLogFile() {
+ return logFile;
+ }
+
+ public File getBeforeExecuteLogFile() {
+ return beforeExecuteLogFile;
+ }
+
+ public File getAfterExecuteLogFile() {
+ return afterExecuteLogFile;
+ }
+
+ public String getDebugHint() {
+ return String.format(DEBUG_HINT, inputFile, rawOutputFile, outputFile, expectedOutputFile,
+ logFile, beforeExecuteLogFile, afterExecuteLogFile,
+ "./itests/qtest/target/tmp/log/hive.log");
+ }
+
+ /**
+ * Filters the sql commands if necessary.
+ * @param commands The array of the sql commands before filtering
+ * @return The filtered array of the sql command strings
+ * @throws IOException File read error
+ */
+ public String[] filterCommands(String[] commands) throws IOException {
+ if (rewriteSourceTables) {
+ for (int i=0; i<commands.length; i++) {
+ if (USE_PATTERN.matcher(commands[i]).matches()) {
+ System.err.println(String.format(USE_COMMAND_WARNING, inputFile, commands[i]));
+ }
+ commands[i] = replaceTableNames(commands[i]);
+ }
+ }
+ return commands;
+ }
+
+ /**
+ * Replace the default src database TABLE_NAMEs in the queries with default.TABLE_NAME, like
+ * src->default.src, srcpart->default.srcpart, so the queries could be run even if the used
+ * database is query specific. This change is only a best effort, since we do not want to parse
+ * the queries, we could not be sure that we do not replace other strings which are not
+ * tablenames. Like 'select src from othertable;'. The q files containing these commands should
+ * be excluded. Only replace the tablenames, if rewriteSourceTables is set.
+ * @param source The original query string
+ * @return The query string where the tablenames are replaced
+ */
+ private String replaceTableNames(String source) {
+ for (String table : srcTables) {
+ source = source.replaceAll("(?is)(\\s+)" + table + "([\\s;\\n\\)])", "$1default." + table
+ + "$2");
+ }
+ return source;
+ }
+
+ /**
+ * The result contains the original queries. To revert them to the original form remove the
+ * 'default' from every default.TABLE_NAME, like default.src->src, default.srcpart->srcpart.
+ * @param source The original query output
+ * @return The query output where the tablenames are replaced
+ */
+ private String revertReplaceTableNames(String source) {
+ for (String table : srcTables) {
+ source = source.replaceAll("(?is)(\\s+)default\\." + table + "([\\s;\\n\\)])", "$1" + table
+ + "$2");
+ }
+ return source;
+ }
+
+ public void filterOutput() throws IOException {
+ String rawOutput = FileUtils.readFileToString(rawOutputFile, "UTF-8");
+ if (rewriteSourceTables) {
+ rawOutput = revertReplaceTableNames(rawOutput);
+ }
+ String filteredOutput = staticFilterSet.filter(specificFilterSet.filter(rawOutput));
+ FileUtils.writeStringToFile(outputFile, filteredOutput);
+ }
+
+ public QTestProcessExecResult compareResults() throws IOException, InterruptedException {
+ if (!expectedOutputFile.exists()) {
+ throw new IOException("Expected results file does not exist: " + expectedOutputFile);
+ }
+ return executeDiff();
+ }
+
+ public void overwriteResults() throws IOException {
+ FileUtils.copyFile(outputFile, expectedOutputFile);
+ }
+
+ private QTestProcessExecResult executeDiff() throws IOException, InterruptedException {
+ List<String> diffCommandArgs = new ArrayList<String>();
+ diffCommandArgs.add("diff");
+
+ // Text file comparison
+ diffCommandArgs.add("-a");
+
+ if (Shell.WINDOWS) {
+ // Ignore changes in the amount of white space
+ diffCommandArgs.add("-b");
+
+ // Files created on Windows machines have different line endings
+ // than files created on Unix/Linux. Windows uses carriage return and line feed
+ // ("\r\n") as a line ending, whereas Unix uses just line feed ("\n").
+ // Also StringBuilder.toString(), Stream to String conversions adds extra
+ // spaces at the end of the line.
+ diffCommandArgs.add("--strip-trailing-cr"); // Strip trailing carriage return on input
+ diffCommandArgs.add("-B"); // Ignore changes whose lines are all blank
+ }
+
+ // Add files to compare to the arguments list
+ diffCommandArgs.add(getQuotedString(expectedOutputFile));
+ diffCommandArgs.add(getQuotedString(outputFile));
+
+ System.out.println("Running: " + org.apache.commons.lang.StringUtils.join(diffCommandArgs,
+ ' '));
+ Process executor = Runtime.getRuntime().exec(diffCommandArgs.toArray(
+ new String[diffCommandArgs.size()]));
+
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintStream out = new PrintStream(bos, true, "UTF-8");
+
+ StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, System.err);
+ StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, System.out, out);
+
+ outPrinter.start();
+ errPrinter.start();
+
+ int result = executor.waitFor();
+
+ outPrinter.join();
+ errPrinter.join();
+
+ executor.waitFor();
+
+ return QTestProcessExecResult.create(result, new String(bos.toByteArray(),
+ StandardCharsets.UTF_8));
+ }
+
+ private static String getQuotedString(File file) {
+ return Shell.WINDOWS ? String.format("\"%s\"", file.getAbsolutePath()) : file.getAbsolutePath();
+ }
+
+ private static class Filter {
+ private final Pattern pattern;
+ private final String replacement;
+
+ public Filter(Pattern pattern, String replacement) {
+ this.pattern = pattern;
+ this.replacement = replacement;
+ }
+ }
+
+ private static class RegexFilterSet {
+ private final List<Filter> regexFilters = new ArrayList<Filter>();
+
+ public RegexFilterSet addFilter(String regex, String replacement) {
+ regexFilters.add(new Filter(Pattern.compile(regex), replacement));
+ return this;
+ }
+
+ public String filter(String input) {
+ for (Filter filter : regexFilters) {
+ input = filter.pattern.matcher(input).replaceAll(filter.replacement);
+ }
+ return input;
+ }
+ }
+
+ // These are the filters which are common for every QTest.
+ // Check specificFilterSet for QTest specific ones.
+ private static RegexFilterSet getStaticFilterSet() {
+ // Pattern to remove the timestamp and other infrastructural info from the out file
+ return new RegexFilterSet()
+ .addFilter("Reading log file: .*\n", "")
+ .addFilter("INFO : ", "")
+ .addFilter(".*/tmp/.*\n", MASK_PATTERN)
+ .addFilter(".*file:.*\n", MASK_PATTERN)
+ .addFilter(".*file\\..*\n", MASK_PATTERN)
+ .addFilter(".*CreateTime.*\n", MASK_PATTERN)
+ .addFilter(".*transient_lastDdlTime.*\n", MASK_PATTERN)
+ .addFilter("(?s)(" + MASK_PATTERN + ")+", MASK_PATTERN);
+ }
+
+ /**
+ * Builder to generate QFile objects. After initializing the builder it is possible the
+ * generate the next QFile object using it's name only.
+ */
+ public static class QFileBuilder {
+ private File queryDirectory;
+ private File logDirectory;
+ private File resultsDirectory;
+ private boolean rewriteSourceTables;
+
+ public QFileBuilder() {
+ }
+
+ public QFileBuilder setQueryDirectory(File queryDirectory) {
+ this.queryDirectory = queryDirectory;
+ return this;
+ }
+
+ public QFileBuilder setLogDirectory(File logDirectory) {
+ this.logDirectory = logDirectory;
+ return this;
+ }
+
+ public QFileBuilder setResultsDirectory(File resultsDirectory) {
+ this.resultsDirectory = resultsDirectory;
+ return this;
+ }
+
+ public QFileBuilder setRewriteSourceTables(boolean rewriteSourceTables) {
+ this.rewriteSourceTables = rewriteSourceTables;
+ return this;
+ }
+
+ public QFile getQFile(String name) throws IOException {
+ QFile result = new QFile();
+ result.name = name;
+ result.inputFile = new File(queryDirectory, name + ".q");
+ result.rawOutputFile = new File(logDirectory, name + ".q.out.raw");
+ result.outputFile = new File(logDirectory, name + ".q.out");
+ result.expectedOutputFile = new File(resultsDirectory, name + ".q.out");
+ result.logFile = new File(logDirectory, name + ".q.beeline");
+ result.beforeExecuteLogFile = new File(logDirectory, name + ".q.beforeExecute.log");
+ result.afterExecuteLogFile = new File(logDirectory, name + ".q.afterExecute.log");
+ result.rewriteSourceTables = rewriteSourceTables;
+ result.specificFilterSet = new RegexFilterSet()
+ .addFilter("(PREHOOK|POSTHOOK): (Output|Input): database:" + name + "\n",
+ "$1: $2: database:default\n")
+ .addFilter("(PREHOOK|POSTHOOK): (Output|Input): " + name + "@", "$1: $2: default@")
+ .addFilter("name(:?) " + name + "\\.(.*)\n", "name$1 default.$2\n")
+ .addFilter("/" + name + ".db/", "/");
+ return result;
+ }
+ }
+}