You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by to...@apache.org on 2019/05/13 22:54:39 UTC
[impala] 03/04: IMPALA-8369 [Test fixes] More test fixes when
running against Hive-3
This is an automated email from the ASF dual-hosted git repository.
todd pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit eedff62f7c5d4c07354d0d7aaa4a40c7e4a8fbbd
Author: Vihang Karajgaonkar <vi...@cloudera.com>
AuthorDate: Tue May 7 13:34:49 2019 -0700
IMPALA-8369 [Test fixes] More test fixes when running against Hive-3
This test fixes CatalogTest, FrontendTest, CatalogObjectTofromThriftTest
by breaking some tests into the ones which are not expected to work on
Hive-3 and then skipping it. It does this by adding a util method in
TestUtils which returns if the environment variable
IMPALA_HIVE_MAJOR_VERSION is >= 3. If this condition is true, it skips
certain tests which use hive_idx_tbl (not supported in data-load against
Hive-3). If it is less than 3 the tests are not skipped so we keep the
test coverage on Hive-2 setups.
Also, fixes the TestCaseLoaderTest which instantiates a embedded HMS
instance. This requires some configuration changes for the embedded
standalone mode as well as adding datanucleus JDO as a test dependency.
Additionally, this patch also fixes test_show_create_table which was
failing on Hive-3 setups due to the additional parameter
bucketing_version available from Hive-3.
Testing Done:
1. Ran the tests when mini-cluster is deployed with USE_CDP_HIVE=true
and made sure that the tests work (or are skipped as expected)
2. Ran the same tests with USE_CDP_HIVE=false to make sure they still
work against HMS-2
Change-Id: If05f74efc481e2b0d26a9c4f6e58cef38605d72c
Reviewed-on: http://gerrit.cloudera.org:8080/13276
Reviewed-by: Zoltan Borok-Nagy <bo...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Todd Lipcon <to...@apache.org>
---
fe/pom.xml | 8 ++++++++
.../impala/catalog/CatalogObjectToFromThriftTest.java | 14 ++++++++++++--
.../test/java/org/apache/impala/catalog/CatalogTest.java | 16 +++++++++++++---
.../java/org/apache/impala/service/FrontendTest.java | 16 +++++++++++++---
.../impala/testutil/EmbeddedMetastoreClientPool.java | 11 +++++++++++
.../test/java/org/apache/impala/testutil/TestUtils.java | 10 ++++++++++
tests/metadata/test_show_create_table.py | 3 ++-
7 files changed, 69 insertions(+), 9 deletions(-)
diff --git a/fe/pom.xml b/fe/pom.xml
index ecd79d3..8827295 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -1019,6 +1019,14 @@ under the License.
</exclusion>
</exclusions>
</dependency>
+ <!-- Needed by tests like TestCaseLoader instantiate HMS in embedded mode which
+ needs datanucleus as test dependency-->
+ <dependency>
+ <groupId>org.datanucleus</groupId>
+ <artifactId>javax.jdo</artifactId>
+ <version>3.2.0-m3</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</profile>
diff --git a/fe/src/test/java/org/apache/impala/catalog/CatalogObjectToFromThriftTest.java b/fe/src/test/java/org/apache/impala/catalog/CatalogObjectToFromThriftTest.java
index 9762a3e..7c5c576 100644
--- a/fe/src/test/java/org/apache/impala/catalog/CatalogObjectToFromThriftTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/CatalogObjectToFromThriftTest.java
@@ -27,6 +27,7 @@ import org.apache.impala.analysis.LiteralExpr;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.SqlCastException;
import org.apache.impala.testutil.CatalogServiceTestCatalog;
+import org.apache.impala.testutil.TestUtils;
import org.apache.impala.thrift.CatalogObjectsConstants;
import org.apache.impala.thrift.TAccessLevel;
import org.apache.impala.thrift.THBaseTable;
@@ -36,6 +37,7 @@ import org.apache.impala.thrift.TTable;
import org.apache.impala.thrift.TTableType;
import org.junit.AfterClass;
import org.junit.Assert;
+import org.junit.Assume;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -209,13 +211,21 @@ public class CatalogObjectToFromThriftTest {
}
@Test
- public void TestTableLoadingErrors() throws ImpalaException {
+ public void TestTableLoadingErrorsForHive2() throws ImpalaException {
+ // run the test only when it is running against Hive-2 since index tables are
+ // skipped during data-load against Hive-3
+ Assume.assumeTrue(
+ "Skipping this test since it is only supported when running against Hive-2",
+ TestUtils.getHiveMajorVersion() == 2);
Table table = catalog_.getOrLoadTable("functional", "hive_index_tbl");
TTable thriftTable = getThriftTable(table);
Assert.assertEquals(thriftTable.tbl_name, "hive_index_tbl");
Assert.assertEquals(thriftTable.db_name, "functional");
+ }
- table = catalog_.getOrLoadTable("functional", "alltypes");
+ @Test
+ public void TestTableLoadingErrors() throws ImpalaException {
+ Table table = catalog_.getOrLoadTable("functional", "alltypes");
HdfsTable hdfsTable = (HdfsTable) table;
// Get any partition with valid HMS parameters to create a
// dummy partition.
diff --git a/fe/src/test/java/org/apache/impala/catalog/CatalogTest.java b/fe/src/test/java/org/apache/impala/catalog/CatalogTest.java
index 36c9919..ad47845 100644
--- a/fe/src/test/java/org/apache/impala/catalog/CatalogTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/CatalogTest.java
@@ -56,6 +56,7 @@ import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.Reference;
import org.apache.impala.service.BackendConfig;
import org.apache.impala.testutil.CatalogServiceTestCatalog;
+import org.apache.impala.testutil.TestUtils;
import org.apache.impala.thrift.TBackendGflags;
import org.apache.impala.thrift.TFunctionBinaryType;
import org.apache.impala.thrift.TGetPartitionStatsRequest;
@@ -67,6 +68,7 @@ import org.apache.impala.thrift.TPrivilegeLevel;
import org.apache.impala.thrift.TPrivilegeScope;
import org.apache.impala.thrift.TTableName;
import org.junit.After;
+import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
@@ -699,18 +701,26 @@ public class CatalogTest {
}
@Test
- public void testLoadingUnsupportedTableTypes() throws CatalogException {
+ public void testLoadingUnsupportedTblTypesOnHive2() throws CatalogException {
+ // run the test only when it is running against Hive-2 since index tables are
+ // skipped during data-load against Hive-3
+ Assume.assumeTrue(
+ "Skipping this test since it is only supported when running against Hive-2",
+ TestUtils.getHiveMajorVersion() == 2);
Table table = catalog_.getOrLoadTable("functional", "hive_index_tbl");
assertTrue(table instanceof IncompleteTable);
IncompleteTable incompleteTable = (IncompleteTable) table;
assertTrue(incompleteTable.getCause() instanceof TableLoadingException);
assertEquals("Unsupported table type 'INDEX_TABLE' for: functional.hive_index_tbl",
incompleteTable.getCause().getMessage());
+ }
+ @Test
+ public void testLoadingUnsupportedTableTypes() throws CatalogException {
// Table with unsupported SerDe library.
- table = catalog_.getOrLoadTable("functional", "bad_serde");
+ Table table = catalog_.getOrLoadTable("functional", "bad_serde");
assertTrue(table instanceof IncompleteTable);
- incompleteTable = (IncompleteTable) table;
+ IncompleteTable incompleteTable = (IncompleteTable) table;
assertTrue(incompleteTable.getCause() instanceof TableLoadingException);
assertEquals("Impala does not support tables of this type. REASON: SerDe" +
" library 'org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe' " +
diff --git a/fe/src/test/java/org/apache/impala/service/FrontendTest.java b/fe/src/test/java/org/apache/impala/service/FrontendTest.java
index b9ef150..78fdbd5 100644
--- a/fe/src/test/java/org/apache/impala/service/FrontendTest.java
+++ b/fe/src/test/java/org/apache/impala/service/FrontendTest.java
@@ -33,17 +33,17 @@ import org.apache.hive.service.rpc.thrift.TGetInfoReq;
import org.apache.hive.service.rpc.thrift.TGetSchemasReq;
import org.apache.hive.service.rpc.thrift.TGetTablesReq;
import org.apache.impala.catalog.Db;
-import org.apache.impala.catalog.PrimitiveType;
import org.apache.impala.catalog.ScalarType;
import org.apache.impala.catalog.Table;
import org.apache.impala.catalog.Type;
import org.apache.impala.common.FrontendTestBase;
import org.apache.impala.common.ImpalaException;
+import org.apache.impala.testutil.TestUtils;
import org.apache.impala.thrift.TMetadataOpRequest;
import org.apache.impala.thrift.TMetadataOpcode;
import org.apache.impala.thrift.TResultRow;
import org.apache.impala.thrift.TResultSet;
-import org.apache.impala.thrift.TTypeNodeType;
+import org.junit.Assume;
import org.junit.Test;
import com.google.common.collect.Lists;
@@ -226,15 +226,25 @@ public class FrontendTest extends FrontendTestBase {
assertEquals(null, row.colVals.get(11).string_val);
}
}
+ }
+
+ @Test
+ public void TestGetTablesWithCommentsOnHive2() throws ImpalaException {
+ // run the test only when it is running against Hive-2 since index tables are
+ // skipped during data-load against Hive-3
+ Assume.assumeTrue(
+ "Skipping this test since it is only supported when running against Hive-2",
+ TestUtils.getHiveMajorVersion() == 2);
// IMPALA-5579: GetTables() should succeed and display the available information for
// tables that cannot be loaded.
+ TMetadataOpRequest req = new TMetadataOpRequest();
req = new TMetadataOpRequest();
req.opcode = TMetadataOpcode.GET_TABLES;
req.get_tables_req = new TGetTablesReq();
req.get_tables_req.setSchemaName("functional");
req.get_tables_req.setTableName("hive_index_tbl");
- resp = execMetadataOp(req);
+ TResultSet resp = execMetadataOp(req);
assertEquals(1, resp.rows.size());
}
diff --git a/fe/src/test/java/org/apache/impala/testutil/EmbeddedMetastoreClientPool.java b/fe/src/test/java/org/apache/impala/testutil/EmbeddedMetastoreClientPool.java
index a606d92..172874e 100644
--- a/fe/src/test/java/org/apache/impala/testutil/EmbeddedMetastoreClientPool.java
+++ b/fe/src/test/java/org/apache/impala/testutil/EmbeddedMetastoreClientPool.java
@@ -55,6 +55,16 @@ public class EmbeddedMetastoreClientPool extends MetaStoreClientPool {
// using embedded HMS
private static final String DEFAULT_PARTITION_EXPRESSION_PROXY_CLASS = "org.apache"
+ ".hadoop.hive.metastore.DefaultPartitionExpressionProxy";
+ // In HMS-3 the default value of "metastore.task.thread.always is set to some classes
+ // which are present in hive-exec. We don't need this for our tests as of 05/07/2019
+ // Setting this to default EventCleanerTask avoid pulling in unnecessary dependencies
+ // from hive-exec for running these tests. Note that this config key is not available
+ // in HMS-2. But adding this is still okay since it only print a warning of unknown
+ // config with no other side-effects
+ private static final String METASTORE_TASK_THREAD_ALWAYS_KEY = "metastore.task"
+ + ".threads.always";
+ private static final String DEFAULT_EVENT_CLEANER_TASK_CLASS = "org.apache.hadoop"
+ + ".hive.metastore.events.EventCleanerTask";
/**
* Generates the HiveConf required to connect to an embedded metastore backed by
@@ -76,6 +86,7 @@ public class EmbeddedMetastoreClientPool extends MetaStoreClientPool {
String.format(CONNECTION_URL_TEMPLATE, dbStorePath.toString()));
conf.set(ConfVars.METASTORE_EXPRESSION_PROXY_CLASS.varname,
DEFAULT_PARTITION_EXPRESSION_PROXY_CLASS);
+ conf.set(METASTORE_TASK_THREAD_ALWAYS_KEY, DEFAULT_EVENT_CLEANER_TASK_CLASS);
// Disabling notification event listeners
conf.set(ConfVars.METASTORE_TRANSACTIONAL_EVENT_LISTENERS.varname, "");
return conf;
diff --git a/fe/src/test/java/org/apache/impala/testutil/TestUtils.java b/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
index 25ab2d7..9a1c852 100644
--- a/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
+++ b/fe/src/test/java/org/apache/impala/testutil/TestUtils.java
@@ -17,6 +17,7 @@
package org.apache.impala.testutil;
+import com.google.common.base.Preconditions;
import java.io.StringReader;
import java.io.StringWriter;
import java.text.SimpleDateFormat;
@@ -383,4 +384,13 @@ public class TestUtils {
public static void assumeKuduIsSupported() {
Assume.assumeTrue(RuntimeEnv.INSTANCE.isKuduSupported());
}
+
+ /**
+ * Returns the hive major version from environment
+ */
+ public static int getHiveMajorVersion() {
+ String hiveMajorVersion = Preconditions.checkNotNull(System.getenv(
+ "IMPALA_HIVE_MAJOR_VERSION"));
+ return Integer.parseInt(hiveMajorVersion);
+ }
}
diff --git a/tests/metadata/test_show_create_table.py b/tests/metadata/test_show_create_table.py
index 092a834..3ce2748 100644
--- a/tests/metadata/test_show_create_table.py
+++ b/tests/metadata/test_show_create_table.py
@@ -35,7 +35,8 @@ class TestShowCreateTable(ImpalaTestSuite):
FILTER_TBL_PROPERTIES = ["transient_lastDdlTime", "numFiles", "numPartitions",
"numRows", "rawDataSize", "totalSize", "COLUMN_STATS_ACCURATE",
"STATS_GENERATED_VIA_STATS_TASK", "last_modified_by",
- "last_modified_time", "numFilesErasureCoded"]
+ "last_modified_time", "numFilesErasureCoded",
+ "bucketing_version"]
@classmethod
def get_workload(self):