You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by lv...@apache.org on 2019/06/10 16:18:39 UTC
[impala] 05/05: IMPALA-8629: (part 1) Add temp KuduStorageHandler
This is an automated email from the ASF dual-hosted git repository.
lv pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit d753600f9cf4bda0c955f0f5eecc6fb41b0c365e
Author: Grant Henke <gh...@cloudera.com>
AuthorDate: Fri Jun 7 14:26:51 2019 -0500
IMPALA-8629: (part 1) Add temp KuduStorageHandler
This patch adds a temporary KuduStorageHandler
so that the Kudu project can change its handler without
breaking the integration. It also disabled any tests that
depend on the specific handler.
A follow up patch will remove the
TEMP_KUDU_STORAGE_HANDLER. adjust the
KUDU_STORAGE_HANDLER value to be the final value,
and re-enable the tests.
Change-Id: Ic9982466699818390fa28efc5ea1aae75b11c12a
Reviewed-on: http://gerrit.cloudera.org:8080/13561
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Thomas Marshall <tm...@cloudera.com>
---
fe/src/main/java/org/apache/impala/catalog/KuduTable.java | 9 ++++++++-
.../java/org/apache/impala/analysis/AnalyzeKuduDDLTest.java | 13 +++++++------
tests/custom_cluster/test_kudu.py | 10 ++++++----
tests/query_test/test_kudu.py | 5 +++--
4 files changed, 24 insertions(+), 13 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/catalog/KuduTable.java b/fe/src/main/java/org/apache/impala/catalog/KuduTable.java
index 0f0540f..1626236 100644
--- a/fe/src/main/java/org/apache/impala/catalog/KuduTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/KuduTable.java
@@ -87,6 +87,12 @@ public class KuduTable extends Table implements FeKuduTable {
public static final String KUDU_STORAGE_HANDLER =
"org.apache.kudu.hive.KuduStorageHandler";
+ // TODO(IMPALA-8629): Remove this after Kudu adjusts its StorageHandler logic.
+ // Once that is done KUDU_STORAGE_HANDLER will be
+ // "org.apache.hadoop.hive.kudu.KuduStorageHandler".
+ public static final String TEMP_KUDU_STORAGE_HANDLER =
+ "org.apache.hadoop.hive.kudu.KuduStorageHandler";
+
// Key to specify the number of tablet replicas.
public static final String KEY_TABLET_REPLICAS = "kudu.num_tablet_replicas";
@@ -140,7 +146,8 @@ public class KuduTable extends Table implements FeKuduTable {
public static boolean isKuduStorageHandler(String handler) {
return handler != null && (handler.equals(KUDU_LEGACY_STORAGE_HANDLER) ||
- handler.equals(KUDU_STORAGE_HANDLER));
+ handler.equals(KUDU_STORAGE_HANDLER) ||
+ handler.equals(TEMP_KUDU_STORAGE_HANDLER));
}
public static boolean isKuduTable(org.apache.hadoop.hive.metastore.api.Table msTbl) {
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeKuduDDLTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeKuduDDLTest.java
index 7a6b8e7..20e8f4e 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeKuduDDLTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeKuduDDLTest.java
@@ -198,13 +198,14 @@ public class AnalyzeKuduDDLTest extends FrontendTestBase {
"partition 30 < values) stored as kudu tblproperties" +
"('kudu.master_addresses' = '%s')", kuduMasters));
// Not using the STORED AS KUDU syntax to specify a Kudu table
- AnalysisError("create table tab (x int) tblproperties (" +
- "'storage_handler'='org.apache.kudu.hive.KuduStorageHandler')",
- CreateTableStmt.KUDU_STORAGE_HANDLER_ERROR_MESSAGE);
+ // TODO(IMPALA-8629): Uncomment tests after Kudu adjusts its StorageHandler logic.
+ // AnalysisError("create table tab (x int) tblproperties (" +
+ // "'storage_handler'='org.apache.kudu.hive.KuduStorageHandler')",
+ // CreateTableStmt.KUDU_STORAGE_HANDLER_ERROR_MESSAGE);
// Creating unpartitioned table results in a warning.
- AnalyzesOk("create table tab (x int primary key) stored as kudu " +
- "tblproperties ('storage_handler'='org.apache.kudu.hive.KuduStorageHandler')",
- "Unpartitioned Kudu tables are inefficient for large data sizes.");
+ // AnalyzesOk("create table tab (x int primary key) stored as kudu " +
+ // "tblproperties ('storage_handler'='org.apache.kudu.hive.KuduStorageHandler')",
+ // "Unpartitioned Kudu tables are inefficient for large data sizes.");
// Invalid value for number of replicas
AnalysisError("create table t (x int primary key) stored as kudu tblproperties (" +
"'kudu.num_tablet_replicas'='1.1')",
diff --git a/tests/custom_cluster/test_kudu.py b/tests/custom_cluster/test_kudu.py
index e3d6cc0..eb87a1e 100644
--- a/tests/custom_cluster/test_kudu.py
+++ b/tests/custom_cluster/test_kudu.py
@@ -168,8 +168,9 @@ class TestKuduHMSIntegration(CustomClusterTestSuite, KuduTestSuite):
assert ["", "EXTERNAL", "TRUE"] in table_desc
assert ["", "kudu.master_addresses", KUDU_MASTER_HOSTS] in table_desc
assert ["", "kudu.table_name", kudu_table.name] in table_desc
- assert ["", "storage_handler", "org.apache.kudu.hive.KuduStorageHandler"] \
- in table_desc
+ # TODO(IMPALA-8629): Uncomment tests after Kudu adjusts its StorageHandler logic.
+ # assert ["", "storage_handler", "org.apache.kudu.hive.KuduStorageHandler"] \
+ # in table_desc
@pytest.mark.execute_serially
def test_implicit_managed_table_props(self, cursor, kudu_client, unique_database):
@@ -190,8 +191,9 @@ class TestKuduHMSIntegration(CustomClusterTestSuite, KuduTestSuite):
assert any("kudu.master_addresses" in s for s in table_desc)
assert ["Table Type:", "MANAGED_TABLE", None] in table_desc
assert ["", "kudu.table_name", "%s.foo" % unique_database] in table_desc
- assert ["", "storage_handler", "org.apache.kudu.hive.KuduStorageHandler"] \
- in table_desc
+ # TODO(IMPALA-8629): Uncomment tests after Kudu adjusts its StorageHandler logic.
+ # assert ["", "storage_handler", "org.apache.kudu.hive.KuduStorageHandler"] \
+ # in table_desc
@pytest.mark.execute_serially
def test_drop_non_empty_db(self, unique_cursor, kudu_client):
diff --git a/tests/query_test/test_kudu.py b/tests/query_test/test_kudu.py
index 1dc5eba..875b5d0 100644
--- a/tests/query_test/test_kudu.py
+++ b/tests/query_test/test_kudu.py
@@ -560,8 +560,9 @@ class TestCreateExternalTable(KuduTestSuite):
assert ["", "EXTERNAL", "TRUE"] in table_desc
assert ["", "kudu.master_addresses", KUDU_MASTER_HOSTS] in table_desc
assert ["", "kudu.table_name", kudu_table.name] in table_desc
- assert ["", "storage_handler", "org.apache.kudu.hive.KuduStorageHandler"] \
- in table_desc
+ # TODO(IMPALA-8629): Uncomment tests after Kudu adjusts its StorageHandler logic.
+ # assert ["", "storage_handler", "org.apache.kudu.hive.KuduStorageHandler"] \
+ # in table_desc
@SkipIfKudu.hms_integration_enabled
def test_col_types(self, cursor, kudu_client):