You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2019/11/17 07:09:43 UTC
[impala] branch master updated (2c2df0b -> e642a5c)
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git.
from 2c2df0b IMPALA-9125: generalize finding DataSink from other fragment
new a567698 IMPALA-9160: Remove references to RangerAuthorizationConfig
new e642a5c IMPALA-9092 : Disable show create table tests on Kudu
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../impala/authorization/ranger/RangerAuthorizationConfig.java | 5 -----
fe/src/test/java/org/apache/impala/catalog/CatalogTest.java | 10 ++++++++--
.../java/org/apache/impala/catalog/local/LocalCatalogTest.java | 2 ++
tests/common/skip.py | 3 +++
tests/metadata/test_ddl.py | 4 ++++
tests/metadata/test_show_create_table.py | 5 ++++-
tests/query_test/test_kudu.py | 5 ++++-
7 files changed, 25 insertions(+), 9 deletions(-)
[impala] 02/02: IMPALA-9092 : Disable show create table tests on
Kudu
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit e642a5cd168dbbe6d7e9ca86ae15caeafc003e65
Author: Vihang Karajgaonkar <vi...@apache.org>
AuthorDate: Thu Nov 7 17:15:42 2019 -0800
IMPALA-9092 : Disable show create table tests on Kudu
This patch temporarily disables the Kudu tests which fail when we try to bump up the CDP
build number due the HMS translation. The tests will be re-enabled back again when the fix
for IMPALA-9092 is submitted.
Testing Done:
1. Bumped up the CDP build number to 1507246 which has the HMS translation in it.
2. Ran core tests and found the tests which are failing for Kudu due to this issue.
3. Ran the failing tests again and confirm they are not failing anymore.
Change-Id: I37c0b6d82372bc6380285afcd94f0c1e123f2eda
Reviewed-on: http://gerrit.cloudera.org:8080/14664
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
fe/src/test/java/org/apache/impala/catalog/CatalogTest.java | 10 ++++++++--
.../java/org/apache/impala/catalog/local/LocalCatalogTest.java | 2 ++
tests/common/skip.py | 3 +++
tests/metadata/test_ddl.py | 4 ++++
tests/metadata/test_show_create_table.py | 5 ++++-
tests/query_test/test_kudu.py | 5 ++++-
6 files changed, 25 insertions(+), 4 deletions(-)
diff --git a/fe/src/test/java/org/apache/impala/catalog/CatalogTest.java b/fe/src/test/java/org/apache/impala/catalog/CatalogTest.java
index 563348e..a737f0b 100644
--- a/fe/src/test/java/org/apache/impala/catalog/CatalogTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/CatalogTest.java
@@ -689,8 +689,14 @@ public class CatalogTest {
// alltypesinsert is created using CREATE TABLE LIKE and is a MANAGED table
table = catalog_.getOrLoadTable("functional", "alltypesinsert", "test");
assertEquals(System.getProperty("user.name"), table.getMetaStoreTable().getOwner());
- assertEquals(TableType.MANAGED_TABLE.toString(),
- table.getMetaStoreTable().getTableType());
+ if (TestUtils.getHiveMajorVersion() == 2) {
+ assertEquals(TableType.MANAGED_TABLE.toString(),
+ table.getMetaStoreTable().getTableType());
+ } else {
+ // in Hive-3 due to HMS translation, this table becomes an external table
+ assertEquals(TableType.EXTERNAL_TABLE.toString(),
+ table.getMetaStoreTable().getTableType());
+ }
}
@Test
diff --git a/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java b/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java
index 059a4b0..34ffe74 100644
--- a/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java
@@ -52,6 +52,7 @@ import org.apache.impala.util.PatternMatcher;
import org.hamcrest.CoreMatchers;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import com.google.common.base.Joiner;
@@ -247,6 +248,7 @@ public class LocalCatalogTest {
assertEquals("SELECT * FROM functional.alltypes", v.getQueryStmt().toSql());
}
+ @Ignore("Ignored until IMPALA-9092 is fixed")
@Test
public void testKuduTable() throws Exception {
LocalKuduTable t = (LocalKuduTable) catalog_.getTable("functional_kudu", "alltypes");
diff --git a/tests/common/skip.py b/tests/common/skip.py
index aae8750..08a8358 100644
--- a/tests/common/skip.py
+++ b/tests/common/skip.py
@@ -217,6 +217,9 @@ class SkipIfHive3:
reason="Kudu is not tested with Hive 3 notifications yet, see IMPALA-8751.")
col_stat_separated_by_engine = pytest.mark.skipif(HIVE_MAJOR_VERSION >= 3,
reason="Hive 3 separates column statistics by engine")
+ kudu_with_hms_translation = pytest.mark.skipif(HIVE_MAJOR_VERSION >= 3,
+ reason="Show create table output is different for HMS translated Kudu tables. "
+ "See IMPALA-9092 for details")
class SkipIfHive2:
diff --git a/tests/metadata/test_ddl.py b/tests/metadata/test_ddl.py
index c51e0f4..ea026c9 100644
--- a/tests/metadata/test_ddl.py
+++ b/tests/metadata/test_ddl.py
@@ -680,7 +680,11 @@ class TestDdlStatements(TestDdlBase):
if HIVE_MAJOR_VERSION > 2:
assert properties['OBJCAPABILITIES'] == 'EXTREAD,EXTWRITE'
+ assert properties['TRANSLATED_TO_EXTERNAL'] == 'TRUE'
+ assert properties['external.table.purge'] == 'TRUE'
del properties['OBJCAPABILITIES']
+ del properties['TRANSLATED_TO_EXTERNAL']
+ del properties['external.table.purge']
assert len(properties) == 2
# The transient_lastDdlTime is variable, so don't verify the value.
assert 'transient_lastDdlTime' in properties
diff --git a/tests/metadata/test_show_create_table.py b/tests/metadata/test_show_create_table.py
index 2a3079c..e813d1b 100644
--- a/tests/metadata/test_show_create_table.py
+++ b/tests/metadata/test_show_create_table.py
@@ -20,7 +20,7 @@ import re
import shlex
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIf
+from tests.common.skip import SkipIf, SkipIfHive3
from tests.common.test_dimensions import create_uncompressed_text_dimension
from tests.util.test_file_parser import QueryTestSectionReader, remove_comments
@@ -54,6 +54,7 @@ class TestShowCreateTable(ImpalaTestSuite):
lambda v: v.get_value('table_format').file_format == 'text' and
v.get_value('table_format').compression_codec == 'none')
+ @SkipIfHive3.kudu_with_hms_translation
def test_show_create_table(self, vector, unique_database):
self.__run_show_create_table_test_case('QueryTest/show-create-table', vector,
unique_database)
@@ -263,6 +264,7 @@ class TestInfraCompat(ImpalaTestSuite):
'l_comment')}]
@SkipIf.kudu_not_supported
+ @SkipIfHive3.kudu_with_hms_translation
@pytest.mark.parametrize('table_primary_keys_map', TABLE_PRIMARY_KEYS_MAPS)
def test_primary_key_parse(self, impala_testinfra_cursor, table_primary_keys_map):
"""
@@ -274,6 +276,7 @@ class TestInfraCompat(ImpalaTestSuite):
table_primary_keys_map['table']) == table_primary_keys_map['primary_keys']
@SkipIf.kudu_not_supported
+ @SkipIfHive3.kudu_with_hms_translation
@pytest.mark.parametrize('table_primary_keys_map', TABLE_PRIMARY_KEYS_MAPS)
def test_load_table_with_primary_key_attr(self, impala_testinfra_cursor,
table_primary_keys_map):
diff --git a/tests/query_test/test_kudu.py b/tests/query_test/test_kudu.py
index 4686526..aa94ea0 100644
--- a/tests/query_test/test_kudu.py
+++ b/tests/query_test/test_kudu.py
@@ -40,7 +40,7 @@ from pytz import utc
from tests.common.environ import ImpalaTestClusterProperties
from tests.common.kudu_test_suite import KuduTestSuite
from tests.common.impala_cluster import ImpalaCluster
-from tests.common.skip import SkipIfNotHdfsMinicluster, SkipIfKudu
+from tests.common.skip import SkipIfNotHdfsMinicluster, SkipIfKudu, SkipIfHive3
from tests.common.test_dimensions import add_exec_option_dimension
from tests.verifiers.metric_verifier import MetricVerifier
@@ -825,6 +825,7 @@ class TestShowCreateTable(KuduTestSuite):
textwrap.dedent(show_create_sql.format(**format_args)).strip()
@SkipIfKudu.hms_integration_enabled
+ @SkipIfHive3.kudu_with_hms_translation
def test_primary_key_and_distribution(self, cursor):
# TODO: Add case with BLOCK_SIZE
self.assert_show_create_equals(cursor,
@@ -926,6 +927,7 @@ class TestShowCreateTable(KuduTestSuite):
kudu_addr=KUDU_MASTER_HOSTS))
@SkipIfKudu.hms_integration_enabled
+ @SkipIfHive3.kudu_with_hms_translation
def test_timestamp_default_value(self, cursor):
create_sql_fmt = """
CREATE TABLE {table} (c INT, d TIMESTAMP,
@@ -991,6 +993,7 @@ class TestShowCreateTable(KuduTestSuite):
kudu_client.delete_table(kudu_table_name)
@SkipIfKudu.hms_integration_enabled
+ @SkipIfHive3.kudu_with_hms_translation
def test_managed_kudu_table_name_with_show_create(self, cursor):
"""Check that the generated kudu.table_name tblproperty is not present with
show create table with managed Kudu tables.
[impala] 01/02: IMPALA-9160: Remove references to
RangerAuthorizationConfig
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit a5676981a67d63b2f0ef92e0ad2aec8fcc9a13a6
Author: Fang-Yu Rao <fa...@cloudera.com>
AuthorDate: Fri Nov 15 15:41:57 2019 -0800
IMPALA-9160: Remove references to RangerAuthorizationConfig
Due to recent changes in
https://issues.apache.org/jira/browse/RANGER-2646, the way Impala
instantiates a Ranger authorization configuration in
RangerAuthorizationConfig.java should change accordingly.
Two steps are thus required.
1. Remove the references to the class of RangerConfiguration in
RangerAuthorizationConfig.java.
2. Instantiate rangerConfig_ using the class of RangerPluginConfig
provided in the newer version of Ranger instead of RangerConfiguration
as is done today.
The second step requires us to bump up CDP_BUILD_NUMBER to 1626038 or a
later number. Due to compatibility issues of Impala with other
components in the toolchain, for now we just perform the first step.
The second step will be implemented in a follow-up JIRA after we can be
sure there will not be any compatibility issue after bumping up
CDP_BUILD_NUMBER.
Change-Id: I397c73d83a9c6f8414e83d3a605ab15cb171daa5
Reviewed-on: http://gerrit.cloudera.org:8080/14721
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../impala/authorization/ranger/RangerAuthorizationConfig.java | 5 -----
1 file changed, 5 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/authorization/ranger/RangerAuthorizationConfig.java b/fe/src/main/java/org/apache/impala/authorization/ranger/RangerAuthorizationConfig.java
index 86df80b..a15b854 100644
--- a/fe/src/main/java/org/apache/impala/authorization/ranger/RangerAuthorizationConfig.java
+++ b/fe/src/main/java/org/apache/impala/authorization/ranger/RangerAuthorizationConfig.java
@@ -20,7 +20,6 @@ package org.apache.impala.authorization.ranger;
import com.google.common.base.Preconditions;
import org.apache.impala.authorization.AuthorizationConfig;
import org.apache.impala.authorization.AuthorizationProvider;
-import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
/**
* Impala authorization config with Ranger.
@@ -29,13 +28,11 @@ public class RangerAuthorizationConfig implements AuthorizationConfig {
private final String serviceType_;
private final String appId_;
private final String serverName_;
- private final RangerConfiguration rangerConfig_;
public RangerAuthorizationConfig(String serviceType, String appId, String serverName) {
serviceType_ = Preconditions.checkNotNull(serviceType);
appId_ = Preconditions.checkNotNull(appId);
serverName_ = Preconditions.checkNotNull(serverName);
- rangerConfig_ = RangerConfiguration.getInstance();
}
@Override
@@ -56,6 +53,4 @@ public class RangerAuthorizationConfig implements AuthorizationConfig {
* Returns the Ranger application ID.
*/
public String getAppId() { return appId_; }
-
- public RangerConfiguration getRangerConfig() { return rangerConfig_; }
}