You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mj...@apache.org on 2017/02/17 23:17:10 UTC
[7/9] incubator-impala git commit: IMPALA-4282: Remove max length
check for type strings.
IMPALA-4282: Remove max length check for type strings.
During analysis, Impala used to enforce the default HMS
limit on the number of characters of a type string.
That enforcement has the benefit of a clear error
message, but has the severe drawback of not having
any workaround because the limit is baked into the code.
Going above the 4000 limit is pretty easy with complex
types, and several users have already run into this
issue.
This patch removes the Impala-side enforcement such
that the 4000 limit can be increased by altering
the corresponding columns in the Metastore's backend
database.
Change-Id: I01f5e503e14feee857bbdf80b3ef4f5a8b57fb2d
Reviewed-on: http://gerrit.cloudera.org:8080/6034
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/0c871529
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/0c871529
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/0c871529
Branch: refs/heads/master
Commit: 0c8715292e93769c71f8160ad85ecd57552ca585
Parents: b7a7636
Author: Alex Behm <al...@cloudera.com>
Authored: Wed Feb 15 23:15:18 2017 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Fri Feb 17 05:30:09 2017 +0000
----------------------------------------------------------------------
.../org/apache/impala/analysis/ColumnDef.java | 9 +----
.../org/apache/impala/util/MetaStoreUtil.java | 3 --
.../apache/impala/analysis/AnalyzeDDLTest.java | 39 --------------------
3 files changed, 1 insertion(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/0c871529/fe/src/main/java/org/apache/impala/analysis/ColumnDef.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/ColumnDef.java b/fe/src/main/java/org/apache/impala/analysis/ColumnDef.java
index 8993acb..d7217f7 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ColumnDef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ColumnDef.java
@@ -176,18 +176,11 @@ public class ColumnDef {
}
Preconditions.checkNotNull(type_);
Preconditions.checkState(type_.isValid());
- // Check HMS constraints of type and comment.
- String typeSql = type_.toSql();
- if (typeSql.length() > MetaStoreUtil.MAX_TYPE_NAME_LENGTH) {
- throw new AnalysisException(String.format(
- "Type of column '%s' exceeds maximum type length of %d characters:\n" +
- "%s has %d characters.", colName_, MetaStoreUtil.MAX_TYPE_NAME_LENGTH,
- typeSql, typeSql.length()));
- }
if (hasKuduOptions()) {
Preconditions.checkNotNull(analyzer);
analyzeKuduOptions(analyzer);
}
+ // Check HMS constraints on comment.
if (comment_ != null &&
comment_.length() > MetaStoreUtil.CREATE_MAX_COMMENT_LENGTH) {
throw new AnalysisException(String.format(
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/0c871529/fe/src/main/java/org/apache/impala/util/MetaStoreUtil.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/util/MetaStoreUtil.java b/fe/src/main/java/org/apache/impala/util/MetaStoreUtil.java
index 95ef700..8c9ac24 100644
--- a/fe/src/main/java/org/apache/impala/util/MetaStoreUtil.java
+++ b/fe/src/main/java/org/apache/impala/util/MetaStoreUtil.java
@@ -44,9 +44,6 @@ public class MetaStoreUtil {
// for Postgres it is 4000, but for most other databases it is 256.
public static final int CREATE_MAX_COMMENT_LENGTH = 256;
- // Maximum length of the string representation of a type that the HMS can store.
- public static final int MAX_TYPE_NAME_LENGTH = 4000;
-
// The longest strings Hive accepts for [serde] property keys.
public static final int MAX_PROPERTY_KEY_LENGTH = 256;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/0c871529/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
index 29c59e8..c0703eb 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
@@ -1799,12 +1799,6 @@ public class AnalyzeDDLTest extends FrontendTestBase {
"Invalid column/field name: ???");
AnalysisError("create table new_table (i int) PARTITIONED BY (`^&*` int)",
"Invalid column/field name: ^&*");
- // Test HMS constraint on type name length.
- AnalyzesOk(String.format("create table t (i %s)",
- genTypeSql(MetaStoreUtil.MAX_TYPE_NAME_LENGTH)));
- AnalysisError(String.format("create table t (i %s)",
- genTypeSql(MetaStoreUtil.MAX_TYPE_NAME_LENGTH + 1)),
- "Type of column 'i' exceeds maximum type length of 4000 characters:");
// Test HMS constraint on comment length.
AnalyzesOk(String.format("create table t (i int comment '%s')",
StringUtils.repeat("c", MetaStoreUtil.CREATE_MAX_COMMENT_LENGTH)));
@@ -1865,39 +1859,6 @@ public class AnalyzeDDLTest extends FrontendTestBase {
}
}
- /**
- * Generates a valid type string with exactly the given number of characters.
- * The type is a struct with at least two fields.
- * The given length must be at least "struct<s:int,c:int>".length() == 19.
- */
- private String genTypeSql(int length) {
- Preconditions.checkState(length >= 19);
- StringBuilder result = new StringBuilder();
- result.append("struct<s:int");
- // The middle fields always have a fixed length.
- int midFieldLen = ",f000:int".length();
- // The last field has a variable length, but this is the minimum.
- int lastFieldMinLen = ",f:int".length();
- int fieldIdx = 0;
- while (result.length() < length - midFieldLen - lastFieldMinLen) {
- String fieldStr = String.format(",f%03d:int", fieldIdx);
- result.append(fieldStr);
- ++fieldIdx;
- }
- Preconditions.checkState(result.length() == length - 1 ||
- result.length() < length - lastFieldMinLen);
- // Generate last field with a variable length.
- if (result.length() < length - 1) {
- int fieldNameLen = length - result.length() - ",:int".length() - 1;
- Preconditions.checkState(fieldNameLen > 0);
- String fieldStr = String.format(",%s:int", StringUtils.repeat("f", fieldNameLen));
- result.append(fieldStr);
- }
- result.append(">");
- Preconditions.checkState(result.length() == length);
- return result.toString();
- }
-
@Test
public void TestAlterKuduTable() {
TestUtils.assumeKuduIsSupported();