You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2019/06/05 04:01:17 UTC

[impala] 01/02: IMPALA-8505: Disallow setting Kudu table id in ALTER TABLE

This is an automated email from the ASF dual-hosted git repository.

tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit 9891597e5a0f9a8a290da5295a1343228b7dafef
Author: Hao Hao <ha...@cloudera.com>
AuthorDate: Wed May 29 23:14:58 2019 -0700

    IMPALA-8505: Disallow setting Kudu table id in ALTER TABLE
    
    This commit disallows explicitly setting the Kudu table id property for
    Kudu tables in a ALTER TABLE set statement. The Kudu table id property
    is generated when the table is created and should not be altered
    afterwards.
    
    This commit also extracts Kudu-related analyzing tests for table alteration,
    so that they can be run with or without Kudu/HMS integration enabled.
    
    Change-Id: Ibc1f3f87054db5e28f72a5b54ea5b3c040c1bf22
    Reviewed-on: http://gerrit.cloudera.org:8080/13465
    Reviewed-by: Grant Henke <gr...@apache.org>
    Reviewed-by: Thomas Marshall <tm...@cloudera.com>
    Reviewed-by: Alexey Serbin <as...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 .../analysis/AlterTableSetTblProperties.java       |  13 ++-
 .../org/apache/impala/analysis/AnalyzeDDLTest.java | 121 --------------------
 .../apache/impala/analysis/AnalyzeKuduDDLTest.java | 127 +++++++++++++++++++++
 3 files changed, 135 insertions(+), 126 deletions(-)

diff --git a/fe/src/main/java/org/apache/impala/analysis/AlterTableSetTblProperties.java b/fe/src/main/java/org/apache/impala/analysis/AlterTableSetTblProperties.java
index fcdc077..92ea8b5 100644
--- a/fe/src/main/java/org/apache/impala/analysis/AlterTableSetTblProperties.java
+++ b/fe/src/main/java/org/apache/impala/analysis/AlterTableSetTblProperties.java
@@ -115,11 +115,7 @@ public class AlterTableSetTblProperties extends AlterTableSetStmt {
   }
 
   private void analyzeKuduTable(Analyzer analyzer) throws AnalysisException {
-    // Checking for 'EXTERNAL' is case-insensitive, see IMPALA-5637.
-    String keyForExternalProperty =
-        MetaStoreUtil.findTblPropKeyCaseInsensitive(tblProperties_, "EXTERNAL");
-
-    // Throw error if kudu.table_name is provided for managed Kudu tables
+    // Throw error if kudu.table_name is provided for managed Kudu tables.
     // TODO IMPALA-6375: Allow setting kudu.table_name for managed Kudu tables
     // if the 'EXTERNAL' property is set to TRUE in the same step.
     if (!Table.isExternalTable(table_.getMetaStoreTable())) {
@@ -127,8 +123,15 @@ public class AlterTableSetTblProperties extends AlterTableSetStmt {
           String.format("Not allowed to set '%s' manually for managed Kudu tables .",
               KuduTable.KEY_TABLE_NAME));
     }
+    // Throw error if kudu.table_id is provided for Kudu tables.
+    AnalysisUtils.throwIfNotNull(tblProperties_.get(KuduTable.KEY_TABLE_ID),
+        String.format("Property '%s' cannot be altered for Kudu tables",
+            KuduTable.KEY_TABLE_ID));
     AuthorizationConfig authzConfig = analyzer.getAuthzConfig();
     if (authzConfig.isEnabled()) {
+      // Checking for 'EXTERNAL' is case-insensitive, see IMPALA-5637.
+      String keyForExternalProperty =
+          MetaStoreUtil.findTblPropKeyCaseInsensitive(tblProperties_, "EXTERNAL");
       if (keyForExternalProperty != null ||
           tblProperties_.containsKey(KuduTable.KEY_MASTER_HOSTS)) {
         String authzServer = authzConfig.getServerName();
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
index 6337612..c466a1f 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
@@ -2516,127 +2516,6 @@ public class AnalyzeDDLTest extends FrontendTestBase {
   }
 
   @Test
-  public void TestAlterKuduTable() {
-    TestUtils.assumeKuduIsSupported();
-    // ALTER TABLE ADD/DROP range partitions
-    String[] addDrop = {"add if not exists", "add", "drop if exists", "drop"};
-    for (String kw: addDrop) {
-      AnalyzesOk(String.format("alter table functional_kudu.testtbl %s range " +
-          "partition 10 <= values < 20", kw));
-      AnalyzesOk(String.format("alter table functional_kudu.testtbl %s range " +
-          "partition value = 30", kw));
-      AnalyzesOk(String.format("alter table functional_kudu.testtbl %s range " +
-          "partition values < 100", kw));
-      AnalyzesOk(String.format("alter table functional_kudu.testtbl %s range " +
-          "partition 10 <= values", kw));
-      AnalyzesOk(String.format("alter table functional_kudu.testtbl %s range " +
-          "partition 1+1 <= values <= factorial(3)", kw));
-      AnalyzesOk(String.format("alter table functional_kudu.jointbl %s range " +
-          "partition (0, '0') < values", kw));
-      AnalyzesOk(String.format("alter table functional_kudu.jointbl %s range " +
-          "partition values <= (1, '1')", kw));
-      AnalyzesOk(String.format("alter table functional_kudu.jointbl %s range " +
-          "partition (0, '0') <= values < (1, '1')", kw));
-      AnalyzesOk(String.format("alter table functional_kudu.jointbl %s range " +
-          "partition value = (-1, 'a')", kw));
-      AnalysisError(String.format("alter table functional.alltypes %s range " +
-          "partition 10 < values < 20", kw), "Table functional.alltypes does not " +
-          "support range partitions: RANGE PARTITION 10 < VALUES < 20");
-      AnalysisError(String.format("alter table functional_kudu.testtbl %s range " +
-          "partition values < isnull(null, null)", kw), "Range partition values " +
-          "cannot be NULL. Range partition: 'PARTITION VALUES < isnull(NULL, NULL)'");
-      AnalysisError(String.format("alter table functional_kudu.jointbl %s range " +
-          "partition (0) < values", kw),
-          "Number of specified range partition values is different than the number of " +
-          "partitioning columns: (1 vs 2). Range partition: 'PARTITION (0) < VALUES'");
-      AnalysisError(String.format("alter table functional_kudu.jointbl %s range " +
-          "partition values < (0, 0)", kw),
-          "Range partition value 0 (type: TINYINT) is not type compatible with " +
-          "partitioning column 'test_name' (type: STRING).");
-    }
-
-    // ALTER TABLE ADD COLUMNS
-    // Columns with different supported data types
-    AnalyzesOk("alter table functional_kudu.testtbl add columns (a1 tinyint null, a2 " +
-        "smallint null, a3 int null, a4 bigint null, a5 string null, a6 float null, " +
-        "a7 double null, a8 boolean null comment 'boolean')");
-    // Decimal types
-    AnalyzesOk("alter table functional_kudu.testtbl add columns (d1 decimal null, d2 " +
-        "decimal(9, 2) null, d3 decimal(15, 15) null, d4 decimal(38, 0) null)");
-    // Complex types
-    AnalysisError("alter table functional_kudu.testtbl add columns ( "+
-        "a struct<f1:int>)", "Kudu tables do not support complex types: " +
-        "a STRUCT<f1:INT>");
-    // Add primary key
-    AnalysisError("alter table functional_kudu.testtbl add columns (a int primary key)",
-        "Cannot add a primary key using an ALTER TABLE ADD COLUMNS statement: " +
-        "a INT PRIMARY KEY");
-    // Columns requiring a default value
-    AnalyzesOk("alter table functional_kudu.testtbl add columns (a1 int not null " +
-        "default 10)");
-    AnalyzesOk("alter table functional_kudu.testtbl add columns (a1 int null " +
-        "default 10)");
-    AnalyzesOk("alter table functional_kudu.testtbl add columns (d1 decimal(9, 2) null " +
-        "default 99.99)");
-    // Other Kudu column options
-    AnalyzesOk("alter table functional_kudu.testtbl add columns (a int encoding rle)");
-    AnalyzesOk("alter table functional_kudu.testtbl add columns (a int compression lz4)");
-    AnalyzesOk("alter table functional_kudu.testtbl add columns (a int block_size 10)");
-
-    // REPLACE columns is not supported for Kudu tables
-    AnalysisError("alter table functional_kudu.testtbl replace columns (a int null)",
-        "ALTER TABLE REPLACE COLUMNS is not supported on Kudu tables");
-    // Conflict with existing column
-    AnalysisError("alter table functional_kudu.testtbl add columns (zip int)",
-        "Column already exists: zip");
-    // Kudu column options on an HDFS table
-    AnalysisError("alter table functional.alltypes add columns (a int not null)",
-        "The specified column options are only supported in Kudu tables: a INT NOT NULL");
-
-    // ALTER TABLE DROP COLUMN
-    AnalyzesOk("alter table functional_kudu.testtbl drop column name");
-    AnalysisError("alter table functional_kudu.testtbl drop column no_col",
-        "Column 'no_col' does not exist in table: functional_kudu.testtbl");
-
-    // ALTER TABLE CHANGE COLUMN on Kudu tables
-    AnalyzesOk("alter table functional_kudu.testtbl change column name new_name string");
-    AnalyzesOk("alter table functional_kudu.testtbl change column zip " +
-        "zip int comment 'comment'");
-    // Unsupported column options
-    AnalysisError("alter table functional_kudu.testtbl change column zip zip_code int " +
-        "encoding rle compression lz4 default 90000", "Unsupported column options in " +
-        "ALTER TABLE CHANGE COLUMN statement: 'zip_code INT ENCODING RLE COMPRESSION " +
-        "LZ4 DEFAULT 90000'. Use ALTER TABLE ALTER COLUMN instead.");
-    // Changing the column type is not supported for Kudu tables
-    AnalysisError("alter table functional_kudu.testtbl change column zip zip bigint",
-        "Cannot change the type of a Kudu column using an ALTER TABLE CHANGE COLUMN " +
-        "statement: (INT vs BIGINT)");
-
-    // Rename the underlying Kudu table is not supported for managed Kudu tables
-    AnalysisError("ALTER TABLE functional_kudu.testtbl SET " +
-        "TBLPROPERTIES ('kudu.table_name' = 'Hans')",
-        "Not allowed to set 'kudu.table_name' manually for managed Kudu tables");
-
-    // TODO IMPALA-6375: Allow setting kudu.table_name for managed Kudu tables
-    // if the 'EXTERNAL' property is set to TRUE in the same step.
-    AnalysisError("ALTER TABLE functional_kudu.testtbl SET " +
-        "TBLPROPERTIES ('EXTERNAL' = 'TRUE','kudu.table_name' = 'Hans')",
-        "Not allowed to set 'kudu.table_name' manually for managed Kudu tables");
-
-    // ALTER TABLE RENAME TO
-    AnalyzesOk("ALTER TABLE functional_kudu.testtbl RENAME TO new_testtbl");
-
-    // ALTER TABLE SORT BY
-    AnalysisError("alter table functional_kudu.alltypes sort by (int_col)",
-        "ALTER TABLE SORT BY not supported on Kudu tables.");
-
-    // ALTER TABLE SET TBLPROPERTIES for sort.columns
-    AnalysisError("alter table functional_kudu.alltypes set tblproperties(" +
-        "'sort.columns'='int_col')",
-        "'sort.columns' table property is not supported for Kudu tables.");
-  }
-
-  @Test
   public void TestCreateAvroTest() {
     String alltypesSchemaLoc =
         "hdfs:///test-warehouse/avro_schemas/functional/alltypes.json";
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeKuduDDLTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeKuduDDLTest.java
index 6ac19b5..7a6b8e7 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeKuduDDLTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeKuduDDLTest.java
@@ -492,4 +492,131 @@ public class AnalyzeKuduDDLTest extends FrontendTestBase {
         String.format("Table property %s should not be specified when creating " +
             "a Kudu table.", KuduTable.KEY_TABLE_ID));
   }
+
+  @Test
+  public void TestAlterKuduTable() {
+    TestUtils.assumeKuduIsSupported();
+    // ALTER TABLE ADD/DROP range partitions
+    String[] addDrop = {"add if not exists", "add", "drop if exists", "drop"};
+    for (String kw: addDrop) {
+      AnalyzesOk(String.format("alter table functional_kudu.testtbl %s range " +
+          "partition 10 <= values < 20", kw));
+      AnalyzesOk(String.format("alter table functional_kudu.testtbl %s range " +
+          "partition value = 30", kw));
+      AnalyzesOk(String.format("alter table functional_kudu.testtbl %s range " +
+          "partition values < 100", kw));
+      AnalyzesOk(String.format("alter table functional_kudu.testtbl %s range " +
+          "partition 10 <= values", kw));
+      AnalyzesOk(String.format("alter table functional_kudu.testtbl %s range " +
+          "partition 1+1 <= values <= factorial(3)", kw));
+      AnalyzesOk(String.format("alter table functional_kudu.jointbl %s range " +
+          "partition (0, '0') < values", kw));
+      AnalyzesOk(String.format("alter table functional_kudu.jointbl %s range " +
+          "partition values <= (1, '1')", kw));
+      AnalyzesOk(String.format("alter table functional_kudu.jointbl %s range " +
+          "partition (0, '0') <= values < (1, '1')", kw));
+      AnalyzesOk(String.format("alter table functional_kudu.jointbl %s range " +
+          "partition value = (-1, 'a')", kw));
+      AnalysisError(String.format("alter table functional.alltypes %s range " +
+          "partition 10 < values < 20", kw), "Table functional.alltypes does not " +
+          "support range partitions: RANGE PARTITION 10 < VALUES < 20");
+      AnalysisError(String.format("alter table functional_kudu.testtbl %s range " +
+          "partition values < isnull(null, null)", kw), "Range partition values " +
+          "cannot be NULL. Range partition: 'PARTITION VALUES < isnull(NULL, NULL)'");
+      AnalysisError(String.format("alter table functional_kudu.jointbl %s range " +
+          "partition (0) < values", kw),
+          "Number of specified range partition values is different than the number of " +
+          "partitioning columns: (1 vs 2). Range partition: 'PARTITION (0) < VALUES'");
+      AnalysisError(String.format("alter table functional_kudu.jointbl %s range " +
+          "partition values < (0, 0)", kw),
+          "Range partition value 0 (type: TINYINT) is not type compatible with " +
+          "partitioning column 'test_name' (type: STRING).");
+    }
+
+    // ALTER TABLE ADD COLUMNS
+    // Columns with different supported data types
+    AnalyzesOk("alter table functional_kudu.testtbl add columns (a1 tinyint null, a2 " +
+        "smallint null, a3 int null, a4 bigint null, a5 string null, a6 float null, " +
+        "a7 double null, a8 boolean null comment 'boolean')");
+    // Decimal types
+    AnalyzesOk("alter table functional_kudu.testtbl add columns (d1 decimal null, d2 " +
+        "decimal(9, 2) null, d3 decimal(15, 15) null, d4 decimal(38, 0) null)");
+    // Complex types
+    AnalysisError("alter table functional_kudu.testtbl add columns ( "+
+        "a struct<f1:int>)", "Kudu tables do not support complex types: " +
+        "a STRUCT<f1:INT>");
+    // Add primary key
+    AnalysisError("alter table functional_kudu.testtbl add columns (a int primary key)",
+        "Cannot add a primary key using an ALTER TABLE ADD COLUMNS statement: " +
+        "a INT PRIMARY KEY");
+    // Columns requiring a default value
+    AnalyzesOk("alter table functional_kudu.testtbl add columns (a1 int not null " +
+        "default 10)");
+    AnalyzesOk("alter table functional_kudu.testtbl add columns (a1 int null " +
+        "default 10)");
+    AnalyzesOk("alter table functional_kudu.testtbl add columns (d1 decimal(9, 2) null " +
+        "default 99.99)");
+    // Other Kudu column options
+    AnalyzesOk("alter table functional_kudu.testtbl add columns (a int encoding rle)");
+    AnalyzesOk("alter table functional_kudu.testtbl add columns (a int compression lz4)");
+    AnalyzesOk("alter table functional_kudu.testtbl add columns (a int block_size 10)");
+
+    // REPLACE columns is not supported for Kudu tables
+    AnalysisError("alter table functional_kudu.testtbl replace columns (a int null)",
+        "ALTER TABLE REPLACE COLUMNS is not supported on Kudu tables");
+    // Conflict with existing column
+    AnalysisError("alter table functional_kudu.testtbl add columns (zip int)",
+        "Column already exists: zip");
+    // Kudu column options on an HDFS table
+    AnalysisError("alter table functional.alltypes add columns (a int not null)",
+        "The specified column options are only supported in Kudu tables: " +
+        "a INT NOT NULL");
+
+    // ALTER TABLE DROP COLUMN
+    AnalyzesOk("alter table functional_kudu.testtbl drop column name");
+    AnalysisError("alter table functional_kudu.testtbl drop column no_col",
+        "Column 'no_col' does not exist in table: functional_kudu.testtbl");
+
+    // ALTER TABLE CHANGE COLUMN on Kudu tables
+    AnalyzesOk("alter table functional_kudu.testtbl change column name new_name string");
+    AnalyzesOk("alter table functional_kudu.testtbl change column zip " +
+        "zip int comment 'comment'");
+    // Unsupported column options
+    AnalysisError("alter table functional_kudu.testtbl change column zip zip_code int " +
+        "encoding rle compression lz4 default 90000", "Unsupported column options in " +
+        "ALTER TABLE CHANGE COLUMN statement: 'zip_code INT ENCODING RLE COMPRESSION " +
+        "LZ4 DEFAULT 90000'. Use ALTER TABLE ALTER COLUMN instead.");
+    // Changing the column type is not supported for Kudu tables
+    AnalysisError("alter table functional_kudu.testtbl change column zip zip bigint",
+        "Cannot change the type of a Kudu column using an ALTER TABLE CHANGE COLUMN " +
+        "statement: (INT vs BIGINT)");
+
+    // Setting kudu.table_id is not allowed for Kudu tables.
+    AnalysisError("ALTER TABLE functional_kudu.testtbl SET " +
+        "TBLPROPERTIES ('kudu.table_id' = '1234')",
+        "Property 'kudu.table_id' cannot be altered for Kudu tables");
+
+    // Rename the underlying Kudu table is not supported for managed Kudu tables.
+    AnalysisError("ALTER TABLE functional_kudu.testtbl SET " +
+        "TBLPROPERTIES ('kudu.table_name' = 'Hans')",
+        "Not allowed to set 'kudu.table_name' manually for managed Kudu tables");
+
+    // TODO IMPALA-6375: Allow setting kudu.table_name for managed Kudu tables
+    // if the 'EXTERNAL' property is set to TRUE in the same step.
+    AnalysisError("ALTER TABLE functional_kudu.testtbl SET " +
+        "TBLPROPERTIES ('EXTERNAL' = 'TRUE','kudu.table_name' = 'Hans')",
+        "Not allowed to set 'kudu.table_name' manually for managed Kudu tables");
+
+    // ALTER TABLE RENAME TO
+    AnalyzesOk("ALTER TABLE functional_kudu.testtbl RENAME TO new_testtbl");
+
+    // ALTER TABLE SORT BY
+    AnalysisError("alter table functional_kudu.alltypes sort by (int_col)",
+        "ALTER TABLE SORT BY not supported on Kudu tables.");
+
+    // ALTER TABLE SET TBLPROPERTIES for sort.columns
+    AnalysisError("alter table functional_kudu.alltypes set tblproperties(" +
+        "'sort.columns'='int_col')",
+        "'sort.columns' table property is not supported for Kudu tables.");
+  }
 }