You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mi...@apache.org on 2024/01/04 18:42:04 UTC
(impala) 03/03: IMPALA-12673: Table migration fails if partition contains '/'
This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 9f01c9bef7b9022d84058cfbeb03cc4f0da33c36
Author: Gabor Kaszab <ga...@cloudera.com>
AuthorDate: Wed Jan 3 15:16:48 2024 +0100
IMPALA-12673: Table migration fails if partition contains '/'
Due to Iceberg #7612 migrating a table to Iceberg resulted in incorrect
data and stats if some of the string partition fields contained '/'
character. As a result we deliberately rejected migrating such tables.
Now that Impala uses an Iceberg version that has the fix we can allow
migrating such tables too.
Change-Id: I05b4ca44c7edb81cee6747f83a5bd82c5a4b5c44
Reviewed-on: http://gerrit.cloudera.org:8080/20845
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../apache/impala/analysis/ConvertTableToIcebergStmt.java | 14 --------------
.../iceberg-migrate-from-external-hdfs-tables.test | 15 ++++++++-------
2 files changed, 8 insertions(+), 21 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/analysis/ConvertTableToIcebergStmt.java b/fe/src/main/java/org/apache/impala/analysis/ConvertTableToIcebergStmt.java
index e5f00ea6b..b8b0ee95f 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ConvertTableToIcebergStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ConvertTableToIcebergStmt.java
@@ -120,20 +120,6 @@ public class ConvertTableToIcebergStmt extends StatementBase {
"location may change");
}
- // TODO: this is a temporary check until https://github.com/apache/iceberg/issues/7612
- // is fixed.
- for (PrunablePartition partition : ((FeFsTable) table).getPartitions()) {
- for (LiteralExpr partitionExpr : partition.getPartitionValues()) {
- if (!partitionExpr.getType().isStringType()) continue;
- String partitionValue = partitionExpr.getStringValue();
- if (partitionValue == null) continue;
- if (partitionValue.contains("/")) {
- throw new AnalysisException ("Can't migrate table with '/' in the partition " +
- "values until Iceberg #7612 is fixed. '" + partitionValue + "'");
- }
- }
- }
-
createSubQueryStrings((FeFsTable) table);
}
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-migrate-from-external-hdfs-tables.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-migrate-from-external-hdfs-tables.test
index a2918744f..9d3fde193 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-migrate-from-external-hdfs-tables.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-migrate-from-external-hdfs-tables.test
@@ -28,8 +28,7 @@ insert into alltypes partition (year, month, date_col, date_string_col)
year,
month,
cast(date_string_col as date format 'MM/DD/YY') as date_col,
- # removing '/' until Iceberg issue #7612 is fixed
- replace(date_string_col, "/", "")
+ date_string_col
from functional.alltypes t;
insert into alltypes partition (year, month, date_col, date_string_col)
values (10000, true, 1, 2, 3, 4, 5.1, 6.2, "str", "2023-05-01 01:02:03", 2023, 5,
@@ -319,6 +318,7 @@ create table special_chars (i int) partitioned by (s string) stored as parquet;
insert into special_chars partition (s='11 22-33&44%55"') values (1);
insert into special_chars partition (s='aa - bb') values (2);
insert into special_chars partition (s=null) values (3);
+insert into special_chars partition (s='11/22/33') values (4);
alter table special_chars convert to iceberg;
---- RESULTS
'Table has been migrated.'
@@ -329,6 +329,7 @@ select * from special_chars;
1,'11 22-33&44%55"'
2,'aa - bb'
3,'NULL'
+4,'11/22/33'
---- TYPES
int, string
====
@@ -347,9 +348,9 @@ select * from special_chars where s is null;
int, string
====
---- QUERY
-create table special_chars_with_slash (i int) partitioned by (s1 string, s2 string) stored as parquet;
-insert into special_chars_with_slash partition (s1='abcde', s2='11/22/33') values (1);
-alter table special_chars_with_slash convert to iceberg;
----- CATCH
-AnalysisException: Can't migrate table with '/' in the partition values until Iceberg #7612 is fixed. '11/22/33'
+select * from special_chars where s='11/22/33';
+---- RESULTS
+4,'11/22/33'
+---- TYPES
+INT,STRING
====