You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by da...@apache.org on 2024/07/20 00:20:42 UTC
(hudi) branch master updated: [HUDI-7997] Fix hivesync to support promotion type correctly (#11642)
This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 79f1768e5ac [HUDI-7997] Fix hivesync to support promotion type correctly (#11642)
79f1768e5ac is described below
commit 79f1768e5ac9c37ce6e453dee0bfa08b13c2803d
Author: Nicolas Paris <ni...@adevinta.com>
AuthorDate: Sat Jul 20 02:20:34 2024 +0200
[HUDI-7997] Fix hivesync to support promotion type correctly (#11642)
---
.../org/apache/hudi/hive/util/HiveSchemaUtil.java | 23 ----------------------
1 file changed, 23 deletions(-)
diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HiveSchemaUtil.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HiveSchemaUtil.java
index f70b023d2a0..baf905094d4 100644
--- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HiveSchemaUtil.java
+++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/HiveSchemaUtil.java
@@ -112,12 +112,6 @@ public class HiveSchemaUtil {
expectedType = expectedType.replaceAll("`", "");
if (!tableColumnType.equalsIgnoreCase(expectedType)) {
- // check for incremental queries, the schema type change is allowed as per evolution
- // rules
- if (!isSchemaTypeUpdateAllowed(tableColumnType, expectedType)) {
- throw new HoodieHiveSyncException("Could not convert field Type from " + tableColumnType + " to "
- + expectedType + " for field " + fieldName);
- }
schemaDiffBuilder.updateTableColumn(fieldName, getExpectedType(newTableSchema, tickSurroundedFieldName));
}
}
@@ -421,23 +415,6 @@ public class HiveSchemaUtil {
return array.toString();
}
- public static boolean isSchemaTypeUpdateAllowed(String prevType, String newType) {
- if (prevType == null || prevType.trim().isEmpty() || newType == null || newType.trim().isEmpty()) {
- return false;
- }
- prevType = prevType.toLowerCase();
- newType = newType.toLowerCase();
- if (prevType.equals(newType)) {
- return true;
- } else if (prevType.equalsIgnoreCase(INT_TYPE_NAME) && newType.equalsIgnoreCase(BIGINT_TYPE_NAME)) {
- return true;
- } else if (prevType.equalsIgnoreCase(FLOAT_TYPE_NAME) && newType.equalsIgnoreCase(DOUBLE_TYPE_NAME)) {
- return true;
- } else {
- return prevType.contains("struct") && newType.toLowerCase().contains("struct");
- }
- }
-
public static String generateSchemaString(MessageType storageSchema) throws IOException {
return generateSchemaString(storageSchema, Collections.EMPTY_LIST);
}