You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2023/06/19 04:33:06 UTC
[doris] branch master updated: [fix](hive) check hive transactional table's file format (#20888)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 170cc46b12 [fix](hive) check hive transactional table's file format (#20888)
170cc46b12 is described below
commit 170cc46b12f1a1fc65e1d6b815d6aef9fd66eb60
Author: Mingyu Chen <mo...@163.com>
AuthorDate: Mon Jun 19 12:33:00 2023 +0800
[fix](hive) check hive transactional table's file format (#20888)
Sometimes we meet a hive table with parameter: "transactional" = "true", but format is parquet, which is not supported.
So we need to check the input format for transactional table.
---
.../apache/doris/catalog/external/HMSExternalTable.java | 14 +++++++++++++-
1 file changed, 13 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
index 64e06783c3..fe6603c60c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
@@ -61,6 +61,7 @@ public class HMSExternalTable extends ExternalTable {
private static final Logger LOG = LogManager.getLogger(HMSExternalTable.class);
private static final Set<String> SUPPORTED_HIVE_FILE_FORMATS;
+ private static final Set<String> SUPPORTED_HIVE_TRANSACTIONAL_FILE_FORMATS;
private static final String TBL_PROP_TXN_PROPERTIES = "transactional_properties";
private static final String TBL_PROP_INSERT_ONLY = "insert_only";
@@ -70,6 +71,9 @@ public class HMSExternalTable extends ExternalTable {
SUPPORTED_HIVE_FILE_FORMATS.add("org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat");
SUPPORTED_HIVE_FILE_FORMATS.add("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat");
SUPPORTED_HIVE_FILE_FORMATS.add("org.apache.hadoop.mapred.TextInputFormat");
+
+ SUPPORTED_HIVE_TRANSACTIONAL_FILE_FORMATS = Sets.newHashSet();
+ SUPPORTED_HIVE_TRANSACTIONAL_FILE_FORMATS.add("org.apache.hadoop.hive.ql.io.orc.OrcInputFormat");
}
private static final Set<String> SUPPORTED_HUDI_FILE_FORMATS;
@@ -191,7 +195,15 @@ public class HMSExternalTable extends ExternalTable {
}
public boolean isHiveTransactionalTable() {
- return dlaType == DLAType.HIVE && AcidUtils.isTransactionalTable(remoteTable);
+ return dlaType == DLAType.HIVE && AcidUtils.isTransactionalTable(remoteTable)
+ && isSupportedTransactionalFileFormat();
+ }
+
+ private boolean isSupportedTransactionalFileFormat() {
+ // Sometimes we meet "transactional" = "true" but format is parquet, which is not supported.
+ // So we need to check the input format for transactional table.
+ String inputFormatName = remoteTable.getSd().getInputFormat();
+ return inputFormatName != null && SUPPORTED_HIVE_TRANSACTIONAL_FILE_FORMATS.contains(inputFormatName);
}
public boolean isFullAcidTable() {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org