You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by ty...@apache.org on 2023/01/17 06:09:16 UTC
[incubator-seatunnel] branch dev updated: [Improve][Connector-V2][Hive] Improve config check logic (#3886)
This is an automated email from the ASF dual-hosted git repository.
tyrantlucifer pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new b4348f6f4 [Improve][Connector-V2][Hive] Improve config check logic (#3886)
b4348f6f4 is described below
commit b4348f6f441bc16ec0680797f241eeace0492c64
Author: lightzhao <40...@users.noreply.github.com>
AuthorDate: Tue Jan 17 14:09:11 2023 +0800
[Improve][Connector-V2][Hive] Improve config check logic (#3886)
* Whether the partition directory contains the partition name. If not, the partition loading method is different to ensure successful partition loading
* partition_dir_expression parameter adds the validation of the partition name to prevent the partition from failing to load
* update change log.
* Fix the problem of duplicate hive partition loading.
* partition_dir_expression parameter adds the validation of the partition name to prevent the partition from failing to load
* revert
* revert
* [Improve][Connector-V2][Hive] Improve config check logic, add the limit of partition expression
Co-authored-by: zhaoliang01 <zh...@58.com>
Co-authored-by: tyrantlucifer <ty...@gmail.com>
---
docs/en/connector-v2/sink/Hive.md | 2 +-
.../apache/seatunnel/connectors/seatunnel/hive/sink/HiveSink.java | 6 ++++++
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/docs/en/connector-v2/sink/Hive.md b/docs/en/connector-v2/sink/Hive.md
index d47621880..62844692f 100644
--- a/docs/en/connector-v2/sink/Hive.md
+++ b/docs/en/connector-v2/sink/Hive.md
@@ -165,4 +165,4 @@ sink {
### Next version
- [Improve] Support kerberos authentication ([3840](https://github.com/apache/incubator-seatunnel/pull/3840))
-
+- [Improve] Added partition_dir_expression validation logic ([3886](https://github.com/apache/incubator-seatunnel/pull/3886))
diff --git a/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSink.java b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSink.java
index 914c2b653..09a6d85d7 100644
--- a/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSink.java
+++ b/seatunnel-connectors-v2/connector-hive/src/main/java/org/apache/seatunnel/connectors/seatunnel/hive/sink/HiveSink.java
@@ -37,6 +37,7 @@ import org.apache.seatunnel.common.config.CheckConfigUtil;
import org.apache.seatunnel.common.config.CheckResult;
import org.apache.seatunnel.common.constants.PluginType;
import org.apache.seatunnel.common.exception.CommonErrorCode;
+import org.apache.seatunnel.connectors.seatunnel.file.config.BaseSinkConfig;
import org.apache.seatunnel.connectors.seatunnel.file.config.FileFormat;
import org.apache.seatunnel.connectors.seatunnel.file.config.HadoopConf;
import org.apache.seatunnel.connectors.seatunnel.file.hdfs.sink.BaseHdfsFileSink;
@@ -83,6 +84,11 @@ public class HiveSink extends BaseHdfsFileSink {
String.format("PluginName: %s, PluginType: %s, Message: %s",
getPluginName(), PluginType.SINK, result.getMsg()));
}
+ if (pluginConfig.hasPath(BaseSinkConfig.PARTITION_DIR_EXPRESSION.key())) {
+ throw new HiveConnectorException(SeaTunnelAPIErrorCode.CONFIG_VALIDATION_FAILED, String.format(
+ "Hive sink connector does not support setting %s", BaseSinkConfig.PARTITION_DIR_EXPRESSION.key()
+ ));
+ }
Pair<String[], Table> tableInfo = HiveConfig.getTableInfo(pluginConfig);
dbName = tableInfo.getLeft()[0];
tableName = tableInfo.getLeft()[1];