You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by he...@apache.org on 2022/06/13 06:46:15 UTC
[incubator-inlong] branch master updated (e8f36c68f -> 1f1ff9827)
This is an automated email from the ASF dual-hosted git repository.
healchow pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-inlong.git
omit e8f36c68f [INLONG-4648][Manager] Read the path of Sort connectors from configuration file (#4651)
omit e1fe7c232 [Improve][Sort] Re-optimize the distribution package of the Sort connectors (#4649)
new f07d60f38 [INLONG-4647][Sort] Re-optimize the distribution package of the Sort connectors (#4649)
new 1f1ff9827 [INLONG-4648][Manager] Read the path of Sort connectors from configuration file (#4651)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (e8f36c68f)
\
N -- N -- N refs/heads/master (1f1ff9827)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
[incubator-inlong] 01/02: [INLONG-4647][Sort] Re-optimize the distribution package of the Sort connectors (#4649)
Posted by he...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
healchow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-inlong.git
commit f07d60f383ac59d35ea13a51399379c083e48409
Author: thexia <37...@users.noreply.github.com>
AuthorDate: Mon Jun 13 12:25:11 2022 +0800
[INLONG-4647][Sort] Re-optimize the distribution package of the Sort connectors (#4649)
---
.../src/main/assemblies/release.xml | 18 ---------------
.../src/main/assemblies/sort-connectors.xml | 26 +++++++++++-----------
2 files changed, 13 insertions(+), 31 deletions(-)
diff --git a/inlong-distribution/src/main/assemblies/release.xml b/inlong-distribution/src/main/assemblies/release.xml
index d02faec4a..bd79cffb2 100644
--- a/inlong-distribution/src/main/assemblies/release.xml
+++ b/inlong-distribution/src/main/assemblies/release.xml
@@ -127,24 +127,6 @@
<outputDirectory>inlong-sort</outputDirectory>
</fileSet>
- <!--basic connector-->
- <fileSet>
- <directory>../inlong-sort/sort-connectors/pulsar/target</directory>
- <outputDirectory>sort-plugin</outputDirectory>
- <includes>
- <include>sort-connector-pulsar-${project.version}.jar</include>
- </includes>
- <fileMode>0644</fileMode>
- </fileSet>
- <fileSet>
- <directory>../inlong-sort/sort-connectors/hive/target</directory>
- <outputDirectory>sort-plugin</outputDirectory>
- <includes>
- <include>sort-connector-hive-${project.version}.jar</include>
- </includes>
- <fileMode>0644</fileMode>
- </fileSet>
-
<!-- package InLong-Dashboard-->
<fileSet>
<directory>../inlong-dashboard/build</directory>
diff --git a/inlong-distribution/src/main/assemblies/sort-connectors.xml b/inlong-distribution/src/main/assemblies/sort-connectors.xml
index b09a99609..4eaacb158 100644
--- a/inlong-distribution/src/main/assemblies/sort-connectors.xml
+++ b/inlong-distribution/src/main/assemblies/sort-connectors.xml
@@ -29,7 +29,7 @@
<!--connector plugin-->
<fileSet>
<directory>../inlong-sort/sort-connectors/hbase/target</directory>
- <outputDirectory>sort-plugin</outputDirectory>
+ <outputDirectory>inlong-sort/connectors</outputDirectory>
<includes>
<include>sort-connector-hbase-${project.version}.jar</include>
</includes>
@@ -37,7 +37,7 @@
</fileSet>
<fileSet>
<directory>../inlong-sort/sort-connectors/hive/target</directory>
- <outputDirectory>sort-plugin</outputDirectory>
+ <outputDirectory>inlong-sort/connectors</outputDirectory>
<includes>
<include>sort-connector-hive-${project.version}.jar</include>
</includes>
@@ -45,7 +45,7 @@
</fileSet>
<fileSet>
<directory>../inlong-sort/sort-connectors/iceberg/target</directory>
- <outputDirectory>sort-plugin</outputDirectory>
+ <outputDirectory>inlong-sort/connectors</outputDirectory>
<includes>
<include>sort-connector-iceberg-${project.version}.jar</include>
</includes>
@@ -53,7 +53,7 @@
</fileSet>
<fileSet>
<directory>../inlong-sort/sort-connectors/jdbc/target</directory>
- <outputDirectory>sort-plugin</outputDirectory>
+ <outputDirectory>inlong-sort/connectors</outputDirectory>
<includes>
<include>sort-connector-jdbc-${project.version}.jar</include>
</includes>
@@ -61,7 +61,7 @@
</fileSet>
<fileSet>
<directory>../inlong-sort/sort-connectors/kafka/target</directory>
- <outputDirectory>sort-plugin</outputDirectory>
+ <outputDirectory>inlong-sort/connectors</outputDirectory>
<includes>
<include>sort-connector-kafka-${project.version}.jar</include>
</includes>
@@ -69,7 +69,7 @@
</fileSet>
<fileSet>
<directory>../inlong-sort/sort-connectors/mongodb-cdc/target</directory>
- <outputDirectory>sort-plugin</outputDirectory>
+ <outputDirectory>inlong-sort/connectors</outputDirectory>
<includes>
<include>sort-connector-mongodb-cdc-${project.version}.jar</include>
</includes>
@@ -77,7 +77,7 @@
</fileSet>
<fileSet>
<directory>../inlong-sort/sort-connectors/mysql-cdc/target</directory>
- <outputDirectory>sort-plugin</outputDirectory>
+ <outputDirectory>inlong-sort/connectors</outputDirectory>
<includes>
<include>sort-connector-mysql-cdc-${project.version}.jar</include>
</includes>
@@ -85,7 +85,7 @@
</fileSet>
<fileSet>
<directory>../inlong-sort/sort-connectors/postgres-cdc/target</directory>
- <outputDirectory>sort-plugin</outputDirectory>
+ <outputDirectory>inlong-sort/connectors</outputDirectory>
<includes>
<include>sort-connector-postgres-cdc-${project.version}.jar</include>
</includes>
@@ -93,7 +93,7 @@
</fileSet>
<fileSet>
<directory>../inlong-sort/sort-connectors/pulsar/target</directory>
- <outputDirectory>sort-plugin</outputDirectory>
+ <outputDirectory>inlong-sort/connectors</outputDirectory>
<includes>
<include>sort-connector-pulsar-${project.version}.jar</include>
</includes>
@@ -101,7 +101,7 @@
</fileSet>
<fileSet>
<directory>../inlong-sort/sort-connectors/sqlserver-cdc/target</directory>
- <outputDirectory>sort-plugin</outputDirectory>
+ <outputDirectory>inlong-sort/connectors</outputDirectory>
<includes>
<include>sort-connector-sqlserver-cdc-${project.version}.jar</include>
</includes>
@@ -109,7 +109,7 @@
</fileSet>
<fileSet>
<directory>../inlong-sort/sort-connectors/oracle-cdc/target</directory>
- <outputDirectory>sort-plugin</outputDirectory>
+ <outputDirectory>inlong-sort/connectors</outputDirectory>
<includes>
<include>sort-connector-oracle-cdc-${project.version}.jar</include>
</includes>
@@ -117,7 +117,7 @@
</fileSet>
<fileSet>
<directory>../inlong-sort/sort-connectors/elasticsearch-6/target</directory>
- <outputDirectory>sort-plugin</outputDirectory>
+ <outputDirectory>inlong-sort/connectors</outputDirectory>
<includes>
<include>sort-connector-elasticsearch6-${project.version}.jar</include>
</includes>
@@ -125,7 +125,7 @@
</fileSet>
<fileSet>
<directory>../inlong-sort/sort-connectors/elasticsearch-7/target</directory>
- <outputDirectory>sort-plugin</outputDirectory>
+ <outputDirectory>inlong-sort/connectors</outputDirectory>
<includes>
<include>sort-connector-elasticsearch7-${project.version}.jar</include>
</includes>
[incubator-inlong] 02/02: [INLONG-4648][Manager] Read the path of Sort connectors from configuration file (#4651)
Posted by he...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
healchow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-inlong.git
commit 1f1ff98272ec3026c8b4d7baadddd4a97bc371cf
Author: woofyzhao <49...@qq.com>
AuthorDate: Mon Jun 13 14:28:52 2022 +0800
[INLONG-4648][Manager] Read the path of Sort connectors from configuration file (#4651)
---
.../manager/plugin/flink/FlinkOperation.java | 36 +++++++++++++++++++---
1 file changed, 32 insertions(+), 4 deletions(-)
diff --git a/inlong-manager/manager-plugins/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkOperation.java b/inlong-manager/manager-plugins/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkOperation.java
index 68cc07d6c..ccefecd73 100644
--- a/inlong-manager/manager-plugins/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkOperation.java
+++ b/inlong-manager/manager-plugins/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkOperation.java
@@ -18,6 +18,7 @@
package org.apache.inlong.manager.plugin.flink;
import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.flink.api.common.JobStatus;
import org.apache.flink.runtime.rest.messages.job.JobDetailsInfo;
@@ -26,8 +27,14 @@ import org.apache.inlong.manager.plugin.flink.dto.FlinkInfo;
import org.apache.inlong.manager.plugin.flink.enums.TaskCommitType;
import org.apache.inlong.manager.plugin.util.FlinkUtils;
+import java.io.BufferedInputStream;
import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.List;
+import java.util.Properties;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
@@ -39,19 +46,34 @@ import static org.apache.flink.api.common.JobStatus.RUNNING;
@Slf4j
public class FlinkOperation {
+ private static final String CONFIG_FILE = "application.properties";
+ private static final String CONNECTOR_DIR_KEY = "sort.connector.dir";
private static final String JOB_TERMINATED_MSG = "the job not found by id %s, "
+ "or task already terminated or savepoint path is null";
private static final String INLONG_MANAGER = "inlong-manager";
private static final String INLONG_SORT = "inlong-sort";
private static final String SORT_JAR_PATTERN = "^sort-dist.*jar$";
- private static final String SORT_PLUGIN = "sort-plugin";
-
+ private static Properties properties;
private final FlinkService flinkService;
public FlinkOperation(FlinkService flinkService) {
this.flinkService = flinkService;
}
+ /**
+ * Get sort connector directory
+ */
+ private static String getConnectorDir(String parent) throws IOException {
+ if (properties == null) {
+ properties = new Properties();
+ String path = Thread.currentThread().getContextClassLoader().getResource("").getPath() + CONFIG_FILE;
+ try (InputStream inputStream = new BufferedInputStream(Files.newInputStream(Paths.get(path)))) {
+ properties.load(inputStream);
+ }
+ }
+ return properties.getProperty(CONNECTOR_DIR_KEY, Paths.get(parent, INLONG_SORT, "connectors").toString());
+ }
+
/**
* Get Sort connector jar patterns from the Flink info.
*/
@@ -139,8 +161,14 @@ public class FlinkOperation {
flinkInfo.setLocalJarPath(jarPath);
log.info("get sort jar path success, path: {}", jarPath);
- String pluginPath = startPath + SORT_PLUGIN;
- List<String> connectorPaths = FlinkUtils.listFiles(pluginPath, getConnectorJarPattern(flinkInfo), -1);
+ String connectorDir = getConnectorDir(startPath);
+ List<String> connectorPaths = FlinkUtils.listFiles(connectorDir, getConnectorJarPattern(flinkInfo), -1);
+ if (CollectionUtils.isEmpty(connectorPaths)) {
+ String message = String.format("no sort connectors found in %s", connectorDir);
+ log.error(message);
+ throw new RuntimeException(message);
+ }
+
flinkInfo.setConnectorJarPaths(connectorPaths);
log.info("get sort connector paths success, paths: {}", connectorPaths);