You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by ki...@apache.org on 2022/10/14 05:57:31 UTC
[incubator-seatunnel] branch dev updated: [improve][e2e][engine] Temporarily disable seatunnel's engine testing (#3070)
This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new babf75ad8 [improve][e2e][engine] Temporarily disable seatunnel's engine testing (#3070)
babf75ad8 is described below
commit babf75ad82166c4187f08528b35fd3b12f05ba0c
Author: Zongwen Li <zo...@apache.org>
AuthorDate: Fri Oct 14 13:57:26 2022 +0800
[improve][e2e][engine] Temporarily disable seatunnel's engine testing (#3070)
---
.../apache/seatunnel/e2e/common/TestSuiteBase.java | 3 +
.../container/seatunnel/SeaTunnelContainer.java | 12 +-
.../e2e/common/junit/DisabledOnContainer.java | 2 +
.../seatunnel/engine/e2e/SeaTunnelContainer.java | 129 +++------------------
4 files changed, 25 insertions(+), 121 deletions(-)
diff --git a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/TestSuiteBase.java b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/TestSuiteBase.java
index 00f2c6502..b098a2058 100644
--- a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/TestSuiteBase.java
+++ b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/TestSuiteBase.java
@@ -17,9 +17,11 @@
package org.apache.seatunnel.e2e.common;
+import org.apache.seatunnel.e2e.common.container.EngineType;
import org.apache.seatunnel.e2e.common.container.TestContainer;
import org.apache.seatunnel.e2e.common.container.TestContainersFactory;
import org.apache.seatunnel.e2e.common.junit.ContainerTestingExtension;
+import org.apache.seatunnel.e2e.common.junit.DisabledOnContainer;
import org.apache.seatunnel.e2e.common.junit.TestCaseInvocationContextProvider;
import org.apache.seatunnel.e2e.common.junit.TestContainers;
import org.apache.seatunnel.e2e.common.junit.TestLoggerExtension;
@@ -35,6 +37,7 @@ import org.testcontainers.containers.Network;
TestCaseInvocationContextProvider.class
})
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+@DisabledOnContainer(value = {}, type = EngineType.SEATUNNEL, disabledReason = "TODO: SeaTunnel engine e2e test isn't completed")
public abstract class TestSuiteBase {
protected static final Network NETWORK = TestContainer.NETWORK;
diff --git a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java
index 526e07e4b..7d77e0c67 100644
--- a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java
+++ b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java
@@ -21,11 +21,12 @@ import static org.apache.seatunnel.e2e.common.util.ContainerUtil.PROJECT_ROOT_PA
import org.apache.seatunnel.e2e.common.container.AbstractTestContainer;
import org.apache.seatunnel.e2e.common.container.ContainerExtendedFactory;
+import org.apache.seatunnel.e2e.common.container.TestContainer;
import org.apache.seatunnel.e2e.common.container.TestContainerId;
+import com.google.auto.service.AutoService;
import lombok.NoArgsConstructor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import lombok.extern.slf4j.Slf4j;
import org.testcontainers.containers.Container;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.output.Slf4jLogConsumer;
@@ -38,11 +39,9 @@ import java.util.Collections;
import java.util.List;
@NoArgsConstructor
-//@AutoService(TestContainer.class)
-// TODO add AutoService after engine feature is ready
+@Slf4j
+@AutoService(TestContainer.class)
public class SeaTunnelContainer extends AbstractTestContainer {
-
- private static final Logger LOG = LoggerFactory.getLogger(SeaTunnelContainer.class);
private static final String JDK_DOCKER_IMAGE = "openjdk:8";
private static final String CLIENT_SHELL = "seatunnel.sh";
private static final String SERVER_SHELL = "seatunnel-cluster.sh";
@@ -62,7 +61,6 @@ public class SeaTunnelContainer extends AbstractTestContainer {
Paths.get(SEATUNNEL_HOME, "config").toString());
server.start();
// execute extra commands
- // TODO copy config file
executeExtraCommands(server);
}
diff --git a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/junit/DisabledOnContainer.java b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/junit/DisabledOnContainer.java
index dcebfa72c..49617b526 100644
--- a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/junit/DisabledOnContainer.java
+++ b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/junit/DisabledOnContainer.java
@@ -22,12 +22,14 @@ import org.apache.seatunnel.e2e.common.container.TestContainer;
import org.apache.seatunnel.e2e.common.container.TestContainerId;
import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Target({ ElementType.TYPE, ElementType.METHOD })
@Retention(RetentionPolicy.RUNTIME)
+@Inherited
public @interface DisabledOnContainer {
/**
diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SeaTunnelContainer.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SeaTunnelContainer.java
index 7c5bafab5..bb6b4495e 100644
--- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SeaTunnelContainer.java
+++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/SeaTunnelContainer.java
@@ -17,132 +17,33 @@
package org.apache.seatunnel.engine.e2e;
-import org.apache.seatunnel.e2e.common.util.ContainerUtil;
-
+import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.junit.jupiter.api.TestInstance;
import org.testcontainers.containers.Container;
-import org.testcontainers.containers.GenericContainer;
-import org.testcontainers.containers.Network;
-import org.testcontainers.containers.output.Slf4jLogConsumer;
-import org.testcontainers.containers.wait.strategy.Wait;
-import org.testcontainers.utility.DockerLoggerFactory;
-import org.testcontainers.utility.MountableFile;
-import java.io.File;
import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CompletableFuture;
-
-public abstract class SeaTunnelContainer {
-
- private static final Logger LOG = LoggerFactory.getLogger(SeaTunnelContainer.class);
- private static final String JDK_DOCKER_IMAGE = "openjdk:8";
- protected static final Network NETWORK = Network.newNetwork();
- private static final Path PROJECT_ROOT_PATH = Paths.get(System.getProperty("user.dir")).getParent().getParent().getParent();
- private static final String SEATUNNEL_HOME = "/tmp/seatunnel";
- private static final String PLUGIN_MAPPING_FILE = "plugin-mapping.properties";
- private static final String SEATUNNEL_BIN = Paths.get(SEATUNNEL_HOME, "bin").toString();
- private static final String SEATUNNEL_LIB = Paths.get(SEATUNNEL_HOME, "lib").toString();
- private static final String SEATUNNEL_CONFIG = Paths.get(SEATUNNEL_HOME, "config").toString();
-
- private static final String SEATUNNEL_CONNECTORS = Paths.get(SEATUNNEL_HOME, "connectors").toString();
- private static final String CLIENT_SHELL = "seatunnel.sh";
- private static final String SERVER_SHELL = "seatunnel-cluster.sh";
- private static GenericContainer<?> SERVER;
+@Slf4j
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public abstract class SeaTunnelContainer extends org.apache.seatunnel.e2e.common.container.seatunnel.SeaTunnelContainer {
+ @Override
@BeforeAll
- public static void before() {
- Map<String, String> mountMapping = getFileMountMapping();
- SERVER = new GenericContainer<>(JDK_DOCKER_IMAGE)
- .withNetwork(NETWORK)
- .withCommand(Paths.get(SEATUNNEL_BIN, SERVER_SHELL).toString())
- .withNetworkAliases("server")
- .withExposedPorts()
- .withLogConsumer(new Slf4jLogConsumer(DockerLoggerFactory.getLogger("seatunnel-engine:" + JDK_DOCKER_IMAGE)))
- .waitingFor(Wait.forLogMessage(".*received new worker register.*\\n", 1));
- mountMapping.forEach(SERVER::withFileSystemBind);
- SERVER.start();
- }
-
- protected static Map<String, String> getFileMountMapping() {
-
- Map<String, String> mountMapping = new HashMap<>();
- // copy lib
- mountMapping.put(PROJECT_ROOT_PATH + "/seatunnel-core/seatunnel-starter/target/seatunnel-starter.jar",
- Paths.get(SEATUNNEL_LIB, "seatunnel-starter.jar").toString());
-
- mountMapping.put(PROJECT_ROOT_PATH + "/seatunnel-engine/seatunnel-engine-common/src/main/resources/",
- SEATUNNEL_CONFIG);
-
- // copy bin
- mountMapping.put(PROJECT_ROOT_PATH + "/seatunnel-core/seatunnel-starter/src/main/bin/",
- Paths.get(SEATUNNEL_BIN).toString());
-
- // copy plugin-mapping.properties
- mountMapping.put(PROJECT_ROOT_PATH + "/plugin-mapping.properties", Paths.get(SEATUNNEL_CONNECTORS, PLUGIN_MAPPING_FILE).toString());
-
- return mountMapping;
- }
-
- @SuppressWarnings("checkstyle:MagicNumber")
- public Container.ExecResult executeSeaTunnelJob(String confFile) throws IOException, InterruptedException {
- ContainerUtil.copyConnectorJarToContainer(SERVER, confFile, "seatunnel-connectors-v2", "connector-", "seatunnel", SEATUNNEL_HOME);
- final String confPath = getResource(confFile);
- if (!new File(confPath).exists()) {
- throw new IllegalArgumentException(confFile + " doesn't exist");
- }
- final String targetConfInContainer = Paths.get("/tmp", confFile).toString();
- SERVER.copyFileToContainer(MountableFile.forHostPath(confPath), targetConfInContainer);
-
- // Running IT use cases under Windows requires replacing \ with /
- String conf = targetConfInContainer.replaceAll("\\\\", "/");
- final List<String> command = new ArrayList<>();
- command.add(Paths.get(SEATUNNEL_BIN, CLIENT_SHELL).toString());
- command.add("--config " + conf);
-
- CompletableFuture.runAsync(() -> {
- try {
- Thread.sleep(15000);
- // cancel server if bash command not return
- SERVER.stop();
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- });
- Container.ExecResult execResult = SERVER.execInContainer("bash", "-c", String.join(" ", command));
- if (execResult.getStdout() != null && execResult.getStdout().length() > 0) {
- LOG.info("\n==================== ExecuteConfigFile: {} STDOUT start ====================\n"
- + "{}"
- + "\n==================== ExecuteConfigFile: {} STDOUT end ====================",
- confFile, execResult.getStdout(), confFile);
- }
- if (execResult.getStderr() != null && execResult.getStderr().length() > 0) {
- LOG.error("\n==================== ExecuteConfigFile: {} STDERR start ====================\n"
- + "{}"
- + "\n==================== ExecuteConfigFile: {} STDERR end ====================",
- confFile, execResult.getStderr(), confFile);
- }
- return execResult;
+ public void startUp() throws Exception {
+ super.startUp();
+ log.info("The TestContainer[{}] is running.", identifier());
}
+ @Override
@AfterAll
- public static void after() {
- if (SERVER != null) {
- SERVER.close();
- }
+ public void tearDown() throws Exception {
+ super.tearDown();
+ log.info("The TestContainer[{}] is closed.", identifier());
}
- private String getResource(String confFile) {
- return System.getProperty("user.dir") + "/src/test/resources" + confFile;
+ public Container.ExecResult executeSeaTunnelJob(String confFile) throws IOException, InterruptedException {
+ return executeJob(confFile);
}
-
}