You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bahir.apache.org by es...@apache.org on 2020/07/28 15:44:10 UTC
[bahir-flink] branch master updated: BAHIR-240: replace docker test
by testcontainer (#89)
This is an automated email from the ASF dual-hosted git repository.
eskabetxe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bahir-flink.git
The following commit(s) were added to refs/heads/master by this push:
new 86cce39 BAHIR-240: replace docker test by testcontainer (#89)
86cce39 is described below
commit 86cce3910bd97fdacd77332c8cfb5db9eb0ebd7b
Author: Joao Boto <bo...@boto.pro>
AuthorDate: Tue Jul 28 17:41:33 2020 +0200
BAHIR-240: replace docker test by testcontainer (#89)
---
.travis.yml | 13 +----
flink-connector-flume/dockers/docker-compose.yml | 57 ---------------------
flink-connector-flume/pom.xml | 18 +++----
.../streaming/connectors/flume/DockerTest.java | 31 ------------
.../connectors/flume/FlumeServerTest.java | 59 ++++++++++++++++++++++
.../streaming/connectors/flume/FlumeSinkTest.java | 5 +-
.../streaming/connectors/flume/FlumeUtilsTest.java | 5 +-
.../test/resources/docker}/conf/sink.conf | 0
.../test/resources/docker}/conf/source.conf | 0
flink-connector-kudu/pom.xml | 1 -
pom.xml | 2 -
11 files changed, 72 insertions(+), 119 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index a75578d..67bf903 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -30,27 +30,16 @@ cache:
install: true
-services:
- - docker
-
jdk:
- oraclejdk8
- openjdk8
env:
- FLINK_VERSION="1.10.0" SCALA_VERSION="2.11"
- - FLINK_VERSION="1.10.0" SCALA_VERSION="2.11" PROJECT="flink-connector-flume" DOCKER="true"
- FLINK_VERSION="1.10.0" SCALA_VERSION="2.12"
- - FLINK_VERSION="1.10.0" SCALA_VERSION="2.12" PROJECT="flink-connector-flume" DOCKER="true"
before_install:
- ./dev/change-scala-version.sh $SCALA_VERSION
script:
- - if [[ $DOCKER == "true" ]]; then
- docker-compose -f "$PROJECT/dockers/docker-compose.yml" up -d;
- mvn -q clean verify -pl $PROJECT -Dscala-$SCALA_VERSION -Dflink.version=$FLINK_VERSION ;
- docker-compose -f "$PROJECT/dockers/docker-compose.yml" down;
- else
- mvn -q clean verify -Dscala-$SCALA_VERSION -Dflink.version=$FLINK_VERSION ;
- fi
+ - mvn -q clean verify -Dscala-$SCALA_VERSION -Dflink.version=$FLINK_VERSION ;
diff --git a/flink-connector-flume/dockers/docker-compose.yml b/flink-connector-flume/dockers/docker-compose.yml
deleted file mode 100644
index 042bd5e..0000000
--- a/flink-connector-flume/dockers/docker-compose.yml
+++ /dev/null
@@ -1,57 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-version: '2'
-
-services:
-
- source:
- image: eskabetxe/flume
- container_name: flume-source
- hostname: 172.25.0.3
- ports:
- - "44444:44444"
- volumes:
- - ./conf/source.conf:/opt/flume-config/flume.conf
- environment:
- - FLUME_AGENT_NAME=docker
- links:
- - "sink:sink"
- networks:
- mynet:
- ipv4_address: 172.25.0.3
-
- sink:
- image: eskabetxe/flume
- container_name: flume-sink
- hostname: 172.25.0.4
- volumes:
- - ./conf/sink.conf:/opt/flume-config/flume.conf
- - ./output:/var/tmp/output
- environment:
- - FLUME_AGENT_NAME=docker
- networks:
- mynet:
- ipv4_address: 172.25.0.4
-
-networks:
- mynet:
- driver: bridge
- ipam:
- config:
- - subnet: 172.25.0.0/24
- IPRange: 172.25.0.2/24,
- gateway: 172.25.0.1
\ No newline at end of file
diff --git a/flink-connector-flume/pom.xml b/flink-connector-flume/pom.xml
index 1f6040a..d1e0f2a 100644
--- a/flink-connector-flume/pom.xml
+++ b/flink-connector-flume/pom.xml
@@ -36,7 +36,7 @@ under the License.
<!-- Allow users to pass custom connector versions -->
<properties>
<flume-ng.version>1.9.0</flume-ng.version>
- <junit.groups>!DockerTest</junit.groups>
+ <testcontainers.version>1.14.3</testcontainers.version>
</properties>
<dependencies>
@@ -61,16 +61,14 @@ under the License.
<scope>test</scope>
</dependency>
- </dependencies>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>testcontainers</artifactId>
+ <version>${testcontainers.version}</version>
+ <scope>test</scope>
+ </dependency>
- <profiles>
- <profile>
- <id>docker-test</id>
- <properties>
- <junit.groups>DockerTest</junit.groups>
- </properties>
- </profile>
- </profiles>
+ </dependencies>
<build>
<plugins>
diff --git a/flink-connector-flume/src/test/java/org/apache/flink/streaming/connectors/flume/DockerTest.java b/flink-connector-flume/src/test/java/org/apache/flink/streaming/connectors/flume/DockerTest.java
deleted file mode 100644
index fcee80b..0000000
--- a/flink-connector-flume/src/test/java/org/apache/flink/streaming/connectors/flume/DockerTest.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.streaming.connectors.flume;
-
-import org.junit.jupiter.api.Tag;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-@Target({ ElementType.TYPE, ElementType.METHOD })
-@Retention(RetentionPolicy.RUNTIME)
-@Tag("DockerTest")
-public @interface DockerTest {
-}
-
diff --git a/flink-connector-flume/src/test/java/org/apache/flink/streaming/connectors/flume/FlumeServerTest.java b/flink-connector-flume/src/test/java/org/apache/flink/streaming/connectors/flume/FlumeServerTest.java
new file mode 100644
index 0000000..466d0ca
--- /dev/null
+++ b/flink-connector-flume/src/test/java/org/apache/flink/streaming/connectors/flume/FlumeServerTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.streaming.connectors.flume;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.utility.MountableFile;
+
+public class FlumeServerTest {
+
+ private static final Integer EXPOSED_PORT = 44444;
+
+ public GenericContainer<?> sink = new GenericContainer<>("eskabetxe/flume")
+
+ .withCopyFileToContainer(MountableFile.forClasspathResource("docker/conf/sink.conf"), "/opt/flume-config/flume.conf")
+ .withEnv("FLUME_AGENT_NAME", "docker");
+
+ public GenericContainer<?> source = new GenericContainer<>("eskabetxe/flume")
+ .withCopyFileToContainer(MountableFile.forClasspathResource("docker/conf/source.conf"), "/opt/flume-config/flume.conf")
+ .withExposedPorts(EXPOSED_PORT)
+ .withEnv("FLUME_AGENT_NAME", "docker")
+ .dependsOn(sink);
+
+ @BeforeEach
+ void start() {
+ sink.start();
+ source.start();
+ }
+
+ @AfterEach
+ void stop() {
+ source.stop();
+ sink.stop();
+ }
+
+ protected String getHost() {
+ return source.getHost();
+ }
+
+ protected Integer getPort() {
+ return source.getMappedPort(EXPOSED_PORT);
+ }
+
+}
diff --git a/flink-connector-flume/src/test/java/org/apache/flink/streaming/connectors/flume/FlumeSinkTest.java b/flink-connector-flume/src/test/java/org/apache/flink/streaming/connectors/flume/FlumeSinkTest.java
index 92aad41..c212fdf 100644
--- a/flink-connector-flume/src/test/java/org/apache/flink/streaming/connectors/flume/FlumeSinkTest.java
+++ b/flink-connector-flume/src/test/java/org/apache/flink/streaming/connectors/flume/FlumeSinkTest.java
@@ -26,8 +26,7 @@ import java.nio.charset.Charset;
import static org.apache.flink.test.util.TestUtils.tryExecute;
-@DockerTest
-public class FlumeSinkTest {
+public class FlumeSinkTest extends FlumeServerTest {
@Test
public void testSink() throws Exception {
@@ -40,7 +39,7 @@ public class FlumeSinkTest {
}
};
- FlumeSink<String> flumeSink = new FlumeSink<>("default", "172.25.0.3", 44444, flumeEventBuilder, 1, 1, 1);
+ FlumeSink<String> flumeSink = new FlumeSink<>("default", getHost(), getPort(), flumeEventBuilder, 1, 1, 1);
environment.fromElements("string1", "string2").addSink(flumeSink);
diff --git a/flink-connector-flume/src/test/java/org/apache/flink/streaming/connectors/flume/FlumeUtilsTest.java b/flink-connector-flume/src/test/java/org/apache/flink/streaming/connectors/flume/FlumeUtilsTest.java
index f810d05..8ff168d 100644
--- a/flink-connector-flume/src/test/java/org/apache/flink/streaming/connectors/flume/FlumeUtilsTest.java
+++ b/flink-connector-flume/src/test/java/org/apache/flink/streaming/connectors/flume/FlumeUtilsTest.java
@@ -20,13 +20,12 @@ import org.apache.flume.api.RpcClient;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
-@DockerTest
-public class FlumeUtilsTest {
+public class FlumeUtilsTest extends FlumeServerTest {
private RpcClient client;
@Test
public void testGetRpcClient() {
- client = FlumeUtils.getRpcClient("default","172.25.0.3", 44444, 1);
+ client = FlumeUtils.getRpcClient("default",getHost(), getPort(), 1);
Assertions.assertNotNull(client);
}
diff --git a/flink-connector-flume/dockers/conf/sink.conf b/flink-connector-flume/src/test/resources/docker/conf/sink.conf
similarity index 100%
rename from flink-connector-flume/dockers/conf/sink.conf
rename to flink-connector-flume/src/test/resources/docker/conf/sink.conf
diff --git a/flink-connector-flume/dockers/conf/source.conf b/flink-connector-flume/src/test/resources/docker/conf/source.conf
similarity index 100%
rename from flink-connector-flume/dockers/conf/source.conf
rename to flink-connector-flume/src/test/resources/docker/conf/source.conf
diff --git a/flink-connector-kudu/pom.xml b/flink-connector-kudu/pom.xml
index 3bbefee..bbf168e 100644
--- a/flink-connector-kudu/pom.xml
+++ b/flink-connector-kudu/pom.xml
@@ -32,7 +32,6 @@
<properties>
<kudu.version>1.11.1</kudu.version>
<mockito.version>1.10.19</mockito.version>
- <junit.groups>!DockerTest</junit.groups>
</properties>
<dependencies>
diff --git a/pom.xml b/pom.xml
index ec334fc..126e79d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -98,7 +98,6 @@
<flink.version>1.10.0</flink.version>
<junit.jupiter.version>5.4.1</junit.jupiter.version>
- <junit.groups></junit.groups>
<scalatest.version>3.0.5</scalatest.version>
<scalacheck.version>1.14.0</scalacheck.version>
@@ -346,7 +345,6 @@
<version>2.22.1</version>
<!-- Note config is repeated in scalatest config -->
<configuration>
- <groups>${junit.groups}</groups>
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
<argLine>-Xmx3g -Xss4096k -XX:MaxPermSize=${MaxPermGen} -XX:ReservedCodeCacheSize=512m</argLine>
<systemProperties>