You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by ju...@apache.org on 2019/02/27 05:10:47 UTC
[bigtop] branch master updated: BIGTOP-3151: Add flink smoke test
This is an automated email from the ASF dual-hosted git repository.
junhe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 3134c0c BIGTOP-3151: Add flink smoke test
3134c0c is described below
commit 3134c0c0b883fbdf7d9254545f72722fe356c8f7
Author: Jun He <ju...@linaro.org>
AuthorDate: Thu Feb 14 15:04:47 2019 +0800
BIGTOP-3151: Add flink smoke test
Add flink to smoke tests.
Change-Id: Ib6a538a97347ec0e77b51c81b09a9d23a1a25a1a
Signed-off-by: Jun He <ju...@linaro.org>
---
bigtop-tests/smoke-tests/flink/TestFlink.groovy | 87 +++++++++++++++++++++++++
bigtop-tests/smoke-tests/flink/build.gradle | 34 ++++++++++
bigtop-tests/smoke-tests/flink/test.data | 11 ++++
build.gradle | 1 +
provisioner/utils/smoke-tests.sh | 1 +
5 files changed, 134 insertions(+)
diff --git a/bigtop-tests/smoke-tests/flink/TestFlink.groovy b/bigtop-tests/smoke-tests/flink/TestFlink.groovy
new file mode 100644
index 0000000..d569f53
--- /dev/null
+++ b/bigtop-tests/smoke-tests/flink/TestFlink.groovy
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.bigtop.itest.hadoop.flink
+
+import org.apache.bigtop.itest.shell.Shell
+import org.apache.commons.logging.Log
+import org.apache.commons.logging.LogFactory
+import org.junit.BeforeClass
+import org.junit.AfterClass
+import org.junit.Test
+
+import static org.junit.Assert.assertEquals
+
+class TestFlink {
+ static private Log LOG = LogFactory.getLog(Object.class)
+
+ static Shell sh = new Shell("/bin/bash -s")
+
+ static String cmdPrefix = "export HADOOP_CONF_DIR=/etc/hadoop/conf;HADOOP_USER_NAME=hdfs"
+ static private final String config_file = "/etc/flink/conf/flink-conf.yaml";
+
+
+ private static void execCommand(String cmd) {
+ LOG.info(cmd)
+
+ sh.exec("$cmdPrefix $cmd")
+ }
+
+ @BeforeClass
+ static void setupTest() {
+ execCommand("hadoop fs -rm -r /flink");
+ execCommand("hadoop fs -mkdir /flink")
+ execCommand("hadoop fs -put test.data /flink/")
+ }
+
+ @AfterClass
+ static void tearDown() {
+ execCommand("hadoop fs -rm -r /flink")
+ }
+
+ @Test
+ void testCheckRestfulAPI() {
+ // read JM address and port from conf
+ execCommand("awk '{if(/jobmanager.rpc.address:/) print \$2}' < "+ config_file);
+ final String jmhost = sh.out.join('\n');
+ execCommand("awk '{if(/jobmanager.web.port:/) print \$2}' < "+config_file);
+ final String webPort = sh.out.join('\n');
+ // check web API
+ execCommand("curl http://"+jmhost+":"+webPort+"/config");
+ final String result = sh.out.join('\n');
+ assert(result.contains("flink-version"));
+ }
+
+ @Test
+ void testWordCountBatch() {
+ execCommand("flink run \$FLINK_HOME/examples/batch/WordCount.jar --input hdfs:///flink/test.data --output hdfs:///tmp/result.txt")
+
+ execCommand("hadoop fs -cat /tmp/result.txt")
+
+ String expected =
+ "black 5\n" +
+ "blue 11\n" +
+ "green 11\n" +
+ "white 5\n" +
+ "yellow 11";
+
+ String actual = sh.out.join('\n')
+
+ assertEquals("Incorrect output", expected, actual)
+ }
+}
diff --git a/bigtop-tests/smoke-tests/flink/build.gradle b/bigtop-tests/smoke-tests/flink/build.gradle
new file mode 100644
index 0000000..05ccdab
--- /dev/null
+++ b/bigtop-tests/smoke-tests/flink/build.gradle
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+def tests_to_include() {
+ return [
+ "TestFlink.groovy"
+ ];
+}
+
+sourceSets {
+ test {
+ groovy {
+ srcDirs = ["${BIGTOP_HOME}/bigtop-tests/smoke-tests/flink/"]
+ }
+ }
+}
+
+test.doFirst {
+ checkEnv(["FLINK_HOME"])
+}
diff --git a/bigtop-tests/smoke-tests/flink/test.data b/bigtop-tests/smoke-tests/flink/test.data
new file mode 100644
index 0000000..6599669
--- /dev/null
+++ b/bigtop-tests/smoke-tests/flink/test.data
@@ -0,0 +1,11 @@
+green blue yellow
+yellow green blue black white
+yellow green blue
+white yellow green blue black
+blue yellow green
+black white yellow green blue
+green blue yellow
+blue black white yellow green
+yellow green blue
+green blue black white yellow
+blue yellow green
diff --git a/build.gradle b/build.gradle
index facdcd5..d7e1045 100644
--- a/build.gradle
+++ b/build.gradle
@@ -74,6 +74,7 @@ rat {
/* Test data with rigid structure and/or binary */
"bigtop-tests/test-artifacts/**/resources/**",
"bigtop-tests/smoke-tests/alluxio/datafile",
+ "bigtop-tests/smoke-tests/flink/test.data",
"bigtop-tests/smoke-tests/hive/passwd.ql",
"bigtop-tests/smoke-tests/phoenix/*.csv",
"bigtop-tests/smoke-tests/phoenix/*.sql",
diff --git a/provisioner/utils/smoke-tests.sh b/provisioner/utils/smoke-tests.sh
index a28f799..4f2ef84 100755
--- a/provisioner/utils/smoke-tests.sh
+++ b/provisioner/utils/smoke-tests.sh
@@ -48,6 +48,7 @@ export SPARK_HOME=${SPARK_HOME:-/usr/lib/spark}
export SQOOP_HOME=${SQOOP_HOME:-/usr/lib/sqoop}
export ZOOKEEPER_HOME=${ZOOKEEPER_HOME:-/usr/lib/zookeeper}
export GIRAPH_HOME=${GIRAPH_HOME:-/usr/lib/giraph}
+export FLINK_HOME=${FLINK_HOME:-/usr/lib/flink}
echo -e "\n===== START TO RUN SMOKE TESTS: $SMOKE_TESTS =====\n"