You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by co...@apache.org on 2015/08/07 23:36:52 UTC
bigtop git commit: 1821: added ignite smoke tests ported from grid
gain tests.
Repository: bigtop
Updated Branches:
refs/heads/master f6eac001a -> 6f2a5fc9f
1821: added ignite smoke tests ported from grid gain tests.
Signed-off-by: Konstantin Boudnik <co...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/6f2a5fc9
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/6f2a5fc9
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/6f2a5fc9
Branch: refs/heads/master
Commit: 6f2a5fc9f6a61dc1163acbfe9387d784f96e313b
Parents: f6eac00
Author: iveselovskiy <iv...@gridgain.com>
Authored: Tue Aug 4 15:12:22 2015 +0300
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Fri Aug 7 14:33:39 2015 -0700
----------------------------------------------------------------------
.../ignite-hadoop/TestIgniteHadoop.groovy | 72 ++++++++++++++++
.../smoke-tests/ignite-hadoop/build.gradle | 53 ++++++++++++
.../ignite-hadoop/conf/core-site.xml | 90 ++++++++++++++++++++
.../ignite-hadoop/conf/log4j.properties | 24 ++++++
.../ignite-hadoop/conf/mapred-site.xml | 66 ++++++++++++++
.../smoke-tests/ignite-hadoop/test.data | 11 +++
pom.xml | 1 +
7 files changed, 317 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6f2a5fc9/bigtop-tests/smoke-tests/ignite-hadoop/TestIgniteHadoop.groovy
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/ignite-hadoop/TestIgniteHadoop.groovy b/bigtop-tests/smoke-tests/ignite-hadoop/TestIgniteHadoop.groovy
new file mode 100644
index 0000000..6358b7f
--- /dev/null
+++ b/bigtop-tests/smoke-tests/ignite-hadoop/TestIgniteHadoop.groovy
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.bigtop.itest.hadoop.ignite
+
+import org.apache.bigtop.itest.shell.Shell
+import org.apache.commons.logging.Log
+import org.apache.commons.logging.LogFactory
+import org.junit.Before
+import org.junit.Test
+
+import static org.junit.Assert.assertEquals
+
+class TestIgniteHadoop {
+ static private Log LOG = LogFactory.getLog(Object.class)
+
+ static Shell sh = new Shell("/bin/bash -s")
+
+ static String hadoopClassPath = "/usr/lib/ignite-hadoop/libs/*:/usr/lib/ignite-hadoop/libs/ignite-hadoop/*"
+
+ static String cmdPrefix = "export HADOOP_CLASSPATH=$hadoopClassPath:\$HADOOP_CLASSPATH; " +
+ "export HADOOP_CONF_DIR=\$(pwd)/conf;"
+
+
+ private static void execCommand(String cmd) {
+ LOG.info(cmd)
+
+ sh.exec("$cmdPrefix $cmd")
+ }
+
+ @Before
+ void cleanFileSystem() {
+ execCommand("hadoop fs -rm -r /gh-input")
+ execCommand("hadoop fs -rm -r /gh-output")
+ }
+
+ @Test
+ void test() {
+ execCommand("hadoop fs -mkdir /gh-input")
+ execCommand("hadoop fs -put test.data /gh-input/")
+
+ execCommand("hadoop jar \$HADOOP_MAPRED_HOME/hadoop-mapreduce-examples.jar wordcount /gh-input /gh-output")
+
+ execCommand("hadoop fs -cat /gh-output/part-r-00000")
+
+ String expected =
+ "black\t5\n" +
+ "blue\t11\n" +
+ "green\t11\n" +
+ "white\t5\n" +
+ "yellow\t11";
+
+ String actual = sh.out.join('\n')
+
+ assertEquals("Incorrect output", expected, actual)
+ }
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6f2a5fc9/bigtop-tests/smoke-tests/ignite-hadoop/build.gradle
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/ignite-hadoop/build.gradle b/bigtop-tests/smoke-tests/ignite-hadoop/build.gradle
new file mode 100644
index 0000000..f0973ff
--- /dev/null
+++ b/bigtop-tests/smoke-tests/ignite-hadoop/build.gradle
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+apply plugin: 'groovy'
+
+repositories {
+ mavenCentral()
+}
+
+dependencies {
+ //needed to avoid groovy not on classpath error.
+ testCompile module('org.codehaus.groovy:groovy:1.8.0')
+ testCompile group: 'org.apache.bigtop.itest', name: 'itest-common', version: itestVersion, transitive: 'true'
+ testCompile group: 'org.apache.hadoop', name: 'hadoop-common', version: hadoopVersion, transitive: 'true'
+ testCompile group: 'org.apache.hadoop', name: 'hadoop-hdfs', version: hadoopVersion, transitive: 'true'
+}
+
+def tests_to_include() {
+ return [
+ "TestIgniteHadoop.groovy"
+ ];
+}
+
+sourceSets {
+ test {
+ resources {
+ srcDirs = [
+ 'conf/'
+ ]
+ }
+ groovy {
+ srcDirs = ["$System.env.BIGTOP_HOME"+"/bigtop-tests/smoke-tests/ignite-hadoop/"]
+ }
+ }
+}
+
+test.doFirst {
+ checkEnv(["BIGTOP_HOME", "HADOOP_MAPRED_HOME"])
+}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6f2a5fc9/bigtop-tests/smoke-tests/ignite-hadoop/conf/core-site.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/ignite-hadoop/conf/core-site.xml b/bigtop-tests/smoke-tests/ignite-hadoop/conf/core-site.xml
new file mode 100644
index 0000000..8b8e634
--- /dev/null
+++ b/bigtop-tests/smoke-tests/ignite-hadoop/conf/core-site.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ This template file contains settings needed to run Apache Hadoop jobs
+ with Apache Ignite's distributed in-memory file system IGFS.
+
+ You can replace '$HADOOP_HOME/etc/hadoop/core-site.xml' file with this one
+ to work with IGFS nodes running on localhost (these local nodes can be
+ a part of distributed cluster though). To work with file system on remote
+ hosts you need to change the host of file system URI to any host running
+ IGFS node.
+
+ Note that Ignite jars must be in Apache Hadoop client classpath to work
+ with this configuration.
+
+ Run script '$IGNITE_HOME/bin/setup-hadoop.{sh|bat}' for Apache Hadoop client setup.
+-->
+
+<configuration>
+ <!--
+ Set default file system to IGFS instance named "igfs" configured in Ignite.
+ -->
+ <property>
+ <name>fs.default.name</name>
+ <value>igfs://igfs@localhost</value>
+ </property>
+
+ <!--
+ Set Hadoop 1.* file system implementation class for IGFS.
+ -->
+ <property>
+ <name>fs.igfs.impl</name>
+ <value>org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem</value>
+ </property>
+
+ <!--
+ Set Hadoop 2.* file system implementation class for IGFS.
+ -->
+ <property>
+ <name>fs.AbstractFileSystem.igfs.impl</name>
+ <value>org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem</value>
+ </property>
+
+ <!--
+ Disallow data node replacement since it does not make sense for IGFS nodes.
+ -->
+ <property>
+ <name>dfs.client.block.write.replace-datanode-on-failure.policy</name>
+ <value>NEVER</value>
+ </property>
+
+ <!--
+ Allow to write the job statistics into IGFS.
+ -->
+ <!--
+ <property>
+ <name>ignite.counters.writer</name>
+ <value>org.apache.ignite.hadoop.fs.IgniteHadoopFileSystemCounterWriter</value>
+ </property>
+ -->
+
+ <!--
+ By default data is placed into the file /user/<user_name>/<job_id>/performance
+ You can override this path with using macro ${USER} that is to injection of submitter user name.
+ -->
+ <!--
+ <property>
+ <name>ignite.counters.fswriter.directory</name>
+ <value>/user/${USER}</value>
+ </property>
+ -->
+</configuration>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6f2a5fc9/bigtop-tests/smoke-tests/ignite-hadoop/conf/log4j.properties
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/ignite-hadoop/conf/log4j.properties b/bigtop-tests/smoke-tests/ignite-hadoop/conf/log4j.properties
new file mode 100644
index 0000000..0723326
--- /dev/null
+++ b/bigtop-tests/smoke-tests/ignite-hadoop/conf/log4j.properties
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Set root logger level to INFO and its only appender to A1.
+log4j.rootLogger=INFO, A1
+
+# A1 is set to be a ConsoleAppender.
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+
+# A1 uses PatternLayout.
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=%-4r [BIGTOP-TEST-LOG %t] %-5p %c %x --- %m%n
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6f2a5fc9/bigtop-tests/smoke-tests/ignite-hadoop/conf/mapred-site.xml
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/ignite-hadoop/conf/mapred-site.xml b/bigtop-tests/smoke-tests/ignite-hadoop/conf/mapred-site.xml
new file mode 100644
index 0000000..a2ed437
--- /dev/null
+++ b/bigtop-tests/smoke-tests/ignite-hadoop/conf/mapred-site.xml
@@ -0,0 +1,66 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ This template file contains settings needed to run Apache Hadoop jobs
+ with Apache Ignite In-Memory Accelerator.
+
+ You can replace '$HADOOP_HOME/etc/hadoop/mapred-site.xml' file with this one
+ to run jobs on localhost (local node can be a part of distributed cluster though).
+ To run jobs on remote host you have to change jobtracker address to the REST address
+ of any running Ignite node.
+
+ Note that Ignite jars must be in Apache Hadoop client classpath to work
+ with this configuration.
+
+ Run script '$IGNITE_HOME/bin/setup-hadoop.{sh|bat}' for Apache Hadoop client setup.
+-->
+
+<configuration>
+ <!--
+ Framework name must be set to 'ignite'.
+ -->
+ <property>
+ <name>mapreduce.framework.name</name>
+ <value>ignite</value>
+ </property>
+
+ <!--
+ Job tracker address must be set to the REST address of any running Ignite node.
+ -->
+ <property>
+ <name>mapreduce.jobtracker.address</name>
+ <value>localhost:11211</value>
+ </property>
+
+ <!-- Parameters for job tuning. -->
+ <!--
+ <property>
+ <name>mapreduce.job.reduces</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>mapreduce.job.maps</name>
+ <value>4</value>
+ </property>
+ -->
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6f2a5fc9/bigtop-tests/smoke-tests/ignite-hadoop/test.data
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/ignite-hadoop/test.data b/bigtop-tests/smoke-tests/ignite-hadoop/test.data
new file mode 100644
index 0000000..6599669
--- /dev/null
+++ b/bigtop-tests/smoke-tests/ignite-hadoop/test.data
@@ -0,0 +1,11 @@
+green blue yellow
+yellow green blue black white
+yellow green blue
+white yellow green blue black
+blue yellow green
+black white yellow green blue
+green blue yellow
+blue black white yellow green
+yellow green blue
+green blue black white yellow
+blue yellow green
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6f2a5fc9/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 455f35a..49e9e1d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -304,6 +304,7 @@
<exclude>bigtop-tests/smoke-tests/kite/sandwiches.csv</exclude>
<exclude>bigtop-tests/smoke-tests/phoenix/*.csv</exclude>
<exclude>bigtop-tests/smoke-tests/phoenix/*.sql</exclude>
+ <exclude>bigtop-tests/smoke-tests/ignite-hadoop/*.data</exclude>
<exclude>**/target/**</exclude>
<exclude>**/build/**</exclude>
<exclude>**/.gradle/**</exclude>