You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2015/05/22 20:04:22 UTC
[1/3] accumulo git commit: ACCUMULO-3847 Launch mapreduce jobs
locally so the jar definitely present.
Repository: accumulo
Updated Branches:
refs/heads/1.7 854277387 -> a39e7d106
refs/heads/master 65267bc7c -> 7eeb4e6ab
ACCUMULO-3847 Launch mapreduce jobs locally so the jar definitely present.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a39e7d10
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a39e7d10
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a39e7d10
Branch: refs/heads/1.7
Commit: a39e7d1063b9fcc8698a6a54568a0c9b5a46e7dc
Parents: 8542773
Author: Josh Elser <jo...@gmail.com>
Authored: Fri May 22 14:03:33 2015 -0400
Committer: Josh Elser <jo...@gmail.com>
Committed: Fri May 22 14:03:33 2015 -0400
----------------------------------------------------------------------
.../standalone/StandaloneClusterControl.java | 27 +++++++-----
.../StandaloneClusterControlTest.java | 44 ++++++++++++++++++++
2 files changed, 61 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a39e7d10/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java
----------------------------------------------------------------------
diff --git a/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java b/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java
index 8f943e0..fea542c 100644
--- a/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java
+++ b/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java
@@ -79,6 +79,10 @@ public class StandaloneClusterControl implements ClusterControl {
this.toolPath = new File(bin, TOOL_SCRIPT).getAbsolutePath();
}
+ String getToolPath() {
+ return this.toolPath;
+ }
+
protected Entry<Integer,String> exec(String hostname, String[] command) throws IOException {
RemoteShell shell = new RemoteShell(hostname, command, options);
try {
@@ -118,10 +122,19 @@ public class StandaloneClusterControl implements ClusterControl {
}
public Entry<Integer,String> execMapreduceWithStdout(Class<?> clz, String[] args) throws IOException {
- File confDir = getConfDir();
- String master = getHosts(new File(confDir, "masters")).get(0);
+ String host = "localhost";
String[] cmd = new String[3 + args.length];
- cmd[0] = toolPath;
+ cmd[0] = getToolPath();
+ cmd[1] = getJarFromClass(clz);
+ cmd[2] = clz.getName();
+ for (int i = 0, j = 3; i < args.length; i++, j++) {
+ cmd[j] = "'" + args[i] + "'";
+ }
+ log.info("Running: '{}' on {}", StringUtils.join(cmd, " "), host);
+ return exec(host, cmd);
+ }
+
+ String getJarFromClass(Class<?> clz) {
CodeSource source = clz.getProtectionDomain().getCodeSource();
if (null == source) {
throw new RuntimeException("Could not get CodeSource for class");
@@ -131,13 +144,7 @@ public class StandaloneClusterControl implements ClusterControl {
if (!jar.endsWith(".jar")) {
throw new RuntimeException("Need to have a jar to run mapreduce: " + jar);
}
- cmd[1] = jar;
- cmd[2] = clz.getName();
- for (int i = 0, j = 3; i < args.length; i++, j++) {
- cmd[j] = "'" + args[i] + "'";
- }
- log.info("Running: '{}' on {}", StringUtils.join(cmd, " "), master);
- return exec(master, cmd);
+ return jar;
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a39e7d10/minicluster/src/test/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControlTest.java
----------------------------------------------------------------------
diff --git a/minicluster/src/test/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControlTest.java b/minicluster/src/test/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControlTest.java
index 22632be..3045ddf 100644
--- a/minicluster/src/test/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControlTest.java
+++ b/minicluster/src/test/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControlTest.java
@@ -17,9 +17,18 @@
package org.apache.accumulo.cluster.standalone;
import static org.junit.Assert.assertEquals;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.aryEq;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import org.easymock.EasyMock;
import org.junit.Test;
+import com.google.common.collect.Maps;
+
public class StandaloneClusterControlTest {
@Test
@@ -35,4 +44,39 @@ public class StandaloneClusterControlTest {
assertEquals(accumuloHome + "/bin/start-server.sh", control.startServerPath);
}
+ @Test
+ public void mapreduceLaunchesLocally() throws Exception {
+ final String toolPath = "/usr/lib/accumulo/bin/tool.sh";
+ final String jar = "/home/user/my_project.jar";
+ final Class<?> clz = Object.class;
+ final String myClass = clz.getName();
+ StandaloneClusterControl control = EasyMock.createMockBuilder(StandaloneClusterControl.class).addMockedMethod("exec", String.class, String[].class)
+ .addMockedMethod("getToolPath").addMockedMethod("getJarFromClass", Class.class).createMock();
+
+ final String[] toolArgs = new String[] {"-u", "user", "-p", "password"};
+ final String[] expectedCommands = new String[3 + toolArgs.length];
+
+ int i = 0;
+ expectedCommands[i++] = toolPath;
+ expectedCommands[i++] = jar;
+ expectedCommands[i++] = myClass;
+ for (int j = 0; j < toolArgs.length; j++) {
+ expectedCommands[i + j] = quote(toolArgs[j]);
+ }
+
+ expect(control.getToolPath()).andReturn(toolPath);
+ expect(control.getJarFromClass(anyObject(Class.class))).andReturn(jar);
+ expect(control.exec(eq("localhost"), aryEq(expectedCommands))).andReturn(Maps.immutableEntry(0, ""));
+
+ replay(control);
+
+ // Give a fake Class -- we aren't verifying the actual class passed in
+ control.execMapreduceWithStdout(clz, toolArgs);
+
+ verify(control);
+ }
+
+ private String quote(String word) {
+ return "'" + word + "'";
+ }
}
[2/3] accumulo git commit: ACCUMULO-3847 Launch mapreduce jobs
locally so the jar definitely present.
Posted by el...@apache.org.
ACCUMULO-3847 Launch mapreduce jobs locally so the jar definitely present.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a39e7d10
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a39e7d10
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a39e7d10
Branch: refs/heads/master
Commit: a39e7d1063b9fcc8698a6a54568a0c9b5a46e7dc
Parents: 8542773
Author: Josh Elser <jo...@gmail.com>
Authored: Fri May 22 14:03:33 2015 -0400
Committer: Josh Elser <jo...@gmail.com>
Committed: Fri May 22 14:03:33 2015 -0400
----------------------------------------------------------------------
.../standalone/StandaloneClusterControl.java | 27 +++++++-----
.../StandaloneClusterControlTest.java | 44 ++++++++++++++++++++
2 files changed, 61 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a39e7d10/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java
----------------------------------------------------------------------
diff --git a/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java b/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java
index 8f943e0..fea542c 100644
--- a/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java
+++ b/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java
@@ -79,6 +79,10 @@ public class StandaloneClusterControl implements ClusterControl {
this.toolPath = new File(bin, TOOL_SCRIPT).getAbsolutePath();
}
+ String getToolPath() {
+ return this.toolPath;
+ }
+
protected Entry<Integer,String> exec(String hostname, String[] command) throws IOException {
RemoteShell shell = new RemoteShell(hostname, command, options);
try {
@@ -118,10 +122,19 @@ public class StandaloneClusterControl implements ClusterControl {
}
public Entry<Integer,String> execMapreduceWithStdout(Class<?> clz, String[] args) throws IOException {
- File confDir = getConfDir();
- String master = getHosts(new File(confDir, "masters")).get(0);
+ String host = "localhost";
String[] cmd = new String[3 + args.length];
- cmd[0] = toolPath;
+ cmd[0] = getToolPath();
+ cmd[1] = getJarFromClass(clz);
+ cmd[2] = clz.getName();
+ for (int i = 0, j = 3; i < args.length; i++, j++) {
+ cmd[j] = "'" + args[i] + "'";
+ }
+ log.info("Running: '{}' on {}", StringUtils.join(cmd, " "), host);
+ return exec(host, cmd);
+ }
+
+ String getJarFromClass(Class<?> clz) {
CodeSource source = clz.getProtectionDomain().getCodeSource();
if (null == source) {
throw new RuntimeException("Could not get CodeSource for class");
@@ -131,13 +144,7 @@ public class StandaloneClusterControl implements ClusterControl {
if (!jar.endsWith(".jar")) {
throw new RuntimeException("Need to have a jar to run mapreduce: " + jar);
}
- cmd[1] = jar;
- cmd[2] = clz.getName();
- for (int i = 0, j = 3; i < args.length; i++, j++) {
- cmd[j] = "'" + args[i] + "'";
- }
- log.info("Running: '{}' on {}", StringUtils.join(cmd, " "), master);
- return exec(master, cmd);
+ return jar;
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a39e7d10/minicluster/src/test/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControlTest.java
----------------------------------------------------------------------
diff --git a/minicluster/src/test/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControlTest.java b/minicluster/src/test/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControlTest.java
index 22632be..3045ddf 100644
--- a/minicluster/src/test/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControlTest.java
+++ b/minicluster/src/test/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControlTest.java
@@ -17,9 +17,18 @@
package org.apache.accumulo.cluster.standalone;
import static org.junit.Assert.assertEquals;
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.aryEq;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import org.easymock.EasyMock;
import org.junit.Test;
+import com.google.common.collect.Maps;
+
public class StandaloneClusterControlTest {
@Test
@@ -35,4 +44,39 @@ public class StandaloneClusterControlTest {
assertEquals(accumuloHome + "/bin/start-server.sh", control.startServerPath);
}
+ @Test
+ public void mapreduceLaunchesLocally() throws Exception {
+ final String toolPath = "/usr/lib/accumulo/bin/tool.sh";
+ final String jar = "/home/user/my_project.jar";
+ final Class<?> clz = Object.class;
+ final String myClass = clz.getName();
+ StandaloneClusterControl control = EasyMock.createMockBuilder(StandaloneClusterControl.class).addMockedMethod("exec", String.class, String[].class)
+ .addMockedMethod("getToolPath").addMockedMethod("getJarFromClass", Class.class).createMock();
+
+ final String[] toolArgs = new String[] {"-u", "user", "-p", "password"};
+ final String[] expectedCommands = new String[3 + toolArgs.length];
+
+ int i = 0;
+ expectedCommands[i++] = toolPath;
+ expectedCommands[i++] = jar;
+ expectedCommands[i++] = myClass;
+ for (int j = 0; j < toolArgs.length; j++) {
+ expectedCommands[i + j] = quote(toolArgs[j]);
+ }
+
+ expect(control.getToolPath()).andReturn(toolPath);
+ expect(control.getJarFromClass(anyObject(Class.class))).andReturn(jar);
+ expect(control.exec(eq("localhost"), aryEq(expectedCommands))).andReturn(Maps.immutableEntry(0, ""));
+
+ replay(control);
+
+ // Give a fake Class -- we aren't verifying the actual class passed in
+ control.execMapreduceWithStdout(clz, toolArgs);
+
+ verify(control);
+ }
+
+ private String quote(String word) {
+ return "'" + word + "'";
+ }
}
[3/3] accumulo git commit: Merge branch '1.7'
Posted by el...@apache.org.
Merge branch '1.7'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7eeb4e6a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7eeb4e6a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7eeb4e6a
Branch: refs/heads/master
Commit: 7eeb4e6ab144dfd0846292921ded2a7b85b1f642
Parents: 65267bc a39e7d1
Author: Josh Elser <jo...@gmail.com>
Authored: Fri May 22 14:04:13 2015 -0400
Committer: Josh Elser <jo...@gmail.com>
Committed: Fri May 22 14:04:13 2015 -0400
----------------------------------------------------------------------
.../standalone/StandaloneClusterControl.java | 27 +++++++-----
.../StandaloneClusterControlTest.java | 44 ++++++++++++++++++++
2 files changed, 61 insertions(+), 10 deletions(-)
----------------------------------------------------------------------