You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2013/11/21 00:33:36 UTC
[17/18] git commit: ACCUMULO-1409 Make Mini support loading native
maps
ACCUMULO-1409 Make Mini support loading native maps
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/211ca036
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/211ca036
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/211ca036
Branch: refs/heads/master
Commit: 211ca036c9cbf2c9aec2a13496dc27db758a19db
Parents: 8f5dc61
Author: Christopher Tubbs <ct...@apache.org>
Authored: Wed Nov 20 17:53:45 2013 -0500
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Wed Nov 20 17:53:45 2013 -0500
----------------------------------------------------------------------
.../minicluster/MiniAccumuloCluster.java | 29 ++++++++------------
.../minicluster/MiniAccumuloConfig.java | 22 +++++++++++++++
.../test/functional/ConfigurableMacIT.java | 3 ++
.../accumulo/test/functional/NativeMapIT.java | 10 +++++--
4 files changed, 43 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/211ca036/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
index 8e7e2ba..0a50747 100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
@@ -53,6 +53,7 @@ import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.master.thrift.MasterGoalState;
import org.apache.accumulo.core.util.Daemon;
import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.StringUtil;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.gc.SimpleGarbageCollector;
import org.apache.accumulo.master.Master;
@@ -61,10 +62,9 @@ import org.apache.accumulo.server.init.Initialize;
import org.apache.accumulo.server.util.PortUtils;
import org.apache.accumulo.server.util.time.SimpleTimer;
import org.apache.accumulo.start.Main;
-import org.apache.commons.configuration.MapConfiguration;
import org.apache.accumulo.start.classloader.vfs.MiniDFSUtil;
import org.apache.accumulo.tserver.TabletServer;
-import org.apache.commons.io.FileUtils;
+import org.apache.commons.configuration.MapConfiguration;
import org.apache.commons.vfs2.FileObject;
import org.apache.commons.vfs2.impl.VFSClassLoader;
import org.apache.hadoop.conf.Configuration;
@@ -235,7 +235,6 @@ public class MiniAccumuloCluster {
ArrayList<String> argList = new ArrayList<String>();
argList.addAll(Arrays.asList(javaBin, "-Dproc=" + clazz.getSimpleName(), "-cp", classpath));
- argList.add("-Djava.library.path=" + config.getLibDir());
argList.addAll(extraJvmOpts);
for (Entry<String,String> sysProp : config.getSystemProperties().entrySet()) {
argList.add(String.format("-D%s=%s", sysProp.getKey(), sysProp.getValue()));
@@ -248,6 +247,9 @@ public class MiniAccumuloCluster {
builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath());
builder.environment().put("ACCUMULO_LOG_DIR", config.getLogDir().getAbsolutePath());
builder.environment().put("ACCUMULO_CLIENT_CONF_PATH", config.getClientConfFile().getAbsolutePath());
+ String ldLibraryPath = StringUtil.join(Arrays.asList(config.getNativeLibPaths()), File.pathSeparator);
+ builder.environment().put("LD_LIBRARY_PATH", ldLibraryPath);
+ builder.environment().put("DYLD_LIBRARY_PATH", ldLibraryPath);
// if we're running under accumulo.start, we forward these env vars
String env = System.getenv("HADOOP_PREFIX");
@@ -353,6 +355,7 @@ public class MiniAccumuloCluster {
File clientConfFile = config.getClientConfFile();
// Write only the properties that correspond to ClientConfiguration properties
writeConfigProperties(clientConfFile, Maps.filterEntries(config.getSiteConfig(), new Predicate<Entry<String,String>>() {
+ @Override
public boolean apply(Entry<String,String> v) {
return ClientConfiguration.ClientProperty.getPropertyByKey(v.getKey()) != null;
}
@@ -375,18 +378,6 @@ public class MiniAccumuloCluster {
zooCfg.store(fileWriter, null);
fileWriter.close();
-
- File nativeMap = new File(config.getLibDir().getAbsolutePath() + "/native/map");
- nativeMap.mkdirs();
- File testRoot = new File(new File(new File(System.getProperty("user.dir")).getParent() + "/server/src/main/c++/nativeMap").getAbsolutePath());
-
- if (testRoot.exists()) {
- for (String file : testRoot.list()) {
- File src = new File(testRoot, file);
- if (src.isFile() && file.startsWith("libNativeMap"))
- FileUtils.copyFile(src, new File(nativeMap, file));
- }
- }
}
private void writeConfig(File file, Iterable<Map.Entry<String,String>> settings) throws IOException {
@@ -400,6 +391,7 @@ public class MiniAccumuloCluster {
fileWriter.append("</configuration>\n");
fileWriter.close();
}
+
private void writeConfigProperties(File file, Map<String,String> settings) throws IOException {
FileWriter fileWriter = new FileWriter(file);
@@ -439,7 +431,7 @@ public class MiniAccumuloCluster {
if (!initialized) {
// sleep a little bit to let zookeeper come up before calling init, seems to work better
long startTime = System.currentTimeMillis();
- while (true) {
+ while (true) {
try {
Socket s = new Socket("localhost", config.getZooKeeperPort());
s.getOutputStream().write("ruok\n".getBytes());
@@ -449,7 +441,7 @@ public class MiniAccumuloCluster {
if (n == 4 && new String(buffer, 0, n).equals("imok"))
break;
} catch (Exception e) {
- if(System.currentTimeMillis() - startTime >= 10000) {
+ if (System.currentTimeMillis() - startTime >= 10000) {
throw new RuntimeException("Zookeeper did not start within 10 seconds . Check the logs in " + config.getLogDir() + " for errors.");
}
UtilWaitThread.sleep(250);
@@ -626,7 +618,8 @@ public class MiniAccumuloCluster {
}
public ClientConfiguration getClientConfig() {
- return new ClientConfiguration(Arrays.asList(new MapConfiguration(config.getSiteConfig()))).withInstance(this.getInstanceName()).withZkHosts(this.getZooKeepers());
+ return new ClientConfiguration(Arrays.asList(new MapConfiguration(config.getSiteConfig()))).withInstance(this.getInstanceName()).withZkHosts(
+ this.getZooKeepers());
}
public FileSystem getFileSystem() {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/211ca036/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
----------------------------------------------------------------------
diff --git a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
index 1af2a1e..2326f95 100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
@@ -61,6 +61,8 @@ public class MiniAccumuloConfig {
private String[] classpathItems = null;
+ private String[] nativePathItems = null;
+
/**
* @param dir
* An empty or nonexistant directory that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava, and
@@ -425,6 +427,26 @@ public class MiniAccumuloConfig {
}
/**
+ * @return the paths to use for loading native libraries
+ *
+ * @since 1.6.0
+ */
+ public String[] getNativeLibPaths() {
+ return this.nativePathItems == null ? new String[0] : this.nativePathItems;
+ }
+
+ /**
+ * Sets the path for processes to use for loading native libraries
+ *
+ * @param nativePathItems
+ * the nativePathItems to set
+ * @since 1.6.0
+ */
+ public void setNativeLibPaths(String... nativePathItems) {
+ this.nativePathItems = nativePathItems;
+ }
+
+ /**
* Sets arbitrary configuration properties.
*
* @since 1.6.0
http://git-wip-us.apache.org/repos/asf/accumulo/blob/211ca036/test/src/test/java/org/apache/accumulo/test/functional/ConfigurableMacIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/ConfigurableMacIT.java b/test/src/test/java/org/apache/accumulo/test/functional/ConfigurableMacIT.java
index 21c2bb7..54585fe 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ConfigurableMacIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ConfigurableMacIT.java
@@ -23,6 +23,7 @@ import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.util.MonitorUtil;
import org.apache.accumulo.minicluster.MiniAccumuloCluster;
import org.apache.accumulo.minicluster.MiniAccumuloConfig;
@@ -41,7 +42,9 @@ public class ConfigurableMacIT extends AbstractMacIT {
@Before
public void setUp() throws Exception {
MiniAccumuloConfig cfg = new MiniAccumuloConfig(createTestDir(this.getClass().getName()), ROOT_PASSWORD);
+ cfg.setNativeLibPaths(NativeMapIT.nativeMapLocation().getAbsolutePath());
configure(cfg);
+ cfg.setProperty(Property.TSERV_NATIVEMAP_ENABLED, Boolean.TRUE.toString());
configureForEnvironment(cfg, getClass(), createSharedTestDir(this.getClass().getName() + "-ssl"));
cluster = new MiniAccumuloCluster(cfg);
cluster.start();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/211ca036/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java b/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
index 408d32c..5a79286 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java
@@ -62,11 +62,15 @@ public class NativeMapIT {
return new Value(String.format("r%09d", v).getBytes());
}
- @BeforeClass
- public static void setUp() {
+ public static File nativeMapLocation() {
File projectDir = new File(System.getProperty("user.dir")).getParentFile();
File nativeMapDir = new File(projectDir, "server/native/target/accumulo-native-" + Constants.VERSION + "/accumulo-native-" + Constants.VERSION);
- NativeMap.loadNativeLib(Collections.singletonList(nativeMapDir));
+ return nativeMapDir;
+ }
+
+ @BeforeClass
+ public static void setUp() {
+ NativeMap.loadNativeLib(Collections.singletonList(nativeMapLocation()));
}
private void verifyIterator(int start, int end, int valueOffset, Iterator<Entry<Key,Value>> iter) {