You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/12/02 21:37:07 UTC
[19/50] [abbrv] accumulo git commit: ACCUMULO-3167 Got BulkFileIT
working against mini and standalone clusters
ACCUMULO-3167 Got BulkFileIT working against mini and standalone clusters
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3342e32d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3342e32d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3342e32d
Branch: refs/heads/metrics2
Commit: 3342e32d2d1d1685df43537acd18200782b58d73
Parents: 5534665
Author: Josh Elser <el...@apache.org>
Authored: Sun Nov 23 23:43:42 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Mon Nov 24 18:08:52 2014 -0500
----------------------------------------------------------------------
.../accumulo/cluster/AccumuloCluster.java | 9 +++++-
.../standalone/StandaloneAccumuloCluster.java | 30 +++++++++++++++++++-
.../impl/MiniAccumuloClusterImpl.java | 1 +
.../accumulo/harness/AccumuloClusterIT.java | 27 ------------------
.../accumulo/test/functional/BulkFileIT.java | 23 +++++++++++----
5 files changed, 55 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3342e32d/minicluster/src/main/java/org/apache/accumulo/cluster/AccumuloCluster.java
----------------------------------------------------------------------
diff --git a/minicluster/src/main/java/org/apache/accumulo/cluster/AccumuloCluster.java b/minicluster/src/main/java/org/apache/accumulo/cluster/AccumuloCluster.java
index d647835..e823f15 100644
--- a/minicluster/src/main/java/org/apache/accumulo/cluster/AccumuloCluster.java
+++ b/minicluster/src/main/java/org/apache/accumulo/cluster/AccumuloCluster.java
@@ -16,12 +16,14 @@
*/
package org.apache.accumulo.cluster;
-import org.apache.accumulo.cluster.ClusterControl;
+import java.io.IOException;
+
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.ClientConfiguration;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.hadoop.fs.FileSystem;
/**
* Defines a minimum required set of methods to run an Accumulo cluster.
@@ -73,4 +75,9 @@ public interface AccumuloCluster {
* @throws Exception
*/
void stop() throws Exception;
+
+ /**
+ * @return the {@link FileSystem} in use by this cluster
+ */
+ FileSystem getFileSystem() throws IOException;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3342e32d/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneAccumuloCluster.java b/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneAccumuloCluster.java
index 4abb046..58536ed 100644
--- a/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneAccumuloCluster.java
+++ b/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneAccumuloCluster.java
@@ -27,8 +27,12 @@ import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.master.state.SetGoalState;
import org.apache.accumulo.minicluster.ServerType;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,7 +44,7 @@ public class StandaloneAccumuloCluster implements AccumuloCluster {
private static final Logger log = LoggerFactory.getLogger(StandaloneAccumuloCluster.class);
private Instance instance;
- private String accumuloHome, accumuloConfDir;
+ private String accumuloHome, accumuloConfDir, hadoopConfDir;
public StandaloneAccumuloCluster(String instanceName, String zookeepers) {
this(new ZooKeeperInstance(instanceName, zookeepers));
@@ -66,6 +70,14 @@ public class StandaloneAccumuloCluster implements AccumuloCluster {
this.accumuloConfDir = accumuloConfDir;
}
+ public String getHadoopConfDir() {
+ return hadoopConfDir;
+ }
+
+ public void setHadoopConfDir(String hadoopConfDir) {
+ this.hadoopConfDir = hadoopConfDir;
+ }
+
@Override
public String getInstanceName() {
return instance.getInstanceName();
@@ -116,4 +128,20 @@ public class StandaloneAccumuloCluster implements AccumuloCluster {
}
}
+ @Override
+ public FileSystem getFileSystem() throws IOException {
+ String confDir = hadoopConfDir;
+ if (null == confDir) {
+ confDir = System.getenv("HADOOP_CONF_DIR");
+ }
+ if (null == confDir) {
+ throw new IllegalArgumentException("Cannot determine HADOOP_CONF_DIR for standalone cluster");
+ }
+ // Using CachedConfiguration will make repeatedly calling this method much faster
+ final Configuration conf = CachedConfiguration.getInstance();
+ conf.addResource(new Path(confDir, "core-site.xml"));
+ // Need hdfs-site.xml for NN HA
+ conf.addResource(new Path(confDir, "hdfs-site.xml"));
+ return FileSystem.get(conf);
+ }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3342e32d/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
----------------------------------------------------------------------
diff --git a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
index 54a6468..50e90b8 100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/impl/MiniAccumuloClusterImpl.java
@@ -714,6 +714,7 @@ public class MiniAccumuloClusterImpl implements AccumuloCluster {
this.getZooKeepers());
}
+ @Override
public FileSystem getFileSystem() {
try {
return FileSystem.get(new URI(dfsUri), new Configuration());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3342e32d/test/src/test/java/org/apache/accumulo/harness/AccumuloClusterIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/harness/AccumuloClusterIT.java b/test/src/test/java/org/apache/accumulo/harness/AccumuloClusterIT.java
index 76bccf5..08c8275 100644
--- a/test/src/test/java/org/apache/accumulo/harness/AccumuloClusterIT.java
+++ b/test/src/test/java/org/apache/accumulo/harness/AccumuloClusterIT.java
@@ -18,22 +18,16 @@ package org.apache.accumulo.harness;
import static org.junit.Assert.fail;
-import java.util.Map;
-
import org.apache.accumulo.cluster.AccumuloCluster;
import org.apache.accumulo.cluster.standalone.StandaloneAccumuloCluster;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.admin.SecurityOperations;
import org.apache.accumulo.core.client.admin.TableOperations;
import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
-import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.harness.conf.AccumuloClusterPropertyConfiguration;
import org.apache.accumulo.harness.conf.AccumuloStandaloneClusterConfiguration;
import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
-import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.junit.After;
import org.junit.Assume;
import org.junit.Before;
@@ -167,27 +161,6 @@ public abstract class AccumuloClusterIT extends AccumuloIT implements MiniCluste
}
}
- @SuppressWarnings("deprecation")
- public FileSystem getFileSystem() {
- try {
- Connector conn = getConnector();
- Map<String,String> conf = conn.instanceOperations().getSystemConfiguration();
- String uri;
- if (conf.containsKey(Property.INSTANCE_VOLUMES.getKey())) {
- String instanceVolumes = conf.get(Property.INSTANCE_VOLUMES.getKey());
- uri = StringUtils.split(instanceVolumes, ',')[0];
- } else if (conf.containsKey(Property.INSTANCE_DFS_URI.getKey())) {
- uri = conf.get(Property.INSTANCE_DFS_URI.getKey());
- } else {
- throw new RuntimeException("No DFS configuraiton present in system configuration");
- }
-
- return FileSystem.get(new Path(uri).toUri(), new Configuration());
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
// TODO Really don't want this here. Will ultimately need to abstract configuration method away from MAConfig
// and change over to something more generic
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3342e32d/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java b/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
index a7cf6bd..3772937 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java
@@ -18,6 +18,7 @@ package org.apache.accumulo.test.functional;
import static com.google.common.base.Charsets.UTF_8;
+import java.io.File;
import java.util.Iterator;
import java.util.Map.Entry;
import java.util.SortedSet;
@@ -32,22 +33,22 @@ import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.file.FileSKVWriter;
import org.apache.accumulo.core.file.rfile.RFile;
import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.volume.VolumeConfiguration;
+import org.apache.accumulo.harness.AccumuloClusterIT;
import org.apache.accumulo.minicluster.MemoryUnit;
import org.apache.accumulo.minicluster.ServerType;
import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
import org.apache.accumulo.server.conf.ServerConfiguration;
-import org.apache.accumulo.server.trace.TraceFileSystem;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
+import org.junit.Assert;
import org.junit.Test;
-public class BulkFileIT extends ConfigurableMacIT {
+public class BulkFileIT extends AccumuloClusterIT {
@Override
- public void configure(MiniAccumuloConfigImpl cfg, Configuration conf) {
+ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration conf) {
cfg.setMemory(ServerType.TABLET_SERVER, 128 * 4, MemoryUnit.MEGABYTE);
}
@@ -67,9 +68,19 @@ public class BulkFileIT extends ConfigurableMacIT {
c.tableOperations().addSplits(tableName, splits);
Configuration conf = new Configuration();
AccumuloConfiguration aconf = ServerConfiguration.getDefaultConfiguration();
- FileSystem fs = TraceFileSystem.wrap(VolumeConfiguration.getDefaultVolume(conf, aconf).getFileSystem());
+ FileSystem fs = getCluster().getFileSystem();
+
+ String rootPath;
+ if (ClusterType.MINI == getClusterType()) {
+ rootPath = new File(System.getProperty("user.dir"), "target").getCanonicalPath();
+ } else if (ClusterType.STANDALONE == getClusterType()) {
+ rootPath = "/tmp";
+ } else {
+ Assert.fail("Cannot compute root path for test");
+ return;
+ }
- String dir = rootPath() + "/bulk_test_diff_files_89723987592_" + getUniqueNames(1)[0];
+ String dir = rootPath + "/bulk_test_diff_files_89723987592_" + getUniqueNames(1)[0];
fs.delete(new Path(dir), true);