You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/01/22 23:10:10 UTC
[02/10] git commit: ACCUMULO-2234 Provide accumulo-site.xml to Mapper
classpath and ensure is used by concrete Instance.
ACCUMULO-2234 Provide accumulo-site.xml to Mapper classpath and ensure is used by concrete Instance.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/57f9b6cf
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/57f9b6cf
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/57f9b6cf
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 57f9b6cfd30b1b2505efbacd2a5ce391dbcd1e0c
Parents: 06f8030
Author: Josh Elser <el...@apache.org>
Authored: Wed Jan 22 16:27:02 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Wed Jan 22 16:27:02 2014 -0500
----------------------------------------------------------------------
.../core/client/mapreduce/InputFormatBase.java | 11 ++++++++-
.../core/client/mapreduce/RangeInputSplit.java | 11 ++++++++-
.../test/continuous/ContinuousVerify.java | 24 ++++++++++++++++++--
test/system/continuous/run-verify.sh | 24 +++++++++++++++++++-
4 files changed, 65 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/57f9b6cf/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
index de97f12..9c23246 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
@@ -58,6 +58,8 @@ import org.apache.accumulo.core.client.impl.Tables;
import org.apache.accumulo.core.client.impl.TabletLocator;
import org.apache.accumulo.core.client.mock.MockInstance;
import org.apache.accumulo.core.client.mock.MockTabletLocator;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.PartialKey;
@@ -735,7 +737,14 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
protected static Instance getInstance(Configuration conf) {
if (conf.getBoolean(MOCK, false))
return new MockInstance(conf.get(INSTANCE_NAME));
- return new ZooKeeperInstance(conf.get(INSTANCE_NAME), conf.get(ZOOKEEPERS));
+
+ ZooKeeperInstance zki = new ZooKeeperInstance(conf.get(INSTANCE_NAME), conf.get(ZOOKEEPERS));
+
+ // Wrap the DefaultConfiguration with a SiteConfiguration
+ AccumuloConfiguration xmlConfig = SiteConfiguration.getInstance(zki.getConfiguration());
+ zki.setConfiguration(xmlConfig);
+
+ return zki;
}
/**
http://git-wip-us.apache.org/repos/asf/accumulo/blob/57f9b6cf/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
index 01921c4..e1107cc 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
@@ -29,6 +29,8 @@ import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.client.mapreduce.InputFormatBase.AccumuloIterator;
import org.apache.accumulo.core.client.mapreduce.InputFormatBase.AccumuloIteratorOption;
import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.PartialKey;
@@ -354,7 +356,14 @@ public class RangeInputSplit extends InputSplit implements Writable {
return null;
}
- return new ZooKeeperInstance(getInstanceName(), getZooKeepers());
+ ZooKeeperInstance zki = new ZooKeeperInstance(getInstanceName(), getZooKeepers());
+
+ // Wrap the DefaultConfiguration with a SiteConfiguration so we use accumulo-site.xml
+ // when it's present
+ AccumuloConfiguration xmlConfig = SiteConfiguration.getInstance(zki.getConfiguration());
+ zki.setConfiguration(xmlConfig);
+
+ return zki;
}
public String getInstanceName() {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/57f9b6cf/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java b/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
index d271d22..6546eea 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
@@ -17,6 +17,7 @@
package org.apache.accumulo.server.test.continuous;
import java.io.IOException;
+import java.net.URI;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -34,6 +35,8 @@ import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.server.test.continuous.ContinuousWalk.BadChecksumException;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
@@ -44,12 +47,14 @@ import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.log4j.Logger;
/**
* A map reduce job that verifies a table created by continuous ingest. It verifies that all referenced nodes are defined.
*/
public class ContinuousVerify extends Configured implements Tool {
+ private static final Logger log = Logger.getLogger(ContinuousVerify.class);
public static final VLongWritable DEF = new VLongWritable(-1);
@@ -150,9 +155,9 @@ public class ContinuousVerify extends Configured implements Tool {
args = argsList.toArray(new String[0]);
- if (args.length != 9) {
+ if (args.length != 10) {
throw new IllegalArgumentException("Usage : " + ContinuousVerify.class.getName()
- + " <instance name> <zookeepers> <user> <pass> <table> <output dir> <max mappers> <num reducers> <scan offline>");
+ + " <instance name> <zookeepers> <user> <pass> <table> <output dir> <max mappers> <num reducers> <scan offline> <sitexml>");
}
String instance = args[0];
@@ -164,6 +169,7 @@ public class ContinuousVerify extends Configured implements Tool {
String maxMaps = args[6];
String reducers = args[7];
boolean scanOffline = Boolean.parseBoolean(args[8]);
+ String siteFile = args[9];
Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
job.setJarByClass(this.getClass());
@@ -210,6 +216,20 @@ public class ContinuousVerify extends Configured implements Tool {
job.setOutputFormatClass(TextOutputFormat.class);
job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", scanOffline);
+
+ Path sitePath = new Path(siteFile);
+ Path siteParentPath = sitePath.getParent();
+ if (null == siteParentPath) {
+ siteParentPath = new Path("/");
+ }
+
+ URI siteUri = new URI("hdfs://" + siteFile);
+
+ log.info("Adding " + siteUri + " to DistributedCache");
+
+ // Make sure that accumulo-site.xml is available for mappers running offline scans
+ // as they need to correctly choose instance.dfs.dir for the installation
+ DistributedCache.addFileToClassPath(siteParentPath, job.getConfiguration(), FileSystem.get(siteUri, job.getConfiguration()));
TextOutputFormat.setOutputPath(job, new Path(outputdir));
http://git-wip-us.apache.org/repos/asf/accumulo/blob/57f9b6cf/test/system/continuous/run-verify.sh
----------------------------------------------------------------------
diff --git a/test/system/continuous/run-verify.sh b/test/system/continuous/run-verify.sh
index edf58b7..f2b7a25 100755
--- a/test/system/continuous/run-verify.sh
+++ b/test/system/continuous/run-verify.sh
@@ -24,5 +24,27 @@ if [ -n "$VERIFY_AUTHS" ] ; then
AUTH_OPT="--auths $VERIFY_AUTHS";
fi
-$ACCUMULO_HOME/bin/tool.sh "$SERVER_LIBJAR" org.apache.accumulo.server.test.continuous.ContinuousVerify -libjars "$SERVER_LIBJAR" $AUTH_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $VERIFY_OUT $VERIFY_MAX_MAPS $VERIFY_REDUCERS $SCAN_OFFLINE
+if [ ! -r $ACCUMULO_CONF_DIR/accumulo-site.xml ]; then
+ echo "Could not find accumulo-site.xml in $ACCUMULO_CONF_DIR"
+ exit 1
+fi
+
+TARGET_DIR="ci-conf-`date '+%s'`"
+hadoop fs -mkdir $TARGET_DIR
+
+if [ $? -ne 0 ]; then
+ echo "Could not create $TAGET_DIR in HDFS"
+ exit 1
+fi
+
+hadoop fs -put $ACCUMULO_CONF_DIR/accumulo-site.xml ${TARGET_DIR}/
+
+if [ $? -ne 0 ]; then
+ echo "Could not upload accumulo-site.xml to HDFS"
+ exit 1
+fi
+
+ABS_DIR="/user/`whoami`/${TARGET_DIR}/accumulo-site.xml"
+
+$ACCUMULO_HOME/bin/tool.sh "$SERVER_LIBJAR" org.apache.accumulo.server.test.continuous.ContinuousVerify -libjars "$SERVER_LIBJAR" $AUTH_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $VERIFY_OUT $VERIFY_MAX_MAPS $VERIFY_REDUCERS --sitefile $ABS_DIR $SCAN_OFFLINE