You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/01/24 03:15:37 UTC
[04/21] git commit: Revert "ACCUMULO-2234 Provide accumulo-site.xml
to Mapper classpath and ensure is used by concrete Instance."
Revert "ACCUMULO-2234 Provide accumulo-site.xml to Mapper classpath and ensure is used by concrete Instance."
This reverts commit 57f9b6cfd30b1b2505efbacd2a5ce391dbcd1e0c.
Conflicts:
src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/66516a0f
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/66516a0f
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/66516a0f
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 66516a0f8900f228294cb8d818e350887937b079
Parents: 9cf94f9
Author: Josh Elser <el...@apache.org>
Authored: Thu Jan 23 17:51:05 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Thu Jan 23 17:51:05 2014 -0500
----------------------------------------------------------------------
.../core/client/mapreduce/InputFormatBase.java | 11 +--------
.../core/client/mapreduce/RangeInputSplit.java | 11 +--------
.../test/continuous/ContinuousVerify.java | 24 ++------------------
test/system/continuous/run-verify.sh | 24 +-------------------
4 files changed, 5 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/66516a0f/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
index 705ddbd..de97f12 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
@@ -58,8 +58,6 @@ import org.apache.accumulo.core.client.impl.Tables;
import org.apache.accumulo.core.client.impl.TabletLocator;
import org.apache.accumulo.core.client.mock.MockInstance;
import org.apache.accumulo.core.client.mock.MockTabletLocator;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.PartialKey;
@@ -737,14 +735,7 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
protected static Instance getInstance(Configuration conf) {
if (conf.getBoolean(MOCK, false))
return new MockInstance(conf.get(INSTANCE_NAME));
-
- ZooKeeperInstance zki = new ZooKeeperInstance(conf.get(INSTANCE_NAME), conf.get(ZOOKEEPERS));
-
- // Wrap the DefaultConfiguration with a SiteConfiguration
- AccumuloConfiguration xmlConfig = SiteConfiguration.getInstance(zki.getConfiguration());
- zki.setConfiguration(xmlConfig);
-
- return zki;
+ return new ZooKeeperInstance(conf.get(INSTANCE_NAME), conf.get(ZOOKEEPERS));
}
/**
http://git-wip-us.apache.org/repos/asf/accumulo/blob/66516a0f/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
----------------------------------------------------------------------
diff --git a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
index e1107cc..01921c4 100644
--- a/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
+++ b/src/core/src/main/java/org/apache/accumulo/core/client/mapreduce/RangeInputSplit.java
@@ -29,8 +29,6 @@ import org.apache.accumulo.core.client.ZooKeeperInstance;
import org.apache.accumulo.core.client.mapreduce.InputFormatBase.AccumuloIterator;
import org.apache.accumulo.core.client.mapreduce.InputFormatBase.AccumuloIteratorOption;
import org.apache.accumulo.core.client.mock.MockInstance;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.PartialKey;
@@ -356,14 +354,7 @@ public class RangeInputSplit extends InputSplit implements Writable {
return null;
}
- ZooKeeperInstance zki = new ZooKeeperInstance(getInstanceName(), getZooKeepers());
-
- // Wrap the DefaultConfiguration with a SiteConfiguration so we use accumulo-site.xml
- // when it's present
- AccumuloConfiguration xmlConfig = SiteConfiguration.getInstance(zki.getConfiguration());
- zki.setConfiguration(xmlConfig);
-
- return zki;
+ return new ZooKeeperInstance(getInstanceName(), getZooKeepers());
}
public String getInstanceName() {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/66516a0f/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java b/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
index 4a5c0cf..6165d2a 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/test/continuous/ContinuousVerify.java
@@ -17,7 +17,6 @@
package org.apache.accumulo.server.test.continuous;
import java.io.IOException;
-import java.net.URI;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -35,8 +34,6 @@ import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.server.test.continuous.ContinuousWalk.BadChecksumException;
import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.filecache.DistributedCache;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
@@ -47,14 +44,12 @@ import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
-import org.apache.log4j.Logger;
/**
* A map reduce job that verifies a table created by continuous ingest. It verifies that all referenced nodes are defined.
*/
public class ContinuousVerify extends Configured implements Tool {
- private static final Logger log = Logger.getLogger(ContinuousVerify.class);
public static final VLongWritable DEF = new VLongWritable(-1);
@@ -155,9 +150,9 @@ public class ContinuousVerify extends Configured implements Tool {
args = argsList.toArray(new String[0]);
- if (args.length != 10) {
+ if (args.length != 9) {
throw new IllegalArgumentException("Usage : " + ContinuousVerify.class.getName()
- + " <instance name> <zookeepers> <user> <pass> <table> <output dir> <max mappers> <num reducers> <scan offline> <sitexml>");
+ + " <instance name> <zookeepers> <user> <pass> <table> <output dir> <max mappers> <num reducers> <scan offline>");
}
String instance = args[0];
@@ -169,7 +164,6 @@ public class ContinuousVerify extends Configured implements Tool {
String maxMaps = args[6];
String reducers = args[7];
boolean scanOffline = Boolean.parseBoolean(args[8]);
- String siteFile = args[9];
Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
job.setJarByClass(this.getClass());
@@ -217,20 +211,6 @@ public class ContinuousVerify extends Configured implements Tool {
job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", scanOffline);
- Path sitePath = new Path(siteFile);
- Path siteParentPath = sitePath.getParent();
- if (null == siteParentPath) {
- siteParentPath = new Path("/");
- }
-
- URI siteUri = new URI("hdfs://" + siteFile);
-
- log.info("Adding " + siteUri + " to DistributedCache");
-
- // Make sure that accumulo-site.xml is available for mappers running offline scans
- // as they need to correctly choose instance.dfs.dir for the installation
- DistributedCache.addFileToClassPath(siteParentPath, job.getConfiguration(), FileSystem.get(siteUri, job.getConfiguration()));
-
TextOutputFormat.setOutputPath(job, new Path(outputdir));
job.waitForCompletion(true);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/66516a0f/test/system/continuous/run-verify.sh
----------------------------------------------------------------------
diff --git a/test/system/continuous/run-verify.sh b/test/system/continuous/run-verify.sh
index f2b7a25..edf58b7 100755
--- a/test/system/continuous/run-verify.sh
+++ b/test/system/continuous/run-verify.sh
@@ -24,27 +24,5 @@ if [ -n "$VERIFY_AUTHS" ] ; then
AUTH_OPT="--auths $VERIFY_AUTHS";
fi
-if [ ! -r $ACCUMULO_CONF_DIR/accumulo-site.xml ]; then
- echo "Could not find accumulo-site.xml in $ACCUMULO_CONF_DIR"
- exit 1
-fi
-
-TARGET_DIR="ci-conf-`date '+%s'`"
-hadoop fs -mkdir $TARGET_DIR
-
-if [ $? -ne 0 ]; then
- echo "Could not create $TAGET_DIR in HDFS"
- exit 1
-fi
-
-hadoop fs -put $ACCUMULO_CONF_DIR/accumulo-site.xml ${TARGET_DIR}/
-
-if [ $? -ne 0 ]; then
- echo "Could not upload accumulo-site.xml to HDFS"
- exit 1
-fi
-
-ABS_DIR="/user/`whoami`/${TARGET_DIR}/accumulo-site.xml"
-
-$ACCUMULO_HOME/bin/tool.sh "$SERVER_LIBJAR" org.apache.accumulo.server.test.continuous.ContinuousVerify -libjars "$SERVER_LIBJAR" $AUTH_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $VERIFY_OUT $VERIFY_MAX_MAPS $VERIFY_REDUCERS --sitefile $ABS_DIR $SCAN_OFFLINE
+$ACCUMULO_HOME/bin/tool.sh "$SERVER_LIBJAR" org.apache.accumulo.server.test.continuous.ContinuousVerify -libjars "$SERVER_LIBJAR" $AUTH_OPT $INSTANCE_NAME $ZOO_KEEPERS $USER $PASS $TABLE $VERIFY_OUT $VERIFY_MAX_MAPS $VERIFY_REDUCERS $SCAN_OFFLINE