You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/12/19 21:45:56 UTC
[1/3] accumulo git commit: ACCUMULO-2591 Ensure binary compat of
TeraSortIngest across Hadoop 1/2
Repository: accumulo
Updated Branches:
refs/heads/1.6 273491784 -> 9a32ec116
refs/heads/master 999115eff -> d798d5121
ACCUMULO-2591 Ensure binary compat of TeraSortIngest across Hadoop 1/2
Use the already present reflection code to access the JobContext
in a way compatible with both hadoop 1 and 2.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9a32ec11
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9a32ec11
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9a32ec11
Branch: refs/heads/1.6
Commit: 9a32ec1169035e49f65ec919f0057a9672a975eb
Parents: 2734917
Author: Josh Elser <el...@apache.org>
Authored: Fri Dec 19 15:44:23 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Fri Dec 19 15:44:23 2014 -0500
----------------------------------------------------------------------
.../accumulo/core/client/mapreduce/AbstractInputFormat.java | 2 +-
.../accumulo/examples/simple/mapreduce/TeraSortIngest.java | 5 +++--
2 files changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a32ec11/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
index 19a674f..e42c3e6 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
@@ -676,7 +676,7 @@ public abstract class AbstractInputFormat<K,V> extends InputFormat<K,V> {
}
// use reflection to pull the Configuration out of the JobContext for Hadoop 1 and Hadoop 2 compatibility
- static Configuration getConfiguration(JobContext context) {
+ public static Configuration getConfiguration(JobContext context) {
try {
Class<?> c = AbstractInputFormat.class.getClassLoader().loadClass("org.apache.hadoop.mapreduce.JobContext");
Method m = c.getMethod("getConfiguration");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a32ec11/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java
index fe71b85..f10ec3e 100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java
@@ -28,6 +28,7 @@ import java.util.Random;
import org.apache.accumulo.core.cli.ClientOnRequiredTable;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
+import org.apache.accumulo.core.client.mapreduce.InputFormatBase;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.hadoop.conf.Configuration;
@@ -165,8 +166,8 @@ public class TeraSortIngest extends Configured implements Tool {
*/
@Override
public List<InputSplit> getSplits(JobContext job) {
- long totalRows = job.getConfiguration().getLong(NUMROWS, 0);
- int numSplits = job.getConfiguration().getInt(NUMSPLITS, 1);
+ long totalRows = InputFormatBase.getConfiguration(job).getLong(NUMROWS, 0);
+ int numSplits = InputFormatBase.getConfiguration(job).getInt(NUMSPLITS, 1);
long rowsPerSplit = totalRows / numSplits;
System.out.println("Generating " + totalRows + " using " + numSplits + " maps with step of " + rowsPerSplit);
ArrayList<InputSplit> splits = new ArrayList<InputSplit>(numSplits);
[3/3] accumulo git commit: Merge branch '1.6'
Posted by el...@apache.org.
Merge branch '1.6'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d798d512
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d798d512
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d798d512
Branch: refs/heads/master
Commit: d798d51211857a1e6d7390e04ee903ca8042fd31
Parents: 999115e 9a32ec1
Author: Josh Elser <el...@apache.org>
Authored: Fri Dec 19 15:45:45 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Fri Dec 19 15:45:45 2014 -0500
----------------------------------------------------------------------
.../accumulo/core/client/mapreduce/AbstractInputFormat.java | 2 +-
.../accumulo/examples/simple/mapreduce/TeraSortIngest.java | 5 +++--
2 files changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d798d512/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d798d512/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java
----------------------------------------------------------------------
diff --cc examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java
index afbc4b0,f10ec3e..ade6ce1
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java
@@@ -25,9 -25,10 +25,10 @@@ import java.util.ArrayList
import java.util.List;
import java.util.Random;
-import org.apache.accumulo.core.cli.ClientOnRequiredTable;
+import org.apache.accumulo.core.cli.MapReduceClientOnRequiredTable;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
+ import org.apache.accumulo.core.client.mapreduce.InputFormatBase;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.hadoop.conf.Configuration;
[2/3] accumulo git commit: ACCUMULO-2591 Ensure binary compat of
TeraSortIngest across Hadoop 1/2
Posted by el...@apache.org.
ACCUMULO-2591 Ensure binary compat of TeraSortIngest across Hadoop 1/2
Use the already present reflection code to access the JobContext
in a way compatible with both hadoop 1 and 2.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9a32ec11
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9a32ec11
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9a32ec11
Branch: refs/heads/master
Commit: 9a32ec1169035e49f65ec919f0057a9672a975eb
Parents: 2734917
Author: Josh Elser <el...@apache.org>
Authored: Fri Dec 19 15:44:23 2014 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Fri Dec 19 15:44:23 2014 -0500
----------------------------------------------------------------------
.../accumulo/core/client/mapreduce/AbstractInputFormat.java | 2 +-
.../accumulo/examples/simple/mapreduce/TeraSortIngest.java | 5 +++--
2 files changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a32ec11/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
index 19a674f..e42c3e6 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AbstractInputFormat.java
@@ -676,7 +676,7 @@ public abstract class AbstractInputFormat<K,V> extends InputFormat<K,V> {
}
// use reflection to pull the Configuration out of the JobContext for Hadoop 1 and Hadoop 2 compatibility
- static Configuration getConfiguration(JobContext context) {
+ public static Configuration getConfiguration(JobContext context) {
try {
Class<?> c = AbstractInputFormat.class.getClassLoader().loadClass("org.apache.hadoop.mapreduce.JobContext");
Method m = c.getMethod("getConfiguration");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/9a32ec11/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java
index fe71b85..f10ec3e 100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java
@@ -28,6 +28,7 @@ import java.util.Random;
import org.apache.accumulo.core.cli.ClientOnRequiredTable;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat;
+import org.apache.accumulo.core.client.mapreduce.InputFormatBase;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.hadoop.conf.Configuration;
@@ -165,8 +166,8 @@ public class TeraSortIngest extends Configured implements Tool {
*/
@Override
public List<InputSplit> getSplits(JobContext job) {
- long totalRows = job.getConfiguration().getLong(NUMROWS, 0);
- int numSplits = job.getConfiguration().getInt(NUMSPLITS, 1);
+ long totalRows = InputFormatBase.getConfiguration(job).getLong(NUMROWS, 0);
+ int numSplits = InputFormatBase.getConfiguration(job).getInt(NUMSPLITS, 1);
long rowsPerSplit = totalRows / numSplits;
System.out.println("Generating " + totalRows + " using " + numSplits + " maps with step of " + rowsPerSplit);
ArrayList<InputSplit> splits = new ArrayList<InputSplit>(numSplits);