You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by md...@apache.org on 2014/03/29 05:50:28 UTC
[1/3] git commit: ACCUMULO-2566 Fix botched merge
Repository: accumulo
Updated Branches:
refs/heads/1.5.2-SNAPSHOT 0d2cd1c06 -> 34a44e700
refs/heads/1.6.0-SNAPSHOT 3354b92af -> 9843911d9
ACCUMULO-2566 Fix botched merge
Fixing a bad merge for TeraSortIngest using reflection to get hadoop
configuration objects. Copied the code from InputFormatBase because
the method we need is not visible and adding it to the public API would
be a terrible, terrible thing to do.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/34a44e70
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/34a44e70
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/34a44e70
Branch: refs/heads/1.5.2-SNAPSHOT
Commit: 34a44e700312a01822e046c89dd892616ef92e42
Parents: 0d2cd1c
Author: Mike Drob <md...@cloudera.com>
Authored: Sat Mar 29 00:48:19 2014 -0400
Committer: Mike Drob <md...@cloudera.com>
Committed: Sat Mar 29 00:48:19 2014 -0400
----------------------------------------------------------------------
.../simple/mapreduce/TeraSortIngest.java | 27 ++++++++++++++++++--
1 file changed, 25 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/34a44e70/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java
index f131e6c..bdcf943 100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java
@@ -21,6 +21,7 @@ package org.apache.accumulo.examples.simple.mapreduce;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
@@ -166,8 +167,8 @@ public class TeraSortIngest extends Configured implements Tool {
*/
@Override
public List<InputSplit> getSplits(JobContext job) {
- long totalRows = InputFormatBase.getConfiguration(job).getLong(NUMROWS, 0);
- int numSplits = InputFormatBase.getConfiguration(job).getInt(NUMSPLITS, 1);
+ long totalRows = getConfiguration(job).getLong(NUMROWS, 0);
+ int numSplits = getConfiguration(job).getInt(NUMSPLITS, 1);
long rowsPerSplit = totalRows / numSplits;
System.out.println("Generating " + totalRows + " using " + numSplits + " maps with step of " + rowsPerSplit);
ArrayList<InputSplit> splits = new ArrayList<InputSplit>(numSplits);
@@ -181,6 +182,28 @@ public class TeraSortIngest extends Configured implements Tool {
return splits;
}
+ // TODO kill this when we drop Hadoop 1 support.
+ private static final Method GET_CONFIGURATION;
+ static {
+ try {
+ GET_CONFIGURATION = Class.forName("org.apache.hadoop.mapreduce.JobContext").getMethod("getConfiguration");
+ } catch (SecurityException e) {
+ throw new RuntimeException("Could not get method.", e);
+ } catch (NoSuchMethodException e) {
+ throw new RuntimeException("Could not get method.", e);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException("Could not get JobContext declaration.", e);
+ }
+ }
+
+ // This is copied from InputFormatBase because the implementation there is not public and we don not really want cruft like this in the public API.
+ private static Configuration getConfiguration(JobContext job) {
+ try {
+ return (Configuration) GET_CONFIGURATION.invoke(job, new Object[0]);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
}
private static String NUMSPLITS = "terasort.overridesplits";
[3/3] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Posted by md...@apache.org.
Merge branch '1.5.2-SNAPSHOT' into 1.6.0-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9843911d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9843911d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9843911d
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 9843911d939bafc2808f3edb30903881cb7fa04d
Parents: 3354b92 34a44e7
Author: Mike Drob <md...@cloudera.com>
Authored: Sat Mar 29 00:49:59 2014 -0400
Committer: Mike Drob <md...@cloudera.com>
Committed: Sat Mar 29 00:49:59 2014 -0400
----------------------------------------------------------------------
----------------------------------------------------------------------
[2/3] git commit: ACCUMULO-2566 Fix botched merge
Posted by md...@apache.org.
ACCUMULO-2566 Fix botched merge
Fixing a bad merge for TeraSortIngest using reflection to get hadoop
configuration objects. Copied the code from InputFormatBase because
the method we need is not visible and adding it to the public API would
be a terrible, terrible thing to do.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/34a44e70
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/34a44e70
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/34a44e70
Branch: refs/heads/1.6.0-SNAPSHOT
Commit: 34a44e700312a01822e046c89dd892616ef92e42
Parents: 0d2cd1c
Author: Mike Drob <md...@cloudera.com>
Authored: Sat Mar 29 00:48:19 2014 -0400
Committer: Mike Drob <md...@cloudera.com>
Committed: Sat Mar 29 00:48:19 2014 -0400
----------------------------------------------------------------------
.../simple/mapreduce/TeraSortIngest.java | 27 ++++++++++++++++++--
1 file changed, 25 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/34a44e70/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java
----------------------------------------------------------------------
diff --git a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java
index f131e6c..bdcf943 100644
--- a/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java
+++ b/examples/simple/src/main/java/org/apache/accumulo/examples/simple/mapreduce/TeraSortIngest.java
@@ -21,6 +21,7 @@ package org.apache.accumulo.examples.simple.mapreduce;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
@@ -166,8 +167,8 @@ public class TeraSortIngest extends Configured implements Tool {
*/
@Override
public List<InputSplit> getSplits(JobContext job) {
- long totalRows = InputFormatBase.getConfiguration(job).getLong(NUMROWS, 0);
- int numSplits = InputFormatBase.getConfiguration(job).getInt(NUMSPLITS, 1);
+ long totalRows = getConfiguration(job).getLong(NUMROWS, 0);
+ int numSplits = getConfiguration(job).getInt(NUMSPLITS, 1);
long rowsPerSplit = totalRows / numSplits;
System.out.println("Generating " + totalRows + " using " + numSplits + " maps with step of " + rowsPerSplit);
ArrayList<InputSplit> splits = new ArrayList<InputSplit>(numSplits);
@@ -181,6 +182,28 @@ public class TeraSortIngest extends Configured implements Tool {
return splits;
}
+ // TODO kill this when we drop Hadoop 1 support.
+ private static final Method GET_CONFIGURATION;
+ static {
+ try {
+ GET_CONFIGURATION = Class.forName("org.apache.hadoop.mapreduce.JobContext").getMethod("getConfiguration");
+ } catch (SecurityException e) {
+ throw new RuntimeException("Could not get method.", e);
+ } catch (NoSuchMethodException e) {
+ throw new RuntimeException("Could not get method.", e);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException("Could not get JobContext declaration.", e);
+ }
+ }
+
+ // This is copied from InputFormatBase because the implementation there is not public and we don not really want cruft like this in the public API.
+ private static Configuration getConfiguration(JobContext job) {
+ try {
+ return (Configuration) GET_CONFIGURATION.invoke(job, new Object[0]);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
}
private static String NUMSPLITS = "terasort.overridesplits";