You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by jw...@apache.org on 2013/02/04 17:12:26 UTC
git commit: CRUNCH-160: CrunchInputFormat needs to create a new
Configuration object for each delegate InputFormat
Updated Branches:
refs/heads/master 41f01c037 -> 6536f4c90
CRUNCH-160: CrunchInputFormat needs to create a new Configuration object for each delegate InputFormat
Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/6536f4c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/6536f4c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/6536f4c9
Branch: refs/heads/master
Commit: 6536f4c909a1b8d952f1c4398195177659d1423d
Parents: 41f01c0
Author: Josh Wills <jw...@apache.org>
Authored: Sun Feb 3 17:51:55 2013 -0800
Committer: Josh Wills <jw...@apache.org>
Committed: Sun Feb 3 17:51:55 2013 -0800
----------------------------------------------------------------------
.../crunch/impl/mr/run/CrunchInputFormat.java | 8 +++-----
1 files changed, 3 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/6536f4c9/crunch/src/main/java/org/apache/crunch/impl/mr/run/CrunchInputFormat.java
----------------------------------------------------------------------
diff --git a/crunch/src/main/java/org/apache/crunch/impl/mr/run/CrunchInputFormat.java b/crunch/src/main/java/org/apache/crunch/impl/mr/run/CrunchInputFormat.java
index d39f67f..eb5dd8a 100644
--- a/crunch/src/main/java/org/apache/crunch/impl/mr/run/CrunchInputFormat.java
+++ b/crunch/src/main/java/org/apache/crunch/impl/mr/run/CrunchInputFormat.java
@@ -42,19 +42,17 @@ public class CrunchInputFormat<K, V> extends InputFormat<K, V> {
@Override
public List<InputSplit> getSplits(JobContext job) throws IOException, InterruptedException {
List<InputSplit> splits = Lists.newArrayList();
- Configuration conf = job.getConfiguration();
+ Configuration base = job.getConfiguration();
Map<FormatBundle, Map<Integer, List<Path>>> formatNodeMap = CrunchInputs.getFormatNodeMap(job);
// First, build a map of InputFormats to Paths
for (Map.Entry<FormatBundle, Map<Integer, List<Path>>> entry : formatNodeMap.entrySet()) {
FormatBundle inputBundle = entry.getKey();
+ Configuration conf = new Configuration(base);
+ inputBundle.configure(conf);
Job jobCopy = new Job(conf);
- inputBundle.configure(jobCopy.getConfiguration());
InputFormat<?, ?> format = (InputFormat<?, ?>) ReflectionUtils.newInstance(inputBundle.getFormatClass(),
jobCopy.getConfiguration());
- if (format instanceof Configurable) {
- ((Configurable) format).setConf(jobCopy.getConfiguration());
- }
for (Map.Entry<Integer, List<Path>> nodeEntry : entry.getValue().entrySet()) {
Integer nodeIndex = nodeEntry.getKey();
List<Path> paths = nodeEntry.getValue();