You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by jw...@apache.org on 2017/03/24 05:18:11 UTC
crunch git commit: CRUNCH-636: amend Make replication factor for
temporary files configurable
Repository: crunch
Updated Branches:
refs/heads/master e176b6166 -> ca8ff16b6
CRUNCH-636: amend Make replication factor for temporary files configurable
Signed-off-by: Josh Wills <jw...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/ca8ff16b
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/ca8ff16b
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/ca8ff16b
Branch: refs/heads/master
Commit: ca8ff16b6172be6cf2be1c185c41593ce4897c53
Parents: e176b61
Author: Attila Sasvari <as...@cloudera.com>
Authored: Thu Mar 23 21:35:36 2017 +0100
Committer: Josh Wills <jw...@apache.org>
Committed: Thu Mar 23 16:22:11 2017 -0700
----------------------------------------------------------------------
.../main/java/org/apache/crunch/impl/mr/plan/JobPrototype.java | 4 ++--
.../java/org/apache/crunch/impl/mr/plan/JobPrototypeTest.java | 5 ++---
2 files changed, 4 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/crunch/blob/ca8ff16b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/JobPrototype.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/JobPrototype.java b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/JobPrototype.java
index d31bfad..7705d4a 100644
--- a/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/JobPrototype.java
+++ b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/JobPrototype.java
@@ -329,8 +329,8 @@ class JobPrototype {
return d;
}
- private void setInitialJobReplicationConfig(Configuration job, String defaultReplication) {
- job.set(DFS_REPLICATION_INITIAL, defaultReplication);
+ private void setInitialJobReplicationConfig(Configuration jobConf, String defaultReplication) {
+ jobConf.set(DFS_REPLICATION_INITIAL, defaultReplication);
}
private static CrunchControlledJob.Hook getHook(
http://git-wip-us.apache.org/repos/asf/crunch/blob/ca8ff16b/crunch-core/src/test/java/org/apache/crunch/impl/mr/plan/JobPrototypeTest.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/test/java/org/apache/crunch/impl/mr/plan/JobPrototypeTest.java b/crunch-core/src/test/java/org/apache/crunch/impl/mr/plan/JobPrototypeTest.java
index 44da01a..e9847c7 100644
--- a/crunch-core/src/test/java/org/apache/crunch/impl/mr/plan/JobPrototypeTest.java
+++ b/crunch-core/src/test/java/org/apache/crunch/impl/mr/plan/JobPrototypeTest.java
@@ -48,7 +48,6 @@ public class JobPrototypeTest {
@Mock private Path mockPath;
@Mock private FileTargetImpl mockTarget;
@Mock private FileSystem mockFs;
- @Mock private DoNode mockNode;
@Mock private PGroupedTableImpl<String, String> mockPgroup;
@Mock private Set<NodePath> mockInputs;
private JobPrototype jobPrototypeUnderTest;
@@ -86,7 +85,6 @@ public class JobPrototypeTest {
@Test
public void initialReplicationFactorUsedFromFileSystem() throws IOException {
- testConfiguration = new Configuration();
HashMultimap<Target, NodePath> targetNodePaths = HashMultimap.create();
targetNodePaths.put(mockTarget, new NodePath());
doReturn(mockPath).when(mockTarget).getPath();
@@ -95,6 +93,7 @@ public class JobPrototypeTest {
c.set("dfs.replication", TEST_INITIAL_DFS_REPLICATION);
doReturn(c).when(mockFs).getConf();
jobPrototypeUnderTest.addReducePaths(targetNodePaths);
+ testConfiguration = new Configuration(false);
jobPrototypeUnderTest.handleInitialReplication(testConfiguration);
assertEquals(TEST_INITIAL_DFS_REPLICATION, testConfiguration.get("dfs.replication.initial"));
@@ -102,7 +101,6 @@ public class JobPrototypeTest {
@Test
public void initialReplicationFactorUsedWhenItCannotBeRetrievedFromFileSystem() throws IOException {
- testConfiguration = new Configuration();
HashMultimap<Target, NodePath> targetNodePaths = HashMultimap.create();
targetNodePaths.put(mockTarget, new NodePath());
doReturn(mockPath).when(mockTarget).getPath();
@@ -110,6 +108,7 @@ public class JobPrototypeTest {
Configuration c = new Configuration();
c.set("dfs.replication", TEST_INITIAL_DFS_REPLICATION);
jobPrototypeUnderTest.addReducePaths(targetNodePaths);
+ testConfiguration = new Configuration(false);
jobPrototypeUnderTest.handleInitialReplication(testConfiguration);
assertEquals("3", testConfiguration.get("dfs.replication.initial")); //default