You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2017/03/03 02:45:33 UTC
[22/50] [abbrv] hadoop git commit: YARN-5703. ReservationAgents are
not correctly configured. Contributed by Manikandan R.
YARN-5703. ReservationAgents are not correctly configured. Contributed by Manikandan R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f5b031d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f5b031d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f5b031d
Branch: refs/heads/YARN-5734
Commit: 5f5b031d1f20cb7f621db41979e963eaa42cf52f
Parents: 4d33683
Author: Naganarasimha <na...@apache.org>
Authored: Mon Feb 27 20:38:29 2017 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Mon Feb 27 20:38:29 2017 +0530
----------------------------------------------------------------------
.../reservation/AbstractReservationSystem.java | 8 ++++++--
.../planning/AlignedPlannerWithGreedy.java | 16 +++++++++-------
.../planning/GreedyReservationAgent.java | 19 ++++++-------------
.../reservation/planning/PlanningAlgorithm.java | 4 ++++
.../reservation/planning/ReservationAgent.java | 8 ++++++++
.../planning/TryManyReservationAgents.java | 5 ++++-
.../reservation/planning/TestAlignedPlanner.java | 3 +++
.../planning/TestGreedyReservationAgent.java | 4 ++--
8 files changed, 42 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
index 8769ca1..5ef4912 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
@@ -451,12 +451,16 @@ public abstract class AbstractReservationSystem extends AbstractService
try {
Class<?> agentClazz = conf.getClassByName(agentClassName);
if (ReservationAgent.class.isAssignableFrom(agentClazz)) {
- return (ReservationAgent) ReflectionUtils.newInstance(agentClazz, conf);
+ ReservationAgent resevertionAgent =
+ (ReservationAgent) agentClazz.newInstance();
+ resevertionAgent.init(conf);
+ return resevertionAgent;
} else {
throw new YarnRuntimeException("Class: " + agentClassName
+ " not instance of " + ReservationAgent.class.getCanonicalName());
}
- } catch (ClassNotFoundException e) {
+ } catch (ClassNotFoundException | InstantiationException
+ | IllegalAccessException e) {
throw new YarnRuntimeException("Could not instantiate Agent: "
+ agentClassName + " for queue: " + queueName, e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/AlignedPlannerWithGreedy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/AlignedPlannerWithGreedy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/AlignedPlannerWithGreedy.java
index b23cf1e..00c2333 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/AlignedPlannerWithGreedy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/AlignedPlannerWithGreedy.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning;
import java.util.LinkedList;
import java.util.List;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ReservationDefinition;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
@@ -35,22 +36,25 @@ import org.slf4j.LoggerFactory;
public class AlignedPlannerWithGreedy implements ReservationAgent {
// Default smoothness factor
- private static final int DEFAULT_SMOOTHNESS_FACTOR = 10;
+ public static final int DEFAULT_SMOOTHNESS_FACTOR = 10;
+ public static final String SMOOTHNESS_FACTOR =
+ "yarn.resourcemanager.reservation-system.smoothness-factor";
// Log
private static final Logger LOG = LoggerFactory
.getLogger(AlignedPlannerWithGreedy.class);
// Smoothness factor
- private final ReservationAgent planner;
+ private ReservationAgent planner;
// Constructor
public AlignedPlannerWithGreedy() {
- this(DEFAULT_SMOOTHNESS_FACTOR);
}
- // Constructor
- public AlignedPlannerWithGreedy(int smoothnessFactor) {
+ @Override
+ public void init(Configuration conf) {
+ int smoothnessFactor =
+ conf.getInt(SMOOTHNESS_FACTOR, DEFAULT_SMOOTHNESS_FACTOR);
// List of algorithms
List<ReservationAgent> listAlg = new LinkedList<ReservationAgent>();
@@ -71,7 +75,6 @@ public class AlignedPlannerWithGreedy implements ReservationAgent {
// 1. Attempt to execute algAligned
// 2. If failed, fall back to algGreedy
planner = new TryManyReservationAgents(listAlg);
-
}
@Override
@@ -119,5 +122,4 @@ public class AlignedPlannerWithGreedy implements ReservationAgent {
return planner.deleteReservation(reservationId, user, plan);
}
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/GreedyReservationAgent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/GreedyReservationAgent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/GreedyReservationAgent.java
index 915a834..1559b97 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/GreedyReservationAgent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/GreedyReservationAgent.java
@@ -46,25 +46,19 @@ public class GreedyReservationAgent implements ReservationAgent {
.getLogger(GreedyReservationAgent.class);
// Greedy planner
- private final ReservationAgent planner;
-
+ private ReservationAgent planner;
public final static String GREEDY_FAVOR_EARLY_ALLOCATION =
"yarn.resourcemanager.reservation-system.favor-early-allocation";
-
public final static boolean DEFAULT_GREEDY_FAVOR_EARLY_ALLOCATION = true;
-
- private final boolean allocateLeft;
+ private boolean allocateLeft;
public GreedyReservationAgent() {
- this(new Configuration());
}
- public GreedyReservationAgent(Configuration yarnConfiguration) {
-
- allocateLeft =
- yarnConfiguration.getBoolean(GREEDY_FAVOR_EARLY_ALLOCATION,
- DEFAULT_GREEDY_FAVOR_EARLY_ALLOCATION);
-
+ @Override
+ public void init(Configuration conf) {
+ allocateLeft = conf.getBoolean(GREEDY_FAVOR_EARLY_ALLOCATION,
+ DEFAULT_GREEDY_FAVOR_EARLY_ALLOCATION);
if (allocateLeft) {
LOG.info("Initializing the GreedyReservationAgent to favor \"early\""
+ " (left) allocations (controlled by parameter: "
@@ -78,7 +72,6 @@ public class GreedyReservationAgent implements ReservationAgent {
planner =
new IterativePlanner(new StageEarliestStartByJobArrival(),
new StageAllocatorGreedyRLE(allocateLeft), allocateLeft);
-
}
public boolean isAllocateLeft(){
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
index e1b508d..199bfa5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning;
import java.util.Map;
import java.util.Map.Entry;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ReservationDefinition;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -206,4 +207,7 @@ public abstract class PlanningAlgorithm implements ReservationAgent {
}
+ @Override
+ public void init(Configuration conf) {
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/ReservationAgent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/ReservationAgent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/ReservationAgent.java
index bdea2f4..52e7055 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/ReservationAgent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/ReservationAgent.java
@@ -17,6 +17,7 @@
*******************************************************************************/
package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ReservationDefinition;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
@@ -70,4 +71,11 @@ public interface ReservationAgent {
public boolean deleteReservation(ReservationId reservationId, String user,
Plan plan) throws PlanningException;
+ /**
+ * Init configuration.
+ *
+ * @param conf Configuration
+ */
+ void init(Configuration conf);
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TryManyReservationAgents.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TryManyReservationAgents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TryManyReservationAgents.java
index 1d37ce5..ab2e1e0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TryManyReservationAgents.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TryManyReservationAgents.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning;
import java.util.LinkedList;
import java.util.List;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ReservationDefinition;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
@@ -110,5 +111,7 @@ public class TryManyReservationAgents implements ReservationAgent {
return plan.deleteReservation(reservationId);
}
-
+ @Override
+ public void init(Configuration conf) {
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java
index 4b01eb9..2645366 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java
@@ -726,8 +726,11 @@ public class TestAlignedPlanner {
QueueMetrics queueMetrics = mock(QueueMetrics.class);
RMContext context = ReservationSystemTestUtil.createMockRMContext();
+ conf.setInt(AlignedPlannerWithGreedy.SMOOTHNESS_FACTOR,
+ AlignedPlannerWithGreedy.DEFAULT_SMOOTHNESS_FACTOR);
// Set planning agent
agent = new AlignedPlannerWithGreedy();
+ agent.init(conf);
// Create Plan
plan =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f5b031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java
index ec11ffe..6d1cfa8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java
@@ -110,8 +110,8 @@ public class TestGreedyReservationAgent {
// setting conf to
conf.setBoolean(GreedyReservationAgent.GREEDY_FAVOR_EARLY_ALLOCATION,
allocateLeft);
-
- agent = new GreedyReservationAgent(conf);
+ agent = new GreedyReservationAgent();
+ agent.init(conf);
QueueMetrics queueMetrics = mock(QueueMetrics.class);
RMContext context = ReservationSystemTestUtil.createMockRMContext();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org