You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2016/09/08 06:35:58 UTC
[08/44] hadoop git commit: YARN-5616. Clean up WeightAdjuster. (Yufei
Gu via kasha)
YARN-5616. Clean up WeightAdjuster. (Yufei Gu via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c0e492e5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c0e492e5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c0e492e5
Branch: refs/heads/YARN-3368
Commit: c0e492e50fa98d423c8a61c94d2d1f9553558b6d
Parents: 680be58
Author: Karthik Kambatla <ka...@apache.org>
Authored: Tue Sep 6 22:40:20 2016 -0700
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Tue Sep 6 22:40:20 2016 -0700
----------------------------------------------------------------------
.../scheduler/fair/FairScheduler.java | 5 --
.../scheduler/fair/NewAppWeightBooster.java | 60 --------------------
.../scheduler/fair/WeightAdjuster.java | 36 ------------
3 files changed, 101 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0e492e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index ac8dbe1..15e2318 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -186,7 +186,6 @@ public class FairScheduler extends
// an app can be reserved on
protected boolean sizeBasedWeight; // Give larger weights to larger jobs
- protected WeightAdjuster weightAdjuster; // Can be null for no weight adjuster
protected boolean continuousSchedulingEnabled; // Continuous Scheduling enabled or not
protected int continuousSchedulingSleepMs; // Sleep time for each pass in continuous scheduling
private Comparator<FSSchedulerNode> nodeAvailableResourceComparator =
@@ -563,10 +562,6 @@ public class FairScheduler extends
weight = Math.log1p(app.getDemand().getMemorySize()) / Math.log(2);
}
weight *= app.getPriority().getPriority();
- if (weightAdjuster != null) {
- // Run weight through the user-supplied weightAdjuster
- weight = weightAdjuster.adjustWeight(app, weight);
- }
ResourceWeights resourceWeights = app.getResourceWeights();
resourceWeights.setWeight((float)weight);
return resourceWeights;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0e492e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java
deleted file mode 100644
index fb32e56..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-
-/**
- * A {@link WeightAdjuster} implementation that gives a weight boost to new jobs
- * for a certain amount of time -- by default, a 3x weight boost for 60 seconds.
- * This can be used to make shorter jobs finish faster, emulating Shortest Job
- * First scheduling while not starving long jobs.
- */
-@Private
-@Unstable
-public class NewAppWeightBooster extends Configured implements WeightAdjuster {
- private static final float DEFAULT_FACTOR = 3;
- private static final long DEFAULT_DURATION = 5 * 60 * 1000;
-
- private float factor;
- private long duration;
-
- public void setConf(Configuration conf) {
- if (conf != null) {
- factor = conf.getFloat("mapred.newjobweightbooster.factor",
- DEFAULT_FACTOR);
- duration = conf.getLong("mapred.newjobweightbooster.duration",
- DEFAULT_DURATION);
- }
- super.setConf(conf);
- }
-
- public double adjustWeight(FSAppAttempt app, double curWeight) {
- long start = app.getStartTime();
- long now = System.currentTimeMillis();
- if (now - start < duration) {
- return curWeight * factor;
- } else {
- return curWeight;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0e492e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java
deleted file mode 100644
index 67364ed..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configurable;
-
-/**
- * A pluggable object for altering the weights of apps in the fair scheduler,
- * which is used for example by {@link NewAppWeightBooster} to give higher
- * weight to new jobs so that short jobs finish faster.
- *
- * May implement {@link Configurable} to access configuration parameters.
- */
-@Private
-@Unstable
-public interface WeightAdjuster {
- public double adjustWeight(FSAppAttempt app, double curWeight);
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org