You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2019/11/12 19:34:46 UTC
[hadoop] branch trunk updated: HADOOP-16656. Document FairCallQueue
configs in core-default.xml. Contributed by Siyao Meng.
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 3ab98ee HADOOP-16656. Document FairCallQueue configs in core-default.xml. Contributed by Siyao Meng.
3ab98ee is described below
commit 3ab98eefa54566e221b4b0f6416dd7065f5fd281
Author: Siyao Meng <sm...@cloudera.com>
AuthorDate: Tue Nov 12 11:33:29 2019 -0800
HADOOP-16656. Document FairCallQueue configs in core-default.xml. Contributed by Siyao Meng.
Signed-off-by: Wei-Chiu Chuang <we...@apache.org>
---
.../src/main/resources/core-default.xml | 187 +++++++++++++++++++++
.../hadoop/conf/TestCommonConfigurationFields.java | 24 +++
2 files changed, 211 insertions(+)
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index cf7a251..b1c7be8 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -2273,6 +2273,193 @@
because the server side is stuck in TIME_WAIT state.
</description>
</property>
+
+<!-- FairCallQueue properties -->
+<!-- See FairCallQueue documentation for a table of all properties -->
+
+<!-- [port_number] is the port used by the IPC server to be configured. -->
+<!-- For example, ipc.8020.callqueue.impl will adjust the call queue -->
+<!-- implementation for the IPC server running at port 8020. -->
+
+<!-- Typically, [port_number] is configured to be the NameNode RPC port, -->
+<!-- i.e. port number in dfs.namenode.rpc-address, or port number in -->
+<!-- fs.defaultFS if dfs.namenode.rpc-address is not explicitly configured. -->
+<property>
+ <name>ipc.[port_number].backoff.enable</name>
+ <value>false</value>
+ <description>Whether or not to enable client backoff when a queue is full.
+ </description>
+</property>
+
+<property>
+ <name>ipc.[port_number].callqueue.impl</name>
+ <value>java.util.concurrent.LinkedBlockingQueue</value>
+ <description>The fully qualified name of a class to use as the implementation
+ of a call queue. The default implementation is
+ java.util.concurrent.LinkedBlockingQueue (FIFO queue).
+ Use org.apache.hadoop.ipc.FairCallQueue for the Fair Call Queue.
+ </description>
+</property>
+
+<property>
+ <name>ipc.[port_number].scheduler.impl</name>
+ <value>org.apache.hadoop.ipc.DefaultRpcScheduler</value>
+ <description>The fully qualified name of a class to use as the
+ implementation of the scheduler. The default implementation is
+ org.apache.hadoop.ipc.DefaultRpcScheduler (no-op scheduler) when not using
+ FairCallQueue. If using FairCallQueue, defaults to
+ org.apache.hadoop.ipc.DecayRpcScheduler. Use
+ org.apache.hadoop.ipc.DecayRpcScheduler in conjunction with the Fair Call
+ Queue.
+ </description>
+</property>
+
+<property>
+ <name>ipc.[port_number].scheduler.priority.levels</name>
+ <value>4</value>
+ <description>How many priority levels to use within the scheduler and call
+ queue. This property applies to RpcScheduler and CallQueue.
+ </description>
+</property>
+
+<property>
+ <name>ipc.[port_number].faircallqueue.multiplexer.weights</name>
+ <value>8,4,2,1</value>
+ <description>How much weight to give to each priority queue. This should be
+ a comma-separated list of length equal to the number of priority levels.
+ Weights descend by a factor of 2 (e.g., for 4 levels: 8,4,2,1).
+ This property applies to WeightedRoundRobinMultiplexer.
+ </description>
+</property>
+
+<property>
+ <name>ipc.[port_number].identity-provider.impl</name>
+ <value>org.apache.hadoop.ipc.UserIdentityProvider</value>
+ <description>The identity provider mapping user requests to their identity.
+ This property applies to DecayRpcScheduler.
+ </description>
+</property>
+
+<property>
+ <name>ipc.[port_number].cost-provider.impl</name>
+ <value>org.apache.hadoop.ipc.DefaultCostProvider</value>
+ <description>The cost provider mapping user requests to their cost. To
+ enable determination of cost based on processing time, use
+ org.apache.hadoop.ipc.WeightedTimeCostProvider.
+ This property applies to DecayRpcScheduler.
+ </description>
+</property>
+
+<property>
+ <name>ipc.[port_number].decay-scheduler.period-ms</name>
+ <value>5000</value>
+ <description>How frequently the decay factor should be applied to the
+ operation counts of users. Higher values have less overhead, but respond
+ less quickly to changes in client behavior.
+ This property applies to DecayRpcScheduler.
+ </description>
+</property>
+
+<property>
+ <name>ipc.[port_number].decay-scheduler.decay-factor</name>
+ <value>0.5</value>
+ <description>When decaying the operation counts of users, the multiplicative
+ decay factor to apply. Higher values will weight older operations more
+ strongly, essentially giving the scheduler a longer memory, and penalizing
+ heavy clients for a longer period of time.
+ This property applies to DecayRpcScheduler.
+ </description>
+</property>
+
+<property>
+ <name>ipc.[port_number].decay-scheduler.thresholds</name>
+ <value>13,25,50</value>
+ <description>The client load threshold, as an integer percentage, for each
+ priority queue. Clients producing less load, as a percent of total
+ operations, than specified at position i will be given priority i. This
+ should be a comma-separated list of length equal to the number of priority
+ levels minus 1 (the last is implicitly 100).
+ Thresholds ascend by a factor of 2 (e.g., for 4 levels: 13,25,50).
+ This property applies to DecayRpcScheduler.
+ </description>
+</property>
+
+<property>
+ <name>ipc.[port_number].decay-scheduler.backoff.responsetime.enable</name>
+ <value>false</value>
+ <description>Whether or not to enable the backoff by response time feature.
+ This property applies to DecayRpcScheduler.
+ </description>
+</property>
+
+<property>
+ <name>ipc.[port_number].decay-scheduler.backoff.responsetime.thresholds</name>
+ <value>10s,20s,30s,40s</value>
+ <description>The response time thresholds, as time durations, for each
+ priority queue. If the average response time for a queue is above this
+ threshold, backoff will occur in lower priority queues. This should be a
+ comma-separated list of length equal to the number of priority levels.
+ Threshold increases by 10s per level (e.g., for 4 levels: 10s,20s,30s,40s)
+ This property applies to DecayRpcScheduler.
+ </description>
+</property>
+
+<property>
+ <name>ipc.[port_number].decay-scheduler.metrics.top.user.count</name>
+ <value>10</value>
+ <description>The number of top (i.e., heaviest) users to emit metric
+ information about. This property applies to DecayRpcScheduler.
+ </description>
+</property>
+
+<property>
+ <name>ipc.[port_number].weighted-cost.lockshared</name>
+ <value>10</value>
+ <description>The weight multiplier to apply to the time spent in the
+ processing phase which holds a shared (read) lock.
+ This property applies to WeightedTimeCostProvider.
+ </description>
+</property>
+
+<property>
+ <name>ipc.[port_number].weighted-cost.lockexclusive</name>
+ <value>100</value>
+ <description>The weight multiplier to apply to the time spent in the
+ processing phase which holds an exclusive (write) lock.
+ This property applies to WeightedTimeCostProvider.
+ </description>
+</property>
+
+<property>
+ <name>ipc.[port_number].weighted-cost.handler</name>
+ <value>1</value>
+ <description>The weight multiplier to apply to the time spent in the
+ HANDLER phase which do not involve holding a lock.
+ See org.apache.hadoop.ipc.ProcessingDetails.Timing for more details on
+ this phase. This property applies to WeightedTimeCostProvider.
+ </description>
+</property>
+
+<property>
+ <name>ipc.[port_number].weighted-cost.lockfree</name>
+ <value>1</value>
+ <description>The weight multiplier to apply to the time spent in the
+ LOCKFREE phase which do not involve holding a lock.
+ See org.apache.hadoop.ipc.ProcessingDetails.Timing for more details on
+ this phase. This property applies to WeightedTimeCostProvider.
+ </description>
+</property>
+
+<property>
+ <name>ipc.[port_number].weighted-cost.response</name>
+ <value>1</value>
+ <description>The weight multiplier to apply to the time spent in the
+ RESPONSE phase which do not involve holding a lock.
+ See org.apache.hadoop.ipc.ProcessingDetails.Timing for more details on
+ this phase. This property applies to WeightedTimeCostProvider.
+ </description>
+</property>
+
<!-- Proxy Configuration -->
<property>
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
index 3a4bcce..e784fe9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
@@ -135,6 +135,30 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase {
xmlPropsToSkipCompare.add("fs.azure.saskey.usecontainersaskeyforallaccess");
xmlPropsToSkipCompare.add("fs.azure.user.agent.prefix");
+ // FairCallQueue configs that includes dynamic ports in its keys
+ xmlPropsToSkipCompare.add("ipc.[port_number].backoff.enable");
+ xmlPropsToSkipCompare.add("ipc.[port_number].callqueue.impl");
+ xmlPropsToSkipCompare.add("ipc.[port_number].scheduler.impl");
+ xmlPropsToSkipCompare.add("ipc.[port_number].scheduler.priority.levels");
+ xmlPropsToSkipCompare.add(
+ "ipc.[port_number].faircallqueue.multiplexer.weights");
+ xmlPropsToSkipCompare.add("ipc.[port_number].identity-provider.impl");
+ xmlPropsToSkipCompare.add("ipc.[port_number].cost-provider.impl");
+ xmlPropsToSkipCompare.add("ipc.[port_number].decay-scheduler.period-ms");
+ xmlPropsToSkipCompare.add("ipc.[port_number].decay-scheduler.decay-factor");
+ xmlPropsToSkipCompare.add("ipc.[port_number].decay-scheduler.thresholds");
+ xmlPropsToSkipCompare.add(
+ "ipc.[port_number].decay-scheduler.backoff.responsetime.enable");
+ xmlPropsToSkipCompare.add(
+ "ipc.[port_number].decay-scheduler.backoff.responsetime.thresholds");
+ xmlPropsToSkipCompare.add(
+ "ipc.[port_number].decay-scheduler.metrics.top.user.count");
+ xmlPropsToSkipCompare.add("ipc.[port_number].weighted-cost.lockshared");
+ xmlPropsToSkipCompare.add("ipc.[port_number].weighted-cost.lockexclusive");
+ xmlPropsToSkipCompare.add("ipc.[port_number].weighted-cost.handler");
+ xmlPropsToSkipCompare.add("ipc.[port_number].weighted-cost.lockfree");
+ xmlPropsToSkipCompare.add("ipc.[port_number].weighted-cost.response");
+
// Deprecated properties. These should eventually be removed from the
// class.
configurationPropsToSkipCompare
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org