You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ww...@apache.org on 2017/06/27 04:55:54 UTC
hadoop git commit: HDFS-12018. Ozone: Misc: Add CBlocks defaults to
ozone-defaults.xml. Contributed by Chen Liang.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7240 102dd20b7 -> f98c3523a
HDFS-12018. Ozone: Misc: Add CBlocks defaults to ozone-defaults.xml. Contributed by Chen Liang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f98c3523
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f98c3523
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f98c3523
Branch: refs/heads/HDFS-7240
Commit: f98c3523a2838a2b62bfb3cff005b38952a90c5b
Parents: 102dd20
Author: Weiwei Yang <ww...@apache.org>
Authored: Tue Jun 27 12:55:08 2017 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Tue Jun 27 12:55:08 2017 +0800
----------------------------------------------------------------------
.../apache/hadoop/cblock/CBlockConfigKeys.java | 19 +--
.../jscsiHelper/ContainerCacheFlusher.java | 8 +-
.../cblock/jscsiHelper/SCSITargetDaemon.java | 7 +-
.../src/main/resources/ozone-default.xml | 159 ++++++++++++++++++-
4 files changed, 173 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f98c3523/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
index e49c6de..35b8961 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
@@ -25,28 +25,19 @@ import static java.lang.Thread.NORM_PRIORITY;
public final class CBlockConfigKeys {
public static final String DFS_CBLOCK_SERVICERPC_ADDRESS_KEY =
"dfs.cblock.servicerpc-address";
- public static final String DFS_CBLOCK_SERVICERPC_PORT_KEY =
- "dfs.cblock.servicerpc.port";
public static final int DFS_CBLOCK_SERVICERPC_PORT_DEFAULT =
9810;
- public static final String DFS_CBLOCK_SERVICERPC_HOSTNAME_KEY =
- "dfs.cblock.servicerpc.hostname";
public static final String DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT =
"0.0.0.0";
- public static final String DFS_CBLOCK_SERVICERPC_ADDRESS_DEFAULT =
- DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT
- + ":" + DFS_CBLOCK_SERVICERPC_PORT_DEFAULT;
public static final String DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY =
"dfs.cblock.jscsi-address";
//The port on CBlockManager node for jSCSI to ask
+ public static final String DFS_CBLOCK_JSCSI_PORT_KEY =
+ "dfs.cblock.jscsi.port";
public static final int DFS_CBLOCK_JSCSI_PORT_DEFAULT =
9811;
- public static final String DFS_CBLOCK_JSCSIRPC_ADDRESS_DEFAULT =
- DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT
- + ":" + DFS_CBLOCK_JSCSI_PORT_DEFAULT;
-
public static final String DFS_CBLOCK_SERVICERPC_BIND_HOST_KEY =
"dfs.cblock.service.rpc-bind-host";
@@ -97,9 +88,9 @@ public final class CBlockConfigKeys {
/**
* Minimum Number of threads that cache pool will use for background I/O.
*/
- public static final String DFS_CBLOCK_CACHE_CORE_POOL_SIZE =
- "dfs.cblock.cache.core.pool.size";
- public static final int DFS_CBLOCK_CACHE_CORE_POOL_SIZE_DEFAULT = 16;
+ public static final String DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE =
+ "dfs.cblock.cache.core.min.pool.size";
+ public static final int DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE_DEFAULT = 16;
/**
* Maximum Number of threads that cache pool will use for background I/O.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f98c3523/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java
index 3998333..1582675 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java
@@ -61,9 +61,9 @@ import static org.apache.hadoop.cblock.CBlockConfigKeys
import static org.apache.hadoop.cblock.CBlockConfigKeys
.DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT;
import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_CACHE_CORE_POOL_SIZE;
+ .DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE;
import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_CACHE_CORE_POOL_SIZE_DEFAULT;
+ .DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE_DEFAULT;
import static org.apache.hadoop.cblock.CBlockConfigKeys
.DFS_CBLOCK_CACHE_KEEP_ALIVE_SECONDS;
import static org.apache.hadoop.cblock.CBlockConfigKeys
@@ -114,8 +114,8 @@ public class ContainerCacheFlusher implements Runnable {
CBlockTargetMetrics metrics) {
int queueSize = config.getInt(DFS_CBLOCK_CACHE_QUEUE_SIZE_KB,
DFS_CBLOCK_CACHE_QUEUE_SIZE_KB_DEFAULT) * 1024;
- int corePoolSize = config.getInt(DFS_CBLOCK_CACHE_CORE_POOL_SIZE,
- DFS_CBLOCK_CACHE_CORE_POOL_SIZE_DEFAULT);
+ int corePoolSize = config.getInt(DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE,
+ DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE_DEFAULT);
int maxPoolSize = config.getInt(DFS_CBLOCK_CACHE_MAX_POOL_SIZE,
DFS_CBLOCK_CACHE_MAX_POOL_SIZE_DEFAULT);
long keepAlive = config.getLong(DFS_CBLOCK_CACHE_KEEP_ALIVE_SECONDS,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f98c3523/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java
index e0aa9ff..9751849 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java
@@ -36,6 +36,7 @@ import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_CONTAINER_SIZ
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_DEFAULT;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_KEY;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_PORT_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_PORT_KEY;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY;
import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY;
@@ -67,6 +68,10 @@ public final class SCSITargetDaemon {
DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_KEY,
DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_DEFAULT
);
+ int cbmPort = ozoneConf.getInt(
+ DFS_CBLOCK_JSCSI_PORT_KEY,
+ DFS_CBLOCK_JSCSI_PORT_DEFAULT
+ );
String scmAddress = ozoneConf.get(OZONE_SCM_CLIENT_BIND_HOST_KEY,
OZONE_SCM_CLIENT_BIND_HOST_DEFAULT);
@@ -82,7 +87,7 @@ public final class SCSITargetDaemon {
ozoneConf.set(OZONE_SCM_DATANODE_ADDRESS_KEY, scmDataodeAddress);
InetSocketAddress cbmAddress = new InetSocketAddress(
- cbmIPAddress, DFS_CBLOCK_JSCSI_PORT_DEFAULT);
+ cbmIPAddress, cbmPort);
long version = RPC.getProtocolVersion(
CBlockServiceProtocolPB.class);
CBlockClientProtocolClientSideTranslatorPB cbmClient =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f98c3523/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml
index d9d2d3e..766da66 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml
@@ -579,5 +579,162 @@
The keytab file for Kerberos authentication in KSM.
</description>
</property>
-
+
+ <property>
+ <name>dfs.cblock.servicerpc-address</name>
+ <value></value>
+ <description>
+ The address that cblock will be bind to, should be a host:port format,
+ this setting is required for cblock server to start.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cblock.jscsi.port</name>
+ <value>9811</value>
+ <description>
+ The port on CBlockManager node for jSCSI to talk to.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.storage.service.handler.count</name>
+ <value>10</value>
+ <description>
+ Default number of handlers for CBlock service rpc.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cblock.service.leveldb.path</name>
+ <value>/tmp/cblock_levelDB.dat</value>
+ <description>
+ Default path for the cblock meta data disk store.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cblock.disk.cache.path</name>
+ <value>/tmp/cblockCacheDB</value>
+ <description>
+ Default path for the cblock local cache.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cblock.trace.io</name>
+ <value>false</value>
+ <description>
+ Default flag for enabling trace io.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cblock.short.circuit.io</name>
+ <value>false</value>
+ <description>
+ Default flag to enable cblock local cache.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cblock.cache.cache.size.in.kb</name>
+ <value>256</value>
+ <description>
+ Default cblock cache queue size, in number of kb.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cblock.cache.core.min.pool.size</name>
+ <value>16</value>
+ <description>
+ Minimum number of thread pool thread that cBlock cache will use for background I/O.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cblock.cache.max.pool.size</name>
+ <value>256</value>
+ <description>
+ Maximum number of thread pool thread that cBlcok cache will use for background I/O.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cblock.cache.keep.alive.seconds</name>
+ <value>60</value>
+ <description>
+ Number of seconds to keep the Thread alive when it is idle.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cblock.cache.thread.priority</name>
+ <value>5</value>
+ <description>
+ Priority of cache flusher thread, affecting the relative
+ performance of write and read.upported values are 1, 5, 10.
+ Use 10 for high priority and 1 for low priority.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cblock.cache.block.buffer.size</name>
+ <value>512</value>
+ <description>
+ Block Buffer size in terms of blockID entries, in number of blockIDs.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cblock.block.buffer.flush.interval.seconds</name>
+ <value>60</value>
+ <description>
+ The interval to flush cblock buffer in seconds.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cblock.jscsi.server.address</name>
+ <value>127.0.0.1</value>
+ <description>
+ The address that jscsi server will be running, should have one
+ local jscsi server for each client that tries to mount cblock.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cblock.jscsi.cblock.server.address</name>
+ <value>127.0.0.1</value>
+ <description>
+ The address local jscsi server will use to talk to cblock manager.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cblock.container.size</name>
+ <value>5</value>
+ <description>
+ The size of ozone container in number of GBs. Note that this is
+ not setting container size for ozone, but rather, this is setting
+ that assumption that cblock manager will make about ozone.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cblock.cache.leveldb.cache.size.mb</name>
+ <value>256</value>
+ <description>
+ The size of LevelDB cache file which uses an off-heap cache in LevelDB.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cblock.cache.max.retry</name>
+ <value>65536</value>
+ <description>
+ The maximum number of retries when writing a block to container.
+ </description>
+ </property>
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org