You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2018/05/22 20:39:47 UTC
[04/22] phoenix git commit: PHOENIX-4732 Improve config defaults for
async secondary index MR jobs
PHOENIX-4732 Improve config defaults for async secondary index MR jobs
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/4849fbe2
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/4849fbe2
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/4849fbe2
Branch: refs/heads/omid2
Commit: 4849fbe2b7160d9cf0e1f05b93ae39b186d3f144
Parents: 7e5c387
Author: James Taylor <jt...@salesforce.com>
Authored: Tue May 8 15:26:44 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Wed May 9 13:35:52 2018 -0700
----------------------------------------------------------------------
.../index/automation/PhoenixMRJobSubmitter.java | 48 ++++++++++++++++++--
.../apache/phoenix/util/PhoenixMRJobUtil.java | 18 +++-----
2 files changed, 52 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4849fbe2/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/automation/PhoenixMRJobSubmitter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/automation/PhoenixMRJobSubmitter.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/automation/PhoenixMRJobSubmitter.java
index 9c447e8..d86802a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/automation/PhoenixMRJobSubmitter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/automation/PhoenixMRJobSubmitter.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.mapreduce.index.IndexTool;
@@ -53,8 +54,8 @@ import org.apache.phoenix.schema.PTable.IndexType;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.types.PDate;
import org.apache.phoenix.util.PhoenixMRJobUtil;
-import org.apache.phoenix.util.UpgradeUtil;
import org.apache.phoenix.util.PhoenixMRJobUtil.MR_SCHEDULER_TYPE;
+import org.apache.phoenix.util.UpgradeUtil;
import org.apache.phoenix.util.ZKBasedMasterElectionUtil;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject;
@@ -69,8 +70,28 @@ public class PhoenixMRJobSubmitter {
// Lock to elect a master node that submits the Phoenix Secondary Index MR Jobs
private static final String PHOENIX_LOCKS_PARENT =
"/phoenix/automated-mr-index-build-leader-election";
+ public static final String PHOENIX_MR_CLIENT_SCANNER_TIMEOUT_PERIOD =
+ "phoenix.mr.client.scanner.timeout.period";
+ public static final String PHOENIX_MR_RPC_TIMEOUT =
+ "phoenix.mr.rpc.timeout";
+ public static final String PHOENIX_MR_TASK_TIMEOUT =
+ "phoenix.mr.task.timeout";
+ public static final String PHOENIX_MR_CLIENT_RETRIES_NUMBER =
+ "phoenix.mr.client.retries.number";
+ public static final String PHOENIX_MR_CLIENT_PAUSE =
+ "phoenix.mr.client.retries.number";
+ public static final String PHOENIX_MR_ZK_RECOVERY_RETRY =
+ "phoenix.mr.zk.recovery.retry";
private static final String AUTO_INDEX_BUILD_LOCK_NAME = "ActiveStandbyElectorLock";
-
+ private static final int DEFAULT_TIMEOUT_IN_MILLIS = 600000;
+ public static final int DEFAULT_MR_CLIENT_SCANNER_TIMEOUT_PERIOD = DEFAULT_TIMEOUT_IN_MILLIS;
+ public static final int DEFAULT_MR_RPC_TIMEOUT = DEFAULT_TIMEOUT_IN_MILLIS;
+ public static final int DEFAULT_MR_TASK_TIMEOUT = DEFAULT_TIMEOUT_IN_MILLIS;
+ // Reduced HBase/Zookeeper Client Retries
+ public static final int DEFAULT_MR_CLIENT_RETRIES_NUMBER = 10;
+ public static final int DEFAULT_MR_CLIENT_PAUSE = 1000;
+ public static final int DEFAULT_MR_ZK_RECOVERY_RETRY = 1;
+
public static final String CANDIDATE_INDEX_INFO_QUERY = "SELECT "
+ PhoenixDatabaseMetaData.INDEX_TYPE + ","
+ PhoenixDatabaseMetaData.DATA_TABLE_NAME + ", "
@@ -103,7 +124,28 @@ public class PhoenixMRJobSubmitter {
}
this.conf = conf;
- PhoenixMRJobUtil.updateTimeoutsToFailFast(conf);
+ // Have Phoenix specific properties for defaults to enable potential override
+ conf.setLong(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD,
+ conf.getLong(PHOENIX_MR_CLIENT_SCANNER_TIMEOUT_PERIOD,
+ DEFAULT_MR_CLIENT_SCANNER_TIMEOUT_PERIOD));
+ conf.setLong(HConstants.HBASE_RPC_TIMEOUT_KEY,
+ conf.getLong(PHOENIX_MR_RPC_TIMEOUT,
+ DEFAULT_MR_RPC_TIMEOUT));
+ conf.setLong(MRJobConfig.TASK_TIMEOUT,
+ conf.getLong(PHOENIX_MR_TASK_TIMEOUT,
+ DEFAULT_MR_TASK_TIMEOUT));
+
+ // Reduced HBase Client Retries
+ conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
+ conf.getInt(PHOENIX_MR_CLIENT_RETRIES_NUMBER,
+ DEFAULT_MR_CLIENT_RETRIES_NUMBER));
+ conf.setInt(HConstants.HBASE_CLIENT_PAUSE,
+ conf.getInt(PHOENIX_MR_CLIENT_PAUSE,
+ DEFAULT_MR_CLIENT_PAUSE));
+ conf.setInt("zookeeper.recovery.retry",
+ conf.getInt(PHOENIX_MR_ZK_RECOVERY_RETRY,
+ DEFAULT_MR_ZK_RECOVERY_RETRY));
+
String schedulerType =
conf.get(PhoenixMRJobUtil.PHOENIX_MR_SCHEDULER_TYPE_NAME,
MR_SCHEDULER_TYPE.NONE.toString());
http://git-wip-us.apache.org/repos/asf/phoenix/blob/4849fbe2/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixMRJobUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixMRJobUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixMRJobUtil.java
index f12d49d..13eae98 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixMRJobUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixMRJobUtil.java
@@ -52,19 +52,17 @@ public class PhoenixMRJobUtil {
private static final String ACTIVE_STANDBY_ELECTOR_LOCK = "ActiveStandbyElectorLock";
private static final String RM_APPS_GET_ENDPOINT = "/ws/v1/cluster/apps";
- // Reduced HBase Client Retries
- private static final int CLIENT_RETRIES_NUMBER = 2;
- private static final long CLIENT_PAUSE_TIME = 1000;
- private static final int ZOOKEEPER_RECOVERY_RETRY_COUNT = 1;
-
public static final String PHOENIX_INDEX_MR_QUEUE_NAME_PROPERTY =
"phoenix.index.mr.scheduler.capacity.queuename";
public static final String PHOENIX_INDEX_MR_MAP_MEMORY_PROPERTY =
"phoenix.index.mr.scheduler.capacity.mapMemoryMB";
+ public static final String PHOENIX_MR_CONCURRENT_MAP_LIMIT_PROPERTY =
+ "phoenix.mr.concurrent.map.limit";
// Default MR Capacity Scheduler Configurations for Phoenix MR Index Build
// Jobs
public static final String DEFAULT_QUEUE_NAME = "default";
+ public static final int DEFAULT_MR_CONCURRENT_MAP_LIMIT = 20;
public static final int DEFAULT_MAP_MEMROY_MB = 5120;
public static final String XMX_OPT = "-Xmx";
@@ -208,12 +206,6 @@ public class PhoenixMRJobUtil {
return rmPort;
}
- public static void updateTimeoutsToFailFast(Configuration conf) {
- conf.set("hbase.client.retries.number", String.valueOf(CLIENT_RETRIES_NUMBER));
- conf.set("zookeeper.recovery.retry", String.valueOf(ZOOKEEPER_RECOVERY_RETRY_COUNT));
- conf.set("hbase.client.pause", String.valueOf(CLIENT_PAUSE_TIME));
- }
-
/**
* This method set the configuration values for Capacity scheduler.
* @param conf - Configuration to which Capacity Queue information to be added
@@ -221,6 +213,10 @@ public class PhoenixMRJobUtil {
public static void updateCapacityQueueInfo(Configuration conf) {
conf.set(MRJobConfig.QUEUE_NAME,
conf.get(PHOENIX_INDEX_MR_QUEUE_NAME_PROPERTY, DEFAULT_QUEUE_NAME));
+
+ conf.setInt(MRJobConfig.JOB_RUNNING_MAP_LIMIT,
+ conf.getInt(PHOENIX_MR_CONCURRENT_MAP_LIMIT_PROPERTY, DEFAULT_MR_CONCURRENT_MAP_LIMIT));
+
int mapMemoryMB = conf.getInt(PHOENIX_INDEX_MR_MAP_MEMORY_PROPERTY, DEFAULT_MAP_MEMROY_MB);
conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMemoryMB);