You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by gj...@apache.org on 2017/07/17 17:17:22 UTC
phoenix git commit: PHOENIX-3948 Enable shorter time outs for
server-side index writes
Repository: phoenix
Updated Branches:
refs/heads/master 48341ae3f -> 7a83b8a1c
PHOENIX-3948 Enable shorter time outs for server-side index writes
Signed-off-by: gjacoby <gj...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/7a83b8a1
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/7a83b8a1
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/7a83b8a1
Branch: refs/heads/master
Commit: 7a83b8a1c4c9cefb6027b9aa2020af1dd18b599a
Parents: 48341ae
Author: Vincent <vi...@gmail.com>
Authored: Fri Jul 14 17:16:40 2017 -0700
Committer: gjacoby <gj...@apache.org>
Committed: Mon Jul 17 10:17:13 2017 -0700
----------------------------------------------------------------------
.../phoenix/end2end/index/MutableIndexFailureIT.java | 5 +++--
.../java/org/apache/phoenix/hbase/index/Indexer.java | 11 +++++++++++
.../phoenix/hbase/index/write/IndexWriterUtils.java | 13 ++++++++++++-
3 files changed, 26 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7a83b8a1/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
index 8e2564d..e3cac67 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest;
import org.apache.phoenix.execute.CommitException;
+import org.apache.phoenix.hbase.index.write.IndexWriterUtils;
import org.apache.phoenix.index.PhoenixIndexFailurePolicy;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.BaseTest;
@@ -120,9 +121,9 @@ public class MutableIndexFailureIT extends BaseTest {
public static void doSetup() throws Exception {
Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(10);
serverProps.put("hbase.coprocessor.region.classes", FailingRegionObserver.class.getName());
- serverProps.put(HConstants.HBASE_CLIENT_RETRIES_NUMBER, "2");
+ serverProps.put(IndexWriterUtils.INDEX_WRITER_RPC_RETRIES_NUMBER, "2");
serverProps.put(HConstants.HBASE_RPC_TIMEOUT_KEY, "10000");
- serverProps.put("hbase.client.pause", "5000");
+ serverProps.put(IndexWriterUtils.INDEX_WRITER_RPC_PAUSE, "5000");
serverProps.put("data.tx.snapshot.dir", "/tmp");
serverProps.put("hbase.balancer.period", String.valueOf(Integer.MAX_VALUE));
serverProps.put(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_ATTRIB, Boolean.TRUE.toString());
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7a83b8a1/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
index 98f8c69..d65ad05 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
@@ -18,6 +18,10 @@
package org.apache.phoenix.hbase.index;
import static org.apache.phoenix.hbase.index.util.IndexManagementUtil.rethrowIndexingException;
+import static org.apache.phoenix.hbase.index.write.IndexWriterUtils.DEFAULT_INDEX_WRITER_RPC_PAUSE;
+import static org.apache.phoenix.hbase.index.write.IndexWriterUtils.DEFAULT_INDEX_WRITER_RPC_RETRIES_NUMBER;
+import static org.apache.phoenix.hbase.index.write.IndexWriterUtils.INDEX_WRITER_RPC_PAUSE;
+import static org.apache.phoenix.hbase.index.write.IndexWriterUtils.INDEX_WRITER_RPC_RETRIES_NUMBER;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
@@ -171,6 +175,13 @@ public class Indexer extends BaseRegionObserver {
*/
clonedConfig.setClass(RpcControllerFactory.CUSTOM_CONTROLLER_CONF_KEY,
InterRegionServerIndexRpcControllerFactory.class, RpcControllerFactory.class);
+ // lower the number of rpc retries. We inherit config from HConnectionManager#setServerSideHConnectionRetries,
+ // which by default uses a multiplier of 10. That is too many retries for our synchronous index writes
+ clonedConfig.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
+ env.getConfiguration().getInt(INDEX_WRITER_RPC_RETRIES_NUMBER,
+ DEFAULT_INDEX_WRITER_RPC_RETRIES_NUMBER));
+ clonedConfig.setInt(HConstants.HBASE_CLIENT_PAUSE, env.getConfiguration()
+ .getInt(INDEX_WRITER_RPC_PAUSE, DEFAULT_INDEX_WRITER_RPC_PAUSE));
DelegateRegionCoprocessorEnvironment indexWriterEnv = new DelegateRegionCoprocessorEnvironment(clonedConfig, env);
// setup the actual index writer
this.writer = new IndexWriter(indexWriterEnv, serverName + "-index-writer");
http://git-wip-us.apache.org/repos/asf/phoenix/blob/7a83b8a1/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
index ea4ec1a..f2885a9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/IndexWriterUtils.java
@@ -68,7 +68,18 @@ public class IndexWriterUtils {
public static final String HTABLE_THREAD_KEY = "hbase.htable.threads.max";
public static final String INDEX_WRITES_THREAD_MAX_PER_REGIONSERVER_KEY = "phoenix.index.writes.threads.max";
public static final String HTABLE_KEEP_ALIVE_KEY = "hbase.htable.threads.keepalivetime";
-
+
+ public static final String INDEX_WRITER_RPC_RETRIES_NUMBER = "phoenix.index.writes.rpc.retries.number";
+ /**
+ * Based on the logic in HBase's AsyncProcess, a default of 11 retries with a pause of 100ms
+ * approximates 48 sec total retry time (factoring in backoffs). The total time should be less
+ * than HBase's rpc timeout (default of 60 sec) or else the client will retry before receiving
+ * the response
+ */
+ public static final int DEFAULT_INDEX_WRITER_RPC_RETRIES_NUMBER = 11;
+ public static final String INDEX_WRITER_RPC_PAUSE = "phoenix.index.writes.rpc.pause";
+ public static final int DEFAULT_INDEX_WRITER_RPC_PAUSE = 100;
+
private IndexWriterUtils() {
// private ctor for utilites
}