You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ka...@apache.org on 2019/07/22 19:55:10 UTC
[phoenix] branch 4.14-HBase-1.3 updated: PHOENIX-5401 Error in
running apache.phoenix.mapreduce.index.automation.PhoenixMRJobSubmitter
This is an automated email from the ASF dual-hosted git repository.
kadir pushed a commit to branch 4.14-HBase-1.3
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.14-HBase-1.3 by this push:
new 2b6acb6 PHOENIX-5401 Error in running apache.phoenix.mapreduce.index.automation.PhoenixMRJobSubmitter
2b6acb6 is described below
commit 2b6acb64b368eb263afa74ee24369992cf8a03ba
Author: Kadir <ko...@salesforce.com>
AuthorDate: Sun Jul 21 17:19:22 2019 -0700
PHOENIX-5401 Error in running apache.phoenix.mapreduce.index.automation.PhoenixMRJobSubmitter
---
.../mapreduce/PhoenixServerBuildIndexInputFormat.java | 4 ----
.../org/apache/phoenix/mapreduce/index/IndexTool.java | 15 +++++++++++++++
2 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormat.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormat.java
index 76d5a83..7bd30e6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormat.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixServerBuildIndexInputFormat.java
@@ -71,7 +71,6 @@ public class PhoenixServerBuildIndexInputFormat<T extends DBWritable> extends Ph
final Properties overridingProps = new Properties();
if(txnScnValue==null && currentScnValue!=null) {
overridingProps.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, currentScnValue);
- overridingProps.put(PhoenixRuntime.BUILD_INDEX_AT_ATTRIB, currentScnValue);
}
if (tenantId != null && configuration.get(PhoenixRuntime.TENANT_ID_ATTRIB) == null){
overridingProps.put(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
@@ -98,9 +97,6 @@ public class PhoenixServerBuildIndexInputFormat<T extends DBWritable> extends Ph
if (txnScnValue != null) {
scan.setAttribute(BaseScannerRegionObserver.TX_SCN, Bytes.toBytes(Long.valueOf(txnScnValue)));
}
-
- // Initialize the query plan so it sets up the parallel scans
- queryPlan.iterator(MapReduceParallelScanGrouper.getInstance());
return queryPlan;
} catch (Exception exception) {
LOGGER.error(String.format("Failed to get the query plan with error [%s]",
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
index 9be39ef..fe193b5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
@@ -84,6 +84,9 @@ import org.apache.phoenix.mapreduce.util.ConnectionUtil;
import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
import org.apache.phoenix.mapreduce.util.PhoenixMapReduceUtil;
import org.apache.phoenix.parse.HintNode.Hint;
+import org.apache.phoenix.query.ConnectionQueryServices;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.PIndexState;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTable.IndexType;
@@ -478,6 +481,17 @@ public class IndexTool extends Configured implements Tool {
private Job configureJobForServerBuildIndex()
throws Exception {
+ long indexRebuildQueryTimeoutMs =
+ configuration.getLong(QueryServices.INDEX_REBUILD_QUERY_TIMEOUT_ATTRIB,
+ QueryServicesOptions.DEFAULT_INDEX_REBUILD_QUERY_TIMEOUT);
+ // Set various phoenix and hbase level timeouts and rpc retries
+ configuration.set(QueryServices.THREAD_TIMEOUT_MS_ATTRIB,
+ Long.toString(indexRebuildQueryTimeoutMs));
+ configuration.set(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD,
+ Long.toString(indexRebuildQueryTimeoutMs));
+ configuration.set(HConstants.HBASE_RPC_TIMEOUT_KEY,
+ Long.toString(indexRebuildQueryTimeoutMs));
+
PhoenixConfigurationUtil.setIndexToolDataTableName(configuration, qDataTable);
PhoenixConfigurationUtil.setIndexToolIndexTableName(configuration, qIndexTable);
@@ -492,6 +506,7 @@ public class IndexTool extends Configured implements Tool {
fs = outputPath.getFileSystem(configuration);
fs.delete(outputPath, true);
+ configuration.set("mapreduce.task.timeout", Long.toString(indexRebuildQueryTimeoutMs));
final String jobName = String.format(INDEX_JOB_NAME_TEMPLATE, schemaName, dataTable, indexTable);
final Job job = Job.getInstance(configuration, jobName);
job.setJarByClass(IndexTool.class);