You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2020/05/18 11:52:32 UTC
[kylin] 02/07: KYLIN-4415 HTable Creation with Retry
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 20353a339054b71ae50386bf08397c780d2244e7
Author: Zhong, Yanghong <nj...@apache.org>
AuthorDate: Tue Mar 10 14:45:03 2020 +0800
KYLIN-4415 HTable Creation with Retry
---
.../org/apache/kylin/common/KylinConfigBase.java | 4 ++
.../kylin/storage/hbase/steps/CreateHTableJob.java | 2 +-
.../kylin/storage/hbase/steps/CubeHTableUtil.java | 78 +++++++++++++++++++---
3 files changed, 73 insertions(+), 11 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 16c07c1..7136a51 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -1259,6 +1259,10 @@ public abstract class KylinConfigBase implements Serializable {
}
}
+ public int getHBaseHTableAvailableRetry() {
+ return Integer.parseInt(getOptional("kylin.storage.hbase.htable-available-retry", "3"));
+ }
+
public int getHBaseRegionCountMin() {
return Integer.parseInt(getOptional("kylin.storage.hbase.min-region-count", "1"));
}
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
index 4b2218b..7c970c2 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CreateHTableJob.java
@@ -117,7 +117,7 @@ public class CreateHTableJob extends AbstractHadoopJob {
splitKeys = getRegionSplitsFromCuboidStatistics(cuboidSizeMap, kylinConfig, cubeSegment,
partitionFilePath.getParent());
- CubeHTableUtil.createHTable(cubeSegment, splitKeys);
+ CubeHTableUtil.createHTable(cubeSegment, splitKeys, true);
// export configuration in advance to avoid connecting to hbase from spark
if (cubeDesc.getEngineType()== IEngineAware.ID_SPARK){
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
index d06c993..369c7bc 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
@@ -19,16 +19,20 @@
package org.apache.kylin.storage.hbase.steps;
import java.io.IOException;
-
import java.util.Locale;
+
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableExistsException;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.TableNotEnabledException;
+import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.exceptions.TimeoutIOException;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.regionserver.BloomType;
@@ -54,13 +58,14 @@ public class CubeHTableUtil {
private static final Logger logger = LoggerFactory.getLogger(CubeHTableUtil.class);
- public static void createHTable(CubeSegment cubeSegment, byte[][] splitKeys) throws IOException {
- String tableName = cubeSegment.getStorageLocationIdentifier();
+ public static void createHTable(CubeSegment cubeSegment, byte[][] splitKeys, boolean continueOnExists)
+ throws IOException {
+ TableName tableName = TableName.valueOf(cubeSegment.getStorageLocationIdentifier());
CubeInstance cubeInstance = cubeSegment.getCubeInstance();
CubeDesc cubeDesc = cubeInstance.getDescriptor();
KylinConfig kylinConfig = cubeDesc.getConfig();
- HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(cubeSegment.getStorageLocationIdentifier()));
+ HTableDescriptor tableDesc = new HTableDescriptor(tableName);
tableDesc.setValue(HTableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName());
tableDesc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix());
tableDesc.setValue(IRealizationConstants.HTableCreationTime, String.valueOf(System.currentTimeMillis()));
@@ -95,16 +100,69 @@ public class CubeHTableUtil {
tableDesc.addFamily(cf);
}
- if (admin.tableExists(TableName.valueOf(tableName))) {
- // admin.disableTable(tableName);
- // admin.deleteTable(tableName);
- throw new RuntimeException("HBase table " + tableName + " exists!");
+ if (admin.tableExists(tableName)) {
+ if (!continueOnExists) {
+ throw new RuntimeException("HBase table " + tableName.toString() + " exists!");
+ } else {
+ logger.warn("HBase table " + tableName + " exists when create HTable, continue the process!");
+ if (admin.isTableEnabled(tableName)) {
+ try {
+ admin.disableTable(tableName);
+ logger.warn("Disabled existing enabled HBase table " + tableName.toString());
+ } catch (TableNotEnabledException e) {
+ logger.warn("HBase table " + tableName + " already disabled.", e);
+ }
+ } else {
+ logger.warn("HBase table exists but in disabled state.");
+ }
+ try {
+ admin.deleteTable(tableName);
+ logger.info("Deleted existing HBase table " + tableName.toString());
+ } catch (TableNotFoundException e) {
+ logger.warn("HBase table " + tableName + " already deleted.", e);
+ }
+ }
}
DeployCoprocessorCLI.deployCoprocessor(tableDesc);
- admin.createTable(tableDesc, splitKeys);
- Preconditions.checkArgument(admin.isTableAvailable(TableName.valueOf(tableName)), "table " + tableName + " created, but is not available due to some reasons");
+ try {
+ admin.createTable(tableDesc, splitKeys);
+ } catch (TableExistsException e) {
+ if (admin.isTableEnabled(tableName)) {
+ logger.warn(
+ "Duplicate create table request send to HMaster, ignore it and continue the process, table "
+ + tableName.toString());
+ } else {
+ logger.warn(
+ "Duplicate create table request send to HMaster, ignore it and continue enable the table "
+ + tableName.toString());
+ admin.enableTable(tableName);
+ }
+ } catch (TimeoutIOException e) {
+ if (admin.isTableEnabled(tableName)) {
+ logger.warn("False alerting?? Detect TimeoutIOException when creating HBase table "
+ + tableName.toString(), e);
+ } else {
+ throw e;
+ }
+ }
+
+ int availableRetry = kylinConfig.getHBaseHTableAvailableRetry();
+ while (availableRetry > 0) {
+ if (!admin.isTableAvailable(tableName)) {
+ logger.warn("Table created but not available, wait for a while...");
+ try {
+ availableRetry--;
+ Thread.sleep(60000);
+ } catch (InterruptedException e) {
+ }
+ } else {
+ break;
+ }
+ }
+
+ Preconditions.checkArgument(admin.isTableAvailable(tableName), "table " + tableName + " created, but is not available due to some reasons");
logger.info("create hbase table " + tableName + " done.");
} finally {
IOUtils.closeQuietly(admin);