You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by nj...@apache.org on 2016/04/18 08:35:16 UTC
[34/50] kylin git commit: KYLIN-1458: Checking the consistency of
cube segment host with the environment after cube migration
KYLIN-1458: Checking the consistency of cube segment host with the environment after cube migration
Signed-off-by: Hongbin Ma <ma...@apache.org>
KYLIN-1458: Add the auto fix if inconsistency exists after cube migration
Signed-off-by: Hongbin Ma <ma...@apache.org>
KYLIN-1458: rename parameters for avoiding misleading
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ad8d7153
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ad8d7153
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ad8d7153
Branch: refs/heads/1.4-rc
Commit: ad8d715389cbd9fd15202bc8507c8534a903cedc
Parents: fbbe50d
Author: yangzhong <ya...@ebay.com>
Authored: Mon Feb 29 09:37:06 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Mar 1 13:39:32 2016 +0800
----------------------------------------------------------------------
.../storage/hbase/util/CubeMigrationCLI.java | 7 +
.../hbase/util/CubeMigrationCheckCLI.java | 195 +++++++++++++++++++
2 files changed, 202 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/ad8d7153/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
index 57f362d..53cffe5 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java
@@ -143,6 +143,8 @@ public class CubeMigrationCLI {
} else {
showOpts();
}
+
+ checkMigrationSuccess(dstConfig, cubeName, true);
}
public static void moveCube(String srcCfgUri, String dstCfgUri, String cubeName, String projectName, String copyAcl, String purgeAndDisable, String overwriteIfExists, String realExecute) throws IOException, InterruptedException {
@@ -150,6 +152,11 @@ public class CubeMigrationCLI {
moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), KylinConfig.createInstanceFromUri(dstCfgUri), cubeName, projectName, copyAcl, purgeAndDisable, overwriteIfExists, realExecute);
}
+ public static void checkMigrationSuccess(KylinConfig kylinConfig, String cubeName, Boolean ifFix) throws IOException{
+ CubeMigrationCheckCLI checkCLI = new CubeMigrationCheckCLI(kylinConfig, ifFix);
+ checkCLI.execute(cubeName);
+ }
+
private static String checkAndGetHbaseUrl() {
String srcMetadataUrl = srcConfig.getMetadataUrl();
String dstMetadataUrl = dstConfig.getMetadataUrl();
http://git-wip-us.apache.org/repos/asf/kylin/blob/ad8d7153/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
new file mode 100644
index 0000000..2762561
--- /dev/null
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCheckCLI.java
@@ -0,0 +1,195 @@
+package org.apache.kylin.storage.hbase.util;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * <p/>
+ * This tool serves for the purpose of
+ * checking the "KYLIN_HOST" property to be consistent with the dst's MetadataUrlPrefix
+ * for all of cube segments' corresponding HTables after migrating a cube
+ * <p/>
+ */
+public class CubeMigrationCheckCLI {
+
+ private static final Logger logger = LoggerFactory.getLogger(CubeMigrationCheckCLI.class);
+
+ private static final Option OPTION_FIX = OptionBuilder.withArgName("fix").hasArg().isRequired(false).withDescription("Fix the inconsistent cube segments' HOST").create("fix");
+
+ private static final Option OPTION_DST_CFG_URI = OptionBuilder.withArgName("dstCfgUri").hasArg().isRequired(false).withDescription("The KylinConfig of the cube’s new home").create("dstCfgUri");
+
+ private static final Option OPTION_CUBE = OptionBuilder.withArgName("cube").hasArg().isRequired(false).withDescription("The name of cube migrated").create("cube");
+
+
+ private KylinConfig dstCfg;
+ private HBaseAdmin hbaseAdmin;
+
+ private List<String> issueExistHTables;
+ private List<String> inconsistentHTables;
+
+ private boolean ifFix = false;
+
+ public static void main(String[] args) throws ParseException, IOException {
+
+ OptionsHelper optionsHelper = new OptionsHelper();
+
+ Options options = new Options();
+ options.addOption(OPTION_FIX);
+ options.addOption(OPTION_DST_CFG_URI);
+ options.addOption(OPTION_CUBE);
+
+ boolean ifFix = false;
+ String dstCfgUri;
+ String cubeName;
+ logger.info("jobs args: " + Arrays.toString(args));
+ try {
+
+ optionsHelper.parseOptions(options, args);
+
+ logger.info("options: '" + options.toString() + "'");
+ logger.info("option value 'fix': '" + optionsHelper.getOptionValue(OPTION_FIX) + "'");
+ ifFix = Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_FIX));
+
+ logger.info("option value 'dstCfgUri': '" + optionsHelper.getOptionValue(OPTION_DST_CFG_URI) + "'");
+ dstCfgUri = optionsHelper.getOptionValue(OPTION_DST_CFG_URI);
+
+ logger.info("option value 'cube': '" + optionsHelper.getOptionValue(OPTION_CUBE) + "'");
+ cubeName = optionsHelper.getOptionValue(OPTION_CUBE);
+
+ } catch (ParseException e) {
+ optionsHelper.printUsage(CubeMigrationCheckCLI.class.getName(), options);
+ throw e;
+ }
+
+ KylinConfig kylinConfig;
+ if(dstCfgUri==null){
+ kylinConfig = KylinConfig.getInstanceFromEnv();
+ }else{
+ kylinConfig = KylinConfig.createInstanceFromUri(dstCfgUri);
+ }
+
+ CubeMigrationCheckCLI checkCLI = new CubeMigrationCheckCLI(kylinConfig, ifFix);
+ checkCLI.execute(cubeName);
+ }
+
+ public void execute() throws IOException{
+ execute(null);
+ }
+
+ public void execute(String cubeName) throws IOException{
+ if(cubeName==null){
+ checkAll();
+ }else {
+ checkCube(cubeName);
+ }
+ fixInconsistent();
+ printIssueExistingHTables();
+ }
+
+ public CubeMigrationCheckCLI(KylinConfig kylinConfig, Boolean isFix) throws IOException{
+ this.dstCfg = kylinConfig;
+ this.ifFix = isFix;
+
+ Configuration conf = HBaseConfiguration.create();
+ hbaseAdmin = new HBaseAdmin(conf);
+
+ issueExistHTables = Lists.newArrayList();
+ inconsistentHTables = Lists.newArrayList();
+ }
+
+ public void checkCube(String cubeName) {
+ List<String> segFullNameList = Lists.newArrayList();
+
+ CubeInstance cube = CubeManager.getInstance(dstCfg).getCube(cubeName);
+ addHTableNamesForCube(cube,segFullNameList);
+
+ check(segFullNameList);
+ }
+
+ public void checkAll(){
+ List<String> segFullNameList = Lists.newArrayList();
+
+ CubeManager cubeMgr = CubeManager.getInstance(dstCfg);
+ for (CubeInstance cube : cubeMgr.listAllCubes()) {
+ addHTableNamesForCube(cube, segFullNameList);
+ }
+
+ check(segFullNameList);
+ }
+
+ public void addHTableNamesForCube(CubeInstance cube, List<String> segFullNameList){
+ for (CubeSegment seg : cube.getSegments()) {
+ String tableName = seg.getStorageLocationIdentifier();
+ segFullNameList.add(tableName+","+cube.getName());
+ }
+ }
+
+ public void check(List<String> segFullNameList){
+ issueExistHTables = Lists.newArrayList();
+ inconsistentHTables = Lists.newArrayList();
+
+ for(String segFullName:segFullNameList){
+ String[] sepNameList = segFullName.split(",");
+ try {
+ HTableDescriptor hTableDescriptor = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0]));
+ String host = hTableDescriptor.getValue(IRealizationConstants.HTableTag);
+ if(!dstCfg.getMetadataUrlPrefix().equalsIgnoreCase(host)){
+ inconsistentHTables.add(segFullName);
+ }
+ }catch (IOException e){
+ issueExistHTables.add(segFullName);
+ continue;
+ }
+ }
+ }
+
+ public void fixInconsistent() throws IOException{
+ if(ifFix == true){
+ for(String segFullName : inconsistentHTables){
+ String[] sepNameList = segFullName.split(",");
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0]));
+ logger.info("Change the host of htable "+sepNameList[0]+"belonging to cube "+sepNameList[1]+" from "+desc.getValue(IRealizationConstants.HTableTag)+" to "+dstCfg.getMetadataUrlPrefix());
+ hbaseAdmin.disableTable(sepNameList[0]);
+ desc.setValue(IRealizationConstants.HTableTag, dstCfg.getMetadataUrlPrefix());
+ hbaseAdmin.modifyTable(sepNameList[0], desc);
+ hbaseAdmin.enableTable(sepNameList[0]);
+ }
+ }else{
+ logger.info("------ Inconsistent HTables Needed To Be Fixed ------");
+ for (String hTable : inconsistentHTables) {
+ String[] sepNameList = hTable.split(",");
+ logger.info(sepNameList[0]+" belonging to cube "+sepNameList[1]);
+ }
+ logger.info("----------------------------------------------------");
+ }
+ }
+
+ public void printIssueExistingHTables(){
+ logger.info("------ HTables exist issues in hbase : not existing, metadata broken ------");
+ for(String segFullName : issueExistHTables){
+ String[] sepNameList = segFullName.split(",");
+ logger.error(sepNameList[0]+" belonging to cube "+sepNameList[1]+" has some issues and cannot be read successfully!!!");
+ }
+ logger.info("----------------------------------------------------");
+ }
+}