You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/09/06 09:29:38 UTC
incubator-kylin git commit: KYLIN-1002 add cleanup tools
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8 cc0ede30a -> cdc33f5b9
KYLIN-1002 add cleanup tools
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/cdc33f5b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/cdc33f5b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/cdc33f5b
Branch: refs/heads/0.8
Commit: cdc33f5b94fa58ac6b648d8e73f1e7dea9dc6d39
Parents: cc0ede3
Author: honma <ho...@ebay.com>
Authored: Sun Sep 6 15:31:30 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Sun Sep 6 15:31:39 2015 +0800
----------------------------------------------------------------------
.../kylin/storage/hbase/util/HBaseClean.java | 127 +++++++++++++++++++
.../kylin/storage/hbase/util/HBaseUsage.java | 70 ++++++++++
2 files changed, 197 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cdc33f5b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
new file mode 100644
index 0000000..1d28283
--- /dev/null
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseClean.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.storage.hbase.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
+import org.apache.kylin.metadata.realization.IRealizationConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * clean hbase tables by tag
+ */
+public class HBaseClean extends AbstractHadoopJob {
+
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_DELETE = OptionBuilder.withArgName("delete").hasArg().isRequired(true).withDescription("actually delete or not").create("delete");
+
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_TAG = OptionBuilder.withArgName("tag").hasArg().isRequired(true).withDescription("the tag of HTable").create("tag");
+
+ protected static final Logger log = LoggerFactory.getLogger(HBaseClean.class);
+ boolean delete = false;
+ String tag = null;
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Options options = new Options();
+
+ log.info("----- jobs args: " + Arrays.toString(args));
+ try {
+ options.addOption(OPTION_DELETE);
+ options.addOption(OPTION_TAG);
+ parseOptions(options, args);
+
+ log.info("options: '" + getOptionsAsString() + "'");
+
+ tag = getOptionValue(OPTION_TAG);
+ delete = Boolean.parseBoolean(getOptionValue(OPTION_DELETE));
+
+ cleanUp();
+
+ return 0;
+ } catch (Exception e) {
+ e.printStackTrace(System.err);
+ throw e;
+ }
+ }
+
+ private void cleanUp() {
+ try {
+ // get all kylin hbase tables
+ Configuration conf = HBaseConfiguration.create();
+ HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
+ HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
+ List<String> allTablesNeedToBeDropped = Lists.newArrayList();
+ for (HTableDescriptor desc : tableDescriptors) {
+ String host = desc.getValue(IRealizationConstants.HTableTag);
+ if (tag.equalsIgnoreCase(host)) {
+ allTablesNeedToBeDropped.add(desc.getTableName().getNameAsString());
+ }
+ }
+
+ if (delete) {
+ // drop tables
+ for (String htableName : allTablesNeedToBeDropped) {
+ log.info("Deleting HBase table " + htableName);
+ if (hbaseAdmin.tableExists(htableName)) {
+ if (hbaseAdmin.isTableEnabled(htableName)) {
+ hbaseAdmin.disableTable(htableName);
+ }
+
+ hbaseAdmin.deleteTable(htableName);
+ log.info("Deleted HBase table " + htableName);
+ } else {
+ log.info("HBase table" + htableName + " does not exist");
+ }
+ }
+ } else {
+ System.out.println("--------------- Tables To Be Dropped ---------------");
+ for (String htableName : allTablesNeedToBeDropped) {
+ System.out.println(htableName);
+ }
+ System.out.println("----------------------------------------------------");
+ }
+
+ hbaseAdmin.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ int exitCode = ToolRunner.run(new HBaseClean(), args);
+ System.exit(exitCode);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/cdc33f5b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
new file mode 100644
index 0000000..ffb1e25
--- /dev/null
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/HBaseUsage.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.storage.hbase.util;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.kylin.metadata.realization.IRealizationConstants;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+public class HBaseUsage {
+
+ public static void main(String[] args) throws IOException {
+ show();
+ }
+
+ private static void show() throws IOException {
+ Map<String, List<String>> envs = Maps.newHashMap();
+
+ // get all kylin hbase tables
+ Configuration conf = HBaseConfiguration.create();
+ HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
+ String tableNamePrefix = IRealizationConstants.SharedHbaseStorageLocationPrefix;
+ HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
+ for (HTableDescriptor desc : tableDescriptors) {
+ String host = desc.getValue(IRealizationConstants.HTableTag);
+ if (StringUtils.isEmpty(host)) {
+ add("unknown", desc.getNameAsString(), envs);
+ } else {
+ add(host, desc.getNameAsString(), envs);
+ }
+ }
+
+ for (Map.Entry<String, List<String>> entry : envs.entrySet()) {
+ System.out.println(entry.getKey() + " has htable count: " + entry.getValue().size());
+ }
+ hbaseAdmin.close();
+ }
+
+ private static void add(String tag, String tableName, Map<String, List<String>> envs) {
+ if (!envs.containsKey(tag)) {
+ envs.put(tag, Lists.<String> newArrayList());
+ }
+ envs.get(tag).add(tableName);
+ }
+}