You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/07/22 08:27:14 UTC
incubator-kylin git commit: KYLIN-901 Add tool for cleanup Kylin
metadata storage
Repository: incubator-kylin
Updated Branches:
refs/heads/0.8 7e8896ac5 -> 8ad54f4f6
KYLIN-901 Add tool for cleanup Kylin metadata storage
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/8ad54f4f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/8ad54f4f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/8ad54f4f
Branch: refs/heads/0.8
Commit: 8ad54f4f6701aebbba2f41d618b34940b53bdcb6
Parents: 7e8896a
Author: shaofengshi <sh...@apache.org>
Authored: Wed Jul 22 14:25:48 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed Jul 22 14:26:17 2015 +0800
----------------------------------------------------------------------
.../job/hadoop/cube/MetadataCleanupJob.java | 160 +++++++++++++++++++
1 file changed, 160 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8ad54f4f/job/src/main/java/org/apache/kylin/job/hadoop/cube/MetadataCleanupJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/MetadataCleanupJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/MetadataCleanupJob.java
new file mode 100644
index 0000000..005254b
--- /dev/null
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/MetadataCleanupJob.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job.hadoop.cube;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.job.hadoop.AbstractHadoopJob;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ */
+public class MetadataCleanupJob extends AbstractHadoopJob {
+
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_DELETE = OptionBuilder.withArgName("delete").hasArg().isRequired(false).withDescription("Delete the unused metadata").create("delete");
+
+ protected static final Logger log = LoggerFactory.getLogger(MetadataCleanupJob.class);
+
+ boolean delete = false;
+
+ private KylinConfig config = null;
+
+ public static final long TIME_THREADSHOLD = 2 * 26 * 3600 * 1000l; // 2 days
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.hadoop.util.Tool#run(java.lang.String[])
+ */
+ @Override
+ public int run(String[] args) throws Exception {
+ Options options = new Options();
+
+ log.info("----- jobs args: " + Arrays.toString(args));
+ try {
+ options.addOption(OPTION_DELETE);
+ parseOptions(options, args);
+
+ log.info("options: '" + getOptionsAsString() + "'");
+ log.info("delete option value: '" + getOptionValue(OPTION_DELETE) + "'");
+ delete = Boolean.parseBoolean(getOptionValue(OPTION_DELETE));
+
+ config = KylinConfig.getInstanceFromEnv();
+
+ cleanup();
+
+ return 0;
+ } catch (Exception e) {
+ e.printStackTrace(System.err);
+ throw e;
+ }
+ }
+
+ private ResourceStore getStore() {
+ return ResourceStore.getStore(config);
+ }
+
+ private boolean isOlderThanThreshold(long resourceTime) {
+ long currentTime = System.currentTimeMillis();
+
+ if (currentTime - resourceTime > TIME_THREADSHOLD)
+ return true;
+ return false;
+ }
+
+ public void cleanup() throws Exception {
+ CubeManager cubeManager = CubeManager.getInstance(config);
+
+ List<String> activeResourceList = Lists.newArrayList();
+ for (org.apache.kylin.cube.CubeInstance cube : cubeManager.listAllCubes()) {
+ for (org.apache.kylin.cube.CubeSegment segment : cube.getSegments()) {
+ activeResourceList.addAll(segment.getSnapshotPaths());
+ activeResourceList.addAll(segment.getDictionaryPaths());
+ activeResourceList.add(segment.getStatisticsResourcePath());
+ }
+ }
+
+ List<String> toDeleteResource = Lists.newArrayList();
+
+ // two level resources, snapshot tables and cube statistics
+ for (String resourceRoot : new String[]{ResourceStore.SNAPSHOT_RESOURCE_ROOT, ResourceStore.CUBE_STATISTICS_ROOT}) {
+ ArrayList<String> snapshotTables = getStore().listResources(resourceRoot);
+
+ for (String snapshotTable : snapshotTables) {
+ ArrayList<String> snapshotNames = getStore().listResources(snapshotTable);
+ if (snapshotNames != null)
+ for (String snapshot : snapshotNames) {
+ if (!activeResourceList.contains(snapshot)) {
+ if (isOlderThanThreshold(getStore().getResourceTimestamp(snapshot)))
+ toDeleteResource.add(snapshot);
+ }
+ }
+ }
+ }
+
+ // three level resources, only dictionaries
+ ArrayList<String> dictTables = getStore().listResources(ResourceStore.DICT_RESOURCE_ROOT);
+
+ for (String table : dictTables) {
+ ArrayList<String> tableColNames = getStore().listResources(table);
+ if (tableColNames != null)
+ for (String tableCol : tableColNames) {
+ ArrayList<String> dictionaries = getStore().listResources(tableCol);
+ if (dictionaries != null)
+ for (String dict : dictionaries)
+ if (!activeResourceList.contains(dict)) {
+ if (isOlderThanThreshold(getStore().getResourceTimestamp(dict)))
+ toDeleteResource.add(dict);
+ }
+ }
+ }
+
+
+ if (toDeleteResource.size() > 0) {
+ logger.info("The following resources have no reference, will be cleaned from metadata store: \n");
+
+ for (String s : toDeleteResource) {
+ logger.info(s);
+ if (delete == true) {
+ getStore().deleteResource(s);
+ }
+ }
+ } else {
+ logger.info("No resource to be cleaned up from metadata store;");
+ }
+
+ }
+
+ public static void main(String[] args) throws Exception {
+ int exitCode = ToolRunner.run(new MetadataCleanupJob(), args);
+ System.exit(exitCode);
+ }
+}