You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/07/22 08:27:14 UTC

incubator-kylin git commit: KYLIN-901 Add tool for cleanup Kylin metadata storage

Repository: incubator-kylin
Updated Branches:
  refs/heads/0.8 7e8896ac5 -> 8ad54f4f6


KYLIN-901 Add tool for cleanup Kylin metadata storage


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/8ad54f4f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/8ad54f4f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/8ad54f4f

Branch: refs/heads/0.8
Commit: 8ad54f4f6701aebbba2f41d618b34940b53bdcb6
Parents: 7e8896a
Author: shaofengshi <sh...@apache.org>
Authored: Wed Jul 22 14:25:48 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed Jul 22 14:26:17 2015 +0800

----------------------------------------------------------------------
 .../job/hadoop/cube/MetadataCleanupJob.java     | 160 +++++++++++++++++++
 1 file changed, 160 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8ad54f4f/job/src/main/java/org/apache/kylin/job/hadoop/cube/MetadataCleanupJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/cube/MetadataCleanupJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/cube/MetadataCleanupJob.java
new file mode 100644
index 0000000..005254b
--- /dev/null
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/cube/MetadataCleanupJob.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job.hadoop.cube;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.job.hadoop.AbstractHadoopJob;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ */
+public class MetadataCleanupJob extends AbstractHadoopJob {
+
+    @SuppressWarnings("static-access")
+    private static final Option OPTION_DELETE = OptionBuilder.withArgName("delete").hasArg().isRequired(false).withDescription("Delete the unused metadata").create("delete");
+
+    protected static final Logger log = LoggerFactory.getLogger(MetadataCleanupJob.class);
+
+    boolean delete = false;
+
+    private KylinConfig config = null;
+
+    public static final long TIME_THREADSHOLD = 2 * 26 * 3600 * 1000l; // 2 days
+
+    /*
+     * (non-Javadoc)
+     * 
+     * @see org.apache.hadoop.util.Tool#run(java.lang.String[])
+     */
+    @Override
+    public int run(String[] args) throws Exception {
+        Options options = new Options();
+
+        log.info("----- jobs args: " + Arrays.toString(args));
+        try {
+            options.addOption(OPTION_DELETE);
+            parseOptions(options, args);
+
+            log.info("options: '" + getOptionsAsString() + "'");
+            log.info("delete option value: '" + getOptionValue(OPTION_DELETE) + "'");
+            delete = Boolean.parseBoolean(getOptionValue(OPTION_DELETE));
+
+            config = KylinConfig.getInstanceFromEnv();
+
+            cleanup();
+
+            return 0;
+        } catch (Exception e) {
+            e.printStackTrace(System.err);
+            throw e;
+        }
+    }
+
+    private ResourceStore getStore() {
+        return ResourceStore.getStore(config);
+    }
+
+    private boolean isOlderThanThreshold(long resourceTime) {
+        long currentTime = System.currentTimeMillis();
+
+        if (currentTime - resourceTime > TIME_THREADSHOLD)
+            return true;
+        return false;
+    }
+
+    public void cleanup() throws Exception {
+        CubeManager cubeManager = CubeManager.getInstance(config);
+
+        List<String> activeResourceList = Lists.newArrayList();
+        for (org.apache.kylin.cube.CubeInstance cube : cubeManager.listAllCubes()) {
+            for (org.apache.kylin.cube.CubeSegment segment : cube.getSegments()) {
+                activeResourceList.addAll(segment.getSnapshotPaths());
+                activeResourceList.addAll(segment.getDictionaryPaths());
+                activeResourceList.add(segment.getStatisticsResourcePath());
+            }
+        }
+
+        List<String> toDeleteResource = Lists.newArrayList();
+
+        // two level resources, snapshot tables and cube statistics
+        for (String resourceRoot : new String[]{ResourceStore.SNAPSHOT_RESOURCE_ROOT, ResourceStore.CUBE_STATISTICS_ROOT}) {
+            ArrayList<String> snapshotTables = getStore().listResources(resourceRoot);
+
+            for (String snapshotTable : snapshotTables) {
+                ArrayList<String> snapshotNames = getStore().listResources(snapshotTable);
+                if (snapshotNames != null)
+                    for (String snapshot : snapshotNames) {
+                        if (!activeResourceList.contains(snapshot)) {
+                            if (isOlderThanThreshold(getStore().getResourceTimestamp(snapshot)))
+                                toDeleteResource.add(snapshot);
+                        }
+                    }
+            }
+        }
+
+        // three level resources, only dictionaries
+        ArrayList<String> dictTables = getStore().listResources(ResourceStore.DICT_RESOURCE_ROOT);
+
+        for (String table : dictTables) {
+            ArrayList<String> tableColNames = getStore().listResources(table);
+            if (tableColNames != null)
+                for (String tableCol : tableColNames) {
+                    ArrayList<String> dictionaries = getStore().listResources(tableCol);
+                    if (dictionaries != null)
+                        for (String dict : dictionaries)
+                            if (!activeResourceList.contains(dict)) {
+                                if (isOlderThanThreshold(getStore().getResourceTimestamp(dict)))
+                                    toDeleteResource.add(dict);
+                            }
+                }
+        }
+
+
+        if (toDeleteResource.size() > 0) {
+            logger.info("The following resources have no reference, will be cleaned from metadata store: \n");
+
+            for (String s : toDeleteResource) {
+                logger.info(s);
+                if (delete == true) {
+                    getStore().deleteResource(s);
+                }
+            }
+        } else {
+            logger.info("No resource to be cleaned up from metadata store;");
+        }
+
+    }
+
+    public static void main(String[] args) throws Exception {
+        int exitCode = ToolRunner.run(new MetadataCleanupJob(), args);
+        System.exit(exitCode);
+    }
+}