You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/05/05 10:39:15 UTC
kylin git commit: KYLIN-1614 Diagnosis tool supports 1.3
Repository: kylin
Updated Branches:
refs/heads/master 51c275f3e -> 7b2b9147d
KYLIN-1614 Diagnosis tool supports 1.3
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7b2b9147
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7b2b9147
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7b2b9147
Branch: refs/heads/master
Commit: 7b2b9147db397b85ec45d17bb3787c9ff0f27a68
Parents: 51c275f
Author: lidongsjtu <li...@apache.org>
Authored: Thu May 5 18:39:31 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu May 5 18:39:51 2016 +0800
----------------------------------------------------------------------
.../kylin/tool/AbstractInfoExtractor.java | 1 +
.../apache/kylin/tool/CubeMetaExtractor.java | 53 ++++++++++------
.../org/apache/kylin/tool/DiagnosisInfoCLI.java | 1 +
.../java/org/apache/kylin/tool/ToolUtil.java | 58 -----------------
.../kylin/tool/util/ResourceStoreUtil.java | 67 ++++++++++++++++++++
.../org/apache/kylin/tool/util/ToolUtil.java | 58 +++++++++++++++++
6 files changed, 162 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b2b9147/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java b/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java
index 0706143..9e4e83c 100644
--- a/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java
@@ -34,6 +34,7 @@ import org.apache.kylin.common.KylinVersion;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.common.util.ZipFileUtils;
+import org.apache.kylin.tool.util.ToolUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b2b9147/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
index bfe56e2..a8b732b 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
@@ -19,7 +19,6 @@
package org.apache.kylin.tool;
import java.io.File;
-import java.io.IOException;
import java.util.List;
import org.apache.commons.cli.Option;
@@ -28,7 +27,6 @@ import org.apache.commons.cli.OptionGroup;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.ResourceTool;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeInstance;
@@ -38,6 +36,7 @@ import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.job.dao.ExecutableDao;
import org.apache.kylin.job.dao.ExecutablePO;
import org.apache.kylin.job.exception.PersistentException;
+import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.badquery.BadQueryHistoryManager;
import org.apache.kylin.metadata.model.DataModelDesc;
@@ -51,6 +50,7 @@ import org.apache.kylin.metadata.realization.RealizationRegistry;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.storage.hybrid.HybridInstance;
import org.apache.kylin.storage.hybrid.HybridManager;
+import org.apache.kylin.tool.util.ResourceStoreUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -178,30 +178,27 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
}
try {
- ResourceStore src = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
- ResourceStore dst = ResourceStore.getStore(KylinConfig.createInstanceFromUri(dest));
+ KylinConfig srcConfig = KylinConfig.getInstanceFromEnv();
+ KylinConfig dstConfig = KylinConfig.createInstanceFromUri(dest);
- for (String path : requiredResources) {
- ResourceTool.copyR(src, dst, path);
- }
+ ResourceStoreUtil.copy(srcConfig, dstConfig, requiredResources);
- for (String path : optionalResources) {
- try {
- ResourceTool.copyR(src, dst, path);
- } catch (Exception e) {
- logger.warn("Exception when copying optional resource {}. May be caused by resource missing. Ignore it.");
- }
+ try {
+ ResourceStoreUtil.copy(srcConfig, dstConfig, optionalResources);
+ } catch (Exception e) {
+ logger.warn("Exception when copying optional resource {}. May be caused by resource missing. Ignore it.");
}
+ ResourceStore dstStore = ResourceStore.getStore(dstConfig);
for (CubeInstance cube : cubesToTrimAndSave) {
- CubeInstance trimmedCube = CubeInstance.getCopyOf(cube);
+ CubeInstance trimmedCube = copyCubeInstance(cube);
trimmedCube.getSegments().clear();
trimmedCube.setUuid(cube.getUuid());
- dst.putResource(trimmedCube.getResourcePath(), trimmedCube, CubeManager.CUBE_SERIALIZER);
+ dstStore.putResource(trimmedCube.getResourcePath(), trimmedCube, CubeManager.CUBE_SERIALIZER);
}
- } catch (IOException e) {
- throw new RuntimeException("IOException", e);
+ } catch (Exception e) {
+ throw new RuntimeException("Exception", e);
}
}
@@ -209,6 +206,20 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
return realizationRegistry.getRealization(realizationEntry.getType(), realizationEntry.getRealization());
}
+ // do not call CubeInstance.getCopyOf() to keep backward compatible with 1.3.x
+ private static CubeInstance copyCubeInstance(CubeInstance cubeInstance) {
+ CubeInstance newCube = new CubeInstance();
+ newCube.setName(cubeInstance.getName());
+ newCube.setSegments(cubeInstance.getSegments());
+ newCube.setDescName(cubeInstance.getDescName());
+ newCube.setStatus(cubeInstance.getStatus());
+ newCube.setOwner(cubeInstance.getOwner());
+ newCube.setCost(cubeInstance.getCost());
+ newCube.setCreateTimeUTC(System.currentTimeMillis());
+ newCube.updateRandomUuid();
+ return newCube;
+ }
+
// private void dealWithStreaming(CubeInstance cube) {
// streamingManager = StreamingManager.getInstance(kylinConfig);
// for (StreamingConfig streamingConfig : streamingManager.listAllStreaming()) {
@@ -219,6 +230,10 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
// }
// }
+ private static String concatCubeDescResourcePath(String descName) {
+ return ResourceStore.CUBE_DESC_RESOURCE_ROOT + "/" + descName + MetadataConstants.FILE_SURFIX;
+ }
+
private void retrieveResourcePath(IRealization realization) {
logger.info("Deal with realization {} of type {}", realization.getName(), realization.getType());
@@ -238,7 +253,9 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
}
addRequired(DataModelDesc.concatResourcePath(modelDesc.getName()));
- addRequired(CubeDesc.concatResourcePath(cubeDesc.getName()));
+
+ // backward compatible with 1.3
+ addRequired(concatCubeDescResourcePath(cubeDesc.getName()));
if (includeSegments) {
addRequired(CubeInstance.concatResourcePath(cube.getName()));
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b2b9147/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
index 884a9d4..be41e74 100644
--- a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
@@ -25,6 +25,7 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.io.FileUtils;
import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.tool.util.ToolUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b2b9147/tool/src/main/java/org/apache/kylin/tool/ToolUtil.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/ToolUtil.java b/tool/src/main/java/org/apache/kylin/tool/ToolUtil.java
deleted file mode 100644
index 494f4fe..0000000
--- a/tool/src/main/java/org/apache/kylin/tool/ToolUtil.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.kylin.tool;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.engine.mr.HadoopUtil;
-import org.apache.kylin.storage.hbase.HBaseConnection;
-
-public class ToolUtil {
- public static String getConfFolder() {
- final String CONF = "conf";
- String path = System.getProperty(KylinConfig.KYLIN_CONF);
- if (StringUtils.isNotEmpty(path)) {
- return path;
- }
- path = KylinConfig.getKylinHome();
- if (StringUtils.isNotEmpty(path)) {
- return path + File.separator + CONF;
- }
- return null;
- }
-
- public static String getHBaseMetaStoreId() throws IOException {
- try (final HBaseAdmin hbaseAdmin = new HBaseAdmin(HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration()))) {
- final String metaStoreName = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
- final HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(metaStoreName));
- return desc.getValue(HBaseConnection.HTABLE_UUID_TAG);
- } catch (Exception e) {
- return null;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b2b9147/tool/src/main/java/org/apache/kylin/tool/util/ResourceStoreUtil.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/util/ResourceStoreUtil.java b/tool/src/main/java/org/apache/kylin/tool/util/ResourceStoreUtil.java
new file mode 100644
index 0000000..c7fd4d9
--- /dev/null
+++ b/tool/src/main/java/org/apache/kylin/tool/util/ResourceStoreUtil.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.kylin.tool.util;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.util.List;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.RawResource;
+import org.apache.kylin.common.persistence.ResourceStore;
+
+/**
+ * Created by dongli on 5/5/16.
+ */
+public class ResourceStoreUtil {
+ public static void copy(KylinConfig srcConfig, KylinConfig dstConfig, List<String> paths) throws Exception {
+ ResourceStore src = ResourceStore.getStore(srcConfig);
+ ResourceStore dst = ResourceStore.getStore(dstConfig);
+ for (String path : paths) {
+ rCopy(src, dst, path);
+ }
+ }
+
+ public static void rCopy(ResourceStore src, ResourceStore dst, String path) throws Exception {
+ Method listResourceMethod = ResourceStore.class.getMethod("listResources", String.class);
+ Iterable<String> children = (Iterable<String>) listResourceMethod.invoke(src, path);
+
+ // case of resource (not a folder)
+ if (children == null) {
+ try {
+ RawResource res = src.getResource(path);
+ if (res != null) {
+ dst.putResource(path, res.inputStream, res.timestamp);
+ res.inputStream.close();
+ } else {
+ System.out.println("Resource not exist for " + path);
+ }
+ } catch (Exception ex) {
+ System.err.println("Failed to open " + path);
+ ex.printStackTrace();
+ }
+ }
+ // case of folder
+ else {
+ for (String child : children)
+ rCopy(src, dst, child);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7b2b9147/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java b/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java
new file mode 100644
index 0000000..3fb2c6c
--- /dev/null
+++ b/tool/src/main/java/org/apache/kylin/tool/util/ToolUtil.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.kylin.tool.util;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.storage.hbase.HBaseConnection;
+
+public class ToolUtil {
+ public static String getConfFolder() {
+ final String CONF = "conf";
+ String path = System.getProperty(KylinConfig.KYLIN_CONF);
+ if (StringUtils.isNotEmpty(path)) {
+ return path;
+ }
+ path = KylinConfig.getKylinHome();
+ if (StringUtils.isNotEmpty(path)) {
+ return path + File.separator + CONF;
+ }
+ return null;
+ }
+
+ public static String getHBaseMetaStoreId() throws IOException {
+ try (final HBaseAdmin hbaseAdmin = new HBaseAdmin(HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration()))) {
+ final String metaStoreName = KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix();
+ final HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(metaStoreName));
+ return desc.getValue(HBaseConnection.HTABLE_UUID_TAG);
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+}