You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/04/28 13:19:29 UTC
[1/2] kylin git commit: KYLIN-1637 Fix hbase mapping on sample cube
desc
Repository: kylin
Updated Branches:
refs/heads/master 35bd66403 -> 88c919489
KYLIN-1637 Fix hbase mapping on sample cube desc
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/88c91948
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/88c91948
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/88c91948
Branch: refs/heads/master
Commit: 88c919489c53b3fe30072d77d070da576ab3e544
Parents: 6a5c187
Author: lidongsjtu <li...@apache.org>
Authored: Thu Apr 28 18:54:10 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Apr 28 18:54:18 2016 +0800
----------------------------------------------------------------------
examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/88c91948/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
----------------------------------------------------------------------
diff --git a/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json b/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
index be1b7a9..32b284a 100644
--- a/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
+++ b/examples/sample_cube/metadata/cube_desc/kylin_sales_cube_desc.json
@@ -161,7 +161,7 @@
"name" : "f2",
"columns" : [ {
"qualifier" : "m",
- "measure_refs" : [ "SELLER_CNT_HLL", "SELLER_FORMAT_CNT" ]
+ "measure_refs" : [ "SELLER_CNT_HLL", "SELLER_FORMAT_CNT", "TOP_SELLER" ]
} ]
} ]
},
[2/2] kylin git commit: KYLIN-1614 Job diagnosis
Posted by li...@apache.org.
KYLIN-1614 Job diagnosis
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6a5c187b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6a5c187b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6a5c187b
Branch: refs/heads/master
Commit: 6a5c187bd962d38e6839353b20106584d8a04107
Parents: 35bd664
Author: lidongsjtu <li...@apache.org>
Authored: Thu Apr 28 14:04:21 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Apr 28 18:54:18 2016 +0800
----------------------------------------------------------------------
build/bin/kylin.sh | 6 +--
.../org/apache/kylin/common/KylinVersion.java | 4 +-
.../java/org/apache/kylin/cube/CubeManager.java | 2 +
.../java/org/apache/kylin/cube/CubeSegment.java | 8 ++-
.../kylin/storage/hbase/HBaseConnection.java | 5 ++
tool/pom.xml | 56 ++++++++++++++------
.../apache/kylin/tool/CubeMetaExtractor.java | 3 --
.../org/apache/kylin/tool/DiagnosisInfoCLI.java | 24 +++++++--
.../apache/kylin/tool/HBaseUsageExtractor.java | 3 --
.../org/apache/kylin/tool/JobInfoExtractor.java | 34 +++++++-----
10 files changed, 101 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6a5c187b/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index af90e4c..34014d3 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -202,16 +202,16 @@ then
elif [ "$1" = "diag" ]
then
project="$2"
- if [ -n "$project" ]
+ if [ -z "$project" ]
then
echo "You need to specify a project for diagnosis."
exit 1
fi
destDir="$3"
- if [ -n "$destDir" ]
+ if [ -z "$destDir" ]
then
- $destDir="$KYLIN_HOME/diagnosis_dump/"
+ destDir="$KYLIN_HOME/diagnosis_dump/"
mkdir -p $destDir
fi
http://git-wip-us.apache.org/repos/asf/kylin/blob/6a5c187b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
index 4876dae..992fd5b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
@@ -120,10 +120,10 @@ public class KylinVersion {
}
public static void main(String[] args) {
- System.out.println(getKylinDetailInformation());
+ System.out.println(getKylinClientInformation());
}
- public static String getKylinDetailInformation() {
+ public static String getKylinClientInformation() {
StringBuilder buf = new StringBuilder();
buf.append("kylin.version:").append(KylinVersion.getCurrentVersion()).append("\n");
http://git-wip-us.apache.org/repos/asf/kylin/blob/6a5c187b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 01b05da..87a866a 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -167,7 +167,9 @@ public class CubeManager implements IRealizationProvider {
DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(),true, col, factTableValueProvider);
if (dictInfo != null) {
+ Dictionary dict = dictInfo.getDictionaryObject();
cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
+ cubeSeg.getRowkeyStats().add(new Object[]{col.getName(), dict.getSize(), dict.getSizeOfId()});
CubeUpdate cubeBuilder = new CubeUpdate(cubeSeg.getCubeInstance());
cubeBuilder.setToUpdateSegs(cubeSeg);
http://git-wip-us.apache.org/repos/asf/kylin/blob/6a5c187b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
index 10642c4..0d8da6f 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeSegment.java
@@ -94,6 +94,9 @@ public class CubeSegment implements Comparable<CubeSegment>, IRealizationSegment
@JsonProperty("index_path")
private String indexPath;
+ @JsonProperty("rowkey_stats")
+ private List<Object[]> rowkeyStats = Lists.newArrayList();
+
private volatile Map<Long, Short> cuboidBaseShards = Maps.newHashMap();//cuboid id ==> base(starting) shard for this cuboid
public CubeDesc getCubeDesc() {
@@ -231,10 +234,13 @@ public class CubeSegment implements Comparable<CubeSegment>, IRealizationSegment
@Override
public String getStorageLocationIdentifier() {
-
return storageLocationIdentifier;
}
+ public List<Object[]> getRowkeyStats() {
+ return rowkeyStats;
+ }
+
public Map<String, String> getDictionaries() {
if (dictionaries == null)
dictionaries = new ConcurrentHashMap<String, String>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/6a5c187b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index 661e8e4..e2f4f3a 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -20,6 +20,7 @@ package org.apache.kylin.storage.hbase;
import java.io.IOException;
import java.util.Map;
+import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.lang.StringUtils;
@@ -46,6 +47,8 @@ import org.slf4j.LoggerFactory;
*/
public class HBaseConnection {
+ public static final String HTABLE_UUID_TAG = "UUID";
+
private static final Logger logger = LoggerFactory.getLogger(HBaseConnection.class);
private static final Map<String, Configuration> ConfigCache = new ConcurrentHashMap<String, Configuration>();
@@ -197,6 +200,8 @@ public class HBaseConnection {
desc.addFamily(fd);
}
}
+
+ desc.setValue(HTABLE_UUID_TAG, UUID.randomUUID().toString());
hbase.createTable(desc);
logger.debug("HTable '" + tableName + "' created");
http://git-wip-us.apache.org/repos/asf/kylin/blob/6a5c187b/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index 7df3f0c..7fc08f4 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -31,11 +31,6 @@
<dependencies>
<dependency>
<groupId>org.apache.kylin</groupId>
- <artifactId>kylin-source-hive</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
<artifactId>kylin-source-kafka</artifactId>
<version>${project.parent.version}</version>
</dependency>
@@ -49,16 +44,6 @@
<artifactId>kylin-engine-mr</artifactId>
<version>${project.parent.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-engine-streaming</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.kylin</groupId>
- <artifactId>kylin-invertedindex</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
<!--Env-->
<dependency>
@@ -73,4 +58,45 @@
</dependency>
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.3</version>
+
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <minimizeJar>false</minimizeJar>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>tool</shadedClassifierName>
+ <artifactSet>
+ <excludes>
+ <exclude>io.netty:*</exclude>
+ <exclude>org.apache.zookeeper:*</exclude>
+ <exclude>net.sf.ehcache:*</exclude>
+ </excludes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
</project>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6a5c187b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
index e74b4eb..6b56201 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
@@ -39,7 +39,6 @@ import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.engine.streaming.StreamingConfig;
import org.apache.kylin.engine.streaming.StreamingManager;
-import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.job.dao.ExecutableDao;
import org.apache.kylin.job.dao.ExecutablePO;
import org.apache.kylin.job.exception.PersistentException;
@@ -321,8 +320,6 @@ public class CubeMetaExtractor extends AbstractApplication {
}
retrieveResourcePath(iRealization);
}
- } else if (realization instanceof IIInstance) {
- logger.warn("Does not support extract II instance or hybrid that contains II");
} else {
logger.warn("Unknown realization type: " + realization.getType());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6a5c187b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
index 98302ae..a403ee2 100644
--- a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
@@ -19,10 +19,9 @@
package org.apache.kylin.tool;
import java.io.File;
-import java.io.FilenameFilter;
+import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
-import java.util.Calendar;
import java.util.Date;
import org.apache.commons.cli.Option;
@@ -30,10 +29,17 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.KylinVersion;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.common.util.ZipFileUtils;
+import org.apache.kylin.engine.mr.HadoopUtil;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -140,7 +146,7 @@ public class DiagnosisInfoCLI extends AbstractApplication {
}
}
- // export os conf - linux
+ // export os (linux)
if (includeLinux) {
File linuxDir = new File(exportDir, "linux");
FileUtils.forceMkdir(linuxDir);
@@ -178,6 +184,11 @@ public class DiagnosisInfoCLI extends AbstractApplication {
FileUtils.writeStringToFile(new File(basicDir, "process"), output);
output = kylinConfig.getCliCommandExecutor().execute("lsb_release -a").getSecond();
FileUtils.writeStringToFile(new File(basicDir, "lsb_release"), output);
+ output = KylinVersion.getKylinClientInformation();
+ FileUtils.writeStringToFile(new File(basicDir, "client"), output);
+ output = getHBaseMetaStoreId();
+ FileUtils.writeStringToFile(new File(basicDir, "client"), output, true);
+
} catch (Exception e) {
logger.warn("Error in export process info.", e);
}
@@ -240,4 +251,11 @@ public class DiagnosisInfoCLI extends AbstractApplication {
}
return null;
}
+
+ private String getHBaseMetaStoreId() throws IOException {
+ HBaseAdmin hbaseAdmin = new HBaseAdmin(HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration()));
+ String metaStoreName = kylinConfig.getMetadataUrlPrefix();
+ HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(metaStoreName));
+ return "MetaStore UUID: " + desc.getValue(HBaseConnection.HTABLE_UUID_TAG);
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6a5c187b/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java b/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java
index 0c1bf47..8d69805 100644
--- a/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java
@@ -30,8 +30,6 @@ import org.apache.commons.cli.Options;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.client.HConnection;
-import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.kylin.common.KylinConfig;
@@ -46,7 +44,6 @@ import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.project.RealizationEntry;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.RealizationRegistry;
-import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/6a5c187b/tool/src/main/java/org/apache/kylin/tool/JobInfoExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/JobInfoExtractor.java b/tool/src/main/java/org/apache/kylin/tool/JobInfoExtractor.java
index 43758e0..11abc38 100644
--- a/tool/src/main/java/org/apache/kylin/tool/JobInfoExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/JobInfoExtractor.java
@@ -32,8 +32,6 @@ import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.ResourceTool;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
-import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
-import org.apache.kylin.job.common.ShellExecutable;
import org.apache.kylin.job.constant.ExecutableConstants;
import org.apache.kylin.job.dao.ExecutableDao;
import org.apache.kylin.job.dao.ExecutablePO;
@@ -42,9 +40,6 @@ import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
-/**
- * Created by dongli on 3/29/16.
- */
public class JobInfoExtractor extends AbstractApplication {
private static final Logger logger = LoggerFactory.getLogger(JobInfoExtractor.class);
@@ -116,7 +111,7 @@ public class JobInfoExtractor extends AbstractApplication {
executeExtraction(dest);
if (includeCube) {
- String cubeName = CubingExecutableUtil.getCubeName(executablePO.getParams());
+ String cubeName = executablePO.getParams().get("cubename");
String[] cubeMetaArgs = { "-cube", cubeName, "-destDir", dest + "cube_" + cubeName + "/", "-includeJobs", "false" };
logger.info("Start to extract related cube: " + StringUtils.join(cubeMetaArgs));
cubeMetaExtractor.execute(cubeMetaArgs);
@@ -125,7 +120,7 @@ public class JobInfoExtractor extends AbstractApplication {
if (includeYarnLogs) {
logger.info("Start to related yarn job logs: " + jobId);
for (String taskId : yarnLogsResources) {
- extractYarnLog(taskId, dest + "yarn_" + jobId + "/");
+ extractYarnLog(taskId, dest + "yarn_" + jobId + "/", true);
}
}
@@ -151,19 +146,30 @@ public class JobInfoExtractor extends AbstractApplication {
}
}
- private void extractYarnLog(String taskId, String dest) throws Exception {
+ private void extractYarnLog(String taskId, String dest, boolean onlySucc) throws Exception {
final Map<String, String> jobInfo = executableDao.getJobOutput(taskId).getInfo();
if (jobInfo.containsKey(ExecutableConstants.MR_JOB_ID)) {
String applicationId = jobInfo.get(ExecutableConstants.MR_JOB_ID).replace("job", "application");
- File destFile = new File(dest + applicationId + ".log");
+ if (!onlySucc || isYarnAppSucc(applicationId)) {
+ File destFile = new File(dest + applicationId + ".log");
- ShellExecutable yarnExec = new ShellExecutable();
- yarnExec.setCmd("yarn logs -applicationId " + applicationId + " > " + destFile.getAbsolutePath());
- yarnExec.setName(yarnExec.getCmd());
+ String yarnCmd = "yarn logs -applicationId " + applicationId + " > " + destFile.getAbsolutePath();
+ logger.info(yarnCmd);
+ kylinConfig.getCliCommandExecutor().execute(yarnCmd);
+ }
+ }
+ }
- logger.info(yarnExec.getCmd());
- kylinConfig.getCliCommandExecutor().execute(yarnExec.getCmd(), null);
+ private boolean isYarnAppSucc(String applicationId) throws IOException {
+ final String yarnCmd = "yarn application -status " + applicationId;
+ final String cmdOutput = kylinConfig.getCliCommandExecutor().execute(yarnCmd).getSecond();
+ final String[] cmdOutputLines = cmdOutput.split("\n");
+ for (String cmdOutputLine : cmdOutputLines) {
+ if (cmdOutputLine.equals("Final-State : SUCCEEDED")) {
+ return true;
+ }
}
+ return false;
}
private void addRequired(String record) {