You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/04/14 10:22:38 UTC
[1/3] kylin git commit: minor, refind a API in rangeutil class
Repository: kylin
Updated Branches:
refs/heads/master f3a33673f -> 4a1bc1945
minor, refind a API in rangeutil class
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2174af25
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2174af25
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2174af25
Branch: refs/heads/master
Commit: 2174af259dbf53246bb366c1d605cf997b38c057
Parents: f3a3367
Author: lidongsjtu <li...@apache.org>
Authored: Thu Apr 14 16:17:54 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Apr 14 16:18:09 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/RangeUtil.java | 25 +++++++++++---------
.../apache/kylin/common/util/RangeUtilTest.java | 9 ++++++-
2 files changed, 22 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2174af25/core-common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/RangeUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
index 96c6d9b..ae6cf51 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/RangeUtil.java
@@ -21,8 +21,11 @@ package org.apache.kylin.common.util;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.Iterator;
import java.util.List;
import java.util.NavigableMap;
+import java.util.Set;
+import java.util.SortedSet;
import com.google.common.collect.BoundType;
import com.google.common.collect.Lists;
@@ -198,23 +201,23 @@ public class RangeUtil {
return sb.toString();
}
- public static List<Range<Integer>> buildRanges(int[] values, boolean needSort) {
- ArrayList ranges = Lists.newArrayList();
+ public static ArrayList<Range<Integer>> buildRanges(SortedSet<Integer> values) {
+ ArrayList<Range<Integer>> ranges = Lists.newArrayList();
- if (values == null || values.length == 0)
+ if (values == null || values.isEmpty())
return ranges;
- if (needSort) {
- Arrays.sort(values);
- }
- int lastBegin = values[0];
+ Iterator<Integer> iter = values.iterator();
+ int lastBegin = iter.next();
int lastEnd = lastBegin;
- for (int index = 1; index < values.length; index++) {
- if (values[index] - lastEnd != 1) {
+ int temp = 0;
+ for (int index = 1; index < values.size(); index++) {
+ temp = iter.next();
+ if (temp - lastEnd != 1) {
ranges.add(Ranges.closed(lastBegin, lastEnd));
- lastBegin = values[index];
+ lastBegin = temp;
}
- lastEnd = values[index];
+ lastEnd = temp;
}
ranges.add(Ranges.closed(lastBegin, lastEnd));
return ranges;
http://git-wip-us.apache.org/repos/asf/kylin/blob/2174af25/core-common/src/test/java/org/apache/kylin/common/util/RangeUtilTest.java
----------------------------------------------------------------------
diff --git a/core-common/src/test/java/org/apache/kylin/common/util/RangeUtilTest.java b/core-common/src/test/java/org/apache/kylin/common/util/RangeUtilTest.java
index bc0937e..11bf4ad 100644
--- a/core-common/src/test/java/org/apache/kylin/common/util/RangeUtilTest.java
+++ b/core-common/src/test/java/org/apache/kylin/common/util/RangeUtilTest.java
@@ -21,13 +21,16 @@ package org.apache.kylin.common.util;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
+import java.util.Set;
import java.util.TreeMap;
+import java.util.TreeSet;
import org.junit.Assert;
import org.junit.Test;
import com.google.common.collect.Range;
import com.google.common.collect.Ranges;
+import com.google.common.collect.Sets;
/**
*/
@@ -69,7 +72,11 @@ public class RangeUtilTest {
@Test
public void testBuildRanges() {
int[] test1 = { 1, 2, 3, 5, 7, 8, 10, 4 };
- List<Range<Integer>> ranges = RangeUtil.buildRanges(test1, true);
+ TreeSet<Integer> treeSet = Sets.newTreeSet();
+ for (int t : test1) {
+ treeSet.add(t);
+ }
+ List<Range<Integer>> ranges = RangeUtil.buildRanges(new TreeSet<Integer>(treeSet));
Assert.assertEquals(3, ranges.size());
}
}
[2/3] kylin git commit: minor, add commit id to diagnosis
Posted by li...@apache.org.
minor, add commit id to diagnosis
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6dffd97a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6dffd97a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6dffd97a
Branch: refs/heads/master
Commit: 6dffd97a422589185b4e3c9c6fafafae9f9285d6
Parents: 2174af2
Author: lidongsjtu <li...@apache.org>
Authored: Thu Apr 14 11:20:49 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Apr 14 16:18:26 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinVersion.java | 24 ++++++++++++++++++++
.../apache/kylin/rest/init/ClientInfoTask.java | 1 +
.../org/apache/kylin/tool/DiagnosisInfoCLI.java | 13 ++++++++++-
3 files changed, 37 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6dffd97a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
index 0700968..1c893f3 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
@@ -16,16 +16,23 @@
*/
package org.apache.kylin.common;
+import java.io.File;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
import javax.annotation.Nullable;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
+
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
public class KylinVersion {
+ private static final String COMMIT_SHA1 = "commit_SHA1";
public int major;
public int minor;
@@ -121,6 +128,7 @@ public class KylinVersion {
StringBuilder buf = new StringBuilder();
buf.append("kylin.version:").append(KylinVersion.getCurrentVersion()).append("\n");
+ buf.append("commit:").append(getGitCommitInfo());
buf.append("os.name:").append(System.getProperty("os.name")).append("\n");
buf.append("os.arch:").append(System.getProperty("os.arch")).append("\n");
buf.append("os.version:").append(System.getProperty("os.version")).append("\n");
@@ -129,4 +137,20 @@ public class KylinVersion {
return buf.toString();
}
+
+ public static String getGitCommitInfo() {
+ try {
+ File commitFile = new File(KylinConfig.getKylinHome(), COMMIT_SHA1);
+ List<String> lines = FileUtils.readLines(commitFile);
+ StringBuilder sb = new StringBuilder();
+ for (String line : lines) {
+ if (!line.startsWith("#")) {
+ sb.append(line).append("\n");
+ }
+ }
+ return sb.toString();
+ } catch (Exception e) {
+ return StringUtils.EMPTY;
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6dffd97a/server/src/main/java/org/apache/kylin/rest/init/ClientInfoTask.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/init/ClientInfoTask.java b/server/src/main/java/org/apache/kylin/rest/init/ClientInfoTask.java
index 1cf54a3..0fc0e69 100644
--- a/server/src/main/java/org/apache/kylin/rest/init/ClientInfoTask.java
+++ b/server/src/main/java/org/apache/kylin/rest/init/ClientInfoTask.java
@@ -38,6 +38,7 @@ public class ClientInfoTask extends InitialTask {
StringBuilder buf = new StringBuilder();
buf.append("kylin.version:").append(KylinVersion.getCurrentVersion()).append("\n");
+ buf.append("commit:").append(KylinVersion.getGitCommitInfo());
buf.append("os.name:").append(System.getProperty("os.name")).append("\n");
buf.append("os.arch:").append(System.getProperty("os.arch")).append("\n");
buf.append("os.version:").append(System.getProperty("os.version")).append("\n");
http://git-wip-us.apache.org/repos/asf/kylin/blob/6dffd97a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
index e0e3f2f..a155b8d 100644
--- a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
@@ -137,7 +137,18 @@ public class DiagnosisInfoCLI extends AbstractApplication {
// export conf
if (includeConf) {
logger.info("Start to extract kylin conf files.");
- FileUtils.copyDirectoryToDirectory(new File(getConfFolder()), new File(exportDest));
+ try {
+ FileUtils.copyDirectoryToDirectory(new File(getConfFolder()), new File(exportDest));
+ } catch (Exception e) {
+ logger.warn("Error in export conf.", e);
+ }
+ }
+
+ // export commit id
+ try {
+ FileUtils.copyFileToDirectory(new File(KylinConfig.getKylinHome(), "commit_SHA1"), new File(exportDest));
+ } catch (Exception e) {
+ logger.warn("Error in export commit id.", e);
}
// compress to zip package
[3/3] kylin git commit: minor, add commit to htable
Posted by li...@apache.org.
minor, add commit to htable
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4a1bc194
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4a1bc194
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4a1bc194
Branch: refs/heads/master
Commit: 4a1bc1945c990e9e9752e19f63d27afd3f840941
Parents: 6dffd97
Author: lidongsjtu <li...@apache.org>
Authored: Thu Apr 14 16:16:48 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Apr 14 16:19:34 2016 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/common/KylinVersion.java | 1 -
.../kylin/metadata/realization/IRealizationConstants.java | 2 ++
.../apache/kylin/storage/hbase/steps/CubeHTableUtil.java | 6 ++++++
.../kylin/storage/hbase/util/DeployCoprocessorCLI.java | 9 +++++++++
4 files changed, 17 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a1bc194/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
index 1c893f3..4ed74cd 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinVersion.java
@@ -23,7 +23,6 @@ import java.util.Set;
import javax.annotation.Nullable;
-import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a1bc194/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
index 6c0a973..f5c6428 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java
@@ -40,4 +40,6 @@ public class IRealizationConstants {
public final static String HTableSegmentTag = "SEGMENT";
+ public final static String HTableGitTag = "GIT_COMMIT";
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a1bc194/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
index 65192c2..182cffe 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/CubeHTableUtil.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
import org.apache.hadoop.hbase.security.User;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.KylinVersion;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeDesc;
@@ -67,6 +68,11 @@ public class CubeHTableUtil {
tableDesc.setValue(IRealizationConstants.HTableOwner, kylinConfig.getKylinOwner());
}
+ String commitInfo = KylinVersion.getGitCommitInfo();
+ if (!StringUtils.isEmpty(commitInfo)) {
+ tableDesc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
+ }
+
//HTableUser is the cube owner, which will be the "user"
tableDesc.setValue(IRealizationConstants.HTableUser, cubeInstance.getOwner());
http://git-wip-us.apache.org/repos/asf/kylin/blob/4a1bc194/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index 8bf06b7..2a5c950 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.KylinVersion;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -51,6 +52,7 @@ import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.invertedindex.IISegment;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
@@ -210,6 +212,13 @@ public class DeployCoprocessorCLI {
desc.removeCoprocessor(IIEndpointClassOld);
}
addCoprocessorOnHTable(desc, hdfsCoprocessorJar);
+
+ // update commit tags
+ String commitInfo = KylinVersion.getGitCommitInfo();
+ if (!StringUtils.isEmpty(commitInfo)) {
+ desc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
+ }
+
hbaseAdmin.modifyTable(tableName, desc);
logger.info("Enable " + tableName);