You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/09/19 01:44:10 UTC
[1/9] incubator-kylin git commit: minor,
fix test case BadQueryDetectorTest
Repository: incubator-kylin
Updated Branches:
refs/heads/1.x c0ef677cb -> faf401a0f
minor, fix test case BadQueryDetectorTest
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/1e9646fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/1e9646fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/1e9646fc
Branch: refs/heads/1.x
Commit: 1e9646fc956d1acaf1d28bcc473ae1335b544236
Parents: c0ef677
Author: Li, Yang <ya...@ebay.com>
Authored: Mon Sep 14 10:56:23 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Mon Sep 14 10:58:02 2015 +0800
----------------------------------------------------------------------
.../kylin/rest/service/BadQueryDetectorTest.java | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/1e9646fc/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java b/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
index b38ee9d..c849efd 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
@@ -35,7 +35,7 @@ public class BadQueryDetectorTest {
String mockSql = "select * from just_a_test";
final ArrayList<String[]> alerts = new ArrayList<>();
- BadQueryDetector badQueryDetector = new BadQueryDetector(5000, alertMB, alertRunningSec);
+ BadQueryDetector badQueryDetector = new BadQueryDetector(alertRunningSec * 1000, alertMB, alertRunningSec);
badQueryDetector.registerNotifier(new BadQueryDetector.Notifier() {
@Override
public void badQueryFound(String adj, int runningSec, String sql) {
@@ -45,19 +45,25 @@ public class BadQueryDetectorTest {
badQueryDetector.start();
{
+ Thread.sleep(1000);
+
SQLRequest sqlRequest = new SQLRequest();
sqlRequest.setSql(mockSql);
badQueryDetector.queryStart(Thread.currentThread(), sqlRequest);
- Thread.sleep(alertRunningSec * 2 * 1000);
+ // make sure bad query check happens twice
+ Thread.sleep((alertRunningSec * 2 + 1) * 1000);
badQueryDetector.queryEnd(Thread.currentThread());
}
badQueryDetector.stop();
- assertEquals(2, alerts.size());
+ assertEquals(3, alerts.size());
+ // first check founds Low mem
assertArrayEquals(new String[] { "Low mem", mockSql }, alerts.get(0));
- assertArrayEquals(new String[] { "Slow", mockSql }, alerts.get(1));
+ // second check founds Low mem & Slow
+ assertArrayEquals(new String[] { "Low mem", mockSql }, alerts.get(1));
+ assertArrayEquals(new String[] { "Slow", mockSql }, alerts.get(2));
}
}
[3/9] incubator-kylin git commit: update pom settings
Posted by li...@apache.org.
update pom settings
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/df00623f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/df00623f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/df00623f
Branch: refs/heads/1.x
Commit: df00623f555f3e35a8e743e766275e2b461db5d8
Parents: e0f5e39
Author: Luke Han <lu...@apache.org>
Authored: Fri Sep 11 13:52:51 2015 +0800
Committer: Luke Han <lu...@apache.org>
Committed: Tue Sep 15 17:55:10 2015 +0800
----------------------------------------------------------------------
pom.xml | 177 ++++++++++++++++++++++++++++++-----------------------------
1 file changed, 91 insertions(+), 86 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/df00623f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index cdce89e..c614478 100644
--- a/pom.xml
+++ b/pom.xml
@@ -596,6 +596,96 @@
</plugin>
</plugins>
</pluginManagement>
+ <plugins>
+ <!-- Apache-RAT checks for files without headers.
+ If run on a messy developer's sandbox, it will fail.
+ This serves as a reminder to only build a release in a clean
+ sandbox! -->
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <numUnapprovedLicenses>0</numUnapprovedLicenses>
+ <excludes>
+ <exclude>DEPENDENCIES</exclude>
+ <exclude>.idea/**</exclude>
+ <exclude>.git/**</exclude>
+ <exclude>.settings/**</exclude>
+ <!-- text files without comments -->
+ <exclude>**/*.csv</exclude>
+ <exclude>**/*.json</exclude>
+ <exclude>**/*.md</exclude>
+ <!-- binary files -->
+ <exclude>**/*.dict</exclude>
+ <exclude>**/*.dic</exclude>
+ <exclude>**/*.snapshot</exclude>
+ <exclude>**/*.pdf</exclude>
+ <exclude>**/*.log</exclude>
+
+ <exclude>**/.checkstyle</exclude>
+ <!--Job's Test Data-->
+ <exclude>**/src/test/resources/data/**</exclude>
+
+ <!-- generated files -->
+ <exclude>**/target/**</exclude>
+ <exclude>lib/**</exclude>
+ <!-- Kylin's website content -->
+ <exclude>**/.sass-cache/**</exclude>
+
+ <exclude>website/**</exclude>
+
+ <!-- tomcat package -->
+ <exclude>tomcat/**</exclude>
+ <!-- front end libary and generated files -->
+ <exclude>webapp/node_modules/**</exclude>
+ <exclude>webapp/dist/**</exclude>
+ <exclude>webapp/app/components/**</exclude>
+ <!-- json configuration file-->
+ <exclude>webapp/.bowerrc</exclude>
+ <exclude>webapp/.jshintrc</exclude>
+ <!-- generated dict files -->
+ <exclude>dictionary/metastore_db/**</exclude>
+
+ <!-- MIT license -->
+ <exclude>webapp/app/css/AdminLTE-fonts.css</exclude>
+ <exclude>webapp/app/css/AdminLTE.css</exclude>
+
+ <!-- jdbc log -->
+ <exclude>jdbc/kylin_jdbc.log*</exclude>
+ <!-- server log -->
+ <exclude>server/logs/**</exclude>
+
+ <!-- HBase MiniCluster Testing Data, for testing only -->
+ <exclude>examples/test_case_data/minicluster/hbase-export.tar.gz</exclude>
+ <exclude>examples/test_case_data/**/*.xml</exclude>
+
+ </excludes>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>verify</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.maven.doxia</groupId>
+ <artifactId>doxia-core</artifactId>
+ <version>1.6</version>
+ <exclusions>
+ <exclusion>
+ <groupId>xerces</groupId>
+ <artifactId>xercesImpl</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ </plugin>
+
+
+ </plugins>
</build>
<modules>
@@ -782,92 +872,7 @@
<argLine>-Xms1G -Xmx4G -XX:PermSize=128M -XX:MaxPermSize=512M</argLine>
</configuration>
</plugin>
- <!-- Apache-RAT checks for files without headers.
- If run on a messy developer's sandbox, it will fail.
- This serves as a reminder to only build a release in a clean
- sandbox! -->
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <configuration>
- <numUnapprovedLicenses>0</numUnapprovedLicenses>
- <excludes>
- <exclude>DEPENDENCIES</exclude>
- <exclude>.idea/**</exclude>
- <exclude>.git/**</exclude>
- <exclude>.settings/**</exclude>
- <!-- text files without comments -->
- <exclude>**/*.csv</exclude>
- <exclude>**/*.json</exclude>
- <exclude>**/*.md</exclude>
- <!-- binary files -->
- <exclude>**/*.dict</exclude>
- <exclude>**/*.dic</exclude>
- <exclude>**/*.snapshot</exclude>
- <exclude>**/*.pdf</exclude>
- <exclude>**/*.log</exclude>
-
- <exclude>**/.checkstyle</exclude>
- <!--Job's Test Data-->
- <exclude>**/src/test/resources/data/**</exclude>
-
- <!-- generated files -->
- <exclude>**/target/**</exclude>
- <exclude>lib/**</exclude>
- <!-- Kylin's website content -->
- <exclude>**/.sass-cache/**</exclude>
-
- <exclude>website/**</exclude>
-
- <!-- tomcat package -->
- <exclude>tomcat/**</exclude>
- <!-- front end libary and generated files -->
- <exclude>webapp/node_modules/**</exclude>
- <exclude>webapp/dist/**</exclude>
- <exclude>webapp/app/components/**</exclude>
- <!-- json configuration file-->
- <exclude>webapp/.bowerrc</exclude>
- <exclude>webapp/.jshintrc</exclude>
- <!-- generated dict files -->
- <exclude>dictionary/metastore_db/**</exclude>
-
- <!-- MIT license -->
- <exclude>webapp/app/css/AdminLTE-fonts.css</exclude>
- <exclude>webapp/app/css/AdminLTE.css</exclude>
-
- <!-- jdbc log -->
- <exclude>jdbc/kylin_jdbc.log*</exclude>
- <!-- server log -->
- <exclude>server/logs/**</exclude>
-
- <!-- HBase MiniCluster Testing Data, for testing only -->
- <exclude>examples/test_case_data/minicluster/hbase-export.tar.gz</exclude>
- <exclude>examples/test_case_data/**/*.xml</exclude>
-
- </excludes>
- </configuration>
- <executions>
- <execution>
- <phase>verify</phase>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <dependencies>
- <dependency>
- <groupId>org.apache.maven.doxia</groupId>
- <artifactId>doxia-core</artifactId>
- <version>1.6</version>
- <exclusions>
- <exclusion>
- <groupId>xerces</groupId>
- <artifactId>xercesImpl</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- </dependencies>
- </plugin>
+
<plugin>
<groupId>net.ju-n.maven.plugins</groupId>
<artifactId>checksum-maven-plugin</artifactId>
[7/9] incubator-kylin git commit: KYLIN-978 GarbageCollectionStep
dropped Hive Intermediate Table but didn't drop external hdfs path
Posted by li...@apache.org.
KYLIN-978 GarbageCollectionStep dropped Hive Intermediate Table but didn't drop external hdfs path
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/fd172821
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/fd172821
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/fd172821
Branch: refs/heads/1.x
Commit: fd172821d9070b1bd704291a164569dc9d920045
Parents: 8fd1404
Author: sunyerui <su...@gmail.com>
Authored: Fri Sep 11 00:04:42 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed Sep 16 21:02:26 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/job/cube/CubingJobBuilder.java | 47 ++++++++++++--------
.../kylin/job/cube/GarbageCollectionStep.java | 38 ++++++++++++----
2 files changed, 58 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fd172821/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java b/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
index ff79286..de75f7d 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
@@ -25,8 +25,6 @@ import java.util.List;
import java.util.TimeZone;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeSegment;
@@ -65,9 +63,11 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
final CubingJob result = initialJob(seg, "BUILD");
final String jobId = result.getId();
final String cuboidRootPath = getJobWorkingDir(jobId) + "/" + seg.getCubeInstance().getName() + "/cuboid/";
+ final List<String> toDeletePathsOnHadoopCluster = Lists.newArrayList();
+ final List<String> toDeletePathsOnHBaseCluster = Lists.newArrayList();
// cubing
- Pair<AbstractExecutable, AbstractExecutable> twoSteps = addCubingSteps(seg, cuboidRootPath, result);
+ Pair<AbstractExecutable, AbstractExecutable> twoSteps = addCubingSteps(seg, cuboidRootPath, result, toDeletePathsOnHadoopCluster);
String intermediateHiveTableStepId = twoSteps.getFirst().getId();
String baseCuboidStepId = twoSteps.getSecond().getId();
@@ -79,7 +79,8 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
final CubeJoinedFlatTableDesc intermediateTableDesc = new CubeJoinedFlatTableDesc(seg.getCubeDesc(), seg);
final String hiveIntermediateTable = this.getIntermediateHiveTableName(intermediateTableDesc, jobId);
- result.addTask(createGarbageCollectionStep(seg, null, hiveIntermediateTable, null));
+ toDeletePathsOnHBaseCluster.add(getJobWorkingDir(jobId));
+ result.addTask(createGarbageCollectionStep(seg, null, hiveIntermediateTable, toDeletePathsOnHadoopCluster, toDeletePathsOnHBaseCluster));
return result;
}
@@ -92,9 +93,14 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
final String jobId = result.getId();
final String appendRootPath = getJobWorkingDir(jobId) + "/" + appendSegment.getCubeInstance().getName() + "/append_cuboid/";
final String mergedRootPath = getJobWorkingDir(jobId) + "/" + appendSegment.getCubeInstance().getName() + "/cuboid/";
+ List<String> mergingSegmentIds = Lists.newArrayList();
+ List<String> mergingCuboidPaths = Lists.newArrayList();
+ List<String> mergingHTables = Lists.newArrayList();
+ final List<String> toDeletePathsOnHadoopCluster = Lists.newArrayList();
+ final List<String> toDeletePathsOnHBaseCluster = Lists.newArrayList();
// cubing the incremental segment
- Pair<AbstractExecutable, AbstractExecutable> twoSteps = addCubingSteps(appendSegment, appendRootPath, result);
+ Pair<AbstractExecutable, AbstractExecutable> twoSteps = addCubingSteps(appendSegment, appendRootPath, result, toDeletePathsOnHadoopCluster);
final String intermediateHiveTableStepId = twoSteps.getFirst().getId();
final String baseCuboidStepId = twoSteps.getSecond().getId();
@@ -103,10 +109,7 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
List<CubeSegment> mergingSegments = mergeSegment.getCubeInstance().getMergingSegments(mergeSegment);
Preconditions.checkState(mergingSegments.size() > 1, "there should be more than 2 segments to merge");
- List<String> mergingSegmentIds = Lists.newArrayList();
- List<String> mergingCuboidPaths = Lists.newArrayList();
- List<String> mergingHTables = Lists.newArrayList();
- List<String> toDeletePaths = Lists.newArrayList();
+
for (CubeSegment merging : mergingSegments) {
mergingSegmentIds.add(merging.getUuid());
mergingHTables.add(merging.getStorageLocationIdentifier());
@@ -115,7 +118,7 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
} else {
mergingCuboidPaths.add(getPathToMerge(merging));
}
- toDeletePaths.add(getJobWorkingDir(merging.getLastBuildJobID()));
+ toDeletePathsOnHadoopCluster.add(getJobWorkingDir(merging.getLastBuildJobID()));
}
// merge cuboid
@@ -126,7 +129,8 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
// update cube info
result.addTask(createUpdateCubeInfoAfterMergeStep(mergeSegment, mergingSegmentIds, convertCuboidToHfileStep.getId(), jobId));
- result.addTask(createGarbageCollectionStep(mergeSegment, mergingHTables, null, toDeletePaths));
+ toDeletePathsOnHBaseCluster.add(getJobWorkingDir(jobId));
+ result.addTask(createGarbageCollectionStep(mergeSegment, mergingHTables, null, toDeletePathsOnHadoopCluster, toDeletePathsOnHBaseCluster));
return result;
}
@@ -143,12 +147,14 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
List<String> mergingSegmentIds = Lists.newArrayList();
List<String> mergingCuboidPaths = Lists.newArrayList();
List<String> mergingHTables = Lists.newArrayList();
- List<String> toDeletePaths = Lists.newArrayList();
+ final List<String> toDeletePathsOnHadoopCluster = Lists.newArrayList();
+ final List<String> toDeletePathsOnHBaseCluster = Lists.newArrayList();
+
for (CubeSegment merging : mergingSegments) {
mergingSegmentIds.add(merging.getUuid());
mergingCuboidPaths.add(getPathToMerge(merging));
mergingHTables.add(merging.getStorageLocationIdentifier());
- toDeletePaths.add(getJobWorkingDir(merging.getLastBuildJobID()));
+ toDeletePathsOnHadoopCluster.add(getJobWorkingDir(merging.getLastBuildJobID()));
}
// merge cuboid
@@ -159,7 +165,8 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
// update cube info
result.addTask(createUpdateCubeInfoAfterMergeStep(seg, mergingSegmentIds, convertCuboidToHfileStep.getId(), jobId));
- result.addTask(createGarbageCollectionStep(seg, mergingHTables, null, toDeletePaths));
+ toDeletePathsOnHBaseCluster.add(getJobWorkingDir(jobId));
+ result.addTask(createGarbageCollectionStep(seg, mergingHTables, null, toDeletePathsOnHadoopCluster, toDeletePathsOnHBaseCluster));
return result;
}
@@ -171,7 +178,7 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
result.addTask(createMergeCuboidDataStep(seg, formattedPath, mergedCuboidPath));
}
- Pair<AbstractExecutable, AbstractExecutable> addCubingSteps(CubeSegment seg, String cuboidRootPath, CubingJob result) {
+ Pair<AbstractExecutable, AbstractExecutable> addCubingSteps(CubeSegment seg, String cuboidRootPath, CubingJob result, List<String> toDeletePaths) {
final int groupRowkeyColumnsCount = seg.getCubeDesc().getRowkey().getNCuboidBuildLevels();
final int totalRowkeyColumnsCount = seg.getCubeDesc().getRowkey().getRowKeyColumns().length;
@@ -199,6 +206,9 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
result.addTask(createNDimensionCuboidStep(seg, cuboidOutputTempPath, dimNum, totalRowkeyColumnsCount));
}
+ toDeletePaths.add(intermediateHiveTableLocation);
+ toDeletePaths.add(factDistinctColumnsPath);
+
return new Pair<AbstractExecutable, AbstractExecutable>(intermediateHiveTableStep, baseCuboidStep);
}
@@ -266,7 +276,7 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
}
private String getRowkeyDistributionOutputPath(CubeSegment seg, String jobId) {
- return getJobWorkingDir(jobId) + "/" + seg.getCubeInstance().getName() + "/rowkey_stats";
+ return HadoopUtil.makeQualifiedPathInHBaseCluster(getJobWorkingDir(jobId) + "/" + seg.getCubeInstance().getName() + "/rowkey_stats");
}
private String getFactDistinctColumnsPath(CubeSegment seg, String jobUuid) {
@@ -460,12 +470,13 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
return result;
}
- private GarbageCollectionStep createGarbageCollectionStep(CubeSegment seg, List<String> oldHtables, String hiveIntermediateTable, List<String> oldHdsfPaths) {
+ private GarbageCollectionStep createGarbageCollectionStep(CubeSegment seg, List<String> oldHtables, String hiveIntermediateTable, List<String> oldHdsfPaths, List<String> oldHdfsPathsOnHBaseCluster) {
GarbageCollectionStep result = new GarbageCollectionStep();
result.setName(ExecutableConstants.STEP_NAME_GARBAGE_COLLECTION);
result.setOldHTables(oldHtables);
result.setOldHiveTable(hiveIntermediateTable);
- result.setOldHdsfPaths(oldHdsfPaths);
+ result.setOldHdfsPaths(oldHdsfPaths);
+ result.setOldHdfsPathsOnHBaseCluster(oldHdfsPathsOnHBaseCluster);
return result;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/fd172821/job/src/main/java/org/apache/kylin/job/cube/GarbageCollectionStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/GarbageCollectionStep.java b/job/src/main/java/org/apache/kylin/job/cube/GarbageCollectionStep.java
index 72cad96..641454c 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/GarbageCollectionStep.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/GarbageCollectionStep.java
@@ -55,6 +55,8 @@ public class GarbageCollectionStep extends AbstractExecutable {
private static final String OLD_HDFS_PATHS = "oldHdfsPaths";
+ private static final String OLD_HDFS_PATHS_ON_HBASE_CLUSTER = "oldHdfsPathsOnHBaseCluster";
+
private static final Logger logger = LoggerFactory.getLogger(GarbageCollectionStep.class);
private StringBuffer output;
@@ -69,8 +71,9 @@ public class GarbageCollectionStep extends AbstractExecutable {
try {
dropHBaseTable(context);
- dropHdfsPath(context);
dropHiveTable(context);
+ dropHdfsPath(context);
+ dropHdfsPathOnHBaseCluster(context);
} catch (IOException e) {
logger.error("job:" + getId() + " execute finished with exception", e);
output.append("\n").append(e.getLocalizedMessage());
@@ -131,13 +134,11 @@ public class GarbageCollectionStep extends AbstractExecutable {
}
}
}
-
- private void dropHdfsPath(ExecutableContext context) throws IOException {
- List<String> oldHdfsPaths = this.getOldHdsfPaths();
+ private void dropHdfsPathOnCluster(List<String> oldHdfsPaths, FileSystem fileSystem) throws IOException {
if (oldHdfsPaths != null && oldHdfsPaths.size() > 0) {
- Configuration hconf = HadoopUtil.getCurrentConfiguration();
- FileSystem fileSystem = FileSystem.get(hconf);
+ logger.debug("Drop HDFS path on FileSystem: " + fileSystem.getUri());
+ output.append("Drop HDFS path on FileSystem: \"" + fileSystem.getUri() + "\" \n");
for (String path : oldHdfsPaths) {
if (path.endsWith("*"))
path = path.substring(0, path.length() - 1);
@@ -152,10 +153,21 @@ public class GarbageCollectionStep extends AbstractExecutable {
output.append("HDFS path not exists: \"" + path + "\" \n");
}
}
-
}
}
+ private void dropHdfsPath(ExecutableContext context) throws IOException {
+ List<String> oldHdfsPaths = this.getOldHdfsPaths();
+ FileSystem fileSystem = FileSystem.get(HadoopUtil.getCurrentConfiguration());
+ dropHdfsPathOnCluster(oldHdfsPaths, fileSystem);
+ }
+
+ private void dropHdfsPathOnHBaseCluster(ExecutableContext context) throws IOException {
+ List<String> oldHdfsPaths = this.getOldHdfsPathsOnHBaseCluster();
+ FileSystem fileSystem = FileSystem.get(HadoopUtil.getCurrentHBaseConfiguration());
+ dropHdfsPathOnCluster(oldHdfsPaths, fileSystem);
+ }
+
public void setOldHTables(List<String> tables) {
setArrayParam(OLD_HTABLES, tables);
}
@@ -164,14 +176,22 @@ public class GarbageCollectionStep extends AbstractExecutable {
return getArrayParam(OLD_HTABLES);
}
- public void setOldHdsfPaths(List<String> paths) {
+ public void setOldHdfsPaths(List<String> paths) {
setArrayParam(OLD_HDFS_PATHS, paths);
}
- private List<String> getOldHdsfPaths() {
+ private List<String> getOldHdfsPaths() {
return getArrayParam(OLD_HDFS_PATHS);
}
+ public void setOldHdfsPathsOnHBaseCluster(List<String> paths) {
+ setArrayParam(OLD_HDFS_PATHS_ON_HBASE_CLUSTER, paths);
+ }
+
+ private List<String> getOldHdfsPathsOnHBaseCluster() {
+ return getArrayParam(OLD_HDFS_PATHS_ON_HBASE_CLUSTER);
+ }
+
private void setArrayParam(String paramKey, List<String> paramValues) {
setParam(paramKey, StringUtils.join(paramValues, ","));
}
[6/9] incubator-kylin git commit: KYLIN-968
CubeSegment.lastBuildJobID is null in new instance but used for rowkey_stats
path
Posted by li...@apache.org.
KYLIN-968 CubeSegment.lastBuildJobID is null in new instance but used for rowkey_stats path
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/8fd1404d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/8fd1404d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/8fd1404d
Branch: refs/heads/1.x
Commit: 8fd1404dd748b6761ef2e51c197ba5ef4c56c11c
Parents: b003921
Author: sunyerui <su...@gmail.com>
Authored: Thu Sep 10 11:23:16 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Wed Sep 16 21:01:59 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/job/cube/CubingJobBuilder.java | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8fd1404d/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java b/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
index 5c3c277..ff79286 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/CubingJobBuilder.java
@@ -206,9 +206,9 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
final String jobId = result.getId();
final String cuboidPath = cuboidRootPath + "*";
- result.addTask(createRangeRowkeyDistributionStep(seg, cuboidPath));
+ result.addTask(createRangeRowkeyDistributionStep(seg, cuboidPath, jobId));
// create htable step
- result.addTask(createCreateHTableStep(seg));
+ result.addTask(createCreateHTableStep(seg, jobId));
// generate hfiles step
final MapReduceExecutable convertCuboidToHfileStep = createConvertCuboidToHfileStep(seg, cuboidPath, jobId);
result.addTask(convertCuboidToHfileStep);
@@ -265,8 +265,8 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
return getJobWorkingDir(seg.getLastBuildJobID()) + "/" + seg.getCubeInstance().getName() + "/cuboid/*";
}
- private String getRowkeyDistributionOutputPath(CubeSegment seg) {
- return getJobWorkingDir(seg.getLastBuildJobID()) + "/" + seg.getCubeInstance().getName() + "/rowkey_stats";
+ private String getRowkeyDistributionOutputPath(CubeSegment seg, String jobId) {
+ return getJobWorkingDir(jobId) + "/" + seg.getCubeInstance().getName() + "/rowkey_stats";
}
private String getFactDistinctColumnsPath(CubeSegment seg, String jobUuid) {
@@ -347,14 +347,14 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
return ndCuboidStep;
}
- private MapReduceExecutable createRangeRowkeyDistributionStep(CubeSegment seg, String inputPath) {
+ private MapReduceExecutable createRangeRowkeyDistributionStep(CubeSegment seg, String inputPath, String jobId) {
MapReduceExecutable rowkeyDistributionStep = new MapReduceExecutable();
rowkeyDistributionStep.setName(ExecutableConstants.STEP_NAME_GET_CUBOID_KEY_DISTRIBUTION);
StringBuilder cmd = new StringBuilder();
appendMapReduceParameters(cmd, seg);
appendExecCmdParameters(cmd, "input", inputPath);
- appendExecCmdParameters(cmd, "output", getRowkeyDistributionOutputPath(seg));
+ appendExecCmdParameters(cmd, "output", getRowkeyDistributionOutputPath(seg, jobId));
appendExecCmdParameters(cmd, "cubename", seg.getCubeInstance().getName());
appendExecCmdParameters(cmd, "jobname", "Kylin_Region_Splits_Calculator_" + seg.getCubeInstance().getName() + "_Step");
@@ -363,12 +363,12 @@ public final class CubingJobBuilder extends AbstractJobBuilder {
return rowkeyDistributionStep;
}
- private HadoopShellExecutable createCreateHTableStep(CubeSegment seg) {
+ private HadoopShellExecutable createCreateHTableStep(CubeSegment seg, String jobId) {
HadoopShellExecutable createHtableStep = new HadoopShellExecutable();
createHtableStep.setName(ExecutableConstants.STEP_NAME_CREATE_HBASE_TABLE);
StringBuilder cmd = new StringBuilder();
appendExecCmdParameters(cmd, "cubename", seg.getCubeInstance().getName());
- appendExecCmdParameters(cmd, "input", getRowkeyDistributionOutputPath(seg) + "/part-r-00000");
+ appendExecCmdParameters(cmd, "input", getRowkeyDistributionOutputPath(seg, jobId) + "/part-r-00000");
appendExecCmdParameters(cmd, "htablename", seg.getStorageLocationIdentifier());
createHtableStep.setJobParams(cmd.toString());
[4/9] incubator-kylin git commit: publish kylin on spark blog
Posted by li...@apache.org.
publish kylin on spark blog
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/f929892d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/f929892d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/f929892d
Branch: refs/heads/1.x
Commit: f929892dc9a190d2f9a728afa1b1f054ea91203d
Parents: df00623
Author: Luke Han <lu...@apache.org>
Authored: Tue Sep 15 17:54:56 2015 +0800
Committer: Luke Han <lu...@apache.org>
Committed: Tue Sep 15 17:55:10 2015 +0800
----------------------------------------------------------------------
website/_docs/howto/howto_backup_hbase.md | 2 +-
website/_docs/howto/howto_backup_metadata.md | 2 +-
website/_docs/howto/howto_build_cube_with_restapi.md | 2 +-
website/_docs/howto/howto_use_restapi.md | 2 +-
website/_docs/howto/howto_use_restapi_in_js.md | 2 +-
website/_posts/blog/2015-09-06-release-v1.0-incubating.md | 10 +++++-----
website/_posts/blog/2015-09-09-fast-cubing-on-spark.md | 9 +++++++--
7 files changed, 17 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f929892d/website/_docs/howto/howto_backup_hbase.md
----------------------------------------------------------------------
diff --git a/website/_docs/howto/howto_backup_hbase.md b/website/_docs/howto/howto_backup_hbase.md
index cc43f27..8714132 100644
--- a/website/_docs/howto/howto_backup_hbase.md
+++ b/website/_docs/howto/howto_backup_hbase.md
@@ -3,7 +3,7 @@ layout: docs
title: How to Clean/Backup HBase Tables
categories: howto
permalink: /docs/howto/howto_backup_hbase.html
-version: v0.7.2
+version: v1.0
since: v0.7.1
---
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f929892d/website/_docs/howto/howto_backup_metadata.md
----------------------------------------------------------------------
diff --git a/website/_docs/howto/howto_backup_metadata.md b/website/_docs/howto/howto_backup_metadata.md
index c1eb760..1934a60 100644
--- a/website/_docs/howto/howto_backup_metadata.md
+++ b/website/_docs/howto/howto_backup_metadata.md
@@ -3,7 +3,7 @@ layout: docs
title: How to Backup Metadata
categories: howto
permalink: /docs/howto/howto_backup_metadata.html
-version: v0.8
+version: v1.0
since: v0.7.1
---
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f929892d/website/_docs/howto/howto_build_cube_with_restapi.md
----------------------------------------------------------------------
diff --git a/website/_docs/howto/howto_build_cube_with_restapi.md b/website/_docs/howto/howto_build_cube_with_restapi.md
index 0fcdeac..ee4759c 100644
--- a/website/_docs/howto/howto_build_cube_with_restapi.md
+++ b/website/_docs/howto/howto_build_cube_with_restapi.md
@@ -3,7 +3,7 @@ layout: docs
title: How to Build Cube with Restful API
categories: howto
permalink: /docs/howto/howto_build_cube_with_restapi.html
-version: v0.8
+version: v1.0
since: v0.7.1
---
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f929892d/website/_docs/howto/howto_use_restapi.md
----------------------------------------------------------------------
diff --git a/website/_docs/howto/howto_use_restapi.md b/website/_docs/howto/howto_use_restapi.md
index cb19556..667334d 100644
--- a/website/_docs/howto/howto_use_restapi.md
+++ b/website/_docs/howto/howto_use_restapi.md
@@ -3,7 +3,7 @@ layout: docs
title: How to Use Restful API
categories: howto
permalink: /docs/howto/howto_use_restapi.html
-version: v0.8
+version: v1.0
since: v0.7.1
---
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f929892d/website/_docs/howto/howto_use_restapi_in_js.md
----------------------------------------------------------------------
diff --git a/website/_docs/howto/howto_use_restapi_in_js.md b/website/_docs/howto/howto_use_restapi_in_js.md
index 45c8974..d9fa273 100644
--- a/website/_docs/howto/howto_use_restapi_in_js.md
+++ b/website/_docs/howto/howto_use_restapi_in_js.md
@@ -3,7 +3,7 @@ layout: docs
title: How to Use Restful API in Javascript
categories: howto
permalink: /docs/howto/howto_use_restapi_in_js.html
-version: v0.8
+version: v1.0
since: v0.7.1
---
Kypin securicty authorization is based on basic access authorization, so when you want to use API in your javascript,you need to involve the authorization info in http headers.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f929892d/website/_posts/blog/2015-09-06-release-v1.0-incubating.md
----------------------------------------------------------------------
diff --git a/website/_posts/blog/2015-09-06-release-v1.0-incubating.md b/website/_posts/blog/2015-09-06-release-v1.0-incubating.md
index 4861eea..085c7a1 100644
--- a/website/_posts/blog/2015-09-06-release-v1.0-incubating.md
+++ b/website/_posts/blog/2015-09-06-release-v1.0-incubating.md
@@ -10,17 +10,17 @@ The Apache Kylin team is pleased to announce the release of Apache Kylin v1.0 (i
To download Apache Kylin v1.0 (incubating) visit the [download](http://kylin.incubator.apache.org/download) page.
-This is a majority release which brings more stable, robust and well management version, Apache Kylin team resolved about 47 issues including bug fixes, improvements, and few new features.
+This is a major release which brings more stable, robust and well management version, Apache Kylin team resolved about 47 issues including bug fixes, improvements, and few new features.
## Change Highlights
__Kylin Core Improvement__
-* Dynamic Data Model has been supported for new added or removed column in data model without rebuild cube from the beginning [KYLIN-867](https://issues.apache.org/jira/browse/KYLIN-867)
+* Dynamic Data Model has been added to supporting adding or removing column in data model without rebuild cube from the beginning [KYLIN-867](https://issues.apache.org/jira/browse/KYLIN-867)
* Upgraded Apache Calcite to 1.3 for more bug fixes and new SQL functions [KYLIN-881](https://issues.apache.org/jira/browse/KYLIN-881)
* Cleanup job enhanced to make sure there’s no garbage files left in OS and HDFS/HBase after job build [KYLIN-926](https://issues.apache.org/jira/browse/KYLIN-926)
* Added setting option for Hive intermediate tables created by Kylin [KYLIN-883](https://issues.apache.org/jira/browse/KYLIN-883)
-* HBase Corprocessor enhanced to imrpove query performance [KYLIN-857](https://issues.apache.org/jira/browse/KYLIN-857)
+* HBase coprocessor enhanced to imrpove query performance [KYLIN-857](https://issues.apache.org/jira/browse/KYLIN-857)
* Kylin System Dashboard for usage, storage, performance [KYLIN-792](https://issues.apache.org/jira/browse/KYLIN-792)
__Main Bug Fixes__
@@ -34,11 +34,11 @@ __Main Bug Fixes__
__Zeppelin Integration__
-[Apache Zeppelin](http://zeppelin.incubator.apache.org/) is a web-based notebook that enables interactive data analytics. The Apache Kylin team has contributed Kylin Interpreter which enable Zeppelin interactive with Kylin from notebook using ANSI SQL, this interpreter could be found from Zeppelin master code repo [here](https://github.com/apache/incubator-zeppelin/tree/master/kylin).
+[Apache Zeppelin](http://zeppelin.incubator.apache.org/) is a web-based notebook that enables interactive data analytics. The Apache Kylin team has contributed Kylin Interpreter which enables Zeppelin interaction with Kylin from notebook using ANSI SQL, this interpreter could be found from Zeppelin master code repo [here](https://github.com/apache/incubator-zeppelin/tree/master/kylin).
__Upgrade__
-We recommend to upgrade to this version from v0.7.x or even more early version for better performance, stablility and more clear one (most of the intermediate files will be cleaned up automatically). Also to keep up to date with community with latest features and supports.
+We recommend to upgrade to this version from v0.7.x or even more early version for better performance, stablility and clear one (most of the intermediate files will be cleaned up automatically). Also to keep up to date with community with latest features and supports.
Any issue or question during upgrade, please send to Apache Kylin dev mailing list: [dev@kylin.incubator.apache.org](mailto:dev@kylin.incubator.apache.org)
_Great thanks to everyone who contributed!_
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f929892d/website/_posts/blog/2015-09-09-fast-cubing-on-spark.md
----------------------------------------------------------------------
diff --git a/website/_posts/blog/2015-09-09-fast-cubing-on-spark.md b/website/_posts/blog/2015-09-09-fast-cubing-on-spark.md
index fd9e56d..10ee46a 100644
--- a/website/_posts/blog/2015-09-09-fast-cubing-on-spark.md
+++ b/website/_posts/blog/2015-09-09-fast-cubing-on-spark.md
@@ -1,5 +1,10 @@
-# Fast Cubing on Spark in Apache Kylin
-
+---
+layout: post-blog
+title: Fast Cubing on Spark in Apache Kylin
+date: 2015-09-09 15:28:00
+author: Qianhao Zhou
+categories: blog
+---
## Preparation
[9/9] incubator-kylin git commit: minor doc update,
add Recommended Hadoop Versions in Installation guide
Posted by li...@apache.org.
minor doc update, add Recommended Hadoop Versions in Installation guide
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/faf401a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/faf401a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/faf401a0
Branch: refs/heads/1.x
Commit: faf401a0ff4fce487336d46ec01eadabba17e505
Parents: dab3798
Author: Li, Yang <ya...@ebay.com>
Authored: Fri Sep 18 15:44:11 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Fri Sep 18 15:44:11 2015 +0800
----------------------------------------------------------------------
website/_docs/install/index.md | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/faf401a0/website/_docs/install/index.md
----------------------------------------------------------------------
diff --git a/website/_docs/install/index.md b/website/_docs/install/index.md
index 66ad738..e900071 100644
--- a/website/_docs/install/index.md
+++ b/website/_docs/install/index.md
@@ -11,12 +11,13 @@ since: v0.7.1
Kylin requires a properly setup hadoop environment to run. Following are the minimal request to run Kylin, for more detial, please check this reference: [Hadoop Environment](hadoop_env.html).
-## Prerequisites on Hadoop
+## Recommended Hadoop Versions
+
+* Hadoop: 2.4 - 2.7
+* Hive: 0.13 - 0.14
+* HBase: 0.98 - 0.99
+* JDK: 1.7+
-* Hadoop: 2.4+
-* Hive: 0.13+
-* HBase: 0.98+
-* JDK: 1.7+
_Tested with Hortonworks HDP 2.2 and Cloudera Quickstart VM 5.1_
[5/9] incubator-kylin git commit: KYLIN-1026 Error message for git
check is not correct in package.sh
Posted by li...@apache.org.
KYLIN-1026 Error message for git check is not correct in package.sh
Signed-off-by: honma <ho...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b0039214
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b0039214
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b0039214
Branch: refs/heads/1.x
Commit: b003921468318b84b9c5ad3ef131877c728225df
Parents: f929892
Author: zkld123 <zk...@sjtu.edu.cn>
Authored: Tue Sep 15 20:27:42 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Sep 15 20:38:57 2015 +0800
----------------------------------------------------------------------
script/package.sh | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b0039214/script/package.sh
----------------------------------------------------------------------
diff --git a/script/package.sh b/script/package.sh
index 759c082..820dce2 100755
--- a/script/package.sh
+++ b/script/package.sh
@@ -26,9 +26,11 @@ else
echo "maven check passed"
fi
+echo "Checking git..."
+
if [ -z "$(command -v git)" ]
then
- echo "Please install maven first so that Kylin packaging can proceed"
+ echo "Please install git first so that Kylin packaging can proceed"
exit 1
else
echo "git check passed"
[8/9] incubator-kylin git commit: KYLIN-971,
set default value for [kylin.rest.timezone] in kylin.properties
Posted by li...@apache.org.
KYLIN-971, set default value for [kylin.rest.timezone] in kylin.properties
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/dab3798c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/dab3798c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/dab3798c
Branch: refs/heads/1.x
Commit: dab3798c5161f967b5150411d7ef0bfe315e9f20
Parents: fd17282
Author: jiazhong <ji...@ebay.com>
Authored: Thu Sep 17 10:50:18 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Thu Sep 17 10:50:18 2015 +0800
----------------------------------------------------------------------
conf/kylin.properties | 3 +++
examples/test_case_data/sandbox/kylin.properties | 3 +++
2 files changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dab3798c/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/conf/kylin.properties b/conf/kylin.properties
index 84a1d46..2b8441a 100644
--- a/conf/kylin.properties
+++ b/conf/kylin.properties
@@ -20,6 +20,9 @@
# List of web servers in use, this enables one web server instance to sync up with other servers.
kylin.rest.servers=localhost:7070
+#set display timezone on UI,format like[GMT+N or GMT-N]
+kylin.rest.timezone=GMT-8
+
# The metadata store in hbase
kylin.metadata.url=kylin_metadata@hbase
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/dab3798c/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index b3ec4d0..e4b20fe 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -20,6 +20,9 @@
# List of web servers in use, this enables one web server instance to sync up with other servers.
kylin.rest.servers=localhost:7070
+#set display timezone on UI,format like[GMT+N or GMT-N]
+kylin.rest.timezone=GMT-8
+
# The metadata store in hbase
kylin.metadata.url=kylin_metadata@hbase
[2/9] incubator-kylin git commit: KYLIN-910,
enable edit 'auto merge time range' on UI
Posted by li...@apache.org.
KYLIN-910, enable edit 'auto merge time range' on UI
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/e0f5e398
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/e0f5e398
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/e0f5e398
Branch: refs/heads/1.x
Commit: e0f5e3983b127649419cbe33c27e647b8e84029f
Parents: 1e9646f
Author: jiazhong <ji...@ebay.com>
Authored: Tue Sep 15 16:46:19 2015 +0800
Committer: jiazhong <ji...@ebay.com>
Committed: Tue Sep 15 16:46:19 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/CubeInstance.java | 23 +----
.../java/org/apache/kylin/cube/CubeManager.java | 2 +-
.../org/apache/kylin/cube/model/CubeDesc.java | 20 ++--
.../kylin/rest/controller/CubeController.java | 1 -
.../rest/controller/CubeDescController.java | 1 -
.../apache/kylin/rest/request/CubeRequest.java | 8 --
.../apache/kylin/rest/service/CubeService.java | 5 +-
webapp/app/js/controllers/cubeAdvanceSetting.js | 57 +++++++++++
webapp/app/js/filters/filter.js | 10 ++
webapp/app/js/model/cubeDescModel.js | 2 +-
.../cubeDesigner/advanced_settings.html | 102 +++++++++++++++----
11 files changed, 167 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e0f5e398/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index e286c2b..81a64e8 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -55,7 +55,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization {
cubeInstance.setStatus(RealizationStatusEnum.DISABLED);
cubeInstance.updateRandomUuid();
cubeInstance.setProjectName(projectName);
- cubeInstance.setRetentionRange(cubeDesc.getRetentionRange());
return cubeInstance;
}
@@ -83,12 +82,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization {
@JsonProperty("create_time_utc")
private long createTimeUTC;
- @JsonProperty("auto_merge_time_ranges")
- private long[] autoMergeTimeRanges;
-
- @JsonProperty("retention_range")
- private long retentionRange = 0;
-
private String projectName;
public List<CubeSegment> getBuildingSegments() {
@@ -355,13 +348,6 @@ public class CubeInstance extends RootPersistentEntity implements IRealization {
this.projectName = projectName;
}
- public long[] getAutoMergeTimeRanges() {
- return autoMergeTimeRanges;
- }
-
- public void setAutoMergeTimeRanges(long[] autoMergeTimeRanges) {
- this.autoMergeTimeRanges = autoMergeTimeRanges;
- }
public boolean needAutoMerge() {
if (!this.getDescriptor().getModel().getPartitionDesc().isPartitioned())
@@ -370,16 +356,9 @@ public class CubeInstance extends RootPersistentEntity implements IRealization {
if (this.getDescriptor().hasHolisticCountDistinctMeasures())
return false;
- return autoMergeTimeRanges != null && autoMergeTimeRanges.length > 0;
- }
-
- public long getRetentionRange() {
- return retentionRange;
+ return this.getDescriptor().getAutoMergeTimeRanges() != null && this.getDescriptor().getAutoMergeTimeRanges().length > 0;
}
- public void setRetentionRange(long retentionRange) {
- this.retentionRange = retentionRange;
- }
@Override
public long getDateRangeStart() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e0f5e398/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index e4c0e4a..b3eeeef 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -676,7 +676,7 @@ public class CubeManager implements IRealizationProvider {
return null;
}
- long[] timeRanges = cube.getAutoMergeTimeRanges();
+ long[] timeRanges = cube.getDescriptor().getAutoMergeTimeRanges();
Arrays.sort(timeRanges);
CubeSegment newSeg = null;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e0f5e398/cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 72280ea..7f9252c 100644
--- a/cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -55,7 +55,6 @@ import org.apache.kylin.metadata.model.TblColRef;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.collect.Lists;
@@ -115,10 +114,10 @@ public class CubeDesc extends RootPersistentEntity {
private String signature;
@JsonProperty("notify_list")
private List<String> notifyList;
- /*
- * this field will not be serialized ,can be deserialized to front javascript
- */
- private long retentionRange;
+ @JsonProperty("auto_merge_time_ranges")
+ private long[] autoMergeTimeRanges;
+ @JsonProperty("retention_range")
+ private long retentionRange = 0;
private Map<String, Map<String, TblColRef>> columnMap = new HashMap<String, Map<String, TblColRef>>();
private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<TblColRef>();
@@ -768,13 +767,20 @@ public class CubeDesc extends RootPersistentEntity {
this.nullStrings = nullStrings;
}
- @JsonProperty
public long getRetentionRange() {
return retentionRange;
}
- @JsonIgnore
public void setRetentionRange(long retentionRange) {
this.retentionRange = retentionRange;
}
+
+ public long[] getAutoMergeTimeRanges() {
+ return autoMergeTimeRanges;
+ }
+
+ public void setAutoMergeTimeRanges(long[] autoMergeTimeRanges) {
+ this.autoMergeTimeRanges = autoMergeTimeRanges;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e0f5e398/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 734ef32..3d0fca3 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -381,7 +381,6 @@ public class CubeController extends BasicController {
try {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
- cube.setRetentionRange(desc.getRetentionRange());
String projectName = (null == cubeRequest.getProject()) ? ProjectInstance.DEFAULT_PROJECT_NAME : cubeRequest.getProject();
desc = cubeService.updateCubeAndDesc(cube, desc, projectName);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e0f5e398/server/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java b/server/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
index ad22bf1..76da4ea 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
@@ -57,7 +57,6 @@ public class CubeDescController {
return null;
}
CubeDesc cSchema = cubeInstance.getDescriptor();
- cSchema.setRetentionRange(cubeInstance.getRetentionRange());
if (cSchema != null) {
return new CubeDesc[] { cSchema };
} else {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e0f5e398/server/src/main/java/org/apache/kylin/rest/request/CubeRequest.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/request/CubeRequest.java b/server/src/main/java/org/apache/kylin/rest/request/CubeRequest.java
index 8af963c..e5acbe4 100644
--- a/server/src/main/java/org/apache/kylin/rest/request/CubeRequest.java
+++ b/server/src/main/java/org/apache/kylin/rest/request/CubeRequest.java
@@ -28,7 +28,6 @@ public class CubeRequest {
private String message;
private String cubeDescName;
private String project;
- private long retentionRange;
public String getUuid() {
return uuid;
@@ -131,11 +130,4 @@ public class CubeRequest {
this.project = project;
}
- public long getRetentionRange() {
- return retentionRange;
- }
-
- public void setRetentionRange(long retentionRange) {
- this.retentionRange = retentionRange;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e0f5e398/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index be356af..fdfd136 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -574,7 +574,8 @@ public class CubeService extends BasicService {
private void keepCubeRetention(String cubeName) {
CubeInstance cube = getCubeManager().getCube(cubeName);
- if (cube.getRetentionRange() > 0) {
+ CubeDesc desc = cube.getDescriptor();
+ if (desc.getRetentionRange() > 0) {
synchronized (CubeService.class) {
cube = getCubeManager().getCube(cubeName);
List<CubeSegment> readySegs = cube.getSegment(SegmentStatusEnum.READY);
@@ -582,7 +583,7 @@ public class CubeService extends BasicService {
int position = readySegs.size() - 1;
while (position >= 0) {
currentRange += (readySegs.get(position).getDateRangeEnd() - readySegs.get(position).getDateRangeStart());
- if (currentRange >= cube.getRetentionRange()) {
+ if (currentRange >= desc.getRetentionRange()) {
break;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e0f5e398/webapp/app/js/controllers/cubeAdvanceSetting.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 7e3d346..742490d 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -30,6 +30,30 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal, cubeConf
//edit model
if ($scope.state.mode === "edit") {
$scope.metaModel = MetaModel;
+ if(!$scope.cubeMetaFrame.auto_merge_time_ranges){
+ $scope.cubeMetaFrame.auto_merge_time_ranges = [604800000,2419200000];
+ }
+ $scope._auto_merge_time_ranges = [];
+ angular.forEach($scope.cubeMetaFrame.auto_merge_time_ranges,function(item){
+ var _day = Math.floor(item/86400000);
+ var _hour = (item%86400000)/3600000;
+
+ var rangeObj = {
+ type:'days',
+ range:0,
+ mills:0
+ }
+ if(_day==0){
+ rangeObj.type = 'hours';
+ rangeObj.range = _hour;
+ rangeObj.mills = rangeObj.range*3600000;
+ }else{
+ rangeObj.type = 'days';
+ rangeObj.range = _day;
+ rangeObj.mills = rangeObj.range*86400000;
+ }
+ $scope._auto_merge_time_ranges.push(rangeObj);
+ })
}
@@ -37,6 +61,39 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal, cubeConf
if (rowkey_column.dictionary === "true") {
rowkey_column.length = 0;
}
+ }
+
+
+ $scope.addNewMergeTimeRange = function(){
+ $scope._auto_merge_time_ranges.push({
+ type:'days',
+ range:0,
+ mills:0
+ })
+ $scope.updateAutoMergeRange();
+ }
+
+ $scope.removeTimeRange = function(arr,index,item){
+ if (index > -1) {
+ arr.splice(index, 1);
+ }
+ $scope.cubeMetaFrame.auto_merge_time_ranges.splice(index,1);
+ }
+
+
+ $scope.refreshAutoMergeTimeRanges = function(list,index,item){
+ if(item.type=='hours'){
+ item.mills = item.range*3600000;
+ }else{
+ item.mills = item.range*86400000;
+ }
+ $scope.cubeMetaFrame.auto_merge_time_ranges[index] = item.mills;
+ }
+ $scope.updateAutoMergeRange = function(){
+ $scope.cubeMetaFrame.auto_merge_time_ranges = [];
+ angular.forEach($scope._auto_merge_time_ranges,function(item){
+ $scope.cubeMetaFrame.auto_merge_time_ranges.push(item.mills);
+ })
}
});
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e0f5e398/webapp/app/js/filters/filter.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/filters/filter.js b/webapp/app/js/filters/filter.js
index f5d4750..2cf8114 100755
--- a/webapp/app/js/filters/filter.js
+++ b/webapp/app/js/filters/filter.js
@@ -163,4 +163,14 @@ KylinApp
return function (item) {
return item/86400000;
}
+ }).filter('timeRangeFormat', function ($filter) {
+ return function (item) {
+ var _day = Math.floor(item/86400000);
+ var _hour = (item%86400000)/3600000;
+ if(_day==0){
+ return _hour +" (Hours)"
+ }else{
+ return _day +" (Days)";
+ }
+ }
});
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e0f5e398/webapp/app/js/model/cubeDescModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/cubeDescModel.js b/webapp/app/js/model/cubeDescModel.js
index 7d20ce7..b10d1e6 100644
--- a/webapp/app/js/model/cubeDescModel.js
+++ b/webapp/app/js/model/cubeDescModel.js
@@ -49,7 +49,7 @@ KylinApp.service('CubeDescModel', function () {
"hbase_mapping": {
"column_family": []
},
- "retentionRange":"0"
+ "retention_range":"0"
};
return cubeMeta;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e0f5e398/webapp/app/partials/cubeDesigner/advanced_settings.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/cubeDesigner/advanced_settings.html b/webapp/app/partials/cubeDesigner/advanced_settings.html
index e5e83f2..02a83fd 100755
--- a/webapp/app/partials/cubeDesigner/advanced_settings.html
+++ b/webapp/app/partials/cubeDesigner/advanced_settings.html
@@ -21,26 +21,86 @@
<div class="form-group">
- <table style="margin-left:42px; width:92%"
- class="table table-hover list">
- <tr>
- <td>
- <!--Cube Size-->
- <div class="form-group" ng-show="userService.hasRole('ROLE_ADMIN')">
- <div class="row">
- <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Cube Size</b></label>
- <div class="col-xs-12 col-sm-6">
- <select class="form-control" ng-if="state.mode=='edit'" style="width: 360px;"
- chosen ng-model="metaModel.model.capacity" required
- ng-options="cc as cc for cc in cubeConfig.cubeCapacities">
- <option value=""></option>
- </select>
- <span ng-if="state.mode=='view'">{{metaModel.model.capacity}}</span>
- </div>
- </div>
+ <table style="margin-left:42px; width:92%" class="table table-hover list">
+ <tr>
+ <td>
+ <!--Cube Size-->
+ <div class="form-group" ng-show="userService.hasRole('ROLE_ADMIN')">
+ <div class="row">
+ <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Cube Size</b></label>
+ <div class="col-xs-12 col-sm-6">
+ <select class="form-control" ng-if="state.mode=='edit'" style="width: 360px;"
+ chosen ng-model="metaModel.model.capacity" required
+ ng-options="cc as cc for cc in cubeConfig.cubeCapacities">
+ <option value=""></option>
+ </select>
+ <span ng-if="state.mode=='view'">{{metaModel.model.capacity}}</span>
+ </div>
+ </div>
+ </div>
+ </td>
+ </tr>
+ <tr>
+ <td>
+ <!--Cube Size-->
+ <div class="form-group">
+ <div class="row">
+ <label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Auto Merge Time Ranges(days)</b></label>
+ <div class="col-xs-12 col-sm-6">
+ <!--retention range is store in CubeInstance, will convert to cubeMetaFrame for front end-->
+ <div class="row" ng-if="state.mode=='edit'">
+ <table class="table">
+ <tbody>
+ <tr ng-repeat="timeRange in _auto_merge_time_ranges">
+ <td class="col-xs-4">
+ <select ng-if="timeRange.type=='hours'"
+ chosen ng-model="timeRange.range"
+ ng-options="rangeTypes as rangeTypes for rangeTypes in [0.5,1,2,4,8] "
+ ng-change="refreshAutoMergeTimeRanges(_auto_merge_time_ranges,$index,timeRange)"
+ style="width: 100% !important;"
+ data-placeholder="hours range"
+ class="chosen-select">
+ </select>
+ <input type="text" ng-if="timeRange.type=='days'" ng-change="refreshAutoMergeTimeRanges(_auto_merge_time_ranges,$index,timeRange)" class="form-control input-sm" placeholder="days range" ng-model="timeRange.range">
+ </td>
+ <td class="col-xs-4">
+ <select chosen ng-model="timeRange.type"
+ ng-options="rangeTypes as rangeTypes for rangeTypes in ['hours','days'] "
+ ng-change="refreshAutoMergeTimeRanges(_auto_merge_time_ranges,$index,timeRange)"
+ style="width: 200px !important;"
+ data-placeholder="select a project"
+ class="chosen-select">
+ </select>
+ </td>
+ <td>
+ <button class="btn btn-xs btn-info" ng-show="state.mode=='edit'" ng-click="removeTimeRange(_auto_merge_time_ranges,$index, timeRange)">
+ <i class="fa fa-minus"></i>
+ </button>
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ <button class="btn btn-xs btn-info" ng-show="state.mode=='edit'" ng-click="addNewMergeTimeRange()">
+ New Merge Range<i class="fa fa-plus"></i>
+ </button>
</div>
- </td>
- </tr>
+ <div class="row" ng-if="state.mode=='view'">
+ <table class="table">
+ <tbody>
+ <tr ng-repeat="timeRange in cubeMetaFrame.auto_merge_time_ranges">
+ <td>
+ {{timeRange | timeRangeFormat}}
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+
+ </div>
+ </div>
+ </div>
+ </td>
+ </tr>
<tr>
<td>
<!--Cube Size-->
@@ -49,9 +109,9 @@
<label class="control-label col-xs-12 col-sm-3 no-padding-right font-color-default"><b>Retention Range(days)</b></label>
<div class="col-xs-12 col-sm-6">
<!--retention range is store in CubeInstance, will convert to cubeMetaFrame for front end-->
- <input type="text" retention-format class="form-control ng-scope ng-pristine ng-valid" placeholder="how many days cube retention" ng-model="cubeMetaFrame.retentionRange" ng-if="state.mode=='edit'">
+ <input type="text" retention-format class="form-control ng-scope ng-pristine ng-valid" placeholder="how many days cube retention" ng-model="cubeMetaFrame.retention_range" ng-if="state.mode=='edit'">
<small class="help-block text-red" ng-show="state.mode=='edit'">(by default it's '0',which will keep all historic cube segments ,or will keep latest [Retention Range] days cube segments )</small>
- <span ng-if="state.mode=='view'">{{cube.retention_range | millisecondsToDay}}</span>
+ <span ng-if="state.mode=='view'">{{cubeMetaFrame.retention_range | millisecondsToDay}}</span>
</div>
</div>
</div>