You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/03/14 11:30:45 UTC
[1/3] kylin git commit: KYLIN-2505,
remove hardcode to avoid possible syntax errors
Repository: kylin
Updated Branches:
refs/heads/master b53f2b6d0 -> e7510c270
KYLIN-2505, remove hardcode to avoid possible syntax errors
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3ce48199
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3ce48199
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3ce48199
Branch: refs/heads/master
Commit: 3ce48199164424734696497ddd3eef90efff9563
Parents: b53f2b6
Author: Cheng Wang <ch...@kyligence.io>
Authored: Mon Mar 13 15:39:07 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Mar 14 14:21:27 2017 +0800
----------------------------------------------------------------------
build/bin/sample.sh | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ce48199/build/bin/sample.sh
----------------------------------------------------------------------
diff --git a/build/bin/sample.sh b/build/bin/sample.sh
index 4c65bcf..f4d63d8 100644
--- a/build/bin/sample.sh
+++ b/build/bin/sample.sh
@@ -44,8 +44,9 @@ if [ "${hive_client_mode}" == "beeline" ]
then
beeline_params=`bash ${KYLIN_HOME}/bin/get-properties.sh kylin.source.hive.beeline-params`
beeline ${beeline_params} -e "CREATE DATABASE IF NOT EXISTS "$sample_database
-#### Caution: hard code the beeline parameter by appending sample database.
- beeline ${beeline_params}"/"$sample_database -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
+ hive2_url=`expr match "${beeline_params}" '.*\(hive2:.*:[0-9]\{4,6\}\)'`
+ beeline_params=${beeline_params//${hive2_url}/${hive2_url}/${sample_database}}
+ beeline ${beeline_params} -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
else
hive -e "CREATE DATABASE IF NOT EXISTS "$sample_database
hive --database $sample_database -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; }
[2/3] kylin git commit: minor,
add option to listAllCubingJobs precisely
Posted by li...@apache.org.
minor, add option to listAllCubingJobs precisely
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/87203624
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/87203624
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/87203624
Branch: refs/heads/master
Commit: 872036241c6f8255a07e0f160ba54c9473bf0501
Parents: 3ce4819
Author: Cheng Wang <ch...@kyligence.io>
Authored: Tue Mar 14 14:52:38 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Mar 14 16:43:35 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/rest/service/CubeService.java | 6 ++---
.../apache/kylin/rest/service/JobService.java | 23 +++++++++++++-------
2 files changed, 18 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/87203624/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 9aa7b2a..a4f7c76 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -218,7 +218,7 @@ public class CubeService extends BasicService {
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newProjectName, boolean forceUpdate) throws IOException, JobException {
- final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
+ final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING), true);
if (!cubingJobs.isEmpty()) {
throw new JobException("Cube schema shouldn't be changed with running job.");
}
@@ -248,7 +248,7 @@ public class CubeService extends BasicService {
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
public void deleteCube(CubeInstance cube) throws IOException, JobException {
- final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING, ExecutableState.ERROR));
+ final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING, ExecutableState.ERROR), true);
if (!cubingJobs.isEmpty()) {
throw new JobException("The cube " + cube.getName() + " has running or failed job, please discard it and try again.");
}
@@ -340,7 +340,7 @@ public class CubeService extends BasicService {
throw new InternalErrorException("Cube " + cubeName + " doesn't contain any READY segment");
}
- final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
+ final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING), true);
if (!cubingJobs.isEmpty()) {
throw new JobException("Enable is not allowed with a running job.");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/87203624/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 9618be1..3f17a81 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -187,7 +187,7 @@ public class JobService extends BasicService implements InitializingBean {
private List<JobInstance> listCubeJobInstance(final String cubeName, final String projectName, List<JobStatusEnum> statusList, final long timeStartInMillis, final long timeEndInMillis) {
Set<ExecutableState> states = convertStatusEnumToStates(statusList);
final Map<String, Output> allOutputs = getExecutableManager().getAllOutputs(timeStartInMillis, timeEndInMillis);
- return Lists.newArrayList(FluentIterable.from(listAllCubingJobs(cubeName, projectName, states, timeStartInMillis, timeEndInMillis, allOutputs)).transform(new Function<CubingJob, JobInstance>() {
+ return Lists.newArrayList(FluentIterable.from(listAllCubingJobs(cubeName, projectName, states, timeStartInMillis, timeEndInMillis, allOutputs, false)).transform(new Function<CubingJob, JobInstance>() {
@Override
public JobInstance apply(CubingJob cubingJob) {
return parseToJobInstance(cubingJob, allOutputs);
@@ -198,7 +198,7 @@ public class JobService extends BasicService implements InitializingBean {
private List<JobInstance> listCubeJobInstance(final String cubeName, final String projectName, List<JobStatusEnum> statusList) {
Set<ExecutableState> states = convertStatusEnumToStates(statusList);
final Map<String, Output> allOutputs = getExecutableManager().getAllOutputs();
- return Lists.newArrayList(FluentIterable.from(listAllCubingJobs(cubeName, projectName, states, allOutputs)).transform(new Function<CubingJob, JobInstance>() {
+ return Lists.newArrayList(FluentIterable.from(listAllCubingJobs(cubeName, projectName, states, allOutputs, false)).transform(new Function<CubingJob, JobInstance>() {
@Override
public JobInstance apply(CubingJob cubingJob) {
return parseToJobInstance(cubingJob, allOutputs);
@@ -484,11 +484,11 @@ public class JobService extends BasicService implements InitializingBean {
return job;
}
- public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, final Map<String, Output> allOutputs) {
- return listAllCubingJobs(cubeName, projectName, statusList, 0L, Long.MAX_VALUE, allOutputs);
+ public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, final Map<String, Output> allOutputs, final boolean bEqual) {
+ return listAllCubingJobs(cubeName, projectName, statusList, 0L, Long.MAX_VALUE, allOutputs, bEqual);
}
- public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, long timeStartInMillis, long timeEndInMillis, final Map<String, Output> allOutputs) {
+ public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, long timeStartInMillis, long timeEndInMillis, final Map<String, Output> allOutputs, final boolean bEqual) {
List<CubingJob> results = Lists.newArrayList(FluentIterable.from(getExecutableManager().getAllAbstractExecutables(timeStartInMillis, timeEndInMillis, CubingJob.class)).filter(new Predicate<AbstractExecutable>() {
@Override
public boolean apply(AbstractExecutable executable) {
@@ -499,7 +499,10 @@ public class JobService extends BasicService implements InitializingBean {
String executableCubeName = CubingExecutableUtil.getCubeName(executable.getParams());
if (executableCubeName == null)
return true;
- return executableCubeName.contains(cubeName);
+ if (bEqual)
+ return executableCubeName.equalsIgnoreCase(cubeName);
+ else
+ return executableCubeName.contains(cubeName);
} else {
return false;
}
@@ -534,11 +537,15 @@ public class JobService extends BasicService implements InitializingBean {
return results;
}
+ public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, final boolean bEqual) {
+ return listAllCubingJobs(cubeName, projectName, statusList, getExecutableManager().getAllOutputs(), bEqual);
+ }
+
public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList) {
- return listAllCubingJobs(cubeName, projectName, statusList, getExecutableManager().getAllOutputs());
+ return listAllCubingJobs(cubeName, projectName, statusList, getExecutableManager().getAllOutputs(), false);
}
public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName) {
- return listAllCubingJobs(cubeName, projectName, EnumSet.allOf(ExecutableState.class), getExecutableManager().getAllOutputs());
+ return listAllCubingJobs(cubeName, projectName, EnumSet.allOf(ExecutableState.class), getExecutableManager().getAllOutputs(), false);
}
}
[3/3] kylin git commit: minor, add extract all projects command
Posted by li...@apache.org.
minor, add extract all projects command
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e7510c27
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e7510c27
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e7510c27
Branch: refs/heads/master
Commit: e7510c270642178ca208389dbfaf856c20f9a17d
Parents: 8720362
Author: Roger Shi <ro...@hotmail.com>
Authored: Tue Mar 14 19:22:58 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Mar 14 19:30:33 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/tool/CubeMetaExtractor.java | 45 +++++++++++++-------
1 file changed, 29 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e7510c27/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
index 35831f7..e370e48 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
@@ -23,8 +23,6 @@ import java.io.IOException;
import java.util.List;
import java.util.Set;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.SerializationFeature;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.OptionGroup;
@@ -52,6 +50,7 @@ import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.project.RealizationEntry;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.RealizationRegistry;
+import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.metadata.streaming.StreamingConfig;
import org.apache.kylin.metadata.streaming.StreamingManager;
@@ -61,8 +60,10 @@ import org.apache.kylin.storage.hybrid.HybridManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -80,6 +81,8 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
private static final Option OPTION_HYBRID = OptionBuilder.withArgName("hybrid").hasArg().isRequired(false).withDescription("Specify which hybrid to extract").create("hybrid");
@SuppressWarnings("static-access")
private static final Option OPTION_PROJECT = OptionBuilder.withArgName("project").hasArg().isRequired(false).withDescription("Specify realizations in which project to extract").create("project");
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_All_PROJECT = OptionBuilder.withArgName("allProjects").hasArg(false).isRequired(false).withDescription("Specify realizations in all projects to extract").create("allProjects");
@SuppressWarnings("static-access")
private static final Option OPTION_STORAGE_TYPE = OptionBuilder.withArgName("storageType").hasArg().isRequired(false).withDescription("Specify the storage type to overwrite. Default is empty, keep origin.").create("storageType");
@@ -130,6 +133,7 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
realizationOrProject.addOption(OPTION_CUBE);
realizationOrProject.addOption(OPTION_PROJECT);
realizationOrProject.addOption(OPTION_HYBRID);
+ realizationOrProject.addOption(OPTION_All_PROJECT);
realizationOrProject.setRequired(true);
options.addOptionGroup(realizationOrProject);
@@ -160,23 +164,15 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
realizationRegistry = RealizationRegistry.getInstance(kylinConfig);
badQueryHistoryManager = BadQueryHistoryManager.getInstance(kylinConfig);
- if (optionsHelper.hasOption(OPTION_PROJECT)) {
+ if (optionsHelper.hasOption(OPTION_All_PROJECT)) {
+ for (ProjectInstance projectInstance : projectManager.listAllProjects()) {
+ requireProject(projectInstance);
+ }
+ } else if (optionsHelper.hasOption(OPTION_PROJECT)) {
String projectNames = optionsHelper.getOptionValue(OPTION_PROJECT);
for (String projectName : projectNames.split(",")) {
ProjectInstance projectInstance = projectManager.getProject(projectName);
- if (projectInstance == null) {
- throw new IllegalArgumentException("Project " + projectName + " does not exist");
- }
- addRequired(projectInstance.getResourcePath());
- List<RealizationEntry> realizationEntries = projectInstance.getRealizationEntries();
- for (RealizationEntry realizationEntry : realizationEntries) {
- retrieveResourcePath(getRealization(realizationEntry));
- }
- List<DataModelDesc> modelDescs = metadataManager.getModels(projectName);
- for (DataModelDesc modelDesc : modelDescs) {
- addRequired(DataModelDesc.concatResourcePath(modelDesc.getName()));
- }
- addOptional(badQueryHistoryManager.getBadQueriesForProject(projectName).getResourcePath());
+ requireProject(projectInstance);
}
} else if (optionsHelper.hasOption(OPTION_CUBE)) {
String cubeNames = optionsHelper.getOptionValue(OPTION_CUBE);
@@ -205,6 +201,22 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
engineOverwrite(new File(exportDir.getAbsolutePath()));
}
+ private void requireProject(ProjectInstance projectInstance) throws IOException {
+ if (projectInstance == null) {
+ throw new IllegalArgumentException("Project " + projectInstance.getName() + " does not exist");
+ }
+ addRequired(projectInstance.getResourcePath());
+ List<RealizationEntry> realizationEntries = projectInstance.getRealizationEntries();
+ for (RealizationEntry realizationEntry : realizationEntries) {
+ retrieveResourcePath(getRealization(realizationEntry));
+ }
+ List<DataModelDesc> modelDescs = metadataManager.getModels(projectInstance.getName());
+ for (DataModelDesc modelDesc : modelDescs) {
+ addRequired(DataModelDesc.concatResourcePath(modelDesc.getName()));
+ }
+ addOptional(badQueryHistoryManager.getBadQueriesForProject(projectInstance.getName()).getResourcePath());
+ }
+
private void executeExtraction(String dest) {
logger.info("The resource paths going to be extracted:");
for (String s : requiredResources) {
@@ -359,6 +371,7 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
logger.warn("It's useless to set includeJobs to true when includeSegments is set to false");
}
+ cube.setStatus(RealizationStatusEnum.DISABLED);
cubesToTrimAndSave.add(cube);
}
} else if (realization instanceof HybridInstance) {