You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/03/15 06:42:52 UTC
[1/2] kylin git commit: minor,
add option to listAllCubingJobs precisely cont.
Repository: kylin
Updated Branches:
refs/heads/master 242b33174 -> e4ed2322b
minor, add option to listAllCubingJobs precisely cont.
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f8528637
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f8528637
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f8528637
Branch: refs/heads/master
Commit: f8528637c17d0aa5260df13c0340c314a60c89c1
Parents: 242b331
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Mar 15 14:39:48 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Mar 15 14:39:58 2017 +0800
----------------------------------------------------------------------
.../kylin/rest/controller/JobController.java | 2 +-
.../apache/kylin/rest/service/CubeService.java | 6 +-
.../apache/kylin/rest/service/JobService.java | 118 +++++++------------
.../kylin/rest/service/JobServiceTest.java | 3 +-
4 files changed, 46 insertions(+), 83 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8528637/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
index 12f9e2e..6dc7887 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
@@ -70,7 +70,7 @@ public class JobController extends BasicController {
JobTimeFilterEnum timeFilter = JobTimeFilterEnum.getByCode(jobRequest.getTimeFilter());
try {
- jobInstanceList = jobService.listAllJobs(jobRequest.getCubeName(), jobRequest.getProjectName(), statusList, jobRequest.getLimit(), jobRequest.getOffset(), timeFilter);
+ jobInstanceList = jobService.searchJobs(jobRequest.getCubeName(), jobRequest.getProjectName(), statusList, jobRequest.getLimit(), jobRequest.getOffset(), timeFilter);
} catch (Exception e) {
logger.error(e.getLocalizedMessage(), e);
throw new InternalErrorException(e);
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8528637/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index ba77f06..a6310d1 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -218,7 +218,7 @@ public class CubeService extends BasicService {
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newProjectName, boolean forceUpdate) throws IOException, JobException {
- final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING), true);
+ final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
if (!cubingJobs.isEmpty()) {
throw new JobException("Cube schema shouldn't be changed with running job.");
}
@@ -248,7 +248,7 @@ public class CubeService extends BasicService {
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
public void deleteCube(CubeInstance cube) throws IOException, JobException {
- final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING, ExecutableState.ERROR), true);
+ final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING, ExecutableState.ERROR));
if (!cubingJobs.isEmpty()) {
throw new JobException("The cube " + cube.getName() + " has running or failed job, please discard it and try again.");
}
@@ -340,7 +340,7 @@ public class CubeService extends BasicService {
throw new InternalErrorException("Cube " + cubeName + " doesn't contain any READY segment");
}
- final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING), true);
+ final List<CubingJob> cubingJobs = jobService.listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
if (!cubingJobs.isEmpty()) {
throw new JobException("Enable is not allowed with a running job.");
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8528637/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 1fafb12..9836766 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -138,74 +138,6 @@ public class JobService extends BasicService implements InitializingBean {
}));
}
- public List<JobInstance> listAllJobs(final String cubeName, final String projectName, final List<JobStatusEnum> statusList, final Integer limitValue, final Integer offsetValue, final JobTimeFilterEnum timeFilter) throws IOException, JobException {
- Integer limit = (null == limitValue) ? 30 : limitValue;
- Integer offset = (null == offsetValue) ? 0 : offsetValue;
- List<JobInstance> jobs = listAllJobs(cubeName, projectName, statusList, timeFilter);
- Collections.sort(jobs);
-
- if (jobs.size() <= offset) {
- return Collections.emptyList();
- }
-
- if ((jobs.size() - offset) < limit) {
- return jobs.subList(offset, jobs.size());
- }
-
- return jobs.subList(offset, offset + limit);
- }
-
- public List<JobInstance> listAllJobs(final String cubeName, final String projectName, final List<JobStatusEnum> statusList, final JobTimeFilterEnum timeFilter) {
- Calendar calendar = Calendar.getInstance();
- calendar.setTime(new Date());
- long timeStartInMillis = getTimeStartInMillis(calendar, timeFilter);
- return listCubeJobInstance(cubeName, projectName, statusList, timeStartInMillis, Long.MAX_VALUE);
- }
-
- @Deprecated
- public List<JobInstance> listAllJobs(final String cubeName, final String projectName, final List<JobStatusEnum> statusList, final Integer limitValue, final Integer offsetValue) throws IOException, JobException {
- Integer limit = (null == limitValue) ? 30 : limitValue;
- Integer offset = (null == offsetValue) ? 0 : offsetValue;
- List<JobInstance> jobs = listAllJobs(cubeName, projectName, statusList);
- Collections.sort(jobs);
-
- if (jobs.size() <= offset) {
- return Collections.emptyList();
- }
-
- if ((jobs.size() - offset) < limit) {
- return jobs.subList(offset, jobs.size());
- }
-
- return jobs.subList(offset, offset + limit);
- }
-
- public List<JobInstance> listAllJobs(final String cubeName, final String projectName, final List<JobStatusEnum> statusList) {
- return listCubeJobInstance(cubeName, projectName, statusList);
- }
-
- private List<JobInstance> listCubeJobInstance(final String cubeName, final String projectName, List<JobStatusEnum> statusList, final long timeStartInMillis, final long timeEndInMillis) {
- Set<ExecutableState> states = convertStatusEnumToStates(statusList);
- final Map<String, Output> allOutputs = getExecutableManager().getAllOutputs(timeStartInMillis, timeEndInMillis);
- return Lists.newArrayList(FluentIterable.from(listAllCubingJobs(cubeName, projectName, states, timeStartInMillis, timeEndInMillis, allOutputs, false)).transform(new Function<CubingJob, JobInstance>() {
- @Override
- public JobInstance apply(CubingJob cubingJob) {
- return parseToJobInstance(cubingJob, allOutputs);
- }
- }));
- }
-
- private List<JobInstance> listCubeJobInstance(final String cubeName, final String projectName, List<JobStatusEnum> statusList) {
- Set<ExecutableState> states = convertStatusEnumToStates(statusList);
- final Map<String, Output> allOutputs = getExecutableManager().getAllOutputs();
- return Lists.newArrayList(FluentIterable.from(listAllCubingJobs(cubeName, projectName, states, allOutputs, false)).transform(new Function<CubingJob, JobInstance>() {
- @Override
- public JobInstance apply(CubingJob cubingJob) {
- return parseToJobInstance(cubingJob, allOutputs);
- }
- }));
- }
-
private Set<ExecutableState> convertStatusEnumToStates(List<JobStatusEnum> statusList) {
Set<ExecutableState> states;
if (statusList == null || statusList.isEmpty()) {
@@ -484,11 +416,44 @@ public class JobService extends BasicService implements InitializingBean {
return job;
}
- public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, final Map<String, Output> allOutputs, final boolean bEqual) {
- return listAllCubingJobs(cubeName, projectName, statusList, 0L, Long.MAX_VALUE, allOutputs, bEqual);
+ /**
+ * currently only support substring match
+ * @return
+ */
+ public List<JobInstance> searchJobs(final String cubeNameSubstring, final String projectName, final List<JobStatusEnum> statusList, final Integer limitValue, final Integer offsetValue, final JobTimeFilterEnum timeFilter) throws IOException, JobException {
+ Integer limit = (null == limitValue) ? 30 : limitValue;
+ Integer offset = (null == offsetValue) ? 0 : offsetValue;
+ List<JobInstance> jobs = searchJobs(cubeNameSubstring, projectName, statusList, timeFilter);
+ Collections.sort(jobs);
+
+ if (jobs.size() <= offset) {
+ return Collections.emptyList();
+ }
+
+ if ((jobs.size() - offset) < limit) {
+ return jobs.subList(offset, jobs.size());
+ }
+
+ return jobs.subList(offset, offset + limit);
+ }
+
+ private List<JobInstance> searchJobs(final String cubeNameSubstring, final String projectName, final List<JobStatusEnum> statusList, final JobTimeFilterEnum timeFilter) {
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTime(new Date());
+ long timeStartInMillis = getTimeStartInMillis(calendar, timeFilter);
+
+ long timeEndInMillis = Long.MAX_VALUE;
+ Set<ExecutableState> states = convertStatusEnumToStates(statusList);
+ final Map<String, Output> allOutputs = getExecutableManager().getAllOutputs(timeStartInMillis, timeEndInMillis);
+ return Lists.newArrayList(FluentIterable.from(searchCubingJobs(cubeNameSubstring, projectName, states, timeStartInMillis, timeEndInMillis, allOutputs, false)).transform(new Function<CubingJob, JobInstance>() {
+ @Override
+ public JobInstance apply(CubingJob cubingJob) {
+ return parseToJobInstance(cubingJob, allOutputs);
+ }
+ }));
}
- public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, long timeStartInMillis, long timeEndInMillis, final Map<String, Output> allOutputs, final boolean bEqual) {
+ public List<CubingJob> searchCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, long timeStartInMillis, long timeEndInMillis, final Map<String, Output> allOutputs, final boolean cubeNameExactMatch) {
List<CubingJob> results = Lists.newArrayList(FluentIterable.from(getExecutableManager().getAllAbstractExecutables(timeStartInMillis, timeEndInMillis, CubingJob.class)).filter(new Predicate<AbstractExecutable>() {
@Override
public boolean apply(AbstractExecutable executable) {
@@ -499,7 +464,7 @@ public class JobService extends BasicService implements InitializingBean {
String executableCubeName = CubingExecutableUtil.getCubeName(executable.getParams());
if (executableCubeName == null)
return true;
- if (bEqual)
+ if (cubeNameExactMatch)
return executableCubeName.equalsIgnoreCase(cubeName);
else
return executableCubeName.contains(cubeName);
@@ -537,15 +502,12 @@ public class JobService extends BasicService implements InitializingBean {
return results;
}
- public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList, final boolean bEqual) {
- return listAllCubingJobs(cubeName, projectName, statusList, getExecutableManager().getAllOutputs(), bEqual);
- }
-
public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName, final Set<ExecutableState> statusList) {
- return listAllCubingJobs(cubeName, projectName, statusList, getExecutableManager().getAllOutputs(), false);
+ return searchCubingJobs(cubeName, projectName, statusList, 0L, Long.MAX_VALUE, getExecutableManager().getAllOutputs(), true);
}
public List<CubingJob> listAllCubingJobs(final String cubeName, final String projectName) {
- return listAllCubingJobs(cubeName, projectName, EnumSet.allOf(ExecutableState.class), getExecutableManager().getAllOutputs(), false);
+ return searchCubingJobs(cubeName, projectName, EnumSet.allOf(ExecutableState.class), 0L, Long.MAX_VALUE, getExecutableManager().getAllOutputs(), true);
}
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/f8528637/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
index 0493be6..4150808 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/JobServiceTest.java
@@ -20,6 +20,7 @@ package org.apache.kylin.rest.service;
import java.io.IOException;
+import org.apache.kylin.job.constant.JobTimeFilterEnum;
import org.apache.kylin.job.exception.JobException;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.junit.Assert;
@@ -44,6 +45,6 @@ public class JobServiceTest extends ServiceTestBase {
Assert.assertNotNull(jobService.getMetadataManager());
Assert.assertNotNull(cacheService.getOLAPDataSource(ProjectInstance.DEFAULT_PROJECT_NAME));
Assert.assertNull(jobService.getJobInstance("job_not_exist"));
- Assert.assertNotNull(jobService.listAllJobs(null, null, null));
+ Assert.assertNotNull(jobService.searchJobs(null, null, null, 0, 0, JobTimeFilterEnum.ALL));
}
}
[2/2] kylin git commit: KYLIN-2280 Port config util
Posted by li...@apache.org.
KYLIN-2280 Port config util
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e4ed2322
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e4ed2322
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e4ed2322
Branch: refs/heads/master
Commit: e4ed2322b7ff37a14a0d741766304f46ea58ba69
Parents: f852863
Author: xiefan46 <95...@qq.com>
Authored: Mon Mar 13 15:41:49 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Mar 15 14:40:18 2017 +0800
----------------------------------------------------------------------
build/bin/kylin.sh | 4 +-
build/bin/kylin_port_replace_util.sh | 97 +++++++++++++++++++++++++++++++
2 files changed, 100 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e4ed2322/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index a87fa78..f30b751 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -116,7 +116,9 @@ then
echo ""
echo "A new Kylin instance is started by $USER. To stop it, run 'kylin.sh stop'"
echo "Check the log at ${KYLIN_HOME}/logs/kylin.log"
- echo "Web UI is at http://<hostname>:7070/kylin"
+ kylin_server_port=`sed -n "s/<Connector port=\"\(.*\)\" protocol=\"HTTP\/1.1\"/\1/"p ${KYLIN_HOME}/tomcat/conf/server.xml`
+ kylin_server_port=`echo ${kylin_server_port}` #ignore white space
+ echo "Web UI is at http://<hostname>:${kylin_server_port}/kylin"
exit 0
# stop command
http://git-wip-us.apache.org/repos/asf/kylin/blob/e4ed2322/build/bin/kylin_port_replace_util.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin_port_replace_util.sh b/build/bin/kylin_port_replace_util.sh
new file mode 100755
index 0000000..a51e60d
--- /dev/null
+++ b/build/bin/kylin_port_replace_util.sh
@@ -0,0 +1,97 @@
+#!/bin/bash
+#exit if find error
+# ============================================================================
+
+set -o pipefail # trace ERR through pipes
+set -o errtrace # trace ERR through 'time command' and other functions
+function error() {
+ SCRIPT="$0" # script name
+ LASTLINE="$1" # line of error occurrence
+ LASTERR="$2" # error code
+ echo "ERROR exit from ${SCRIPT} : line ${LASTLINE} with exit code ${LASTERR}"
+ exit 1
+}
+trap 'error ${LINENO} ${?}' ERR
+
+
+#check input parameters
+if [ $# -eq 0 ]; then
+ echo "Usage : port_offset_util.sh set PORT_OFFSET --> Modify all conflict ports base on a offset"
+ echo "Usage : port_offset_util.sh reset --> Recover to original setting"
+ exit 0
+fi
+
+#check kylin home
+if [ -z "$KYLIN_HOME" ]
+then
+ echo 'Please make sure KYLIN_HOME has been set'
+ exit 1
+else
+ echo "KYLIN_HOME is set to ${KYLIN_HOME}"
+fi
+
+#variables
+TOMCAT_BACKUP_FILE="${KYLIN_HOME}/tomcat/conf/server.xml.backup"
+TOMCAT_CONFIG_FILE="${KYLIN_HOME}/tomcat/conf/server.xml"
+KYLIN_CONFIG_FILE="${KYLIN_HOME}/conf/kylin.properties"
+KYLIN_BACKUP_FILE="${KYLIN_HOME}/conf/kylin.properties.backup"
+TOMCAT_PORT_LIST=(9005 7070 9443 7443 9009)
+KYLIN_DEFAULT_PORT=7070
+
+if [ "$1" == "set" ]
+then
+ OFFSET=$2
+ echo "Port offset is : ${OFFSET}"
+
+ #check config file exist
+ if [ ! -f ${KYLIN_CONFIG_FILE} ] || [ ! -f ${TOMCAT_CONFIG_FILE} ]; then
+ echo "Some of the config file not exist"
+ exit 1
+ fi
+
+ #back or reset
+ if [ ! -f ${KYLIN_BACKUP_FILE} ]; then #backup if not exist
+ cp -f ${KYLIN_CONFIG_FILE} ${KYLIN_BACKUP_FILE}
+ else
+ cp -r ${KYLIN_BACKUP_FILE} ${KYLIN_CONFIG_FILE} #reset if exist
+ fi
+
+ if [ ! -f ${TOMCAT_BACKUP_FILE} ]; then #backup if not exist
+ cp -f ${TOMCAT_CONFIG_FILE} ${TOMCAT_BACKUP_FILE}
+ else
+ cp -r ${TOMCAT_BACKUP_FILE} ${TOMCAT_CONFIG_FILE} #reset if exist
+ fi
+
+ #replace ports in kylin.properties
+ new_kylin_port=`expr ${KYLIN_DEFAULT_PORT} + ${OFFSET}`
+
+ sed -i "s/kylin.server.cluster-servers=\(.*\).*:\(.*\)/kylin.server.cluster-servers=\1:${new_kylin_port}/g" ${KYLIN_CONFIG_FILE}
+
+ echo "New kylin port is : ${new_kylin_port}"
+
+ #replace ports in server.xml
+
+ for port in ${TOMCAT_PORT_LIST[@]}
+ do
+ new_port=`expr ${port} + ${OFFSET} `
+ #echo "Replace old port : ${port} to new port : ${new_port}"
+ sed -i "s/$port/${new_port}/g" ${TOMCAT_CONFIG_FILE}
+
+ done
+ echo "Files below modified:"
+ echo ${KYLIN_CONFIG_FILE}
+ echo ${TOMCAT_CONFIG_FILE}
+elif [ "$1" == "reset" ]
+then
+ #reset kylin.properties
+ cp -f ${KYLIN_BACKUP_FILE} ${KYLIN_CONFIG_FILE}
+ cp -f ${TOMCAT_BACKUP_FILE} ${TOMCAT_CONFIG_FILE}
+ rm -f ${KYLIN_BACKUP_FILE}
+ rm -f ${TOMCAT_BACKUP_FILE}
+ echo "Files below reset to original:"
+ echo ${KYLIN_CONFIG_FILE}
+ echo ${TOMCAT_CONFIG_FILE}
+else
+ echo "Unrecognized command"
+ exit 1
+fi