You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/10/26 03:31:48 UTC
[1/2] incubator-kylin git commit: KYLIN-1041 fix UT
Repository: incubator-kylin
Updated Branches:
refs/heads/2.x-staging 7e66013ea -> 06aeb994e
KYLIN-1041 fix UT
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/3cc3501e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/3cc3501e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/3cc3501e
Branch: refs/heads/2.x-staging
Commit: 3cc3501ecc3791a63d29a4c8ca3347c027216043
Parents: 7e66013
Author: honma <ho...@ebay.com>
Authored: Mon Oct 26 10:35:52 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon Oct 26 10:35:52 2015 +0800
----------------------------------------------------------------------
.../kylin/rest/controller/CubeController.java | 42 +++++++++-----------
.../rest/controller/CubeControllerTest.java | 7 ++++
.../hbase/cube/v2/CubeHBaseEndpointRPC.java | 2 +-
3 files changed, 27 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3cc3501e/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 9ce6452..d60cf54 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -47,7 +47,6 @@ import org.apache.kylin.rest.exception.NotFoundException;
import org.apache.kylin.rest.request.CubeRequest;
import org.apache.kylin.rest.request.CubeSegmentRequest;
import org.apache.kylin.rest.request.JobBuildRequest;
-import org.apache.kylin.rest.request.StreamingRequest;
import org.apache.kylin.rest.response.GeneralResponse;
import org.apache.kylin.rest.response.HBaseResponse;
import org.apache.kylin.rest.service.CubeService;
@@ -93,7 +92,7 @@ public class CubeController extends BasicController {
@Autowired
private JobService jobService;
- @RequestMapping(value = "", method = {RequestMethod.GET})
+ @RequestMapping(value = "", method = { RequestMethod.GET })
@ResponseBody
public List<CubeInstance> getCubes(@RequestParam(value = "cubeName", required = false) String cubeName, @RequestParam(value = "modelName", required = false) String modelName, @RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value = "limit", required = false) Integer limit, @RequestParam(value = "offset", required = false) Integer offset) {
return cubeService.getCubes(cubeName, projectName, modelName, limit, offset);
@@ -107,7 +106,7 @@ public class CubeController extends BasicController {
* @throws UnknownHostException
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/segs/{segmentName}/sql", method = {RequestMethod.GET})
+ @RequestMapping(value = "/{cubeName}/segs/{segmentName}/sql", method = { RequestMethod.GET })
@ResponseBody
public GeneralResponse getSql(@PathVariable String cubeName, @PathVariable String segmentName) {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
@@ -129,7 +128,7 @@ public class CubeController extends BasicController {
* @param notifyList
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/notify_list", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/notify_list", method = { RequestMethod.PUT })
@ResponseBody
public void updateNotifyList(@PathVariable String cubeName, @RequestBody List<String> notifyList) {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
@@ -147,7 +146,7 @@ public class CubeController extends BasicController {
}
- @RequestMapping(value = "/{cubeName}/cost", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/cost", method = { RequestMethod.PUT })
@ResponseBody
public CubeInstance updateCubeCost(@PathVariable String cubeName, @RequestParam(value = "cost") int cost) {
try {
@@ -159,7 +158,7 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}/coprocessor", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/coprocessor", method = { RequestMethod.PUT })
@ResponseBody
public Map<String, Boolean> updateCubeCoprocessor(@PathVariable String cubeName, @RequestParam(value = "force") String force) {
try {
@@ -177,7 +176,7 @@ public class CubeController extends BasicController {
*
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/segs/{segmentName}/refresh_lookup", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/segs/{segmentName}/refresh_lookup", method = { RequestMethod.PUT })
@ResponseBody
public CubeInstance rebuildLookupSnapshot(@PathVariable String cubeName, @PathVariable String segmentName, @RequestParam(value = "lookupTable") String lookupTable) {
try {
@@ -195,7 +194,7 @@ public class CubeController extends BasicController {
* @return
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/rebuild", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/rebuild", method = { RequestMethod.PUT })
@ResponseBody
public JobInstance rebuild(@PathVariable String cubeName, @RequestBody JobBuildRequest jobBuildRequest) {
try {
@@ -212,7 +211,7 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}/disable", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/disable", method = { RequestMethod.PUT })
@ResponseBody
public CubeInstance disableCube(@PathVariable String cubeName) {
try {
@@ -230,7 +229,7 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}/purge", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/purge", method = { RequestMethod.PUT })
@ResponseBody
public CubeInstance purgeCube(@PathVariable String cubeName) {
try {
@@ -248,7 +247,7 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}/enable", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/enable", method = { RequestMethod.PUT })
@ResponseBody
public CubeInstance enableCube(@PathVariable String cubeName) {
try {
@@ -265,7 +264,7 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}", method = {RequestMethod.DELETE})
+ @RequestMapping(value = "/{cubeName}", method = { RequestMethod.DELETE })
@ResponseBody
public void deleteCube(@PathVariable String cubeName) {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
@@ -275,8 +274,8 @@ public class CubeController extends BasicController {
//drop related StreamingConfig KafkaConfig if exist
try {
- List<StreamingConfig> configs= streamingService.listAllStreamingConfigs(cubeName);
- for(StreamingConfig config:configs){
+ List<StreamingConfig> configs = streamingService.listAllStreamingConfigs(cubeName);
+ for (StreamingConfig config : configs) {
try {
streamingService.dropStreamingConfig(config);
} catch (IOException e) {
@@ -310,7 +309,7 @@ public class CubeController extends BasicController {
* @return Table metadata array
* @throws IOException
*/
- @RequestMapping(value = "", method = {RequestMethod.POST})
+ @RequestMapping(value = "", method = { RequestMethod.POST })
@ResponseBody
public CubeRequest saveCubeDesc(@RequestBody CubeRequest cubeRequest) {
@@ -337,7 +336,7 @@ public class CubeController extends BasicController {
}
//streaming Cube
- if (cubeRequest.getStreamingCube().equals("true")) {
+ if (cubeRequest.getStreamingCube() != null && cubeRequest.getStreamingCube().equals("true")) {
StreamingConfig streamingConfig = deserializeStreamingDesc(cubeRequest);
KafkaConfig kafkaConfig = deserializeKafkaDesc(cubeRequest);
if (streamingConfig == null) {
@@ -392,7 +391,6 @@ public class CubeController extends BasicController {
}
-
cubeRequest.setUuid(desc.getUuid());
cubeRequest.setSuccessful(true);
return cubeRequest;
@@ -405,7 +403,7 @@ public class CubeController extends BasicController {
* @throws JsonProcessingException
* @throws IOException
*/
- @RequestMapping(value = "", method = {RequestMethod.PUT})
+ @RequestMapping(value = "", method = { RequestMethod.PUT })
@ResponseBody
public CubeRequest updateCubeDesc(@RequestBody CubeRequest cubeRequest) throws JsonProcessingException {
@@ -486,7 +484,6 @@ public class CubeController extends BasicController {
}
-
String descData = JsonUtil.writeValueAsIndentString(desc);
cubeRequest.setCubeDescData(descData);
cubeRequest.setSuccessful(true);
@@ -499,7 +496,7 @@ public class CubeController extends BasicController {
* @return true
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/hbase", method = {RequestMethod.GET})
+ @RequestMapping(value = "/{cubeName}/hbase", method = { RequestMethod.GET })
@ResponseBody
public List<HBaseResponse> getHBaseInfo(@PathVariable String cubeName) {
List<HBaseResponse> hbase = new ArrayList<HBaseResponse>();
@@ -536,7 +533,7 @@ public class CubeController extends BasicController {
return hbase;
}
- @RequestMapping(value = "/{cubeName}/segments", method = {RequestMethod.POST})
+ @RequestMapping(value = "/{cubeName}/segments", method = { RequestMethod.POST })
@ResponseBody
public CubeSegmentRequest appendSegment(@PathVariable String cubeName, @RequestBody CubeSegmentRequest cubeSegmentRequest) {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
@@ -619,7 +616,6 @@ public class CubeController extends BasicController {
return desc;
}
-
private KafkaConfig deserializeKafkaDesc(CubeRequest cubeRequest) {
KafkaConfig desc = null;
try {
@@ -643,7 +639,7 @@ public class CubeController extends BasicController {
*/
private String omitMessage(List<String> errors) {
StringBuffer buffer = new StringBuffer();
- for (Iterator<String> iterator = errors.iterator(); iterator.hasNext(); ) {
+ for (Iterator<String> iterator = errors.iterator(); iterator.hasNext();) {
String string = (String) iterator.next();
buffer.append(string);
buffer.append("\n");
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3cc3501e/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
index 189bad2..c313cc3 100644
--- a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
@@ -28,6 +28,7 @@ import org.apache.kylin.rest.request.CubeRequest;
import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.JobService;
import org.apache.kylin.rest.service.ServiceTestBase;
+import org.apache.kylin.rest.service.StreamingService;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -48,6 +49,8 @@ public class CubeControllerTest extends ServiceTestBase {
CubeService cubeService;
@Autowired
JobService jobService;
+ @Autowired
+ StreamingService streamingService;
@Before
public void setup() throws Exception {
@@ -56,8 +59,12 @@ public class CubeControllerTest extends ServiceTestBase {
cubeController = new CubeController();
cubeController.setCubeService(cubeService);
cubeController.setJobService(jobService);
+ cubeController.setStreamingService(streamingService);
+
+
cubeDescController = new CubeDescController();
cubeDescController.setCubeService(cubeService);
+
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/3cc3501e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
index 86bc42d..8557da9 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java
@@ -46,10 +46,10 @@ import org.apache.kylin.gridtable.GTInfo;
import org.apache.kylin.gridtable.GTRecord;
import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.IGTScanner;
+import org.apache.kylin.storage.hbase.HBaseConnection;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitRequest.IntList;
import org.apache.kylin.storage.hbase.cube.v2.coprocessor.endpoint.generated.CubeVisitProtos.CubeVisitResponse.Stats;
-import org.apache.kylin.storage.hbase.steps.HBaseConnection;
import com.google.common.base.Function;
import com.google.common.collect.Collections2;
[2/2] incubator-kylin git commit: KYLIN-1093 fix UT
Posted by ma...@apache.org.
KYLIN-1093 fix UT
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/06aeb994
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/06aeb994
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/06aeb994
Branch: refs/heads/2.x-staging
Commit: 06aeb994ed277272a84506b62be18bc12d9c28ed
Parents: 3cc3501
Author: honma <ho...@ebay.com>
Authored: Mon Oct 26 10:36:07 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon Oct 26 10:36:07 2015 +0800
----------------------------------------------------------------------
.../test_case_data/localmeta/kylin.properties | 2 +-
.../minicluster/b-kylin/kylin.properties | 57 --------------------
.../minicluster/b-kylin/meta/kylin.properties | 54 -------------------
.../test_case_data/minicluster/kylin.properties | 4 +-
4 files changed, 3 insertions(+), 114 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/06aeb994/examples/test_case_data/localmeta/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/kylin.properties b/examples/test_case_data/localmeta/kylin.properties
index bdd6113..92773a1 100644
--- a/examples/test_case_data/localmeta/kylin.properties
+++ b/examples/test_case_data/localmeta/kylin.properties
@@ -7,7 +7,7 @@ kylin.rest.servers=localhost:7070
kylin.metadata.url=
# The storage for final cube file in hbase
-kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure
+kylin.storage.url=hbase
# Temp folder in hdfs, make sure user has the right access to the hdfs directory
kylin.hdfs.working.dir=/kylin
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/06aeb994/examples/test_case_data/minicluster/b-kylin/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/b-kylin/kylin.properties b/examples/test_case_data/minicluster/b-kylin/kylin.properties
deleted file mode 100644
index 968ee6a..0000000
--- a/examples/test_case_data/minicluster/b-kylin/kylin.properties
+++ /dev/null
@@ -1,57 +0,0 @@
-## Config for Kylin Engine ##
-
-# List of web servers in use, this enables one web server instance to sync up with other servers.
-kylin.rest.servers=ADMIN:KYLIN@localhost
-
-# The metadata store in hbase
-kylin.metadata.url=kylin_metadata@hbase:sandbox.hortonworks.com:2181:/hbase-unsecure
-
-# The storage for final cube file in hbase
-kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure
-
-# Temp folder in hdfs, make sure user has the right access to the hdfs directory
-kylin.hdfs.working.dir=/kylin
-
-kylin.job.mapreduce.default.reduce.input.mb=500
-
-# If true, job engine will not assume that hadoop CLI reside on the same server as it self
-# you will have to specify kylin.job.remote.cli.hostname, kylin.job.remote.cli.username and kylin.job.remote.cli.password
-kylin.job.run.as.remote.cmd=true
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.hostname=sandbox.hortonworks.com
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.username=root
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.password=hadoop
-
-# Used by test cases to prepare synthetic data for sample cube
-kylin.job.remote.cli.working.dir=/tmp/kylin
-
-# Max count of concurrent jobs running
-kylin.job.concurrent.max.limit=10
-
-# Time interval to check hadoop job status
-kylin.job.yarn.app.rest.check.interval.seconds=1
-
-
-## Config for Restful APP ##
-# database connection settings:
-ldap.server=
-ldap.username=
-ldap.password=
-ldap.user.searchBase=
-ldap.user.searchPattern=
-ldap.user.groupSearchBase=
-ldap.service.searchBase=OU=
-ldap.service.searchPattern=
-ldap.service.groupSearchBase=
-acl.adminRole=
-acl.defaultRole=
-ganglia.group=
-ganglia.port=8664
-
-###########################config info for sandbox#######################
-kylin.sandbox=true
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/06aeb994/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties b/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties
deleted file mode 100644
index 6caf9cc..0000000
--- a/examples/test_case_data/minicluster/b-kylin/meta/kylin.properties
+++ /dev/null
@@ -1,54 +0,0 @@
-## Config for Kylin Engine ##
-
-# List of web servers in use, this enables one web server instance to sync up with other servers.
-kylin.rest.servers=ADMIN:KYLIN@localhost
-
-# The metadata store in hbase
-kylin.metadata.url=kylin_metadata@hbase:sandbox.hortonworks.com:2181:/hbase-unsecure
-
-# The storage for final cube file in hbase
-kylin.storage.url=hbase:sandbox.hortonworks.com:2181:/hbase-unsecure
-
-# Temp folder in hdfs, make sure user has the right access to the hdfs directory
-kylin.hdfs.working.dir=/kylin
-
-kylin.job.mapreduce.default.reduce.input.mb=500
-
-# If true, job engine will not assume that hadoop CLI reside on the same server as it self
-# you will have to specify kylin.job.remote.cli.hostname, kylin.job.remote.cli.username and kylin.job.remote.cli.password
-kylin.job.run.as.remote.cmd=true
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.hostname=sandbox.hortonworks.com
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.username=root
-
-# Only necessary when kylin.job.run.as.remote.cmd=true
-kylin.job.remote.cli.password=hadoop
-
-# Used by test cases to prepare synthetic data for sample cube
-kylin.job.remote.cli.working.dir=/tmp/kylin
-
-# Max count of concurrent jobs running
-kylin.job.concurrent.max.limit=10
-
-# Time interval to check hadoop job status
-kylin.job.yarn.app.rest.check.interval.seconds=1
-
-
-## Config for Restful APP ##
-# database connection settings:
-ldap.server=
-ldap.username=
-ldap.password=
-ldap.user.searchBase=
-ldap.user.searchPattern=
-ldap.user.groupSearchBase=
-ldap.service.searchBase=OU=
-ldap.service.searchPattern=
-ldap.service.groupSearchBase=
-acl.adminRole=
-acl.defaultRole=
-ganglia.group=
-ganglia.port=8664
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/06aeb994/examples/test_case_data/minicluster/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/minicluster/kylin.properties b/examples/test_case_data/minicluster/kylin.properties
index 3b0033b..abd5714 100644
--- a/examples/test_case_data/minicluster/kylin.properties
+++ b/examples/test_case_data/minicluster/kylin.properties
@@ -4,10 +4,10 @@
kylin.rest.servers=ADMIN:KYLIN@localhost
# The metadata store in hbase
-kylin.metadata.url=kylin_metadata@hbase:minicluster:/hbase-unsecure
+kylin.metadata.url=kylin_metadata@hbase
# The storage for final cube file in hbase
-kylin.storage.url=hbase:minicluster:/hbase-unsecure
+kylin.storage.url=hbase
# Temp folder in hdfs, make sure user has the right access to the hdfs directory
kylin.hdfs.working.dir=/kylin