You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2016/03/28 11:58:02 UTC
kylin git commit: KYLIN-1540 REST API for deleting segment
Repository: kylin
Updated Branches:
refs/heads/master 425d56bb5 -> 2e03c9c38
KYLIN-1540 REST API for deleting segment
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2e03c9c3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2e03c9c3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2e03c9c3
Branch: refs/heads/master
Commit: 2e03c9c38a02635ce245ca8e6fe5157e7f1db618
Parents: 425d56b
Author: shaofengshi <sh...@apache.org>
Authored: Mon Mar 28 17:53:19 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Mar 28 17:57:54 2016 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 3 +
.../org/apache/kylin/cube/CubeInstance.java | 2 +-
...st_kylin_cube_with_slr_ready_3_segments.json | 88 ++++++++++++++++++++
.../test_case_data/sandbox/kylin.properties | 2 +-
.../kylin/rest/controller/CubeController.java | 27 ++++++
.../apache/kylin/rest/service/CubeService.java | 32 +++++--
.../rest/controller/CubeControllerTest.java | 47 ++++++++++-
7 files changed, 190 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2e03c9c3/build/conf/kylin.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties
index 289afd1..5fb4e36 100644
--- a/build/conf/kylin.properties
+++ b/build/conf/kylin.properties
@@ -15,6 +15,9 @@
# limitations under the License.
#
+# kylin server's mode
+kylin.server.mode=all
+
# optional information for the owner of kylin platform, it can be your team's email
# currently it will be attached to each kylin's htable attribute
kylin.owner=whoami@kylin.apache.org
http://git-wip-us.apache.org/repos/asf/kylin/blob/2e03c9c3/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
index d89e736..aa063a6 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java
@@ -296,7 +296,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization,
public CubeSegment getSegment(String name, SegmentStatusEnum status) {
for (CubeSegment segment : segments) {
- if ((null != segment.getName() && segment.getName().equals(name)) && segment.getStatus() == status) {
+ if ((null != segment.getName() && segment.getName().equals(name)) && (status == null || segment.getStatus() == status)) {
return segment;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2e03c9c3/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready_3_segments.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready_3_segments.json b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready_3_segments.json
new file mode 100644
index 0000000..e39de87
--- /dev/null
+++ b/examples/test_case_data/localmeta/cube/test_kylin_cube_with_slr_ready_3_segments.json
@@ -0,0 +1,88 @@
+{
+ "uuid" : "3eaca32a-a33e-4b69-83dd-0bb8b1f8c53c",
+ "last_modified" : 1404098141020,
+ "name" : "test_kylin_cube_with_slr_ready_3_segments",
+ "owner" : null,
+ "descriptor" : "test_kylin_cube_with_slr_desc",
+ "cost" : 50,
+ "segments" : [ {
+ "uuid" : "1aeca32a-a33e-4b69-83dd-xxe8b1f8dddd",
+ "name" : "19691231160000_20131112000000",
+ "storage_location_identifier" : "KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_READY-F24668F6-DCFF-4CB6-A89B-77F1119DF8FA",
+ "date_range_start" : 1384240200000,
+ "date_range_end" : 1384243200000,
+ "status" : "READY",
+ "size_kb" : 7801,
+ "source_records" : 10000,
+ "source_records_size" : 608012,
+ "last_build_time" : 1404098140902,
+ "last_build_job_id" : "f24668f6-dcff-4cb6-a89b-77f1119df8fa",
+ "binary_signature" : null,
+ "dictionaries" : {
+ "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/16d8185c-ee6b-4f8c-a919-756d9809f937.dict",
+ "DEFAULT.TEST_KYLIN_FACT/SLR_SEGMENT_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict",
+ "DEFAULT.TEST_KYLIN_FACT/LSTG_SITE_ID" : "/dict/TEST_SITES/SITE_ID/652bd393-678a-4f16-a504-fd8ce1229355.dict",
+ "EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/2a44ff38-f64b-42e7-9fcf-66afccac8047.dict",
+ "DEFAULT.TEST_KYLIN_FACT/CAL_DT" : "/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict",
+ "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict",
+ "EDW.TEST_CAL_DT/CAL_DT" : "/dict/TEST_CAL_DT/CAL_DT/ed0c3451-593c-494c-9019-64f63fcb0b8e.dict",
+ "DEFAULT.TEST_KYLIN_FACT/LEAF_CATEG_ID" : "/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/4243889f-bc81-4807-a975-7041bbbf35e7.dict",
+ "EDW.TEST_SITES/SITE_ID" : "/dict/TEST_SITES/SITE_ID/ff7e8943-ac0f-4e66-b9ed-510f6a0b875d.dict",
+ "DEFAULT.TEST_CATEGORY_GROUPINGS/META_CATEG_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/aceae914-4246-4251-a0c2-692fe7a300df.dict"
+ },
+ "snapshots" : {
+ "EDW.TEST_SELLER_TYPE_DIM" : "/table_snapshot/TEST_SELLER_TYPE_DIM.csv/4fe75ccd-9b24-4cdf-ac9d-b4038e947f89.snapshot",
+ "EDW.TEST_CAL_DT" : "/table_snapshot/TEST_CAL_DT.csv/8ff1339e-f804-47f3-b42c-1d4fa4ff0cf7.snapshot",
+ "DEFAULT.TEST_CATEGORY_GROUPINGS" : "/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/e172b442-ae10-447e-9071-c7dbb2bb38cc.snapshot",
+ "EDW.TEST_SITES" : "/table_snapshot/TEST_SITES.csv/28130338-fcf4-429e-91b0-cd8dfd397280.snapshot"
+ }
+ }, {
+ "uuid" : "2aeca32a-a33e-4b69-83dd-xxe8b1f8dddd",
+ "name" : "20131112000000_20131212000000",
+ "storage_location_identifier" : "KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_READY-F24668F6-DCFF-4CB6-A89B-77F1119DF8FB",
+ "date_range_start" : 1384243200000,
+ "date_range_end" : 1386835200000,
+ "status" : "READY",
+ "size_kb" : 7801,
+ "source_records" : 10000,
+ "source_records_size" : 608012,
+ "last_build_time" : 1404098140902,
+ "last_build_job_id" : "f24668f6-dcff-4cb6-a89b-77f1119df8fb",
+ "binary_signature" : null,
+ "dictionaries" : {
+ "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL2_NAME/16d8185c-ee6b-4f8c-a919-756d9809f937.dict",
+ "DEFAULT.TEST_KYLIN_FACT/SLR_SEGMENT_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/c6bf9b51-6e90-4337-8082-4e2fdf78307f.dict",
+ "DEFAULT.TEST_KYLIN_FACT/LSTG_SITE_ID" : "/dict/TEST_SITES/SITE_ID/652bd393-678a-4f16-a504-fd8ce1229355.dict",
+ "EDW.TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD" : "/dict/TEST_SELLER_TYPE_DIM/SELLER_TYPE_CD/2a44ff38-f64b-42e7-9fcf-66afccac8047.dict",
+ "DEFAULT.TEST_KYLIN_FACT/CAL_DT" : "/dict/PREDEFINED/date(yyyy-mm-dd)/64ac4f82-f2af-476e-85b9-f0805001014e.dict",
+ "DEFAULT.TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/CATEG_LVL3_NAME/ad09f2d5-054a-4e1b-a776-7cc07399a6c1.dict",
+ "EDW.TEST_CAL_DT/CAL_DT" : "/dict/TEST_CAL_DT/CAL_DT/ed0c3451-593c-494c-9019-64f63fcb0b8e.dict",
+ "DEFAULT.TEST_KYLIN_FACT/LEAF_CATEG_ID" : "/dict/TEST_CATEGORY_GROUPINGS/LEAF_CATEG_ID/4243889f-bc81-4807-a975-7041bbbf35e7.dict",
+ "EDW.TEST_SITES/SITE_ID" : "/dict/TEST_SITES/SITE_ID/ff7e8943-ac0f-4e66-b9ed-510f6a0b875d.dict",
+ "DEFAULT.TEST_CATEGORY_GROUPINGS/META_CATEG_NAME" : "/dict/TEST_CATEGORY_GROUPINGS/META_CATEG_NAME/aceae914-4246-4251-a0c2-692fe7a300df.dict"
+ },
+ "snapshots" : {
+ "EDW.TEST_SELLER_TYPE_DIM" : "/table_snapshot/TEST_SELLER_TYPE_DIM.csv/4fe75ccd-9b24-4cdf-ac9d-b4038e947f89.snapshot",
+ "EDW.TEST_CAL_DT" : "/table_snapshot/TEST_CAL_DT.csv/8ff1339e-f804-47f3-b42c-1d4fa4ff0cf7.snapshot",
+ "DEFAULT.TEST_CATEGORY_GROUPINGS" : "/table_snapshot/TEST_CATEGORY_GROUPINGS.csv/e172b442-ae10-447e-9071-c7dbb2bb38cc.snapshot",
+ "EDW.TEST_SITES" : "/table_snapshot/TEST_SITES.csv/28130338-fcf4-429e-91b0-cd8dfd397280.snapshot"
+ }
+ }, {
+ "uuid" : "3aeca32a-a33e-4b69-83dd-xxe8b1f8dddd",
+ "name" : "20131212000000_20140112000000",
+ "storage_location_identifier" : "KYLIN-CUBE-TEST_KYLIN_CUBE_WITH_SLR_READY-F24668F6-DCFF-4CB6-A89B-77F1119DF8FB",
+ "date_range_start" : 1386835200000,
+ "date_range_end" : 1389484800000,
+ "status" : "NEW",
+ "size_kb" : 0,
+ "source_records" : 0,
+ "source_records_size" : 0,
+ "last_build_time" : 1404098140902,
+ "last_build_job_id" : "",
+ "binary_signature" : null
+ } ],
+ "status" : "READY",
+ "create_time" : null,
+ "notify_list" : null,
+ "auto_merge_time_ranges" : [2595000000]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/2e03c9c3/examples/test_case_data/sandbox/kylin.properties
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties
index f1ecd64..2fa647d 100644
--- a/examples/test_case_data/sandbox/kylin.properties
+++ b/examples/test_case_data/sandbox/kylin.properties
@@ -63,7 +63,7 @@ kylin.job.concurrent.max.limit=10
kylin.job.mapreduce.max.reducer.number=5
#the percentage of the sampling, default 100%
-kylin.job.cubing.inMem.sampling.percent=100
+kylin.job.cubing.inmem.sampling.percent=100
# The cut size for hbase region, in GB.
# E.g, for cube whose capacity be marked as "SMALL", split region per 10GB by default
http://git-wip-us.apache.org/repos/asf/kylin/blob/2e03c9c3/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index b17378c..6c62b52 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -209,6 +209,33 @@ public class CubeController extends BasicController {
}
/**
+ * Delete a cube segment
+ *
+ * @throws IOException
+ */
+ @RequestMapping(value = "/{cubeName}/segs/{segmentName}", method = { RequestMethod.DELETE })
+ @ResponseBody
+ public CubeInstance deleteSegment(@PathVariable String cubeName, @PathVariable String segmentName) {
+ CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
+
+ if (cube == null) {
+ throw new InternalErrorException("Cannot find cube " + cubeName);
+ }
+
+ CubeSegment segment = cube.getSegment(segmentName, null);
+ if (segment == null) {
+ throw new InternalErrorException("Cannot find segment '" + segmentName + "'");
+ }
+
+ try {
+ return cubeService.deleteSegment(cube, segmentName);
+ } catch (Exception e) {
+ logger.error(e.getLocalizedMessage(), e);
+ throw new InternalErrorException(e.getLocalizedMessage());
+ }
+ }
+
+ /**
* Send a rebuild cube job
*
* @param cubeName Cube ID
http://git-wip-us.apache.org/repos/asf/kylin/blob/2e03c9c3/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index eb15beb..8ed0802 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -19,17 +19,11 @@
package org.apache.kylin.rest.service;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Date;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.WeakHashMap;
+import java.util.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.mapred.Merger;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -527,6 +521,28 @@ public class CubeService extends BasicService {
return cube;
}
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
+ public CubeInstance deleteSegment(CubeInstance cube, String segmentName) throws IOException {
+
+ if (!segmentName.equals(cube.getSegments().get(0).getName()) && !segmentName.equals(cube.getSegments().get(cube.getSegments().size() - 1).getName())) {
+ throw new IllegalArgumentException("Cannot delete segment '" + segmentName + "' as it is neither the first nor the last segment.");
+ }
+ CubeSegment toDelete = null;
+ for (CubeSegment seg : cube.getSegments()) {
+ if (seg.getName().equals(segmentName)) {
+ toDelete = seg;
+ }
+ }
+
+ if (toDelete.getStatus() != SegmentStatusEnum.READY) {
+ throw new IllegalArgumentException("Cannot delete segment '" + segmentName + "' as its status is not READY. Discard the on-going job for it.");
+ }
+
+ CubeUpdate update = new CubeUpdate(cube);
+ update.setToRemoveSegs(new CubeSegment[]{toDelete});
+ return CubeManager.getInstance(getConfig()).updateCube(update);
+ }
+
/**
* purge the cube
*
http://git-wip-us.apache.org/repos/asf/kylin/blob/2e03c9c3/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
index 1230886..20ddd58 100644
--- a/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/controller/CubeControllerTest.java
@@ -24,6 +24,7 @@ import java.util.List;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.request.CubeRequest;
import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.JobService;
@@ -68,7 +69,7 @@ public class CubeControllerTest extends ServiceTestBase {
@Test
public void testBasics() throws IOException {
- CubeDesc[] cubes = (CubeDesc[]) cubeDescController.getCube("test_kylin_cube_with_slr_ready");
+ CubeDesc[] cubes = cubeDescController.getCube("test_kylin_cube_with_slr_ready");
Assert.assertNotNull(cubes);
Assert.assertNotNull(cubeController.getSql("test_kylin_cube_with_slr_ready", "20130331080000_20131212080000"));
Assert.assertNotNull(cubeController.getCubes(null, null, null, 0, 5));
@@ -121,4 +122,48 @@ public class CubeControllerTest extends ServiceTestBase {
cubeController.deleteCube(newCubeName);
}
+ @Test (expected=InternalErrorException.class)
+ public void testDeleteSegmentNew() throws IOException {
+ String cubeName = "test_kylin_cube_with_slr_ready_3_segments";
+ CubeDesc[] cubes = cubeDescController.getCube(cubeName);
+ Assert.assertNotNull(cubes);
+
+ cubeController.deleteSegment(cubeName, "20131212000000_20140112000000");
+ }
+
+ @Test (expected=InternalErrorException.class)
+ public void testDeleteSegmentNotExist() throws IOException {
+ String cubeName = "test_kylin_cube_with_slr_ready_3_segments";
+ CubeDesc[] cubes = cubeDescController.getCube(cubeName);
+ Assert.assertNotNull(cubes);
+
+ cubeController.deleteSegment(cubeName, "not_exist_segment");
+ }
+
+
+ @Test (expected=InternalErrorException.class)
+ public void testDeleteSegmentInMiddle() throws IOException {
+ String cubeName = "test_kylin_cube_with_slr_ready_3_segments";
+ CubeDesc[] cubes = cubeDescController.getCube(cubeName);
+ Assert.assertNotNull(cubes);
+
+ cubeController.deleteSegment(cubeName, "20131112000000_20131212000000");
+ }
+
+ @Test
+ public void testDeleteSegmentFromHead() throws IOException {
+ String cubeName = "test_kylin_cube_with_slr_ready_3_segments";
+ CubeDesc[] cubes = cubeDescController.getCube(cubeName);
+ Assert.assertNotNull(cubes);
+
+ int segNumber = cubeService.getCubeManager().getCube(cubeName).getSegments().size();
+
+ cubeController.deleteSegment(cubeName, "19691231160000_20131112000000");
+
+ int newSegNumber = cubeService.getCubeManager().getCube(cubeName).getSegments().size();
+
+ Assert.assertTrue(segNumber == newSegNumber + 1);
+ }
+
+
}