You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2021/05/27 03:04:57 UTC
[kylin] 02/02: [KYLIN-4948] Provide an API to allow users to adjust
cuboids manually
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 082df34f2d304d1e6237c09047cf2c008c5e7df0
Author: yangjiang <ya...@ebay.com>
AuthorDate: Mon Apr 12 09:52:22 2021 +0800
[KYLIN-4948] Provide an API to allow users to adjust cuboids manually
---
.../kylin/rest/controller/CubeController.java | 59 ++++++++++++++++++++++
.../kylin/rest/request/JobOptimizeRequest2.java | 40 +++++++++++++++
2 files changed, 99 insertions(+)
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index c514bd4..7df1527 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -70,6 +70,7 @@ import org.apache.kylin.rest.request.CubeRequest;
import org.apache.kylin.rest.request.JobBuildRequest;
import org.apache.kylin.rest.request.JobBuildRequest2;
import org.apache.kylin.rest.request.JobOptimizeRequest;
+import org.apache.kylin.rest.request.JobOptimizeRequest2;
import org.apache.kylin.rest.response.CubeInstanceResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse;
import org.apache.kylin.rest.response.EnvelopeResponse;
@@ -440,6 +441,64 @@ public class CubeController extends BasicController {
}
/**
+ * Send a optimize cube job for delete or add cuboid
+ *
+ * @param cubeName Cube ID
+ * @param jobOptimizeRequest method (add or delete), cuboidsRecommend
+ * @return JobInstance of CheckpointExecutable
+ */
+ @RequestMapping(value = "/{cubeName}/optimize2", method = {RequestMethod.PUT})
+ @ResponseBody
+ public JobInstance optimize(@PathVariable String cubeName, @RequestBody JobOptimizeRequest2 jobOptimizeRequest) {
+ try {
+ String submitter = SecurityContextHolder.getContext().getAuthentication().getName();
+
+ checkCubeExists(cubeName);
+ CubeInstance cube = jobService.getCubeManager().getCube(cubeName);
+
+ Set<Long> cuboidIds = cube.getCuboidScheduler().getAllCuboidIds();
+ Set<Long> cuboidsAdd = jobOptimizeRequest.getCuboidsAdd();
+ Set<Long> cuboidsDelete = jobOptimizeRequest.getCuboidsDelete();
+ Set<Long> result = new HashSet<>(cuboidIds);
+
+ if (cuboidsAdd == null && cuboidsDelete == null) {
+ throw new BadRequestException("must use cuboidsAdd or cuboidsDelete in request body.");
+ }
+
+ if (cuboidsAdd != null && cuboidsAdd.size() != 0) {
+ result.addAll(cuboidsAdd);
+ logger.info(
+ "Add cuboid cubeName: " + cubeName + " contained cuboids: " + Sets.intersection(cuboidIds, cuboidsAdd));
+ cuboidsAdd.removeAll(cuboidIds);
+ logger.info("Add cuboid cubeName: " + cubeName + " add cuboids: " + cuboidsAdd);
+ } else {
+ logger.info(cubeName + " no cuboids to add.");
+ }
+
+ if (cuboidsDelete != null && cuboidsDelete.size() != 0) {
+ result.removeAll(cuboidsDelete);
+ logger.info("Remove cuboid cubeName: " + cubeName + " remove cuboids: "
+ + Sets.intersection(cuboidIds, cuboidsDelete));
+ cuboidsDelete.removeAll(cuboidIds);
+ logger.info("Remove cuboid cubeName: " + cubeName + " missing cuboids: " + cuboidsDelete);
+ } else {
+ logger.info(cubeName + " no cuboids to delete.");
+ }
+
+ return jobService.submitOptimizeJob(cube, result, submitter).getFirst();
+ } catch (BadRequestException e) {
+ logger.error(e.getLocalizedMessage(), e);
+ throw e;
+ } catch (JobException e) {
+ logger.error(e.getLocalizedMessage(), e);
+ throw new BadRequestException(e.getLocalizedMessage());
+ } catch (Exception e) {
+ logger.error(e.getLocalizedMessage(), e);
+ throw new InternalErrorException(e.getLocalizedMessage(), e);
+ }
+ }
+
+ /**
* Send a optimize cube segment job
*
* @param cubeName Cube ID
diff --git a/server-base/src/main/java/org/apache/kylin/rest/request/JobOptimizeRequest2.java b/server-base/src/main/java/org/apache/kylin/rest/request/JobOptimizeRequest2.java
new file mode 100644
index 0000000..75204fb
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/request/JobOptimizeRequest2.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.request;
+
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class JobOptimizeRequest2 {
+ private Set<String> cuboidsAdd;
+ private Set<String> cuboidsDelete;
+
+ public Set<Long> getCuboidsAdd() {
+ if (cuboidsAdd == null)
+ return null;
+ return cuboidsAdd.stream().map(s -> Long.parseLong(s)).collect(Collectors.toSet());
+ }
+
+ public Set<Long> getCuboidsDelete() {
+ if (cuboidsDelete == null)
+ return null;
+ return cuboidsDelete.stream().map(s -> Long.parseLong(s)).collect(Collectors.toSet());
+ }
+
+}