You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2021/05/08 04:10:46 UTC
[kylin] branch master updated: [KYLIN-4948] Provide an API to allow
users to adjust cuboids manually
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new 91da578 [KYLIN-4948] Provide an API to allow users to adjust cuboids manually
91da578 is described below
commit 91da578b0594aa0b7c9bcda25bd92e3a83e54d0c
Author: yangjiang <ya...@ebay.com>
AuthorDate: Mon Apr 12 09:52:22 2021 +0800
[KYLIN-4948] Provide an API to allow users to adjust cuboids manually
---
.../kylin/rest/controller/CubeController.java | 60 ++++++++++++++++++++++
.../kylin/rest/request/JobOptimizeRequest2.java | 40 +++++++++++++++
2 files changed, 100 insertions(+)
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 8821284..64186c5 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
@@ -70,6 +71,7 @@ import org.apache.kylin.rest.request.CubeRequest;
import org.apache.kylin.rest.request.JobBuildRequest;
import org.apache.kylin.rest.request.JobBuildRequest2;
import org.apache.kylin.rest.request.JobOptimizeRequest;
+import org.apache.kylin.rest.request.JobOptimizeRequest2;
import org.apache.kylin.rest.request.LookupSnapshotBuildRequest;
import org.apache.kylin.rest.response.CubeInstanceResponse;
import org.apache.kylin.rest.response.CuboidTreeResponse;
@@ -547,6 +549,64 @@ public class CubeController extends BasicController {
}
/**
+ * Send a optimize cube job for delete or add cuboid
+ *
+ * @param cubeName Cube ID
+ * @param jobOptimizeRequest method (add or delete), cuboidsRecommend
+ * @return JobInstance of CheckpointExecutable
+ */
+ @RequestMapping(value = "/{cubeName}/optimize2", method = {RequestMethod.PUT})
+ @ResponseBody
+ public JobInstance optimize(@PathVariable String cubeName, @RequestBody JobOptimizeRequest2 jobOptimizeRequest) {
+ try {
+ String submitter = SecurityContextHolder.getContext().getAuthentication().getName();
+
+ checkCubeExists(cubeName);
+ CubeInstance cube = jobService.getCubeManager().getCube(cubeName);
+
+ Set<Long> cuboidIds = cube.getCuboidScheduler().getAllCuboidIds();
+ Set<Long> cuboidsAdd = jobOptimizeRequest.getCuboidsAdd();
+ Set<Long> cuboidsDelete = jobOptimizeRequest.getCuboidsDelete();
+ Set<Long> result = new HashSet<>(cuboidIds);
+
+ if (cuboidsAdd == null && cuboidsDelete == null) {
+ throw new BadRequestException("must use cuboidsAdd or cuboidsDelete in request body.");
+ }
+
+ if (cuboidsAdd != null && cuboidsAdd.size() != 0) {
+ result.addAll(cuboidsAdd);
+ logger.info(
+ "Add cuboid cubeName: " + cubeName + " contained cuboids: " + Sets.intersection(cuboidIds, cuboidsAdd));
+ cuboidsAdd.removeAll(cuboidIds);
+ logger.info("Add cuboid cubeName: " + cubeName + " add cuboids: " + cuboidsAdd);
+ } else {
+ logger.info(cubeName + " no cuboids to add.");
+ }
+
+ if (cuboidsDelete != null && cuboidsDelete.size() != 0) {
+ result.removeAll(cuboidsDelete);
+ logger.info("Remove cuboid cubeName: " + cubeName + " remove cuboids: "
+ + Sets.intersection(cuboidIds, cuboidsDelete));
+ cuboidsDelete.removeAll(cuboidIds);
+ logger.info("Remove cuboid cubeName: " + cubeName + " missing cuboids: " + cuboidsDelete);
+ } else {
+ logger.info(cubeName + " no cuboids to delete.");
+ }
+
+ return jobService.submitOptimizeJob(cube, result, submitter).getFirst();
+ } catch (BadRequestException e) {
+ logger.error(e.getLocalizedMessage(), e);
+ throw e;
+ } catch (JobException e) {
+ logger.error(e.getLocalizedMessage(), e);
+ throw new BadRequestException(e.getLocalizedMessage());
+ } catch (Exception e) {
+ logger.error(e.getLocalizedMessage(), e);
+ throw new InternalErrorException(e.getLocalizedMessage(), e);
+ }
+ }
+
+ /**
* Send a optimize cube segment job
*
* @param cubeName Cube ID
diff --git a/server-base/src/main/java/org/apache/kylin/rest/request/JobOptimizeRequest2.java b/server-base/src/main/java/org/apache/kylin/rest/request/JobOptimizeRequest2.java
new file mode 100644
index 0000000..75204fb
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/request/JobOptimizeRequest2.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.request;
+
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class JobOptimizeRequest2 {
+ private Set<String> cuboidsAdd;
+ private Set<String> cuboidsDelete;
+
+ public Set<Long> getCuboidsAdd() {
+ if (cuboidsAdd == null)
+ return null;
+ return cuboidsAdd.stream().map(s -> Long.parseLong(s)).collect(Collectors.toSet());
+ }
+
+ public Set<Long> getCuboidsDelete() {
+ if (cuboidsDelete == null)
+ return null;
+ return cuboidsDelete.stream().map(s -> Long.parseLong(s)).collect(Collectors.toSet());
+ }
+
+}