You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/12/31 13:20:03 UTC
[46/50] [abbrv] kylin git commit: Merge commit
'5f2eff68d80ea6264d7590e14c052114c3cd6b74'
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/server-base/src/main/java/org/apache/kylin/rest/controller/DashboardController.java
----------------------------------------------------------------------
diff --cc server-base/src/main/java/org/apache/kylin/rest/controller/DashboardController.java
index 0000000,35ba615..ee9fdcd
mode 000000,100644..100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/DashboardController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/DashboardController.java
@@@ -1,0 -1,129 +1,129 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+ package org.apache.kylin.rest.controller;
+
++import java.util.List;
++
+ import org.apache.kylin.cube.CubeInstance;
+ import org.apache.kylin.metadata.project.ProjectInstance;
+ import org.apache.kylin.metrics.MetricsManager;
+ import org.apache.kylin.rest.request.SQLRequest;
+ import org.apache.kylin.rest.response.MetricsResponse;
+ import org.apache.kylin.rest.response.SQLResponse;
+ import org.apache.kylin.rest.service.CubeService;
+ import org.apache.kylin.rest.service.DashboardService;
+ import org.apache.kylin.rest.service.QueryService;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ import org.springframework.beans.factory.annotation.Autowired;
+ import org.springframework.security.access.AccessDeniedException;
+ import org.springframework.stereotype.Controller;
+ import org.springframework.web.bind.annotation.PathVariable;
+ import org.springframework.web.bind.annotation.RequestMapping;
+ import org.springframework.web.bind.annotation.RequestMethod;
+ import org.springframework.web.bind.annotation.RequestParam;
+ import org.springframework.web.bind.annotation.ResponseBody;
+
-import java.util.List;
-
+ @Controller
+ @RequestMapping(value = "/dashboard")
+ public class DashboardController extends BasicController {
+ private static final Logger logger = LoggerFactory.getLogger(DashboardController.class);
+
+ @Autowired
+ private DashboardService dashboardService;
+
+ @Autowired
+ private QueryService queryService;
+
+ @Autowired
+ private CubeService cubeService;
+
+ @RequestMapping(value = "/metric/cube", method = { RequestMethod.GET })
+ @ResponseBody
+ public MetricsResponse getCubeMetrics(@RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value = "cubeName", required = false) String cubeName) {
+ checkAuthorization(projectName);
+ return dashboardService.getCubeMetrics(projectName, cubeName);
+ }
+
+ @RequestMapping(value = "/metric/query", method = RequestMethod.GET)
+ @ResponseBody
+ public MetricsResponse getQueryMetrics(@RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value = "cubeName", required = false) String cubeName, @RequestParam(value = "startTime") String startTime, @RequestParam(value = "endTime") String endTime) {
+ checkAuthorization(projectName);
+ MetricsResponse queryMetrics = new MetricsResponse();
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
+ String sql = dashboardService.getQueryMetricsSQL(startTime, endTime, projectName, cubeName);
+ sqlRequest.setSql(sql);
- SQLResponse sqlResponse = queryService.queryWithoutSecure(sqlRequest);
++ SQLResponse sqlResponse = queryService.doQueryWithCache(sqlRequest);
+ if(!sqlResponse.getIsException()){
+ queryMetrics.increase("queryCount", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(0)));
+ queryMetrics.increase("avgQueryLatency", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(1)));
+ queryMetrics.increase("maxQueryLatency", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(2)));
+ queryMetrics.increase("minQueryLatency", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(3)));
+ }
+ return queryMetrics;
+ }
+
+ @RequestMapping(value = "/metric/job", method = RequestMethod.GET)
+ @ResponseBody
+ public MetricsResponse getJobMetrics(@RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value = "cubeName", required = false) String cubeName, @RequestParam(value = "startTime") String startTime, @RequestParam(value = "endTime") String endTime) {
+ checkAuthorization(projectName);
+ MetricsResponse jobMetrics = new MetricsResponse();
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
+ String sql = dashboardService.getJobMetricsSQL(startTime, endTime, projectName, cubeName);
+ sqlRequest.setSql(sql);
- SQLResponse sqlResponse = queryService.queryWithoutSecure(sqlRequest);
++ SQLResponse sqlResponse = queryService.doQueryWithCache(sqlRequest);
+ if(!sqlResponse.getIsException()){
+ jobMetrics.increase("jobCount", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(0)));
+ jobMetrics.increase("avgJobBuildTime", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(1)));
+ jobMetrics.increase("maxJobBuildTime", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(2)));
+ jobMetrics.increase("minJobBuildTime", dashboardService.getMetricValue(sqlResponse.getResults().get(0).get(3)));
+ }
+ return jobMetrics;
+ }
+
+ @RequestMapping(value = "/chart/{category}/{metric}/{dimension}", method = RequestMethod.GET)
+ @ResponseBody
+ public MetricsResponse getChartData(@PathVariable String dimension, @PathVariable String metric, @PathVariable String category, @RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value = "cubeName", required = false) String cubeName, @RequestParam(value = "startTime") String startTime, @RequestParam(value = "endTime") String endTime) {
+ checkAuthorization(projectName);
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setProject(MetricsManager.SYSTEM_PROJECT);
+ String sql = dashboardService.getChartSQL(startTime, endTime, projectName, cubeName, dimension, metric, category);
+ sqlRequest.setSql(sql);
- return dashboardService.transformChartData(queryService.queryWithoutSecure(sqlRequest));
++ return dashboardService.transformChartData(queryService.doQueryWithCache(sqlRequest));
+ }
+
+ private void checkAuthorization(String projectName){
+ if (projectName!=null && !projectName.isEmpty()) {
+ ProjectInstance project = dashboardService.getProjectManager().getProject(projectName);
+ try {
+ dashboardService.checkAuthorization(project);
+ } catch (AccessDeniedException e) {
+ List<CubeInstance> cubes = cubeService.listAllCubes(null, projectName, null, true);
+ if (cubes.isEmpty()) {
+ throw new AccessDeniedException("Access is denied");
+ }
+ }
+ } else {
+ dashboardService.checkAuthorization();
+ }
+ }
+ }
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java
----------------------------------------------------------------------
diff --cc server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java
index 48a8e58,18ef867..e595804
--- a/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java
@@@ -21,6 -27,14 +27,13 @@@ import javax.annotation.concurrent.Thre
import org.apache.hadoop.metrics2.MetricsException;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.kylin.common.KylinConfig;
+ import org.apache.kylin.common.QueryContext;
-import org.apache.kylin.metadata.project.ProjectInstance;
+ import org.apache.kylin.metrics.MetricsManager;
+ import org.apache.kylin.metrics.lib.impl.RecordEvent;
+ import org.apache.kylin.metrics.lib.impl.TimedRecordEvent;
+ import org.apache.kylin.metrics.property.QueryCubePropertyEnum;
+ import org.apache.kylin.metrics.property.QueryPropertyEnum;
+ import org.apache.kylin.metrics.property.QueryRPCPropertyEnum;
import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.SQLResponse;
import org.slf4j.Logger;
@@@ -63,6 -88,139 +87,143 @@@ public class QueryMetricsFacade
update(getQueryMetrics(cubeMetricName), sqlResponse);
}
+ /**
+ * report query related metrics
+ */
+ private static void updateMetricsToReservoir(SQLRequest sqlRequest, SQLResponse sqlResponse) {
+ if (!KylinConfig.getInstanceFromEnv().isKylinMetricsReporterForQueryEnabled()) {
+ return;
+ }
+ String user = SecurityContextHolder.getContext().getAuthentication().getName();
+ if (user == null) {
+ user = "unknown";
+ }
+ for (QueryContext.RPCStatistics entry : QueryContext.current().getRpcStatisticsList()) {
+ RecordEvent rpcMetricsEvent = new TimedRecordEvent(
+ KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectQueryRpcCall());
+ setRPCWrapper(rpcMetricsEvent, //
- ProjectInstance.getNormalizedProjectName(sqlRequest.getProject()), entry.getRealizationName(),
++ norm(sqlRequest.getProject()), entry.getRealizationName(),
+ entry.getRpcServer(), entry.getException());
+ setRPCStats(rpcMetricsEvent, //
+ entry.getCallTimeMs(), entry.getSkippedRows(), entry.getScannedRows(), entry.getReturnedRows(),
+ entry.getAggregatedRows());
+ //For update rpc level related metrics
+ MetricsManager.getInstance().update(rpcMetricsEvent);
+ }
+ long sqlHashCode = getSqlHashCode(sqlRequest.getSql());
+ for (QueryContext.CubeSegmentStatisticsResult contextEntry : sqlResponse.getCubeSegmentStatisticsList()) {
+ RecordEvent queryMetricsEvent = new TimedRecordEvent(
+ KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectQuery());
+ setQueryWrapper(queryMetricsEvent, //
+ user, sqlHashCode, sqlResponse.isStorageCacheUsed() ? "CACHE" : contextEntry.getQueryType(),
- ProjectInstance.getNormalizedProjectName(sqlRequest.getProject()), contextEntry.getRealization(),
++ norm(sqlRequest.getProject()), contextEntry.getRealization(),
+ contextEntry.getRealizationType(),
+ sqlResponse.getThrowable());
+
+ long totalStorageReturnCount = 0L;
+ for (Map<String, QueryContext.CubeSegmentStatistics> cubeEntry : contextEntry.getCubeSegmentStatisticsMap()
+ .values()) {
+ for (QueryContext.CubeSegmentStatistics segmentEntry : cubeEntry.values()) {
+ RecordEvent cubeSegmentMetricsEvent = new TimedRecordEvent(
+ KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectQueryCube());
+
+ setCubeWrapper(cubeSegmentMetricsEvent, //
- ProjectInstance.getNormalizedProjectName(sqlRequest.getProject()),
++ norm(sqlRequest.getProject()),
+ segmentEntry.getCubeName(), segmentEntry.getSegmentName(), segmentEntry.getSourceCuboidId(),
+ segmentEntry.getTargetCuboidId(), segmentEntry.getFilterMask());
+
+ setCubeStats(cubeSegmentMetricsEvent, //
+ segmentEntry.getCallCount(), segmentEntry.getCallTimeSum(), segmentEntry.getCallTimeMax(),
+ segmentEntry.getStorageSkippedRows(), segmentEntry.getStorageScannedRows(),
+ segmentEntry.getStorageReturnedRows(), segmentEntry.getStorageAggregatedRows(),
+ segmentEntry.isIfSuccess(), 1.0 / cubeEntry.size());
+
+ totalStorageReturnCount += segmentEntry.getStorageReturnedRows();
+ //For update cube segment level related query metrics
+ MetricsManager.getInstance().update(cubeSegmentMetricsEvent);
+ }
+ }
+ setQueryStats(queryMetricsEvent, //
+ sqlResponse.getDuration(), sqlResponse.getResults().size(), totalStorageReturnCount);
+ //For update query level metrics
+ MetricsManager.getInstance().update(queryMetricsEvent);
+ }
+ }
+
++ private static String norm(String project) {
++ return project.toUpperCase();
++ }
++
+ private static void setRPCWrapper(RecordEvent metricsEvent, String projectName, String realizationName,
+ String rpcServer, Throwable throwable) {
+ metricsEvent.put(QueryRPCPropertyEnum.PROJECT.toString(), projectName);
+ metricsEvent.put(QueryRPCPropertyEnum.REALIZATION.toString(), realizationName);
+ metricsEvent.put(QueryRPCPropertyEnum.RPC_SERVER.toString(), rpcServer);
+ metricsEvent.put(QueryRPCPropertyEnum.EXCEPTION.toString(),
+ throwable == null ? "NULL" : throwable.getClass().getName());
+ }
+
+ private static void setRPCStats(RecordEvent metricsEvent, long callTimeMs, long skipCount, long scanCount,
+ long returnCount, long aggrCount) {
+ metricsEvent.put(QueryRPCPropertyEnum.CALL_TIME.toString(), callTimeMs);
+ metricsEvent.put(QueryRPCPropertyEnum.SKIP_COUNT.toString(), skipCount); //Number of skips on region servers based on region meta or fuzzy filter
+ metricsEvent.put(QueryRPCPropertyEnum.SCAN_COUNT.toString(), scanCount); //Count scanned by region server
+ metricsEvent.put(QueryRPCPropertyEnum.RETURN_COUNT.toString(), returnCount);//Count returned by region server
+ metricsEvent.put(QueryRPCPropertyEnum.AGGR_FILTER_COUNT.toString(), scanCount - returnCount); //Count filtered & aggregated by coprocessor
+ metricsEvent.put(QueryRPCPropertyEnum.AGGR_COUNT.toString(), aggrCount); //Count aggregated by coprocessor
+ }
+
+ private static void setCubeWrapper(RecordEvent metricsEvent, String projectName, String cubeName,
+ String segmentName, long sourceCuboidId, long targetCuboidId, long filterMask) {
+ metricsEvent.put(QueryCubePropertyEnum.PROJECT.toString(), projectName);
+ metricsEvent.put(QueryCubePropertyEnum.CUBE.toString(), cubeName);
+ metricsEvent.put(QueryCubePropertyEnum.SEGMENT.toString(), segmentName);
+ metricsEvent.put(QueryCubePropertyEnum.CUBOID_SOURCE.toString(), sourceCuboidId);
+ metricsEvent.put(QueryCubePropertyEnum.CUBOID_TARGET.toString(), targetCuboidId);
+ metricsEvent.put(QueryCubePropertyEnum.IF_MATCH.toString(), sourceCuboidId == targetCuboidId);
+ metricsEvent.put(QueryCubePropertyEnum.FILTER_MASK.toString(), filterMask);
+ }
+
+ private static void setCubeStats(RecordEvent metricsEvent, long callCount, long callTimeSum, long callTimeMax,
+ long skipCount, long scanCount, long returnCount, long aggrCount, boolean ifSuccess, double weightPerHit) {
+ metricsEvent.put(QueryCubePropertyEnum.CALL_COUNT.toString(), callCount);
+ metricsEvent.put(QueryCubePropertyEnum.TIME_SUM.toString(), callTimeSum);
+ metricsEvent.put(QueryCubePropertyEnum.TIME_MAX.toString(), callTimeMax);
+ metricsEvent.put(QueryCubePropertyEnum.SKIP_COUNT.toString(), skipCount);
+ metricsEvent.put(QueryCubePropertyEnum.SCAN_COUNT.toString(), scanCount);
+ metricsEvent.put(QueryCubePropertyEnum.RETURN_COUNT.toString(), returnCount);
+ metricsEvent.put(QueryCubePropertyEnum.AGGR_FILTER_COUNT.toString(), scanCount - returnCount);
+ metricsEvent.put(QueryCubePropertyEnum.AGGR_COUNT.toString(), aggrCount);
+ metricsEvent.put(QueryCubePropertyEnum.IF_SUCCESS.toString(), ifSuccess);
+ metricsEvent.put(QueryCubePropertyEnum.WEIGHT_PER_HIT.toString(), weightPerHit);
+ }
+
+ private static void setQueryWrapper(RecordEvent metricsEvent, String user, long queryHashCode, String queryType,
+ String projectName, String realizationName, int realizationType, Throwable throwable) {
+ metricsEvent.put(QueryPropertyEnum.USER.toString(), user);
+ metricsEvent.put(QueryPropertyEnum.ID_CODE.toString(), queryHashCode);
+ metricsEvent.put(QueryPropertyEnum.TYPE.toString(), queryType);
+ metricsEvent.put(QueryPropertyEnum.PROJECT.toString(), projectName);
+ metricsEvent.put(QueryPropertyEnum.REALIZATION.toString(), realizationName);
+ metricsEvent.put(QueryPropertyEnum.REALIZATION_TYPE.toString(), realizationType);
+ metricsEvent.put(QueryPropertyEnum.EXCEPTION.toString(),
+ throwable == null ? "NULL" : throwable.getClass().getName());
+ }
+
+ private static void setQueryStats(RecordEvent metricsEvent, long callTimeMs, long returnCountByCalcite,
+ long returnCountByStorage) {
+ metricsEvent.put(QueryPropertyEnum.TIME_COST.toString(), callTimeMs);
+ metricsEvent.put(QueryPropertyEnum.CALCITE_RETURN_COUNT.toString(), returnCountByCalcite);
+ metricsEvent.put(QueryPropertyEnum.STORAGE_RETURN_COUNT.toString(), returnCountByStorage);
+ long countAggrAndFilter = returnCountByStorage - returnCountByCalcite;
+ if (countAggrAndFilter < 0) {
+ countAggrAndFilter = 0;
+ logger.warn(returnCountByStorage + " rows returned by storage less than " + returnCountByCalcite
+ + " rows returned by calcite");
+ }
+ metricsEvent.put(QueryPropertyEnum.AGGR_FILTER_COUNT.toString(), countAggrAndFilter);
+ }
+
private static void update(QueryMetrics queryMetrics, SQLResponse sqlResponse) {
try {
incrQueryCount(queryMetrics, sqlResponse);
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/server-base/src/main/java/org/apache/kylin/rest/msg/Message.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/server-base/src/main/java/org/apache/kylin/rest/response/CubeInstanceResponse.java
----------------------------------------------------------------------
diff --cc server-base/src/main/java/org/apache/kylin/rest/response/CubeInstanceResponse.java
index 724568b,3ee28b0..63c305f
--- a/server-base/src/main/java/org/apache/kylin/rest/response/CubeInstanceResponse.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/response/CubeInstanceResponse.java
@@@ -19,9 -19,6 +19,7 @@@
package org.apache.kylin.rest.response;
import org.apache.kylin.cube.CubeInstance;
- import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.metadata.model.ISourceAware;
- import org.apache.kylin.metadata.model.SegmentStatusEnum;
import com.fasterxml.jackson.annotation.JsonProperty;
@@@ -30,92 -28,52 +28,74 @@@
@SuppressWarnings("serial")
public class CubeInstanceResponse extends CubeInstance {
- public void setProject(String project) {
+ @JsonProperty("project")
+ private String project;
+ @JsonProperty("model")
+ private String model;
+ @JsonProperty("is_streaming")
+ private boolean isStreaming;
+ @JsonProperty("partitionDateColumn")
+ private String partitionDateColumn;
+ @JsonProperty("partitionDateStart")
+ private long partitionDateStart;
+ @JsonProperty("isStandardPartitioned")
+ private boolean isStandardPartitioned;
+ @JsonProperty("size_kb")
+ private long sizeKB;
+ @JsonProperty("input_records_count")
+ private long inputRecordCnt;
+ @JsonProperty("input_records_size")
+ private long inputRecordSizeMB;
+
+ public CubeInstanceResponse(CubeInstance cube, String project) {
+
this.project = project;
- }
- public void setModel(String model) {
- this.model = model;
- }
+ if (cube == null)
+ return;
- public void setIs_streaming(boolean is_streaming) {
- this.is_streaming = is_streaming;
- }
+ setUuid(cube.getUuid());
+ setVersion(cube.getVersion());
+ setName(cube.getName());
+ setOwner(cube.getOwner());
+ setDescName(cube.getDescName());
+ setCost(cube.getCost());
+ setStatus(cube.getStatus());
+ setSegments(cube.getSegments());
+ setCreateTimeUTC(cube.getCreateTimeUTC());
+ setLastModified(cube.getDescriptor().getLastModified());
+
+ this.model = cube.getDescriptor().getModelName();
+ this.partitionDateStart = cube.getDescriptor().getPartitionDateStart();
+ // cuz model doesn't have a state to label a model is broken,
+ // so in some case the model can not be loaded due to some check failed,
+ // but the cube in this model can still be loaded.
+ if (cube.getModel() != null) {
+ this.partitionDateColumn = cube.getModel().getPartitionDesc().getPartitionDateColumn();
+ this.isStandardPartitioned = cube.getModel().isStandardPartitionedDateColumn();
+ this.isStreaming = cube.getModel().getRootFactTable().getTableDesc()
+ .getSourceType() == ISourceAware.ID_STREAMING;
+ }
- public void setDraft(boolean isDraft) {
- this.isDraft = isDraft;
+ initSizeKB();
+ initInputRecordCount();
+ initInputRecordSizeMB();
}
- public void setPartitionDateColumn(String partitionDateColumn) {
- this.partitionDateColumn = partitionDateColumn;
+ protected void setModel(String model) {
+ this.model = model;
}
- public void setPartitionDateStart(long partitionDateStart) {
- this.partitionDateStart = partitionDateStart;
+ protected void initSizeKB() {
- long sizeKb = 0L;
-
- for (CubeSegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
- sizeKb += cubeSegment.getSizeKB();
- }
-
- this.sizeKB = sizeKb;
++ this.sizeKB = super.getSizeKB();
}
- @JsonProperty("project")
- private String project;
- @JsonProperty("model")
- private String model;
- @JsonProperty("is_streaming")
- private boolean is_streaming;
- @JsonProperty("is_draft")
- private boolean isDraft;
- @JsonProperty("partitionDateColumn")
- private String partitionDateColumn;
- @JsonProperty("partitionDateStart")
- private long partitionDateStart;
+ protected void initInputRecordCount() {
- long inputRecordCount = 0L;
-
- for (CubeSegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
- inputRecordCount += cubeSegment.getInputRecords();
- }
-
- this.inputRecordCnt = inputRecordCount;
++ this.inputRecordCnt = super.getInputRecordCount();
+ }
- public CubeInstanceResponse(CubeInstance cubeInstance) {
- setUuid(cubeInstance.getUuid());
- setVersion(cubeInstance.getVersion());
- setName(cubeInstance.getName());
- setOwner(cubeInstance.getOwner());
- setDescName(cubeInstance.getDescName());
- setCost(cubeInstance.getCost());
- setStatus(cubeInstance.getStatus());
- setSegments(cubeInstance.getSegments());
- setCreateTimeUTC(cubeInstance.getCreateTimeUTC());
+ protected void initInputRecordSizeMB() {
- long inputRecordSize = 0L;
-
- for (CubeSegment cubeSegment : this.getSegments(SegmentStatusEnum.READY)) {
- inputRecordSize += cubeSegment.getInputRecordsSize();
- }
-
- this.inputRecordSizeMB = inputRecordSize;
++ this.inputRecordSizeMB = super.getInputRecordSizeMB();
}
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
----------------------------------------------------------------------
diff --cc server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
index c2a2d28,a41e21b..0bdf037
--- a/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
@@@ -63,7 -72,7 +72,9 @@@ public class SQLResponse implements Ser
protected boolean queryPushDown = false;
+ protected byte[] queryStatistics;
++
+ protected String traceUrl = null;
public SQLResponse() {
}
@@@ -178,11 -206,25 +198,33 @@@
this.storageCacheUsed = storageCacheUsed;
}
+ public String getTraceUrl() {
+ return traceUrl;
+ }
+
+ public void setTraceUrl(String traceUrl) {
+ this.traceUrl = traceUrl;
+ }
++
+ @JsonIgnore
+ public List<QueryContext.CubeSegmentStatisticsResult> getCubeSegmentStatisticsList() {
+ try {
+ return queryStatistics == null ? Lists.<QueryContext.CubeSegmentStatisticsResult> newArrayList()
+ : (List<QueryContext.CubeSegmentStatisticsResult>) SerializationUtils.deserialize(queryStatistics);
+ } catch (Exception e) { // deserialize exception should not block query
+ logger.warn("Error while deserialize queryStatistics due to " + e);
+ return Lists.newArrayList();
+ }
+ }
+
+ public void setCubeSegmentStatisticsList(
+ List<QueryContext.CubeSegmentStatisticsResult> cubeSegmentStatisticsList) {
+ try {
+ this.queryStatistics = cubeSegmentStatisticsList == null ? null
+ : SerializationUtils.serialize((Serializable) cubeSegmentStatisticsList);
+ } catch (Exception e) { // serialize exception should not block query
+ logger.warn("Error while serialize queryStatistics due to " + e);
+ this.queryStatistics = null;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/server-base/src/main/java/org/apache/kylin/rest/security/AuthoritiesPopulator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --cc server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index a02715e,435b874..2e93a81
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@@ -43,11 -47,8 +48,10 @@@ import org.apache.kylin.measure.percent
import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.draft.Draft;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.SegmentRange;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
- import org.apache.kylin.metadata.model.Segments;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.project.RealizationEntry;
@@@ -59,7 -60,8 +63,9 @@@ import org.apache.kylin.rest.exception.
import org.apache.kylin.rest.msg.Message;
import org.apache.kylin.rest.msg.MsgPicker;
import org.apache.kylin.rest.request.MetricsRequest;
+import org.apache.kylin.rest.response.CubeInstanceResponse;
+ import org.apache.kylin.rest.response.CuboidTreeResponse;
+ import org.apache.kylin.rest.response.CuboidTreeResponse.NodeInfo;
import org.apache.kylin.rest.response.HBaseResponse;
import org.apache.kylin.rest.response.MetricsResponse;
import org.apache.kylin.rest.security.AclPermission;
@@@ -469,17 -488,10 +476,11 @@@ public class CubeService extends BasicS
aclEvaluate.hasProjectOperationPermission(cube.getProjectInstance());
Message msg = MsgPicker.getMsg();
- Segments<CubeSegment> existing = cube.getSegments();
- if (existing.size() > 0 && !segmentName.equals(existing.get(0).getName())
- && !segmentName.equals(existing.get(existing.size() - 1).getName())) {
- throw new BadRequestException(String.format(msg.getDELETE_NOT_FIRST_LAST_SEG(), segmentName));
- }
-
CubeSegment toDelete = null;
- for (CubeSegment seg : existing) {
+ for (CubeSegment seg : cube.getSegments()) {
if (seg.getName().equals(segmentName)) {
toDelete = seg;
+ break;
}
}
@@@ -491,7 -503,14 +492,12 @@@
throw new BadRequestException(String.format(msg.getDELETE_NOT_READY_SEG(), segmentName));
}
+ if (!segmentName.equals(cube.getSegments().get(0).getName())
+ && !segmentName.equals(cube.getSegments().get(cube.getSegments().size() - 1).getName())) {
+ logger.warn(String.format(msg.getDELETE_SEGMENT_CAUSE_GAPS(), cube.getName(), segmentName));
+ }
+
- CubeUpdate update = new CubeUpdate(cube);
- update.setToRemoveSegs(new CubeSegment[] { toDelete });
- return CubeManager.getInstance(getConfig()).updateCube(update);
+ return CubeManager.getInstance(getConfig()).updateCubeDropSegments(cube, toDelete);
}
protected void releaseAllJobs(CubeInstance cube) {
@@@ -512,7 -531,12 +518,12 @@@
*/
private void releaseAllSegments(CubeInstance cube) throws IOException {
releaseAllJobs(cube);
- CubeManager.getInstance(getConfig()).updateCubeDropSegments(cube, cube.getSegments());
+
- CubeUpdate update = new CubeUpdate(cube);
++ CubeUpdate update = new CubeUpdate(cube.latestCopyForWrite());
+ update.setToRemoveSegs(cube.getSegments().toArray(new CubeSegment[cube.getSegments().size()]));
+ update.setCuboids(Maps.<Long, Long>newHashMap());
+ update.setCuboidsRecommend(Sets.<Long>newHashSet());
+ CubeManager.getInstance(getConfig()).updateCube(update);
}
public void updateOnNewSegmentReady(String cubeName) {
@@@ -752,7 -769,83 +763,87 @@@
}
}
+ public CubeInstanceResponse createCubeInstanceResponse(CubeInstance cube) {
+ return new CubeInstanceResponse(cube, projectService.getProjectOfCube(cube.getName()));
+ }
++
+ public CuboidTreeResponse getCuboidTreeResponse(CuboidScheduler cuboidScheduler, Map<Long, Long> rowCountMap,
+ Map<Long, Long> hitFrequencyMap, Map<Long, Long> queryMatchMap, Set<Long> currentCuboidSet) {
+ long baseCuboidId = cuboidScheduler.getBaseCuboidId();
+ int dimensionCount = Long.bitCount(baseCuboidId);
+
+ // get cube query count total
+ long cubeQueryCount = 0L;
+ if (hitFrequencyMap != null) {
+ for (long queryCount : hitFrequencyMap.values()) {
+ cubeQueryCount += queryCount;
+ }
+ }
+
+ NodeInfo root = generateNodeInfo(baseCuboidId, dimensionCount, cubeQueryCount, rowCountMap, hitFrequencyMap,
+ queryMatchMap, currentCuboidSet);
+
+ List<NodeInfo> nodeQueue = Lists.newLinkedList();
+ nodeQueue.add(root);
+ while (!nodeQueue.isEmpty()) {
+ NodeInfo parentNode = nodeQueue.remove(0);
+ for (long childId : cuboidScheduler.getSpanningCuboid(parentNode.getId())) {
+ NodeInfo childNode = generateNodeInfo(childId, dimensionCount, cubeQueryCount, rowCountMap,
+ hitFrequencyMap, queryMatchMap, currentCuboidSet);
+ parentNode.addChild(childNode);
+ nodeQueue.add(childNode);
+ }
+ }
+
+ CuboidTreeResponse result = new CuboidTreeResponse();
+ result.setRoot(root);
+ return result;
+ }
+
+ private NodeInfo generateNodeInfo(long cuboidId, int dimensionCount, long cubeQueryCount,
+ Map<Long, Long> rowCountMap, Map<Long, Long> hitFrequencyMap, Map<Long, Long> queryMatchMap,
+ Set<Long> currentCuboidSet) {
+ Long queryCount = hitFrequencyMap == null || hitFrequencyMap.get(cuboidId) == null ? 0L
+ : hitFrequencyMap.get(cuboidId);
+ float queryRate = cubeQueryCount <= 0 ? 0 : queryCount.floatValue() / cubeQueryCount;
+ long queryExactlyMatchCount = queryMatchMap == null || queryMatchMap.get(cuboidId) == null ? 0L
+ : queryMatchMap.get(cuboidId);
+ boolean ifExist = currentCuboidSet.contains(cuboidId);
+ long rowCount = rowCountMap == null ? 0L : rowCountMap.get(cuboidId);
+
+ NodeInfo node = new NodeInfo();
+ node.setId(cuboidId);
+ node.setName(Cuboid.getDisplayName(cuboidId, dimensionCount));
+ node.setQueryCount(queryCount);
+ node.setQueryRate(queryRate);
+ node.setExactlyMatchCount(queryExactlyMatchCount);
+ node.setExisted(ifExist);
+ node.setRowCount(rowCount);
+ return node;
+ }
+
+ /** cube planner services */
+ public Map<Long, Long> formatQueryCount(List<List<String>> orgQueryCount) {
+ Map<Long, Long> formattedQueryCount = Maps.newLinkedHashMap();
+ for (List<String> hit : orgQueryCount) {
+ formattedQueryCount.put(Long.parseLong(hit.get(0)), (long) Double.parseDouble(hit.get(1)));
+ }
+ return formattedQueryCount;
+ }
+
+ public Map<Long, Map<Long, Long>> formatRollingUpCount(List<List<String>> orgRollingUpCount) {
+ Map<Long, Map<Long, Long>> formattedRollingUpCount = Maps.newLinkedHashMap();
+ for (List<String> rollingUp : orgRollingUpCount) {
+ Map<Long, Long> childMap = Maps.newLinkedHashMap();
+ childMap.put(Long.parseLong(rollingUp.get(1)), (long) Double.parseDouble(rollingUp.get(2)));
+ formattedRollingUpCount.put(Long.parseLong(rollingUp.get(0)), childMap);
+ }
+ return formattedRollingUpCount;
+ }
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION')")
+ public Map<Long, Long> getRecommendCuboidStatistics(CubeInstance cube, Map<Long, Long> hitFrequencyMap,
+ Map<Long, Map<Long, Long>> rollingUpCountSourceMap) throws IOException {
+ return CuboidRecommenderUtil.getRecommendCuboidList(cube, hitFrequencyMap, rollingUpCountSourceMap);
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java
----------------------------------------------------------------------
diff --cc server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java
index 0000000,f1084f3..63bc4fe
mode 000000,100644..100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/DashboardService.java
@@@ -1,0 -1,333 +1,333 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+ package org.apache.kylin.rest.service;
+
+ import java.util.ArrayList;
+ import java.util.List;
+
-import com.google.common.collect.Lists;
+ import org.apache.kylin.cube.CubeInstance;
+ import org.apache.kylin.metadata.project.ProjectInstance;
+ import org.apache.kylin.metadata.project.RealizationEntry;
+ import org.apache.kylin.metadata.realization.RealizationType;
+ import org.apache.kylin.metrics.MetricsManager;
+ import org.apache.kylin.metrics.lib.impl.TimePropertyEnum;
+ import org.apache.kylin.metrics.property.JobPropertyEnum;
+ import org.apache.kylin.metrics.property.QueryPropertyEnum;
+ import org.apache.kylin.rest.constant.Constant;
+ import org.apache.kylin.rest.exception.BadRequestException;
+ import org.apache.kylin.rest.response.MetricsResponse;
+ import org.apache.kylin.rest.response.SQLResponse;
+ import org.apache.kylin.storage.hybrid.HybridInstance;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ import org.springframework.beans.factory.annotation.Autowired;
+ import org.springframework.security.access.AccessDeniedException;
+ import org.springframework.security.access.prepost.PreAuthorize;
+ import org.springframework.stereotype.Component;
+
+ import com.google.common.base.Strings;
++import com.google.common.collect.Lists;
+
+ @Component("dashboardService")
+ public class DashboardService extends BasicService {
+
+ private static final Logger logger = LoggerFactory.getLogger(DashboardService.class);
+
+ @Autowired
+ private CubeService cubeService;
+
+ private enum CategoryEnum {QUERY, JOB}
+
+ private enum QueryDimensionEnum {
+ PROJECT(QueryPropertyEnum.PROJECT.toString()),
+ CUBE(QueryPropertyEnum.REALIZATION.toString()),
+ DAY(TimePropertyEnum.DAY_DATE.toString()),
+ WEEK(TimePropertyEnum.WEEK_BEGIN_DATE.toString()),
+ MONTH(TimePropertyEnum.MONTH.toString());
+ private final String sql;
+
+ QueryDimensionEnum(String sql) {
+ this.sql = sql;
+ }
+
+ public String toSQL() {
+ return this.sql;
+ }
+ };
+
+ private enum JobDimensionEnum {
+ PROJECT(JobPropertyEnum.PROJECT.toString()),
+ CUBE(JobPropertyEnum.CUBE.toString()),
+ DAY(TimePropertyEnum.DAY_DATE.toString()),
+ WEEK(TimePropertyEnum.WEEK_BEGIN_DATE.toString()),
+ MONTH(TimePropertyEnum.MONTH.toString());
+ private final String sql;
+
+ JobDimensionEnum(String sql) {
+ this.sql = sql;
+ }
+
+ public String toSQL() {
+ return this.sql;
+ }
+ };
+
+ private enum QueryMetricEnum {
+ QUERY_COUNT("count(*)"),
+ AVG_QUERY_LATENCY("sum(" + QueryPropertyEnum.TIME_COST.toString() + ")/(count(" + QueryPropertyEnum.TIME_COST.toString() + "))"),
+ MAX_QUERY_LATENCY("max(" + QueryPropertyEnum.TIME_COST.toString() + ")"),
+ MIN_QUERY_LATENCY("min(" + QueryPropertyEnum.TIME_COST.toString() + ")");
+
+ private final String sql;
+
+ QueryMetricEnum(String sql) {
+ this.sql = sql;
+ }
+
+ public String toSQL() {
+ return this.sql;
+ }
+ }
+
+ private enum JobMetricEnum {
+ JOB_COUNT("count(*)"),
+ AVG_JOB_BUILD_TIME("sum(" + JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")/count(" + JobPropertyEnum.PER_BYTES_TIME_COST + ")"),
+ MAX_JOB_BUILD_TIME("max(" + JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")"),
+ MIN_JOB_BUILD_TIME("min(" + JobPropertyEnum.PER_BYTES_TIME_COST.toString() + ")");
+
+ private final String sql;
+
+ JobMetricEnum(String sql) {
+ this.sql = sql;
+ }
+
+ public String toSQL() {
+ return this.sql;
+ }
+ }
+
+ public MetricsResponse getCubeMetrics(String projectName, String cubeName) {
+ MetricsResponse cubeMetrics = new MetricsResponse();
+ Float totalCubeSize = 0f;
+ long totalRecoadSize = 0;
+ List<CubeInstance> cubeInstances = cubeService.listAllCubes(cubeName, projectName, null, true);
+ Integer totalCube = cubeInstances.size();
+ if (projectName == null) {
+ totalCube += getHybridManager().listHybridInstances().size();
+ } else {
+ ProjectInstance project = getProjectManager().getProject(projectName);
+ totalCube += project.getRealizationCount(RealizationType.HYBRID);
+ }
+ Float minCubeExpansion = Float.POSITIVE_INFINITY;
+ Float maxCubeExpansion = Float.NEGATIVE_INFINITY;
+ cubeMetrics.increase("totalCube", totalCube.floatValue());
+ for (CubeInstance cubeInstance : cubeInstances) {
- if (cubeInstance.getInputRecordSize() > 0) {
++ if (cubeInstance.getInputRecordSizeMB() > 0) {
+ totalCubeSize += cubeInstance.getSizeKB();
- totalRecoadSize += cubeInstance.getInputRecordSize();
- Float cubeExpansion = new Float(cubeInstance.getSizeKB()) * 1024 / cubeInstance.getInputRecordSize();
++ totalRecoadSize += cubeInstance.getInputRecordSizeMB();
++ Float cubeExpansion = new Float(cubeInstance.getSizeKB()) * 1024 / cubeInstance.getInputRecordSizeMB();
+ if (cubeExpansion > maxCubeExpansion) {
+ maxCubeExpansion = cubeExpansion;
+ }
+ if (cubeExpansion < minCubeExpansion) {
+ minCubeExpansion = cubeExpansion;
+ }
+ }
+ }
+ Float avgCubeExpansion = 0f;
+ if (totalRecoadSize != 0) {
+ avgCubeExpansion = totalCubeSize * 1024 / totalRecoadSize;
+ }
+ cubeMetrics.increase("avgCubeExpansion", avgCubeExpansion);
+ cubeMetrics.increase("maxCubeExpansion", maxCubeExpansion == Float.NEGATIVE_INFINITY ? 0 : maxCubeExpansion);
+ cubeMetrics.increase("minCubeExpansion", minCubeExpansion == Float.POSITIVE_INFINITY ? 0 : minCubeExpansion);
+ return cubeMetrics;
+ }
+
+ private List<CubeInstance> getCubeByHybrid(HybridInstance hybridInstance) {
+ List<CubeInstance> cubeInstances = Lists.newArrayList();
+ List<RealizationEntry> realizationEntries = hybridInstance.getRealizationEntries();
+ for (RealizationEntry realizationEntry : realizationEntries) {
+ String reName = realizationEntry.getRealization();
+ if (RealizationType.CUBE == realizationEntry.getType()) {
+ CubeInstance cubeInstance = getCubeManager().getCube(reName);
+ cubeInstances.add(cubeInstance);
+ } else if (RealizationType.HYBRID == realizationEntry.getType()) {
+ HybridInstance innerHybridInstance = getHybridManager().getHybridInstance(reName);
+ cubeInstances.addAll(getCubeByHybrid(innerHybridInstance));
+ }
+ }
+ return cubeInstances;
+ }
+
+ public String getQueryMetricsSQL(String startTime, String endTime, String projectName, String cubeName) {
+ String[] metrics = new String[] {QueryMetricEnum.QUERY_COUNT.toSQL(), QueryMetricEnum.AVG_QUERY_LATENCY.toSQL(), QueryMetricEnum.MAX_QUERY_LATENCY.toSQL(), QueryMetricEnum.MIN_QUERY_LATENCY.toSQL()};
+ List<String> filters = getBaseFilters(CategoryEnum.QUERY, projectName, startTime, endTime);
+ filters = addCubeFilter(filters, CategoryEnum.QUERY, cubeName);
+ return createSql(null, metrics, getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQuery()), filters.toArray(new String[filters.size()]));
+ }
+
+ public String getJobMetricsSQL(String startTime, String endTime, String projectName, String cubeName) {
+ String[] metrics = new String[] {JobMetricEnum.JOB_COUNT.toSQL(), JobMetricEnum.AVG_JOB_BUILD_TIME.toSQL(), JobMetricEnum.MAX_JOB_BUILD_TIME.toSQL(), JobMetricEnum.MIN_JOB_BUILD_TIME.toSQL()};
+ List<String> filters = getBaseFilters(CategoryEnum.JOB, projectName, startTime, endTime);
+ filters = addCubeFilter(filters, CategoryEnum.JOB, cubeName);
+ return createSql(null, metrics, getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectJob()), filters.toArray(new String[filters.size()]));
+ }
+
+ public String getChartSQL(String startTime, String endTime, String projectName, String cubeName, String dimension, String metric, String category) {
+ try{
+ CategoryEnum categoryEnum = CategoryEnum.valueOf(category);
+ String table = "";
+ String[] dimensionSQL = null;
+ String[] metricSQL = null;
+
+ if(categoryEnum == CategoryEnum.QUERY) {
+ dimensionSQL = new String[] {QueryDimensionEnum.valueOf(dimension).toSQL()};
+ metricSQL = new String[] {QueryMetricEnum.valueOf(metric).toSQL()};
+ table = getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectQuery());
+ } else if (categoryEnum == CategoryEnum.JOB) {
+ dimensionSQL = new String[] {JobDimensionEnum.valueOf(dimension).toSQL()};
+ metricSQL = new String[] {JobMetricEnum.valueOf(metric).toSQL()};
+ table = getMetricsManager().getSystemTableFromSubject(getConfig().getKylinMetricsSubjectJob());
+ }
+
+ List<String> filters = getBaseFilters(categoryEnum, projectName, startTime, endTime);
+ filters = addCubeFilter(filters, categoryEnum, cubeName);
+
+ return createSql(dimensionSQL, metricSQL, table, filters.toArray(new String[filters.size()]));
+ } catch (IllegalArgumentException e) {
+ String message = "Generate dashboard chart sql failed. Please double check the input parameter: dimension, metric or category.";
+ logger.error(message, e);
+ throw new BadRequestException(message + " Caused by: " + e.getMessage(), null, e.getCause());
+ }
+ }
+
+ public MetricsResponse transformChartData(SQLResponse sqlResponse) {
+ if(!sqlResponse.getIsException()){
+ MetricsResponse metrics = new MetricsResponse();
+ List<List<String>> results = sqlResponse.getResults();
+ for (List<String> result : results) {
+ String dimension = result.get(0);
+ if (dimension !=null && !dimension.isEmpty()) {
+ String metric = result.get(1);
+ metrics.increase(dimension, getMetricValue(metric));
+ }
+ }
+ return metrics;
+ }
+ return null;
+ }
+
+ public Float getMetricValue(String value) {
+ if (value == null || value.isEmpty()) {
+ return 0f;
+ } else {
+ return Float.valueOf(value);
+ }
+ }
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#project, 'ADMINISTRATION')")
+ public void checkAuthorization(ProjectInstance project) throws AccessDeniedException {
+ }
+
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
+ public void checkAuthorization() throws AccessDeniedException{
+ }
+
+ private List<String> getBaseFilters(CategoryEnum category, String projectName, String startTime, String endTime) {
+ List<String> filters = new ArrayList<String>();
+ String project = "";
+ if (category == CategoryEnum.QUERY) {
+ project = QueryDimensionEnum.PROJECT.toSQL();
+ } else {
+ project = JobDimensionEnum.PROJECT.toSQL();
+ }
+ filters.add(TimePropertyEnum.DAY_DATE.toString() + " >= '" + startTime + "'");
+ filters.add(TimePropertyEnum.DAY_DATE.toString() + " <= '" + endTime + "'");
+ if (!Strings.isNullOrEmpty(projectName)) {
- filters.add(project + " ='" + ProjectInstance.getNormalizedProjectName(projectName) + "'");
++ filters.add(project + " ='" + projectName.toUpperCase() + "'");
+ } else {
+ filters.add(project + " <> '" + MetricsManager.SYSTEM_PROJECT + "'");
+ }
+ return filters;
+ }
+
+ private List<String> addCubeFilter(List<String> baseFilter, CategoryEnum category, String cubeName) {
+ if (category == CategoryEnum.QUERY) {
+ baseFilter.add(QueryPropertyEnum.EXCEPTION.toString() + " = 'NULL'");
+ if (!Strings.isNullOrEmpty(cubeName)) {
+ baseFilter.add(QueryPropertyEnum.REALIZATION + " = '" + cubeName + "'");
+ }
+ } else if (category == CategoryEnum.JOB && !Strings.isNullOrEmpty(cubeName)) {
+ HybridInstance hybridInstance = getHybridManager().getHybridInstance(cubeName);
+ if (null != hybridInstance) {
+ StringBuffer cubeNames = new StringBuffer();
+ for (CubeInstance cube:getCubeByHybrid(hybridInstance)) {
+ cubeNames.append(",'" + cube.getName() + "'");
+ }
+ baseFilter.add(JobPropertyEnum.CUBE.toString() + " IN (" + cubeNames.substring(1) + ")");
+ } else {
+ baseFilter.add(JobPropertyEnum.CUBE.toString() + " ='" + cubeName + "'");
+ }
+ }
+ return baseFilter;
+ }
+
+ private String createSql(String[] dimensions, String[] metrics, String category, String[] filters) {
+ StringBuffer baseSQL = new StringBuffer("select ");
+ StringBuffer groupBy = new StringBuffer("");
+ if (dimensions != null && dimensions.length > 0) {
+ groupBy.append(" group by ");
+ StringBuffer dimensionSQL = new StringBuffer("");
+ for (String dimension : dimensions) {
+ dimensionSQL.append(",");
+ dimensionSQL.append(dimension);
+ }
+ baseSQL.append(dimensionSQL.substring(1));
+ groupBy.append(dimensionSQL.substring(1));
+ }
+ if (metrics != null && metrics.length > 0) {
+ StringBuffer metricSQL = new StringBuffer("");
+ for (String metric : metrics) {
+ metricSQL.append(",");
+ metricSQL.append(metric);
+ }
+ if (groupBy.length() > 0) {
+ baseSQL.append(metricSQL);
+ } else {
+ baseSQL.append(metricSQL.substring(1));
+ }
+ }
+ baseSQL.append(" from ");
+ baseSQL.append(category);
+ if (filters != null && filters.length > 0) {
+ StringBuffer filterSQL = new StringBuffer(" where ");
+ filterSQL.append(filters[0]);
+ for(int i = 1; i < filters.length; i++) {
+ filterSQL.append(" and ");
+ filterSQL.append(filters[i]);
+ }
+ baseSQL.append(filterSQL.toString());
+ }
+ baseSQL.append(groupBy);
+
+ return baseSQL.toString();
+ }
+ }
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
----------------------------------------------------------------------
diff --cc server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index 1dfa1de,89e996d..cbbf711
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@@ -34,11 -34,13 +34,14 @@@ import org.apache.commons.lang3.StringU
import org.apache.directory.api.util.Strings;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.ClassUtil;
+ import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
+ import org.apache.kylin.cube.CubeUpdate;
import org.apache.kylin.cube.model.CubeBuildTypeEnum;
import org.apache.kylin.engine.EngineFactory;
+ import org.apache.kylin.engine.mr.BatchOptimizeJobCheckpointBuilder;
import org.apache.kylin.engine.mr.CubingJob;
import org.apache.kylin.engine.mr.common.JobInfoConverter;
import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
@@@ -270,6 -284,137 +283,132 @@@ public class JobService extends BasicSe
return jobInstance;
}
+ public Pair<JobInstance, List<JobInstance>> submitOptimizeJob(CubeInstance cube, Set<Long> cuboidsRecommend,
+ String submitter) throws IOException, JobException {
+
+ Pair<JobInstance, List<JobInstance>> result = submitOptimizeJobInternal(cube, cuboidsRecommend, submitter);
+ accessService.init(result.getFirst(), null);
+ accessService.inherit(result.getFirst(), cube);
+ for (JobInstance jobInstance : result.getSecond()) {
+ accessService.init(jobInstance, null);
+ accessService.inherit(jobInstance, cube);
+ }
+
+ return result;
+ }
+
+ private Pair<JobInstance, List<JobInstance>> submitOptimizeJobInternal(CubeInstance cube,
+ Set<Long> cuboidsRecommend, String submitter) throws IOException {
+ Message msg = MsgPicker.getMsg();
+
+ if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
+ throw new BadRequestException(String.format(msg.getBUILD_BROKEN_CUBE(), cube.getName()));
+ }
+
+ checkCubeDescSignature(cube);
+ checkAllowOptimization(cube, cuboidsRecommend);
+
+ CubeSegment[] optimizeSegments = null;
+ try {
+ /** Add optimize segments */
+ optimizeSegments = getCubeManager().optimizeSegments(cube, cuboidsRecommend);
+ List<JobInstance> optimizeJobInstances = Lists.newLinkedList();
+
+ /** Add optimize jobs */
+ List<AbstractExecutable> optimizeJobList = Lists.newArrayListWithExpectedSize(optimizeSegments.length);
+ for (CubeSegment optimizeSegment : optimizeSegments) {
+ DefaultChainedExecutable optimizeJob = EngineFactory.createBatchOptimizeJob(optimizeSegment, submitter);
+ getExecutableManager().addJob(optimizeJob);
+
+ optimizeJobList.add(optimizeJob);
+ optimizeJobInstances.add(getSingleJobInstance(optimizeJob));
+ }
+
+ /** Add checkpoint job for batch jobs */
+ CheckpointExecutable checkpointJob = new BatchOptimizeJobCheckpointBuilder(cube, submitter).build();
+ checkpointJob.addTaskListForCheck(optimizeJobList);
+
+ getExecutableManager().addJob(checkpointJob);
+
+ return new Pair(getCheckpointJobInstance(checkpointJob), optimizeJobInstances);
+ } catch (Exception e) {
+ if (optimizeSegments != null) {
+ logger.error("Job submission might failed for NEW segments {}, will clean the NEW segments from cube",
+ optimizeSegments);
+ try {
+ // Remove this segments
- CubeUpdate cubeBuilder = new CubeUpdate(cube);
- cubeBuilder.setToRemoveSegs(optimizeSegments);
- getCubeManager().updateCube(cubeBuilder);
++ getCubeManager().updateCubeDropSegments(cube, optimizeSegments);
+ } catch (Exception ee) {
+ // swallow the exception
+ logger.error("Clean New segments failed, ignoring it", e);
+ }
+ }
+ throw e;
+ }
+ }
+
+ public JobInstance submitRecoverSegmentOptimizeJob(CubeSegment segment, String submitter)
+ throws IOException, JobException {
+ CubeInstance cubeInstance = segment.getCubeInstance();
+
+ checkCubeDescSignature(cubeInstance);
+
+ String cubeName = cubeInstance.getName();
+ List<JobInstance> jobInstanceList = searchJobsByCubeName(cubeName, null,
+ Lists.newArrayList(JobStatusEnum.NEW, JobStatusEnum.PENDING, JobStatusEnum.ERROR),
+ JobTimeFilterEnum.ALL, JobSearchMode.CHECKPOINT_ONLY);
+ if (jobInstanceList.size() > 1) {
+ throw new IllegalStateException("Exist more than one CheckpointExecutable for cube " + cubeName);
+ } else if (jobInstanceList.size() == 0) {
+ throw new IllegalStateException("There's no CheckpointExecutable for cube " + cubeName);
+ }
+ CheckpointExecutable checkpointExecutable = (CheckpointExecutable) getExecutableManager()
+ .getJob(jobInstanceList.get(0).getId());
+
+ AbstractExecutable toBeReplaced = null;
+ for (AbstractExecutable taskForCheck : checkpointExecutable.getSubTasksForCheck()) {
+ if (taskForCheck instanceof CubingJob) {
+ CubingJob subCubingJob = (CubingJob) taskForCheck;
+ String segmentName = CubingExecutableUtil.getSegmentName(subCubingJob.getParams());
+ if (segmentName != null && segmentName.equals(segment.getName())) {
+ String segmentID = CubingExecutableUtil.getSegmentId(subCubingJob.getParams());
+ CubeSegment beingOptimizedSegment = cubeInstance.getSegmentById(segmentID);
+ if (beingOptimizedSegment != null) { // beingOptimizedSegment exists & should not be recovered
+ throw new IllegalStateException("Segment " + beingOptimizedSegment.getName() + "-"
+ + beingOptimizedSegment.getUuid()
+ + " still exists. Please delete it or discard the related optimize job first!!!");
+ }
+ toBeReplaced = taskForCheck;
+ break;
+ }
+ }
+ }
+ if (toBeReplaced == null) {
+ throw new IllegalStateException("There's no CubingJob for segment " + segment.getName()
+ + " in CheckpointExecutable " + checkpointExecutable.getName());
+ }
+
+ /** Add CubingJob for the related segment **/
+ CubeSegment optimizeSegment = getCubeManager().appendSegment(cubeInstance, segment.getTSRange());
- CubeUpdate cubeBuilder = new CubeUpdate(cubeInstance);
- cubeBuilder.setToAddSegs(optimizeSegment);
- getCubeManager().updateCube(cubeBuilder);
+
+ DefaultChainedExecutable optimizeJob = EngineFactory.createBatchOptimizeJob(optimizeSegment, submitter);
+
+ getExecutableManager().addJob(optimizeJob);
+
+ JobInstance optimizeJobInstance = getSingleJobInstance(optimizeJob);
+ accessService.init(optimizeJobInstance, null);
+ accessService.inherit(optimizeJobInstance, cubeInstance);
+
+ /** Update the checkpoint job */
+ checkpointExecutable.getSubTasksForCheck().set(checkpointExecutable.getSubTasksForCheck().indexOf(toBeReplaced),
+ optimizeJob);
+
+ getExecutableManager().updateCheckpointJob(checkpointExecutable.getId(),
+ checkpointExecutable.getSubTasksForCheck());
+
+ return optimizeJobInstance;
+ }
+
private void checkCubeDescSignature(CubeInstance cube) {
Message msg = MsgPicker.getMsg();
@@@ -278,8 -423,48 +417,52 @@@
String.format(msg.getINCONSISTENT_CUBE_DESC_SIGNATURE(), cube.getDescriptor()));
}
+ private void checkAllowBuilding(CubeInstance cube) {
- Segments<CubeSegment> readyPendingSegments = cube.getSegments(SegmentStatusEnum.READY_PENDING);
- if (readyPendingSegments.size() > 0) {
- throw new BadRequestException("The cube " + cube.getName() + " has READY_PENDING segments "
- + readyPendingSegments + ". It's not allowed for building");
++ if (cube.getConfig().isCubePlannerEnabled()) {
++ Segments<CubeSegment> readyPendingSegments = cube.getSegments(SegmentStatusEnum.READY_PENDING);
++ if (readyPendingSegments.size() > 0) {
++ throw new BadRequestException("The cube " + cube.getName() + " has READY_PENDING segments "
++ + readyPendingSegments + ". It's not allowed for building");
++ }
+ }
+ }
+
+ private void checkAllowParallelBuilding(CubeInstance cube) {
- if (cube.getCuboids() == null) {
- Segments<CubeSegment> cubeSegments = cube.getSegments();
- if (cubeSegments.size() > 0 && cubeSegments.getSegments(SegmentStatusEnum.READY).size() <= 0) {
- throw new BadRequestException("The cube " + cube.getName() + " has segments " + cubeSegments
- + ", but none of them is READY. It's not allowed for parallel building");
++ if (cube.getConfig().isCubePlannerEnabled()) {
++ if (cube.getCuboids() == null) {
++ Segments<CubeSegment> cubeSegments = cube.getSegments();
++ if (cubeSegments.size() > 0 && cubeSegments.getSegments(SegmentStatusEnum.READY).size() <= 0) {
++ throw new BadRequestException("The cube " + cube.getName() + " has segments " + cubeSegments
++ + ", but none of them is READY. It's not allowed for parallel building");
++ }
+ }
+ }
+ }
+
+ private void checkAllowOptimization(CubeInstance cube, Set<Long> cuboidsRecommend) {
+ Segments<CubeSegment> buildingSegments = cube.getBuildingSegments();
+ if (buildingSegments.size() > 0) {
+ throw new BadRequestException("The cube " + cube.getName() + " has building segments " + buildingSegments
+ + ". It's not allowed for optimization");
+ }
+ long baseCuboid = cube.getCuboidScheduler().getBaseCuboidId();
+ if (!cuboidsRecommend.contains(baseCuboid)) {
+ throw new BadRequestException("The recommend cuboids should contain the base cuboid " + baseCuboid);
+ }
+ Set<Long> currentCuboidSet = cube.getCuboidScheduler().getAllCuboidIds();
+ if (currentCuboidSet.equals(cuboidsRecommend)) {
+ throw new BadRequestException(
+ "The recommend cuboids are the same as the current cuboids. It's no need to do optimization.");
+ }
+ }
+
public JobInstance getJobInstance(String uuid) {
- return getSingleJobInstance(getExecutableManager().getJob(uuid));
+ AbstractExecutable job = getExecutableManager().getJob(uuid);
+ if (job instanceof CheckpointExecutable) {
+ return getCheckpointJobInstance(job);
+ } else {
+ return getSingleJobInstance(job);
+ }
}
public Output getOutput(String id) {
@@@ -338,19 -544,90 +548,84 @@@
getExecutableManager().discardJob(job.getId());
return job;
}
- CubeInstance cubeInstance = getCubeManager().getCube(job.getRelatedCube());
+
+ logger.info("Cancel job [" + job.getId() + "] trigger by "
+ + SecurityContextHolder.getContext().getAuthentication().getName());
+ if (job.getStatus() == JobStatusEnum.FINISHED) {
+ throw new IllegalStateException(
+ "The job " + job.getId() + " has already been finished and cannot be discarded.");
+ }
+ if (job.getStatus() == JobStatusEnum.DISCARDED) {
+ return job;
+ }
+
+ AbstractExecutable executable = getExecutableManager().getJob(job.getId());
+ if (executable instanceof CubingJob) {
+ cancelCubingJobInner((CubingJob) executable);
+ } else if (executable instanceof CheckpointExecutable) {
+ cancelCheckpointJobInner((CheckpointExecutable) executable);
+ } else {
+ getExecutableManager().discardJob(executable.getId());
+ }
+ return job;
+ }
+
+ private void cancelCubingJobInner(CubingJob cubingJob) throws IOException {
+ CubeInstance cubeInstance = getCubeManager().getCube(CubingExecutableUtil.getCubeName(cubingJob.getParams()));
// might not a cube job
- final String segmentIds = job.getRelatedSegment();
- for (String segmentId : StringUtils.split(segmentIds)) {
- final CubeSegment segment = cubeInstance.getSegmentById(segmentId);
- if (segment != null && (segment.getStatus() == SegmentStatusEnum.NEW || segment.getTSRange().end.v == 0)) {
- // Remove this segment
- getCubeManager().updateCubeDropSegments(cubeInstance, segment);
+ final String segmentIds = CubingExecutableUtil.getSegmentId(cubingJob.getParams());
+ if (!StringUtils.isEmpty(segmentIds)) {
- List<CubeSegment> toRemoveSegments = Lists.newLinkedList();
+ for (String segmentId : StringUtils.split(segmentIds)) {
+ final CubeSegment segment = cubeInstance.getSegmentById(segmentId);
+ if (segment != null
+ && (segment.getStatus() == SegmentStatusEnum.NEW || segment.getTSRange().end.v == 0)) {
+ // Remove this segment
- toRemoveSegments.add(segment);
++ getCubeManager().updateCubeDropSegments(cubeInstance, segment);
+ }
}
- if (!toRemoveSegments.isEmpty()) {
- CubeUpdate cubeBuilder = new CubeUpdate(cubeInstance);
- cubeBuilder.setToRemoveSegs(toRemoveSegments.toArray(new CubeSegment[toRemoveSegments.size()]));
- getCubeManager().updateCube(cubeBuilder);
- }
}
- getExecutableManager().discardJob(job.getId());
+ getExecutableManager().discardJob(cubingJob.getId());
+ }
- return job;
+ private void cancelCheckpointJobInner(CheckpointExecutable checkpointExecutable) throws IOException {
+ List<String> segmentIdList = Lists.newLinkedList();
+ List<String> jobIdList = Lists.newLinkedList();
+ jobIdList.add(checkpointExecutable.getId());
+ setRelatedIdList(checkpointExecutable, segmentIdList, jobIdList);
+
+ CubeInstance cubeInstance = getCubeManager()
+ .getCube(CubingExecutableUtil.getCubeName(checkpointExecutable.getParams()));
+ if (!segmentIdList.isEmpty()) {
+ List<CubeSegment> toRemoveSegments = Lists.newLinkedList();
+ for (String segmentId : segmentIdList) {
+ final CubeSegment segment = cubeInstance.getSegmentById(segmentId);
+ if (segment != null && segment.getStatus() != SegmentStatusEnum.READY) {
+ toRemoveSegments.add(segment);
+ }
+ }
+
+ CubeUpdate cubeBuilder = new CubeUpdate(cubeInstance);
+ cubeBuilder.setToRemoveSegs(toRemoveSegments.toArray(new CubeSegment[toRemoveSegments.size()]));
+ cubeBuilder.setCuboidsRecommend(Sets.<Long> newHashSet()); //Set recommend cuboids to be null
+ getCubeManager().updateCube(cubeBuilder);
+ }
+
+ for (String jobId : jobIdList) {
+ getExecutableManager().discardJob(jobId);
+ }
+ }
+
+ private void setRelatedIdList(CheckpointExecutable checkpointExecutable, List<String> segmentIdList,
+ List<String> jobIdList) {
+ for (AbstractExecutable taskForCheck : checkpointExecutable.getSubTasksForCheck()) {
+ jobIdList.add(taskForCheck.getId());
+ if (taskForCheck instanceof CubingJob) {
+ segmentIdList.addAll(Lists
+ .newArrayList(StringUtils.split(CubingExecutableUtil.getSegmentId(taskForCheck.getParams()))));
+ } else if (taskForCheck instanceof CheckpointExecutable) {
+ setRelatedIdList((CheckpointExecutable) taskForCheck, segmentIdList, jobIdList);
+ }
+ }
}
public JobInstance pauseJob(JobInstance job) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d50b269/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
----------------------------------------------------------------------
diff --cc server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
index 600c901,17a9a8c..6a21fad
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java
@@@ -136,11 -132,6 +136,11 @@@ public class ModelService extends Basic
public DataModelDesc createModelDesc(String projectName, DataModelDesc desc) throws IOException {
aclEvaluate.hasProjectWritePermission(getProjectManager().getProject(projectName));
+ Message msg = MsgPicker.getMsg();
-
+ if (getDataModelManager().getDataModelDesc(desc.getName()) != null) {
+ throw new BadRequestException(String.format(msg.getDUPLICATE_MODEL_NAME(), desc.getName()));
+ }
++
DataModelDesc createdDesc = null;
String owner = SecurityContextHolder.getContext().getAuthentication().getName();
createdDesc = getDataModelManager().createDataModelDesc(desc, projectName, owner);