You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/06/25 23:34:24 UTC
[12/13] kylin git commit: KYLIN-1823 split code from kylin-server
into kylin-server-base
http://git-wip-us.apache.org/repos/asf/kylin/blob/1a124e68/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
new file mode 100644
index 0000000..9315a20
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -0,0 +1,615 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.controller;
+
+import java.io.IOException;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.model.CubeBuildTypeEnum;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
+import org.apache.kylin.dimension.DimensionEncodingFactory;
+import org.apache.kylin.engine.streaming.StreamingConfig;
+import org.apache.kylin.job.JobInstance;
+import org.apache.kylin.job.JoinedFlatTable;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.rest.exception.BadRequestException;
+import org.apache.kylin.rest.exception.ForbiddenException;
+import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.exception.NotFoundException;
+import org.apache.kylin.rest.request.CubeRequest;
+import org.apache.kylin.rest.request.JobBuildRequest;
+import org.apache.kylin.rest.request.JobBuildRequest2;
+import org.apache.kylin.rest.response.GeneralResponse;
+import org.apache.kylin.rest.response.HBaseResponse;
+import org.apache.kylin.rest.service.CubeService;
+import org.apache.kylin.rest.service.JobService;
+import org.apache.kylin.rest.service.KafkaConfigService;
+import org.apache.kylin.rest.service.StreamingService;
+import org.apache.kylin.source.kafka.config.KafkaConfig;
+import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.security.access.AccessDeniedException;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonMappingException;
+
+import com.google.common.collect.Sets;
+
+/**
+ * CubeController is defined as Restful API entrance for UI.
+ */
+@Controller
+@RequestMapping(value = "/cubes")
+public class CubeController extends BasicController {
+ private static final Logger logger = LoggerFactory.getLogger(CubeController.class);
+
+ @Autowired
+ private StreamingService streamingService;
+
+ @Autowired
+ private KafkaConfigService kafkaConfigService;
+
+ @Autowired
+ private CubeService cubeService;
+
+ @Autowired
+ private JobService jobService;
+
+ @RequestMapping(value = "", method = { RequestMethod.GET })
+ @ResponseBody
+ public List<CubeInstance> getCubes(@RequestParam(value = "cubeName", required = false) String cubeName, @RequestParam(value = "modelName", required = false) String modelName, @RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value = "limit", required = false) Integer limit, @RequestParam(value = "offset", required = false) Integer offset) {
+ return cubeService.getCubes(cubeName, projectName, modelName, limit, offset);
+ }
+
+ @RequestMapping(value = "validEncodings", method = { RequestMethod.GET })
+ @ResponseBody
+ public Set<String> getValidEncodings() {
+ Set<String> encodings;
+ try {
+ encodings = DimensionEncodingFactory.getValidEncodings();
+ } catch (Exception e) {
+ return Sets.newTreeSet();
+ }
+ return encodings;
+ }
+
+ @RequestMapping(value = "/{cubeName}", method = { RequestMethod.GET })
+ @ResponseBody
+ public CubeInstance getCube(@PathVariable String cubeName) {
+ CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
+ if (cube == null) {
+ throw new InternalErrorException("Cannot find cube " + cubeName);
+ }
+ return cube;
+ }
+
+ /**
+ * Get hive SQL of the cube
+ *
+ * @param cubeName Cube Name
+ * @return
+ * @throws UnknownHostException
+ * @throws IOException
+ */
+ @RequestMapping(value = "/{cubeName}/segs/{segmentName}/sql", method = { RequestMethod.GET })
+ @ResponseBody
+ public GeneralResponse getSql(@PathVariable String cubeName, @PathVariable String segmentName) {
+ CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
+ CubeDesc cubeDesc = cube.getDescriptor();
+ CubeSegment cubeSegment = cube.getSegment(segmentName, SegmentStatusEnum.READY);
+ CubeJoinedFlatTableDesc flatTableDesc = new CubeJoinedFlatTableDesc(cubeDesc, cubeSegment);
+ String sql = JoinedFlatTable.generateSelectDataStatement(flatTableDesc);
+
+ GeneralResponse repsonse = new GeneralResponse();
+ repsonse.setProperty("sql", sql);
+
+ return repsonse;
+ }
+
+ /**
+ * Update cube notify list
+ *
+ * @param cubeName
+ * @param notifyList
+ * @throws IOException
+ */
+ @RequestMapping(value = "/{cubeName}/notify_list", method = { RequestMethod.PUT })
+ @ResponseBody
+ public void updateNotifyList(@PathVariable String cubeName, @RequestBody List<String> notifyList) {
+ CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
+
+ if (cube == null) {
+ throw new InternalErrorException("Cannot find cube " + cubeName);
+ }
+
+ try {
+ cubeService.updateCubeNotifyList(cube, notifyList);
+ } catch (Exception e) {
+ logger.error(e.getLocalizedMessage(), e);
+ throw new InternalErrorException(e.getLocalizedMessage());
+ }
+
+ }
+
+ @RequestMapping(value = "/{cubeName}/cost", method = { RequestMethod.PUT })
+ @ResponseBody
+ public CubeInstance updateCubeCost(@PathVariable String cubeName, @RequestParam(value = "cost") int cost) {
+ try {
+ return cubeService.updateCubeCost(cubeName, cost);
+ } catch (Exception e) {
+ String message = "Failed to update cube cost: " + cubeName + " : " + cost;
+ logger.error(message, e);
+ throw new InternalErrorException(message + " Caused by: " + e.getMessage(), e);
+ }
+ }
+
+ @RequestMapping(value = "/{cubeName}/coprocessor", method = { RequestMethod.PUT })
+ @ResponseBody
+ public Map<String, Boolean> updateCubeCoprocessor(@PathVariable String cubeName, @RequestParam(value = "force") String force) {
+ try {
+ ObserverEnabler.updateCubeOverride(cubeName, force);
+ return ObserverEnabler.getCubeOverrides();
+ } catch (Exception e) {
+ String message = "Failed to update cube coprocessor: " + cubeName + " : " + force;
+ logger.error(message, e);
+ throw new InternalErrorException(message + " Caused by: " + e.getMessage(), e);
+ }
+ }
+
+ /**
+ * Force rebuild a cube's lookup table snapshot
+ *
+ * @throws IOException
+ */
+ @RequestMapping(value = "/{cubeName}/segs/{segmentName}/refresh_lookup", method = { RequestMethod.PUT })
+ @ResponseBody
+ public CubeInstance rebuildLookupSnapshot(@PathVariable String cubeName, @PathVariable String segmentName, @RequestParam(value = "lookupTable") String lookupTable) {
+ try {
+ return cubeService.rebuildLookupSnapshot(cubeName, segmentName, lookupTable);
+ } catch (IOException e) {
+ logger.error(e.getLocalizedMessage(), e);
+ throw new InternalErrorException(e.getLocalizedMessage());
+ }
+ }
+
+ /**
+ * Delete a cube segment
+ *
+ * @throws IOException
+ */
+ @RequestMapping(value = "/{cubeName}/segs/{segmentName}", method = { RequestMethod.DELETE })
+ @ResponseBody
+ public CubeInstance deleteSegment(@PathVariable String cubeName, @PathVariable String segmentName) {
+ CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
+
+ if (cube == null) {
+ throw new InternalErrorException("Cannot find cube " + cubeName);
+ }
+
+ CubeSegment segment = cube.getSegment(segmentName, null);
+ if (segment == null) {
+ throw new InternalErrorException("Cannot find segment '" + segmentName + "'");
+ }
+
+ try {
+ return cubeService.deleteSegment(cube, segmentName);
+ } catch (Exception e) {
+ logger.error(e.getLocalizedMessage(), e);
+ throw new InternalErrorException(e.getLocalizedMessage());
+ }
+ }
+
+ /** Build/Rebuild a cube segment */
+ @RequestMapping(value = "/{cubeName}/build", method = { RequestMethod.PUT })
+ @ResponseBody
+ public JobInstance build(@PathVariable String cubeName, @RequestBody JobBuildRequest req) {
+ return rebuild(cubeName, req);
+ }
+
+ /** Build/Rebuild a cube segment */
+ @RequestMapping(value = "/{cubeName}/rebuild", method = { RequestMethod.PUT })
+ @ResponseBody
+ public JobInstance rebuild(@PathVariable String cubeName, @RequestBody JobBuildRequest req) {
+ return buildInternal(cubeName, req.getStartTime(), req.getEndTime(), 0, 0, req.getBuildType(), req.isForce() || req.isForceMergeEmptySegment());
+ }
+
+ /** Build/Rebuild a cube segment by source offset */
+ @RequestMapping(value = "/{cubeName}/build2", method = { RequestMethod.PUT })
+ @ResponseBody
+ public JobInstance build(@PathVariable String cubeName, @RequestBody JobBuildRequest2 req) {
+ return rebuild(cubeName, req);
+ }
+
+ /** Build/Rebuild a cube segment by source offset */
+ @RequestMapping(value = "/{cubeName}/rebuild2", method = { RequestMethod.PUT })
+ @ResponseBody
+ public JobInstance rebuild(@PathVariable String cubeName, @RequestBody JobBuildRequest2 req) {
+ return buildInternal(cubeName, 0, 0, req.getStartSourceOffset(), req.getEndSourceOffset(), req.getBuildType(), req.isForce());
+ }
+
+ private JobInstance buildInternal(String cubeName, long startTime, long endTime, //
+ long startOffset, long endOffset, String buildType, boolean force) {
+ try {
+ String submitter = SecurityContextHolder.getContext().getAuthentication().getName();
+ CubeInstance cube = jobService.getCubeManager().getCube(cubeName);
+ return jobService.submitJob(cube, startTime, endTime, startOffset, endOffset, //
+ CubeBuildTypeEnum.valueOf(buildType), force, submitter);
+ } catch (Exception e) {
+ logger.error(e.getLocalizedMessage(), e);
+ throw new InternalErrorException(e.getLocalizedMessage());
+ }
+ }
+
+ @RequestMapping(value = "/{cubeName}/disable", method = { RequestMethod.PUT })
+ @ResponseBody
+ public CubeInstance disableCube(@PathVariable String cubeName) {
+ try {
+ CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
+
+ if (cube == null) {
+ throw new InternalErrorException("Cannot find cube " + cubeName);
+ }
+
+ return cubeService.disableCube(cube);
+ } catch (Exception e) {
+ String message = "Failed to disable cube: " + cubeName;
+ logger.error(message, e);
+ throw new InternalErrorException(message + " Caused by: " + e.getMessage(), e);
+ }
+ }
+
+ @RequestMapping(value = "/{cubeName}/purge", method = { RequestMethod.PUT })
+ @ResponseBody
+ public CubeInstance purgeCube(@PathVariable String cubeName) {
+ try {
+ CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
+
+ if (cube == null) {
+ throw new InternalErrorException("Cannot find cube " + cubeName);
+ }
+
+ return cubeService.purgeCube(cube);
+ } catch (Exception e) {
+ String message = "Failed to purge cube: " + cubeName;
+ logger.error(message, e);
+ throw new InternalErrorException(message + " Caused by: " + e.getMessage(), e);
+ }
+ }
+
+ @RequestMapping(value = "/{cubeName}/clone", method = { RequestMethod.PUT })
+ @ResponseBody
+ public CubeInstance cloneCube(@PathVariable String cubeName, @RequestBody CubeRequest cubeRequest) {
+ String newCubeName = cubeRequest.getCubeName();
+ String project = cubeRequest.getProject();
+
+ CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
+ if (cube == null) {
+ throw new InternalErrorException("Cannot find cube " + cubeName);
+ }
+ CubeDesc cubeDesc = cube.getDescriptor();
+ CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc);
+ newCubeDesc.setName(newCubeName);
+
+ CubeInstance newCube;
+ try {
+ newCube = cubeService.createCubeAndDesc(newCubeName, project, newCubeDesc);
+
+ //reload to avoid shallow clone
+ cubeService.getCubeDescManager().reloadCubeDescLocal(newCubeName);
+ } catch (IOException e) {
+ throw new InternalErrorException("Failed to clone cube ", e);
+ }
+
+ return newCube;
+
+ }
+
+ @RequestMapping(value = "/{cubeName}/enable", method = { RequestMethod.PUT })
+ @ResponseBody
+ public CubeInstance enableCube(@PathVariable String cubeName) {
+ try {
+ CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
+ if (null == cube) {
+ throw new InternalErrorException("Cannot find cube " + cubeName);
+ }
+
+ return cubeService.enableCube(cube);
+ } catch (Exception e) {
+ String message = "Failed to enable cube: " + cubeName;
+ logger.error(message, e);
+ throw new InternalErrorException(message + " Caused by: " + e.getMessage(), e);
+ }
+ }
+
+ @RequestMapping(value = "/{cubeName}", method = { RequestMethod.DELETE })
+ @ResponseBody
+ public void deleteCube(@PathVariable String cubeName) {
+ CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
+ if (null == cube) {
+ throw new NotFoundException("Cube with name " + cubeName + " not found..");
+ }
+
+ //drop Cube
+ try {
+ cubeService.deleteCube(cube);
+ } catch (Exception e) {
+ logger.error(e.getLocalizedMessage(), e);
+ throw new InternalErrorException("Failed to delete cube. " + " Caused by: " + e.getMessage(), e);
+ }
+
+ }
+
+ /**
+ * save cubeDesc
+ *
+ * @return Table metadata array
+ * @throws IOException
+ */
+ @RequestMapping(value = "", method = { RequestMethod.POST })
+ @ResponseBody
+ public CubeRequest saveCubeDesc(@RequestBody CubeRequest cubeRequest) {
+
+ CubeDesc desc = deserializeCubeDesc(cubeRequest);
+ if (desc == null) {
+ cubeRequest.setMessage("CubeDesc is null.");
+ return cubeRequest;
+ }
+ String name = CubeService.getCubeNameFromDesc(desc.getName());
+ if (StringUtils.isEmpty(name)) {
+ logger.info("Cube name should not be empty.");
+ throw new BadRequestException("Cube name should not be empty.");
+ }
+
+ try {
+ desc.setUuid(UUID.randomUUID().toString());
+ String projectName = (null == cubeRequest.getProject()) ? ProjectInstance.DEFAULT_PROJECT_NAME : cubeRequest.getProject();
+ cubeService.createCubeAndDesc(name, projectName, desc);
+ } catch (Exception e) {
+ logger.error("Failed to deal with the request.", e);
+ throw new InternalErrorException(e.getLocalizedMessage(), e);
+ }
+
+ cubeRequest.setUuid(desc.getUuid());
+ cubeRequest.setSuccessful(true);
+ return cubeRequest;
+ }
+
+ /**
+ * update CubDesc
+ *
+ * @return Table metadata array
+ * @throws JsonProcessingException
+ * @throws IOException
+ */
+ @RequestMapping(value = "", method = { RequestMethod.PUT })
+ @ResponseBody
+ public CubeRequest updateCubeDesc(@RequestBody CubeRequest cubeRequest) throws JsonProcessingException {
+
+ //update cube
+ CubeDesc desc = deserializeCubeDesc(cubeRequest);
+ CubeDesc oldCubeDesc;
+ boolean isCubeDescFreeEditable;
+
+ if (desc == null) {
+ return cubeRequest;
+ }
+
+ // Check if the cube is editable
+ isCubeDescFreeEditable = cubeService.isCubeDescFreeEditable(desc);
+
+ String projectName = (null == cubeRequest.getProject()) ? ProjectInstance.DEFAULT_PROJECT_NAME : cubeRequest.getProject();
+ try {
+ CubeInstance cube = cubeService.getCubeManager().getCube(cubeRequest.getCubeName());
+
+ if (cube == null) {
+ String error = "The cube named " + cubeRequest.getCubeName() + " does not exist ";
+ updateRequest(cubeRequest, false, error);
+ return cubeRequest;
+ }
+
+ //cube renaming is not allowed
+ if (!cube.getDescriptor().getName().equalsIgnoreCase(desc.getName())) {
+ String error = "Cube Desc renaming is not allowed: desc.getName(): " + desc.getName() + ", cubeRequest.getCubeName(): " + cubeRequest.getCubeName();
+ updateRequest(cubeRequest, false, error);
+ return cubeRequest;
+ }
+
+ oldCubeDesc = cube.getDescriptor();
+ if (isCubeDescFreeEditable || oldCubeDesc.consistentWith(desc)) {
+ desc = cubeService.updateCubeAndDesc(cube, desc, projectName, true);
+ } else {
+ logger.warn("Won't update the cube desc due to inconsistency");
+ updateRequest(cubeRequest, false, "CubeDesc " + desc.getName() + " is inconsistent with existing. Try purge that cube first or avoid updating key cube desc fields.");
+ return cubeRequest;
+ }
+ } catch (AccessDeniedException accessDeniedException) {
+ throw new ForbiddenException("You don't have right to update this cube.");
+ } catch (Exception e) {
+ logger.error("Failed to deal with the request:" + e.getLocalizedMessage(), e);
+ throw new InternalErrorException("Failed to deal with the request: " + e.getLocalizedMessage());
+ }
+
+ if (!desc.getError().isEmpty()) {
+ logger.warn("Cube " + desc.getName() + " fail to update because " + desc.getError());
+ updateRequest(cubeRequest, false, omitMessage(desc.getError()));
+ return cubeRequest;
+ }
+
+ String descData = JsonUtil.writeValueAsIndentString(desc);
+ cubeRequest.setCubeDescData(descData);
+ cubeRequest.setSuccessful(true);
+ return cubeRequest;
+ }
+
+ /**
+ * get Hbase Info
+ *
+ * @return true
+ * @throws IOException
+ */
+ @RequestMapping(value = "/{cubeName}/hbase", method = { RequestMethod.GET })
+ @ResponseBody
+ public List<HBaseResponse> getHBaseInfo(@PathVariable String cubeName) {
+ List<HBaseResponse> hbase = new ArrayList<HBaseResponse>();
+
+ CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
+ if (null == cube) {
+ throw new InternalErrorException("Cannot find cube " + cubeName);
+ }
+
+ List<CubeSegment> segments = cube.getSegments();
+
+ for (CubeSegment segment : segments) {
+ String tableName = segment.getStorageLocationIdentifier();
+ HBaseResponse hr = null;
+
+ // Get info of given table.
+ try {
+ hr = cubeService.getHTableInfo(tableName);
+ } catch (IOException e) {
+ logger.error("Failed to calcuate size of HTable \"" + tableName + "\".", e);
+ }
+
+ if (null == hr) {
+ logger.info("Failed to calcuate size of HTable \"" + tableName + "\".");
+ hr = new HBaseResponse();
+ }
+
+ hr.setTableName(tableName);
+ hr.setDateRangeStart(segment.getDateRangeStart());
+ hr.setDateRangeEnd(segment.getDateRangeEnd());
+ hbase.add(hr);
+ }
+
+ return hbase;
+ }
+
+ private CubeDesc deserializeCubeDesc(CubeRequest cubeRequest) {
+ CubeDesc desc = null;
+ try {
+ logger.debug("Saving cube " + cubeRequest.getCubeDescData());
+ desc = JsonUtil.readValue(cubeRequest.getCubeDescData(), CubeDesc.class);
+ } catch (JsonParseException e) {
+ logger.error("The cube definition is not valid.", e);
+ updateRequest(cubeRequest, false, e.getMessage());
+ } catch (JsonMappingException e) {
+ logger.error("The cube definition is not valid.", e);
+ updateRequest(cubeRequest, false, e.getMessage());
+ } catch (IOException e) {
+ logger.error("Failed to deal with the request.", e);
+ throw new InternalErrorException("Failed to deal with the request:" + e.getMessage(), e);
+ }
+ return desc;
+ }
+
+ private StreamingConfig deserializeStreamingDesc(CubeRequest cubeRequest) {
+ StreamingConfig desc = null;
+ try {
+ logger.debug("Saving StreamingConfig " + cubeRequest.getStreamingData());
+ desc = JsonUtil.readValue(cubeRequest.getStreamingData(), StreamingConfig.class);
+ } catch (JsonParseException e) {
+ logger.error("The StreamingConfig definition is not valid.", e);
+ updateRequest(cubeRequest, false, e.getMessage());
+ } catch (JsonMappingException e) {
+ logger.error("The data StreamingConfig definition is not valid.", e);
+ updateRequest(cubeRequest, false, e.getMessage());
+ } catch (IOException e) {
+ logger.error("Failed to deal with the request.", e);
+ throw new InternalErrorException("Failed to deal with the request:" + e.getMessage(), e);
+ }
+ return desc;
+ }
+
+ private KafkaConfig deserializeKafkaDesc(CubeRequest cubeRequest) {
+ KafkaConfig desc = null;
+ try {
+ logger.debug("Saving KafkaConfig " + cubeRequest.getKafkaData());
+ desc = JsonUtil.readValue(cubeRequest.getKafkaData(), KafkaConfig.class);
+ } catch (JsonParseException e) {
+ logger.error("The KafkaConfig definition is not valid.", e);
+ updateRequest(cubeRequest, false, e.getMessage());
+ } catch (JsonMappingException e) {
+ logger.error("The data KafkaConfig definition is not valid.", e);
+ updateRequest(cubeRequest, false, e.getMessage());
+ } catch (IOException e) {
+ logger.error("Failed to deal with the request.", e);
+ throw new InternalErrorException("Failed to deal with the request:" + e.getMessage(), e);
+ }
+ return desc;
+ }
+
+ /**
+ * @return
+ */
+ private String omitMessage(List<String> errors) {
+ StringBuffer buffer = new StringBuffer();
+ for (Iterator<String> iterator = errors.iterator(); iterator.hasNext();) {
+ String string = (String) iterator.next();
+ buffer.append(string);
+ buffer.append("\n");
+ }
+ return buffer.toString();
+ }
+
+ private void updateRequest(CubeRequest request, boolean success, String message) {
+ request.setCubeDescData("");
+ request.setSuccessful(success);
+ request.setMessage(message);
+ }
+
+ public void setCubeService(CubeService cubeService) {
+ this.cubeService = cubeService;
+ }
+
+ public void setJobService(JobService jobService) {
+ this.jobService = jobService;
+ }
+
+ public void setStreamingService(StreamingService streamingService) {
+ this.streamingService = streamingService;
+ }
+
+ public void setKafkaConfigService(KafkaConfigService kafkaConfigService) {
+ this.kafkaConfigService = kafkaConfigService;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1a124e68/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
new file mode 100644
index 0000000..61584a3
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeDescController.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.controller;
+
+import java.io.IOException;
+
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.rest.service.CubeService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+/**
+ * @author xduo
+ *
+ */
+@Controller
+@RequestMapping(value = "/cube_desc")
+public class CubeDescController {
+
+ @Autowired
+ private CubeService cubeService;
+
+ /**
+ * Get detail information of the "Cube ID"
+ *
+ * @param cubeDescName
+ * Cube ID
+ * @return
+ * @throws IOException
+ */
+ @RequestMapping(value = "/{cubeName}", method = { RequestMethod.GET })
+ @ResponseBody
+ public CubeDesc[] getCube(@PathVariable String cubeName) {
+ CubeInstance cubeInstance = cubeService.getCubeManager().getCube(cubeName);
+ if (cubeInstance == null) {
+ return null;
+ }
+ CubeDesc cSchema = cubeInstance.getDescriptor();
+ if (cSchema != null) {
+ return new CubeDesc[] { cSchema };
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Get detail information of the "Cube ID"
+ * return CubeDesc instead of CubeDesc[]
+ *
+ * @param cubeDescName
+ * Cube ID
+ * @return
+ * @throws IOException
+ */
+ @RequestMapping(value = "/{cubeName}/desc", method = { RequestMethod.GET })
+ @ResponseBody
+ public CubeDesc getDesc(@PathVariable String cubeName) {
+ CubeInstance cubeInstance = cubeService.getCubeManager().getCube(cubeName);
+ if (cubeInstance == null) {
+ return null;
+ }
+ CubeDesc cSchema = cubeInstance.getDescriptor();
+ if (cSchema != null) {
+ return cSchema;
+ } else {
+ return null;
+ }
+ }
+
+ public void setCubeService(CubeService cubeService) {
+ this.cubeService = cubeService;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1a124e68/server-base/src/main/java/org/apache/kylin/rest/controller/DiagnosisController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/DiagnosisController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/DiagnosisController.java
new file mode 100644
index 0000000..b9da9b2
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/DiagnosisController.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.kylin.rest.controller;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.List;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.kylin.metadata.badquery.BadQueryEntry;
+import org.apache.kylin.metadata.badquery.BadQueryHistory;
+import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.service.DiagnosisService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+import com.google.common.collect.Lists;
+
+@Controller
+@RequestMapping(value = "/diag")
+public class DiagnosisController {
+
+ private static final Logger logger = LoggerFactory.getLogger(DiagnosisController.class);
+
+ @Autowired
+ private DiagnosisService dgService;
+
+ /**
+ * Get bad query history
+ */
+ @RequestMapping(value = "/{project}/sql", method = { RequestMethod.GET })
+ @ResponseBody
+ public List<BadQueryEntry> getBadQuerySql(@PathVariable String project) {
+
+ List<BadQueryEntry> badEntry = Lists.newArrayList();
+ try {
+ BadQueryHistory badQueryHistory = dgService.getProjectBadQueryHistory(project);
+ badEntry.addAll(badQueryHistory.getEntries());
+ } catch (IOException e) {
+ throw new InternalErrorException(e + " Caused by: " + e.getMessage(), e);
+ }
+
+ return badEntry;
+ }
+
+ /**
+ * Get diagnosis information for project
+ */
+ @RequestMapping(value = "/project/{project}/download", method = { RequestMethod.GET })
+ @ResponseBody
+ public void dumpProjectDiagnosisInfo(@PathVariable String project, final HttpServletRequest request, final HttpServletResponse response) {
+ String filePath;
+ try {
+ filePath = dgService.dumpProjectDiagnosisInfo(project);
+ } catch (IOException e) {
+ throw new InternalErrorException(e + " Caused by: " + e.getMessage(), e);
+ }
+
+ setDownloadResponse(filePath, response);
+ }
+
+ /**
+ * Get diagnosis information for job
+ */
+ @RequestMapping(value = "/job/{jobId}/download", method = { RequestMethod.GET })
+ @ResponseBody
+ public void dumpJobDiagnosisInfo(@PathVariable String jobId, final HttpServletRequest request, final HttpServletResponse response) {
+ String filePath;
+ try {
+ filePath = dgService.dumpJobDiagnosisInfo(jobId);
+ } catch (IOException e) {
+ throw new InternalErrorException(e + " Caused by: " + e.getMessage(), e);
+ }
+
+ setDownloadResponse(filePath, response);
+ }
+
+ private void setDownloadResponse(String downloadFile, final HttpServletResponse response) {
+ File file = new File(downloadFile);
+ try (InputStream fileInputStream = new FileInputStream(file); OutputStream output = response.getOutputStream();) {
+ response.reset();
+ response.setContentType("application/octet-stream");
+ response.setContentLength((int) (file.length()));
+ response.setHeader("Content-Disposition", "attachment; filename=\"" + file.getName() + "\"");
+ IOUtils.copyLarge(fileInputStream, output);
+ output.flush();
+ } catch (IOException e) {
+ throw new InternalErrorException(e + " Caused by: " + e.getMessage(), e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1a124e68/server-base/src/main/java/org/apache/kylin/rest/controller/ExternalFilterController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/ExternalFilterController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/ExternalFilterController.java
new file mode 100644
index 0000000..7baa639
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/ExternalFilterController.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.controller;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.metadata.model.ExternalFilterDesc;
+import org.apache.kylin.rest.request.ExternalFilterRequest;
+import org.apache.kylin.rest.service.ExtFilterService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+import com.google.common.collect.Lists;
+
+/**
+ * @author jiazhong
+ */
+@Controller
+@RequestMapping(value = "/extFilter")
+public class ExternalFilterController extends BasicController {
+ private static final Logger logger = LoggerFactory.getLogger(ExternalFilterController.class);
+
+ @Autowired
+ private ExtFilterService extFilterService;
+
+ @RequestMapping(value = "/saveExtFilter", method = { RequestMethod.POST })
+ @ResponseBody
+ public Map<String, String> saveExternalFilter(@RequestBody ExternalFilterRequest request) throws IOException {
+ Map<String, String> result = new HashMap();
+ String filterProject = request.getProject();
+ ExternalFilterDesc desc = JsonUtil.readValue(request.getExtFilter(), ExternalFilterDesc.class);
+ desc.setUuid(UUID.randomUUID().toString());
+ extFilterService.saveExternalFilter(desc);
+ extFilterService.syncExtFilterToProject(new String[] { desc.getName() }, filterProject);
+ result.put("success", "true");
+ return result;
+ }
+
+ @RequestMapping(value = "/updateExtFilter", method = { RequestMethod.PUT })
+ @ResponseBody
+ public Map<String, String> updateExternalFilter(@RequestBody ExternalFilterRequest request) throws IOException {
+ Map<String, String> result = new HashMap();
+ ExternalFilterDesc desc = JsonUtil.readValue(request.getExtFilter(), ExternalFilterDesc.class);
+ extFilterService.updateExternalFilter(desc);
+ extFilterService.syncExtFilterToProject(new String[] { desc.getName() }, request.getProject());
+ result.put("success", "true");
+ return result;
+ }
+
+ @RequestMapping(value = "/{filter}/{project}", method = { RequestMethod.DELETE })
+ @ResponseBody
+ public Map<String, String> removeFilter(@PathVariable String filter, @PathVariable String project) throws IOException {
+ Map<String, String> result = new HashMap<String, String>();
+ extFilterService.removeExtFilterFromProject(filter, project);
+ extFilterService.removeExternalFilter(filter);
+ result.put("success", "true");
+ return result;
+ }
+
+ @RequestMapping(value = "", method = { RequestMethod.GET })
+ @ResponseBody
+ public List<ExternalFilterDesc> getExternalFilters(@RequestParam(value = "project", required = true) String project) throws IOException {
+ List<ExternalFilterDesc> filterDescs = Lists.newArrayList();
+ filterDescs.addAll(extFilterService.getProjectManager().listExternalFilterDescs(project).values());
+ return filterDescs;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1a124e68/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
new file mode 100644
index 0000000..dce3847
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.controller;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TimeZone;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.ClassUtil;
+import org.apache.kylin.job.JobInstance;
+import org.apache.kylin.job.Scheduler;
+import org.apache.kylin.job.SchedulerFactory;
+import org.apache.kylin.job.constant.JobStatusEnum;
+import org.apache.kylin.job.constant.JobTimeFilterEnum;
+import org.apache.kylin.job.engine.JobEngineConfig;
+import org.apache.kylin.job.exception.SchedulerException;
+import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.lock.JobLock;
+import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.request.JobListRequest;
+import org.apache.kylin.rest.service.JobService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+/**
+ *
+ */
+@Controller
+@RequestMapping(value = "jobs")
+public class JobController extends BasicController implements InitializingBean {
+ private static final Logger logger = LoggerFactory.getLogger(JobController.class);
+
+ @Autowired
+ private JobService jobService;
+
+ private JobLock jobLock;
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.springframework.beans.factory.InitializingBean#afterPropertiesSet()
+ */
+ @SuppressWarnings("unchecked")
+ @Override
+ public void afterPropertiesSet() throws Exception {
+
+ String timeZone = jobService.getConfig().getTimeZone();
+ TimeZone tzone = TimeZone.getTimeZone(timeZone);
+ TimeZone.setDefault(tzone);
+
+ final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ final Scheduler<AbstractExecutable> scheduler = (Scheduler<AbstractExecutable>) SchedulerFactory.scheduler(kylinConfig.getSchedulerType());
+
+ jobLock = (JobLock) ClassUtil.newInstance(kylinConfig.getJobControllerLock());
+
+ new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ scheduler.init(new JobEngineConfig(kylinConfig), jobLock);
+ while (!scheduler.hasStarted()) {
+ logger.error("scheduler has not been started");
+ Thread.sleep(1000);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }).start();
+
+ Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ scheduler.shutdown();
+ } catch (SchedulerException e) {
+ logger.error("error occurred to shutdown scheduler", e);
+ }
+ }
+ }));
+ }
+
+ /**
+ * get all cube jobs
+ *
+ * @return
+ * @throws IOException
+ */
+ @RequestMapping(value = "", method = { RequestMethod.GET })
+ @ResponseBody
+ public List<JobInstance> list(JobListRequest jobRequest) {
+
+ List<JobInstance> jobInstanceList = Collections.emptyList();
+ List<JobStatusEnum> statusList = new ArrayList<JobStatusEnum>();
+
+ if (null != jobRequest.getStatus()) {
+ for (int status : jobRequest.getStatus()) {
+ statusList.add(JobStatusEnum.getByCode(status));
+ }
+ }
+
+ JobTimeFilterEnum timeFilter = JobTimeFilterEnum.getByCode(jobRequest.getTimeFilter());
+
+ try {
+ jobInstanceList = jobService.listAllJobs(jobRequest.getCubeName(), jobRequest.getProjectName(), statusList, jobRequest.getLimit(), jobRequest.getOffset(), timeFilter);
+ } catch (Exception e) {
+ logger.error(e.getLocalizedMessage(), e);
+ throw new InternalErrorException(e);
+ }
+ return jobInstanceList;
+ }
+
+ /**
+ * Get a cube job
+ *
+ * @return
+ * @throws IOException
+ */
+ @RequestMapping(value = "/{jobId}", method = { RequestMethod.GET })
+ @ResponseBody
+ public JobInstance get(@PathVariable String jobId) {
+ JobInstance jobInstance = null;
+ try {
+ jobInstance = jobService.getJobInstance(jobId);
+ } catch (Exception e) {
+ logger.error(e.getLocalizedMessage(), e);
+ throw new InternalErrorException(e);
+ }
+
+ return jobInstance;
+ }
+
+ /**
+ * Get a job step output
+ *
+ * @return
+ * @throws IOException
+ */
+ @RequestMapping(value = "/{jobId}/steps/{stepId}/output", method = { RequestMethod.GET })
+ @ResponseBody
+ public Map<String, String> getStepOutput(@PathVariable String jobId, @PathVariable String stepId) {
+ Map<String, String> result = new HashMap<String, String>();
+ result.put("jobId", jobId);
+ result.put("stepId", String.valueOf(stepId));
+ result.put("cmd_output", jobService.getExecutableManager().getOutput(stepId).getVerboseMsg());
+ return result;
+ }
+
+ /**
+ * Resume a cube job
+ *
+ * @return
+ * @throws IOException
+ */
+ @RequestMapping(value = "/{jobId}/resume", method = { RequestMethod.PUT })
+ @ResponseBody
+ public JobInstance resume(@PathVariable String jobId) {
+ try {
+ final JobInstance jobInstance = jobService.getJobInstance(jobId);
+ jobService.resumeJob(jobInstance);
+ return jobService.getJobInstance(jobId);
+ } catch (Exception e) {
+ logger.error(e.getLocalizedMessage(), e);
+ throw new InternalErrorException(e);
+ }
+ }
+
+ /**
+ * Cancel a job
+ *
+ * @return
+ * @throws IOException
+ */
+ @RequestMapping(value = "/{jobId}/cancel", method = { RequestMethod.PUT })
+ @ResponseBody
+ public JobInstance cancel(@PathVariable String jobId) {
+
+ try {
+ final JobInstance jobInstance = jobService.getJobInstance(jobId);
+ return jobService.cancelJob(jobInstance);
+ } catch (Exception e) {
+ logger.error(e.getLocalizedMessage(), e);
+ throw new InternalErrorException(e);
+ }
+
+ }
+
+ public void setJobService(JobService jobService) {
+ this.jobService = jobService;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1a124e68/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
new file mode 100644
index 0000000..5f6a91b
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/ModelController.java
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.controller;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.rest.exception.BadRequestException;
+import org.apache.kylin.rest.exception.ForbiddenException;
+import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.exception.NotFoundException;
+import org.apache.kylin.rest.request.ModelRequest;
+import org.apache.kylin.rest.service.ModelService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.security.access.AccessDeniedException;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonMappingException;
+
+/**
+ * ModelController is defined as Restful API entrance for UI.
+ *
+ * @author jiazhong
+ */
+@Controller
+@RequestMapping(value = "/models")
+public class ModelController extends BasicController {
+ private static final Logger logger = LoggerFactory.getLogger(ModelController.class);
+
+ @Autowired
+ private ModelService modelService;
+
+ @RequestMapping(value = "", method = { RequestMethod.GET })
+ @ResponseBody
+ public List<DataModelDesc> getModels(@RequestParam(value = "modelName", required = false) String modelName, @RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value = "limit", required = false) Integer limit, @RequestParam(value = "offset", required = false) Integer offset) {
+ try {
+ return modelService.getModels(modelName, projectName, limit, offset);
+ } catch (IOException e) {
+ logger.error("Failed to deal with the request:" + e.getLocalizedMessage(), e);
+ throw new InternalErrorException("Failed to deal with the request: " + e.getLocalizedMessage());
+ }
+ }
+
+ /**
+ *
+ * create model
+ * @throws java.io.IOException
+ */
+ @RequestMapping(value = "", method = { RequestMethod.POST })
+ @ResponseBody
+ public ModelRequest saveModelDesc(@RequestBody ModelRequest modelRequest) {
+ //Update Model
+ DataModelDesc modelDesc = deserializeDataModelDesc(modelRequest);
+ if (modelDesc == null || StringUtils.isEmpty(modelDesc.getName())) {
+ return modelRequest;
+ }
+
+ if (StringUtils.isEmpty(modelDesc.getName())) {
+ logger.info("Model name should not be empty.");
+ throw new BadRequestException("Model name should not be empty.");
+ }
+
+ try {
+ modelDesc.setUuid(UUID.randomUUID().toString());
+ String projectName = (null == modelRequest.getProject()) ? ProjectInstance.DEFAULT_PROJECT_NAME : modelRequest.getProject();
+
+ modelService.createModelDesc(projectName, modelDesc);
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ logger.error("Failed to deal with the request:" + e.getLocalizedMessage(), e);
+ throw new InternalErrorException("Failed to deal with the request: " + e.getLocalizedMessage());
+ }
+
+ modelRequest.setUuid(modelDesc.getUuid());
+ modelRequest.setSuccessful(true);
+ return modelRequest;
+ }
+
+ @RequestMapping(value = "", method = { RequestMethod.PUT })
+ @ResponseBody
+ public ModelRequest updateModelDesc(@RequestBody ModelRequest modelRequest) throws JsonProcessingException {
+ DataModelDesc modelDesc = deserializeDataModelDesc(modelRequest);
+ if (modelDesc == null) {
+ return modelRequest;
+ }
+ try {
+ modelDesc = modelService.updateModelAndDesc(modelDesc);
+ } catch (AccessDeniedException accessDeniedException) {
+ throw new ForbiddenException("You don't have right to update this model.");
+ } catch (Exception e) {
+ logger.error("Failed to deal with the request:" + e.getLocalizedMessage(), e);
+ throw new InternalErrorException("Failed to deal with the request: " + e.getLocalizedMessage());
+ }
+
+ if (modelDesc.getError().isEmpty()) {
+ modelRequest.setSuccessful(true);
+ } else {
+ logger.warn("Model " + modelDesc.getName() + " fail to update because " + modelDesc.getError());
+ updateRequest(modelRequest, false, omitMessage(modelDesc.getError()));
+ }
+ String descData = JsonUtil.writeValueAsIndentString(modelDesc);
+ modelRequest.setModelDescData(descData);
+ return modelRequest;
+ }
+
+ @RequestMapping(value = "/{modelName}", method = { RequestMethod.DELETE })
+ @ResponseBody
+ public void deleteModel(@PathVariable String modelName) {
+ DataModelDesc desc = modelService.getMetadataManager().getDataModelDesc(modelName);
+ if (null == desc) {
+ throw new NotFoundException("Data Model with name " + modelName + " not found..");
+ }
+ try {
+ modelService.dropModel(desc);
+ } catch (Exception e) {
+ logger.error(e.getLocalizedMessage(), e);
+ throw new InternalErrorException("Failed to delete model. " + " Caused by: " + e.getMessage(), e);
+ }
+ }
+
+ @RequestMapping(value = "/{modelName}/clone", method = { RequestMethod.PUT })
+ @ResponseBody
+ public ModelRequest cloneModel(@PathVariable String modelName, @RequestBody ModelRequest modelRequest) {
+ String project = modelRequest.getProject();
+ MetadataManager metaManager = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ DataModelDesc modelDesc = metaManager.getDataModelDesc(modelName);
+ String newModelName = modelRequest.getModelName();
+
+ if (StringUtils.isEmpty(project)) {
+ logger.info("Project name should not be empty.");
+ throw new BadRequestException("Project name should not be empty.");
+ }
+
+ if (modelDesc == null || StringUtils.isEmpty(modelName)) {
+ logger.info("Model does not exist.");
+ throw new BadRequestException("Model does not exist.");
+ }
+
+ if (StringUtils.isEmpty(newModelName)) {
+ logger.info("New model name is empty.");
+ throw new BadRequestException("New model name is empty.");
+ }
+
+ DataModelDesc newModelDesc = DataModelDesc.getCopyOf(modelDesc);
+ newModelDesc.setName(newModelName);
+ try {
+ newModelDesc = modelService.createModelDesc(project, newModelDesc);
+
+ //reload avoid shallow
+ metaManager.reloadDataModelDesc(newModelName);
+ } catch (IOException e) {
+ throw new InternalErrorException("failed to clone DataModelDesc", e);
+ }
+
+ modelRequest.setUuid(newModelDesc.getUuid());
+ modelRequest.setSuccessful(true);
+ return modelRequest;
+ }
+
+ private DataModelDesc deserializeDataModelDesc(ModelRequest modelRequest) {
+ DataModelDesc desc = null;
+ try {
+ logger.debug("Saving MODEL " + modelRequest.getModelDescData());
+ desc = JsonUtil.readValue(modelRequest.getModelDescData(), DataModelDesc.class);
+ } catch (JsonParseException e) {
+ logger.error("The data model definition is not valid.", e);
+ updateRequest(modelRequest, false, e.getMessage());
+ } catch (JsonMappingException e) {
+ logger.error("The data model definition is not valid.", e);
+ updateRequest(modelRequest, false, e.getMessage());
+ } catch (IOException e) {
+ logger.error("Failed to deal with the request.", e);
+ throw new InternalErrorException("Failed to deal with the request:" + e.getMessage(), e);
+ }
+ return desc;
+ }
+
+ private void updateRequest(ModelRequest request, boolean success, String message) {
+ request.setModelDescData("");
+ request.setSuccessful(success);
+ request.setMessage(message);
+ }
+
+ public void setModelService(ModelService modelService) {
+ this.modelService = modelService;
+ }
+
+ /**
+ * @param errors
+ * @return
+ */
+ private String omitMessage(List<String> errors) {
+ StringBuffer buffer = new StringBuffer();
+ for (Iterator<String> iterator = errors.iterator(); iterator.hasNext();) {
+ String string = (String) iterator.next();
+ buffer.append(string);
+ buffer.append("\n");
+ }
+ return buffer.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1a124e68/server-base/src/main/java/org/apache/kylin/rest/controller/ModelDescController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/ModelDescController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/ModelDescController.java
new file mode 100644
index 0000000..4171afd
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/ModelDescController.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.controller;
+
+import java.io.IOException;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+/**
+ * @author jiazhong
+ *
+ */
+@Controller
+@RequestMapping(value = "/model")
+public class ModelDescController extends BasicController {
+
+ /**
+ * Get detail information of the "Model ID"
+ *
+ * @param modelDescName
+ * Model ID
+ * @return
+ * @throws IOException
+ */
+ @RequestMapping(value = "/{model_name}", method = { RequestMethod.GET })
+ @ResponseBody
+ public DataModelDesc getModel(@PathVariable String model_name) {
+ MetadataManager metaManager = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
+ DataModelDesc modeDesc = metaManager.getDataModelDesc(model_name);
+ return modeDesc;
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1a124e68/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
new file mode 100644
index 0000000..f829fff
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/ProjectController.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.controller;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.persistence.AclEntity;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.rest.constant.Constant;
+import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.request.CreateProjectRequest;
+import org.apache.kylin.rest.request.UpdateProjectRequest;
+import org.apache.kylin.rest.service.AccessService;
+import org.apache.kylin.rest.service.CubeService;
+import org.apache.kylin.rest.service.ProjectService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.security.acls.domain.GrantedAuthoritySid;
+import org.springframework.security.acls.domain.PrincipalSid;
+import org.springframework.security.acls.model.AccessControlEntry;
+import org.springframework.security.acls.model.Acl;
+import org.springframework.security.core.Authentication;
+import org.springframework.security.core.GrantedAuthority;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.security.core.userdetails.UserDetails;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+/**
+ * @author xduo
+ */
+@Controller
+@RequestMapping(value = "/projects")
+public class ProjectController extends BasicController {
+ private static final Logger logger = LoggerFactory.getLogger(ProjectController.class);
+
+ @Autowired
+ private ProjectService projectService;
+ @Autowired
+ private AccessService accessService;
+ @Autowired
+ private CubeService cubeService;
+
+ /**
+ * Get available project list
+ *
+ * @return Table metadata array
+ * @throws IOException
+ */
+ @RequestMapping(value = "", method = { RequestMethod.GET })
+ @ResponseBody
+ public List<ProjectInstance> getProjects(@RequestParam(value = "limit", required = false) Integer limit, @RequestParam(value = "offset", required = false) Integer offset) {
+ return projectService.listAllProjects(limit, offset);
+ }
+
+ @RequestMapping(value = "/readable", method = { RequestMethod.GET })
+ @ResponseBody
+ public List<ProjectInstance> getReadableProjects(@RequestParam(value = "limit", required = false) Integer limit, @RequestParam(value = "offset", required = false) Integer offset) {
+ List<ProjectInstance> readableProjects = new ArrayList<ProjectInstance>();
+ //list all projects first
+ List<ProjectInstance> projectInstances = projectService.listAllProjects(limit, offset);
+
+ //get user infomation
+ Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
+ UserDetails userDetails = null;
+ if (authentication == null) {
+ logger.debug("authentication is null.");
+ throw new InternalErrorException("Can not find authentication infomation.");
+ }
+ if (authentication.getPrincipal() instanceof UserDetails) {
+ logger.debug("authentication.getPrincipal() is " + authentication.getPrincipal());
+ userDetails = (UserDetails) authentication.getPrincipal();
+ }
+ if (authentication.getDetails() instanceof UserDetails) {
+ logger.debug("authentication.getDetails() is " + authentication.getDetails());
+ userDetails = (UserDetails) authentication.getDetails();
+ }
+
+ //check if ROLE_ADMIN return all,also get user role list
+ List<String> userAuthority = new ArrayList<>();
+ for (GrantedAuthority auth : authentication.getAuthorities()) {
+ userAuthority.add(auth.getAuthority());
+ if (auth.getAuthority().equals(Constant.ROLE_ADMIN))
+ return projectInstances;
+ }
+ String userName = userDetails.getUsername();
+ for (ProjectInstance projectInstance : projectInstances) {
+ if (projectInstance == null) {
+ continue;
+ }
+
+ boolean hasProjectPermission = false;
+ AclEntity ae = accessService.getAclEntity("ProjectInstance", projectInstance.getId());
+ Acl projectAcl = accessService.getAcl(ae);
+ //project no Acl info will be skipped
+ if (projectAcl != null) {
+
+ //project owner has permission
+ if (((PrincipalSid) projectAcl.getOwner()).getPrincipal().equals(userName)) {
+ readableProjects.add(projectInstance);
+ continue;
+ }
+
+ //check project permission and role
+ for (AccessControlEntry ace : projectAcl.getEntries()) {
+ if (ace.getSid() instanceof PrincipalSid && ((PrincipalSid) ace.getSid()).getPrincipal().equals(userName)) {
+ hasProjectPermission = true;
+ readableProjects.add(projectInstance);
+ break;
+
+ } else if (ace.getSid() instanceof GrantedAuthoritySid) {
+ String projectAuthority = ((GrantedAuthoritySid) ace.getSid()).getGrantedAuthority();
+ if (userAuthority.contains(projectAuthority)) {
+ hasProjectPermission = true;
+ readableProjects.add(projectInstance);
+ break;
+ }
+
+ }
+
+ }
+ }
+
+ if (!hasProjectPermission) {
+ List<CubeInstance> cubeInstances = cubeService.listAllCubes(projectInstance.getName());
+
+ for (CubeInstance cubeInstance : cubeInstances) {
+ if (cubeInstance == null) {
+ continue;
+ }
+ boolean hasCubePermission = false;
+ AclEntity cubeAe = accessService.getAclEntity("CubeInstance", cubeInstance.getId());
+ Acl cubeAcl = accessService.getAcl(cubeAe);
+ //cube no Acl info will not be used to filter project
+ if (cubeAcl != null) {
+ //cube owner will have permission to read project
+ if (((PrincipalSid) cubeAcl.getOwner()).getPrincipal().equals(userName)) {
+ hasProjectPermission = true;
+ break;
+ }
+ for (AccessControlEntry cubeAce : cubeAcl.getEntries()) {
+
+ if (cubeAce.getSid() instanceof PrincipalSid && ((PrincipalSid) cubeAce.getSid()).getPrincipal().equals(userName)) {
+ hasCubePermission = true;
+ break;
+ } else if (cubeAce.getSid() instanceof GrantedAuthoritySid) {
+ String cubeAuthority = ((GrantedAuthoritySid) cubeAce.getSid()).getGrantedAuthority();
+ if (userAuthority.contains(cubeAuthority)) {
+ hasCubePermission = true;
+ break;
+ }
+
+ }
+ }
+ }
+ if (hasCubePermission) {
+ hasProjectPermission = true;
+ break;
+ }
+ }
+ if (hasProjectPermission) {
+ readableProjects.add(projectInstance);
+ }
+ }
+
+ }
+ return readableProjects;
+ }
+
+ @RequestMapping(value = "", method = { RequestMethod.POST })
+ @ResponseBody
+ public ProjectInstance saveProject(@RequestBody CreateProjectRequest projectRequest) {
+ if (StringUtils.isEmpty(projectRequest.getName())) {
+ throw new InternalErrorException("A project name must be given to create a project");
+ }
+
+ ProjectInstance createdProj = null;
+ try {
+ createdProj = projectService.createProject(projectRequest);
+ } catch (Exception e) {
+ logger.error("Failed to deal with the request.", e);
+ throw new InternalErrorException(e.getLocalizedMessage());
+ }
+
+ return createdProj;
+ }
+
+ @RequestMapping(value = "", method = { RequestMethod.PUT })
+ @ResponseBody
+ public ProjectInstance updateProject(@RequestBody UpdateProjectRequest projectRequest) {
+ if (StringUtils.isEmpty(projectRequest.getFormerProjectName())) {
+ throw new InternalErrorException("A project name must be given to update a project");
+ }
+
+ ProjectInstance updatedProj = null;
+ try {
+ ProjectInstance currentProject = projectService.getProjectManager().getProject(projectRequest.getFormerProjectName());
+ updatedProj = projectService.updateProject(projectRequest, currentProject);
+ } catch (Exception e) {
+ logger.error("Failed to deal with the request.", e);
+ throw new InternalErrorException(e.getLocalizedMessage());
+ }
+
+ return updatedProj;
+ }
+
+ @RequestMapping(value = "/{projectName}", method = { RequestMethod.DELETE })
+ @ResponseBody
+ public void deleteProject(@PathVariable String projectName) {
+ try {
+
+ ProjectInstance project = projectService.getProjectManager().getProject(projectName);
+ projectService.deleteProject(projectName, project);
+ } catch (Exception e) {
+ logger.error(e.getLocalizedMessage(), e);
+ throw new InternalErrorException("Failed to delete project. " + " Caused by: " + e.getMessage(), e);
+ }
+ }
+
+ public void setProjectService(ProjectService projectService) {
+ this.projectService = projectService;
+ }
+
+ public void setAccessService(AccessService accessService) {
+ this.accessService = accessService;
+ }
+
+ public void setCubeService(CubeService cubeService) {
+ this.cubeService = cubeService;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1a124e68/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
new file mode 100644
index 0000000..93b71ad
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.controller;
+
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.annotation.PostConstruct;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.debug.BackdoorToggles;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.rest.constant.Constant;
+import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.model.Query;
+import org.apache.kylin.rest.model.SelectedColumnMeta;
+import org.apache.kylin.rest.model.TableMeta;
+import org.apache.kylin.rest.request.MetaRequest;
+import org.apache.kylin.rest.request.PrepareSqlRequest;
+import org.apache.kylin.rest.request.SQLRequest;
+import org.apache.kylin.rest.request.SaveSqlRequest;
+import org.apache.kylin.rest.response.SQLResponse;
+import org.apache.kylin.rest.service.QueryService;
+import org.apache.kylin.rest.util.QueryUtil;
+import org.apache.kylin.storage.exception.ScanOutOfLimitException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.security.access.AccessDeniedException;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.supercsv.io.CsvListWriter;
+import org.supercsv.io.ICsvListWriter;
+import org.supercsv.prefs.CsvPreference;
+
+import com.google.common.base.Preconditions;
+
+import net.sf.ehcache.Cache;
+import net.sf.ehcache.CacheManager;
+import net.sf.ehcache.Element;
+
+/**
+ * Handle query requests.
+ *
+ * @author xduo
+ */
+@Controller
+public class QueryController extends BasicController {
+
+ private static final Logger logger = LoggerFactory.getLogger(QueryController.class);
+
+ public static final String SUCCESS_QUERY_CACHE = "StorageCache";
+ public static final String EXCEPTION_QUERY_CACHE = "ExceptionQueryCache";
+
+ @Autowired
+ private QueryService queryService;
+
+ @Autowired
+ private CacheManager cacheManager;
+
+ @PostConstruct
+ public void init() throws IOException {
+ Preconditions.checkNotNull(cacheManager, "cacheManager is not injected yet");
+ }
+
+ @RequestMapping(value = "/query", method = RequestMethod.POST)
+ @ResponseBody
+ public SQLResponse query(@RequestBody SQLRequest sqlRequest) {
+ return doQueryWithCache(sqlRequest);
+ }
+
+ // TODO should be just "prepare" a statement, get back expected ResultSetMetaData
+ @RequestMapping(value = "/query/prestate", method = RequestMethod.POST, produces = "application/json")
+ @ResponseBody
+ public SQLResponse prepareQuery(@RequestBody PrepareSqlRequest sqlRequest) {
+ return doQueryWithCache(sqlRequest);
+ }
+
+ @RequestMapping(value = "/saved_queries", method = RequestMethod.POST)
+ @ResponseBody
+ public void saveQuery(@RequestBody SaveSqlRequest sqlRequest) throws IOException {
+ String creator = SecurityContextHolder.getContext().getAuthentication().getName();
+ Query newQuery = new Query(sqlRequest.getName(), sqlRequest.getProject(), sqlRequest.getSql(), sqlRequest.getDescription());
+
+ queryService.saveQuery(creator, newQuery);
+ }
+
+ @RequestMapping(value = "/saved_queries/{id}", method = RequestMethod.DELETE)
+ @ResponseBody
+ public void removeQuery(@PathVariable String id) throws IOException {
+ String creator = SecurityContextHolder.getContext().getAuthentication().getName();
+ queryService.removeQuery(creator, id);
+ }
+
+ @RequestMapping(value = "/saved_queries", method = RequestMethod.GET)
+ @ResponseBody
+ public List<Query> getQueries() throws IOException {
+ String creator = SecurityContextHolder.getContext().getAuthentication().getName();
+ return queryService.getQueries(creator);
+ }
+
+ @RequestMapping(value = "/query/format/{format}", method = RequestMethod.GET)
+ @ResponseBody
+ public void downloadQueryResult(@PathVariable String format, SQLRequest sqlRequest, HttpServletResponse response) {
+ SQLResponse result = doQueryWithCache(sqlRequest);
+ response.setContentType("text/" + format + ";charset=utf-8");
+ response.setHeader("Content-Disposition", "attachment; filename=\"result." + format + "\"");
+ ICsvListWriter csvWriter = null;
+
+ try {
+ csvWriter = new CsvListWriter(response.getWriter(), CsvPreference.STANDARD_PREFERENCE);
+
+ List<String> headerList = new ArrayList<String>();
+
+ for (SelectedColumnMeta column : result.getColumnMetas()) {
+ headerList.add(column.getName());
+ }
+
+ String[] headers = new String[headerList.size()];
+ csvWriter.writeHeader(headerList.toArray(headers));
+
+ for (List<String> row : result.getResults()) {
+ csvWriter.write(row);
+ }
+ } catch (IOException e) {
+ logger.error("", e);
+ } finally {
+ IOUtils.closeQuietly(csvWriter);
+ }
+ }
+
+ @RequestMapping(value = "/tables_and_columns", method = RequestMethod.GET)
+ @ResponseBody
+ public List<TableMeta> getMetadata(MetaRequest metaRequest) {
+ try {
+ return queryService.getMetadata(metaRequest.getProject());
+ } catch (SQLException e) {
+ logger.error(e.getLocalizedMessage(), e);
+ throw new InternalErrorException(e.getLocalizedMessage(), e);
+ }
+ }
+
+ private SQLResponse doQueryWithCache(SQLRequest sqlRequest) {
+ try {
+ BackdoorToggles.setToggles(sqlRequest.getBackdoorToggles());
+
+ String sql = sqlRequest.getSql();
+ String project = sqlRequest.getProject();
+ logger.info("Using project: " + project);
+ logger.info("The original query: " + sql);
+
+ String serverMode = KylinConfig.getInstanceFromEnv().getServerMode();
+ if (!(Constant.SERVER_MODE_QUERY.equals(serverMode.toLowerCase()) || Constant.SERVER_MODE_ALL.equals(serverMode.toLowerCase()))) {
+ throw new InternalErrorException("Query is not allowed in " + serverMode + " mode.");
+ }
+
+ if (!sql.toLowerCase().contains("select")) {
+ logger.debug("Directly return exception as not supported");
+ throw new InternalErrorException("Not Supported SQL.");
+ }
+
+ long startTime = System.currentTimeMillis();
+
+ SQLResponse sqlResponse = searchQueryInCache(sqlRequest);
+ try {
+ if (null == sqlResponse) {
+ sqlResponse = queryService.query(sqlRequest);
+
+ long durationThreshold = KylinConfig.getInstanceFromEnv().getQueryDurationCacheThreshold();
+ long scancountThreshold = KylinConfig.getInstanceFromEnv().getQueryScanCountCacheThreshold();
+ sqlResponse.setDuration(System.currentTimeMillis() - startTime);
+ logger.info("Stats of SQL response: isException: {}, duration: {}, total scan count {}", //
+ new String[] { String.valueOf(sqlResponse.getIsException()), String.valueOf(sqlResponse.getDuration()), String.valueOf(sqlResponse.getTotalScanCount()) });
+ if (!sqlResponse.getIsException() && (sqlResponse.getDuration() > durationThreshold || sqlResponse.getTotalScanCount() > scancountThreshold)) {
+ cacheManager.getCache(SUCCESS_QUERY_CACHE).put(new Element(sqlRequest, sqlResponse));
+ }
+ } else {
+ sqlResponse.setDuration(System.currentTimeMillis() - startTime);
+ }
+
+ checkQueryAuth(sqlResponse);
+
+ } catch (Throwable e) { // calcite may throw AssertError
+ logger.error("Exception when execute sql", e);
+ String errMsg = QueryUtil.makeErrorMsgUserFriendly(e);
+
+ sqlResponse = new SQLResponse(null, null, 0, true, errMsg);
+
+ // for exception queries, only cache ScanOutOfLimitException
+ if (e instanceof ScanOutOfLimitException) {
+ Cache exceptionCache = cacheManager.getCache(EXCEPTION_QUERY_CACHE);
+ exceptionCache.put(new Element(sqlRequest, sqlResponse));
+ }
+ }
+
+ queryService.logQuery(sqlRequest, sqlResponse);
+
+ if (sqlResponse.getIsException())
+ throw new InternalErrorException(sqlResponse.getExceptionMessage());
+
+ return sqlResponse;
+
+ } finally {
+ BackdoorToggles.cleanToggles();
+ }
+ }
+
+ private SQLResponse searchQueryInCache(SQLRequest sqlRequest) {
+ SQLResponse response = null;
+ Cache exceptionCache = cacheManager.getCache(EXCEPTION_QUERY_CACHE);
+ Cache successCache = cacheManager.getCache(SUCCESS_QUERY_CACHE);
+
+ if (KylinConfig.getInstanceFromEnv().isQueryCacheEnabled() && !BackdoorToggles.getDisableCache()) {
+ if (exceptionCache.get(sqlRequest) != null) {
+ logger.info("The sqlResponse is found in EXCEPTION_QUERY_CACHE");
+ Element element = exceptionCache.get(sqlRequest);
+ response = (SQLResponse) element.getObjectValue();
+ response.setHitExceptionCache(true);
+ } else if (successCache.get(sqlRequest) != null) {
+ logger.info("The sqlResponse is found in SUCCESS_QUERY_CACHE");
+ Element element = successCache.get(sqlRequest);
+ response = (SQLResponse) element.getObjectValue();
+ response.setStorageCacheUsed(true);
+ }
+ }
+
+ return response;
+ }
+
+ private void checkQueryAuth(SQLResponse sqlResponse) throws AccessDeniedException {
+ if (!sqlResponse.getIsException() && KylinConfig.getInstanceFromEnv().isQuerySecureEnabled()) {
+ CubeInstance cubeInstance = this.queryService.getCubeManager().getCube(sqlResponse.getCube());
+ queryService.checkAuthorization(cubeInstance);
+ }
+ }
+
+ public void setQueryService(QueryService queryService) {
+ this.queryService = queryService;
+ }
+
+ public void setCacheManager(CacheManager cacheManager) {
+ this.cacheManager = cacheManager;
+ }
+
+}