You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/04/01 09:48:24 UTC
kylin git commit: put ext filter into project L2 cache
Repository: kylin
Updated Branches:
refs/heads/master c86de1a8e -> 0e662a350
put ext filter into project L2 cache
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0e662a35
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0e662a35
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0e662a35
Branch: refs/heads/master
Commit: 0e662a350b88ad4b1194874af19f25590286a52a
Parents: c86de1a
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Apr 1 15:47:50 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Apr 1 15:47:50 2016 +0800
----------------------------------------------------------------------
.../metadata/filter/UDF/MassInTupleFilter.java | 1 +
.../kylin/metadata/project/ProjectL2Cache.java | 37 +++++++++++++++-----
.../kylin/metadata/project/ProjectManager.java | 9 +++++
.../controller/ExternalFilterController.java | 37 ++++++++++----------
.../kylin/rest/service/ExtFilterService.java | 21 -----------
5 files changed, 58 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/0e662a35/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
index 0cb416c..5889c61 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
@@ -32,6 +32,7 @@ import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.filter.function.Functions;
import org.apache.kylin.metadata.model.ExternalFilterDesc;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/kylin/blob/0e662a35/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
index e8004e1..3006773 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
@@ -25,6 +25,7 @@ import java.util.Set;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.ExternalFilterDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TableDesc;
@@ -59,6 +60,16 @@ class ProjectL2Cache {
projectCaches.clear();
}
+ public ExternalFilterDesc getExternalFilterDesc(String project, String extFilterName) {
+ ProjectCache prjCache = getCache(project);
+ return prjCache.extFilters.get(extFilterName);
+ }
+
+ public Map<String, ExternalFilterDesc> listExternalFilterDesc(String project) {
+ ProjectCache prjCache = getCache(project);
+ return Collections.unmodifiableMap(prjCache.extFilters);
+ }
+
public List<TableDesc> listDefinedTables(String project) {
ProjectCache prjCache = getCache(project);
List<TableDesc> result = Lists.newArrayListWithCapacity(prjCache.tables.size());
@@ -159,7 +170,7 @@ class ProjectL2Cache {
private ProjectCache loadCache(String project) {
logger.info("Loading L2 project cache for " + project);
- ProjectCache result = new ProjectCache(project);
+ ProjectCache projectCache = new ProjectCache(project);
ProjectInstance pi = mgr.getProject(project);
@@ -171,30 +182,39 @@ class ProjectL2Cache {
for (String tableName : pi.getTables()) {
TableDesc tableDesc = metaMgr.getTableDesc(tableName);
if (tableDesc != null) {
- result.tables.put(tableDesc.getIdentity(), new TableCache(tableDesc));
+ projectCache.tables.put(tableDesc.getIdentity(), new TableCache(tableDesc));
} else {
logger.warn("Table '" + tableName + "' defined under project '" + project + "' is not found");
}
}
+ for (String extFilterName : pi.getExtFilters()) {
+ ExternalFilterDesc filterDesc = metaMgr.getExtFilterDesc(extFilterName);
+ if (filterDesc != null) {
+ projectCache.extFilters.put(extFilterName, filterDesc);
+ } else {
+ logger.warn("External Filter '" + extFilterName + "' defined under project '" + project + "' is not found");
+ }
+ }
+
RealizationRegistry registry = RealizationRegistry.getInstance(mgr.getConfig());
for (RealizationEntry entry : pi.getRealizationEntries()) {
IRealization realization = registry.getRealization(entry.getType(), entry.getRealization());
if (realization != null) {
- result.realizations.add(realization);
+ projectCache.realizations.add(realization);
} else {
logger.warn("Realization '" + entry + "' defined under project '" + project + "' is not found");
}
}
- for (IRealization realization : result.realizations) {
- if (sanityCheck(result, realization)) {
- mapTableToRealization(result, realization);
- markExposedTablesAndColumns(result, realization);
+ for (IRealization realization : projectCache.realizations) {
+ if (sanityCheck(projectCache, realization)) {
+ mapTableToRealization(projectCache, realization);
+ markExposedTablesAndColumns(projectCache, realization);
}
}
- return result;
+ return projectCache;
}
// check all columns reported by realization does exists
@@ -257,6 +277,7 @@ class ProjectL2Cache {
private Map<String, TableCache> tables = Maps.newHashMap();
private Set<TableDesc> exposedTables = Sets.newHashSet();
private Set<IRealization> realizations = Sets.newHashSet();
+ private Map<String, ExternalFilterDesc> extFilters = Maps.newHashMap();
ProjectCache(String project) {
this.project = project;
http://git-wip-us.apache.org/repos/asf/kylin/blob/0e662a35/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
index 65b3719..b42b7e2 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
@@ -21,6 +21,7 @@ package org.apache.kylin.metadata.project;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@@ -352,6 +353,14 @@ public class ProjectManager {
return projects;
}
+ public ExternalFilterDesc getExternalFilterDesc(String project, String extFilter) {
+ return l2Cache.getExternalFilterDesc(project, extFilter);
+ }
+
+ public Map<String, ExternalFilterDesc> listExternalFilterDescs(String project) {
+ return l2Cache.listExternalFilterDesc(project);
+ }
+
public List<TableDesc> listDefinedTables(String project) throws IOException {
return l2Cache.listDefinedTables(norm(project));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0e662a35/server/src/main/java/org/apache/kylin/rest/controller/ExternalFilterController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/ExternalFilterController.java b/server/src/main/java/org/apache/kylin/rest/controller/ExternalFilterController.java
index b0b7569..5801eb5 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/ExternalFilterController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/ExternalFilterController.java
@@ -18,6 +18,12 @@
package org.apache.kylin.rest.controller;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.metadata.model.ExternalFilterDesc;
import org.apache.kylin.rest.request.ExternalFilterRequest;
@@ -26,13 +32,14 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
-import org.springframework.web.bind.annotation.*;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.ResponseBody;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
+import com.google.common.collect.Lists;
/**
* @author jiazhong
@@ -45,8 +52,6 @@ public class ExternalFilterController extends BasicController {
@Autowired
private ExtFilterService extFilterService;
-
-
@RequestMapping(value = "/saveExtFilter", method = { RequestMethod.POST })
@ResponseBody
public Map<String, String> saveExternalFilter(@RequestBody ExternalFilterRequest request) throws IOException {
@@ -55,8 +60,8 @@ public class ExternalFilterController extends BasicController {
ExternalFilterDesc desc = JsonUtil.readValue(request.getExtFilter(), ExternalFilterDesc.class);
desc.setUuid(UUID.randomUUID().toString());
extFilterService.saveExternalFilter(desc);
- extFilterService.syncExtFilterToProject(new String[]{desc.getName()},filterProject);
- result.put("success","true");
+ extFilterService.syncExtFilterToProject(new String[] { desc.getName() }, filterProject);
+ result.put("success", "true");
return result;
}
@@ -66,7 +71,7 @@ public class ExternalFilterController extends BasicController {
Map<String, String> result = new HashMap();
ExternalFilterDesc desc = JsonUtil.readValue(request.getExtFilter(), ExternalFilterDesc.class);
extFilterService.updateExternalFilter(desc);
- result.put("success","true");
+ result.put("success", "true");
return result;
}
@@ -74,13 +79,12 @@ public class ExternalFilterController extends BasicController {
@ResponseBody
public Map<String, String> unLoadHiveTables(@PathVariable String filter, @PathVariable String project) throws IOException {
Map<String, String> result = new HashMap<String, String>();
- extFilterService.removeExtFilterFromProject(filter,project);
+ extFilterService.removeExtFilterFromProject(filter, project);
extFilterService.removeExternalFilter(filter);
result.put("success", "true");
return result;
}
-
/**
* Get available table list of the input database
*
@@ -90,12 +94,9 @@ public class ExternalFilterController extends BasicController {
@RequestMapping(value = "", method = { RequestMethod.GET })
@ResponseBody
public List<ExternalFilterDesc> getExternalFilters(@RequestParam(value = "project", required = true) String project) throws IOException {
- List<ExternalFilterDesc> filterDescs;
- filterDescs = extFilterService.listProjectFilters(project);
+ List<ExternalFilterDesc> filterDescs = Lists.newArrayList();
+ filterDescs.addAll(extFilterService.getProjectManager().listExternalFilterDescs(project).values());
return filterDescs;
}
-
-
-
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/0e662a35/server/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java b/server/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java
index f41996d..6577a12 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java
@@ -33,11 +33,6 @@ import org.springframework.stereotype.Component;
import java.io.IOException;
import java.util.List;
-/**
- * Stateless & lightweight service facade of cube management functions.
- *
- * @author jiazhong
- */
@Component("extFilterService")
public class ExtFilterService extends BasicService {
private static final Logger logger = LoggerFactory.getLogger(ExtFilterService.class);
@@ -47,22 +42,6 @@ public class ExtFilterService extends BasicService {
private AccessService accessService;
-
- public List<ExternalFilterDesc> listProjectFilters(String project) throws IOException {
- List<ExternalFilterDesc> filterDescs = Lists.newArrayList();
- if(project == null){
- return filterDescs;
- }
- ProjectInstance projectInstance = getProjectManager().getProject(project);
- for(String filter:projectInstance.getExtFilters()){
- ExternalFilterDesc filterDesc = getMetadataManager().getExtFilterDesc(filter);
- if(filterDesc!=null){
- filterDescs.add(filterDesc);
- }
- }
- return filterDescs;
- }
-
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
public void saveExternalFilter(ExternalFilterDesc desc) throws IOException {
if(getMetadataManager().getExtFilterDesc(desc.getName()) != null){