You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2018/02/09 13:24:20 UTC
[1/3] kylin git commit: KYLIN-2672 Only clean necessary cache for
CubeMigrationCLI
Repository: kylin
Updated Branches:
refs/heads/master 139c303d4 -> 509906641
KYLIN-2672 Only clean necessary cache for CubeMigrationCLI
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/65c2315d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/65c2315d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/65c2315d
Branch: refs/heads/master
Commit: 65c2315d4e5396b50294c9bbe03f3a0307c92904
Parents: 139c303
Author: kangkaisen <ka...@meituan.com>
Authored: Mon Dec 25 18:33:31 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Fri Feb 9 21:19:09 2018 +0800
----------------------------------------------------------------------
.../kylin/common/restclient/RestClient.java | 20 +++++++
.../java/org/apache/kylin/cube/CubeManager.java | 2 +-
.../kylin/metadata/TableMetadataManager.java | 12 ++++
.../apache/kylin/metadata/model/TableDesc.java | 2 +-
.../kylin/metadata/project/ProjectL2Cache.java | 4 ++
.../kylin/metadata/project/ProjectManager.java | 6 +-
.../kylin/rest/controller/CacheController.java | 10 ++++
.../rest/request/CubeMigrationRequest.java | 62 ++++++++++++++++++++
.../apache/kylin/rest/service/CacheService.java | 34 ++++++++++-
.../org/apache/kylin/tool/CubeMigrationCLI.java | 28 ++++++---
10 files changed, 168 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/65c2315d/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
index 02045ae..e1cc13c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
+++ b/core-common/src/main/java/org/apache/kylin/common/restclient/RestClient.java
@@ -274,6 +274,26 @@ public class RestClient {
return response;
}
+ public void clearCacheForCubeMigration(String cube, String project, String model, Map<String, String> tableToProjects) throws IOException{
+ String url = baseUrl + "/cache/migration";
+ HttpPost post = new HttpPost(url);
+
+ post.addHeader("Accept", "application/json, text/plain, */*");
+ post.addHeader("Content-Type", "application/json");
+
+ HashMap<String, Object> paraMap = new HashMap<String, Object>();
+ paraMap.put("cube", cube);
+ paraMap.put("project", project);
+ paraMap.put("model", model);
+ paraMap.put("tableToProjects", tableToProjects);
+ String jsonMsg = JsonUtil.writeValueAsString(paraMap);
+ post.setEntity(new StringEntity(jsonMsg, "UTF-8"));
+ HttpResponse response = client.execute(post);
+ if (response.getStatusLine().getStatusCode() != 200) {
+ throw new IOException("Invalid response " + response.getStatusLine().getStatusCode());
+ }
+ }
+
private HashMap dealResponse(HttpResponse response) throws IOException {
if (response.getStatusLine().getStatusCode() != 200) {
throw new IOException("Invalid response " + response.getStatusLine().getStatusCode());
http://git-wip-us.apache.org/repos/asf/kylin/blob/65c2315d/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 8bdb5aa..9c52e8b 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -386,7 +386,7 @@ public class CubeManager implements IRealizationProvider {
}
// for internal
- CubeInstance reloadCubeQuietly(String cubeName) {
+ public CubeInstance reloadCubeQuietly(String cubeName) {
try (AutoLock lock = cubeMapLock.lockForWrite()) {
CubeInstance cube = crud.reloadQuietly(cubeName);
if (cube != null)
http://git-wip-us.apache.org/repos/asf/kylin/blob/65c2315d/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java
index f09c47c..2308df4 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java
@@ -151,6 +151,12 @@ public class TableMetadataManager {
}
}
+ public void reloadSourceTable(String table, String project) {
+ try (AutoLock lock = srcTableMapLock.lockForWrite()) {
+ srcTableCrud.reloadQuietly(TableDesc.concatResourcePath(table, project));
+ }
+ }
+
public List<TableDesc> listAllTables(String prj) {
try (AutoLock lock = srcTableMapLock.lockForWrite()) {
return Lists.newArrayList(getAllTablesMap(prj).values());
@@ -314,6 +320,12 @@ public class TableMetadataManager {
}
}
+ public void reloadTableExt(String table, String project) {
+ try (AutoLock lock = srcExtMapLock.lockForWrite()) {
+ srcExtCrud.reloadQuietly(TableExtDesc.concatResourcePath(table, project));
+ }
+ }
+
/**
* Get table extended info. Keys are defined in {@link MetadataConstants}
*
http://git-wip-us.apache.org/repos/asf/kylin/blob/65c2315d/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index 68bc5e9..be278de 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -57,7 +57,7 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
// this method should only used for getting dest path when copying from src to dest.
// if you want to get table's src path, use getResourcePath() instead.
- private static String concatResourcePath(String tableIdentity, String prj) {
+ public static String concatResourcePath(String tableIdentity, String prj) {
return concatRawResourcePath(makeResourceName(tableIdentity, prj));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/65c2315d/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
index 70b6a12..6e09ae8 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
@@ -187,6 +187,10 @@ class ProjectL2Cache {
return result;
}
+ public void reloadCacheByProject(String project) {
+ projectCaches.put(project, loadCache(project));
+ }
+
private ProjectCache loadCache(String project) {
logger.debug("Loading L2 project cache for " + project);
ProjectCache projectCache = new ProjectCache(project);
http://git-wip-us.apache.org/repos/asf/kylin/blob/65c2315d/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
index 0dd364d..aae692d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
@@ -117,10 +117,14 @@ public class ProjectManager {
l2Cache.clear();
}
+ public void reloadProjectL2Cache(String project) {
+ l2Cache.reloadCacheByProject(project);
+ }
+
public ProjectInstance reloadProjectQuietly(String project) throws IOException {
try (AutoLock lock = prjMapLock.lockForWrite()) {
ProjectInstance prj = crud.reloadQuietly(project);
- clearL2Cache();
+ reloadProjectL2Cache(project);
return prj;
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/65c2315d/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java
index 992094b..08b7cc4 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.metadata.cachesync.Broadcaster;
+import org.apache.kylin.rest.request.CubeMigrationRequest;
import org.apache.kylin.rest.service.CacheService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -29,6 +30,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;
@@ -73,6 +75,14 @@ public class CacheController extends BasicController {
cacheService.notifyMetadataChange(Broadcaster.SYNC_ALL, Broadcaster.Event.UPDATE, Broadcaster.SYNC_ALL);
}
+ @RequestMapping(value = "/migration", method = RequestMethod.POST)
+ @ResponseBody
+ public void clearCacheForCubeMigration(@RequestBody CubeMigrationRequest request) throws IOException {
+ cacheService.clearCacheForCubeMigration(request.getCube(), request.getProject(), request.getModel(), request.getTableToProjects());
+
+ cacheService.cleanDataCache(request.getProject());
+ }
+
public void setCacheService(CacheService cacheService) {
this.cacheService = cacheService;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/65c2315d/server-base/src/main/java/org/apache/kylin/rest/request/CubeMigrationRequest.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/request/CubeMigrationRequest.java b/server-base/src/main/java/org/apache/kylin/rest/request/CubeMigrationRequest.java
new file mode 100644
index 0000000..175fb59
--- /dev/null
+++ b/server-base/src/main/java/org/apache/kylin/rest/request/CubeMigrationRequest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.rest.request;
+
+import java.util.Map;
+
+public class CubeMigrationRequest {
+ private String cube;
+ private String model;
+ private String project;
+
+ private Map<String, String> tableToProjects;//For KYLIN-2717 compatibility, the project of old table will be NULL
+
+ public String getProject() {
+ return project;
+ }
+
+ public void setProject(String project) {
+ this.project = project;
+ }
+
+ public String getCube() {
+ return cube;
+ }
+
+ public void setCube(String cube) {
+ this.cube = cube;
+ }
+
+ public String getModel() {
+ return model;
+ }
+
+ public void setModel(String model) {
+ this.model = model;
+ }
+
+ public Map<String, String> getTableToProjects() {
+ return tableToProjects;
+ }
+
+ public void setTableToProjects(Map<String, String> tableToProjects) {
+ this.tableToProjects = tableToProjects;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/65c2315d/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
index b61309e..a8771ed 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -20,6 +20,7 @@ package org.apache.kylin.rest.service;
import java.io.IOException;
+import java.util.Map;
import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
import org.apache.kylin.storage.hbase.HBaseConnection;
@@ -114,7 +115,7 @@ public class CacheService extends BasicService implements InitializingBean {
broadcaster.notifyListener(entity, event, cacheKey);
}
- protected void cleanDataCache(String project) {
+ public void cleanDataCache(String project) {
if (cacheManager != null) {
logger.info("cleaning cache for project " + project + " (currently remove all entries)");
cacheManager.getCache(QueryService.SUCCESS_QUERY_CACHE).removeAll();
@@ -133,4 +134,35 @@ public class CacheService extends BasicService implements InitializingBean {
}
}
+ public void clearCacheForCubeMigration(String cube, String project, String model, Map<String, String> tableToProjects) throws IOException {
+ //the metadata reloading must be in order
+
+ //table must before model
+ for (Map.Entry<String, String> entry : tableToProjects.entrySet()) {
+ //For KYLIN-2717 compatibility, use tableProject not project
+ getTableManager().reloadSourceTable(entry.getKey(), entry.getValue());
+ getTableManager().reloadTableExt(entry.getKey(), entry.getValue());
+ }
+ logger.info("reload table cache done");
+
+ //ProjectInstance cache must before cube and model cache, as the new cubeDesc init and model reloading relays on latest ProjectInstance cache
+ getProjectManager().reloadProjectQuietly(project);
+ logger.info("reload project cache done");
+
+ //model must before cube desc
+ getDataModelManager().reloadDataModel(model);
+ logger.info("reload model cache done");
+
+ //cube desc must before cube instance
+ getCubeDescManager().reloadCubeDescLocal(cube);
+ logger.info("reload cubeDesc cache done");
+
+ getCubeManager().reloadCubeQuietly(cube);
+ logger.info("reload cube cache done");
+
+ //reload project l2cache again after cube cache, because the project L2 cache relay on latest cube cache
+ getProjectManager().reloadProjectL2Cache(project);
+ logger.info("reload project l2cache done");
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/65c2315d/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
index a4a6ab9..5426b62 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
@@ -54,7 +54,6 @@ import org.apache.kylin.dict.lookup.SnapshotManager;
import org.apache.kylin.dict.lookup.SnapshotTable;
import org.apache.kylin.engine.mr.JobBuilderSupport;
import org.apache.kylin.metadata.MetadataConstants;
-import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
@@ -129,8 +128,16 @@ public class CubeMigrationCLI extends AbstractApplication {
throws IOException, InterruptedException {
moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), KylinConfig.createInstanceFromUri(dstCfgUri), cubeName,
- projectName, Boolean.parseBoolean(copyAcl), Boolean.parseBoolean(purgeAndDisable),
- Boolean.parseBoolean(overwriteIfExists), Boolean.parseBoolean(realExecute), true);
+ projectName, copyAcl, purgeAndDisable, overwriteIfExists, realExecute);
+ }
+
+ public void moveCube(KylinConfig srcCfg, KylinConfig dstCfg, String cubeName, String projectName, String copyAcl,
+ String purgeAndDisable, String overwriteIfExists, String realExecute)
+ throws IOException, InterruptedException {
+
+ moveCube(srcCfg, dstCfg, cubeName, projectName, Boolean.parseBoolean(copyAcl),
+ Boolean.parseBoolean(purgeAndDisable), Boolean.parseBoolean(overwriteIfExists),
+ Boolean.parseBoolean(realExecute), true);
}
public void moveCube(String srcCfgUri, String dstCfgUri, String cubeName, String projectName, String copyAcl,
@@ -187,12 +194,12 @@ public class CubeMigrationCLI extends AbstractApplication {
if (migrateSegment) {
checkMigrationSuccess(dstConfig, cubeName, true);
}
- updateMeta(dstConfig);
+ updateMeta(dstConfig, projectName, cubeName, cube.getModel());
} else {
showOpts();
}
}
-
+
public void checkMigrationSuccess(KylinConfig kylinConfig, String cubeName, Boolean ifFix) throws IOException {
CubeMigrationCheckCLI checkCLI = new CubeMigrationCheckCLI(kylinConfig, ifFix);
checkCLI.execute(cubeName);
@@ -619,7 +626,7 @@ public class CubeMigrationCLI extends AbstractApplication {
}
}
}
-
+
private String renameTableWithinProject(String srcItem) {
if (dstProject != null && srcItem.contains(ResourceStore.TABLE_RESOURCE_ROOT)) {
String tableIdentity = TableDesc.parseResourcePath(srcItem).getFirst();
@@ -631,13 +638,18 @@ public class CubeMigrationCLI extends AbstractApplication {
return srcItem;
}
- private void updateMeta(KylinConfig config) {
+ private void updateMeta(KylinConfig config, String projectName, String cubeName, DataModelDesc model) {
String[] nodes = config.getRestServers();
+ Map<String, String> tableToProjects = new HashMap<>();
+ for (TableRef tableRef : model.getAllTables()) {
+ tableToProjects.put(tableRef.getTableIdentity(), tableRef.getTableDesc().getProject());
+ }
+
for (String node : nodes) {
RestClient restClient = new RestClient(node);
try {
logger.info("update meta cache for " + node);
- restClient.wipeCache(Broadcaster.SYNC_ALL, Broadcaster.Event.UPDATE.getType(), Broadcaster.SYNC_ALL);
+ restClient.clearCacheForCubeMigration(cubeName, projectName, model.getName(), tableToProjects);
} catch (IOException e) {
logger.error(e.getMessage());
}
[2/3] kylin git commit: KYLIN-2672 minor code review
Posted by bi...@apache.org.
KYLIN-2672 minor code review
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5cecbce2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5cecbce2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5cecbce2
Branch: refs/heads/master
Commit: 5cecbce26301a40bd436132efe1bc611ed86779a
Parents: 65c2315
Author: Li Yang <li...@apache.org>
Authored: Mon Feb 5 18:40:48 2018 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Fri Feb 9 21:19:52 2018 +0800
----------------------------------------------------------------------
core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java | 1 -
.../java/org/apache/kylin/metadata/TableMetadataManager.java | 4 ++--
.../java/org/apache/kylin/metadata/project/ProjectL2Cache.java | 2 +-
.../main/java/org/apache/kylin/rest/service/CacheService.java | 4 ++--
4 files changed, 5 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cecbce2/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 9c52e8b..dc370e2 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -385,7 +385,6 @@ public class CubeManager implements IRealizationProvider {
}
}
- // for internal
public CubeInstance reloadCubeQuietly(String cubeName) {
try (AutoLock lock = cubeMapLock.lockForWrite()) {
CubeInstance cube = crud.reloadQuietly(cubeName);
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cecbce2/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java
index 2308df4..42233b7 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/TableMetadataManager.java
@@ -151,7 +151,7 @@ public class TableMetadataManager {
}
}
- public void reloadSourceTable(String table, String project) {
+ public void reloadSourceTableQuietly(String table, String project) {
try (AutoLock lock = srcTableMapLock.lockForWrite()) {
srcTableCrud.reloadQuietly(TableDesc.concatResourcePath(table, project));
}
@@ -320,7 +320,7 @@ public class TableMetadataManager {
}
}
- public void reloadTableExt(String table, String project) {
+ public void reloadTableExtQuietly(String table, String project) {
try (AutoLock lock = srcExtMapLock.lockForWrite()) {
srcExtCrud.reloadQuietly(TableExtDesc.concatResourcePath(table, project));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cecbce2/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
index 6e09ae8..1663c8d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
@@ -187,7 +187,7 @@ class ProjectL2Cache {
return result;
}
- public void reloadCacheByProject(String project) {
+ void reloadCacheByProject(String project) {
projectCaches.put(project, loadCache(project));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/5cecbce2/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
index a8771ed..10ab90b 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -140,8 +140,8 @@ public class CacheService extends BasicService implements InitializingBean {
//table must before model
for (Map.Entry<String, String> entry : tableToProjects.entrySet()) {
//For KYLIN-2717 compatibility, use tableProject not project
- getTableManager().reloadSourceTable(entry.getKey(), entry.getValue());
- getTableManager().reloadTableExt(entry.getKey(), entry.getValue());
+ getTableManager().reloadSourceTableQuietly(entry.getKey(), entry.getValue());
+ getTableManager().reloadTableExtQuietly(entry.getKey(), entry.getValue());
}
logger.info("reload table cache done");
[3/3] kylin git commit: KYLIN-2672 Add UT
Posted by bi...@apache.org.
KYLIN-2672 Add UT
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/50990664
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/50990664
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/50990664
Branch: refs/heads/master
Commit: 509906641dc819ecc03bb1050bcdc33e1236e6cf
Parents: 5cecbce
Author: kangkaisen <ka...@meituan.com>
Authored: Fri Feb 9 15:53:02 2018 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Fri Feb 9 21:20:03 2018 +0800
----------------------------------------------------------------------
.../rest/controller/CacheControllerTest.java | 54 +++++++++++++++++++-
1 file changed, 53 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/50990664/server/src/test/java/org/apache/kylin/rest/controller/CacheControllerTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/controller/CacheControllerTest.java b/server/src/test/java/org/apache/kylin/rest/controller/CacheControllerTest.java
index c2e21cc..3466a36 100644
--- a/server/src/test/java/org/apache/kylin/rest/controller/CacheControllerTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/controller/CacheControllerTest.java
@@ -19,7 +19,17 @@
package org.apache.kylin.rest.controller;
import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.JsonSerializer;
+import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.cube.CubeDescManager;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.TableRef;
+import org.apache.kylin.rest.request.CubeMigrationRequest;
import org.apache.kylin.rest.service.CacheService;
import org.apache.kylin.rest.service.ServiceTestBase;
import org.junit.Before;
@@ -27,6 +37,9 @@ import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
/**
* @author shaoshi
*/
@@ -48,7 +61,46 @@ public class CacheControllerTest extends ServiceTestBase {
@Test
public void testBasics() throws IOException {
-
cacheController.wipeCache("cube_desc", "drop", "test_kylin_cube_with_slr_desc");
}
+
+ @Test
+ public void testClearCacheForCubeMigration() throws IOException {
+ KylinConfig config = KylinConfig.getInstanceFromEnv();
+ String CUBENAME = "test_kylin_cube_without_slr_desc";
+
+ CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
+ CubeDesc cubeDesc = cubeDescManager.getCubeDesc(CUBENAME);
+ DataModelDesc modelDesc = cubeDesc.getModel();
+ Map<String, String> tableToProjects = new HashMap<>();
+ for (TableRef tableRef : modelDesc.getAllTables()) {
+ tableToProjects.put(tableRef.getTableIdentity(), tableRef.getTableDesc().getProject());
+ }
+
+ String uuid = cubeDesc.getUuid();
+ String signature = cubeDesc.getSignature();
+
+ assertEquals(cubeDesc.getRetentionRange(), 0);
+
+ //update cubeDesc
+ cubeDesc.setRetentionRange(2018);
+ cubeDesc.updateRandomUuid();
+
+ //directly update metadata in store to simulate cube migration
+ Serializer<CubeDesc> cubeDescSerializer = new JsonSerializer<CubeDesc>(CubeDesc.class);
+ getStore().putResource(cubeDesc.getResourcePath(), cubeDesc, cubeDescSerializer);
+
+ CubeMigrationRequest request = new CubeMigrationRequest();
+ request.setCube(cubeDesc.getName());
+ request.setModel(modelDesc.getName());
+ request.setProject(modelDesc.getProject());
+ request.setTableToProjects(tableToProjects);
+
+ cacheController.clearCacheForCubeMigration(request);
+
+ assertEquals(2018, cubeDescManager.getCubeDesc(CUBENAME).getRetentionRange());
+ assertEquals(signature, cubeDescManager.getCubeDesc(CUBENAME).getSignature());
+ assertNotEquals(uuid, cubeDescManager.getCubeDesc(CUBENAME).getUuid());
+ }
+
}