You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/05/25 08:11:34 UTC
[02/11] incubator-kylin git commit: fix ci: clean broadcase module
fix ci: clean broadcase module
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/f5695644
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/f5695644
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/f5695644
Branch: refs/heads/new697
Commit: f56956443e456b497a3aaf3a2cc4626f58e72f5c
Parents: cd7b918
Author: honma <ho...@ebay.com>
Authored: Fri May 15 17:32:18 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon May 25 11:22:58 2015 +0800
----------------------------------------------------------------------
.../kylin/common/restclient/Broadcaster.java | 11 +-
.../common/restclient/MultiValueCache.java | 88 ----------
.../common/restclient/SingleValueCache.java | 9 +-
.../org/apache/kylin/cube/CubeDescManager.java | 25 ++-
.../java/org/apache/kylin/cube/CubeManager.java | 161 +++++++++----------
.../apache/kylin/cube/CubeManagerCacheTest.java | 11 +-
.../kylin/invertedindex/IIDescManager.java | 8 +-
.../apache/kylin/invertedindex/IIManager.java | 101 ++++++------
.../invertedindex/IIDescManagerTest.java | 2 +-
.../kylin/job/cube/MergeDictionaryStep.java | 15 +-
.../job/cube/UpdateCubeInfoAfterBuildStep.java | 9 +-
.../job/hadoop/invertedindex/IIBulkLoadJob.java | 2 +-
.../kylin/job/BuildCubeWithEngineTest.java | 2 +-
.../apache/kylin/job/BuildIIWithEngineTest.java | 19 ++-
.../apache/kylin/job/BuildIIWithStreamTest.java | 31 ++--
.../java/org/apache/kylin/job/DeployUtil.java | 27 ++--
.../job/hadoop/cube/MergeCuboidMapperTest.java | 31 ++--
.../apache/kylin/metadata/MetadataManager.java | 33 ++--
.../kylin/metadata/project/ProjectManager.java | 77 ++++-----
.../kylin/rest/controller/CubeController.java | 60 +++----
.../apache/kylin/rest/service/BasicService.java | 15 +-
.../apache/kylin/rest/service/CacheService.java | 132 +++++++--------
.../apache/kylin/rest/service/CubeService.java | 55 +++----
.../apache/kylin/rest/service/JobService.java | 82 +++++-----
.../kylin/rest/service/ProjectService.java | 23 +--
.../kylin/rest/service/CacheServiceTest.java | 51 +++---
.../kylin/rest/service/CubeServiceTest.java | 14 +-
27 files changed, 465 insertions(+), 629 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java b/common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java
index 6fbe413..f745410 100644
--- a/common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java
+++ b/common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java
@@ -35,9 +35,6 @@ import java.util.concurrent.atomic.AtomicLong;
/**
* Broadcast kylin event out
- *
- * @author jianliu
- *
*/
public class Broadcaster {
@@ -113,11 +110,11 @@ public class Broadcaster {
return counter.getAndSet(0);
}
- public static enum EVENT {
+ public enum EVENT {
CREATE("create"), UPDATE("update"), DROP("drop");
private String text;
- private EVENT(String text) {
+ EVENT(String text) {
this.text = text;
}
@@ -136,11 +133,11 @@ public class Broadcaster {
}
}
- public static enum TYPE {
+ public enum TYPE {
ALL("all"), CUBE("cube"), CUBE_DESC("cube_desc"), PROJECT("project"), INVERTED_INDEX("inverted_index"), INVERTED_INDEX_DESC("ii_desc"), TABLE("table"), DATA_MODEL("data_model"), HYBRID("hybrid");
private String text;
- private TYPE(String text) {
+ TYPE(String text) {
this.text = text;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/common/src/main/java/org/apache/kylin/common/restclient/MultiValueCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/restclient/MultiValueCache.java b/common/src/main/java/org/apache/kylin/common/restclient/MultiValueCache.java
deleted file mode 100644
index b479969..0000000
--- a/common/src/main/java/org/apache/kylin/common/restclient/MultiValueCache.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.common.restclient;
-
-import java.util.Set;
-
-import com.google.common.collect.HashMultimap;
-
-/**
- * @author xjiang
- *
- */
-public class MultiValueCache<K, V> extends AbstractRestCache<K, V> {
-
- private final HashMultimap<K, V> innerCache;
-
- public MultiValueCache(Broadcaster.TYPE syncType) {
- super(syncType);
- innerCache = HashMultimap.create();
- }
-
- public void put(K key, V value) {
- Broadcaster.EVENT eventType = innerCache.containsKey(key) ? Broadcaster.EVENT.UPDATE : Broadcaster.EVENT.CREATE;
- synchronized (this) {
- innerCache.put(key, value);
- }
- syncRemote(key, eventType);
- }
-
- public void putLocal(K key, V value) {
- synchronized (this) {
- innerCache.put(key, value);
- }
- }
-
- public void remove(K key) {
- if (innerCache.containsKey(key)) {
- innerCache.removeAll(key);
- syncRemote(key, Broadcaster.EVENT.DROP);
- }
- }
-
- public void removeLocal(K key) {
- if (innerCache.containsKey(key)) {
- innerCache.removeAll(key);
- }
- }
-
- public void clear() {
- innerCache.clear();
- }
-
- public int size() {
- return innerCache.size();
- }
-
- public Set<V> get(K key) {
- return innerCache.get(key);
- }
-
- public Set<K> keySet() {
- return innerCache.keySet();
- }
-
- public boolean containsKey(Object key) {
- return innerCache.containsKey(key);
- }
-
- public boolean containsEntry(Object key, Object value) {
- return innerCache.containsEntry(key, value);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java b/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
index 9848201..fb44206 100644
--- a/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
+++ b/common/src/main/java/org/apache/kylin/common/restclient/SingleValueCache.java
@@ -43,8 +43,10 @@ public abstract class SingleValueCache<K, V> extends AbstractRestCache<K, V> {
}
public void put(K key, V value) {
- final V result = innerCache.put(key, value);
- if (result == null) {
+ //enforce all cache changes coming from REST
+ //final V result = innerCache.put(key, value);
+
+ if (!innerCache.containsKey(key)) {
syncRemote(key, Broadcaster.EVENT.CREATE);
} else {
syncRemote(key, Broadcaster.EVENT.UPDATE);
@@ -57,7 +59,8 @@ public abstract class SingleValueCache<K, V> extends AbstractRestCache<K, V> {
public void remove(K key) {
if (innerCache.containsKey(key)) {
- innerCache.remove(key);
+ //enforce all cache changes coming from REST
+ //innerCache.remove(key);
syncRemote(key, Broadcaster.EVENT.DROP);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java b/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
index ac5fc2b..d406da1 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeDescManager.java
@@ -18,26 +18,25 @@
package org.apache.kylin.cube;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.cube.model.validation.CubeMetadataValidator;
-import org.apache.kylin.cube.model.validation.ValidateContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.Serializer;
import org.apache.kylin.common.restclient.Broadcaster;
+import org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.validation.CubeMetadataValidator;
+import org.apache.kylin.cube.model.validation.ValidateContext;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.MetadataManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
/**
* Manager class for CubeDesc; extracted from #CubeManager
@@ -108,7 +107,7 @@ public class CubeDescManager {
* @param name
* @throws IOException
*/
- public CubeDesc reloadCubeDesc(String name) throws IOException {
+ public CubeDesc reloadCubeDescLocal(String name) throws IOException {
// Save Source
String path = CubeDesc.getCubeDescResourcePath(name);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
----------------------------------------------------------------------
diff --git a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 76a6c6d..b986c36 100644
--- a/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -52,7 +52,6 @@ import java.io.IOException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
-
/**
* @author yangli9
*/
@@ -138,7 +137,7 @@ public class CubeManager implements IRealizationProvider {
Iterator<CubeInstance> it = list.iterator();
while (it.hasNext()) {
CubeInstance ci = it.next();
- if (descName.equals(ci.getDescName())) {
+ if (descName.equalsIgnoreCase(ci.getDescName())) {
result.add(ci);
}
}
@@ -154,7 +153,7 @@ public class CubeManager implements IRealizationProvider {
DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, factColumnsPath);
cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
- saveResource(cubeSeg.getCubeInstance());
+ updateCube(cubeSeg.getCubeInstance(), false);
return dictInfo;
}
@@ -192,7 +191,7 @@ public class CubeManager implements IRealizationProvider {
cubeSeg.putSnapshotResPath(lookupTable, snapshot.getResourcePath());
- saveResource(cubeSeg.getCubeInstance());
+ updateCube(cubeSeg.getCubeInstance(), false);
return snapshot;
}
@@ -202,25 +201,26 @@ public class CubeManager implements IRealizationProvider {
logger.info("Dropping cube '" + cubeName + "'");
// load projects before remove cube from project
- ResourceStore store = getStore();
-
// delete cube instance and cube desc
CubeInstance cube = getCube(cubeName);
- if (deleteDesc && cube.getDescriptor() != null)
- store.deleteResource(cube.getDescriptor().getResourcePath());
-
- store.deleteResource(cube.getResourcePath());
+ if (deleteDesc && cube.getDescriptor() != null) {
+ CubeDescManager.getInstance(config).removeCubeDesc(cube.getDescriptor());
+ }
+ removeCube(cube);
// delete cube from project
ProjectManager.getInstance(config).removeRealizationsFromProjects(RealizationType.CUBE, cubeName);
- // clean cube cache
- this.afterCubeDropped(cube);
-
return cube;
}
+ private void removeCube(CubeInstance cube) throws IOException {
+ // remove cube and update cache
+ getStore().deleteResource(cube.getResourcePath());
+ cubeMap.remove(cube.getName());
+ }
+
// sync on update
public CubeInstance createCube(String cubeName, String projectName, CubeDesc desc, String owner) throws IOException {
logger.info("Creating cube '" + projectName + "-->" + cubeName + "' from desc '" + desc.getName() + "'");
@@ -228,16 +228,27 @@ public class CubeManager implements IRealizationProvider {
// save cube resource
CubeInstance cube = CubeInstance.create(cubeName, projectName, desc);
cube.setOwner(owner);
- saveResource(cube);
+ updateCube(cube, false);
ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.CUBE, cubeName, projectName, owner);
return cube;
}
- public CubeInstance updateCube(CubeInstance cube) throws IOException {
+ /**
+ * if not sure whether to enable updateProject, just use it
+ */
+ public CubeInstance updateCube(CubeInstance cube, boolean updateProject) throws IOException {
+
logger.info("Updating cube instance '" + cube.getName());
- saveResource(cube);
+ getStore().putResource(cube.getResourcePath(), cube, CUBE_SERIALIZER);
+ cubeMap.put(cube.getName(), cube);
+
+ if (updateProject) {
+ logger.info("Updating project instance for cube:'" + cube.getName());
+ ProjectManager.getInstance(config).updateProject(RealizationType.CUBE, cube.getName());
+ }
+
return cube;
}
@@ -258,7 +269,7 @@ public class CubeManager implements IRealizationProvider {
cube.getSegments().add(appendSegment);
cube.getSegments().add(mergeSegment);
Collections.sort(cube.getSegments());
- updateCube(cube);
+ updateCube(cube, false);
return new Pair<CubeSegment, CubeSegment>(appendSegment, mergeSegment);
}
@@ -277,19 +288,18 @@ public class CubeManager implements IRealizationProvider {
validateNewSegments(cube, newSegment);
cube.getSegments().add(newSegment);
Collections.sort(cube.getSegments());
- updateCube(cube);
+ updateCube(cube, false);
return newSegment;
}
-
public CubeSegment refreshSegment(CubeInstance cube, long startDate, long endDate) throws IOException {
checkNoBuildingSegment(cube);
CubeSegment newSegment = newSegment(cube, startDate, endDate);
cube.getSegments().add(newSegment);
Collections.sort(cube.getSegments());
- updateCube(cube);
+ updateCube(cube, false);
return newSegment;
}
@@ -304,7 +314,7 @@ public class CubeManager implements IRealizationProvider {
validateNewSegments(cube, newSegment);
cube.getSegments().add(newSegment);
Collections.sort(cube.getSegments());
- updateCube(cube);
+ updateCube(cube, false);
return newSegment;
}
@@ -371,28 +381,17 @@ public class CubeManager implements IRealizationProvider {
*
* @param cubeName
*/
- public void loadCubeCache(String cubeName) {
+ public void reloadCubeLocal(String cubeName) {
try {
- loadCubeInstance(CubeInstance.concatResourcePath(cubeName));
+ reloadCubeLocalAt(CubeInstance.concatResourcePath(cubeName));
} catch (IOException e) {
logger.error(e.getLocalizedMessage(), e);
}
}
- /**
- * After cube deletion, remove cube related cache
- *
- * @param cube
- */
- public void removeCubeCache(CubeInstance cube) {
- final String cubeName = cube.getName().toUpperCase();
- cubeMap.remove(cubeName);
- usedStorageLocation.removeAll(cubeName);
- }
-
- public void removeCubeCacheLocal(String cubeName) {
+ public void removeCubeLocal(String cubeName) {
cubeMap.removeLocal(cubeName);
- usedStorageLocation.removeAll(cubeName);
+ usedStorageLocation.removeAll(cubeName.toUpperCase());
}
public LookupStringTable getLookupTable(CubeSegment cubeSegment, DimensionDesc dim) {
@@ -412,18 +411,41 @@ public class CubeManager implements IRealizationProvider {
}
}
- private void saveResource(CubeInstance cube) throws IOException {
- ResourceStore store = getStore();
- store.putResource(cube.getResourcePath(), cube, CUBE_SERIALIZER);
- this.afterCubeUpdated(cube);
- }
+ private CubeSegment newSegment(CubeInstance cubeInstance, long startDate, long endDate) {
+ if (startDate >= endDate)
+ throw new IllegalArgumentException("New segment range invalid, start date must be earlier than end date, " + startDate + " < " + endDate);
- private void afterCubeUpdated(CubeInstance updatedCube) {
- cubeMap.put(updatedCube.getName(), updatedCube);
+ CubeSegment segment = new CubeSegment();
+ String incrementalSegName = CubeSegment.getSegmentName(startDate, endDate);
+ segment.setUuid(UUID.randomUUID().toString());
+ segment.setName(incrementalSegName);
+ Date creatTime = new Date();
+ segment.setCreateTimeUTC(creatTime.getTime());
+ segment.setDateRangeStart(startDate);
+ segment.setDateRangeEnd(endDate);
+ segment.setStatus(SegmentStatusEnum.NEW);
+ segment.setStorageLocationIdentifier(generateStorageLocation());
+
+ segment.setCubeInstance(cubeInstance);
+
+ segment.validate();
+ return segment;
}
- private void afterCubeDropped(CubeInstance droppedCube) {
- removeCubeCache(droppedCube);
+ private String generateStorageLocation() {
+ String namePrefix = IRealizationConstants.CubeHbaseStorageLocationPrefix;
+ String tableName = "";
+ do {
+ StringBuffer sb = new StringBuffer();
+ sb.append(namePrefix);
+ Random ran = new Random();
+ for (int i = 0; i < HBASE_TABLE_LENGTH; i++) {
+ sb.append(ALPHA_NUM.charAt(ran.nextInt(ALPHA_NUM.length())));
+ }
+ tableName = sb.toString();
+ } while (this.usedStorageLocation.containsValue(tableName));
+
+ return tableName;
}
public CubeSegment autoMergeCubeSegments(CubeInstance cube) throws IOException {
@@ -480,43 +502,6 @@ public class CubeManager implements IRealizationProvider {
return null;
}
- private CubeSegment newSegment(CubeInstance cubeInstance, long startDate, long endDate) {
- if (startDate >= endDate)
- throw new IllegalArgumentException("New segment range invalid, start date must be earlier than end date, " + startDate + " < " + endDate);
-
- CubeSegment segment = new CubeSegment();
- String incrementalSegName = CubeSegment.getSegmentName(startDate, endDate);
- segment.setUuid(UUID.randomUUID().toString());
- segment.setName(incrementalSegName);
- Date creatTime = new Date();
- segment.setCreateTimeUTC(creatTime.getTime());
- segment.setDateRangeStart(startDate);
- segment.setDateRangeEnd(endDate);
- segment.setStatus(SegmentStatusEnum.NEW);
- segment.setStorageLocationIdentifier(generateStorageLocation());
-
- segment.setCubeInstance(cubeInstance);
-
- segment.validate();
- return segment;
- }
-
- private String generateStorageLocation() {
- String namePrefix = IRealizationConstants.CubeHbaseStorageLocationPrefix;
- String tableName = "";
- do {
- StringBuffer sb = new StringBuffer();
- sb.append(namePrefix);
- Random ran = new Random();
- for (int i = 0; i < HBASE_TABLE_LENGTH; i++) {
- sb.append(ALPHA_NUM.charAt(ran.nextInt(ALPHA_NUM.length())));
- }
- tableName = sb.toString();
- } while (this.usedStorageLocation.containsValue(tableName));
-
- return tableName;
- }
-
public void promoteNewlyBuiltSegments(CubeInstance cube, CubeSegment... newSegments) throws IOException {
List<CubeSegment> tobe = calculateToBeSegments(cube);
@@ -542,7 +527,7 @@ public class CubeManager implements IRealizationProvider {
cube.setStatus(RealizationStatusEnum.READY);
logger.info("Promoting cube " + cube + ", new segments " + newSegments);
- saveResource(cube);
+ updateCube(cube, true);
}
private void validateNewSegments(CubeInstance cube, CubeSegment... newSegments) {
@@ -579,7 +564,7 @@ public class CubeManager implements IRealizationProvider {
}
firstSeg.validate();
- for (int i = 0, j = 1; j < tobe.size(); ) {
+ for (int i = 0, j = 1; j < tobe.size();) {
CubeSegment is = tobe.get(i);
CubeSegment js = tobe.get(j);
js.validate();
@@ -644,16 +629,16 @@ public class CubeManager implements IRealizationProvider {
logger.debug("Loading Cube from folder " + store.getReadableResourcePath(ResourceStore.CUBE_RESOURCE_ROOT));
for (String path : paths) {
- loadCubeInstance(path);
+ reloadCubeLocalAt(path);
}
logger.debug("Loaded " + paths.size() + " Cube(s)");
}
- private synchronized CubeInstance loadCubeInstance(String path) throws IOException {
+ private synchronized CubeInstance reloadCubeLocalAt(String path) throws IOException {
ResourceStore store = getStore();
- CubeInstance cubeInstance = null;
+ CubeInstance cubeInstance;
try {
cubeInstance = store.getResource(path, CubeInstance.class, CUBE_SERIALIZER);
cubeInstance.setConfig(config);
@@ -668,7 +653,7 @@ public class CubeManager implements IRealizationProvider {
cubeMap.putLocal(cubeName, cubeInstance);
for (CubeSegment segment : cubeInstance.getSegments()) {
- usedStorageLocation.put(cubeName, segment.getStorageLocationIdentifier());
+ usedStorageLocation.put(cubeName.toUpperCase(), segment.getStorageLocationIdentifier());
}
return cubeInstance;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
----------------------------------------------------------------------
diff --git a/cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java b/cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
index 5e64e01..7c74993 100644
--- a/cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
+++ b/cube/src/test/java/org/apache/kylin/cube/CubeManagerCacheTest.java
@@ -18,18 +18,17 @@
package org.apache.kylin.cube;
-import static org.junit.Assert.*;
-
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.metadata.MetadataManager;
+import static org.junit.Assert.assertEquals;
/**
* @author yangli9
@@ -68,7 +67,7 @@ public class CubeManagerCacheTest extends LocalFileMetadataTestCase {
assertEquals(RealizationStatusEnum.DISABLED, createdCube.getStatus());
createdCube.setStatus(RealizationStatusEnum.DESCBROKEN);
- cubeManager.updateCube(createdCube);
+ cubeManager.updateCube(createdCube,true);
assertEquals(RealizationStatusEnum.DESCBROKEN, cubeManager.getCube("a_whole_new_cube").getStatus());
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIDescManager.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIDescManager.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIDescManager.java
index ae38955..cf0af86 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIDescManager.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIDescManager.java
@@ -18,6 +18,7 @@
package org.apache.kylin.invertedindex;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -27,7 +28,6 @@ import org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.MetadataManager;
-import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -90,7 +90,7 @@ public class IIDescManager {
reloadAllIIDesc();
}
- public List<IIDesc> listAllDesc(){
+ public List<IIDesc> listAllDesc() {
return new ArrayList<IIDesc>(iiDescMap.values());
}
@@ -105,7 +105,7 @@ public class IIDescManager {
* @param name
* @throws IOException
*/
- public IIDesc reloadIIDesc(String name) throws IOException {
+ public IIDesc reloadIIDescLocal(String name) throws IOException {
// Save Source
String path = IIDesc.getIIDescResourcePath(name);
@@ -170,7 +170,7 @@ public class IIDescManager {
}
public void removeIIDescLocal(String name) throws IOException {
- iiDescMap.remove(name);
+ iiDescMap.removeLocal(name);
}
private void reloadAllIIDesc() throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
index c1a0765..ea40ebd 100644
--- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
+++ b/invertedindex/src/main/java/org/apache/kylin/invertedindex/IIManager.java
@@ -18,6 +18,8 @@
package org.apache.kylin.invertedindex;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
@@ -31,6 +33,8 @@ import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.metadata.realization.IRealizationProvider;
@@ -39,7 +43,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
/**
@@ -92,7 +99,7 @@ public class IIManager implements IRealizationProvider {
private CaseInsensitiveStringCache<IIInstance> iiMap = new CaseInsensitiveStringCache<IIInstance>(Broadcaster.TYPE.INVERTED_INDEX);
// for generation hbase table name of a new segment
- private HashSet<String> usedStorageLocation = new HashSet<String>();
+ private Multimap<String, String> usedStorageLocation = HashMultimap.create();
private IIManager(KylinConfig config) throws IOException {
logger.info("Initializing IIManager with config " + config);
@@ -106,7 +113,6 @@ public class IIManager implements IRealizationProvider {
}
public IIInstance getII(String iiName) {
- iiName = iiName.toUpperCase();
return iiMap.get(iiName);
}
@@ -137,7 +143,7 @@ public class IIManager implements IRealizationProvider {
DictionaryInfo dict = dictMgr.buildDictionary(iiDesc.getModel(), "true", column, factColumnsPath);
iiSeg.putDictResPath(column, dict.getResourcePath());
}
- saveResource(iiSeg.getIIInstance());
+ updateII(iiSeg.getIIInstance(), false);
}
/**
@@ -168,58 +174,55 @@ public class IIManager implements IRealizationProvider {
if (this.getII(ii.getName()) != null)
throw new IllegalArgumentException("The II name '" + ii.getName() + "' already exists.");
- // other logic is the same as update.
- return updateII(ii);
- }
-
- public void updateIIStreamingOffset(String iiName, int partition, long offset) throws IOException {
-
- }
-
-
- public IIInstance updateII(IIInstance ii) throws IOException {
- logger.info("Updating II instance '" + ii.getName());
-
- // save resource
- saveResource(ii);
-
- logger.info("II with " + ii.getSegments().size() + " segments is saved");
+ this.updateII(ii, false);
+ String projectName = (null == ii.getProjectName()) ? ProjectInstance.DEFAULT_PROJECT_NAME : ii.getProjectName();
+ ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.INVERTED_INDEX, ii.getName(), projectName, ii.getOwner());
return ii;
}
- public void loadIICache(String iiName) {
+ public void reloadIILocal(String iiName) {
try {
- loadIIInstance(IIInstance.concatResourcePath(iiName));
+ reloadIILocalAt(IIInstance.concatResourcePath(iiName));
} catch (IOException e) {
logger.error(e.getLocalizedMessage(), e);
}
}
- public void removeIICache(IIInstance ii) {
- iiMap.remove(ii.getName());
+ public IIInstance dropII(String iiName, boolean deleteDesc) throws IOException {
+ logger.info("Dropping II '" + iiName + "'");
+
+ IIInstance ii = getII(iiName);
- for (IISegment segment : ii.getSegments()) {
- usedStorageLocation.remove(segment.getName());
+ if (deleteDesc && ii.getDescriptor() != null) {
+ IIDescManager.getInstance(config).removeIIDesc(ii.getDescriptor());
}
+
+ removeII(ii);
+ ProjectManager.getInstance(config).removeRealizationsFromProjects(RealizationType.INVERTED_INDEX, iiName);
+
+ return ii;
}
- public void removeIILocalCache(String name) {
- iiMap.removeLocal(name);
- //TODO
- // for (IISegment segment : ii.getSegments()) {
- // usedStorageLocation.remove(segment.getName());
- // }
+ private void removeII(IIInstance ii) throws IOException {
+ getStore().deleteResource(ii.getResourcePath());
+ iiMap.remove(ii.getName());
}
- private void saveResource(IIInstance ii) throws IOException {
- ResourceStore store = getStore();
- store.putResource(ii.getResourcePath(), ii, II_SERIALIZER);
- this.afterIIUpdated(ii);
+ public void removeIILocal(String name) {
+ iiMap.removeLocal(name);
+ usedStorageLocation.removeAll(name.toUpperCase());
}
- private void afterIIUpdated(IIInstance updatedII) {
- iiMap.put(updatedII.getName(), updatedII);
+ public void updateII(IIInstance ii, boolean updateProject) throws IOException {
+ logger.info("Updating II instance : " + ii.getName());
+ getStore().putResource(ii.getResourcePath(), ii, II_SERIALIZER);
+ iiMap.put(ii.getName(), ii);
+
+ if (updateProject) {
+ logger.info("Updating project instance for ii: " + ii.getName());
+ ProjectManager.getInstance(config).updateProject(RealizationType.INVERTED_INDEX, ii.getName());
+ }
}
/**
@@ -251,7 +254,7 @@ public class IIManager implements IRealizationProvider {
int idx = (int) (Math.random() * ALPHA_NUM.length());
sb.append(ALPHA_NUM.charAt(idx));
}
- if (usedStorageLocation.contains(sb.toString())) {
+ if (usedStorageLocation.containsValue(sb.toString())) {
continue;
} else {
return sb.toString();
@@ -266,31 +269,31 @@ public class IIManager implements IRealizationProvider {
logger.debug("Loading II from folder " + store.getReadableResourcePath(ResourceStore.II_RESOURCE_ROOT));
for (String path : paths) {
- loadIIInstance(path);
+ reloadIILocalAt(path);
}
logger.debug("Loaded " + paths.size() + " II(s)");
}
- private synchronized IIInstance loadIIInstance(String path) throws IOException {
+ private synchronized IIInstance reloadIILocalAt(String path) throws IOException {
ResourceStore store = getStore();
logger.debug("Loading IIInstance " + store.getReadableResourcePath(path));
- IIInstance IIInstance = null;
+ IIInstance ii = null;
try {
- IIInstance = store.getResource(path, IIInstance.class, II_SERIALIZER);
- IIInstance.setConfig(config);
+ ii = store.getResource(path, IIInstance.class, II_SERIALIZER);
+ ii.setConfig(config);
- if (StringUtils.isBlank(IIInstance.getName()))
+ if (StringUtils.isBlank(ii.getName()))
throw new IllegalStateException("IIInstance name must not be blank");
- iiMap.putLocal(IIInstance.getName(), IIInstance);
+ iiMap.putLocal(ii.getName(), ii);
- for (IISegment segment : IIInstance.getSegments()) {
- usedStorageLocation.add(segment.getName());
+ for (IISegment segment : ii.getSegments()) {
+ usedStorageLocation.put(ii.getName().toUpperCase(), segment.getStorageLocationIdentifier());
}
- return IIInstance;
+ return ii;
} catch (Exception e) {
logger.error("Error during load ii instance " + path, e);
return null;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
----------------------------------------------------------------------
diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
index a13a0d1..a456dcd 100644
--- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
+++ b/invertedindex/src/test/java/org/apache/kylin/invertedindex/invertedindex/IIDescManagerTest.java
@@ -95,7 +95,7 @@ public class IIDescManagerTest extends LocalFileMetadataTestCase {
Assert.assertEquals(desc, newDesc);
// reload the cache
- mgr.reloadIIDesc(TEST_II_DESC_NAME);
+ mgr.reloadIIDescLocal(TEST_II_DESC_NAME);
newDesc = mgr.getIIDesc(TEST_II_DESC_NAME);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java b/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java
index b9aa7a4..d0a7db3 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/MergeDictionaryStep.java
@@ -18,16 +18,8 @@
package org.apache.kylin.job.cube;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang.StringUtils;
-
import com.google.common.collect.Lists;
+import org.apache.commons.lang.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
@@ -42,6 +34,9 @@ import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
import org.apache.kylin.metadata.model.TblColRef;
+import java.io.IOException;
+import java.util.*;
+
public class MergeDictionaryStep extends AbstractExecutable {
private static final String CUBE_NAME = "cubeName";
@@ -68,7 +63,7 @@ public class MergeDictionaryStep extends AbstractExecutable {
makeDictForNewSegment(conf, cube, newSegment, mergingSegments);
makeSnapshotForNewSegment(cube, newSegment, mergingSegments);
- mgr.updateCube(cube);
+ mgr.updateCube(cube,false);
return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
} catch (IOException e) {
logger.error("fail to merge dictionary or lookup snapshots", e);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java b/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
index e9c2a2c..8140003 100644
--- a/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
+++ b/job/src/main/java/org/apache/kylin/job/cube/UpdateCubeInfoAfterBuildStep.java
@@ -18,11 +18,8 @@
package org.apache.kylin.job.cube;
-import java.io.IOException;
-
-import org.apache.commons.lang.StringUtils;
-
import com.google.common.base.Preconditions;
+import org.apache.commons.lang.StringUtils;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
@@ -33,6 +30,8 @@ import org.apache.kylin.job.execution.ExecutableContext;
import org.apache.kylin.job.execution.ExecuteResult;
import org.apache.kylin.job.execution.Output;
+import java.io.IOException;
+
/**
*/
public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
@@ -128,7 +127,7 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
if (segmentReady) {
cubeManager.promoteNewlyBuiltSegments(cube, segment);
} else {
- cubeManager.updateCube(cube);
+ cubeManager.updateCube(cube, true);
}
return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java
index f636af4..145a3c8 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java
@@ -64,7 +64,7 @@ public class IIBulkLoadJob extends AbstractHadoopJob {
IISegment seg = ii.getFirstSegment();
seg.setStorageLocationIdentifier(tableName);
seg.setStatus(SegmentStatusEnum.READY);
- mgr.updateII(ii);
+ mgr.updateII(ii,true);
return hbaseExitCode;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index 812eb9d..547a6e7 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -263,7 +263,7 @@ public class BuildCubeWithEngineTest {
private void clearSegment(String cubeName) throws Exception {
CubeInstance cube = cubeManager.getCube(cubeName);
cube.getSegments().clear();
- cubeManager.updateCube(cube);
+ cubeManager.updateCube(cube,true);
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
index af8682e..9c0219a 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
@@ -61,7 +61,7 @@ public class BuildIIWithEngineTest {
private DefaultScheduler scheduler;
protected ExecutableManager jobService;
- protected static final String[] TEST_II_INSTANCES = new String[]{ "test_kylin_ii_inner_join", "test_kylin_ii_left_join"};
+ protected static final String[] TEST_II_INSTANCES = new String[] { "test_kylin_ii_inner_join", "test_kylin_ii_left_join" };
private static final Log logger = LogFactory.getLog(BuildIIWithEngineTest.class);
@@ -104,7 +104,7 @@ public class BuildIIWithEngineTest {
}
jobEngineConfig = new JobEngineConfig(kylinConfig);
for (String jobId : jobService.getAllJobIds()) {
- if(jobService.getJob(jobId) instanceof IIJob){
+ if (jobService.getJob(jobId) instanceof IIJob) {
jobService.deleteJob(jobId);
}
}
@@ -115,7 +115,7 @@ public class BuildIIWithEngineTest {
IIInstance ii = iiManager.getII(iiInstance);
if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
ii.setStatus(RealizationStatusEnum.DISABLED);
- iiManager.updateII(ii);
+ iiManager.updateII(ii, true);
}
}
}
@@ -127,7 +127,7 @@ public class BuildIIWithEngineTest {
IIInstance ii = iiManager.getII(iiInstance);
if (ii.getStatus() != RealizationStatusEnum.READY) {
ii.setStatus(RealizationStatusEnum.READY);
- iiManager.updateII(ii);
+ iiManager.updateII(ii, true);
}
}
backup();
@@ -137,7 +137,7 @@ public class BuildIIWithEngineTest {
@Ignore
public void testBuildII() throws Exception {
- String[] testCase = new String[]{"buildIIInnerJoin", "buildIILeftJoin"};
+ String[] testCase = new String[] { "buildIIInnerJoin", "buildIILeftJoin" };
ExecutorService executorService = Executors.newFixedThreadPool(testCase.length);
final CountDownLatch countDownLatch = new CountDownLatch(testCase.length);
List<Future<List<String>>> tasks = Lists.newArrayListWithExpectedSize(testCase.length);
@@ -183,10 +183,9 @@ public class BuildIIWithEngineTest {
}
protected List<String> buildIIInnerJoin() throws Exception {
- return buildII(TEST_II_INSTANCES[0]);
+ return buildII(TEST_II_INSTANCES[0]);
}
-
protected List<String> buildIILeftJoin() throws Exception {
return buildII(TEST_II_INSTANCES[1]);
}
@@ -208,14 +207,14 @@ public class BuildIIWithEngineTest {
private void clearSegment(String iiName) throws Exception {
IIInstance ii = iiManager.getII(iiName);
ii.getSegments().clear();
- iiManager.updateII(ii);
+ iiManager.updateII(ii,true);
}
private String buildSegment(String iiName, long startDate, long endDate) throws Exception {
IIInstance iiInstance = iiManager.getII(iiName);
IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
iiInstance.getSegments().add(segment);
- iiManager.updateII(iiInstance);
+ iiManager.updateII(iiInstance, true);
IIJobBuilder iiJobBuilder = new IIJobBuilder(jobEngineConfig);
IIJob job = iiJobBuilder.buildJob(segment);
jobService.addJob(job);
@@ -224,7 +223,7 @@ public class BuildIIWithEngineTest {
}
private int cleanupOldStorage() throws Exception {
- String[] args = {"--delete", "true"};
+ String[] args = { "--delete", "true" };
int exitCode = ToolRunner.run(new StorageCleanupJob(), args);
return exitCode;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index 65faad7..6bceab7 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -34,17 +34,7 @@
package org.apache.kylin.job;
-import static org.junit.Assert.fail;
-
-import java.io.File;
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingDeque;
-
+import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ToolRunner;
@@ -77,7 +67,16 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Lists;
+import java.io.File;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingDeque;
+
+import static org.junit.Assert.fail;
/**
*/
@@ -109,7 +108,7 @@ public class BuildIIWithStreamTest {
IIInstance ii = iiManager.getII(iiInstance);
if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
ii.setStatus(RealizationStatusEnum.DISABLED);
- iiManager.updateII(ii);
+ iiManager.updateII(ii,true);
}
}
}
@@ -169,7 +168,7 @@ public class BuildIIWithStreamTest {
private void clearSegment(String iiName) throws Exception {
IIInstance ii = iiManager.getII(iiName);
ii.getSegments().clear();
- iiManager.updateII(ii);
+ iiManager.updateII(ii,true);
}
private IISegment createSegment(String iiName) throws Exception {
@@ -186,7 +185,7 @@ public class BuildIIWithStreamTest {
IIInstance iiInstance = iiManager.getII(iiName);
IISegment segment = iiManager.buildSegment(iiInstance, startDate, endDate);
iiInstance.getSegments().add(segment);
- iiManager.updateII(iiInstance);
+ iiManager.updateII(iiInstance,true);
return segment;
}
@@ -243,7 +242,7 @@ public class BuildIIWithStreamTest {
IIInstance ii = iiManager.getII(iiName);
if (ii.getStatus() != RealizationStatusEnum.READY) {
ii.setStatus(RealizationStatusEnum.READY);
- iiManager.updateII(ii);
+ iiManager.updateII(ii,true);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/job/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/DeployUtil.java b/job/src/test/java/org/apache/kylin/job/DeployUtil.java
index 329eef1..6d348bd 100644
--- a/job/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/job/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -18,35 +18,30 @@
package org.apache.kylin.job;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.InputStream;
-
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.job.dataGen.FactTableGenerator;
-import org.apache.maven.model.Model;
-import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
-import org.apache.tools.ant.filters.StringInputStream;
-import org.codehaus.plexus.util.FileUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.ResourceTool;
import org.apache.kylin.common.util.AbstractKylinTestCase;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.HiveClient;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.job.dataGen.FactTableGenerator;
import org.apache.kylin.job.hadoop.hive.SqlHiveDataTypeMapping;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.maven.model.Model;
+import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
+import org.apache.tools.ant.filters.StringInputStream;
+import org.codehaus.plexus.util.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
public class DeployUtil {
@SuppressWarnings("unused")
@@ -65,7 +60,7 @@ public class DeployUtil {
// update cube desc signature.
for (CubeInstance cube : CubeManager.getInstance(config()).listAllCubes()) {
cube.getDescriptor().setSignature(cube.getDescriptor().calculateSignature());
- CubeManager.getInstance(config()).updateCube(cube);
+ CubeManager.getInstance(config()).updateCube(cube,true);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
index 1d0b16b..9f7e5f9 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/cube/MergeCuboidMapperTest.java
@@ -18,34 +18,29 @@
package org.apache.kylin.job.hadoop.cube;
-import java.io.File;
-import java.io.IOException;
-import java.text.ParseException;
-import java.util.ArrayList;
-import java.util.List;
-
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.dict.DictionaryGenerator;
-import org.apache.kylin.dict.DictionaryInfo;
-import org.apache.kylin.dict.DictionaryManager;
-import org.apache.kylin.dict.TrieDictionary;
+import org.apache.kylin.dict.*;
import org.apache.kylin.dict.lookup.TableSignature;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.ProjectManager;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.List;
import static org.junit.Assert.assertTrue;
@@ -139,7 +134,7 @@ public class MergeCuboidMapperTest extends LocalFileMetadataTestCase {
// cubeManager.saveResource(segment.getCubeInstance());
// cubeManager.afterCubeUpdated(segment.getCubeInstance());
- cubeManager.updateCube(cube);
+ cubeManager.updateCube(cube,true);
isFirstSegment = false;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 331bd83..2833218 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -18,23 +18,6 @@
package org.apache.kylin.metadata;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.*;
-import java.util.Map.Entry;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.metadata.project.ProjectManager;
-import org.apache.kylin.metadata.project.RealizationEntry;
-import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.RealizationRegistry;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.kylin.common.KylinConfig;
@@ -45,8 +28,22 @@ import org.apache.kylin.common.restclient.Broadcaster;
import org.apache.kylin.common.restclient.CaseInsensitiveStringCache;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.project.RealizationEntry;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metadata.realization.RealizationRegistry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import javax.xml.crypto.Data;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.*;
+import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
/**
* Serves (and caches) metadata for Kylin instance.
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
----------------------------------------------------------------------
diff --git a/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java b/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
index 029179f..76b1c0f 100644
--- a/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
+++ b/metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
@@ -102,20 +102,19 @@ public class ProjectManager {
logger.debug("Loading Project from folder " + store.getReadableResourcePath(ResourceStore.PROJECT_RESOURCE_ROOT));
for (String path : paths) {
- reloadProjectAt(path);
+ reloadProjectLocalAt(path);
}
wireProjectAndRealizations(projectMap.values());
logger.debug("Loaded " + projectMap.size() + " Project(s)");
}
- public ProjectInstance reloadProject(String project) throws IOException {
- return reloadProjectAt(ProjectInstance.concatResourcePath(project));
+ public ProjectInstance reloadProjectLocal(String project) throws IOException {
+ return reloadProjectLocalAt(ProjectInstance.concatResourcePath(project));
}
- private ProjectInstance reloadProjectAt(String path) throws IOException {
- ResourceStore store = getStore();
+ private ProjectInstance reloadProjectLocalAt(String path) throws IOException {
- ProjectInstance projectInstance = store.getResource(path, ProjectInstance.class, PROJECT_SERIALIZER);
+ ProjectInstance projectInstance = getStore().getResource(path, ProjectInstance.class, PROJECT_SERIALIZER);
if (projectInstance == null) {
logger.warn("reload project at path:" + path + " not found, this:" + this.toString());
return null;
@@ -159,16 +158,16 @@ public class ProjectManager {
}
public ProjectInstance createProject(String projectName, String owner, String description) throws IOException {
- logger.info("Creating project '" + projectName);
+ logger.info("Creating project " + projectName);
ProjectInstance currentProject = getProject(projectName);
if (currentProject == null) {
- currentProject = ProjectInstance.create(projectName, owner, description, null,null);
+ currentProject = ProjectInstance.create(projectName, owner, description, null, null);
} else {
throw new IllegalStateException("The project named " + projectName + "already exists");
}
- saveResource(currentProject);
+ updateProject(currentProject);
return currentProject;
}
@@ -189,21 +188,30 @@ public class ProjectManager {
logger.info("Dropping project '" + projectInstance.getName() + "'");
- deleteResource(projectInstance);
+ removeProject(projectInstance);
return projectInstance;
}
+ //passive update due to underlying realization update
+ public void updateProject(RealizationType type, String realizationName) throws IOException {
+ for (ProjectInstance proj : findProjects(type, realizationName)) {
+ updateProject(proj);
+ }
+ }
+
+ //update project itself
public ProjectInstance updateProject(ProjectInstance project, String newName, String newDesc) throws IOException {
if (!project.getName().equals(newName)) {
ProjectInstance newProject = this.createProject(newName, project.getOwner(), newDesc);
- // FIXME table lost??
+
newProject.setCreateTimeUTC(project.getCreateTimeUTC());
newProject.recordUpdateTime(System.currentTimeMillis());
newProject.setRealizationEntries(project.getRealizationEntries());
+ newProject.setTables(project.getTables());
- deleteResource(project);
- saveResource(newProject);
+ removeProject(project);
+ updateProject(newProject);
return newProject;
} else {
@@ -213,12 +221,24 @@ public class ProjectManager {
if (project.getUuid() == null)
project.updateRandomUuid();
- saveResource(project);
+ updateProject(project);
return project;
}
}
+ private void updateProject(ProjectInstance prj) throws IOException {
+ getStore().putResource(prj.getResourcePath(), prj, PROJECT_SERIALIZER);
+ projectMap.put(norm(prj.getName()), prj); // triggers update broadcast
+ clearL2Cache();
+ }
+
+ private void removeProject(ProjectInstance proj) throws IOException {
+ getStore().deleteResource(proj.getResourcePath());
+ projectMap.remove(norm(proj.getName()));
+ clearL2Cache();
+ }
+
public boolean isModelInProject(String projectName, String modelName) {
return this.getProject(projectName).containsModel(modelName);
}
@@ -231,7 +251,7 @@ public class ProjectManager {
public void removeModelFromProjects(String modelName) throws IOException {
for (ProjectInstance projectInstance : findProjects(modelName)) {
projectInstance.removeModel(modelName);
- saveResource(projectInstance);
+ updateProject(projectInstance);
}
}
@@ -239,10 +259,10 @@ public class ProjectManager {
String newProjectName = ProjectInstance.getNormalizedProjectName(project);
ProjectInstance newProject = getProject(newProjectName);
if (newProject == null) {
- throw new IllegalArgumentException("Project "+newProjectName+" does not exist.");
+ throw new IllegalArgumentException("Project " + newProjectName + " does not exist.");
}
newProject.addModel(modelName);
- saveResource(newProject);
+ updateProject(newProject);
return newProject;
}
@@ -252,7 +272,6 @@ public class ProjectManager {
return addRealizationToProject(type, realizationName, newProjectName, owner);
}
-
private ProjectInstance addRealizationToProject(RealizationType type, String realizationName, String project, String user) throws IOException {
String newProjectName = norm(project);
ProjectInstance newProject = getProject(newProjectName);
@@ -260,7 +279,7 @@ public class ProjectManager {
newProject = this.createProject(newProjectName, user, "This is a project automatically added when adding realization " + realizationName + "(" + type + ")");
}
newProject.addRealizationEntry(type, realizationName);
- saveResource(newProject);
+ updateProject(newProject);
return newProject;
}
@@ -268,7 +287,7 @@ public class ProjectManager {
public void removeRealizationsFromProjects(RealizationType type, String realizationName) throws IOException {
for (ProjectInstance projectInstance : findProjects(type, realizationName)) {
projectInstance.removeRealization(type, realizationName);
- saveResource(projectInstance);
+ updateProject(projectInstance);
}
}
@@ -283,26 +302,10 @@ public class ProjectManager {
projectInstance.addTable(table.getIdentity());
}
- saveResource(projectInstance);
+ updateProject(projectInstance);
return projectInstance;
}
- private void saveResource(ProjectInstance prj) throws IOException {
- ResourceStore store = getStore();
- store.putResource(prj.getResourcePath(), prj, PROJECT_SERIALIZER);
-
- prj = reloadProjectAt(prj.getResourcePath());
- projectMap.put(norm(prj.getName()), prj); // triggers update broadcast
- clearL2Cache();
- }
-
- private void deleteResource(ProjectInstance proj) throws IOException {
- ResourceStore store = getStore();
- store.deleteResource(proj.getResourcePath());
- projectMap.remove(norm(proj.getName()));
- clearL2Cache();
- }
-
public List<ProjectInstance> findProjects(RealizationType type, String realizationName) {
List<ProjectInstance> result = Lists.newArrayList();
for (ProjectInstance prj : projectMap.values()) {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index b1f9b4a..466a007 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -20,25 +20,18 @@ package org.apache.kylin.rest.controller;
import java.io.IOException;
import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
+import java.util.*;
import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.model.CubeBuildTypeEnum;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
import org.apache.kylin.job.JobInstance;
import org.apache.kylin.job.JoinedFlatTable;
import org.apache.kylin.job.exception.JobException;
-import org.apache.kylin.cube.model.CubeJoinedFlatTableDesc;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.rest.exception.BadRequestException;
@@ -58,12 +51,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.AccessDeniedException;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Controller;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestMethod;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.*;
import com.codahale.metrics.annotation.Metered;
import com.fasterxml.jackson.core.JsonParseException;
@@ -86,10 +74,10 @@ public class CubeController extends BasicController {
@Autowired
private JobService jobService;
- @RequestMapping(value = "", method = {RequestMethod.GET})
+ @RequestMapping(value = "", method = { RequestMethod.GET })
@ResponseBody
@Metered(name = "listCubes")
- public List<CubeInstance> getCubes(@RequestParam(value = "cubeName", required = false) String cubeName,@RequestParam(value = "modelName", required = false) String modelName, @RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value="limit",required = false) Integer limit, @RequestParam(value = "offset" ,required = false) Integer offset) {
+ public List<CubeInstance> getCubes(@RequestParam(value = "cubeName", required = false) String cubeName, @RequestParam(value = "modelName", required = false) String modelName, @RequestParam(value = "projectName", required = false) String projectName, @RequestParam(value = "limit", required = false) Integer limit, @RequestParam(value = "offset", required = false) Integer offset) {
return cubeService.getCubes(cubeName, projectName, modelName, limit, offset);
}
@@ -101,7 +89,7 @@ public class CubeController extends BasicController {
* @throws UnknownHostException
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/segs/{segmentName}/sql", method = {RequestMethod.GET})
+ @RequestMapping(value = "/{cubeName}/segs/{segmentName}/sql", method = { RequestMethod.GET })
@ResponseBody
public GeneralResponse getSql(@PathVariable String cubeName, @PathVariable String segmentName) {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
@@ -123,7 +111,7 @@ public class CubeController extends BasicController {
* @param notifyList
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/notify_list", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/notify_list", method = { RequestMethod.PUT })
@ResponseBody
public void updateNotifyList(@PathVariable String cubeName, @RequestBody List<String> notifyList) {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
@@ -141,7 +129,7 @@ public class CubeController extends BasicController {
}
- @RequestMapping(value = "/{cubeName}/cost", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/cost", method = { RequestMethod.PUT })
@ResponseBody
@Metered(name = "updateCubeCost")
public CubeInstance updateCubeCost(@PathVariable String cubeName, @RequestParam(value = "cost") int cost) {
@@ -154,7 +142,7 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}/coprocessor", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/coprocessor", method = { RequestMethod.PUT })
@ResponseBody
public Map<String, Boolean> updateCubeCoprocessor(@PathVariable String cubeName, @RequestParam(value = "force") String force) {
try {
@@ -172,7 +160,7 @@ public class CubeController extends BasicController {
*
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/segs/{segmentName}/refresh_lookup", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/segs/{segmentName}/refresh_lookup", method = { RequestMethod.PUT })
@ResponseBody
public CubeInstance rebuildLookupSnapshot(@PathVariable String cubeName, @PathVariable String segmentName, @RequestParam(value = "lookupTable") String lookupTable) {
try {
@@ -191,7 +179,7 @@ public class CubeController extends BasicController {
* @throws SchedulerException
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/rebuild", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/rebuild", method = { RequestMethod.PUT })
@ResponseBody
public JobInstance rebuild(@PathVariable String cubeName, @RequestBody JobBuildRequest jobBuildRequest) {
try {
@@ -208,7 +196,7 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}/disable", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/disable", method = { RequestMethod.PUT })
@ResponseBody
@Metered(name = "disableCube")
public CubeInstance disableCube(@PathVariable String cubeName) {
@@ -227,7 +215,7 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}/purge", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/purge", method = { RequestMethod.PUT })
@ResponseBody
@Metered(name = "purgeCube")
public CubeInstance purgeCube(@PathVariable String cubeName) {
@@ -246,7 +234,7 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}/enable", method = {RequestMethod.PUT})
+ @RequestMapping(value = "/{cubeName}/enable", method = { RequestMethod.PUT })
@ResponseBody
@Metered(name = "enableCube")
public CubeInstance enableCube(@PathVariable String cubeName) {
@@ -264,7 +252,7 @@ public class CubeController extends BasicController {
}
}
- @RequestMapping(value = "/{cubeName}", method = {RequestMethod.DELETE})
+ @RequestMapping(value = "/{cubeName}", method = { RequestMethod.DELETE })
@ResponseBody
@Metered(name = "deleteCube")
public void deleteCube(@PathVariable String cubeName) {
@@ -287,7 +275,7 @@ public class CubeController extends BasicController {
* @return Table metadata array
* @throws IOException
*/
- @RequestMapping(value = "", method = {RequestMethod.POST})
+ @RequestMapping(value = "", method = { RequestMethod.POST })
@ResponseBody
@Metered(name = "saveCube")
public CubeRequest saveCubeDesc(@RequestBody CubeRequest cubeRequest) {
@@ -324,7 +312,7 @@ public class CubeController extends BasicController {
* @throws JsonProcessingException
* @throws IOException
*/
- @RequestMapping(value = "", method = {RequestMethod.PUT})
+ @RequestMapping(value = "", method = { RequestMethod.PUT })
@ResponseBody
@Metered(name = "updateCube")
public CubeRequest updateCubeDesc(@RequestBody CubeRequest cubeRequest) throws JsonProcessingException {
@@ -338,11 +326,17 @@ public class CubeController extends BasicController {
// Check if the cube is editable
if (!cubeService.isCubeDescEditable(desc)) {
- String error = "Cube desc " + desc.getName().toUpperCase() + " is not editable.";
+ String error = "Purge the related cube before editing its desc. Desc name: " + desc.getName();
updateRequest(cubeRequest, false, error);
return cubeRequest;
}
+ //cube renaming:
+ if (!cubeRequest.getCubeName().equalsIgnoreCase(CubeService.getCubeNameFromDesc(desc.getName()))) {
+ deleteCube(cubeRequest.getCubeName());
+ saveCubeDesc(cubeRequest);
+ }
+
try {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeRequest.getCubeName());
String projectName = (null == cubeRequest.getProject()) ? ProjectInstance.DEFAULT_PROJECT_NAME : cubeRequest.getProject();
@@ -373,7 +367,7 @@ public class CubeController extends BasicController {
* @return true
* @throws IOException
*/
- @RequestMapping(value = "/{cubeName}/hbase", method = {RequestMethod.GET})
+ @RequestMapping(value = "/{cubeName}/hbase", method = { RequestMethod.GET })
@ResponseBody
@Metered(name = "getHBaseInfo")
public List<HBaseResponse> getHBaseInfo(@PathVariable String cubeName) {
@@ -429,14 +423,12 @@ public class CubeController extends BasicController {
return desc;
}
-
-
/**
* @return
*/
private String omitMessage(List<String> errors) {
StringBuffer buffer = new StringBuffer();
- for (Iterator<String> iterator = errors.iterator(); iterator.hasNext(); ) {
+ for (Iterator<String> iterator = errors.iterator(); iterator.hasNext();) {
String string = (String) iterator.next();
buffer.append(string);
buffer.append("\n");
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
index c0f366d..0ae9376 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/BasicService.java
@@ -23,15 +23,15 @@ import com.google.common.base.Predicate;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.Lists;
import com.google.common.io.Files;
-
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.invertedindex.IIDescManager;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.job.cube.CubingJob;
-import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.execution.AbstractExecutable;
+import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.manager.ExecutableManager;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.project.ProjectInstance;
@@ -41,7 +41,6 @@ import org.apache.kylin.query.enumerator.OLAPQuery;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
import org.apache.kylin.rest.controller.QueryController;
-import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.cache.annotation.CacheEvict;
@@ -49,7 +48,6 @@ import org.springframework.cache.annotation.Caching;
import org.springframework.jdbc.datasource.DriverManagerDataSource;
import javax.sql.DataSource;
-
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
@@ -78,6 +76,7 @@ public abstract class BasicService {
}
public void removeOLAPDataSource(String project) {
+ logger.info("removeOLAPDataSource is called for project " + project);
if (StringUtils.isEmpty(project))
throw new IllegalArgumentException("removeOLAPDataSource: project name not given");
@@ -85,9 +84,9 @@ public abstract class BasicService {
olapDataSources.remove(project);
}
- public static void resetOLAPDataSources() {
+ public static void removeAllOLAPDataSources() {
// brutal, yet simplest way
- logger.info("resetOLAPDataSources is called.");
+ logger.info("removeAllOLAPDataSources is called.");
olapDataSources.clear();
}
@@ -135,7 +134,7 @@ public abstract class BasicService {
public void cleanDataCache() {
CubeManager.clearCache();
ProjectManager.clearCache();
- BasicService.resetOLAPDataSources();
+ removeAllOLAPDataSources();
}
public final KylinConfig getKylinConfig() {
@@ -184,7 +183,7 @@ public abstract class BasicService {
if (cubeName == null) {
return true;
}
- return ((CubingJob) executable).getCubeName().equalsIgnoreCase(cubeName);
+ return ((CubingJob) executable).getCubeName().equalsIgnoreCase(cubeName);
} else {
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/f5695644/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 6ea563f..9b89932 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -24,16 +24,13 @@ import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.model.CubeBuildTypeEnum;
import org.apache.kylin.invertedindex.IIDescManager;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.job.cube.CubingJob;
import org.apache.kylin.job.cube.CubingJobBuilder;
import org.apache.kylin.job.engine.JobEngineConfig;
-import org.apache.kylin.job.exception.JobException;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.project.ProjectManager;
-import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.controller.QueryController;
import org.slf4j.Logger;
@@ -44,7 +41,6 @@ import org.springframework.cache.annotation.Caching;
import org.springframework.stereotype.Component;
import java.io.IOException;
-import java.util.List;
/**
*/
@@ -61,58 +57,48 @@ public class CacheService extends BasicService {
final String log = "rebuild cache type: " + cacheType + " name:" + cacheKey;
try {
switch (cacheType) {
- case CUBE:
- getCubeManager().loadCubeCache(cacheKey);
- cleanProjectCacheByRealization(RealizationType.CUBE, cacheKey);
- mergeCubeOnNewSegmentReady(cacheKey);
- break;
- case CUBE_DESC:
- getCubeDescManager().reloadCubeDesc(cacheKey);
- break;
- case PROJECT:
- getProjectManager().reloadProject(cacheKey);
- break;
- case INVERTED_INDEX:
- getIIManager().loadIICache(cacheKey);
- cleanProjectCacheByRealization(RealizationType.INVERTED_INDEX, cacheKey);
- break;
- case INVERTED_INDEX_DESC:
- getIIDescManager().reloadIIDesc(cacheKey);
- break;
- case TABLE:
- getMetadataManager().reloadTableCache(cacheKey);
- IIDescManager.clearCache();
- CubeDescManager.clearCache();
- break;
- case DATA_MODEL:
- getMetadataManager().reloadDataModelDesc(cacheKey);
- IIDescManager.clearCache();
- CubeDescManager.clearCache();
- break;
- case ALL:
- getMetadataManager().reload();
- CubeDescManager.clearCache();
- CubeManager.clearCache();
- IIDescManager.clearCache();
- IIManager.clearCache();
- ProjectManager.clearCache();
- BasicService.resetOLAPDataSources();
- break;
- default:
- throw new RuntimeException("invalid cacheType:" + cacheType);
+ case CUBE:
+ getCubeManager().reloadCubeLocal(cacheKey);
+ mergeCubeOnNewSegmentReady(cacheKey);
+ break;
+ case CUBE_DESC:
+ getCubeDescManager().reloadCubeDescLocal(cacheKey);
+ break;
+ case PROJECT:
+ ProjectInstance projectInstance = getProjectManager().reloadProjectLocal(cacheKey);
+ removeOLAPDataSource(projectInstance.getName());
+ break;
+ case INVERTED_INDEX:
+ getIIManager().reloadIILocal(cacheKey);
+ break;
+ case INVERTED_INDEX_DESC:
+ getIIDescManager().reloadIIDescLocal(cacheKey);
+ break;
+ case TABLE:
+ getMetadataManager().reloadTableCache(cacheKey);
+ IIDescManager.clearCache();
+ CubeDescManager.clearCache();
+ break;
+ case DATA_MODEL:
+ getMetadataManager().reloadDataModelDesc(cacheKey);
+ IIDescManager.clearCache();
+ CubeDescManager.clearCache();
+ break;
+ case ALL:
+ getMetadataManager().reload();
+ CubeDescManager.clearCache();
+ CubeManager.clearCache();
+ IIDescManager.clearCache();
+ IIManager.clearCache();
+ ProjectManager.clearCache();
+ removeAllOLAPDataSources();
+ break;
+ default:
+ throw new RuntimeException("invalid cacheType:" + cacheType);
}
} catch (IOException e) {
throw new RuntimeException("error " + log, e);
}
-
- }
-
- private void cleanProjectCacheByRealization(RealizationType type, String realizationName) throws IOException {
- List<ProjectInstance> projectInstances = getProjectManager().findProjects(type, realizationName);
- for (ProjectInstance pi : projectInstances) {
- getProjectManager().reloadProject(pi.getName());
- removeOLAPDataSource(pi.getName());
- }
}
@Caching(evict = { @CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true) })
@@ -120,27 +106,27 @@ public class CacheService extends BasicService {
final String log = "remove cache type: " + cacheType + " name:" + cacheKey;
try {
switch (cacheType) {
- case CUBE:
- getCubeManager().removeCubeCacheLocal(cacheKey);
- break;
- case CUBE_DESC:
- getCubeDescManager().removeLocalCubeDesc(cacheKey);
- break;
- case PROJECT:
- ProjectManager.clearCache();
- break;
- case INVERTED_INDEX:
- getIIManager().removeIILocalCache(cacheKey);
- break;
- case INVERTED_INDEX_DESC:
- getIIDescManager().removeIIDescLocal(cacheKey);
- break;
- case TABLE:
- throw new UnsupportedOperationException(log);
- case DATA_MODEL:
- throw new UnsupportedOperationException(log);
- default:
- throw new RuntimeException("invalid cacheType:" + cacheType);
+ case CUBE:
+ getCubeManager().removeCubeLocal(cacheKey);
+ break;
+ case CUBE_DESC:
+ getCubeDescManager().removeLocalCubeDesc(cacheKey);
+ break;
+ case PROJECT:
+ ProjectManager.clearCache();
+ break;
+ case INVERTED_INDEX:
+ getIIManager().removeIILocal(cacheKey);
+ break;
+ case INVERTED_INDEX_DESC:
+ getIIDescManager().removeIIDescLocal(cacheKey);
+ break;
+ case TABLE:
+ throw new UnsupportedOperationException(log);
+ case DATA_MODEL:
+ throw new UnsupportedOperationException(log);
+ default:
+ throw new RuntimeException("invalid cacheType:" + cacheType);
}
} catch (IOException e) {
throw new RuntimeException("error " + log, e);