You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/08/29 00:34:55 UTC
[kylin] branch 2.4.x updated: KYLIN-3456 cube level's snapshot
config does not work
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch 2.4.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/2.4.x by this push:
new a755ab0 KYLIN-3456 cube level's snapshot config does not work
a755ab0 is described below
commit a755ab0f67a8292d919a1d265c78a7d74be5209c
Author: Xinbei Fu <23...@users.noreply.github.com>
AuthorDate: Tue Aug 28 11:01:33 2018 +0800
KYLIN-3456 cube level's snapshot config does not work
---
.../java/org/apache/kylin/cube/CubeManager.java | 2 +-
.../apache/kylin/dict/lookup/SnapshotManager.java | 6 +--
.../apache/kylin/dict/lookup/LookupTableTest.java | 43 ++++++++++++++++++++++
.../lookup/LookupSnapshotToMetaStoreStep.java | 2 +-
.../kylin/source/hive/ITSnapshotManagerTest.java | 2 +-
5 files changed, 49 insertions(+), 6 deletions(-)
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
index 3f4c576..7f81cb3 100755
--- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java
@@ -1127,7 +1127,7 @@ public class CubeManager implements IRealizationProvider {
TableDesc tableDesc = new TableDesc(metaMgr.getTableDesc(lookupTable, segCopy.getProject()));
IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc);
- SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc);
+ SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cubeSeg.getConfig());
CubeDesc cubeDesc = cubeSeg.getCubeDesc();
if (!cubeDesc.isGlobalSnapshotTable(lookupTable)) {
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
index dd90b33..d1895ba 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
@@ -115,7 +115,7 @@ public class SnapshotManager {
snapshotCache.invalidate(resourcePath);
}
- public SnapshotTable buildSnapshot(IReadableTable table, TableDesc tableDesc) throws IOException {
+ public SnapshotTable buildSnapshot(IReadableTable table, TableDesc tableDesc, KylinConfig cubeConfig) throws IOException {
SnapshotTable snapshot = new SnapshotTable(table, tableDesc.getIdentity());
snapshot.updateRandomUuid();
@@ -125,8 +125,8 @@ public class SnapshotManager {
return getSnapshotTable(dup);
}
- if (snapshot.getSignature().getSize() / 1024 / 1024 > config.getTableSnapshotMaxMB()) {
- throw new IllegalStateException("Table snapshot should be no greater than " + config.getTableSnapshotMaxMB() //
+ if ((float) snapshot.getSignature().getSize() / 1024 / 1024 > cubeConfig.getTableSnapshotMaxMB()) {
+ throw new IllegalStateException("Table snapshot should be no greater than " + cubeConfig.getTableSnapshotMaxMB() //
+ " MB, but " + tableDesc + " size is " + snapshot.getSignature().getSize());
}
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
index f53e8fe..5251c1c 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/LookupTableTest.java
@@ -18,6 +18,7 @@
package org.apache.kylin.dict.lookup;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -30,6 +31,7 @@ import org.apache.kylin.common.util.Pair;
import org.apache.kylin.dict.TrieDictionaryForest;
import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.source.IReadableTable;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -115,6 +117,47 @@ public class LookupTableTest extends LocalFileMetadataTestCase {
}
+ @Test
+ public void testSnapshotOverVolume() throws IOException {
+ KylinConfig cubeConfig = KylinConfig.createKylinConfig(getTestConfig());
+ cubeConfig.setProperty("kylin.snapshot.max-mb", "0");
+ String tableName = "DEFAULT.TEST_KYLIN_FACT";
+ TableDesc tableDesc = TableMetadataManager.getInstance(getTestConfig()).getTableDesc(tableName, "default");
+ IReadableTable mockTable = new IReadableTable() {
+ @Override
+ public TableReader getReader() throws IOException {
+ return new TableReader() {
+ @Override
+ public boolean next() throws IOException {
+ return false;
+ }
+ @Override
+ public String[] getRow() {
+ return new String[0];
+ }
+ @Override
+ public void close() throws IOException {
+ }
+ };
+ }
+ @Override
+ public TableSignature getSignature() throws IOException {
+ return new IReadableTable.TableSignature("", 2 * 1024 * 1024,
+ System.currentTimeMillis());
+ }
+ @Override
+ public boolean exists() throws IOException {
+ return false;
+ }
+ };
+ SnapshotManager snapshotManager = getSnapshotManager();
+ try {
+ snapshotManager.buildSnapshot(mockTable, tableDesc, cubeConfig).getResourcePath();
+ } catch (IllegalStateException ex) {
+ Assert.assertTrue(ex.getLocalizedMessage().startsWith("Table snapshot should be no greater than 0 MB"));
+ }
+ }
+
private String millis(String dateStr) {
return String.valueOf(DateFormat.stringToMillis(dateStr));
}
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/lookup/LookupSnapshotToMetaStoreStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/lookup/LookupSnapshotToMetaStoreStep.java
index 783ded0..c64694c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/lookup/LookupSnapshotToMetaStoreStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/lookup/LookupSnapshotToMetaStoreStep.java
@@ -63,7 +63,7 @@ public class LookupSnapshotToMetaStoreStep extends AbstractExecutable {
TableDesc tableDesc = metaMgr.getTableDesc(lookupTableName, cube.getProject());
IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc);
logger.info("take snapshot for table:" + lookupTableName);
- SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc);
+ SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cube.getConfig());
logger.info("update snapshot path to cube metadata");
if (cubeDesc.isGlobalSnapshotTable(lookupTableName)) {
diff --git a/kylin-it/src/test/java/org/apache/kylin/source/hive/ITSnapshotManagerTest.java b/kylin-it/src/test/java/org/apache/kylin/source/hive/ITSnapshotManagerTest.java
index 031da29..872f570 100644
--- a/kylin-it/src/test/java/org/apache/kylin/source/hive/ITSnapshotManagerTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/source/hive/ITSnapshotManagerTest.java
@@ -57,7 +57,7 @@ public class ITSnapshotManagerTest extends HBaseMetadataTestCase {
String tableName = "EDW.TEST_SITES";
TableDesc tableDesc = TableMetadataManager.getInstance(getTestConfig()).getTableDesc(tableName, "default");
IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc);
- String snapshotPath = snapshotMgr.buildSnapshot(hiveTable, tableDesc).getResourcePath();
+ String snapshotPath = snapshotMgr.buildSnapshot(hiveTable, tableDesc, getTestConfig()).getResourcePath();
snapshotMgr.wipeoutCache();