You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2018/01/25 02:47:33 UTC
[1/2] kylin git commit: KYLIN-3086 When you load Hive table metadata
from server, the intermediate tables should not be loaded.
Repository: kylin
Updated Branches:
refs/heads/master ff2d2727e -> e03c76a40
KYLIN-3086 When you load Hive table metadata from server, the intermediate tables should not be loaded.
Signed-off-by: Billy Liu <bi...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6346deb9
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6346deb9
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6346deb9
Branch: refs/heads/master
Commit: 6346deb9fb718144cc25880e5217e2bbe5645c09
Parents: ff2d272
Author: peng.jianhua <pe...@zte.com.cn>
Authored: Thu Jan 25 09:23:39 2018 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Thu Jan 25 10:34:47 2018 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConstant.java | 24 ++++++++++++++++++++
.../cube/model/CubeJoinedFlatTableDesc.java | 5 ++--
.../apache/kylin/metadata/model/TableDesc.java | 3 ++-
.../ITDoggedCubeBuilderStressTest.java | 4 +++-
.../inmemcubing/ITDoggedCubeBuilderTest.java | 4 +++-
.../inmemcubing/ITInMemCubeBuilderTest.java | 9 ++++++--
.../kylin/rest/job/StorageCleanupJob.java | 6 +++--
.../apache/kylin/rest/service/TableService.java | 14 +++++++++++-
.../storage/hbase/util/StorageCleanupJob.java | 5 ++--
9 files changed, 62 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6346deb9/core-common/src/main/java/org/apache/kylin/common/KylinConstant.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConstant.java b/core-common/src/main/java/org/apache/kylin/common/KylinConstant.java
new file mode 100644
index 0000000..d8073b7
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConstant.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.common;
+
+public class KylinConstant {
+
+ public static final String KYLIN_INTERMEDIATE_PREFIX = "kylin_intermediate_";
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6346deb9/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
index 81087f8..231cd54 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
@@ -22,6 +22,7 @@ import java.io.Serializable;
import java.util.List;
import java.util.Map;
+import org.apache.kylin.common.KylinConstant;
import org.apache.kylin.common.util.BytesSplitter;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.metadata.model.DataModelDesc;
@@ -74,9 +75,9 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc, Serializab
protected String makeTableName(CubeDesc cubeDesc, CubeSegment cubeSegment) {
if (cubeSegment == null) {
- return "kylin_intermediate_" + cubeDesc.getName().toLowerCase();
+ return KylinConstant.KYLIN_INTERMEDIATE_PREFIX + cubeDesc.getName().toLowerCase();
} else {
- return "kylin_intermediate_" + cubeDesc.getName().toLowerCase() + "_"
+ return KylinConstant.KYLIN_INTERMEDIATE_PREFIX + cubeDesc.getName().toLowerCase() + "_"
+ cubeSegment.getUuid().replaceAll("-", "_");
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6346deb9/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index e456d9a..f9d549b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -22,6 +22,7 @@ import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
+import org.apache.kylin.common.KylinConstant;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.Pair;
@@ -45,7 +46,7 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
private static final Logger logger = LoggerFactory.getLogger(TableDesc.class);
private static final String TABLE_TYPE_VIRTUAL_VIEW = "VIRTUAL_VIEW";
- private static final String materializedTableNamePrefix = "kylin_intermediate_";
+ private static final String materializedTableNamePrefix = KylinConstant.KYLIN_INTERMEDIATE_PREFIX;
public static String concatRawResourcePath(String nameOnPath) {
return ResourceStore.TABLE_RESOURCE_ROOT + "/" + nameOnPath + ".json";
http://git-wip-us.apache.org/repos/asf/kylin/blob/6346deb9/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
index fa2d792..f56beb0 100644
--- a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
@@ -26,6 +26,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.KylinConstant;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeInstance;
@@ -66,7 +67,8 @@ public class ITDoggedCubeBuilderStressTest extends LocalFileMetadataTestCase {
CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
cube = cubeManager.getCube("ssb");
- flatTable = LOCALMETA_TEST_DATA + "/data/kylin_intermediate_ssb_19920101000000_19920201000000.csv";
+ flatTable = LOCALMETA_TEST_DATA + "/data/" + KylinConstant.KYLIN_INTERMEDIATE_PREFIX
+ + "ssb_19920101000000_19920201000000.csv";
dictionaryMap = ITInMemCubeBuilderTest.getDictionaryMap(cube, flatTable);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6346deb9/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
index 865cdbb..52a1c6d 100644
--- a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
@@ -33,6 +33,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.KylinConstant;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeInstance;
@@ -70,7 +71,8 @@ public class ITDoggedCubeBuilderTest extends LocalFileMetadataTestCase {
CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
cube = cubeManager.getCube("ssb");
- flatTable = LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/data/kylin_intermediate_ssb_19920101000000_19920201000000.csv";
+ flatTable = LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/data/" + KylinConstant.KYLIN_INTERMEDIATE_PREFIX
+ + "ssb_19920101000000_19920201000000.csv";
dictionaryMap = ITInMemCubeBuilderTest.getDictionaryMap(cube, flatTable);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6346deb9/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
index f1b65b0..77a35a2 100644
--- a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
@@ -33,6 +33,7 @@ import java.util.concurrent.Future;
import org.apache.commons.io.FileUtils;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.KylinConstant;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeInstance;
@@ -83,13 +84,17 @@ public class ITInMemCubeBuilderTest extends LocalFileMetadataTestCase {
@Test
public void testSSBCubeMore() throws Exception {
testBuild("ssb", //
- LOCALMETA_TEST_DATA + "/data/kylin_intermediate_ssb_19920101000000_19920201000000.csv", 7000, 4);
+ LOCALMETA_TEST_DATA + "/data/" + KylinConstant.KYLIN_INTERMEDIATE_PREFIX
+ + "ssb_19920101000000_19920201000000.csv",
+ 7000, 4);
}
@Test
public void testSSBCube() throws Exception {
testBuild("ssb", //
- LOCALMETA_TEST_DATA + "/data/kylin_intermediate_ssb_19920101000000_19920201000000.csv", 1000, 1);
+ LOCALMETA_TEST_DATA + "/data/" + KylinConstant.KYLIN_INTERMEDIATE_PREFIX
+ + "ssb_19920101000000_19920201000000.csv",
+ 1000, 1);
}
public void testBuild(String cubeName, String flatTable, int nInpRows, int nThreads) throws Exception {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6346deb9/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
index d1cb176..189e827 100755
--- a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
@@ -38,6 +38,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.KylinConstant;
+import org.apache.kylin.common.metrics.common.MetricsConstant;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.HadoopUtil;
@@ -192,7 +194,7 @@ public class StorageCleanupJob extends AbstractApplication {
JobEngineConfig engineConfig = new JobEngineConfig(KylinConfig.getInstanceFromEnv());
final CliCommandExecutor cmdExec = config.getCliCommandExecutor();
final int uuidLength = 36;
- final String preFix = "kylin_intermediate_";
+ final String preFix = KylinConstant.KYLIN_INTERMEDIATE_PREFIX;
final String uuidPattern = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
ISourceMetadataExplorer explr = SourceFactory.getDefaultSource().getSourceMetadataExplorer();
@@ -200,7 +202,7 @@ public class StorageCleanupJob extends AbstractApplication {
Iterable<String> kylinIntermediates = Iterables.filter(hiveTableNames, new Predicate<String>() {
@Override
public boolean apply(@Nullable String input) {
- return input != null && input.startsWith("kylin_intermediate_");
+ return input != null && input.startsWith(KylinConstant.KYLIN_INTERMEDIATE_PREFIX);
}
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/6346deb9/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
index 8a6dd96..caa7130 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -29,7 +29,10 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import javax.annotation.Nullable;
+
import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConstant;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeManager;
@@ -61,6 +64,8 @@ import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Component;
import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
import com.google.common.collect.LinkedHashMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.SetMultimap;
@@ -324,7 +329,14 @@ public class TableService extends BasicService {
*/
public List<String> getHiveTableNames(String database) throws Exception {
ISourceMetadataExplorer explr = SourceFactory.getDefaultSource().getSourceMetadataExplorer();
- return explr.listTables(database);
+ List<String> hiveTableNames = explr.listTables(database);
+ Iterable<String> kylinApplicationTableNames = Iterables.filter(hiveTableNames, new Predicate<String>() {
+ @Override
+ public boolean apply(@Nullable String input) {
+ return input != null && !input.startsWith(KylinConstant.KYLIN_INTERMEDIATE_PREFIX);
+ }
+ });
+ return Lists.newArrayList(kylinApplicationTableNames);
}
private TableDescResponse cloneTableDesc(TableDesc table, String prj) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6346deb9/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index 2ae21b4..f81de6c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.KylinConstant;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.HadoopUtil;
@@ -251,13 +252,13 @@ public class StorageCleanupJob extends AbstractApplication {
final KylinConfig config = KylinConfig.getInstanceFromEnv();
final CliCommandExecutor cmdExec = config.getCliCommandExecutor();
final int uuidLength = 36;
- final String preFix = "kylin_intermediate_";
+ final String preFix = KylinConstant.KYLIN_INTERMEDIATE_PREFIX;
final String uuidPattern = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
final String useDatabaseHql = "USE " + config.getHiveDatabaseForIntermediateTable() + ";";
final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.addStatement(useDatabaseHql);
- hiveCmdBuilder.addStatement("show tables " + "\'kylin_intermediate_*\'" + "; ");
+ hiveCmdBuilder.addStatement("show tables " + "\'" + KylinConstant.KYLIN_INTERMEDIATE_PREFIX + "*\'" + "; ");
Pair<Integer, String> result = cmdExec.execute(hiveCmdBuilder.build());
[2/2] kylin git commit: KYLIN-3086 code review,
reuse MetadataConstants
Posted by bi...@apache.org.
KYLIN-3086 code review, reuse MetadataConstants
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e03c76a4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e03c76a4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e03c76a4
Branch: refs/heads/master
Commit: e03c76a4098d08e606ecb819340b153effb38b6e
Parents: 6346deb
Author: Billy Liu <bi...@apache.org>
Authored: Thu Jan 25 10:47:24 2018 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Thu Jan 25 10:47:24 2018 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConstant.java | 24 --------------------
.../cube/model/CubeJoinedFlatTableDesc.java | 6 ++---
.../kylin/metadata/MetadataConstants.java | 1 +
.../apache/kylin/metadata/model/TableDesc.java | 5 ++--
.../ITDoggedCubeBuilderStressTest.java | 4 ++--
.../inmemcubing/ITDoggedCubeBuilderTest.java | 4 ++--
.../inmemcubing/ITInMemCubeBuilderTest.java | 6 ++---
.../kylin/rest/job/StorageCleanupJob.java | 7 +++---
.../apache/kylin/rest/service/TableService.java | 4 ++--
.../storage/hbase/util/StorageCleanupJob.java | 6 ++---
10 files changed, 21 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e03c76a4/core-common/src/main/java/org/apache/kylin/common/KylinConstant.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConstant.java b/core-common/src/main/java/org/apache/kylin/common/KylinConstant.java
deleted file mode 100644
index d8073b7..0000000
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConstant.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-package org.apache.kylin.common;
-
-public class KylinConstant {
-
- public static final String KYLIN_INTERMEDIATE_PREFIX = "kylin_intermediate_";
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/e03c76a4/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
index 231cd54..4a1cfc0 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeJoinedFlatTableDesc.java
@@ -22,9 +22,9 @@ import java.io.Serializable;
import java.util.List;
import java.util.Map;
-import org.apache.kylin.common.KylinConstant;
import org.apache.kylin.common.util.BytesSplitter;
import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
@@ -75,9 +75,9 @@ public class CubeJoinedFlatTableDesc implements IJoinedFlatTableDesc, Serializab
protected String makeTableName(CubeDesc cubeDesc, CubeSegment cubeSegment) {
if (cubeSegment == null) {
- return KylinConstant.KYLIN_INTERMEDIATE_PREFIX + cubeDesc.getName().toLowerCase();
+ return MetadataConstants.KYLIN_INTERMEDIATE_PREFIX + cubeDesc.getName().toLowerCase();
} else {
- return KylinConstant.KYLIN_INTERMEDIATE_PREFIX + cubeDesc.getName().toLowerCase() + "_"
+ return MetadataConstants.KYLIN_INTERMEDIATE_PREFIX + cubeDesc.getName().toLowerCase() + "_"
+ cubeSegment.getUuid().replaceAll("-", "_");
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e03c76a4/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataConstants.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataConstants.java b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataConstants.java
index c2d8569..6a79cad 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataConstants.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataConstants.java
@@ -52,4 +52,5 @@ public interface MetadataConstants {
public static final String TABLE_EXD_DELIM = "delim";
public static final String TABLE_EXD_DEFAULT_VALUE = "unknown";
+ public static final String KYLIN_INTERMEDIATE_PREFIX = "kylin_intermediate_";
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e03c76a4/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index f9d549b..68bc5e9 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -22,11 +22,11 @@ import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
-import org.apache.kylin.common.KylinConstant;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.StringSplitter;
+import org.apache.kylin.metadata.MetadataConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,7 +46,6 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
private static final Logger logger = LoggerFactory.getLogger(TableDesc.class);
private static final String TABLE_TYPE_VIRTUAL_VIEW = "VIRTUAL_VIEW";
- private static final String materializedTableNamePrefix = KylinConstant.KYLIN_INTERMEDIATE_PREFIX;
public static String concatRawResourcePath(String nameOnPath) {
return ResourceStore.TABLE_RESOURCE_ROOT + "/" + nameOnPath + ".json";
@@ -351,7 +350,7 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
}
public String getMaterializedName() {
- return materializedTableNamePrefix + database.getName() + "_" + name;
+ return MetadataConstants.KYLIN_INTERMEDIATE_PREFIX + database.getName() + "_" + name;
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/e03c76a4/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
index f56beb0..be3d759 100644
--- a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderStressTest.java
@@ -26,13 +26,13 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.KylinConstant;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.junit.AfterClass;
@@ -67,7 +67,7 @@ public class ITDoggedCubeBuilderStressTest extends LocalFileMetadataTestCase {
CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
cube = cubeManager.getCube("ssb");
- flatTable = LOCALMETA_TEST_DATA + "/data/" + KylinConstant.KYLIN_INTERMEDIATE_PREFIX
+ flatTable = LOCALMETA_TEST_DATA + "/data/" + MetadataConstants.KYLIN_INTERMEDIATE_PREFIX
+ "ssb_19920101000000_19920201000000.csv";
dictionaryMap = ITInMemCubeBuilderTest.getDictionaryMap(cube, flatTable);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e03c76a4/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
index 52a1c6d..8fcf9ed 100644
--- a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITDoggedCubeBuilderTest.java
@@ -33,13 +33,13 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.KylinConstant;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.junit.AfterClass;
@@ -71,7 +71,7 @@ public class ITDoggedCubeBuilderTest extends LocalFileMetadataTestCase {
CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
cube = cubeManager.getCube("ssb");
- flatTable = LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/data/" + KylinConstant.KYLIN_INTERMEDIATE_PREFIX
+ flatTable = LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/data/" + MetadataConstants.KYLIN_INTERMEDIATE_PREFIX
+ "ssb_19920101000000_19920201000000.csv";
dictionaryMap = ITInMemCubeBuilderTest.getDictionaryMap(cube, flatTable);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e03c76a4/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
----------------------------------------------------------------------
diff --git a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
index 77a35a2..9bfc18e 100644
--- a/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/cube/inmemcubing/ITInMemCubeBuilderTest.java
@@ -33,7 +33,6 @@ import java.util.concurrent.Future;
import org.apache.commons.io.FileUtils;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.KylinConstant;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.cube.CubeInstance;
@@ -45,6 +44,7 @@ import org.apache.kylin.dict.DictionaryGenerator;
import org.apache.kylin.dict.IterableDictionaryValueEnumerator;
import org.apache.kylin.engine.EngineFactory;
import org.apache.kylin.gridtable.GTRecord;
+import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
import org.apache.kylin.metadata.model.MeasureDesc;
@@ -84,7 +84,7 @@ public class ITInMemCubeBuilderTest extends LocalFileMetadataTestCase {
@Test
public void testSSBCubeMore() throws Exception {
testBuild("ssb", //
- LOCALMETA_TEST_DATA + "/data/" + KylinConstant.KYLIN_INTERMEDIATE_PREFIX
+ LOCALMETA_TEST_DATA + "/data/" + MetadataConstants.KYLIN_INTERMEDIATE_PREFIX
+ "ssb_19920101000000_19920201000000.csv",
7000, 4);
}
@@ -92,7 +92,7 @@ public class ITInMemCubeBuilderTest extends LocalFileMetadataTestCase {
@Test
public void testSSBCube() throws Exception {
testBuild("ssb", //
- LOCALMETA_TEST_DATA + "/data/" + KylinConstant.KYLIN_INTERMEDIATE_PREFIX
+ LOCALMETA_TEST_DATA + "/data/" + MetadataConstants.KYLIN_INTERMEDIATE_PREFIX
+ "ssb_19920101000000_19920201000000.csv",
1000, 1);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e03c76a4/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
index 189e827..7730ee3 100755
--- a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanupJob.java
@@ -38,8 +38,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.KylinConstant;
-import org.apache.kylin.common.metrics.common.MetricsConstant;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.HadoopUtil;
@@ -53,6 +51,7 @@ import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableManager;
import org.apache.kylin.job.execution.ExecutableState;
+import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.source.ISourceMetadataExplorer;
import org.apache.kylin.source.SourceFactory;
import org.slf4j.Logger;
@@ -194,7 +193,7 @@ public class StorageCleanupJob extends AbstractApplication {
JobEngineConfig engineConfig = new JobEngineConfig(KylinConfig.getInstanceFromEnv());
final CliCommandExecutor cmdExec = config.getCliCommandExecutor();
final int uuidLength = 36;
- final String preFix = KylinConstant.KYLIN_INTERMEDIATE_PREFIX;
+ final String preFix = MetadataConstants.KYLIN_INTERMEDIATE_PREFIX;
final String uuidPattern = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
ISourceMetadataExplorer explr = SourceFactory.getDefaultSource().getSourceMetadataExplorer();
@@ -202,7 +201,7 @@ public class StorageCleanupJob extends AbstractApplication {
Iterable<String> kylinIntermediates = Iterables.filter(hiveTableNames, new Predicate<String>() {
@Override
public boolean apply(@Nullable String input) {
- return input != null && input.startsWith(KylinConstant.KYLIN_INTERMEDIATE_PREFIX);
+ return input != null && input.startsWith(MetadataConstants.KYLIN_INTERMEDIATE_PREFIX);
}
});
http://git-wip-us.apache.org/repos/asf/kylin/blob/e03c76a4/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
index caa7130..d737a6a 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -32,7 +32,6 @@ import java.util.UUID;
import javax.annotation.Nullable;
import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConstant;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.CubeManager;
@@ -41,6 +40,7 @@ import org.apache.kylin.engine.mr.common.MapReduceExecutable;
import org.apache.kylin.job.execution.DefaultChainedExecutable;
import org.apache.kylin.job.execution.ExecutableManager;
import org.apache.kylin.job.execution.ExecutableState;
+import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.TableMetadataManager;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
@@ -333,7 +333,7 @@ public class TableService extends BasicService {
Iterable<String> kylinApplicationTableNames = Iterables.filter(hiveTableNames, new Predicate<String>() {
@Override
public boolean apply(@Nullable String input) {
- return input != null && !input.startsWith(KylinConstant.KYLIN_INTERMEDIATE_PREFIX);
+ return input != null && !input.startsWith(MetadataConstants.KYLIN_INTERMEDIATE_PREFIX);
}
});
return Lists.newArrayList(kylinApplicationTableNames);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e03c76a4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
index f81de6c..725d819 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/StorageCleanupJob.java
@@ -44,7 +44,6 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.KylinConstant;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.HadoopUtil;
@@ -59,6 +58,7 @@ import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.execution.AbstractExecutable;
import org.apache.kylin.job.execution.ExecutableManager;
import org.apache.kylin.job.execution.ExecutableState;
+import org.apache.kylin.metadata.MetadataConstants;
import org.apache.kylin.metadata.realization.IRealizationConstants;
import org.apache.kylin.storage.hbase.HBaseConnection;
import org.slf4j.Logger;
@@ -252,13 +252,13 @@ public class StorageCleanupJob extends AbstractApplication {
final KylinConfig config = KylinConfig.getInstanceFromEnv();
final CliCommandExecutor cmdExec = config.getCliCommandExecutor();
final int uuidLength = 36;
- final String preFix = KylinConstant.KYLIN_INTERMEDIATE_PREFIX;
+ final String preFix = MetadataConstants.KYLIN_INTERMEDIATE_PREFIX;
final String uuidPattern = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
final String useDatabaseHql = "USE " + config.getHiveDatabaseForIntermediateTable() + ";";
final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
hiveCmdBuilder.addStatement(useDatabaseHql);
- hiveCmdBuilder.addStatement("show tables " + "\'" + KylinConstant.KYLIN_INTERMEDIATE_PREFIX + "*\'" + "; ");
+ hiveCmdBuilder.addStatement("show tables " + "\'" + MetadataConstants.KYLIN_INTERMEDIATE_PREFIX + "*\'" + "; ");
Pair<Integer, String> result = cmdExec.execute(hiveCmdBuilder.build());